diff --git a/.github/workflows/slo-2dc.yml b/.github/workflows/slo-2dc.yml new file mode 100644 index 00000000..56310e63 --- /dev/null +++ b/.github/workflows/slo-2dc.yml @@ -0,0 +1,95 @@ +name: Hosted 2DC SLO + +on: + pull_request: + paths: + - .github/workflows/slo-2dc.yml + - tests/slo/bridge/** + - tests/slo/src/bridge_probe.py + - tests/slo/requirements.txt + workflow_dispatch: + inputs: + server_image: + description: YDB server image to qualify (resolved to a digest) + default: cr.yandex/crpb3d0up7s6a7iuofo9/ydb-appteam-ci/local-ydb:trunk + required: true + +permissions: + contents: read + +jobs: + bridge: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + concurrency: + group: hosted-2dc-${{ github.ref }} + cancel-in-progress: true + env: + WORKLOAD_MEMORY_LIMIT: 1G + WORKLOAD_CPU_LIMIT: '1.00' + WORKLOAD_CURRENT_IMAGE: py-slo-bridge:current + REQUESTED_IMAGE: ${{ inputs.server_image || 'cr.yandex/crpb3d0up7s6a7iuofo9/ydb-appteam-ci/local-ydb@sha256:6c227c994f83a3c34398f8ac7e46e10f9e2eefa35c9c4ac7775950d9920e7566' }} + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: ydb-platform/ydb-slo-action + ref: 54c3c03de1ce8c6a2d8b0828b328bd5db82ef2bb + path: .slo-action + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install the Compose version required by the action + run: | + set -euo pipefail + mkdir -p "$HOME/.docker/cli-plugins" + curl -fsSL --retry 3 \ + https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-linux-x86_64 \ + -o "$HOME/.docker/cli-plugins/docker-compose" + chmod +x "$HOME/.docker/cli-plugins/docker-compose" + docker compose version + docker compose up --help | grep -F -- --quiet-build + - name: Record runner and resolve server image + run: | + set -euo pipefail + mkdir -p bridge-artifacts + free -h + df -h + docker version + docker compose version + docker pull --platform linux/amd64 "$REQUESTED_IMAGE" + docker image inspect "$REQUESTED_IMAGE" > bridge-artifacts/server-image.json + SERVER_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$REQUESTED_IMAGE") + printf '%s\n' "$SERVER_IMAGE" > bridge-artifacts/server-image.txt + python tests/slo/bridge/prepare.py .slo-action "$SERVER_IMAGE" + - name: Build current Python SDK workload + run: docker build -f tests/slo/Dockerfile -t py-slo-bridge:current . + - name: Run all five Bridge scenarios + id: slo + uses: ./.slo-action/init + timeout-minutes: 35 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + workload_name: sync-query + workload_duration: '900' + workload_current_ref: current + workload_current_image: py-slo-bridge:current + workload_current_command: >- + --read-rps 50 --write-rps 10 --read-threads 2 --write-threads 2 + --read-timeout 30000 --write-timeout 30000 + bridge_mode: true + disable_compose_profiles: extra-nodes + fail_on_workload_error: true + - name: Verify scenario coverage, SDK results and data + if: always() + timeout-minutes: 3 + env: + ACTION_OUTCOME: ${{ steps.slo.outcome }} + run: python tests/slo/bridge/verify.py + - name: Upload qualification evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: python-hosted-2dc + path: bridge-artifacts/ + retention-days: 14 diff --git a/tests/slo/README.md b/tests/slo/README.md index 88d46e72..a39a8383 100644 --- a/tests/slo/README.md +++ b/tests/slo/README.md @@ -1,5 +1,39 @@ # SLO workload +## GitHub-hosted Bridge qualification + +`.github/workflows/slo-2dc.yml` runs the existing sync Query workload on +`ubuntu-24.04`, without the `SLO` label or a self-hosted runner. It also runs +an independent Query driver that verifies read-after-write results and audits +every acknowledged row after the faults. + +The workflow pins `ydb-slo-action` v2 by commit. The PR configuration pins a +resolved server digest; manual dispatch accepts a server image and records its +resolved digest. The selected server is passed as the upstream Dockerfile's +`YDB_IMAGE` build argument. No server binaries or action lifecycle code are +modified. Compose 2.40.3 is installed explicitly because the action requires +`up --quiet-build`, which is absent from some hosted runner images. +The hosted profile uses two storage nodes, two database nodes, one +CPU per YDB actor system and bounded node/workload memory. + +This is a 15-minute functional qualification at 50 reads/s and 10 writes/s, +not a performance baseline for the larger self-hosted runner. All five upstream +Bridge scenarios must finish within the workload interval. Process exit zero +alone is insufficient: the verification step requires successful native reads +and writes, no final native operation errors, a successful independent data +audit and no OOM-killed nodes. `bridge-artifacts/result.json` contains the +combined verdict; logs and the resolved server image are uploaded on failure. + +To qualify another image after the workflow is available for dispatch: + +```bash +gh workflow run slo-2dc.yml --ref -f server_image= +``` + +The files in `tests/slo/bridge/` adapt and verify a checked-out action on the +ephemeral runner. They do not start a local Docker stack. The existing `SLO` +workflow and its label-triggered self-hosted jobs remain unchanged. + SLO is the type of test where app based on ydb-sdk is tested against falling YDB cluster nodes, tablets, network (that is possible situations for distributed DBs with hundreds of nodes) diff --git a/tests/slo/bridge/prepare.py b/tests/slo/bridge/prepare.py new file mode 100644 index 00000000..838c8638 --- /dev/null +++ b/tests/slo/bridge/prepare.py @@ -0,0 +1,59 @@ +"""Configure a pinned Action checkout for a bounded GitHub-hosted run.""" + +import argparse +import copy +import json +import subprocess +from pathlib import Path + + +def prepare(action, image): + deploy = action.resolve() / "deploy" + compose = deploy / "compose.bridge.yml" + rendered = subprocess.check_output( + [ + "docker", + "compose", + "-f", + str(compose), + "config", + "--no-interpolate", + "--format", + "json", + ], + text=True, + ) + config = json.loads(rendered) + for service in config["services"].values(): + if "build" in service and service["build"]["context"].endswith("/ydb"): + service["build"].setdefault("args", {})["YDB_IMAGE"] = image + service["mem_limit"] = "2g" + for name in ("workload-current", "workload-baseline"): + config["services"][name]["deploy"]["resources"]["reservations"]["memory"] = "256m" + for filename in ("config.yaml", "config.bridge.yaml"): + path = deploy / "ydb/rootfs/opt/ydb.tech/ydbd/cfg" / filename + text = path.read_text() + if text.count("cpu_count: 4") != 1 or text.count("hard_limit_bytes: 17179869184") != 1: + raise RuntimeError(f"Unexpected resource configuration: {path}") + path.write_text( + text.replace("cpu_count: 4", "cpu_count: 1").replace( + "hard_limit_bytes: 17179869184", "hard_limit_bytes: 1610612736" + ) + ) + probe = copy.deepcopy(config["services"]["workload-current"]) + probe.update( + container_name="ydb-bridge-probe", entrypoint=["python", "/src/tests/slo/src/bridge_probe.py"], command=[] + ) + probe.pop("deploy", None) + probe["mem_limit"] = "512m" + config["services"]["bridge-probe"] = probe + compose.write_text(json.dumps(config, indent=2) + "\n") + subprocess.run(["docker", "compose", "-f", str(compose), "config", "--quiet"], check=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("action", type=Path) + parser.add_argument("image") + args = parser.parse_args() + prepare(args.action, args.image) diff --git a/tests/slo/bridge/verify.py b/tests/slo/bridge/verify.py new file mode 100644 index 00000000..b01c0b20 --- /dev/null +++ b/tests/slo/bridge/verify.py @@ -0,0 +1,98 @@ +"""Check actual chaos completion, native observations and acknowledged data.""" + +import json +import math +import os +import subprocess +import urllib.parse +import urllib.request +from pathlib import Path + +SCENARIOS = [ + "01-switchover", + "02-takedown-synchronized", + "03-takedown-primary", + "04-failover-synchronized", + "05-failover-primary", +] + + +def command(*args, check=True): + return subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=check).stdout + + +def main(): + artifacts = Path("bridge-artifacts") + artifacts.mkdir(exist_ok=True) + result = {"errors": [], "action_outcome": os.environ.get("ACTION_OUTCOME")} + for name in ("ydb-workload-current", "ydb-bridge-probe", "ydb-chaos-monkey"): + (artifacts / f"{name}.log").write_text(command("docker", "logs", name, check=False)) + try: + if result["action_outcome"] != "success": + result["errors"].append("Action did not complete successfully") + chaos = json.loads(command("docker", "inspect", "ydb-chaos-monkey"))[0]["State"] + result["chaos"] = chaos + log = (artifacts / "ydb-chaos-monkey.log").read_text() + if chaos["Status"] != "exited" or chaos["ExitCode"] != 0 or "Chaos Monkey finished" not in log: + result["errors"].append("Chaos sequence did not finish successfully during the workload") + for scenario in SCENARIOS: + if f"Running scenario: {scenario}" not in log: + result["errors"].append(f"Scenario not executed: {scenario}") + probe_state = json.loads(command("docker", "inspect", "ydb-bridge-probe"))[0]["State"] + if probe_state["Running"]: + command("docker", "exec", "ydb-bridge-probe", "touch", "/tmp/bridge-probe-stop") + command("docker", "wait", "ydb-bridge-probe") + command("docker", "cp", "ydb-bridge-probe:/tmp/bridge-probe.json", str(artifacts / "probe.json")) + result["probe"] = json.loads((artifacts / "probe.json").read_text()) + if not result["probe"].get("passed"): + result["errors"].append("Independent data probe failed") + ip = command( + "docker", + "inspect", + "--format", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + "ydb-prometheus", + ).strip() + + def metric(expression): + url = f"http://{ip}:9090/api/v1/query?" + urllib.parse.urlencode({"query": expression}) + with urllib.request.urlopen(url, timeout=15) as response: + data = json.load(response) + if data["status"] != "success": + raise RuntimeError(data) + return sum(float(item["value"][1]) for item in data["data"]["result"]) + + observations = {} + for operation in ("read", "write"): + count = metric(f'sum(sdk_operations_success_total{{ref="current",operation_type="{operation}"}})') + observations[operation] = count + if not math.isfinite(count) or count <= 0: + result["errors"].append(f"No successful native {operation} observations") + observations["errors"] = metric('sum(sdk_operations_failure_total{ref="current"})') + if observations["errors"]: + result["errors"].append("Native workload reported final operation failures") + result["native_observations"] = observations + nodes = json.loads( + command( + "docker", + "inspect", + "ydb-storage-1", + "ydb-storage-2", + "ydb-database-1", + "ydb-database-2", + ) + ) + result["node_states"] = {node["Name"]: node["State"] for node in nodes} + result["node_restarts"] = {node["Name"]: node["RestartCount"] for node in nodes} + if any(node["State"]["OOMKilled"] or node["RestartCount"] for node in nodes): + result["errors"].append("YDB node was OOM killed or unexpectedly restarted") + except Exception as error: + result["errors"].append(repr(error)) + result["passed"] = not result["errors"] + (artifacts / "result.json").write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/slo/requirements.txt b/tests/slo/requirements.txt index 68f516f1..d9021cc5 100644 --- a/tests/slo/requirements.txt +++ b/tests/slo/requirements.txt @@ -4,6 +4,6 @@ hdrhistogram==0.10.3 # OpenTelemetry (OTLP/HTTP exporter) # NOTE: OpenTelemetry 1.39.1 requires Python >= 3.9. -opentelemetry-api==1.45.1 +opentelemetry-api==1.39.1 opentelemetry-sdk==1.39.1 opentelemetry-exporter-otlp-proto-http==1.39.1 diff --git a/tests/slo/src/bridge_probe.py b/tests/slo/src/bridge_probe.py new file mode 100644 index 00000000..fff5111c --- /dev/null +++ b/tests/slo/src/bridge_probe.py @@ -0,0 +1,90 @@ +"""Verify acknowledged data with one Query driver throughout Bridge chaos.""" + +import json +import os +import time +import traceback +import uuid +from pathlib import Path + +import ydb + + +def main(): + result = {"successes": 0, "errors": [], "missing": [], "verified": False} + table = "bridge_probe_" + uuid.uuid4().hex + confirmed = {} + try: + config = ydb.DriverConfig( + os.environ["YDB_ENDPOINT"], + database=os.environ["YDB_DATABASE"], + credentials=ydb.AnonymousCredentials(), + ) + with ydb.Driver(config) as driver: + driver.wait(timeout=60) + with ydb.QuerySessionPool(driver) as pool: + version = pool.execute_with_retries("SELECT CAST(version() AS Utf8) AS version;") + result["server_version"] = version[0].rows[0].version + pool.execute_with_retries(f"CREATE TABLE `{table}` (id Uint64, payload Utf8, PRIMARY KEY(id));") + deadline = time.monotonic() + int(os.environ.get("WORKLOAD_DURATION", "900")) + 180 + retry = ydb.RetrySettings( + max_retries=10, + max_session_acquire_timeout=2, + backoff_slot_duration=0.1, + backoff_ceiling=3, + idempotent=True, + ) + seq = 0 + while time.monotonic() < deadline and not Path("/tmp/bridge-probe-stop").exists(): + seq += 1 + payload = f"{table}_{seq}" + started = time.monotonic() + try: + + def write_read(tx): + return list( + tx.execute( + f"UPSERT INTO `{table}` (id, payload) VALUES " + f'(CAST({seq} AS Uint64), CAST("{payload}" AS Utf8)); ' + f"SELECT payload FROM `{table}` WHERE id = CAST({seq} AS Uint64);", + settings=ydb.BaseRequestSettings().with_timeout(10), + ) + ) + + rows = pool.retry_tx_sync( + write_read, + tx_mode=ydb.QuerySerializableReadWrite(), + retry_settings=retry, + ) + if len(rows[0].rows) != 1 or rows[0].rows[0].payload != payload: + raise AssertionError(f"Incorrect read-after-write result for {seq}") + confirmed[seq] = payload + result["successes"] += 1 + print(json.dumps({"seq": seq, "ok": True, "seconds": time.monotonic() - started}), flush=True) + except AssertionError: + raise + except ydb.Error as error: + result["errors"].append({"seq": seq, "error": repr(error)}) + print(json.dumps({"seq": seq, "ok": False, "error": repr(error)}), flush=True) + time.sleep(0.5) + rows = pool.execute_with_retries(f"SELECT id, payload FROM `{table}`;") + stored = {row.id: row.payload for row in rows[0].rows} + result["missing"] = [seq for seq, payload in confirmed.items() if stored.get(seq) != payload] + result["verified"] = True + except Exception as error: + result["fatal"] = repr(error) + traceback.print_exc() + result["passed"] = bool( + result["successes"] + and result["verified"] + and not result["errors"] + and not result["missing"] + and not result.get("fatal") + ) + Path("/tmp/bridge-probe.json").write_text(json.dumps(result, indent=2)) + print(json.dumps(result), flush=True) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main())