Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 7 additions & 39 deletions .github/workflows/slo-report.yml
Original file line number Diff line number Diff line change
@@ -1,57 +1,25 @@
name: slo-report

# Runs in the base-repo context (not the fork's), so it has the
# pull-requests: write permission needed to publish reports and edit
# labels — which fork-triggered SLO runs do not.

# Report from the base repository without checking out pull request code.
on:
workflow_run:
workflows: ["SLO"]
types:
- completed
types: [completed]

jobs:
publish-slo-report:
# Only act on runs that actually executed to completion. A "skipped"
# conclusion means the matrix was gated out (no "SLO" label) — nothing to
# publish. A "cancelled" conclusion means the run was superseded mid-flight
# (e.g. by `cancel-in-progress` when a new run started) — there is no real
# report and the label must NOT be stripped, otherwise it gets removed long
# before the actual workload finishes. Success/failure are genuine endings.
if: >-
github.event.workflow_run.conclusion == 'success' ||
github.event.workflow_run.conclusion == 'failure'
if: github.event.workflow_run.event == 'pull_request'
name: Publish YDB SLO Report
runs-on: ubuntu-latest
permissions:
actions: read
checks: write
contents: read
pull-requests: write
steps:
- name: Publish YDB SLO Report
uses: ydb-platform/ydb-slo-action/report@v2
uses: ydb-platform/ydb-slo-action/report@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
github_run_id: ${{ github.event.workflow_run.id }}

remove-slo-label:
# Same gating as the report job: only consume the "SLO" label once a run has
# genuinely finished (success/failure). Removing it on a "cancelled" run
# would strip the label mid-flight when a run is superseded.
if: >-
github.event.workflow_run.conclusion == 'success' ||
github.event.workflow_run.conclusion == 'failure'
name: Remove SLO Label
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- name: Remove SLO label from PR
env:
PRS: ${{ toJSON(github.event.workflow_run.pull_requests) }}
REPO: ${{ github.event.workflow_run.repository.full_name }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
PR=$(jq -r '.[0].number' <<<"$PRS")
gh pr edit "$PR" --repo "$REPO" --remove-label SLO
github_workflows: SLO
remove_label: SLO
20 changes: 14 additions & 6 deletions .github/workflows/slo.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,21 @@ jobs:
sdk:
- name: sync-table
command: "--read-rps 1000 --write-rps 100"
workload_type: table
- name: sync-query
command: "--read-rps 1000 --write-rps 100"
workload_type: table
- name: async-query
command: "--read-rps 1000 --write-rps 100"
workload_type: table
- name: sync-topic
command: "--write-rps 200 --write-threads 8 --read-threads 8"
workload_type: topic
metrics_yaml_path: sdk-current/tests/slo/metrics-topic.yaml
thresholds_yaml_path: sdk-current/tests/slo/thresholds-topic.yaml
- name: async-topic
command: "--write-rps 200 --write-threads 8 --read-threads 8"
workload_type: topic
metrics_yaml_path: sdk-current/tests/slo/metrics-topic.yaml
thresholds_yaml_path: sdk-current/tests/slo/thresholds-topic.yaml

Expand Down Expand Up @@ -117,35 +122,38 @@ jobs:
-t "ydb-app-current" \
"$GITHUB_WORKSPACE/sdk-current"

# Build baseline: baseline SDK with the current workload runner
# (Dockerfile + tests/slo/), so the runner-side contract changes
# (entrypoint, metrics format) apply uniformly to both images.
# Build baseline: baseline SDK with the current shared checker.
# Keep its Dockerfile while copying runner-side contract changes.
cp "$GITHUB_WORKSPACE/sdk-baseline/tests/slo/Dockerfile" \
"$RUNNER_TEMP/slo-baseline.Dockerfile"
rm -rf "$GITHUB_WORKSPACE/sdk-baseline/tests/slo"
cp -r "$GITHUB_WORKSPACE/sdk-current/tests/slo" \
"$GITHUB_WORKSPACE/sdk-baseline/tests/slo"
cp "$RUNNER_TEMP/slo-baseline.Dockerfile" \
"$GITHUB_WORKSPACE/sdk-baseline/tests/slo/Dockerfile"
docker build \
-f "$GITHUB_WORKSPACE/sdk-baseline/tests/slo/Dockerfile" \
-t "ydb-app-baseline" \
"$GITHUB_WORKSPACE/sdk-baseline"

- name: Run SLO Tests
uses: ydb-platform/ydb-slo-action/init@v2
uses: ydb-platform/ydb-slo-action/init@d66bd1e10f893e5f70d542c4430a3cffb6d6f9f8 # v3 workload metrics, Action PR81
timeout-minutes: 30
with:
github_issue: ${{ github.event.pull_request.number }}
github_token: ${{ secrets.GITHUB_TOKEN }}
workload_name: ${{ matrix.sdk.name }}
workload_duration: "600"
fail_on_workload_error: true
workload_current_ref: ${{ github.head_ref || github.ref_name }}
workload_current_image: ydb-app-current
workload_current_command: ${{ matrix.sdk.command }}
workload_baseline_ref: ${{ steps.baseline.outputs.ref }}
workload_baseline_image: ydb-app-baseline
workload_baseline_command: ${{ matrix.sdk.command }}
workload_type: ${{ matrix.sdk.workload_type }}
# Custom metrics (e.g. topic e2e latency / loss) merged on top of the
# action defaults; empty for workloads that don't set it.
metrics_yaml_path: ${{ matrix.sdk.metrics_yaml_path }}
# Per-scenario threshold overrides (topics make read_latency neutral).
# Ignored by action versions without ydb-slo-action#57; self-activates
# once per-scenario thresholds land in the consumed tag.
thresholds_yaml_path: ${{ matrix.sdk.thresholds_yaml_path }}
1 change: 1 addition & 0 deletions test-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
attrs==21.2.0
aiolimiter==1.1.0
bcrypt==3.2.0
black==25.11.0
build
Expand Down
7 changes: 3 additions & 4 deletions tests/slo/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
# syntax=docker/dockerfile:1

# This image packages the Python SLO workload runner for ydb-slo-action v2.
#
# This image packages the Python SLO workload runner for ydb-slo-action v3.
# The action launches the same image for `current` and `baseline` workloads
# in parallel and injects these env vars (see tests/slo/docker-entrypoint.sh):
#
# WORKLOAD_NAME sync-table | sync-query | topic
# WORKLOAD_REF "current" / "main" — used as the `ref` metric label
# WORKLOAD_NAME sync-table | sync-query | async-query | sync-topic | async-topic
# WORKLOAD_REF current ref / baseline ref — used as the `ref` metric label
# WORKLOAD_DURATION seconds
# YDB_ENDPOINT grpc://ydb:2136
# YDB_DATABASE /Root/testdb
Expand Down
2 changes: 1 addition & 1 deletion tests/slo/docker-entrypoint.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/sh
# Workload entrypoint used by ydb-slo-action v2.
# Workload entrypoint used by ydb-slo-action v3.
#
# The action launches the same image for both `current` and `baseline` workloads
# in parallel; both must be able to schema-prepare and then run.
Expand Down
77 changes: 69 additions & 8 deletions tests/slo/src/jobs/async_table_jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import logging
import time
from random import randint
from uuid import uuid4

# `aiolimiter` is a runtime dependency (see `tests/slo/requirements.txt`).
from aiolimiter import AsyncLimiter # pyright: ignore[reportMissingImports]
Expand All @@ -12,7 +13,8 @@
import ydb.aio

from .base import AsyncBaseJobManager
from .table_jobs import READ_QUERY_TEMPLATE, WRITE_QUERY_TEMPLATE
from .table_jobs import CONTROL_READ_QUERY, CONTROL_SCHEMA, READ_QUERY_TEMPLATE, WRITE_QUERY_TEMPLATE
from .table_jobs import check_control_result, control_row

logger = logging.getLogger(__name__)

Expand All @@ -33,12 +35,60 @@ async def run_tests(self):
if self.workload_type != "async-query":
raise ValueError(f"Unsupported async workload type: {self.workload_type}")

await self._run_control_checks()
tasks = [
*self._run_query_read_jobs(),
*self._run_query_write_jobs(),
*self._run_metric_job(),
]
await asyncio.gather(*tasks)
await self._wait_jobs(tasks)

async def _run_control_checks(self):
table = f"{self.table_name}_control_{uuid4().hex}"
write = WRITE_QUERY_TEMPLATE.format(table).replace("Digest::NumericHash($object_id)", "0")
types = (
ydb.PrimitiveType.Uint64,
ydb.PrimitiveType.Utf8,
ydb.PrimitiveType.Double,
ydb.PrimitiveType.Timestamp,
)
failure = None
async with ydb.aio.QuerySessionPool(self.aio_driver) as pool:
await pool.execute_with_retries(CONTROL_SCHEMA.format(table))
try:
for generation in (1, 2):
expected = control_row(generation)
values = {f"${key}": value for key, value in expected.items() if key != "object_hash"}
values = {key: (value, kind) for (key, value), kind in zip(values.items(), types)}
for query, params, operation, timeout, checker in (
(write, values, OP_TYPE_WRITE, self.args.write_timeout, None),
(
CONTROL_READ_QUERY.format(table),
{},
OP_TYPE_READ,
self.args.read_timeout,
lambda rows: check_control_result(rows, expected),
),
):
await self._execute_query(
pool,
query,
params,
ydb.BaseRequestSettings().with_timeout(timeout / 1000),
ydb.RetrySettings(idempotent=True, max_session_acquire_timeout=timeout / 1000),
(operation,),
check_result_cb=checker,
)
except BaseException as error:
failure = error
raise
finally:
try:
await pool.execute_with_retries(f"DROP TABLE `{table}`;")
except BaseException:
if failure is None:
raise
logger.exception("Cannot clean up control table %s", table)

def _run_query_read_jobs(self):
logger.info("Start async query read jobs")
Expand Down Expand Up @@ -118,7 +168,7 @@ async def _run_query_writes(self, query, limiter):

logger.info("Stop async query write workload")

async def _execute_query(self, pool, query, params, request_settings, retry_setting, labels):
async def _execute_query(self, pool, query, params, request_settings, retry_setting, labels, check_result_cb=None):
attempt = 0
error = None

Expand All @@ -127,18 +177,29 @@ async def callee(session):
attempt += 1

result_stream = await session.execute(query, parameters=params, settings=request_settings)
async for _ in result_stream:
pass
rows = []
async for part in result_stream:
rows.extend(part.rows)
return rows

ts = self.metrics.start(labels)

try:
await pool.retry_operation_async(callee, retry_setting)
rows = await pool.retry_operation_async(callee, retry_setting)
if check_result_cb:
check_result_cb(rows)
return rows
except asyncio.CancelledError:
raise
except ydb.Error as err:
error = err
self.record_failure(err)
logger.exception("[labels: %s] Cannot retry error:", labels)
raise
except BaseException as err:
error = err
self.record_failure(err)
logger.exception("[labels: %s] Unexpected error:", labels)

self.metrics.stop(labels, ts, attempts=attempt, error=error)
raise
finally:
self.metrics.stop(labels, ts, attempts=attempt, error=error)
48 changes: 42 additions & 6 deletions tests/slo/src/jobs/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,9 @@ class SyncRateLimiter:
threads sharing the same instance.
"""

def __init__(self, min_interval_s: float) -> None:
def __init__(self, min_interval_s: float, stop_event: threading.Event | None = None) -> None:
self._min_interval_s = max(0.0, float(min_interval_s))
self._stop_event = stop_event
self._lock = threading.Lock()
self._next_allowed_ts = 0.0 # monotonic timestamp

Expand All @@ -37,7 +38,10 @@ def __enter__(self):

# Sleep outside the lock so other threads can observe the schedule.
if sleep_for > 0:
time.sleep(sleep_for)
if self._stop_event is None:
time.sleep(sleep_for)
elif self._stop_event.wait(sleep_for):
return self

def __exit__(self, exc_type, exc, tb) -> bool:
# Do not suppress exceptions.
Expand All @@ -49,6 +53,26 @@ def __init__(self, driver, args, metrics):
self.driver: ydb.Driver = driver
self.args = args
self.metrics = metrics
self._failure = None
self._failure_lock = threading.Lock()
self._stop = threading.Event()

def record_failure(self, error):
with self._failure_lock:
if self._failure is None:
self._failure = error
self._stop.set()

def raise_if_failed(self):
if self._failure is not None:
raise self._failure

def _run_worker(self, worker, *args):
try:
worker(*args)
except BaseException as error:
self.record_failure(error)
logger.exception("Workload worker failed")

@abstractmethod
def run_tests(self):
Expand All @@ -59,12 +83,12 @@ def _run_metric_job(self):
return []

report_period_ms = max(1, int(self.args.report_period))
limiter = SyncRateLimiter(min_interval_s=report_period_ms / 1000.0)
limiter = SyncRateLimiter(min_interval_s=report_period_ms / 1000.0, stop_event=self._stop)

future = threading.Thread(
name="slo_metrics_sender",
target=self._metric_sender,
args=(limiter, self.args.time),
target=self._run_worker,
args=(self._metric_sender, limiter, self.args.time),
)
future.start()
return [future]
Expand All @@ -73,8 +97,10 @@ def _metric_sender(self, limiter, runtime):
start_time = time.time()
logger.info("Start push metrics")

while time.time() - start_time < runtime:
while not self._stop.is_set() and time.time() - start_time < runtime:
with limiter:
if self._stop.is_set():
break
self.metrics.push()

logger.info("Stop push metrics")
Expand All @@ -91,6 +117,16 @@ class AsyncBaseJobManager(BaseJobManager):
async def run_tests(self):
pass

async def _wait_jobs(self, tasks):
try:
await asyncio.gather(*tasks)
except BaseException as error:
self.record_failure(error)
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
self.raise_if_failed()

def _run_metric_job(self):
if not getattr(self.args, "otlp_endpoint", None):
return []
Expand Down
Loading
Loading