diff --git a/.github/workflows/slo-2dc.yml b/.github/workflows/slo-2dc.yml new file mode 100644 index 00000000..a8084e51 --- /dev/null +++ b/.github/workflows/slo-2dc.yml @@ -0,0 +1,99 @@ +name: Hosted 2DC SLO + +on: + pull_request: + paths: + - .github/workflows/slo-2dc.yml + - tests/slo/** + - ydb/driver.py + - ydb/resolver.py + - ydb/pool.py + - ydb/aio/driver.py + - ydb/aio/pool.py + workflow_dispatch: + inputs: + server_image: + description: YDB server image to qualify (resolved to a digest) + default: cr.yandex/crpb3d0up7s6a7iuofo9/ydb-appteam-ci/local-ydb:trunk + required: true + +permissions: + contents: read + +jobs: + bridge: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + concurrency: + group: hosted-2dc-${{ github.ref }} + cancel-in-progress: true + env: + WORKLOAD_MEMORY_LIMIT: 1G + WORKLOAD_CPU_LIMIT: '1.00' + WORKLOAD_CURRENT_IMAGE: py-slo-bridge:current + REQUESTED_IMAGE: ${{ inputs.server_image || 'cr.yandex/crpb3d0up7s6a7iuofo9/ydb-appteam-ci/local-ydb@sha256:bce9caa6a1a6aac9b513510b46f61c4d4b58c1f88d937bcb54c20dc1bda6c8d4' }} + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: ydb-platform/ydb-slo-action + ref: 54c3c03de1ce8c6a2d8b0828b328bd5db82ef2bb + path: .slo-action + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install the Compose version required by the action + run: | + set -euo pipefail + mkdir -p "$HOME/.docker/cli-plugins" + curl -fsSL --retry 3 \ + https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-linux-x86_64 \ + -o "$HOME/.docker/cli-plugins/docker-compose" + chmod +x "$HOME/.docker/cli-plugins/docker-compose" + docker compose version + docker compose up --help | grep -F -- --quiet-build + - name: Record runner and resolve server image + run: | + set -euo pipefail + mkdir -p bridge-artifacts + free -h + df -h + docker version + docker compose version + docker pull --platform linux/amd64 "$REQUESTED_IMAGE" + docker image inspect "$REQUESTED_IMAGE" > bridge-artifacts/server-image.json + SERVER_IMAGE=$(docker image inspect --format '{{index .RepoDigests 0}}' "$REQUESTED_IMAGE") + printf '%s\n' "$SERVER_IMAGE" > bridge-artifacts/server-image.txt + python tests/slo/bridge/prepare.py .slo-action "$SERVER_IMAGE" + - name: Build current Python SDK workload + run: docker build -f tests/slo/Dockerfile -t py-slo-bridge:current . + - name: Run all five Bridge scenarios + id: slo + uses: ./.slo-action/init + timeout-minutes: 20 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + workload_name: sync-query + workload_duration: '600' + workload_current_ref: current + workload_current_image: py-slo-bridge:current + workload_current_command: >- + --read-rps 50 --write-rps 10 --read-threads 2 --write-threads 2 + --read-timeout 30000 --write-timeout 30000 + --prefer-primary-pile + bridge_mode: true + disable_compose_profiles: extra-nodes + fail_on_workload_error: true + - name: Verify scenario coverage and workload recovery + if: always() + timeout-minutes: 3 + env: + ACTION_OUTCOME: ${{ steps.slo.outcome }} + run: python tests/slo/bridge/verify.py + - name: Upload qualification evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: python-hosted-2dc + path: bridge-artifacts/ + retention-days: 14 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a8302ae..714f276c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Add opt-in PRIMARY/PROMOTED Bridge pile preference with synchronized-pile fallback for sync and async drivers + ## 3.35.0 ## * Include streaming queries in scheme directory listings and path descriptions * Add timeout to the token exchange request so a stalled token endpoint cannot hang credential refresh diff --git a/docs/driver.rst b/docs/driver.rst index d5876240..e94ce103 100644 --- a/docs/driver.rst +++ b/docs/driver.rst @@ -48,6 +48,9 @@ DriverConfig * - ``discovery_request_timeout`` - ``10`` - Timeout in seconds for each discovery request. + * - ``prefer_primary_pile`` + - ``False`` + - Prefer PRIMARY/PROMOTED Bridge endpoints with synchronized-pile fallback; no effect without Bridge metadata. * - ``table_client_settings`` - ``None`` - :class:`~ydb.TableClientSettings` instance to configure the Table service client. @@ -62,6 +65,29 @@ DriverConfig - gRPC-level compression (e.g. ``grpc.Compression.Gzip``). +Preferring the primary Bridge pile +---------------------------------------- + +Enable this policy explicitly for sync or async drivers: + +.. code-block:: python + + config = ydb.DriverConfig( + endpoint="grpc://discovery.example:2135", + database="/Root/mydb", + prefer_primary_pile=True, + ) + driver = ydb.Driver(config) + +PRIMARY and PROMOTED endpoints take priority over SYNCHRONIZED endpoints. +Other or unknown pile states are last-resort candidates. The policy follows role +changes from discovery without recreating the driver, and takes precedence over +local-DC preferences while Bridge metadata is present, even with ``use_all_nodes=True``. +Without Bridge metadata the existing balancing behavior is preserved. +Requests bound to a session retain their node affinity; the option does not move +an existing session or perform a server-side failover. It is disabled by default. + + Creating a Driver ----------------- diff --git a/tests/aio/test_discovery_detect_local_dc.py b/tests/aio/test_discovery_detect_local_dc.py index ad534291..ca89c1a2 100644 --- a/tests/aio/test_discovery_detect_local_dc.py +++ b/tests/aio/test_discovery_detect_local_dc.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- +import asyncio import pytest from unittest.mock import MagicMock, patch, AsyncMock from ydb import driver, connection +from ydb import _apis, resolver from ydb.aio import pool, _utilities @@ -141,3 +143,178 @@ def mock_init(self, endpoint, driver_config, endpoint_options=None): await discovery.execute_discovery() assert detect_mock.call_count == 0, "detect_local_dc should NOT be called when use_all_nodes=True" + + +class PrimaryConnection: + failed = set() + created = [] + + def __init__(self, endpoint, driver_config, endpoint_options=None): + self.endpoint = endpoint + self.node_id = endpoint_options.node_id + self.callbacks = [] + self.closed = False + self.created.append(self) + + async def connection_ready(self, ready_timeout): + if self.endpoint in self.failed: + raise asyncio.TimeoutError() + + def add_cleanup_callback(self, callback): + self.callbacks.append(callback) + + async def close(self, grace=30): + self.closed = True + for callback in self.callbacks: + callback(self) + + +def primary_result(states, nodes=((1, "A"), (2, "B"))): + message = _apis.ydb_discovery.ListEndpointsResult(self_location="local") + for node_id, pile in nodes: + message.endpoints.add( + address=f"node{node_id}", + port=2135, + node_id=node_id, + location="local" if pile == "A" else "remote", + bridge_pile_name=pile, + ) + for name, state in states.items(): + message.pile_states.add(pile_name=name, state=state) + response = _apis.ydb_discovery.ListEndpointsResponse() + response.operation.ready = True + response.operation.status = 400000 + response.operation.result.Pack(message) + return resolver.DiscoveryResult.from_response(None, response, use_all_nodes=True) + + +def primary_discovery(monkeypatch, result, enabled=True, use_all_nodes=True, failed=()): + PrimaryConnection.failed, PrimaryConnection.created = set(failed), [] + monkeypatch.setattr(pool, "Connection", PrimaryConnection) + config = driver.DriverConfig("grpc://test:2135", "/local", use_all_nodes=use_all_nodes, prefer_primary_pile=enabled) + discovery = pool.Discovery(pool.ConnectionsCache(use_all_nodes), config) + discovery._resolver.resolve = AsyncMock(return_value=result) + return discovery + + +@pytest.mark.asyncio +async def test_primary_role_change_and_affinity(monkeypatch): + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3})) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1} + assert (await discovery._cache.get(connection.EndpointKey("node2:2135", 2))).node_id == 2 + discovery._resolver.resolve.return_value = primary_result({"A": 3, "B": 1}) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {2} + assert len(PrimaryConnection.created) == 2 + assert not any(conn.closed for conn in PrimaryConnection.created) + assert (await discovery._cache.get(connection.EndpointKey("node1:2135", 1))).node_id == 1 + await (await discovery._cache.get()).close() + assert (await discovery._cache.get()).node_id == 1 + + +@pytest.mark.asyncio +async def test_primary_off_and_ordinary_cluster_locality(monkeypatch): + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}), enabled=False) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1, 2} + discovery = primary_discovery(monkeypatch, primary_result({}), use_all_nodes=False) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1} + discovery._resolver.resolve.return_value = primary_result({"A": 3, "B": 1}) + assert await discovery.execute_discovery() + assert (await discovery._cache.get()).node_id == 2 + discovery._resolver.resolve.return_value = primary_result({}) + assert await discovery.execute_discovery() + assert (await discovery._cache.get()).node_id == 1 + + +@pytest.mark.asyncio +async def test_primary_promoted_and_unknown_states(monkeypatch): + discovery = primary_discovery( + monkeypatch, + primary_result({"A": 1, "B": 2, "C": 3, "D": 99}, ((1, "A"), (2, "B"), (3, "C"), (4, "D"), (5, "missing"))), + ) + assert await discovery.execute_discovery() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {1, 2} + await (await discovery._cache.get(connection.EndpointKey("node1:2135", 1))).close() + await (await discovery._cache.get(connection.EndpointKey("node2:2135", 2))).close() + assert (await discovery._cache.get()).node_id == 3 + await (await discovery._cache.get()).close() + assert {(await discovery._cache.get()).node_id for _ in range(10)} == {4, 5} + + +@pytest.mark.asyncio +async def test_primary_full_cache_admission_and_failed_connection(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + discovery = primary_discovery(monkeypatch, primary_result({"A": 1}, old)) + assert await discovery.execute_discovery() and discovery._cache.size == 9 + discovery._resolver.resolve.return_value = primary_result({"A": 3, "B": 1}, old + ((10, "B"),)) + PrimaryConnection.failed = {"node10:2135"} + assert await discovery.execute_discovery() + assert discovery._cache.size == 9 + assert not any(conn.closed for conn in PrimaryConnection.created if conn.node_id != 10) + PrimaryConnection.failed.clear() + assert await discovery.execute_discovery() + assert (await discovery._cache.get()).node_id == 10 + assert discovery._cache.size == 9 + created_count = len(PrimaryConnection.created) + assert await discovery.execute_discovery() + assert len(PrimaryConnection.created) == created_count + + +@pytest.mark.asyncio +async def test_primary_cache_preserves_synchronized_fallback(monkeypatch): + nodes = tuple((i, "A") for i in range(1, 11)) + ((11, "B"),) + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}, nodes)) + assert await discovery.execute_discovery() and discovery._cache.size == 9 + assert discovery._cache.already_exists("node11:2135") + for conn in discovery._cache.values(): + if conn.node_id != 11: + await conn.close() + assert (await discovery._cache.get()).node_id == 11 + + +@pytest.mark.asyncio +async def test_primary_alias_retirement_keeps_node_affinity(): + cache = pool.ConnectionsCache() + options = connection.EndpointOptions(node_id=1) + old = PrimaryConnection("old:2135", None, options) + new = PrimaryConnection("new:2135", None, options) + cache.add(old) + cache.add(new) + await old.close() + assert await cache.get(connection.EndpointKey(None, 1)) is new + + +@pytest.mark.asyncio +async def test_primary_driver_keyword_and_config_override(monkeypatch): + from ydb import topic + from ydb.aio import driver as aio_driver, table, scheme, coordination + + seen = [] + monkeypatch.setattr(pool.ConnectionPool, "__init__", lambda self, config: seen.append(config)) + for module, name in ( + (topic, "TopicClientAsyncIO"), + (table, "TableClient"), + (scheme, "SchemeClient"), + (coordination, "CoordinationClient"), + ): + monkeypatch.setattr(module, name, MagicMock()) + aio_driver.Driver(endpoint="grpc://test:2135", database="/local", prefer_primary_pile=True) + assert seen[-1].prefer_primary_pile + config = aio_driver.DriverConfig("grpc://test:2135", "/local", prefer_primary_pile=True) + aio_driver.Driver(config) + assert seen[-1].prefer_primary_pile + aio_driver.Driver(config, prefer_primary_pile=False) + assert not seen[-1].prefer_primary_pile + + +@pytest.mark.asyncio +async def test_primary_discovery_cancellation_closes_candidate(monkeypatch): + discovery = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3})) + monkeypatch.setattr(PrimaryConnection, "connection_ready", AsyncMock(side_effect=asyncio.CancelledError)) + with pytest.raises(asyncio.CancelledError): + await discovery.execute_discovery() + assert len(PrimaryConnection.created) == 1 + assert PrimaryConnection.created[0].closed diff --git a/tests/slo/README.md b/tests/slo/README.md index 88d46e72..6821ee48 100644 --- a/tests/slo/README.md +++ b/tests/slo/README.md @@ -1,5 +1,21 @@ # SLO workload +## GitHub-hosted Bridge SLO + +`.github/workflows/slo-2dc.yml` runs the standard five Bridge scenarios from the +pinned SLO Action v2 on `ubuntu-24.04`. The ordinary `sync-query` workload runs for +10 minutes with `--prefer-primary-pile` enabled. + +The runner uses two storage and two database nodes, `erasure=none` and SectorMap disks. +The adapter bounds resources and cold-start readiness time, restricts bootstrap DNS +to active compute nodes, and sets a 30-second gap between fault scenarios to fit the +10-minute workload. The upstream scenario scripts and SDK discovery intervals are unchanged. + +The job requires all five scenarios to complete, successful recent reads and writes, +and no workload failure, OOM or unexpected container restart. Transient operation +errors are recorded. Server image metadata, workload and chaos logs, final container +states and counters are retained in the `python-hosted-2dc` artifact for 14 days. + SLO is the type of test where app based on ydb-sdk is tested against falling YDB cluster nodes, tablets, network (that is possible situations for distributed DBs with hundreds of nodes) diff --git a/tests/slo/bridge/.gitignore b/tests/slo/bridge/.gitignore new file mode 100644 index 00000000..c18dd8d8 --- /dev/null +++ b/tests/slo/bridge/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/tests/slo/bridge/prepare.py b/tests/slo/bridge/prepare.py new file mode 100644 index 00000000..b0b14bdb --- /dev/null +++ b/tests/slo/bridge/prepare.py @@ -0,0 +1,73 @@ +"""Configure a pinned Action checkout for a bounded GitHub-hosted run.""" + +import argparse +import json +import subprocess +from pathlib import Path + + +def prepare(action, image): + deploy = action.resolve() / "deploy" + compose = deploy / "compose.bridge.yml" + rendered = subprocess.check_output( + [ + "docker", + "compose", + "-f", + str(compose), + "config", + "--no-interpolate", + "--format", + "json", + ], + text=True, + ) + config = json.loads(rendered) + for service in config["services"].values(): + if "build" in service and service["build"]["context"].endswith("/ydb"): + service["build"].setdefault("args", {})["YDB_IMAGE"] = image + service["mem_limit"] = "2g" + for name in ("workload-current", "workload-baseline"): + config["services"][name]["deploy"]["resources"]["reservations"]["memory"] = "256m" + service = config["services"][name] + hosts = service.get("extra_hosts", []) + if isinstance(hosts, dict): + hosts = [ + f"{host}:{address}" + for host, addresses in hosts.items() + for address in (addresses if isinstance(addresses, list) else [addresses]) + ] + service["extra_hosts"] = [host for host in hosts if not host.startswith(("ydb:", "ydb="))] + [ + "ydb:172.28.0.11", + "ydb:172.28.0.12", + ] + chaos = config["services"]["chaos-monkey"] + environment = chaos["environment"] + if isinstance(environment, list): + environment = {item.partition("=")[0]: item.partition("=")[2] if "=" in item else None for item in environment} + chaos["environment"] = {**environment, "CHAOS_SCENARIO_DELAY": "30"} + for filename in ("config.yaml", "config.bridge.yaml"): + path = deploy / "ydb/rootfs/opt/ydb.tech/ydbd/cfg" / filename + text = path.read_text() + if text.count("cpu_count: 4") != 1 or text.count("hard_limit_bytes: 17179869184") != 1: + raise RuntimeError(f"Unexpected resource configuration: {path}") + path.write_text( + text.replace("cpu_count: 4", "cpu_count: 1").replace( + "hard_limit_bytes: 17179869184", "hard_limit_bytes: 1610612736" + ) + ) + readiness = deploy / "ydb/rootfs/opt/ydb.tech/scripts/ydbd/check-readiness.sh" + text = readiness.read_text() + if text.count("local max_attempts=30") != 2: + raise RuntimeError("Unexpected readiness implementation") + readiness.write_text(text.replace("local max_attempts=30", "local max_attempts=150", 1)) + compose.write_text(json.dumps(config, indent=2) + "\n") + subprocess.run(["docker", "compose", "-f", str(compose), "config", "--quiet"], check=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("action", type=Path) + parser.add_argument("image") + args = parser.parse_args() + prepare(args.action, args.image) diff --git a/tests/slo/bridge/verify.py b/tests/slo/bridge/verify.py new file mode 100644 index 00000000..14257cb4 --- /dev/null +++ b/tests/slo/bridge/verify.py @@ -0,0 +1,124 @@ +"""Verify completion and recovery of the GitHub-hosted Bridge workload.""" + +import json +import math +import os +from pathlib import Path +import subprocess +import urllib.parse +import urllib.request + +SCENARIOS = ( + "01-switchover", + "02-takedown-synchronized", + "03-takedown-primary", + "04-failover-synchronized", + "05-failover-primary", +) + + +def command(*args, check=True, timeout=90, stderr=subprocess.STDOUT): + return subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=stderr, check=check, timeout=timeout).stdout + + +def check_chaos(log, state, restarts): + errors = [] + if state["Status"] != "exited" or state["ExitCode"] != 0 or restarts or "Chaos Monkey finished" not in log: + errors.append("Chaos sequence did not finish without a process restart") + for scenario in SCENARIOS: + if f"Running scenario: {scenario}" not in log: + errors.append(f"Scenario not executed: {scenario}") + if log.count("] Recovered: ") != len(SCENARIOS): + errors.append("Expected five completed chaos recoveries") + return errors + + +def main(): + artifacts = Path("bridge-artifacts") + artifacts.mkdir(exist_ok=True) + result = {"errors": [], "action_outcome": os.environ.get("ACTION_OUTCOME")} + for name in ("ydb-workload-current", "ydb-chaos-monkey"): + (artifacts / f"{name}.log").write_text(command("docker", "logs", name, check=False)) + try: + (artifacts / "healthcheck.json").write_text( + command( + "docker", + "exec", + "ydb-storage-1", + "/opt/ydb.tech/ydb/bin/ydb", + "-e", + "grpc://localhost:2136", + "monitoring", + "healthcheck", + "--format", + "json", + check=False, + timeout=30, + ) + ) + except Exception as error: + (artifacts / "healthcheck-error.txt").write_text(repr(error)) + try: + if result["action_outcome"] != "success": + result["errors"].append("Action did not complete successfully") + chaos_info = json.loads(command("docker", "inspect", "ydb-chaos-monkey"))[0] + chaos = chaos_info["State"] + log = (artifacts / "ydb-chaos-monkey.log").read_text() + result["errors"].extend(check_chaos(log, chaos, chaos_info["RestartCount"])) + result["completed_scenarios"] = len(SCENARIOS) if not result["errors"] else None + ip = command( + "docker", + "inspect", + "--format", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + "ydb-prometheus", + ).strip() + + def metric(expression): + url = f"http://{ip}:9090/api/v1/query?" + urllib.parse.urlencode({"query": expression}) + with urllib.request.urlopen(url, timeout=15) as response: + data = json.load(response) + if data["status"] != "success": + raise RuntimeError(data) + return sum(float(item["value"][1]) for item in data["data"]["result"]) + + observations = {} + for operation in ("read", "write"): + selector = f'sdk_operations_success_total{{ref="current",operation_type="{operation}"}}' + count, recent = metric(f"sum({selector})"), metric(f"sum(increase({selector}[120s]))") + observations[operation], observations[f"recent_{operation}"] = count, recent + if not math.isfinite(count) or not math.isfinite(recent) or count <= 0 or recent <= 0: + result["errors"].append(f"No recent successful native {operation} observations") + observations["errors"] = metric('sum(sdk_operations_failure_total{ref="current"})') + result["native_observations"] = observations + containers = json.loads( + command( + "docker", + "inspect", + "ydb-storage-1", + "ydb-storage-2", + "ydb-database-1", + "ydb-database-2", + "ydb-workload-current", + ) + ) + result["container_states"] = { + node["Name"]: {"state": node["State"], "restarts": node["RestartCount"]} for node in containers + } + if any(node["State"]["OOMKilled"] or node["RestartCount"] for node in containers): + result["errors"].append("A container was OOM killed or unexpectedly restarted") + for node in containers: + if node["Name"] == "/ydb-workload-current" and ( + node["State"]["Status"] != "exited" or node["State"]["ExitCode"] != 0 + ): + result["errors"].append(f"Workload process did not exit successfully: {node['Name']}") + except Exception as error: + result["errors"].append(repr(error)) + result["passed"] = not result["errors"] + (artifacts / "result.json").write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/slo/requirements.txt b/tests/slo/requirements.txt index 68f516f1..d9021cc5 100644 --- a/tests/slo/requirements.txt +++ b/tests/slo/requirements.txt @@ -4,6 +4,6 @@ hdrhistogram==0.10.3 # OpenTelemetry (OTLP/HTTP exporter) # NOTE: OpenTelemetry 1.39.1 requires Python >= 3.9. -opentelemetry-api==1.45.1 +opentelemetry-api==1.39.1 opentelemetry-sdk==1.39.1 opentelemetry-exporter-otlp-proto-http==1.39.1 diff --git a/tests/slo/src/options.py b/tests/slo/src/options.py index 200c3c55..9f289df4 100644 --- a/tests/slo/src/options.py +++ b/tests/slo/src/options.py @@ -1,4 +1,10 @@ import argparse +import ydb + + +def get_driver_config(args): + options = {"prefer_primary_pile": True} if getattr(args, "prefer_primary_pile", False) else {} + return ydb.DriverConfig(args.endpoint, database=args.db, grpc_keep_alive_timeout=5000, **options) def add_common_options(parser): @@ -7,6 +13,7 @@ def add_common_options(parser): parser.add_argument("-t", "--table-name", default="key_value", help="Table name") parser.add_argument("--debug", action="store_true", help="Enable debug logging") parser.add_argument("--async", action="store_true", help="Use async mode for operations") + parser.add_argument("--prefer-primary-pile", action="store_true", help="Prefer PRIMARY/PROMOTED Bridge endpoints") def make_table_create_parser(subparsers): diff --git a/tests/slo/src/root_runner.py b/tests/slo/src/root_runner.py index 2c8c2e75..5334a7ff 100644 --- a/tests/slo/src/root_runner.py +++ b/tests/slo/src/root_runner.py @@ -5,6 +5,7 @@ from typing import Dict from core.metrics import WORKLOAD +from options import get_driver_config from runners.topic_runner import TopicRunner from runners.table_runner import TableRunner from runners.base import BaseRunner @@ -45,11 +46,7 @@ def run_command(self, args): def _run_sync_command(self, args, runner_instance, command): """Run command in synchronous mode""" - driver_config = ydb.DriverConfig( - args.endpoint, - database=args.db, - grpc_keep_alive_timeout=5000, - ) + driver_config = get_driver_config(args) with ydb.Driver(driver_config) as driver: driver.wait(timeout=300) @@ -71,11 +68,7 @@ def _run_sync_command(self, args, runner_instance, command): async def _run_async_command(self, args, runner_instance, command): """Run command in asynchronous mode""" - driver_config = ydb.DriverConfig( - args.endpoint, - database=args.db, - grpc_keep_alive_timeout=5000, - ) + driver_config = get_driver_config(args) async with ydb.aio.Driver(driver_config) as driver: await driver.wait(timeout=300) diff --git a/tests/test_discovery_detect_local_dc.py b/tests/test_discovery_detect_local_dc.py index 874ffba7..77f30a79 100644 --- a/tests/test_discovery_detect_local_dc.py +++ b/tests/test_discovery_detect_local_dc.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from unittest.mock import Mock, MagicMock, patch from ydb import driver, pool, _utilities, connection +from ydb import _apis, resolver class MockEndpointInfo: @@ -118,3 +119,194 @@ def test_detect_local_dc_skipped_when_use_all_nodes_true(): discovery.execute_discovery() assert detect_mock.call_count == 0, "detect_local_dc should NOT be called when use_all_nodes=True" + + +class PrimaryConnection: + def __init__(self, endpoint, node_id): + self.endpoint, self.node_id = endpoint, node_id + self.closed = False + self.callbacks = [] + + def add_cleanup_callback(self, callback): + self.callbacks.append(callback) + + def close(self): + self.closed = True + for callback in self.callbacks: + callback(self) + + +def primary_result(states, nodes=((1, "A"), (2, "B"))): + message = _apis.ydb_discovery.ListEndpointsResult(self_location="local") + for node_id, pile in nodes: + message.endpoints.add( + address=f"node{node_id}", + port=2135, + node_id=node_id, + location="local" if pile == "A" else "remote", + bridge_pile_name=pile, + ) + for name, state in states.items(): + message.pile_states.add(pile_name=name, state=state) + response = _apis.ydb_discovery.ListEndpointsResponse() + response.operation.ready = True + response.operation.status = 400000 + response.operation.result.Pack(message) + return resolver.DiscoveryResult.from_response(None, response, use_all_nodes=True) + + +def primary_discovery(monkeypatch, result, enabled=True, use_all_nodes=True, failed=()): + created = [] + + def ready(endpoint, config, timeout, endpoint_options): + if endpoint in failed: + return None + conn = PrimaryConnection(endpoint, endpoint_options.node_id) + created.append(conn) + return conn + + monkeypatch.setattr(connection.Connection, "ready_factory", ready) + config = driver.DriverConfig("grpc://test:2135", "/local", use_all_nodes=use_all_nodes, prefer_primary_pile=enabled) + discovery = pool.Discovery(pool.ConnectionsCache(use_all_nodes), config) + discovery._resolver = MagicMock() + discovery._resolver.context_resolve.return_value.__enter__.return_value = result + return discovery, created + + +def test_primary_role_change_updates_existing_connections_and_affinity(monkeypatch): + discovery, created = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3})) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1} + assert discovery._cache.get(connection.EndpointKey("node2:2135", 2)).node_id == 2 + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result({"A": 3, "B": 1}) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {2} + assert len(created) == 2 and not any(conn.closed for conn in created) + assert discovery._cache.get(connection.EndpointKey("node1:2135", 1)).node_id == 1 + discovery._cache.get().close() + assert discovery._cache.get().node_id == 1 + + +def test_primary_option_is_off_by_default_and_ordinary_clusters_keep_locality(monkeypatch): + assert not driver.DriverConfig("grpc://test:2135").prefer_primary_pile + discovery, _ = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}), enabled=False) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1, 2} + discovery, created = primary_discovery(monkeypatch, primary_result({}), use_all_nodes=False) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1} + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result({"A": 3, "B": 1}) + assert discovery.execute_discovery() + assert discovery._cache.get().node_id == 2 + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result({}) + assert discovery.execute_discovery() + assert discovery._cache.get().node_id == 1 + assert len(created) == 2 + + +def test_primary_promoted_synchronized_and_unknown_fallback(monkeypatch): + result = primary_result({"A": 1, "B": 2, "C": 3, "D": 99}, ((1, "A"), (2, "B"), (3, "C"), (4, "D"), (5, "missing"))) + discovery, _ = primary_discovery(monkeypatch, result) + assert discovery.execute_discovery() + assert {discovery._cache.get().node_id for _ in range(10)} == {1, 2} + discovery._cache.get(connection.EndpointKey("node1:2135", 1)).close() + discovery._cache.get(connection.EndpointKey("node2:2135", 2)).close() + assert discovery._cache.get().node_id == 3 + discovery._cache.get().close() + assert {discovery._cache.get().node_id for _ in range(10)} == {4, 5} + + +def test_primary_new_endpoint_enters_full_cache(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + discovery, created = primary_discovery(monkeypatch, primary_result({"A": 1}, old)) + assert discovery.execute_discovery() and discovery._cache.size == 9 + discovery._resolver.context_resolve.return_value.__enter__.return_value = primary_result( + {"A": 3, "B": 1}, old + ((10, "B"),) + ) + assert discovery.execute_discovery() + assert discovery._cache.get().node_id == 10 + assert discovery._cache.size == 9 and sum(conn.closed for conn in created) == 1 + assert discovery.execute_discovery() + assert discovery._cache.size == 9 and len(created) == 10 + + +def test_failed_primary_does_not_evict_full_fallback_cache(monkeypatch): + old = tuple((i, "A") for i in range(1, 10)) + discovery, created = primary_discovery( + monkeypatch, primary_result({"A": 3, "B": 1}, old + ((10, "B"),)), failed={"node10:2135"} + ) + assert discovery.execute_discovery() + assert discovery._cache.size == 9 and not any(conn.closed for conn in created) + assert discovery._cache.get().node_id in range(1, 10) + + +def test_primary_cache_retains_synchronized_fallback(monkeypatch): + nodes = tuple((i, "A") for i in range(1, 11)) + ((11, "B"),) + discovery, _ = primary_discovery(monkeypatch, primary_result({"A": 1, "B": 3}, nodes)) + assert discovery.execute_discovery() + assert discovery._cache.size == 9 + assert discovery._cache.already_exists("node11:2135") + for conn in discovery._cache.values(): + if conn.node_id != 11: + conn.close() + assert discovery._cache.get().node_id == 11 + + +def test_primary_connection_retirement_preserves_other_address_node_affinity(): + cache = pool.ConnectionsCache() + old, new = PrimaryConnection("old:2135", 1), PrimaryConnection("new:2135", 1) + cache.add(old) + cache.add(new) + old.close() + assert cache.get(connection.EndpointKey(None, 1)) is new + + +def test_primary_driver_keyword_and_config_override(monkeypatch): + from ydb import topic, table, scheme, coordination + + seen = [] + monkeypatch.setattr(pool.ConnectionPool, "__init__", lambda self, config: seen.append(config)) + for module, name in ( + (topic, "TopicClient"), + (table, "TableClient"), + (scheme, "SchemeClient"), + (coordination, "CoordinationClient"), + ): + monkeypatch.setattr(module, name, MagicMock()) + driver.Driver(endpoint="grpc://test:2135", database="/local", prefer_primary_pile=True) + assert seen[-1].prefer_primary_pile + config = driver.DriverConfig("grpc://test:2135", "/local", prefer_primary_pile=True) + driver.Driver(config) + assert seen[-1].prefer_primary_pile + driver.Driver(config, prefer_primary_pile=False) + assert not seen[-1].prefer_primary_pile + + +def test_slo_primary_option_keeps_legacy_baseline_compatible(monkeypatch): + from tests.slo.src import options + + args = options.get_root_parser().parse_args( + ["table-run", "grpc://host:2136", "/Root/testdb", "--prefer-primary-pile"] + ) + assert options.get_driver_config(args).prefer_primary_pile + original = driver.DriverConfig + + def legacy_config(endpoint, database, grpc_keep_alive_timeout): + return original(endpoint, database=database, grpc_keep_alive_timeout=grpc_keep_alive_timeout) + + monkeypatch.setattr(options.ydb, "DriverConfig", legacy_config) + args = options.get_root_parser().parse_args(["table-run", "grpc://host:2136", "/Root/testdb"]) + assert not args.prefer_primary_pile + assert not options.get_driver_config(args).prefer_primary_pile + + +def test_bridge_slo_requires_all_completed_scenarios(): + from tests.slo.bridge.verify import SCENARIOS, check_chaos + + log = "\n".join(f"Running scenario: {name}\n[time] Recovered: {name}" for name in SCENARIOS) + log += "\nChaos Monkey finished" + state = {"Status": "exited", "ExitCode": 0} + assert not check_chaos(log, state, 0) + assert check_chaos(log.replace("Running scenario: 05-failover-primary", "skipped"), state, 0) + assert check_chaos(log.replace("] Recovered: 05-failover-primary", "] incomplete"), state, 0) + assert check_chaos(log, state, 1) diff --git a/ydb/aio/driver.py b/ydb/aio/driver.py index 88221e94..e1532366 100644 --- a/ydb/aio/driver.py +++ b/ydb/aio/driver.py @@ -56,11 +56,16 @@ def __init__( database: Optional[str] = None, root_certificates: Optional[bytes] = None, credentials: Optional["Credentials"] = None, + *, + prefer_primary_pile: Optional[bool] = None, **kwargs: Any, ) -> None: from .. import topic # local import for prevent cycle import error from . import coordination # local import for prevent cycle import error + if prefer_primary_pile is not None: + kwargs["prefer_primary_pile"] = prefer_primary_pile + config = get_config( driver_config, connection_string, diff --git a/ydb/aio/pool.py b/ydb/aio/pool.py index 15027436..c971b26e 100644 --- a/ydb/aio/pool.py +++ b/ydb/aio/pool.py @@ -3,6 +3,7 @@ import asyncio import logging import random +import grpc from typing import Any, Callable, Optional, Tuple, TYPE_CHECKING, cast from ydb import issues @@ -47,15 +48,7 @@ async def get( # async version with different Connection type if preferred_endpoint is not None and preferred_endpoint.endpoint in self.connections: return self.connections[preferred_endpoint.endpoint] # type: ignore[return-value] - for conn_lst in self.conn_lst_order: - try: - endpoint, connection = conn_lst.popitem(last=False) - conn_lst[endpoint] = connection - return connection # type: ignore[return-value] - except KeyError: - continue - - raise issues.ConnectionLost("Couldn't find valid connection") + return self._get_balanced_connection() # type: ignore[return-value] def add(self, connection: Optional[Connection], preferred: bool = False) -> bool: # type: ignore[override] # async Connection type if connection is None: @@ -81,7 +74,8 @@ def complete_discovery(self, error: Optional[Exception]) -> None: self._fast_fail_event.set() def remove(self, connection: Connection) -> None: # type: ignore[override] # async Connection type - self.connections_by_node_id.pop(connection.node_id, None) + if cast(Any, self.connections_by_node_id.get(connection.node_id)) is connection: + self.connections_by_node_id.pop(connection.node_id, None) self.preferred.pop(connection.endpoint, None) self.connections.pop(connection.endpoint, None) self.outdated.pop(connection.endpoint, None) @@ -147,9 +141,11 @@ async def execute_discovery(self) -> bool: self._cache.make_outdated(cached_endpoint) local_dc = resolve_details.self_location + priorities = resolve_details.bridge_priorities(True) if self._driver_config.prefer_primary_pile else None + self._cache.set_bridge_priorities(priorities) # Detect local DC using TCP latency if enabled and preferred is meaningful - if self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: + if priorities is None and self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: # Use only endpoints that match the SSL requirements for detection ssl_filtered_endpoints = [ endpoint @@ -200,13 +196,24 @@ async def execute_discovery(self) -> bool: endpoint, endpoint_options, ) in resolved_endpoint.endpoints_with_options(): - if self._cache.size >= self._max_size or self._cache.already_exists(endpoint): + if self._cache.already_exists(endpoint) or not self._cache.connection_slot(endpoint, self._max_size)[0]: continue ready_connection = Connection(endpoint, self._driver_config, endpoint_options=endpoint_options) - await ready_connection.connection_ready(ready_timeout=self._ready_timeout) - - self._cache.add(ready_connection, preferred) + try: + await ready_connection.connection_ready(ready_timeout=self._ready_timeout) + except BaseException as error: + await ready_connection.close(grace=0) + if priorities is not None and isinstance(error, (asyncio.TimeoutError, grpc.RpcError)): + continue + raise + allowed, replacement = self._cache.connection_slot(endpoint, self._max_size) + if allowed: + if replacement is not None: + self._cache.make_outdated(replacement) + self._cache.add(ready_connection, preferred) + else: + await ready_connection.close() await self._cache.cleanup_outdated() return self._cache.size > 0 diff --git a/ydb/driver.py b/ydb/driver.py index 98864980..0d6361f9 100644 --- a/ydb/driver.py +++ b/ydb/driver.py @@ -113,6 +113,7 @@ class DriverConfig(object): "compression", "disable_discovery", "detect_local_dc", + "prefer_primary_pile", "_additional_sdk_headers", ) @@ -141,6 +142,7 @@ def __init__( disable_discovery: bool = False, detect_local_dc: bool = False, *, + prefer_primary_pile: bool = False, _additional_sdk_headers: Tuple[str, ...] = (), ) -> None: """ @@ -167,6 +169,9 @@ def __init__( server-provided self_location. **Note**: This option only affects endpoint selection when use_all_nodes=False.\ When use_all_nodes=True (default), all endpoints are used regardless of detected location. :param _additional_sdk_headers: Reserved for SDK integrations (e.g. dbapi, sqlalchemy). Do not use in application code. + :param prefer_primary_pile: Prefer PRIMARY/PROMOTED Bridge endpoints, with synchronized-pile fallback.\ + Disabled by default and has no effect without Bridge metadata. Overrides locality while in Bridge mode;\ + existing sessions retain node affinity. """ self.endpoint = endpoint @@ -196,6 +201,7 @@ def __init__( self.compression = compression self.disable_discovery = disable_discovery self.detect_local_dc = detect_local_dc + self.prefer_primary_pile = prefer_primary_pile self._additional_sdk_headers = _additional_sdk_headers def set_database(self, database: str) -> "DriverConfig": @@ -301,6 +307,8 @@ def __init__( database: Optional[str] = None, root_certificates: Optional[bytes] = None, credentials: Optional["Credentials"] = None, + *, + prefer_primary_pile: Optional[bool] = None, **kwargs: Any, ) -> None: """ @@ -313,6 +321,7 @@ def __init__( :param endpoint: An endpoint specified in the following format: ://: :param database: A database path :param credentials: A credentials. If not specified credentials constructed by default. + :param prefer_primary_pile: Override the DriverConfig PRIMARY-pile preference when specified. """ from . import topic # local import for prevent cycle import error from . import coordination # local import for prevent cycle import error @@ -325,6 +334,8 @@ def __init__( root_certificates, credentials, ) + if prefer_primary_pile is not None: + driver_config.prefer_primary_pile = prefer_primary_pile super(Driver, self).__init__(driver_config) diff --git a/ydb/pool.py b/ydb/pool.py index 2ba1c33a..50ee37ea 100644 --- a/ydb/pool.py +++ b/ydb/pool.py @@ -7,7 +7,7 @@ from concurrent import futures import collections import random -from typing import Any, Callable, ContextManager, List, Optional, Set, Tuple, TYPE_CHECKING +from typing import Any, Callable, ContextManager, Dict, List, Optional, Set, Tuple, TYPE_CHECKING from . import connection as connection_impl, issues, resolver, _utilities, tracing from .observability.tracing import SpanName, create_ydb_span @@ -37,6 +37,48 @@ def __init__(self, use_all_nodes: bool = False, tracer: tracing.Tracer = tracing self.use_all_nodes = use_all_nodes self.conn_lst_order = (self.connections,) if self.use_all_nodes else (self.preferred, self.connections) self.fast_fail_subscriptions: Set["futures.Future[None]"] = set() + self._bridge_priorities: Optional[Dict[str, int]] = None + + def set_bridge_priorities(self, priorities: Optional[Dict[str, int]]) -> None: + with self.lock: + self._bridge_priorities = priorities + + def connection_slot(self, endpoint: str, max_size: int) -> Tuple[bool, Optional[Connection]]: + with self.lock: + active = [conn for key, conn in self.connections.items() if key not in self.outdated] + if len(active) < max_size: + return True, None + if self._bridge_priorities is None or not active: + return False, None + priorities = self._bridge_priorities + target = priorities.get(endpoint, 2) + worst = max(active, key=lambda conn: priorities.get(conn.endpoint, 2)) + ranks = [priorities.get(conn.endpoint, 2) for conn in active] + if target < max(ranks): + # Keep one synchronized connection for fallback when the cache has room for both tiers. + if max_size > 1 and target == 0 and max(ranks) == 1 and ranks.count(1) == 1 and 0 in ranks: + return False, None + return True, worst + if max_size > 1 and target == 1 and 1 not in ranks and ranks.count(0) > 1: + return True, next(conn for conn in active if priorities.get(conn.endpoint, 2) == 0) + return False, None + + def _get_balanced_connection(self) -> Connection: + if self._bridge_priorities is not None and self.connections: + priorities = self._bridge_priorities + best = min(priorities.get(endpoint, 2) for endpoint in self.connections) + for endpoint, conn in self.connections.items(): + if priorities.get(endpoint, 2) == best: + self.connections.move_to_end(endpoint) + return conn + for conn_lst in self.conn_lst_order: + try: + endpoint, conn = conn_lst.popitem(last=False) + conn_lst[endpoint] = conn + return conn + except KeyError: + continue + raise issues.ConnectionLost("Couldn't find valid connection") def add(self, connection: Optional[Connection], preferred: bool = False) -> bool: if connection is None: @@ -142,20 +184,14 @@ def get(self, preferred_endpoint: Optional[EndpointKey] = None) -> Connection: if preferred_endpoint is not None and preferred_endpoint.endpoint in self.connections: return self.connections[preferred_endpoint.endpoint] - for conn_lst in self.conn_lst_order: - try: - endpoint, connection = conn_lst.popitem(last=False) - conn_lst[endpoint] = connection - tracing.trace(self.tracer, {"found_in_lists": True}) - return connection - except KeyError: - continue - - raise issues.ConnectionLost("Couldn't find valid connection") + connection = self._get_balanced_connection() + tracing.trace(self.tracer, {"found_in_lists": True}) + return connection def remove(self, connection: Connection) -> None: with self.lock: - self.connections_by_node_id.pop(connection.node_id, None) + if self.connections_by_node_id.get(connection.node_id) is connection: + self.connections_by_node_id.pop(connection.node_id, None) self.preferred.pop(connection.endpoint, None) self.connections.pop(connection.endpoint, None) self.outdated.pop(connection.endpoint, None) @@ -238,9 +274,11 @@ def execute_discovery(self) -> bool: self._cache.make_outdated(cached_endpoint) local_dc = resolve_details.self_location + priorities = resolve_details.bridge_priorities(True) if self._driver_config.prefer_primary_pile else None + self._cache.set_bridge_priorities(priorities) # Detect local DC using TCP latency if enabled and preferred is meaningful - if self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: + if priorities is None and self._driver_config.detect_local_dc and not self._driver_config.use_all_nodes: # Use only endpoints that match the SSL requirements for detection ssl_filtered_endpoints = [ endpoint @@ -291,7 +329,10 @@ def execute_discovery(self) -> bool: endpoint, endpoint_options, ) in resolved_endpoint.endpoints_with_options(): - if self._cache.size >= self._max_size or self._cache.already_exists(endpoint): + if ( + self._cache.already_exists(endpoint) + or not self._cache.connection_slot(endpoint, self._max_size)[0] + ): continue ready_connection = connection_impl.Connection.ready_factory( @@ -300,7 +341,14 @@ def execute_discovery(self) -> bool: self._ready_timeout, endpoint_options=endpoint_options, ) - self._cache.add(ready_connection, preferred) + if ready_connection is not None: + allowed, replacement = self._cache.connection_slot(endpoint, self._max_size) + if allowed: + if replacement is not None: + self._cache.make_outdated(replacement) + self._cache.add(ready_connection, preferred) + else: + ready_connection.close() self._cache.cleanup_outdated() diff --git a/ydb/resolver.py b/ydb/resolver.py index 40a04980..fd66108d 100644 --- a/ydb/resolver.py +++ b/ydb/resolver.py @@ -7,14 +7,16 @@ import random import itertools import typing -from typing import Any, ContextManager, List, Optional, Iterator +from typing import Any, ContextManager, Dict, List, Optional, Iterator from . import connection as conn_impl, driver, issues, settings as settings_impl, _apis # Workaround for good IDE and universal for runtime if typing.TYPE_CHECKING: from ._grpc.v4.protos import ydb_discovery_pb2 + from ._grpc.v4.protos import ydb_bridge_common_pb2 else: from ._grpc.common.protos import ydb_discovery_pb2 + from ._grpc.common.protos import ydb_bridge_common_pb2 logger = logging.getLogger(__name__) @@ -31,6 +33,7 @@ class EndpointInfo(object): "ipv6_addrs", "ssl_target_name_override", "node_id", + "bridge_pile_name", ) def __init__(self, endpoint_info: ydb_discovery_pb2.EndpointInfo): @@ -43,6 +46,7 @@ def __init__(self, endpoint_info: ydb_discovery_pb2.EndpointInfo): self.ipv6_addrs = tuple(endpoint_info.ip_v6) self.ssl_target_name_override = endpoint_info.ssl_target_name_override self.node_id = endpoint_info.node_id + self.bridge_pile_name = endpoint_info.bridge_pile_name def endpoints_with_options(self) -> typing.Generator[typing.Tuple[str, conn_impl.EndpointOptions], None, None]: ssl_target_name_override = None @@ -95,9 +99,27 @@ def _list_endpoints_request_factory(connection_params: driver.DriverConfig) -> _ class DiscoveryResult(object): - def __init__(self, self_location: str, endpoints: "list[EndpointInfo]"): + def __init__( + self, self_location: str, endpoints: "list[EndpointInfo]", pile_states: Optional[Dict[str, int]] = None + ): self.self_location = self_location self.endpoints = endpoints + self.pile_states = pile_states or {} + + def bridge_priorities(self, enabled: bool) -> Optional[Dict[str, int]]: + if not enabled or not self.pile_states: + return None + priorities = {} + for info in self.endpoints: + state = self.pile_states.get(info.bridge_pile_name, ydb_bridge_common_pb2.PileState.UNSPECIFIED) + priority = ( + 0 + if state in (ydb_bridge_common_pb2.PileState.PRIMARY, ydb_bridge_common_pb2.PileState.PROMOTED) + else (1 if state == ydb_bridge_common_pb2.PileState.SYNCHRONIZED else 2) + ) + for endpoint, _ in info.endpoints_with_options(): + priorities[endpoint] = priority + return priorities def __str__(self): return "DiscoveryResult " % ( @@ -139,7 +161,7 @@ def from_response( result.extend(local_endpoints) result.extend(different_endpoints) - return cls(message.self_location, result) + return cls(message.self_location, result, {pile.pile_name: pile.state for pile in message.pile_states}) class DiscoveryEndpointsResolver(object):