From 46cbf19c7b58505368d5e1955f5139b525c6382b Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Wed, 4 Nov 2026 12:44:20 +0200 Subject: [PATCH 1/4] Implement Agent operation lifecycle coordination --- .../assert-operation-lifecycle/action.yml | 48 ++ .../assert-operation-lifecycle/assert.py | 301 +++++++ .../create-fixture/fixtures/manifest.py | 56 ++ .../operations/lifecycle/action.yml | 61 ++ .github/workflows/rpm-installed-test.yml | 8 + docs/index.md | 8 +- docs/sfp-operation-lifecycle.md | 70 ++ mkdocs.yml | 1 + pyproject.toml | 2 +- .../_journal/sqlite/operation_admissions.py | 191 ++++- src/stateforge_agent/_runtime/daemon.py | 4 + src/stateforge_agent/_sfp/__init__.py | 4 + .../_sfp/_runtime/sessions.py | 108 ++- .../_sfp/_services/manifest/declaration.py | 4 + .../operation_admission/coordinator.py | 11 + .../_services/operation_admission/store.py | 25 + .../_services/operation_lifecycle/__init__.py | 1 + .../_services/operation_lifecycle/client.py | 188 +++++ .../operation_lifecycle/coordinator.py | 387 +++++++++ .../_services/operation_lifecycle/models.py | 105 +++ tests/test_operation_admission.py | 3 + tests/test_operation_lifecycle.py | 748 ++++++++++++++++++ tests/test_rpm_automation.py | 4 +- uv.lock | 4 +- 24 files changed, 2331 insertions(+), 11 deletions(-) create mode 100644 .github/actions/agent/assert-operation-lifecycle/action.yml create mode 100644 .github/actions/agent/assert-operation-lifecycle/assert.py create mode 100644 .github/actions/rpm/test-installed/operations/lifecycle/action.yml create mode 100644 docs/sfp-operation-lifecycle.md create mode 100644 src/stateforge_agent/_sfp/_services/operation_lifecycle/__init__.py create mode 100644 src/stateforge_agent/_sfp/_services/operation_lifecycle/client.py create mode 100644 src/stateforge_agent/_sfp/_services/operation_lifecycle/coordinator.py create mode 100644 src/stateforge_agent/_sfp/_services/operation_lifecycle/models.py create mode 100644 tests/test_operation_lifecycle.py diff --git a/.github/actions/agent/assert-operation-lifecycle/action.yml b/.github/actions/agent/assert-operation-lifecycle/action.yml new file mode 100644 index 0000000..a9213a1 --- /dev/null +++ b/.github/actions/agent/assert-operation-lifecycle/action.yml @@ -0,0 +1,48 @@ +name: Assert Agent Operation Lifecycle +description: Exercise RPM-installed Agent lifecycle observation against a real Provider. + +inputs: + container-name: + description: Fedora container running the installed Agent and Provider. + required: true + provider-instance-id: + description: Provider installation identity published by the fixture. + required: true + provider-state-name: + description: Fixture state directory containing operation evidence. + required: false + default: provider-a + evidence-directory: + description: Host directory receiving sanitized lifecycle evidence. + required: true + +runs: + using: composite + steps: + - name: Exercise installed Agent Operation Lifecycle + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + CONTAINER_NAME: ${{ inputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + PROVIDER_INSTANCE_ID: ${{ inputs.provider-instance-id }} + PROVIDER_STATE_NAME: ${{ inputs.provider-state-name }} + run: | + set -euo pipefail + mkdir -p "${EVIDENCE_DIRECTORY}" + verifier_directory="/opt/stateforge-agent-tests/operation-lifecycle" + verifier="${verifier_directory}/assert.py" + state_directory="/var/lib/stateforge-provider-fixtures/${PROVIDER_STATE_NAME}" + docker exec "${CONTAINER_NAME}" mkdir -p "${verifier_directory}" + docker cp "${ACTION_PATH}/assert.py" \ + "${CONTAINER_NAME}:${verifier}" + set +e + docker exec "${CONTAINER_NAME}" /usr/bin/python3 "${verifier}" \ + --provider-instance-id "${PROVIDER_INSTANCE_ID}" \ + --provider-state-name "${PROVIDER_STATE_NAME}" \ + > "${EVIDENCE_DIRECTORY}/operation-lifecycle.json" + status=$? + set -e + docker exec "${CONTAINER_NAME}" cat "${state_directory}/events.log" \ + > "${EVIDENCE_DIRECTORY}/operation-events.txt" 2>&1 || true + exit "${status}" diff --git a/.github/actions/agent/assert-operation-lifecycle/assert.py b/.github/actions/agent/assert-operation-lifecycle/assert.py new file mode 100644 index 0000000..534606b --- /dev/null +++ b/.github/actions/agent/assert-operation-lifecycle/assert.py @@ -0,0 +1,301 @@ +"""Exercise lifecycle observation through the RPM-installed Agent package.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from datetime import UTC, datetime, timedelta +from importlib.metadata import version +from pathlib import Path +from uuid import UUID, uuid4 + +from stateforge_protocol.sfp.operation_lifecycle.v1 import ( + operation_lifecycle_pb2, +) + +from stateforge_agent._journal import SqliteJournal +from stateforge_agent._sfp import ( + OperationAdmissionRequest, + ProviderAvailabilityState, + ProviderSessionManager, + SfpVersion, +) +from stateforge_agent._sfp._runtime.descriptor import load_provider_descriptor +from stateforge_agent._sfp._services.operation_admission.coordinator import ( + OperationAdmissionCoordinator, +) +from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAttempt, +) +from stateforge_agent._sfp._services.operation_lifecycle.coordinator import ( + OperationLifecycleCoordinator, +) +from stateforge_agent._sfp._services.operation_lifecycle.models import ( + OperationStatus, +) + +_AGENT_JOURNAL = Path("/var/lib/stateforge-agent/journal/agent.db") +_DISCOVERY_DIRECTORY = Path("/run/stateforge-agent/providers") +_STATE_DIRECTORY = Path("/var/lib/stateforge-provider-fixtures") +_OPERATION = "stateforge.test/lifecycle@1" +_UNCANCELLABLE = "stateforge.test/lifecycle.uncancellable@1" +_TERMINAL = frozenset( + { + "OPERATION_STATE_SUCCEEDED", + "OPERATION_STATE_FAILED", + "OPERATION_STATE_CANCELLED", + } +) +_COMPLETE_PERCENTAGE = 100 +_TERMINAL_NUMBERS = frozenset( + { + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + operation_lifecycle_pb2.OPERATION_STATE_FAILED, + operation_lifecycle_pb2.OPERATION_STATE_CANCELLED, + } +) + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--provider-instance-id", type=UUID, required=True) + parser.add_argument("--provider-state-name", required=True) + return parser.parse_args() + + +def _events(path: Path) -> tuple[str, ...]: + return tuple(path.read_text(encoding="utf-8").splitlines()) + + +async def _wait_status( + manager: ProviderSessionManager, + provider_id: UUID, + operation_id: UUID, + wanted: set[str] | frozenset[str], +) -> OperationStatus: + async with asyncio.timeout(20): + while True: + status = await manager.operation_status(provider_id, operation_id) + if status is not None and status.state in wanted: + return status + await asyncio.sleep(0.05) + + +async def _wait_journal_terminal( + journal: SqliteJournal, + provider_id: UUID, + operation_id: UUID, +) -> OperationAttempt: + store = journal.operation_admissions(maximum_unresolved_attempts=32) + async with asyncio.timeout(20): + while True: + attempt = await store.get(provider_id, operation_id) + if attempt is not None and attempt.lifecycle_snapshot is not None: + state = attempt.lifecycle_snapshot.state + if state in _TERMINAL_NUMBERS: + return attempt + await asyncio.sleep(0.05) + + +async def _restart_agent() -> None: + process = await asyncio.create_subprocess_exec( + "/usr/bin/systemctl", "restart", "stateforge-agent.service" + ) + if await process.wait() != 0: + raise AssertionError("Installed Agent service restart failed") + + +async def _manager( + journal: SqliteJournal, + provider_id: UUID, +) -> ProviderSessionManager: + store = journal.operation_admissions(maximum_unresolved_attempts=32) + await store.initialize() + manager = ProviderSessionManager( + agent_runtime_id=uuid4(), + agent_software_version=version("stateforge-agent"), + supported_sfp_versions=(SfpVersion(major=1, minor=0),), + capability_assessment_retry_initial_delay_seconds=0.05, + operation_admission=OperationAdmissionCoordinator(store), + operation_lifecycle=OperationLifecycleCoordinator( + store, retry_initial_delay_seconds=0.05 + ), + ) + await manager.establish( + load_provider_descriptor( + _DISCOVERY_DIRECTORY / f"{provider_id}.sfp.json", + discovery_directory=_DISCOVERY_DIRECTORY, + ) + ) + async with asyncio.timeout(15): + while True: + availability = manager.availability_status(provider_id) + if ( + availability is not None + and availability.state is ProviderAvailabilityState.AVAILABLE + ): + return manager + await asyncio.sleep(0.05) + + +async def _admit( + manager: ProviderSessionManager, + provider_id: UUID, + value: str, + *, + identifier: str = _OPERATION, +) -> UUID: + result = await manager.admit_operation( + OperationAdmissionRequest( + provider_instance_id=provider_id, + operation_identifier=identifier, + payload={"value": value}, + execution_deadline=datetime.now(UTC) + timedelta(minutes=3), + execution_context={"scenario": "installed-operation-lifecycle"}, + ) + ) + return result.operation_id + + +async def _run( # noqa: C901, PLR0912, PLR0915 - scenario assertions + arguments: argparse.Namespace, +) -> dict[str, object]: + provider_id: UUID = arguments.provider_instance_id + state_directory = _STATE_DIRECTORY / arguments.provider_state_name + event_log = state_directory / "events.log" + journal = SqliteJournal(_AGENT_JOURNAL) + await journal.open() + manager = await _manager(journal, provider_id) + seen_ids: list[UUID] = [] + try: + success_id = await _admit(manager, provider_id, "success") + seen_ids.append(success_id) + success = await _wait_status( + manager, provider_id, success_id, _TERMINAL + ) + if success.state != "OPERATION_STATE_SUCCEEDED": + raise AssertionError("Installed Provider success was not observed") + if success.result != {"value": "success"}: + raise AssertionError("Typed result was not normalized") + if success.progress_percentage != _COMPLETE_PERCENTAGE: + raise AssertionError("Latest progress was not retained") + + held_id = await _admit(manager, provider_id, "held") + seen_ids.append(held_id) + await _wait_status( + manager, provider_id, held_id, {"OPERATION_STATE_RUNNING"} + ) + finally: + await manager.close() + + # The operation stays in the Provider while the observer and Agent restart. + await _restart_agent() + (state_directory / "lifecycle-release").touch() + persisted = await _wait_journal_terminal(journal, provider_id, held_id) + if ( + persisted.lifecycle_snapshot is None + or persisted.lifecycle_snapshot.state + != operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED + ): + raise AssertionError("Daemon did not resume outstanding observation") + + manager = await _manager(journal, provider_id) + try: + cancel_id = await _admit(manager, provider_id, "cancel") + seen_ids.append(cancel_id) + await _wait_status( + manager, provider_id, cancel_id, {"OPERATION_STATE_RUNNING"} + ) + requested = await manager.cancel_operation(provider_id, cancel_id) + if requested.state != "OPERATION_STATE_CANCELLATION_REQUESTED": + raise AssertionError("Cancellation request fabricated an outcome") + if requested.cancellation_disposition != ( + "CANCELLATION_DISPOSITION_REQUESTED" + ): + raise AssertionError("Provider cancellation decision was lost") + (state_directory / "cancellation-release").touch() + cancelled = await _wait_status( + manager, provider_id, cancel_id, _TERMINAL + ) + if cancelled.state != "OPERATION_STATE_CANCELLED": + raise AssertionError("Confirmed cancellation was not retained") + if cancelled.side_effects != "SIDE_EFFECT_STATE_PRESENT": + raise AssertionError("Cancellation effects were not retained") + + failure_id = await _admit(manager, provider_id, "failure") + seen_ids.append(failure_id) + failed = await _wait_status( + manager, provider_id, failure_id, _TERMINAL + ) + if failed.state != "OPERATION_STATE_FAILED" or failed.failure_code != ( + "stateforge.test.fixture_failure" + ): + raise AssertionError("Structured failure was not retained") + if failed.cleanup != "CLEANUP_STATE_INCOMPLETE": + raise AssertionError("Partial cleanup evidence was lost") + + unsupported_id = await _admit( + manager, provider_id, "unsupported", identifier=_UNCANCELLABLE + ) + seen_ids.append(unsupported_id) + await _wait_status( + manager, provider_id, unsupported_id, {"OPERATION_STATE_RUNNING"} + ) + unsupported = await manager.cancel_operation( + provider_id, unsupported_id + ) + if unsupported.cancellation_disposition != ( + "CANCELLATION_DISPOSITION_UNSUPPORTED" + ): + raise AssertionError("Unsupported cancellation was misclassified") + (state_directory / "uncancellable-release").touch() + await _wait_status(manager, provider_id, unsupported_id, _TERMINAL) + + already_terminal = await manager.cancel_operation( + provider_id, success_id + ) + if already_terminal.cancellation_disposition != ( + "CANCELLATION_DISPOSITION_ALREADY_TERMINAL" + ): + raise AssertionError("Terminal cancellation was misclassified") + if already_terminal.state != "OPERATION_STATE_SUCCEEDED": + raise AssertionError("Terminal success was replaced") + finally: + await manager.close() + await journal.close() + + events = _events(event_log) + for operation_id in seen_ids: + dispatches = sum( + event.endswith(str(operation_id)) + and event.startswith("operation:") + for event in events + ) + if dispatches != 1: + raise AssertionError("Watch or Cancel redispatched business work") + return { + "agent_version": version("stateforge-agent"), + "protocol_version": version("stateforge-protocol"), + "provider_version": version("stateforge-provider"), + "operation_ids": [str(item) for item in seen_ids], + "daemon_resumed_after_restart": True, + "typed_success_and_progress": True, + "requested_then_confirmed_cancelled": True, + "partial_failure_evidence": True, + "unsupported_and_already_terminal": True, + "no_redispatch": True, + } + + +def main() -> None: + """Emit sanitized installed-scenario evidence as JSON.""" + sys.stdout.write( + json.dumps(asyncio.run(_run(_arguments())), indent=2, sort_keys=True) + + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/.github/actions/provider/create-fixture/fixtures/manifest.py b/.github/actions/provider/create-fixture/fixtures/manifest.py index 3584959..25ac04e 100644 --- a/.github/actions/provider/create-fixture/fixtures/manifest.py +++ b/.github/actions/provider/create-fixture/fixtures/manifest.py @@ -23,6 +23,12 @@ ) from stateforge_provider import ( HealthStatus, + OperationCancelledError, + OperationCleanup, + OperationContext, + OperationEffects, + OperationFailedError, + OperationSideEffects, Platform, Provider, ProviderModel, @@ -113,6 +119,7 @@ def __init__( ) self._event_log = Path(os.environ["STATEFORGE_TEST_EVENT_LOG"]) self._target_path = Path(os.environ["STATEFORGE_TEST_TARGET_PATH"]) + self._state_directory = state_directory async def on_startup(self) -> None: """Install deterministic health transitions for system tests.""" @@ -228,6 +235,55 @@ async def indeterminate(self, request: FixtureInput) -> FixtureResult: self._record("operation:indeterminate") return FixtureResult(value=request.value) + @operation("stateforge.test/lifecycle@1", cancellable=True) + async def lifecycle( + self, request: FixtureInput, context: OperationContext + ) -> FixtureResult: + """Expose deterministic real execution for Agent lifecycle tests.""" + self._record(f"operation:lifecycle:{context.operation_id}") + await context.report_progress("Started", percentage=0) + if request.value == "held": + await self._wait_for_marker("lifecycle-release") + elif request.value in {"cancel", "cancel-success"}: + await context.wait_for_cancellation() + await context.report_progress("Stopping", percentage=50) + await self._wait_for_marker("cancellation-release") + if request.value == "cancel": + raise OperationCancelledError( + OperationEffects( + side_effects=OperationSideEffects.PRESENT, + cleanup=OperationCleanup.COMPLETED, + explanation="Temporary fixture state was removed.", + ) + ) + elif request.value == "failure": + raise OperationFailedError( + code="stateforge.test.fixture_failure", + explanation="Fixture operation failed after partial work.", + effects=OperationEffects( + side_effects=OperationSideEffects.PRESENT, + cleanup=OperationCleanup.INCOMPLETE, + explanation="Fixture work remains for inspection.", + ), + ) + await context.report_progress("Finished", percentage=100) + return FixtureResult(value=request.value) + + @operation("stateforge.test/lifecycle.uncancellable@1") + async def lifecycle_uncancellable( + self, request: FixtureInput, context: OperationContext + ) -> FixtureResult: + """Keep one operation active while cancellation is unsupported.""" + self._record(f"operation:uncancellable:{context.operation_id}") + await self._wait_for_marker("uncancellable-release") + return FixtureResult(value=request.value) + + async def _wait_for_marker(self, name: str) -> None: + """Wait for the installed verifier to release one execution.""" + marker = self._state_directory / name + while not marker.exists(): # noqa: ASYNC110 - cross-process marker + await asyncio.sleep(0.05) + @trigger("stateforge.test/runtime.started@1") async def runtime_started(self) -> AsyncIterator[FixtureTriggerPayload]: """Publish one event through the decorated public Provider API.""" diff --git a/.github/actions/rpm/test-installed/operations/lifecycle/action.yml b/.github/actions/rpm/test-installed/operations/lifecycle/action.yml new file mode 100644 index 0000000..5396cf9 --- /dev/null +++ b/.github/actions/rpm/test-installed/operations/lifecycle/action.yml @@ -0,0 +1,61 @@ +name: Test installed Operation Lifecycle +description: Verify the RPM Agent watches and cancels real Provider work. + +inputs: + rpm-directory: + description: Directory containing the exact RPM artifact under test. + required: false + default: dist/rpm + evidence-directory: + description: Directory receiving this scenario's evidence. + required: false + default: dist/agent-system-test-evidence/operation-lifecycle + fixture-directory: + description: Directory containing the exact Provider fixture bundle. + required: false + default: dist/provider-fixture + +runs: + using: composite + steps: + - name: Prepare installed Agent + id: setup + uses: ./.github/actions/rpm/test-installed/setup + with: + rpm-directory: ${{ inputs.rpm-directory }} + evidence-directory: ${{ inputs.evidence-directory }} + fixture-directory: ${{ inputs.fixture-directory }} + + - name: Start operation-capable Provider + id: provider + uses: ./.github/actions/provider/start + with: + container-name: ${{ steps.setup.outputs.container-name }} + fixture-path: ${{ steps.setup.outputs.fixture-path }} + wheel-directory: ${{ steps.setup.outputs.wheel-directory }} + provider-name: provider-a + provider-identity: stateforge.test.operation_lifecycle + + - name: Require Agent readiness + uses: ./.github/actions/agent/assert-availability + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + endpoint-generation: ${{ steps.provider.outputs.endpoint-generation }} + state: available + journal-cursor: ${{ steps.provider.outputs.agent-journal-cursor }} + evidence-directory: ${{ inputs.evidence-directory }}/availability + + - name: Exercise Operation Lifecycle through the installed Agent + uses: ./.github/actions/agent/assert-operation-lifecycle + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Preserve evidence and clean up + if: ${{ always() }} + uses: ./.github/actions/rpm/test-installed/cleanup + with: + container-name: ${{ steps.setup.outputs.container-name }} + evidence-directory: ${{ inputs.evidence-directory }} diff --git a/.github/workflows/rpm-installed-test.yml b/.github/workflows/rpm-installed-test.yml index 035a922..fe9e0cd 100644 --- a/.github/workflows/rpm-installed-test.yml +++ b/.github/workflows/rpm-installed-test.yml @@ -108,6 +108,7 @@ jobs: - capability-mismatch - capability-limits - operation-admission + - operation-lifecycle - provider-availability - trigger-publication - trigger-storage-failure @@ -161,6 +162,13 @@ jobs: evidence-directory: dist/agent-system-test-evidence/${{ matrix.scenario }} fixture-directory: dist/provider-fixture + - name: Run Operation Lifecycle scenario + if: ${{ matrix.scenario == 'operation-lifecycle' }} + uses: ./.github/actions/rpm/test-installed/operations/lifecycle + with: + evidence-directory: dist/agent-system-test-evidence/${{ matrix.scenario }} + fixture-directory: dist/provider-fixture + - name: Run multiple-Providers scenario if: ${{ matrix.scenario == 'multiple-providers' }} uses: ./.github/actions/rpm/test-installed/lifecycle/multiple-providers diff --git a/docs/index.md b/docs/index.md index 1a9ac37..8b1c2aa 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,7 +35,9 @@ The current runtime can: them; and - admit manifest-declared operations only after current capability evidence, persist their immutable identities before transport, and recover uncertain - responses without dispatching duplicate work. + responses without dispatching duplicate work; and +- watch admitted operations, retain their latest progress and terminal + outcomes, and request cooperative operation-scoped cancellation. The runtime reports a Provider as connected only after both session setup and manifest validation succeed. Operational eligibility is stricter: the exact @@ -59,3 +61,7 @@ Operation Admission is similarly fenced to the exact available session, manifest, declaration, and capability evidence. A lost response is retried with the same durable operation identity; the Provider resolves the existing record rather than running the operation twice. + +Operation Lifecycle resumes Watch from the last durable sequence after an +interruption. A cancellation request is recorded separately from the final +outcome; only a terminal Provider snapshot confirms that execution stopped. diff --git a/docs/sfp-operation-lifecycle.md b/docs/sfp-operation-lifecycle.md new file mode 100644 index 0000000..9cb2cb4 --- /dev/null +++ b/docs/sfp-operation-lifecycle.md @@ -0,0 +1,70 @@ +# Operation Lifecycle + +After Operation Admission proves that a Provider accepted an operation, the +Agent watches that operation's current state through SFP. Watching does not +start execution again. It uses the admission record's operation ID and request +fingerprint, and it resumes from the last sequence retained in the Agent +journal. + +```mermaid +sequenceDiagram + participant J as Agent journal + participant A as Agent lifecycle coordinator + participant P as Provider + + A->>J: Read accepted operation and last sequence + A->>P: WatchOperation(operation ID, fingerprint, cursor) + P-->>A: Current snapshot and later updates + A->>A: Validate session, admission identity, sequence, result schema + A->>J: Retain latest snapshot + Note over A,P: A lost stream never changes the business outcome + A->>J: Read last retained sequence + A->>P: WatchOperation(same operation, new cursor) + P-->>A: Current state or immutable terminal replay +``` + +## Local status + +`ProviderSessionManager.operation_status(provider_instance_id, operation_id)` +returns the Agent's durable local `OperationStatus`, or `None` for an unknown +identity. It does not make a Provider call. The status separates admission +from execution state and includes the latest sequence and progress, a typed +successful result, or a structured failure/cancellation outcome with side +effects and cleanup evidence. A known accepted operation without a retained +snapshot has an unknown execution state; the Agent does not infer success or +failure from admission alone. + +Only accepted records with a pinned result schema are watched. The Agent +validates each terminal success against the result schema captured at +admission, even if a later manifest revision changes the declaration. The +journal preserves one latest snapshot, not an unlimited progress history. +It rejects conflicting snapshots at the same sequence and never replaces a +terminal outcome. + +## Cancellation + +`ProviderSessionManager.cancel_operation(provider_instance_id, operation_id)` +requests cooperative cancellation of one admitted operation. The Provider's +response distinguishes `REQUESTED`, `ALREADY_REQUESTED`, `UNSUPPORTED`, and +`ALREADY_TERMINAL`. `REQUESTED` means the Provider has received the request; +it does **not** mean execution has stopped. Only a validated terminal +`CANCELLED` snapshot confirms cancellation. Success or failure may win the +race after a cancellation request, and the Agent retains that actual terminal +outcome. A timeout, lost Watch, Agent restart, or health loss does not +fabricate a terminal state or cancel unrelated operations. + +## Interruption and recovery + +The coordinator bounds simultaneous Watch calls. On a retryable stream +failure it reconnects to the same admitted identity using the last durable +sequence; on a replacement Provider session it uses the new session fences. +Observer interruption changes no business execution. When the packaged Agent +restarts, it scans accepted nonterminal records and resumes observation after +establishing a current Provider session. Complete state-loss reconciliation +belongs to the later Disconnection and Recovery milestone. + +The installed-system scenario uses the exact Agent RPM plus Protocol and +Provider wheels. It verifies real progress, typed success, structured failure, +cooperative cancellation, unsupported and already-terminal decisions, +resumption after Agent service restart, and exactly one Provider dispatch per +operation identity. diff --git a/mkdocs.yml b/mkdocs.yml index bedff0b..8e8cd94 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -20,6 +20,7 @@ nav: - Configuration: configuration.md - Capability Assessment: sfp-capability-assessment.md - Operation Admission: sfp-operation-admission.md + - Operation Lifecycle: sfp-operation-lifecycle.md - Trigger Publication: sfp-trigger-publication.md - RPM CI/CD: ci-cd.md - Internal reference: diff --git a/pyproject.toml b/pyproject.toml index 6d94c25..468f065 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ environments = ["python_version == '3.13'"] [tool.uv.sources] -stateforge-protocol = { git = "https://github.com/stateforge-org/stateforge-protocol.git", rev = "e4d67e8fd7abbb077e09c41664e6f2945760037e" } +stateforge-protocol = { git = "https://github.com/stateforge-org/stateforge-protocol.git", rev = "9504be252d9cd15ac53fe479bfb09a3007a16aca" } [tool.hatch.build.targets.wheel] diff --git a/src/stateforge_agent/_journal/sqlite/operation_admissions.py b/src/stateforge_agent/_journal/sqlite/operation_admissions.py index a007391..889215d 100644 --- a/src/stateforge_agent/_journal/sqlite/operation_admissions.py +++ b/src/stateforge_agent/_journal/sqlite/operation_admissions.py @@ -11,6 +11,12 @@ from stateforge_protocol.sfp.operation_admission.v1 import ( operation_admission_pb2, ) +from stateforge_protocol.sfp.operation_lifecycle.v1 import ( + operation_lifecycle_pb2, +) +from stateforge_protocol.sfp.operation_lifecycle.v1 import ( + validation as lifecycle_validation, +) from stateforge_agent._sfp._services.operation_admission.models import ( OperationAdmissionDisposition, @@ -26,7 +32,14 @@ ) _SCHEMA_COMPONENT = "operation_admission" -_SCHEMA_VERSION = 1 +_SCHEMA_VERSION = 2 +_TERMINAL_STATES = frozenset( + { + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + operation_lifecycle_pb2.OPERATION_STATE_FAILED, + operation_lifecycle_pb2.OPERATION_STATE_CANCELLED, + } +) _Result = TypeVar("_Result") @@ -64,6 +77,8 @@ async def create_intent( provider_instance_id: UUID, invocation: operation_admission_pb2.OperationInvocation, request_fingerprint: str, + *, + result_schema_json: bytes | None = None, ) -> OperationIntentResult: """Atomically insert or classify one immutable invocation.""" serialized = invocation.SerializeToString(deterministic=True) @@ -79,6 +94,10 @@ def create(connection: sqlite3.Connection) -> OperationIntentResult: decision = ( OperationIntentDecision.EXISTING if existing.request_fingerprint == request_fingerprint + and ( + result_schema_json is None + or existing.result_schema_json == result_schema_json + ) else OperationIntentDecision.CONFLICT ) return OperationIntentResult(decision, existing) @@ -96,17 +115,19 @@ def create(connection: sqlite3.Connection) -> OperationIntentResult: operation_id, invocation, request_fingerprint, + result_schema_json, state, created_at, updated_at, attempt_count - ) VALUES (?, ?, ?, ?, ?, ?, ?, 0) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0) """, ( str(provider_instance_id), invocation.operation_id, serialized, request_fingerprint, + result_schema_json, OperationAttemptState.UNRESOLVED.value, now, now, @@ -298,6 +319,135 @@ def read( return await self._journal.transaction(read) + async def accepted_nonterminal(self) -> tuple[OperationAttempt, ...]: + """Return accepted records that still need observation.""" + + def read( + connection: sqlite3.Connection, + ) -> tuple[OperationAttempt, ...]: + rows = connection.execute( + f"{self._SELECT} WHERE state = ? AND lifecycle_terminal = 0 " + "ORDER BY created_at", + (OperationAttemptState.ACCEPTED.value,), + ).fetchall() + return tuple(self._attempt_from_row(row) for row in rows) + + return await self._journal.transaction(read) + + async def record_lifecycle_snapshot( + self, + provider_instance_id: UUID, + operation_id: UUID, + snapshot: operation_lifecycle_pb2.OperationSnapshot, + *, + cancellation_disposition: int | None = None, + ) -> OperationAttempt: + """Commit newer evidence without replacing a terminal outcome.""" + lifecycle_validation.validate_operation_snapshot(snapshot) + serialized = snapshot.SerializeToString(deterministic=True) + now = datetime.now(UTC).isoformat() + + def record( # noqa: PLR0912 - atomic replay and terminal decisions + connection: sqlite3.Connection, + ) -> OperationAttempt: + current = self._select( + connection, provider_instance_id, operation_id + ) + if ( + current is None + or current.state is not OperationAttemptState.ACCEPTED + or current.result is None + ): + raise ValueError( + "Lifecycle evidence has no accepted Agent intent" + ) + admission = snapshot.admission_record + if ( + admission.operation_id != str(operation_id) + or admission.request_fingerprint != current.request_fingerprint + or snapshot.operation_identifier + != current.invocation.operation_identifier + or snapshot.manifest_revision + != current.invocation.contract.manifest_revision + or admission.admitted_endpoint_generation + != current.result.admitted_endpoint_generation + or admission.admitted_at.ToDatetime(tzinfo=UTC) + != current.result.admitted_at + or admission.record_expires_at.ToDatetime(tzinfo=UTC) + != current.result.record_expires_at + ): + raise ValueError( + "Lifecycle evidence does not match durable admission" + ) + previous = current.lifecycle_snapshot + retained_bytes = serialized + terminal = int(snapshot.state in _TERMINAL_STATES) + if previous is not None: + if previous.state in _TERMINAL_STATES: + if snapshot.sequence < previous.sequence: + retained_bytes = previous.SerializeToString( + deterministic=True + ) + terminal = 1 + else: + lifecycle_validation.validate_terminal_replay( + previous, snapshot + ) + elif snapshot.sequence == previous.sequence: + if snapshot != previous: + raise ValueError( + "Lifecycle sequence changed without advancing" + ) + elif snapshot.sequence > previous.sequence: + if snapshot.sequence == previous.sequence + 1: + lifecycle_validation.validate_snapshot_transition( + previous, snapshot + ) + elif ( + snapshot.admission_record != previous.admission_record + or snapshot.operation_identifier + != previous.operation_identifier + or snapshot.manifest_revision + != previous.manifest_revision + ): + raise ValueError( + "Coalesced lifecycle evidence changed identity" + ) + else: + retained_bytes = previous.SerializeToString( + deterministic=True + ) + terminal = 0 + connection.execute( + """ + UPDATE operation_admission_attempts + SET lifecycle_snapshot = ?, lifecycle_terminal = ?, + cancellation_disposition = COALESCE( + ?, cancellation_disposition + ), + updated_at = ? + WHERE provider_instance_id = ? AND operation_id = ? + """, + ( + retained_bytes, + terminal, + cancellation_disposition, + now, + str(provider_instance_id), + str(operation_id), + ), + ) + updated = self._select( + connection, provider_instance_id, operation_id + ) + if updated is None: + raise sqlite3.DatabaseError( + "Lifecycle evidence disappeared during commit" + ) + return updated + + return await self._journal.transaction(record) + _SELECT = """ SELECT provider_instance_id, operation_id, invocation, request_fingerprint, state, created_at, updated_at, @@ -307,7 +457,8 @@ def read( record_expires_at, admitted_endpoint_generation, rejection_reason, rejection_retryable, rejection_retry_after_seconds, rejection_explanation, - rejection_grpc_status + rejection_grpc_status, result_schema_json, + lifecycle_snapshot, cancellation_disposition FROM operation_admission_attempts """ @@ -376,6 +527,19 @@ def _attempt_from_row(row: tuple[object, ...]) -> OperationAttempt: last_session_id=(_string(row[9]) if row[9] is not None else None), result=result, rejection=rejection, + result_schema_json=( + _bytes(row[22]) if row[22] is not None else None + ), + lifecycle_snapshot=( + operation_lifecycle_pb2.OperationSnapshot.FromString( + _bytes(row[23]) + ) + if row[23] is not None + else None + ), + cancellation_disposition=( + _integer(row[24]) if row[24] is not None else None + ), ) @staticmethod @@ -405,7 +569,7 @@ def _initialize(connection: sqlite3.Connection) -> None: "SELECT version FROM journal_schemas WHERE component = ?", (_SCHEMA_COMPONENT,), ).fetchone() - if row is not None and row[0] != _SCHEMA_VERSION: + if row is not None and row[0] not in {1, _SCHEMA_VERSION}: raise sqlite3.DatabaseError( "Unsupported Operation Admission journal schema" ) @@ -436,10 +600,29 @@ def _initialize(connection: sqlite3.Connection) -> None: rejection_retry_after_seconds REAL, rejection_explanation TEXT, rejection_grpc_status TEXT, + result_schema_json BLOB, + lifecycle_snapshot BLOB, + lifecycle_terminal INTEGER NOT NULL DEFAULT 0, + cancellation_disposition INTEGER, PRIMARY KEY (provider_instance_id, operation_id) ) WITHOUT ROWID """ ) + if row is not None and row[0] == 1: + for column in ( + "result_schema_json BLOB", + "lifecycle_snapshot BLOB", + "lifecycle_terminal INTEGER NOT NULL DEFAULT 0", + "cancellation_disposition INTEGER", + ): + connection.execute( + "ALTER TABLE operation_admission_attempts " + f"ADD COLUMN {column}" + ) + connection.execute( + "UPDATE journal_schemas SET version = ? WHERE component = ?", + (_SCHEMA_VERSION, _SCHEMA_COMPONENT), + ) connection.execute( """ INSERT OR IGNORE INTO journal_schemas (component, version) diff --git a/src/stateforge_agent/_runtime/daemon.py b/src/stateforge_agent/_runtime/daemon.py index 16b1e81..ff3e6f7 100644 --- a/src/stateforge_agent/_runtime/daemon.py +++ b/src/stateforge_agent/_runtime/daemon.py @@ -42,6 +42,9 @@ from stateforge_agent._sfp._services.operation_admission.coordinator import ( OperationAdmissionCoordinator, ) +from stateforge_agent._sfp._services.operation_lifecycle.coordinator import ( + OperationLifecycleCoordinator, +) from stateforge_agent._sfp._services.trigger_publication.coordinator import ( TriggerPublicationCoordinator, ) @@ -138,6 +141,7 @@ def reload_provider_descriptor( configuration.providers.operation_admission_retry_initial_delay_seconds ), ), + operation_lifecycle=OperationLifecycleCoordinator(operation_store), ) try: diff --git a/src/stateforge_agent/_sfp/__init__.py b/src/stateforge_agent/_sfp/__init__.py index 85d1674..17ef863 100644 --- a/src/stateforge_agent/_sfp/__init__.py +++ b/src/stateforge_agent/_sfp/__init__.py @@ -52,6 +52,9 @@ OperationAdmissionPreconditionError, OperationAdmissionUnresolvedError, ) +from stateforge_agent._sfp._services.operation_lifecycle.models import ( + OperationStatus, +) from stateforge_agent._sfp.version import SfpVersion __all__ = [ @@ -70,6 +73,7 @@ "OperationAdmissionResult", "OperationAdmissionUnresolvedError", "OperationRejection", + "OperationStatus", "ProviderAvailabilityState", "ProviderAvailabilityStatus", "ProviderConnectionState", diff --git a/src/stateforge_agent/_sfp/_runtime/sessions.py b/src/stateforge_agent/_sfp/_runtime/sessions.py index e996430..658f133 100644 --- a/src/stateforge_agent/_sfp/_runtime/sessions.py +++ b/src/stateforge_agent/_sfp/_runtime/sessions.py @@ -63,6 +63,14 @@ OperationAdmissionRequest, OperationAdmissionResult, ) +from stateforge_agent._sfp._services.operation_lifecycle.coordinator import ( + OperationLifecycleCoordinator, + OperationLifecycleLease, + OperationLifecyclePreconditionError, +) +from stateforge_agent._sfp._services.operation_lifecycle.models import ( + OperationStatus, +) from stateforge_agent._sfp._services.trigger_publication.client import ( SfpTriggerPublicationError, SfpTriggerPublicationStaleError, @@ -195,6 +203,7 @@ def __init__( ) = None, trigger_publication: TriggerPublicationCoordinator | None = None, operation_admission: OperationAdmissionCoordinator | None = None, + operation_lifecycle: OperationLifecycleCoordinator | None = None, ) -> None: """Create an empty Provider session registry.""" if not isinstance(agent_runtime_id, UUID): @@ -261,6 +270,7 @@ def __init__( ) self._trigger_publication = trigger_publication self._operation_admission = operation_admission + self._operation_lifecycle = operation_lifecycle self._providers: dict[UUID, _ProviderRuntimeState] = {} self._closed = False self._lock = asyncio.Lock() @@ -341,12 +351,14 @@ async def admit_operation( lease = await self._operation_admission_lease( request.provider_instance_id ) - return await coordinator.admit( + result = await coordinator.admit( lease, request, assessment, assert_current=self._assert_operation_admission_is_current, ) + await self._resume_operation_observation(request.provider_instance_id) + return result async def resolve_operation( self, @@ -371,12 +383,93 @@ async def resolve_operation( execution_context=execution_context, ) lease = await self._operation_admission_lease(provider_instance_id) - return await coordinator.submit( + result = await coordinator.submit( lease, operation_id, assessment, assert_current=self._assert_operation_admission_is_current, ) + await self._resume_operation_observation(provider_instance_id) + return result + + async def operation_status( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationStatus | None: + """Return durable local knowledge of one operation.""" + coordinator = self._operation_lifecycle + if coordinator is None: + raise OperationLifecyclePreconditionError( + "Operation Lifecycle is not configured" + ) + return await coordinator.status(provider_instance_id, operation_id) + + async def cancel_operation( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationStatus: + """Request cancellation for one admitted operation only.""" + coordinator = self._operation_lifecycle + if coordinator is None: + raise OperationLifecyclePreconditionError( + "Operation Lifecycle is not configured" + ) + lease = await self._operation_lifecycle_lease(provider_instance_id) + return await coordinator.cancel( + lease, + operation_id, + assert_current=self._assert_operation_lifecycle_is_current, + ) + + async def _resume_operation_observation( + self, provider_instance_id: UUID + ) -> None: + coordinator = self._operation_lifecycle + if coordinator is None: + return + try: + lease = await self._operation_lifecycle_lease(provider_instance_id) + except OperationLifecyclePreconditionError: + return + await coordinator.resume( + lease, + assert_current=self._assert_operation_lifecycle_is_current, + ) + + async def _operation_lifecycle_lease( + self, provider_instance_id: UUID + ) -> OperationLifecycleLease: + """Use the current channel even when Health blocks new admission.""" + async with self._lock: + provider = self._providers.get(provider_instance_id) + if provider is None or provider.active_session is None: + raise OperationLifecyclePreconditionError( + "Provider has no active session for Operation Lifecycle" + ) + session = provider.active_session + return OperationLifecycleLease( + provider_instance_id=session.provider_instance_id, + endpoint_generation=session.endpoint_generation, + session_id=session.session_id, + channel=session._channel, + session_token=session, + ) + + async def _assert_operation_lifecycle_is_current( + self, lease: OperationLifecycleLease + ) -> None: + async with self._lock: + provider = self._providers.get(lease.provider_instance_id) + if ( + self._closed + or provider is None + or provider.active_session is not lease.session_token + ): + raise OperationLifecyclePreconditionError( + "Operation Lifecycle session is no longer current" + ) async def _operation_admission_lease( self, @@ -623,6 +716,7 @@ async def _establish_once( await _cancel_task(replaced_health_watcher) if replaced_session is not None: await replaced_session._channel.close() + await self._resume_operation_observation(result.provider_instance_id) return result async def _connect_candidate( @@ -770,6 +864,10 @@ async def remove(self, provider_instance_id: UUID) -> None: await _cancel_task(watcher) await _cancel_task(health_watcher) await _cancel_task(trigger_watcher) + if session is not None and self._operation_lifecycle is not None: + await self._operation_lifecycle.suspend( + provider_instance_id, session + ) if session is not None: await session._channel.close() @@ -851,6 +949,8 @@ async def close(self) -> None: await asyncio.gather(*health_watchers, return_exceptions=True) if trigger_watchers: await asyncio.gather(*trigger_watchers, return_exceptions=True) + if self._operation_lifecycle is not None: + await self._operation_lifecycle.close() for session in sessions: await session._channel.close() self._capability_assessment.clear() @@ -1427,6 +1527,10 @@ async def _invalidate_lost_session( ) await _cancel_task(health_watcher) await _cancel_task(trigger_watcher) + if self._operation_lifecycle is not None: + await self._operation_lifecycle.suspend( + session.provider_instance_id, session + ) await session._channel.close() await self._schedule_retry(session.provider_instance_id) diff --git a/src/stateforge_agent/_sfp/_services/manifest/declaration.py b/src/stateforge_agent/_sfp/_services/manifest/declaration.py index ae9901f..569d406 100644 --- a/src/stateforge_agent/_sfp/_services/manifest/declaration.py +++ b/src/stateforge_agent/_sfp/_services/manifest/declaration.py @@ -33,6 +33,7 @@ class ManifestOperationDeclaration: """Expose the manifest facts needed to preflight one operation.""" input_schema: Mapping[str, Any] + result_schema: Mapping[str, Any] implements: frozenset[str] @@ -89,15 +90,18 @@ def operation_declarations( ): raise RuntimeError("Validated manifest cache entry is corrupt") input_schema = declaration.get("inputSchema") + result_schema = declaration.get("resultSchema") implements = declaration.get("implements") if ( not isinstance(input_schema, dict) + or not isinstance(result_schema, dict) or not isinstance(implements, list) or any(not isinstance(item, str) for item in implements) ): raise RuntimeError("Validated manifest cache entry is corrupt") declarations[identifier] = ManifestOperationDeclaration( input_schema=MappingProxyType(input_schema), + result_schema=MappingProxyType(result_schema), implements=frozenset(implements), ) return MappingProxyType(declarations) diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py b/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py index f205da2..4c21495 100644 --- a/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py +++ b/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py @@ -4,6 +4,7 @@ import asyncio import hashlib +import json from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass, field from datetime import UTC, datetime @@ -189,11 +190,21 @@ async def prepare( ) ) fingerprint = validation.compute_request_fingerprint(invocation) + declaration = lease.operation_declarations[ + request.operation_identifier + ] + result_schema_json = json.dumps( + dict(declaration.result_schema), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") try: stored = await self._store.create_intent( lease.provider_instance_id, invocation, fingerprint, + result_schema_json=result_schema_json, ) except JournalError as error: raise OperationAdmissionCapacityError( diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/store.py b/src/stateforge_agent/_sfp/_services/operation_admission/store.py index aae2727..6fc0d6f 100644 --- a/src/stateforge_agent/_sfp/_services/operation_admission/store.py +++ b/src/stateforge_agent/_sfp/_services/operation_admission/store.py @@ -11,6 +11,9 @@ from stateforge_protocol.sfp.operation_admission.v1 import ( operation_admission_pb2, ) +from stateforge_protocol.sfp.operation_lifecycle.v1 import ( + operation_lifecycle_pb2, +) from stateforge_agent._sfp._services.operation_admission.models import ( OperationAdmissionResult, @@ -54,6 +57,11 @@ class OperationAttempt: last_session_id: str | None result: OperationAdmissionResult | None = None rejection: OperationRejection | None = None + result_schema_json: bytes | None = field(default=None, repr=False) + lifecycle_snapshot: operation_lifecycle_pb2.OperationSnapshot | None = ( + field(default=None, repr=False, compare=False) + ) + cancellation_disposition: int | None = None @dataclass(frozen=True, slots=True) @@ -77,6 +85,8 @@ async def create_intent( provider_instance_id: UUID, invocation: operation_admission_pb2.OperationInvocation, request_fingerprint: str, + *, + result_schema_json: bytes | None = None, ) -> OperationIntentResult: """Atomically create or classify one immutable operation identity.""" @@ -118,3 +128,18 @@ async def record_rejected( @abstractmethod async def unresolved(self) -> tuple[OperationAttempt, ...]: """Return unresolved attempts available for restart recovery.""" + + @abstractmethod + async def accepted_nonterminal(self) -> tuple[OperationAttempt, ...]: + """Return admitted work whose terminal state is not yet retained.""" + + @abstractmethod + async def record_lifecycle_snapshot( + self, + provider_instance_id: UUID, + operation_id: UUID, + snapshot: operation_lifecycle_pb2.OperationSnapshot, + *, + cancellation_disposition: int | None = None, + ) -> OperationAttempt: + """Atomically retain a validated snapshot and cancellation decision.""" diff --git a/src/stateforge_agent/_sfp/_services/operation_lifecycle/__init__.py b/src/stateforge_agent/_sfp/_services/operation_lifecycle/__init__.py new file mode 100644 index 0000000..650ac85 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/operation_lifecycle/__init__.py @@ -0,0 +1 @@ +"""Coordinate observation and cancellation of admitted SFP operations.""" diff --git a/src/stateforge_agent/_sfp/_services/operation_lifecycle/client.py b/src/stateforge_agent/_sfp/_services/operation_lifecycle/client.py new file mode 100644 index 0000000..14d3280 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/operation_lifecycle/client.py @@ -0,0 +1,188 @@ +"""Call the canonical SFP Operation Lifecycle service.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Protocol + +import grpc +from google.protobuf.message import DecodeError +from stateforge_protocol.sfp.operation_lifecycle.v1 import ( + contract, + operation_lifecycle_pb2, + validation, +) + +from stateforge_agent._configuration_validation import require_finite_positive + + +class SfpOperationLifecycleError(RuntimeError): + """Report an invalid or failed lifecycle exchange without payload data.""" + + +class SfpOperationLifecycleProtocolError(SfpOperationLifecycleError): + """Report malformed, oversized, or inconsistent Provider evidence.""" + + +class SfpOperationLifecycleTransportError(SfpOperationLifecycleError): + """Retain the gRPC status of one inconclusive lifecycle call.""" + + def __init__(self, code: grpc.StatusCode) -> None: + super().__init__(f"Operation Lifecycle transport failed: {code.name}") + self.code = code + + +class OperationLifecycleRequester(Protocol): + """Describe the bounded lifecycle calls used by Agent coordination.""" + + def watch( + self, + channel: grpc.aio.Channel, + request: operation_lifecycle_pb2.WatchOperationRequest, + ) -> AsyncIterator[operation_lifecycle_pb2.WatchOperationResponse]: + """Yield validated responses from one Watch call.""" + ... + + async def cancel( + self, + channel: grpc.aio.Channel, + request: operation_lifecycle_pb2.CancelOperationRequest, + ) -> operation_lifecycle_pb2.CancelOperationResponse: + """Return one validated cancellation decision.""" + ... + + +def _method_path(name: str) -> str: + service = operation_lifecycle_pb2.DESCRIPTOR.services_by_name[ + "OperationLifecycleService" + ] + return f"/{service.full_name}/{service.methods_by_name[name].name}" + + +def _decode_watch( + payload: bytes, +) -> operation_lifecycle_pb2.WatchOperationResponse: + if len(payload) > contract.MAX_RESPONSE_BYTES: + raise SfpOperationLifecycleProtocolError( + "Operation Lifecycle Watch response exceeds its limit" + ) + try: + return operation_lifecycle_pb2.WatchOperationResponse.FromString( + payload + ) + except DecodeError as error: + raise SfpOperationLifecycleProtocolError( + "Operation Lifecycle Watch response is malformed" + ) from error + + +def _decode_cancel( + payload: bytes, +) -> operation_lifecycle_pb2.CancelOperationResponse: + if len(payload) > contract.MAX_RESPONSE_BYTES: + raise SfpOperationLifecycleProtocolError( + "Operation Lifecycle Cancel response exceeds its limit" + ) + try: + return operation_lifecycle_pb2.CancelOperationResponse.FromString( + payload + ) + except DecodeError as error: + raise SfpOperationLifecycleProtocolError( + "Operation Lifecycle Cancel response is malformed" + ) from error + + +class OperationLifecycleClient: + """Use one multiplexed Provider channel without redispatching work.""" + + def __init__(self, *, timeout_seconds: float = 5.0) -> None: + self._timeout_seconds = require_finite_positive( + timeout_seconds, name="Operation Lifecycle timeout" + ) + + def watch( + self, + channel: grpc.aio.Channel, + request: operation_lifecycle_pb2.WatchOperationRequest, + ) -> AsyncIterator[operation_lifecycle_pb2.WatchOperationResponse]: + """Watch one already admitted identity with bounded responses.""" + return self._watch(channel, request) + + async def _watch( + self, + channel: grpc.aio.Channel, + request: operation_lifecycle_pb2.WatchOperationRequest, + ) -> AsyncIterator[operation_lifecycle_pb2.WatchOperationResponse]: + try: + validation.validate_watch_operation_request(request) + except validation.OperationLifecycleValidationError as error: + raise SfpOperationLifecycleProtocolError( + "Operation Lifecycle Watch request is invalid" + ) from error + call = channel.unary_stream( + _method_path("WatchOperation"), + request_serializer=( + operation_lifecycle_pb2.WatchOperationRequest.SerializeToString + ), + response_deserializer=_decode_watch, + ) + previous_sequence: int | None = None + terminal = False + try: + async for response in call(request): + try: + validation.validate_watch_exchange( + request, + response, + previous_sequence=previous_sequence, + ) + except validation.OperationLifecycleValidationError as error: + raise SfpOperationLifecycleProtocolError( + "Operation Lifecycle Watch exchange is invalid" + ) from error + previous_sequence = response.snapshot.sequence + terminal = response.snapshot.state in { + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + operation_lifecycle_pb2.OPERATION_STATE_FAILED, + operation_lifecycle_pb2.OPERATION_STATE_CANCELLED, + } + yield response + if not terminal: + raise SfpOperationLifecycleProtocolError( + "Operation Lifecycle Watch closed without a terminal " + "outcome" + ) + except grpc.aio.AioRpcError as error: + raise SfpOperationLifecycleTransportError(error.code()) from error + + async def cancel( + self, + channel: grpc.aio.Channel, + request: operation_lifecycle_pb2.CancelOperationRequest, + ) -> operation_lifecycle_pb2.CancelOperationResponse: + """Request cooperative cancellation without assuming completion.""" + try: + validation.validate_cancel_operation_request(request) + except validation.OperationLifecycleValidationError as error: + raise SfpOperationLifecycleProtocolError( + "Operation Lifecycle Cancel request is invalid" + ) from error + call = channel.unary_unary( + _method_path("CancelOperation"), + request_serializer=( + operation_lifecycle_pb2.CancelOperationRequest.SerializeToString + ), + response_deserializer=_decode_cancel, + ) + try: + response = await call(request, timeout=self._timeout_seconds) + except grpc.aio.AioRpcError as error: + raise SfpOperationLifecycleTransportError(error.code()) from error + try: + validation.validate_cancel_exchange(request, response) + except validation.OperationLifecycleValidationError as error: + raise SfpOperationLifecycleProtocolError( + "Operation Lifecycle Cancel exchange is invalid" + ) from error + return response diff --git a/src/stateforge_agent/_sfp/_services/operation_lifecycle/coordinator.py b/src/stateforge_agent/_sfp/_services/operation_lifecycle/coordinator.py new file mode 100644 index 0000000..cfe5e77 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/operation_lifecycle/coordinator.py @@ -0,0 +1,387 @@ +"""Retain Agent-side observation of admitted Provider operations.""" + +from __future__ import annotations + +import asyncio +import json +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from uuid import UUID + +import grpc +from google.protobuf.json_format import MessageToDict +from jsonschema import Draft202012Validator +from stateforge_protocol.sfp.operation_admission.v1 import ( + operation_admission_pb2, +) +from stateforge_protocol.sfp.operation_lifecycle.v1 import ( + operation_lifecycle_pb2, +) + +from stateforge_agent._configuration_validation import ( + require_bounded_integer, + require_finite_positive, +) +from stateforge_agent._journal.base import JournalError +from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAdmissionStore, + OperationAttempt, + OperationAttemptState, +) +from stateforge_agent._sfp._services.operation_lifecycle.client import ( + OperationLifecycleClient, + OperationLifecycleRequester, + SfpOperationLifecycleError, + SfpOperationLifecycleProtocolError, + SfpOperationLifecycleTransportError, +) +from stateforge_agent._sfp._services.operation_lifecycle.models import ( + OperationStatus, +) + +_LOGGER = logging.getLogger(__name__) +_RETRYABLE_CODES = frozenset( + { + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.RESOURCE_EXHAUSTED, + } +) +_TERMINAL_STATES = frozenset( + { + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + operation_lifecycle_pb2.OPERATION_STATE_FAILED, + operation_lifecycle_pb2.OPERATION_STATE_CANCELLED, + } +) + + +class OperationLifecyclePreconditionError(RuntimeError): + """Reject observation or cancellation without durable admission.""" + + +@dataclass(frozen=True, slots=True) +class OperationLifecycleLease: + """Identify one live Provider session used only for lifecycle RPCs.""" + + provider_instance_id: UUID + endpoint_generation: int + session_id: str + channel: grpc.aio.Channel = field(repr=False, compare=False) + session_token: object = field(repr=False, compare=False) + + +OperationLifecycleCurrentnessCheck = Callable[ + [OperationLifecycleLease], Awaitable[None] +] + + +@dataclass(slots=True) +class _ProviderObservation: + """Keep one Provider's active session and observer tasks together.""" + + lease: OperationLifecycleLease + assert_current: OperationLifecycleCurrentnessCheck + tasks: dict[UUID, asyncio.Task[None]] = field(default_factory=dict) + blocked: set[UUID] = field(default_factory=set) + + +class OperationLifecycleCoordinator: + """Observe retained operations without initiating business execution.""" + + def __init__( + self, + store: OperationAdmissionStore, + *, + maximum_active_observers: int = 16, + retry_initial_delay_seconds: float = 0.25, + retry_max_delay_seconds: float = 5.0, + client: OperationLifecycleRequester | None = None, + ) -> None: + self._store = store + self._maximum_active_observers = require_bounded_integer( + maximum_active_observers, + name="maximum active operation observers", + minimum=1, + maximum=1024, + ) + self._retry_initial_delay_seconds = require_finite_positive( + retry_initial_delay_seconds, + name="Operation Lifecycle retry delay", + ) + self._retry_max_delay_seconds = require_finite_positive( + retry_max_delay_seconds, + name="Operation Lifecycle maximum retry delay", + ) + if retry_max_delay_seconds < retry_initial_delay_seconds: + raise ValueError( + "Operation Lifecycle maximum retry delay is too small" + ) + self._client = client or OperationLifecycleClient() + self._providers: dict[UUID, _ProviderObservation] = {} + self._lock = asyncio.Lock() + self._closed = False + + async def resume( + self, + lease: OperationLifecycleLease, + *, + assert_current: OperationLifecycleCurrentnessCheck, + ) -> None: + """Resume accepted work on the current session without dispatch.""" + async with self._lock: + if self._closed: + return + previous = self._providers.get(lease.provider_instance_id) + if ( + previous is not None + and previous.lease.session_token is not lease.session_token + ): + for task in previous.tasks.values(): + task.cancel() + previous = None + if previous is None: + self._providers[lease.provider_instance_id] = ( + _ProviderObservation(lease, assert_current) + ) + await self._fill_observer_slots() + + async def status( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationStatus | None: + """Return current durable local knowledge without a Provider call.""" + attempt = await self._store.get(provider_instance_id, operation_id) + return ( + OperationStatus.from_attempt(attempt) + if attempt is not None + else None + ) + + async def cancel( + self, + lease: OperationLifecycleLease, + operation_id: UUID, + *, + assert_current: OperationLifecycleCurrentnessCheck, + ) -> OperationStatus: + """Request cooperative cancellation and retain its exact decision.""" + attempt = await self._accepted_attempt( + lease.provider_instance_id, operation_id + ) + request = operation_lifecycle_pb2.CancelOperationRequest( + session_binding=_binding(lease), + operation_id=str(operation_id), + expected_request_fingerprint=attempt.request_fingerprint, + ) + await assert_current(lease) + response = await self._client.cancel(lease.channel, request) + await assert_current(lease) + return await self._retain( + attempt, + response.snapshot, + cancellation_disposition=response.disposition, + ) + + async def close(self) -> None: + """Stop local observers without cancelling Provider operations.""" + async with self._lock: + self._closed = True + tasks = tuple( + task + for provider in self._providers.values() + for task in provider.tasks.values() + ) + self._providers.clear() + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + async def suspend( + self, provider_instance_id: UUID, session_token: object + ) -> None: + """Detach observers after session loss without changing work state.""" + async with self._lock: + provider = self._providers.get(provider_instance_id) + if ( + provider is None + or provider.lease.session_token is not session_token + ): + return + self._providers.pop(provider_instance_id) + tasks = tuple(provider.tasks.values()) if provider else () + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + async def _fill_observer_slots(self) -> None: + outstanding = await self._store.accepted_nonterminal() + async with self._lock: + if self._closed: + return + active = sum( + len(provider.tasks) for provider in self._providers.values() + ) + for attempt in outstanding: + if active >= self._maximum_active_observers: + break + provider = self._providers.get(attempt.provider_instance_id) + if ( + provider is None + or attempt.result_schema_json is None + or attempt.operation_id in provider.tasks + or attempt.operation_id in provider.blocked + ): + continue + provider.tasks[attempt.operation_id] = asyncio.create_task( + self._observe(provider, attempt.operation_id), + name=f"stateforge-operation-watch-{attempt.operation_id}", + ) + active += 1 + + async def _observe( + self, + provider: _ProviderObservation, + operation_id: UUID, + ) -> None: + lease = provider.lease + delay = self._retry_initial_delay_seconds + blocked = False + try: + while not self._closed: + attempt = await self._accepted_attempt( + lease.provider_instance_id, operation_id + ) + snapshot = attempt.lifecycle_snapshot + if snapshot is not None and snapshot.state in _TERMINAL_STATES: + return + request = operation_lifecycle_pb2.WatchOperationRequest( + session_binding=_binding(lease), + operation_id=str(operation_id), + expected_request_fingerprint=attempt.request_fingerprint, + after_sequence=( + snapshot.sequence if snapshot is not None else 0 + ), + ) + try: + await provider.assert_current(lease) + await self._watch_once(provider, attempt, request) + return + except SfpOperationLifecycleTransportError as error: + if error.code not in _RETRYABLE_CODES: + _LOGGER.warning( + "Operation observation stopped: operation_id=%s " + "transport=%s", + operation_id, + error.code.name, + ) + blocked = True + return + except SfpOperationLifecycleError: + _LOGGER.warning( + "Operation observation rejected invalid evidence: " + "operation_id=%s", + operation_id, + ) + blocked = True + return + await asyncio.sleep(delay) + delay = min(delay * 2, self._retry_max_delay_seconds) + except asyncio.CancelledError: + raise + except (JournalError, OperationLifecyclePreconditionError): + blocked = True + _LOGGER.exception( + "Operation observation cannot retain evidence: " + "operation_id=%s", + operation_id, + ) + finally: + async with self._lock: + current = self._providers.get(lease.provider_instance_id) + if current is provider: + current.tasks.pop(operation_id, None) + if blocked: + current.blocked.add(operation_id) + if not self._closed: + await self._fill_observer_slots() + + async def _watch_once( + self, + provider: _ProviderObservation, + attempt: OperationAttempt, + request: operation_lifecycle_pb2.WatchOperationRequest, + ) -> None: + """Retain one stream until its terminal snapshot or failure.""" + async for response in self._client.watch( + provider.lease.channel, request + ): + await provider.assert_current(provider.lease) + await self._retain(attempt, response.snapshot) + if response.snapshot.state in _TERMINAL_STATES: + return + raise SfpOperationLifecycleProtocolError( + "Operation Lifecycle Watch ended without terminal evidence" + ) + + async def _accepted_attempt( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationAttempt: + attempt = await self._store.get(provider_instance_id, operation_id) + if ( + attempt is None + or attempt.state is not OperationAttemptState.ACCEPTED + or attempt.result is None + or attempt.result_schema_json is None + ): + raise OperationLifecyclePreconditionError( + "Operation Lifecycle requires a pinned admitted identity" + ) + return attempt + + async def _retain( + self, + attempt: OperationAttempt, + snapshot: operation_lifecycle_pb2.OperationSnapshot, + *, + cancellation_disposition: int | None = None, + ) -> OperationStatus: + if snapshot.state == operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED: + if attempt.result_schema_json is None: + raise OperationLifecyclePreconditionError( + "Operation result contract was not pinned" + ) + schema = json.loads(attempt.result_schema_json) + result = MessageToDict(snapshot.success.result) + if not Draft202012Validator(schema).is_valid(result): + raise SfpOperationLifecycleProtocolError( + "Operation result violates its pinned schema" + ) + try: + retained = await self._store.record_lifecycle_snapshot( + attempt.provider_instance_id, + attempt.operation_id, + snapshot, + cancellation_disposition=cancellation_disposition, + ) + except ValueError as error: + raise SfpOperationLifecycleProtocolError( + "Operation evidence violates its retained contract" + ) from error + return OperationStatus.from_attempt(retained) + + +def _binding( + lease: OperationLifecycleLease, +) -> operation_admission_pb2.OperationSessionBinding: + return operation_admission_pb2.OperationSessionBinding( + expected_provider_instance_id=str(lease.provider_instance_id), + expected_endpoint_generation=lease.endpoint_generation, + expected_session_id=lease.session_id, + ) diff --git a/src/stateforge_agent/_sfp/_services/operation_lifecycle/models.py b/src/stateforge_agent/_sfp/_services/operation_lifecycle/models.py new file mode 100644 index 0000000..ba79108 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/operation_lifecycle/models.py @@ -0,0 +1,105 @@ +"""Expose Agent-normalized operation state without protobuf types.""" + +from __future__ import annotations + +from dataclasses import dataclass +from uuid import UUID + +from google.protobuf.json_format import MessageToDict +from stateforge_protocol.sfp.operation_lifecycle.v1 import ( + operation_lifecycle_pb2, +) + +from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAttempt, +) + + +@dataclass(frozen=True, slots=True) +class OperationStatus: + """Describe one durable Agent observation of admitted Provider work.""" + + provider_instance_id: UUID + operation_id: UUID + admission_state: str + state: str | None + sequence: int | None + progress_message: str | None + progress_percentage: float | None + result: dict[str, object] | None + failure_code: str | None + failure_explanation: str | None + side_effects: str | None + cleanup: str | None + effects_explanation: str | None + cancellation_disposition: str | None + + @classmethod + def from_attempt(cls, attempt: OperationAttempt) -> OperationStatus: + """Normalize one journal record for local coordination callers.""" + snapshot = attempt.lifecycle_snapshot + progress = ( + snapshot.progress + if snapshot is not None and snapshot.HasField("progress") + else None + ) + failure = ( + snapshot.failure + if snapshot is not None and snapshot.HasField("failure") + else None + ) + effects = None + if failure is not None: + effects = failure.effects + elif snapshot is not None and snapshot.HasField("cancellation"): + effects = snapshot.cancellation.effects + result: dict[str, object] | None = None + if snapshot is not None and snapshot.HasField("success"): + result = MessageToDict(snapshot.success.result) + disposition = attempt.cancellation_disposition + return cls( + provider_instance_id=attempt.provider_instance_id, + operation_id=attempt.operation_id, + admission_state=attempt.state.value, + state=( + operation_lifecycle_pb2.OperationState.Name(snapshot.state) + if snapshot is not None + else None + ), + sequence=snapshot.sequence if snapshot is not None else None, + progress_message=( + progress.message if progress is not None else None + ), + progress_percentage=( + progress.percentage + if progress is not None and progress.HasField("percentage") + else None + ), + result=result, + failure_code=failure.code if failure is not None else None, + failure_explanation=( + failure.explanation if failure is not None else None + ), + side_effects=( + operation_lifecycle_pb2.SideEffectState.Name( + effects.side_effects + ) + if effects is not None + else None + ), + cleanup=( + operation_lifecycle_pb2.CleanupState.Name(effects.cleanup) + if effects is not None + else None + ), + effects_explanation=( + effects.explanation if effects is not None else None + ), + cancellation_disposition=( + operation_lifecycle_pb2.CancellationDisposition.DESCRIPTOR.values_by_number[ + disposition + ].name + if disposition is not None + else None + ), + ) diff --git a/tests/test_operation_admission.py b/tests/test_operation_admission.py index 1e1dd85..f519fe5 100644 --- a/tests/test_operation_admission.py +++ b/tests/test_operation_admission.py @@ -79,6 +79,7 @@ "required": ["unit"], "additionalProperties": False, } +_RESULT_SCHEMA = {"type": "object"} class _ScriptedClient(OperationAdmissionRequester): @@ -136,6 +137,7 @@ def _lease(*, generation: int = 1) -> OperationAdmissionLease: operation_declarations={ _OPERATION: ManifestOperationDeclaration( input_schema=_INPUT_SCHEMA, + result_schema=_RESULT_SCHEMA, implements=frozenset({_PROFILE}), ) }, @@ -478,6 +480,7 @@ async def test_preflight_rejects_invalid_payload_or_profile( "unit": {"type": "string", "minLength": 1} }, }, + result_schema=_RESULT_SCHEMA, implements=frozenset({_PROFILE}), ) }, diff --git a/tests/test_operation_lifecycle.py b/tests/test_operation_lifecycle.py new file mode 100644 index 0000000..f9bee99 --- /dev/null +++ b/tests/test_operation_lifecycle.py @@ -0,0 +1,748 @@ +"""Agent lifecycle observation, cancellation, and durable evidence tests.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta +from pathlib import Path +from uuid import UUID, uuid4 + +import grpc +import pytest +from google.protobuf import timestamp_pb2 +from google.protobuf.json_format import ParseDict +from stateforge_protocol.sfp.operation_admission.v1 import ( + operation_admission_pb2, +) +from stateforge_protocol.sfp.operation_admission.v1 import ( + validation as admission_validation, +) +from stateforge_protocol.sfp.operation_lifecycle.v1 import ( + operation_lifecycle_pb2, +) + +from stateforge_agent._journal import SqliteJournal +from stateforge_agent._sfp._services.operation_admission.models import ( + OperationAdmissionDisposition, + OperationAdmissionResult, +) +from stateforge_agent._sfp._services.operation_admission.store import ( + OperationAdmissionStore, +) +from stateforge_agent._sfp._services.operation_lifecycle.client import ( + OperationLifecycleClient, + OperationLifecycleRequester, + SfpOperationLifecycleProtocolError, + SfpOperationLifecycleTransportError, +) +from stateforge_agent._sfp._services.operation_lifecycle.coordinator import ( + OperationLifecycleCoordinator, + OperationLifecycleLease, + OperationLifecyclePreconditionError, +) + +_PROVIDER_ID = UUID("7f7db7d8-68b4-4f71-9769-065b5c4f941c") +_OPERATION_IDENTIFIER = "stateforge.test/inspect@1" +_MANIFEST_REVISION = "sha256:" + "a" * 64 +_RESULT_SCHEMA = { + "type": "object", + "properties": {"value": {"type": "integer"}}, + "required": ["value"], + "additionalProperties": False, +} + + +def _timestamp(value: datetime) -> timestamp_pb2.Timestamp: + timestamp = timestamp_pb2.Timestamp() + timestamp.FromDatetime(value) + return timestamp + + +async def _accepted( + path: Path, +) -> tuple[SqliteJournal, OperationAdmissionStore, OperationAdmissionResult]: + journal = SqliteJournal(path) + await journal.open() + store = journal.operation_admissions(maximum_unresolved_attempts=16) + await store.initialize() + operation_id = uuid4() + now = datetime.now(UTC).replace(microsecond=0) + invocation = operation_admission_pb2.OperationInvocation( + operation_id=str(operation_id), + operation_identifier=_OPERATION_IDENTIFIER, + execution_deadline=_timestamp(now + timedelta(hours=1)), + contract=operation_admission_pb2.OperationContractContext( + manifest_revision=_MANIFEST_REVISION, + ), + ) + ParseDict({}, invocation.payload) + ParseDict({"requester": "test"}, invocation.execution_context) + fingerprint = admission_validation.compute_request_fingerprint(invocation) + await store.create_intent( + _PROVIDER_ID, + invocation, + fingerprint, + result_schema_json=json.dumps(_RESULT_SCHEMA).encode(), + ) + result = OperationAdmissionResult( + provider_instance_id=_PROVIDER_ID, + endpoint_generation=1, + session_id="session-1", + manifest_revision=_MANIFEST_REVISION, + operation_id=operation_id, + request_fingerprint=fingerprint, + disposition=OperationAdmissionDisposition.ACCEPTED_NEW, + admitted_at=now, + record_expires_at=now + timedelta(hours=2), + admitted_endpoint_generation=1, + ) + await store.record_accepted(result) + return journal, store, result + + +def _snapshot( + admission: OperationAdmissionResult, + state: int, + sequence: int, + *, + result: dict[str, object] | None = None, +) -> operation_lifecycle_pb2.OperationSnapshot: + snapshot = operation_lifecycle_pb2.OperationSnapshot( + admission_record=operation_admission_pb2.OperationAdmissionRecord( + operation_id=str(admission.operation_id), + request_fingerprint=admission.request_fingerprint, + admitted_at=_timestamp(admission.admitted_at), + record_expires_at=_timestamp(admission.record_expires_at), + admitted_endpoint_generation=admission.admitted_endpoint_generation, + ), + operation_identifier=_OPERATION_IDENTIFIER, + manifest_revision=_MANIFEST_REVISION, + sequence=sequence, + state=state, + updated_at=_timestamp( + admission.admitted_at + timedelta(seconds=sequence - 1) + ), + ) + if result is not None: + ParseDict(result, snapshot.success.result) + return snapshot + + +def _response( + admission: OperationAdmissionResult, + snapshot: operation_lifecycle_pb2.OperationSnapshot, + *, + generation: int = 1, + session_id: str = "session-1", +) -> operation_lifecycle_pb2.WatchOperationResponse: + return operation_lifecycle_pb2.WatchOperationResponse( + provider_instance_id=str(admission.provider_instance_id), + endpoint_generation=generation, + session_id=session_id, + snapshot=snapshot, + ) + + +class _ScriptedClient(OperationLifecycleRequester): + def __init__( + self, + pages: list[ + list[ + operation_lifecycle_pb2.WatchOperationResponse + | SfpOperationLifecycleTransportError + ] + | SfpOperationLifecycleTransportError + ], + cancel_response: operation_lifecycle_pb2.CancelOperationResponse + | None = None, + ) -> None: + self.pages = pages + self.cancel_response = cancel_response + self.watch_requests: list[ + operation_lifecycle_pb2.WatchOperationRequest + ] = [] + self.cancel_requests: list[ + operation_lifecycle_pb2.CancelOperationRequest + ] = [] + + async def watch( + self, + channel: grpc.aio.Channel, + request: operation_lifecycle_pb2.WatchOperationRequest, + ) -> AsyncIterator[operation_lifecycle_pb2.WatchOperationResponse]: + del channel + self.watch_requests.append(request) + page = self.pages.pop(0) + if isinstance(page, Exception): + raise page + for response in page: + if isinstance(response, Exception): + raise response + yield response + + async def cancel( + self, + channel: grpc.aio.Channel, + request: operation_lifecycle_pb2.CancelOperationRequest, + ) -> operation_lifecycle_pb2.CancelOperationResponse: + del channel + self.cancel_requests.append(request) + if self.cancel_response is None: + raise AssertionError("Unexpected cancellation call") + return self.cancel_response + + +def _lease(generation: int = 1) -> OperationLifecycleLease: + return OperationLifecycleLease( + provider_instance_id=_PROVIDER_ID, + endpoint_generation=generation, + session_id=f"session-{generation}", + channel=grpc.aio.insecure_channel( + "unix:/tmp/unused-stateforge-test.sock" + ), + session_token=object(), + ) + + +async def _current(lease: OperationLifecycleLease) -> None: + del lease + + +async def _until_terminal( + coordinator: OperationLifecycleCoordinator, + operation_id: UUID, +) -> None: + for _ in range(100): + status = await coordinator.status(_PROVIDER_ID, operation_id) + if status is not None and status.state in { + "OPERATION_STATE_SUCCEEDED", + "OPERATION_STATE_FAILED", + "OPERATION_STATE_CANCELLED", + }: + return + await asyncio.sleep(0) + raise AssertionError("Operation did not reach a terminal observation") + + +async def _until_watch_count(client: _ScriptedClient, expected: int) -> None: + async with asyncio.timeout(2): + while len(client.watch_requests) < expected: # noqa: ASYNC110 + await asyncio.sleep(0.001) + + +@pytest.mark.asyncio +async def test_watch_retains_typed_terminal_and_does_not_dispatch( + tmp_path: Path, +) -> None: + journal, store, admitted = await _accepted(tmp_path / "agent.db") + accepted = _snapshot( + admitted, operation_lifecycle_pb2.OPERATION_STATE_ACCEPTED, 1 + ) + running = _snapshot( + admitted, operation_lifecycle_pb2.OPERATION_STATE_RUNNING, 2 + ) + succeeded = _snapshot( + admitted, + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + 3, + result={"value": 7}, + ) + client = _ScriptedClient( + [ + [ + _response(admitted, item) + for item in (accepted, running, succeeded) + ] + ] + ) + coordinator = OperationLifecycleCoordinator(store, client=client) + lease = _lease() + try: + await coordinator.resume(lease, assert_current=_current) + await _until_terminal(coordinator, admitted.operation_id) + status = await coordinator.status(_PROVIDER_ID, admitted.operation_id) + assert status is not None + assert status.result == {"value": 7.0} + assert status.sequence == succeeded.sequence + assert len(client.watch_requests) == 1 + assert client.watch_requests[0].after_sequence == 0 + assert await store.accepted_nonterminal() == () + finally: + await coordinator.close() + await lease.channel.close() + await journal.close() + + +@pytest.mark.asyncio +async def test_invalid_terminal_result_is_not_recorded( + tmp_path: Path, +) -> None: + journal, store, admitted = await _accepted(tmp_path / "agent.db") + invalid = _snapshot( + admitted, + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + 3, + result={"value": "not an integer"}, + ) + client = _ScriptedClient([[_response(admitted, invalid)]]) + coordinator = OperationLifecycleCoordinator(store, client=client) + lease = _lease() + try: + await coordinator.resume(lease, assert_current=_current) + await _until_watch_count(client, 1) + status = await coordinator.status(_PROVIDER_ID, admitted.operation_id) + assert status is not None and status.state is None + assert len(client.watch_requests) == 1 + finally: + await coordinator.close() + await lease.channel.close() + await journal.close() + + +@pytest.mark.asyncio +async def test_cancel_request_is_not_a_cancelled_outcome( + tmp_path: Path, +) -> None: + journal, store, admitted = await _accepted(tmp_path / "agent.db") + requested = _snapshot( + admitted, + operation_lifecycle_pb2.OPERATION_STATE_CANCELLATION_REQUESTED, + 3, + ) + cancel_response = operation_lifecycle_pb2.CancelOperationResponse( + provider_instance_id=str(_PROVIDER_ID), + endpoint_generation=1, + session_id="session-1", + disposition=operation_lifecycle_pb2.CANCELLATION_DISPOSITION_REQUESTED, + snapshot=requested, + ) + client = _ScriptedClient([], cancel_response) + coordinator = OperationLifecycleCoordinator(store, client=client) + lease = _lease() + try: + status = await coordinator.cancel( + lease, admitted.operation_id, assert_current=_current + ) + assert status.state == "OPERATION_STATE_CANCELLATION_REQUESTED" + assert status.cancellation_disposition == ( + "CANCELLATION_DISPOSITION_REQUESTED" + ) + assert status.state != "OPERATION_STATE_CANCELLED" + assert len(client.cancel_requests) == 1 + assert client.cancel_requests[0].operation_id == str( + admitted.operation_id + ) + finally: + await coordinator.close() + await lease.channel.close() + await journal.close() + + +@pytest.mark.asyncio +async def test_unknown_cancellation_never_contacts_provider( + tmp_path: Path, +) -> None: + journal, store, _ = await _accepted(tmp_path / "agent.db") + client = _ScriptedClient([]) + coordinator = OperationLifecycleCoordinator(store, client=client) + lease = _lease() + try: + with pytest.raises(OperationLifecyclePreconditionError): + await coordinator.cancel(lease, uuid4(), assert_current=_current) + assert client.cancel_requests == [] + finally: + await coordinator.close() + await lease.channel.close() + await journal.close() + + +@pytest.mark.asyncio +async def test_terminal_outcome_cannot_be_replaced( + tmp_path: Path, +) -> None: + journal, store, admitted = await _accepted(tmp_path / "agent.db") + success = _snapshot( + admitted, + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + 3, + result={"value": 7}, + ) + different = _snapshot( + admitted, + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + 3, + result={"value": 9}, + ) + try: + await store.record_lifecycle_snapshot( + _PROVIDER_ID, admitted.operation_id, success + ) + with pytest.raises(ValueError): + await store.record_lifecycle_snapshot( + _PROVIDER_ID, admitted.operation_id, different + ) + retained = await store.get(_PROVIDER_ID, admitted.operation_id) + assert retained is not None and retained.lifecycle_snapshot == success + finally: + await journal.close() + + +@pytest.mark.asyncio +async def test_watch_retry_preserves_cursor_and_never_redispatches( + tmp_path: Path, +) -> None: + journal, store, admitted = await _accepted(tmp_path / "agent.db") + accepted = _snapshot( + admitted, operation_lifecycle_pb2.OPERATION_STATE_ACCEPTED, 1 + ) + succeeded = _snapshot( + admitted, + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + 3, + result={"value": 7}, + ) + client = _ScriptedClient( + [ + [ + _response(admitted, accepted), + SfpOperationLifecycleTransportError( + grpc.StatusCode.UNAVAILABLE + ), + ], + [_response(admitted, succeeded)], + ] + ) + coordinator = OperationLifecycleCoordinator( + store, client=client, retry_initial_delay_seconds=0.001 + ) + lease = _lease() + try: + await coordinator.resume(lease, assert_current=_current) + await _until_terminal(coordinator, admitted.operation_id) + assert [ + request.after_sequence for request in client.watch_requests + ] == [ + 0, + 1, + ] + assert client.cancel_requests == [] + finally: + await coordinator.close() + await lease.channel.close() + await journal.close() + + +@pytest.mark.asyncio +async def test_restarted_observer_resumes_from_durable_cursor( + tmp_path: Path, +) -> None: + path = tmp_path / "agent.db" + journal, store, admitted = await _accepted(path) + running = _snapshot( + admitted, operation_lifecycle_pb2.OPERATION_STATE_RUNNING, 2 + ) + await store.record_lifecycle_snapshot( + _PROVIDER_ID, admitted.operation_id, running + ) + await journal.close() + reopened = SqliteJournal(path) + await reopened.open() + reopened_store = reopened.operation_admissions( + maximum_unresolved_attempts=16 + ) + await reopened_store.initialize() + success = _snapshot( + admitted, + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + 4, + result={"value": 7}, + ) + client = _ScriptedClient([[_response(admitted, success)]]) + coordinator = OperationLifecycleCoordinator(reopened_store, client=client) + lease = _lease(2) + try: + await coordinator.resume(lease, assert_current=_current) + await _until_terminal(coordinator, admitted.operation_id) + assert [ + request.after_sequence for request in client.watch_requests + ] == [running.sequence] + assert client.watch_requests[ + 0 + ].session_binding.expected_session_id == ("session-2") + finally: + await coordinator.close() + await lease.channel.close() + await reopened.close() + + +@pytest.mark.asyncio +async def test_completion_wins_over_late_cancellation_response( + tmp_path: Path, +) -> None: + journal, store, admitted = await _accepted(tmp_path / "agent.db") + success = _snapshot( + admitted, + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + 5, + result={"value": 7}, + ) + await store.record_lifecycle_snapshot( + _PROVIDER_ID, admitted.operation_id, success + ) + requested = _snapshot( + admitted, + operation_lifecycle_pb2.OPERATION_STATE_CANCELLATION_REQUESTED, + 3, + ) + client = _ScriptedClient( + [], + operation_lifecycle_pb2.CancelOperationResponse( + provider_instance_id=str(_PROVIDER_ID), + endpoint_generation=1, + session_id="session-1", + disposition=( + operation_lifecycle_pb2.CANCELLATION_DISPOSITION_REQUESTED + ), + snapshot=requested, + ), + ) + coordinator = OperationLifecycleCoordinator(store, client=client) + lease = _lease() + try: + status = await coordinator.cancel( + lease, admitted.operation_id, assert_current=_current + ) + assert status.state == "OPERATION_STATE_SUCCEEDED" + assert status.sequence == success.sequence + assert status.result == {"value": 7.0} + assert status.cancellation_disposition == ( + "CANCELLATION_DISPOSITION_REQUESTED" + ) + finally: + await coordinator.close() + await lease.channel.close() + await journal.close() + + +@pytest.mark.asyncio +async def test_stale_session_cannot_retain_watch_evidence( + tmp_path: Path, +) -> None: + journal, store, admitted = await _accepted(tmp_path / "agent.db") + accepted = _snapshot( + admitted, operation_lifecycle_pb2.OPERATION_STATE_ACCEPTED, 1 + ) + client = _ScriptedClient([[_response(admitted, accepted)]]) + coordinator = OperationLifecycleCoordinator(store, client=client) + lease = _lease() + current_checks = 0 + + async def replaced(_lease: OperationLifecycleLease) -> None: + nonlocal current_checks + current_checks += 1 + if current_checks > 1: + raise OperationLifecyclePreconditionError("replaced") + + try: + await coordinator.resume(lease, assert_current=replaced) + await _until_watch_count(client, 1) + for _ in range(20): + await asyncio.sleep(0) + status = await coordinator.status(_PROVIDER_ID, admitted.operation_id) + assert status is not None and status.state is None + finally: + await coordinator.close() + await lease.channel.close() + await journal.close() + + +@pytest.mark.asyncio +async def test_wrong_admission_fingerprint_never_replaces_evidence( + tmp_path: Path, +) -> None: + journal, store, admitted = await _accepted(tmp_path / "agent.db") + invalid = _snapshot( + admitted, operation_lifecycle_pb2.OPERATION_STATE_ACCEPTED, 1 + ) + invalid.admission_record.request_fingerprint = "sha256:" + "b" * 64 + try: + with pytest.raises(ValueError): + await store.record_lifecycle_snapshot( + _PROVIDER_ID, admitted.operation_id, invalid + ) + attempt = await store.get(_PROVIDER_ID, admitted.operation_id) + assert attempt is not None and attempt.lifecycle_snapshot is None + finally: + await journal.close() + + +@pytest.mark.asyncio +async def test_real_grpc_client_validates_watch_and_cancel_correlation( + tmp_path: Path, +) -> None: + journal, _, admitted = await _accepted(tmp_path / "agent.db") + accepted = _snapshot( + admitted, operation_lifecycle_pb2.OPERATION_STATE_ACCEPTED, 1 + ) + success = _snapshot( + admitted, + operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, + 3, + result={"value": 7}, + ) + seen: list[str] = [] + + async def watch(request, _context): # type: ignore[no-untyped-def] + seen.append(f"watch:{request.operation_id}:{request.after_sequence}") + yield _response(admitted, accepted) + coalesced = _response(admitted, success) + coalesced.updates_skipped = True + yield coalesced + + async def cancel(request, _context): # type: ignore[no-untyped-def] + seen.append(f"cancel:{request.operation_id}") + return operation_lifecycle_pb2.CancelOperationResponse( + provider_instance_id=str(_PROVIDER_ID), + endpoint_generation=1, + session_id="session-1", + disposition=( + operation_lifecycle_pb2.CANCELLATION_DISPOSITION_ALREADY_TERMINAL + ), + snapshot=success, + ) + + service = operation_lifecycle_pb2.DESCRIPTOR.services_by_name[ + "OperationLifecycleService" + ].full_name + server = grpc.aio.server() + server.add_generic_rpc_handlers( + ( + grpc.method_handlers_generic_handler( + service, + { + "WatchOperation": grpc.unary_stream_rpc_method_handler( + watch, + request_deserializer=( + operation_lifecycle_pb2.WatchOperationRequest.FromString + ), + response_serializer=( + operation_lifecycle_pb2.WatchOperationResponse.SerializeToString + ), + ), + "CancelOperation": grpc.unary_unary_rpc_method_handler( + cancel, + request_deserializer=( + operation_lifecycle_pb2.CancelOperationRequest.FromString + ), + response_serializer=( + operation_lifecycle_pb2.CancelOperationResponse.SerializeToString + ), + ), + }, + ), + ) + ) + socket_path = Path("/private/tmp") / f"sf-{uuid4().hex[:12]}.sock" + address = f"unix:{socket_path}" + server.add_insecure_port(address) + await server.start() + channel = grpc.aio.insecure_channel(address) + client = OperationLifecycleClient() + binding = operation_admission_pb2.OperationSessionBinding( + expected_provider_instance_id=str(_PROVIDER_ID), + expected_endpoint_generation=1, + expected_session_id="session-1", + ) + try: + request = operation_lifecycle_pb2.WatchOperationRequest( + session_binding=binding, + operation_id=str(admitted.operation_id), + expected_request_fingerprint=admitted.request_fingerprint, + ) + observed = [ + response async for response in client.watch(channel, request) + ] + assert [item.snapshot.sequence for item in observed] == [1, 3] + decision = await client.cancel( + channel, + operation_lifecycle_pb2.CancelOperationRequest( + session_binding=binding, + operation_id=str(admitted.operation_id), + expected_request_fingerprint=admitted.request_fingerprint, + ), + ) + assert decision.snapshot == success + assert seen == [ + f"watch:{admitted.operation_id}:0", + f"cancel:{admitted.operation_id}", + ] + finally: + await channel.close() + await server.stop(0) + socket_path.unlink(missing_ok=True) + await journal.close() + + +@pytest.mark.asyncio +async def test_real_grpc_client_rejects_foreign_watch_response( + tmp_path: Path, +) -> None: + journal, _, admitted = await _accepted(tmp_path / "agent.db") + foreign = _response( + admitted, + _snapshot( + admitted, operation_lifecycle_pb2.OPERATION_STATE_ACCEPTED, 1 + ), + ) + foreign.provider_instance_id = str(uuid4()) + + async def watch(_request, _context): # type: ignore[no-untyped-def] + yield foreign + + service = operation_lifecycle_pb2.DESCRIPTOR.services_by_name[ + "OperationLifecycleService" + ].full_name + server = grpc.aio.server() + server.add_generic_rpc_handlers( + ( + grpc.method_handlers_generic_handler( + service, + { + "WatchOperation": grpc.unary_stream_rpc_method_handler( + watch, + request_deserializer=( + operation_lifecycle_pb2.WatchOperationRequest.FromString + ), + response_serializer=( + operation_lifecycle_pb2.WatchOperationResponse.SerializeToString + ), + ) + }, + ), + ) + ) + socket_path = Path("/private/tmp") / f"sf-{uuid4().hex[:12]}.sock" + address = f"unix:{socket_path}" + server.add_insecure_port(address) + await server.start() + channel = grpc.aio.insecure_channel(address) + try: + request = operation_lifecycle_pb2.WatchOperationRequest( + session_binding=operation_admission_pb2.OperationSessionBinding( + expected_provider_instance_id=str(_PROVIDER_ID), + expected_endpoint_generation=1, + expected_session_id="session-1", + ), + operation_id=str(admitted.operation_id), + expected_request_fingerprint=admitted.request_fingerprint, + ) + with pytest.raises(SfpOperationLifecycleProtocolError): + async for _ in OperationLifecycleClient().watch(channel, request): + pass + finally: + await channel.close() + await server.stop(0) + socket_path.unlink(missing_ok=True) + await journal.close() diff --git a/tests/test_rpm_automation.py b/tests/test_rpm_automation.py index 20daa93..af5d00d 100644 --- a/tests/test_rpm_automation.py +++ b/tests/test_rpm_automation.py @@ -10,7 +10,7 @@ DEPENDENCY_COUNT = 2 HEALTH_SCENARIO_PROVIDER_COUNT = 3 MANIFEST_ASSERTION_COUNT = 6 -AVAILABILITY_ASSERTION_COUNT = 12 +AVAILABILITY_ASSERTION_COUNT = 13 OPERATION_PROVIDER_COUNT = 2 @@ -437,6 +437,7 @@ def test_action_library_has_expected_domain_boundaries() -> None: "agent/assert-capability-boundary", "agent/assert-capability-assessment", "agent/assert-operation-admission", + "agent/assert-operation-lifecycle", "agent/assert-trigger-publication", "agent/collect-evidence", "agent/install", @@ -456,6 +457,7 @@ def test_action_library_has_expected_domain_boundaries() -> None: "rpm/test-installed/health/provider-availability", "rpm/test-installed/lifecycle/multiple-providers", "rpm/test-installed/operations/admission", + "rpm/test-installed/operations/lifecycle", "rpm/test-installed/setup", "rpm/test-installed/smoke/manifest-exchange", "rpm/test-installed/triggers/publication", diff --git a/uv.lock b/uv.lock index b5305f2..d8cbf1f 100644 --- a/uv.lock +++ b/uv.lock @@ -1506,7 +1506,7 @@ requires-dist = [ { name = "grpcio-health-checking", specifier = ">=1.83" }, { name = "grpcio-status", specifier = ">=1.83" }, { name = "jsonschema", specifier = ">=4.25,<5" }, - { name = "stateforge-protocol", git = "https://github.com/stateforge-org/stateforge-protocol.git?rev=e4d67e8fd7abbb077e09c41664e6f2945760037e" }, + { name = "stateforge-protocol", git = "https://github.com/stateforge-org/stateforge-protocol.git?rev=9504be252d9cd15ac53fe479bfb09a3007a16aca" }, { name = "watchfiles", specifier = ">=1.2,<2" }, ] @@ -1534,7 +1534,7 @@ docs = [ [[package]] name = "stateforge-protocol" version = "0.1.0" -source = { git = "https://github.com/stateforge-org/stateforge-protocol.git?rev=e4d67e8fd7abbb077e09c41664e6f2945760037e#e4d67e8fd7abbb077e09c41664e6f2945760037e" } +source = { git = "https://github.com/stateforge-org/stateforge-protocol.git?rev=9504be252d9cd15ac53fe479bfb09a3007a16aca#9504be252d9cd15ac53fe479bfb09a3007a16aca" } dependencies = [ { name = "grpcio" }, { name = "grpcio-health-checking" }, From f235b6b1c06c706965832cd1c80effa79edc9951 Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Wed, 4 Nov 2026 12:55:12 +0200 Subject: [PATCH 2/4] Require pinned result schemas for operation intents --- .../_journal/sqlite/operation_admissions.py | 32 ++++--------------- .../_services/operation_admission/store.py | 4 +-- .../operation_lifecycle/coordinator.py | 6 ---- tests/test_operation_admission_store.py | 18 +++++++++++ 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/src/stateforge_agent/_journal/sqlite/operation_admissions.py b/src/stateforge_agent/_journal/sqlite/operation_admissions.py index 889215d..552dcdb 100644 --- a/src/stateforge_agent/_journal/sqlite/operation_admissions.py +++ b/src/stateforge_agent/_journal/sqlite/operation_admissions.py @@ -32,7 +32,7 @@ ) _SCHEMA_COMPONENT = "operation_admission" -_SCHEMA_VERSION = 2 +_SCHEMA_VERSION = 1 _TERMINAL_STATES = frozenset( { operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED, @@ -78,7 +78,7 @@ async def create_intent( invocation: operation_admission_pb2.OperationInvocation, request_fingerprint: str, *, - result_schema_json: bytes | None = None, + result_schema_json: bytes, ) -> OperationIntentResult: """Atomically insert or classify one immutable invocation.""" serialized = invocation.SerializeToString(deterministic=True) @@ -94,10 +94,7 @@ def create(connection: sqlite3.Connection) -> OperationIntentResult: decision = ( OperationIntentDecision.EXISTING if existing.request_fingerprint == request_fingerprint - and ( - result_schema_json is None - or existing.result_schema_json == result_schema_json - ) + and existing.result_schema_json == result_schema_json else OperationIntentDecision.CONFLICT ) return OperationIntentResult(decision, existing) @@ -527,9 +524,7 @@ def _attempt_from_row(row: tuple[object, ...]) -> OperationAttempt: last_session_id=(_string(row[9]) if row[9] is not None else None), result=result, rejection=rejection, - result_schema_json=( - _bytes(row[22]) if row[22] is not None else None - ), + result_schema_json=_bytes(row[22]), lifecycle_snapshot=( operation_lifecycle_pb2.OperationSnapshot.FromString( _bytes(row[23]) @@ -569,7 +564,7 @@ def _initialize(connection: sqlite3.Connection) -> None: "SELECT version FROM journal_schemas WHERE component = ?", (_SCHEMA_COMPONENT,), ).fetchone() - if row is not None and row[0] not in {1, _SCHEMA_VERSION}: + if row is not None and row[0] != _SCHEMA_VERSION: raise sqlite3.DatabaseError( "Unsupported Operation Admission journal schema" ) @@ -600,7 +595,7 @@ def _initialize(connection: sqlite3.Connection) -> None: rejection_retry_after_seconds REAL, rejection_explanation TEXT, rejection_grpc_status TEXT, - result_schema_json BLOB, + result_schema_json BLOB NOT NULL, lifecycle_snapshot BLOB, lifecycle_terminal INTEGER NOT NULL DEFAULT 0, cancellation_disposition INTEGER, @@ -608,21 +603,6 @@ def _initialize(connection: sqlite3.Connection) -> None: ) WITHOUT ROWID """ ) - if row is not None and row[0] == 1: - for column in ( - "result_schema_json BLOB", - "lifecycle_snapshot BLOB", - "lifecycle_terminal INTEGER NOT NULL DEFAULT 0", - "cancellation_disposition INTEGER", - ): - connection.execute( - "ALTER TABLE operation_admission_attempts " - f"ADD COLUMN {column}" - ) - connection.execute( - "UPDATE journal_schemas SET version = ? WHERE component = ?", - (_SCHEMA_VERSION, _SCHEMA_COMPONENT), - ) connection.execute( """ INSERT OR IGNORE INTO journal_schemas (component, version) diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/store.py b/src/stateforge_agent/_sfp/_services/operation_admission/store.py index 6fc0d6f..e068813 100644 --- a/src/stateforge_agent/_sfp/_services/operation_admission/store.py +++ b/src/stateforge_agent/_sfp/_services/operation_admission/store.py @@ -55,9 +55,9 @@ class OperationAttempt: attempt_count: int last_endpoint_generation: int | None last_session_id: str | None + result_schema_json: bytes = field(repr=False) result: OperationAdmissionResult | None = None rejection: OperationRejection | None = None - result_schema_json: bytes | None = field(default=None, repr=False) lifecycle_snapshot: operation_lifecycle_pb2.OperationSnapshot | None = ( field(default=None, repr=False, compare=False) ) @@ -86,7 +86,7 @@ async def create_intent( invocation: operation_admission_pb2.OperationInvocation, request_fingerprint: str, *, - result_schema_json: bytes | None = None, + result_schema_json: bytes, ) -> OperationIntentResult: """Atomically create or classify one immutable operation identity.""" diff --git a/src/stateforge_agent/_sfp/_services/operation_lifecycle/coordinator.py b/src/stateforge_agent/_sfp/_services/operation_lifecycle/coordinator.py index cfe5e77..7cc5bbc 100644 --- a/src/stateforge_agent/_sfp/_services/operation_lifecycle/coordinator.py +++ b/src/stateforge_agent/_sfp/_services/operation_lifecycle/coordinator.py @@ -232,7 +232,6 @@ async def _fill_observer_slots(self) -> None: provider = self._providers.get(attempt.provider_instance_id) if ( provider is None - or attempt.result_schema_json is None or attempt.operation_id in provider.tasks or attempt.operation_id in provider.blocked ): @@ -338,7 +337,6 @@ async def _accepted_attempt( attempt is None or attempt.state is not OperationAttemptState.ACCEPTED or attempt.result is None - or attempt.result_schema_json is None ): raise OperationLifecyclePreconditionError( "Operation Lifecycle requires a pinned admitted identity" @@ -353,10 +351,6 @@ async def _retain( cancellation_disposition: int | None = None, ) -> OperationStatus: if snapshot.state == operation_lifecycle_pb2.OPERATION_STATE_SUCCEEDED: - if attempt.result_schema_json is None: - raise OperationLifecyclePreconditionError( - "Operation result contract was not pinned" - ) schema = json.loads(attempt.result_schema_json) result = MessageToDict(snapshot.success.result) if not Draft202012Validator(schema).is_valid(result): diff --git a/tests/test_operation_admission_store.py b/tests/test_operation_admission_store.py index 47d4291..cbc9977 100644 --- a/tests/test_operation_admission_store.py +++ b/tests/test_operation_admission_store.py @@ -31,6 +31,7 @@ _OTHER_PROVIDER_ID = UUID("1d40d0d5-ff2a-4780-bc3d-c4a7c55fc3a2") _MANIFEST_REVISION = "sha256:" + ("a" * 64) _ATTEMPTED_ENDPOINT_GENERATION = 4 +_RESULT_SCHEMA_JSON = b'{"type":"object"}' def _invocation( @@ -81,17 +82,26 @@ async def test_intent_is_immutable_and_survives_journal_restart( _PROVIDER_ID, invocation, fingerprint, + result_schema_json=_RESULT_SCHEMA_JSON, ) duplicate = await store.create_intent( _PROVIDER_ID, invocation, fingerprint, + result_schema_json=_RESULT_SCHEMA_JSON, ) conflicting = _invocation(operation_id, unit="sshd.service") conflict = await store.create_intent( _PROVIDER_ID, conflicting, validation.compute_request_fingerprint(conflicting), + result_schema_json=_RESULT_SCHEMA_JSON, + ) + schema_conflict = await store.create_intent( + _PROVIDER_ID, + invocation, + fingerprint, + result_schema_json=b'{"type":"string"}', ) await journal.close() @@ -104,6 +114,7 @@ async def test_intent_is_immutable_and_survives_journal_restart( assert created.decision is OperationIntentDecision.CREATED assert duplicate.decision is OperationIntentDecision.EXISTING assert conflict.decision is OperationIntentDecision.CONFLICT + assert schema_conflict.decision is OperationIntentDecision.CONFLICT assert persisted is not None assert persisted.request_fingerprint == fingerprint assert persisted.invocation == invocation @@ -121,16 +132,19 @@ async def test_unresolved_capacity_and_provider_identity_are_isolated( _PROVIDER_ID, first, validation.compute_request_fingerprint(first), + result_schema_json=_RESULT_SCHEMA_JSON, ) same_identity_other_provider = await store.create_intent( _OTHER_PROVIDER_ID, first, validation.compute_request_fingerprint(first), + result_schema_json=_RESULT_SCHEMA_JSON, ) exhausted = await store.create_intent( _PROVIDER_ID, second, validation.compute_request_fingerprint(second), + result_schema_json=_RESULT_SCHEMA_JSON, ) finally: await journal.close() @@ -156,11 +170,13 @@ async def test_concurrent_intent_creation_cannot_exceed_capacity( _PROVIDER_ID, first, validation.compute_request_fingerprint(first), + result_schema_json=_RESULT_SCHEMA_JSON, ), store.create_intent( _PROVIDER_ID, second, validation.compute_request_fingerprint(second), + result_schema_json=_RESULT_SCHEMA_JSON, ), ) unresolved = await store.unresolved() @@ -192,11 +208,13 @@ async def test_transport_success_and_rejection_are_durable( _PROVIDER_ID, accepted_invocation, accepted_fingerprint, + result_schema_json=_RESULT_SCHEMA_JSON, ) await store.create_intent( _PROVIDER_ID, rejected_invocation, rejected_fingerprint, + result_schema_json=_RESULT_SCHEMA_JSON, ) attempted = await store.record_transport_attempt( _PROVIDER_ID, From 269f008cb200b9245967c0960eb6aafe4d1db5b3 Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Wed, 4 Nov 2026 13:53:58 +0200 Subject: [PATCH 3/4] refactor agent provider session runtime --- src/stateforge_agent/_sfp/__init__.py | 4 +- .../_sfp/_runtime/availability.py | 213 +++ .../_sfp/_runtime/connections.py | 194 +++ src/stateforge_agent/_sfp/_runtime/leases.py | 185 +++ src/stateforge_agent/_sfp/_runtime/models.py | 103 ++ .../_sfp/_runtime/operations.py | 236 +++ .../_sfp/_runtime/reconnect.py | 196 +++ .../_sfp/_runtime/sessions.py | 1271 +++-------------- src/stateforge_agent/_sfp/_runtime/state.py | 86 ++ .../_sfp/_runtime/watchers/__init__.py | 1 + .../_sfp/_runtime/watchers/connection.py | 174 +++ .../_sfp/_runtime/watchers/health.py | 123 ++ .../_sfp/_runtime/watchers/triggers.py | 121 ++ tests/test_sfp_session.py | 6 +- 14 files changed, 1825 insertions(+), 1088 deletions(-) create mode 100644 src/stateforge_agent/_sfp/_runtime/availability.py create mode 100644 src/stateforge_agent/_sfp/_runtime/connections.py create mode 100644 src/stateforge_agent/_sfp/_runtime/leases.py create mode 100644 src/stateforge_agent/_sfp/_runtime/models.py create mode 100644 src/stateforge_agent/_sfp/_runtime/operations.py create mode 100644 src/stateforge_agent/_sfp/_runtime/reconnect.py create mode 100644 src/stateforge_agent/_sfp/_runtime/state.py create mode 100644 src/stateforge_agent/_sfp/_runtime/watchers/__init__.py create mode 100644 src/stateforge_agent/_sfp/_runtime/watchers/connection.py create mode 100644 src/stateforge_agent/_sfp/_runtime/watchers/health.py create mode 100644 src/stateforge_agent/_sfp/_runtime/watchers/triggers.py diff --git a/src/stateforge_agent/_sfp/__init__.py b/src/stateforge_agent/_sfp/__init__.py index 17ef863..5ecc393 100644 --- a/src/stateforge_agent/_sfp/__init__.py +++ b/src/stateforge_agent/_sfp/__init__.py @@ -5,12 +5,14 @@ SfpIncompatibleError, SfpManifestError, ) -from stateforge_agent._sfp._runtime.sessions import ( +from stateforge_agent._sfp._runtime.models import ( ProviderAvailabilityState, ProviderAvailabilityStatus, ProviderConnectionState, ProviderConnectionStatus, ProviderSession, +) +from stateforge_agent._sfp._runtime.sessions import ( ProviderSessionManager, ) from stateforge_agent._sfp._services.capability_assessment import ( diff --git a/src/stateforge_agent/_sfp/_runtime/availability.py b/src/stateforge_agent/_sfp/_runtime/availability.py new file mode 100644 index 0000000..4f879d1 --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/availability.py @@ -0,0 +1,213 @@ +"""Apply Health observations to manager-owned Provider availability state.""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Mapping +from uuid import UUID + +from grpc_health.v1 import health_pb2 + +from stateforge_agent._sfp._runtime.models import ( + ProviderAvailabilityState, + ProviderAvailabilityStatus, + ProviderRuntimeState, + ProviderSession, +) +from stateforge_agent._sfp._runtime.state import _availability_status +from stateforge_agent._sfp._runtime.watchers.triggers import ( + ProviderTriggerWatcher, +) +from stateforge_agent._sfp._services.capability_assessment.coordinator import ( + CapabilityAssessmentCoordinator, +) +from stateforge_agent._sfp._services.trigger_publication.client import ( + SfpTriggerPublicationStaleError, +) +from stateforge_agent._sfp._services.trigger_publication.coordinator import ( + TriggerPublicationCoordinator, + TriggerPublicationLease, +) + +_LOGGER = logging.getLogger(__name__) + + +class ProviderAvailabilityController: + """Fence Health and Trigger observations against the active session. + + The manager supplies its one registry and lock. This controller stores no + parallel session state; every read or transition uses that shared lock. + """ + + def __init__( + self, + *, + lock: asyncio.Lock, + providers: Mapping[UUID, ProviderRuntimeState], + capability_assessment: CapabilityAssessmentCoordinator, + trigger_publication: TriggerPublicationCoordinator | None, + retry_initial_delay_seconds: float, + retry_max_delay_seconds: float, + ) -> None: + """Bind availability changes to the manager-owned registry.""" + self._lock = lock + self._providers = providers + self._capability_assessment = capability_assessment + self._trigger_publication = trigger_publication + self._trigger_monitor = ProviderTriggerWatcher( + coordinator=trigger_publication, + retry_initial_delay_seconds=retry_initial_delay_seconds, + retry_max_delay_seconds=retry_max_delay_seconds, + assert_current=self._assert_trigger_stream_is_current, + on_finished=self._finish_trigger_watcher, + ) + + async def watch_triggers( + self, + session: ProviderSession, + availability_generation: int, + ) -> None: + """Consume trigger publications for one exact available snapshot.""" + await self._trigger_monitor.watch(session, availability_generation) + + async def mark_health_watching(self, session: ProviderSession) -> bool: + """Fail closed before opening or reopening a current Health Watch.""" + return await self.set_availability( + session, + ProviderAvailabilityState.WATCHING, + "awaiting Provider Health status", + ) + + async def apply_health_status( + self, + session: ProviderSession, + status: int, + ) -> bool: + """Apply one ordered Health status to its exact current session.""" + if status == health_pb2.HealthCheckResponse.SERVING: + state = ProviderAvailabilityState.AVAILABLE + reason = "Provider Health reports SERVING" + elif status == health_pb2.HealthCheckResponse.NOT_SERVING: + state = ProviderAvailabilityState.NOT_SERVING + reason = "Provider Health reports NOT_SERVING" + elif status == health_pb2.HealthCheckResponse.SERVICE_UNKNOWN: + state = ProviderAvailabilityState.SERVICE_UNKNOWN + reason = "Provider-wide Health service is unknown" + else: + state = ProviderAvailabilityState.UNKNOWN + reason = "Provider Health reports UNKNOWN" + return await self.set_availability(session, state, reason) + + async def set_availability( + self, + session: ProviderSession, + state: ProviderAvailabilityState, + reason: str, + ) -> bool: + """Set health-derived eligibility for the current fenced session.""" + async with self._lock: + provider = self._providers.get(session.provider_instance_id) + if provider is None or provider.active_session is not session: + return False + self.record_availability( + provider, + _availability_status(session, state, reason), + ) + return True + + async def finish_health_watcher(self, session: ProviderSession) -> None: + """Clear only the current task's Health watcher slot.""" + async with self._lock: + provider = self._providers.get(session.provider_instance_id) + if ( + provider is not None + and provider.health_watcher is asyncio.current_task() + ): + provider.health_watcher = None + + def record_availability( + self, + provider: ProviderRuntimeState, + status: ProviderAvailabilityStatus, + ) -> None: + """Record one transition, invalidate evidence, and sync triggers.""" + previous = provider.availability_status + provider.availability_status = status + if previous != status: + provider.availability_generation += 1 + self._capability_assessment.discard_provider( + status.provider_instance_id + ) + _LOGGER.info( + "Provider availability changed: " + "provider_instance_id=%s endpoint_generation=%s " + "state=%s eligible=%s reason=%s", + status.provider_instance_id, + status.endpoint_generation, + status.state, + status.eligible, + status.reason, + ) + self._synchronize_trigger_watcher(provider) + + def _synchronize_trigger_watcher( + self, + provider: ProviderRuntimeState, + ) -> None: + """Run Trigger Publication only when the session is eligible.""" + watcher = provider.trigger_watcher + status = provider.availability_status + session = provider.active_session + if ( + self._trigger_publication is None + or status is None + or not status.eligible + or session is None + or not session.manifest.trigger_schemas + ): + provider.trigger_watcher = None + if watcher is not None: + watcher.cancel() + return + if watcher is not None and not watcher.done(): + return + provider.trigger_watcher = asyncio.create_task( + self.watch_triggers(session, provider.availability_generation), + name=f"stateforge-sfp-triggers-{session.provider_instance_id}", + ) + + async def _finish_trigger_watcher( + self, + session: ProviderSession, + ) -> None: + """Clear only the current task's Trigger watcher slot.""" + async with self._lock: + provider = self._providers.get(session.provider_instance_id) + if ( + provider is not None + and provider.trigger_watcher is asyncio.current_task() + ): + provider.trigger_watcher = None + + async def _assert_trigger_stream_is_current( + self, + lease: TriggerPublicationLease, + ) -> None: + """Reject events from a replaced or unavailable Trigger session.""" + async with self._lock: + provider = self._providers.get(lease.provider_instance_id) + if ( + provider is None + or provider.active_session is not lease.session_token + or provider.availability_generation + != lease.availability_generation + ): + raise SfpTriggerPublicationStaleError( + "Trigger Publication session is no longer current" + ) + status = provider.availability_status + if status is None or not status.eligible: + raise SfpTriggerPublicationStaleError( + "Provider is unavailable for Trigger Publication" + ) diff --git a/src/stateforge_agent/_sfp/_runtime/connections.py b/src/stateforge_agent/_sfp/_runtime/connections.py new file mode 100644 index 0000000..8bb0ce9 --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/connections.py @@ -0,0 +1,194 @@ +"""Establish provisional Provider sessions before manager-owned promotion.""" + +from __future__ import annotations + +import asyncio +from typing import Protocol +from uuid import UUID + +import grpc + +from stateforge_agent._sfp._runtime.channel import create_provider_channel +from stateforge_agent._sfp._runtime.descriptor import ( + ProviderEndpointDescriptor, +) +from stateforge_agent._sfp._runtime.errors import SfpConnectionError +from stateforge_agent._sfp._runtime.models import ProviderSession +from stateforge_agent._sfp._services.bootstrap.client import perform_handshake +from stateforge_agent._sfp._services.manifest.client import ManifestRetriever +from stateforge_agent._sfp.version import SfpVersion + + +class ProviderConnectionOwner(Protocol): + """Expose manager-owned transitions needed during connection setup.""" + + async def _begin_attempt( + self, + descriptor: ProviderEndpointDescriptor, + ) -> tuple[ProviderSession | None, object | None]: ... + + async def _mark_handshaking( + self, + descriptor: ProviderEndpointDescriptor, + attempt: object, + ) -> bool: ... + + async def _mark_retrieving_manifest( + self, + descriptor: ProviderEndpointDescriptor, + attempt: object, + ) -> bool: ... + + async def _promote_candidate( + self, + *, + candidate: ProviderSession, + descriptor: ProviderEndpointDescriptor, + attempt: object, + ) -> tuple[ProviderSession, ProviderSession | None, bool]: ... + + async def _connection_attempt_failed( + self, + descriptor: ProviderEndpointDescriptor, + attempt: object, + ) -> None: ... + + async def _activate_candidate( + self, + session: ProviderSession, + replaced_session: ProviderSession | None, + *, + promoted: bool, + ) -> None: ... + + def get(self, provider_instance_id: UUID) -> ProviderSession | None: ... + + +class ProviderConnectionLifecycle: + """Coordinate channel setup while delegating state changes to its owner.""" + + def __init__( + self, + *, + owner: ProviderConnectionOwner, + agent_runtime_id: UUID, + agent_software_version: str, + supported_sfp_versions: tuple[SfpVersion, ...], + channel_ready_timeout_seconds: float, + handshake_timeout_seconds: float, + manifest_client: ManifestRetriever, + ) -> None: + """Bind connection I/O to manager-owned endpoint/session fences.""" + self._owner = owner + self._agent_runtime_id = agent_runtime_id + self._agent_software_version = agent_software_version + self._supported_sfp_versions = supported_sfp_versions + self._channel_ready_timeout_seconds = channel_ready_timeout_seconds + self._handshake_timeout_seconds = handshake_timeout_seconds + self._manifest_client = manifest_client + + async def establish( + self, + descriptor: ProviderEndpointDescriptor, + ) -> ProviderSession: + """Promote a validated channel only if its attempt remains current.""" + current_session, attempt = await self._owner._begin_attempt(descriptor) + if current_session is not None: + return current_session + if attempt is None: + raise RuntimeError("Connection attempt was not initialized") + + channel = create_provider_channel(f"unix:{descriptor.socket_path}") + try: + candidate = await self._connect_candidate( + channel=channel, + descriptor=descriptor, + attempt=attempt, + ) + if candidate is None: + await channel.close() + surviving_session = self._owner.get( + descriptor.provider_instance_id + ) + if surviving_session is None: + raise SfpConnectionError( + "Provider connection attempt was superseded or removed" + ) + return surviving_session + ( + result, + replaced_session, + promoted, + ) = await self._owner._promote_candidate( + candidate=candidate, + descriptor=descriptor, + attempt=attempt, + ) + except BaseException: + await channel.close() + await self._owner._connection_attempt_failed( + descriptor, + attempt, + ) + raise + + if not promoted: + await channel.close() + await self._owner._activate_candidate( + result, + replaced_session, + promoted=promoted, + ) + return result + + async def _connect_candidate( + self, + *, + channel: grpc.aio.Channel, + descriptor: ProviderEndpointDescriptor, + attempt: object, + ) -> ProviderSession | None: + """Handshake and retrieve the manifest on one provisional channel.""" + try: + await asyncio.wait_for( + channel.channel_ready(), + timeout=self._channel_ready_timeout_seconds, + ) + except TimeoutError as error: + raise SfpConnectionError( + "Provider gRPC channel did not become ready before the " + f"deadline: {descriptor.socket_path}" + ) from error + + if not await self._owner._mark_handshaking(descriptor, attempt): + return None + + handshake = await perform_handshake( + channel, + descriptor, + agent_runtime_id=self._agent_runtime_id, + agent_software_version=self._agent_software_version, + supported_sfp_versions=self._supported_sfp_versions, + timeout_seconds=self._handshake_timeout_seconds, + ) + if not await self._owner._mark_retrieving_manifest( + descriptor, + attempt, + ): + return None + manifest = await self._manifest_client.retrieve( + channel, + descriptor, + sfp_version=handshake.selected_sfp_version, + session_id=handshake.session_id, + ) + return ProviderSession( + provider_instance_id=descriptor.provider_instance_id, + endpoint_generation=descriptor.endpoint_generation, + socket_path=descriptor.socket_path, + selected_sfp_version=handshake.selected_sfp_version, + provider_runtime_id=handshake.provider_runtime_id, + session_id=handshake.session_id, + manifest=manifest, + _channel=channel, + ) diff --git a/src/stateforge_agent/_sfp/_runtime/leases.py b/src/stateforge_agent/_sfp/_runtime/leases.py new file mode 100644 index 0000000..47cb3df --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/leases.py @@ -0,0 +1,185 @@ +"""Build and validate fenced leases over manager-owned Provider state.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Mapping +from uuid import UUID + +from stateforge_agent._sfp._runtime.models import ( + ProviderRuntimeState, +) +from stateforge_agent._sfp._services.capability_assessment.client import ( + SfpCapabilityAssessmentStaleError, + SfpCapabilityAssessmentUnavailableError, +) +from stateforge_agent._sfp._services.capability_assessment.coordinator import ( + CapabilityAssessmentLease, +) +from stateforge_agent._sfp._services.operation_admission.client import ( + SfpOperationAdmissionStaleError, +) +from stateforge_agent._sfp._services.operation_admission.coordinator import ( + OperationAdmissionLease, + OperationAdmissionPreconditionError, +) +from stateforge_agent._sfp._services.operation_lifecycle.coordinator import ( + OperationLifecycleLease, + OperationLifecyclePreconditionError, +) + + +class ProviderSessionLeases: + """Snapshot current leases while the manager's one lock is held. + + The manager remains the owner of both the lock and Provider registry. + This collaborator only reads those objects to create and validate fences. + """ + + def __init__( + self, + *, + lock: asyncio.Lock, + providers: Mapping[UUID, ProviderRuntimeState], + is_closed: Callable[[], bool], + ) -> None: + """Bind lease operations to the manager-owned registry and lock.""" + self._lock = lock + self._providers = providers + self._is_closed = is_closed + + async def _operation_lifecycle_lease( + self, + provider_instance_id: UUID, + ) -> OperationLifecycleLease: + """Use the current channel even when Health blocks new admission.""" + async with self._lock: + provider = self._providers.get(provider_instance_id) + if provider is None or provider.active_session is None: + raise OperationLifecyclePreconditionError( + "Provider has no active session for Operation Lifecycle" + ) + session = provider.active_session + return OperationLifecycleLease( + provider_instance_id=session.provider_instance_id, + endpoint_generation=session.endpoint_generation, + session_id=session.session_id, + channel=session._channel, + session_token=session, + ) + + async def _assert_operation_lifecycle_is_current( + self, + lease: OperationLifecycleLease, + ) -> None: + """Reject lifecycle work whose exact active session was replaced.""" + async with self._lock: + provider = self._providers.get(lease.provider_instance_id) + if ( + self._is_closed() + or provider is None + or provider.active_session is not lease.session_token + ): + raise OperationLifecyclePreconditionError( + "Operation Lifecycle session is no longer current" + ) + + async def _operation_admission_lease( + self, + provider_instance_id: UUID, + ) -> OperationAdmissionLease: + """Snapshot current operation declarations and availability fences.""" + async with self._lock: + provider = self._providers.get(provider_instance_id) + if provider is None or provider.active_session is None: + raise OperationAdmissionPreconditionError( + "Provider has no active SFP session" + ) + status = provider.availability_status + if status is None or not status.eligible: + raise OperationAdmissionPreconditionError( + "Provider is not available for Operation Admission" + ) + session = provider.active_session + return OperationAdmissionLease( + provider_instance_id=session.provider_instance_id, + endpoint_generation=session.endpoint_generation, + session_id=session.session_id, + manifest_revision=session.manifest.key.revision, + availability_generation=provider.availability_generation, + operation_declarations=session.manifest.operation_declarations, + channel=session._channel, + session_token=session, + ) + + async def _assert_operation_admission_is_current( + self, + lease: OperationAdmissionLease, + ) -> None: + """Fence admission to the exact session and available generation.""" + async with self._lock: + provider = self._providers.get(lease.provider_instance_id) + if ( + provider is None + or provider.active_session is not lease.session_token + or provider.availability_generation + != lease.availability_generation + ): + raise SfpOperationAdmissionStaleError( + "Operation Admission session is no longer current" + ) + status = provider.availability_status + if status is None or not status.eligible: + raise SfpOperationAdmissionStaleError( + "Provider is no longer available for Operation Admission" + ) + + async def _capability_assessment_lease( + self, + provider_instance_id: UUID, + ) -> CapabilityAssessmentLease: + """Snapshot one active session and its availability fence.""" + async with self._lock: + provider = self._providers.get(provider_instance_id) + if provider is None or provider.active_session is None: + raise SfpCapabilityAssessmentUnavailableError( + "Provider has no active SFP session" + ) + session = provider.active_session + return CapabilityAssessmentLease( + provider_instance_id=session.provider_instance_id, + endpoint_generation=session.endpoint_generation, + session_id=session.session_id, + manifest_revision=session.manifest.key.revision, + availability_generation=provider.availability_generation, + operation_identifiers=session.manifest.operation_identifiers, + channel=session._channel, + session_token=session, + ) + + async def _assert_capability_assessment_is_current( + self, + lease: CapabilityAssessmentLease, + ) -> None: + """Fence assessment to the exact session and availability snapshot.""" + async with self._lock: + provider = self._providers.get(lease.provider_instance_id) + if ( + provider is None + or provider.active_session is not lease.session_token + ): + raise SfpCapabilityAssessmentStaleError( + "Capability Assessment session is no longer current" + ) + if ( + provider.availability_generation + != lease.availability_generation + ): + raise SfpCapabilityAssessmentStaleError( + "Provider availability changed during assessment" + ) + status = provider.availability_status + if status is None or not status.eligible: + raise SfpCapabilityAssessmentUnavailableError( + "Provider is not currently available for assessment" + ) diff --git a/src/stateforge_agent/_sfp/_runtime/models.py b/src/stateforge_agent/_sfp/_runtime/models.py new file mode 100644 index 0000000..b1048f4 --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/models.py @@ -0,0 +1,103 @@ +"""Define immutable Provider session and status snapshots.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path +from uuid import UUID + +import grpc + +from stateforge_agent._sfp._runtime.descriptor import ( + ProviderEndpointDescriptor, +) +from stateforge_agent._sfp._services.manifest.declaration import ( + ManifestDeclaration, +) +from stateforge_agent._sfp.version import SfpVersion + + +class ProviderConnectionState(StrEnum): + """Describe the Agent's local connection state for one Provider.""" + + UNAVAILABLE = "unavailable" + CONNECTING = "connecting" + HANDSHAKING = "handshaking" + RETRIEVING_MANIFEST = "retrieving_manifest" + CONNECTED = "connected" + INCOMPATIBLE = "incompatible" + MANIFEST_REJECTED = "manifest_rejected" + REPLACING_CONNECTING = "replacing_connecting" + REPLACING_HANDSHAKING = "replacing_handshaking" + REPLACING_RETRIEVING_MANIFEST = "replacing_retrieving_manifest" + + +class ProviderAvailabilityState(StrEnum): + """Describe whether one current Provider may accept new operations.""" + + UNAVAILABLE = "unavailable" + WATCHING = "watching" + AVAILABLE = "available" + UNKNOWN = "unknown" + NOT_SERVING = "not_serving" + SERVICE_UNKNOWN = "service_unknown" + WATCH_UNSUPPORTED = "watch_unsupported" + + +@dataclass(frozen=True, slots=True) +class ProviderSession: + """Represent one negotiated session bound to an endpoint generation.""" + + provider_instance_id: UUID + endpoint_generation: int + socket_path: Path + selected_sfp_version: SfpVersion + provider_runtime_id: UUID + session_id: str + manifest: ManifestDeclaration + _channel: grpc.aio.Channel = field(repr=False, compare=False) + + +@dataclass(frozen=True, slots=True) +class ProviderConnectionStatus: + """Expose an immutable snapshot of local Provider connection state.""" + + provider_instance_id: UUID + state: ProviderConnectionState + active_endpoint_generation: int | None + candidate_endpoint_generation: int | None + + +@dataclass(frozen=True, slots=True) +class ProviderAvailabilityStatus: + """Expose an immutable snapshot of Provider operation eligibility.""" + + provider_instance_id: UUID + state: ProviderAvailabilityState + endpoint_generation: int | None + session_id: str | None + manifest_revision: str | None + reason: str + + @property + def eligible(self) -> bool: + """Return whether new operation admission is currently allowed.""" + return self.state is ProviderAvailabilityState.AVAILABLE + + +@dataclass(slots=True) +class ProviderRuntimeState: + """Store the mutable state for one Provider registry entry.""" + + desired_descriptor: ProviderEndpointDescriptor | None = None + active_session: ProviderSession | None = None + connection_status: ProviderConnectionStatus | None = None + availability_status: ProviderAvailabilityStatus | None = None + attempt_token: object | None = None + channel_watcher: asyncio.Task[None] | None = None + health_watcher: asyncio.Task[None] | None = None + trigger_watcher: asyncio.Task[None] | None = None + retry_task: asyncio.Task[None] | None = None + availability_generation: int = 0 diff --git a/src/stateforge_agent/_sfp/_runtime/operations.py b/src/stateforge_agent/_sfp/_runtime/operations.py new file mode 100644 index 0000000..1c2e558 --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/operations.py @@ -0,0 +1,236 @@ +"""Expose operation-facing APIs over manager-owned session leases.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Protocol +from uuid import UUID + +from google.protobuf.json_format import MessageToDict + +from stateforge_agent._sfp._services.capability_assessment.coordinator import ( + CapabilityAssessmentCoordinator, + CapabilityAssessmentLease, +) +from stateforge_agent._sfp._services.capability_assessment.models import ( + CapabilityAssessment, +) +from stateforge_agent._sfp._services.operation_admission.coordinator import ( + OperationAdmissionCoordinator, + OperationAdmissionLease, + OperationAdmissionPreconditionError, +) +from stateforge_agent._sfp._services.operation_admission.models import ( + OperationAdmissionRequest, + OperationAdmissionResult, +) +from stateforge_agent._sfp._services.operation_lifecycle.coordinator import ( + OperationLifecycleCoordinator, + OperationLifecycleLease, + OperationLifecyclePreconditionError, +) +from stateforge_agent._sfp._services.operation_lifecycle.models import ( + OperationStatus, +) + + +class ProviderOperationSessions(Protocol): + """Provide leases and fence checks owned by the session manager.""" + + async def _capability_assessment_lease( + self, + provider_instance_id: UUID, + ) -> CapabilityAssessmentLease: ... + + async def _assert_capability_assessment_is_current( + self, + lease: CapabilityAssessmentLease, + ) -> None: ... + + async def _operation_admission_lease( + self, + provider_instance_id: UUID, + ) -> OperationAdmissionLease: ... + + async def _assert_operation_admission_is_current( + self, + lease: OperationAdmissionLease, + ) -> None: ... + + async def _operation_lifecycle_lease( + self, + provider_instance_id: UUID, + ) -> OperationLifecycleLease: ... + + async def _assert_operation_lifecycle_is_current( + self, + lease: OperationLifecycleLease, + ) -> None: ... + + +class ProviderOperations: + """Delegate operation APIs while keeping session fences in the manager.""" + + def __init__( + self, + *, + sessions: ProviderOperationSessions, + capability_assessment: CapabilityAssessmentCoordinator, + operation_admission: OperationAdmissionCoordinator | None, + operation_lifecycle: OperationLifecycleCoordinator | None, + ) -> None: + """Bind operation coordinators to the manager's lease boundary.""" + self._sessions = sessions + self._capability_assessment = capability_assessment + self._operation_admission = operation_admission + self._operation_lifecycle = operation_lifecycle + + async def assess_capabilities( + self, + provider_instance_id: UUID, + operation_identifiers: tuple[str, ...], + *, + execution_context: Mapping[str, object] | None = None, + ) -> CapabilityAssessment: + """Read current Provider evidence without authorizing execution.""" + lease = await self._sessions._capability_assessment_lease( + provider_instance_id + ) + return await self._capability_assessment.assess( + lease, + operation_identifiers=operation_identifiers, + execution_context=execution_context, + assert_current=( + self._sessions._assert_capability_assessment_is_current + ), + ) + + async def cached_capability_assessment( + self, + provider_instance_id: UUID, + operation_identifiers: tuple[str, ...], + *, + execution_context: Mapping[str, object] | None = None, + ) -> CapabilityAssessment | None: + """Return exact-scope prior evidence, never an eligibility decision.""" + lease = await self._sessions._capability_assessment_lease( + provider_instance_id + ) + return await self._capability_assessment.cached( + lease, + operation_identifiers=operation_identifiers, + execution_context=execution_context, + assert_current=( + self._sessions._assert_capability_assessment_is_current + ), + ) + + async def admit_operation( + self, + request: OperationAdmissionRequest, + ) -> OperationAdmissionResult: + """Preflight and submit one newly authorized generic operation.""" + coordinator = self._operation_admission + if coordinator is None: + raise OperationAdmissionPreconditionError( + "Operation Admission is not configured" + ) + assessment = await self.assess_capabilities( + request.provider_instance_id, + (request.operation_identifier,), + execution_context=request.execution_context, + ) + lease = await self._sessions._operation_admission_lease( + request.provider_instance_id + ) + result = await coordinator.admit( + lease, + request, + assessment, + assert_current=self._sessions._assert_operation_admission_is_current, + ) + await self.resume_observation(request.provider_instance_id) + return result + + async def resolve_operation( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationAdmissionResult: + """Resolve one durable uncertain identity through a current session.""" + coordinator = self._operation_admission + if coordinator is None: + raise OperationAdmissionPreconditionError( + "Operation Admission is not configured" + ) + attempt = await coordinator.attempt(provider_instance_id, operation_id) + if attempt is None: + raise OperationAdmissionPreconditionError( + "Operation Admission identity is not durable" + ) + execution_context = MessageToDict(attempt.invocation.execution_context) + assessment = await self.assess_capabilities( + provider_instance_id, + (attempt.invocation.operation_identifier,), + execution_context=execution_context, + ) + lease = await self._sessions._operation_admission_lease( + provider_instance_id + ) + result = await coordinator.submit( + lease, + operation_id, + assessment, + assert_current=self._sessions._assert_operation_admission_is_current, + ) + await self.resume_observation(provider_instance_id) + return result + + async def operation_status( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationStatus | None: + """Return durable local knowledge of one operation.""" + coordinator = self._operation_lifecycle + if coordinator is None: + raise OperationLifecyclePreconditionError( + "Operation Lifecycle is not configured" + ) + return await coordinator.status(provider_instance_id, operation_id) + + async def cancel_operation( + self, + provider_instance_id: UUID, + operation_id: UUID, + ) -> OperationStatus: + """Request cancellation for one admitted operation only.""" + coordinator = self._operation_lifecycle + if coordinator is None: + raise OperationLifecyclePreconditionError( + "Operation Lifecycle is not configured" + ) + lease = await self._sessions._operation_lifecycle_lease( + provider_instance_id + ) + return await coordinator.cancel( + lease, + operation_id, + assert_current=self._sessions._assert_operation_lifecycle_is_current, + ) + + async def resume_observation(self, provider_instance_id: UUID) -> None: + """Resume observing durable operations without resubmitting them.""" + coordinator = self._operation_lifecycle + if coordinator is None: + return + try: + lease = await self._sessions._operation_lifecycle_lease( + provider_instance_id + ) + except OperationLifecyclePreconditionError: + return + await coordinator.resume( + lease, + assert_current=self._sessions._assert_operation_lifecycle_is_current, + ) diff --git a/src/stateforge_agent/_sfp/_runtime/reconnect.py b/src/stateforge_agent/_sfp/_runtime/reconnect.py new file mode 100644 index 0000000..8e10a14 --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/reconnect.py @@ -0,0 +1,196 @@ +"""Schedule Provider reconnects against the manager-owned registry.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Coroutine, MutableMapping +from typing import Any +from uuid import UUID + +from stateforge_agent._sfp._runtime.availability import ( + ProviderAvailabilityController, +) +from stateforge_agent._sfp._runtime.descriptor import ( + ProviderEndpointDescriptor, +) +from stateforge_agent._sfp._runtime.errors import SfpConnectionError +from stateforge_agent._sfp._runtime.models import ( + ProviderConnectionState, + ProviderConnectionStatus, + ProviderRuntimeState, +) +from stateforge_agent._sfp._runtime.state import ( + _connected_status, + _unavailable_status, +) + +ProviderDescriptorLoader = Callable[[UUID], ProviderEndpointDescriptor] +RetryLoop = Callable[[UUID], Coroutine[Any, Any, None]] + + +class ProviderReconnectController: + """Own retry-task coordination without owning a second state registry.""" + + def __init__( + self, + *, + lock: asyncio.Lock, + providers: MutableMapping[UUID, ProviderRuntimeState], + is_closed: Callable[[], bool], + descriptor_loader: ProviderDescriptorLoader | None, + availability: ProviderAvailabilityController, + ) -> None: + """Bind retry coordination to the session manager's state and lock.""" + self._lock = lock + self._providers = providers + self._is_closed = is_closed + self._descriptor_loader = descriptor_loader + self._availability = availability + + async def schedule_retry( + self, + provider_instance_id: UUID, + retry_loop: RetryLoop, + ) -> None: + """Ensure exactly one retry loop exists for a desired Provider.""" + async with self._lock: + provider = self._providers.get(provider_instance_id) + if ( + self._is_closed() + or provider is None + or provider.desired_descriptor is None + or provider.attempt_token is not None + ): + return + existing = provider.retry_task + if existing is not None and not existing.done(): + return + provider.retry_task = asyncio.create_task( + retry_loop(provider_instance_id), + name=f"stateforge-sfp-retry-{provider_instance_id}", + ) + + async def finish_retry( + self, + provider_instance_id: UUID, + connected: bool, + retry_loop: RetryLoop, + ) -> None: + """Clear its task slot and retry if a newer target superseded it.""" + reschedule = False + async with self._lock: + provider = self._providers.get(provider_instance_id) + if ( + provider is not None + and provider.retry_task is asyncio.current_task() + ): + provider.retry_task = None + desired = provider.desired_descriptor + active = provider.active_session + reschedule = ( + connected + and not self._is_closed() + and desired is not None + and ( + active is None + or active.endpoint_generation + < desired.endpoint_generation + ) + ) + if reschedule: + await self.schedule_retry(provider_instance_id, retry_loop) + + async def load_retry_descriptor( + self, + provider_instance_id: UUID, + ) -> ProviderEndpointDescriptor | None: + """Reload and retain the current validated retry target.""" + async with self._lock: + provider = self._providers.get(provider_instance_id) + if ( + self._is_closed() + or provider is None + or provider.desired_descriptor is None + ): + return None + desired = provider.desired_descriptor + loader = self._descriptor_loader + if loader is None: + return desired + + loaded = loader(provider_instance_id) + if loaded.provider_instance_id != provider_instance_id: + raise SfpConnectionError( + "Reloaded descriptor returned a different Provider identity" + ) + async with self._lock: + provider = self._providers.get(provider_instance_id) + if ( + self._is_closed() + or provider is None + or provider.desired_descriptor is None + ): + return None + current = provider.desired_descriptor + if loaded.endpoint_generation < current.endpoint_generation: + raise SfpConnectionError( + "Reloaded descriptor endpoint generation moved backwards" + ) + provider.desired_descriptor = loaded + return loaded + + async def mark_incompatible( + self, + descriptor: ProviderEndpointDescriptor, + ) -> None: + """Record incompatibility unless an older session still survives.""" + async with self._lock: + provider = self._providers.setdefault( + descriptor.provider_instance_id, + ProviderRuntimeState(), + ) + session = provider.active_session + if session is not None: + provider.connection_status = _connected_status(session) + return + provider.connection_status = ProviderConnectionStatus( + provider_instance_id=descriptor.provider_instance_id, + state=ProviderConnectionState.INCOMPATIBLE, + active_endpoint_generation=None, + candidate_endpoint_generation=None, + ) + self._availability.record_availability( + provider, + _unavailable_status( + descriptor.provider_instance_id, + "Provider is SFP-incompatible", + ), + ) + + async def mark_manifest_rejected( + self, + descriptor: ProviderEndpointDescriptor, + ) -> None: + """Record rejection unless an older validated session survives.""" + async with self._lock: + provider = self._providers.setdefault( + descriptor.provider_instance_id, + ProviderRuntimeState(), + ) + session = provider.active_session + if session is not None: + provider.connection_status = _connected_status(session) + return + provider.connection_status = ProviderConnectionStatus( + provider_instance_id=descriptor.provider_instance_id, + state=ProviderConnectionState.MANIFEST_REJECTED, + active_endpoint_generation=None, + candidate_endpoint_generation=None, + ) + self._availability.record_availability( + provider, + _unavailable_status( + descriptor.provider_instance_id, + "Provider manifest was rejected", + ), + ) diff --git a/src/stateforge_agent/_sfp/_runtime/sessions.py b/src/stateforge_agent/_sfp/_runtime/sessions.py index 658f133..d75b257 100644 --- a/src/stateforge_agent/_sfp/_runtime/sessions.py +++ b/src/stateforge_agent/_sfp/_runtime/sessions.py @@ -3,20 +3,17 @@ from __future__ import annotations import asyncio -import logging -import secrets from collections.abc import Callable, Mapping -from dataclasses import dataclass, field -from enum import StrEnum -from pathlib import Path from uuid import UUID -import grpc -from google.protobuf.json_format import MessageToDict -from grpc_health.v1 import health_pb2 from stateforge_protocol.sfp import primitives -from stateforge_agent._sfp._runtime.channel import create_provider_channel +from stateforge_agent._sfp._runtime.availability import ( + ProviderAvailabilityController, +) +from stateforge_agent._sfp._runtime.connections import ( + ProviderConnectionLifecycle, +) from stateforge_agent._sfp._runtime.descriptor import ( ProviderEndpointDescriptor, ) @@ -25,15 +22,37 @@ SfpIncompatibleError, SfpManifestError, ) -from stateforge_agent._sfp._services.bootstrap.client import perform_handshake +from stateforge_agent._sfp._runtime.leases import ProviderSessionLeases +from stateforge_agent._sfp._runtime.models import ( + ProviderAvailabilityState, + ProviderAvailabilityStatus, + ProviderConnectionState, + ProviderConnectionStatus, + ProviderRuntimeState, + ProviderSession, +) +from stateforge_agent._sfp._runtime.operations import ProviderOperations +from stateforge_agent._sfp._runtime.reconnect import ( + ProviderReconnectController, +) +from stateforge_agent._sfp._runtime.state import ( + _availability_status, + _connected_status, + _matches, + _unavailable_after, + _unavailable_status, +) +from stateforge_agent._sfp._runtime.watchers.connection import ( + ProviderConnectionWatcher, +) +from stateforge_agent._sfp._runtime.watchers.health import ( + ProviderHealthMonitor, +) from stateforge_agent._sfp._services.capability_assessment.client import ( CapabilityAssessmentRequester, - SfpCapabilityAssessmentStaleError, - SfpCapabilityAssessmentUnavailableError, ) from stateforge_agent._sfp._services.capability_assessment.coordinator import ( CapabilityAssessmentCoordinator, - CapabilityAssessmentLease, ) from stateforge_agent._sfp._services.capability_assessment.models import ( CapabilityAssessment, @@ -41,23 +60,13 @@ from stateforge_agent._sfp._services.health.client import ( HealthClient, HealthWatcher, - SfpHealthError, - SfpHealthWatchError, ) from stateforge_agent._sfp._services.manifest.client import ( ManifestClient, ManifestRetriever, ) -from stateforge_agent._sfp._services.manifest.declaration import ( - ManifestDeclaration, -) -from stateforge_agent._sfp._services.operation_admission.client import ( - SfpOperationAdmissionStaleError, -) from stateforge_agent._sfp._services.operation_admission.coordinator import ( OperationAdmissionCoordinator, - OperationAdmissionLease, - OperationAdmissionPreconditionError, ) from stateforge_agent._sfp._services.operation_admission.models import ( OperationAdmissionRequest, @@ -65,116 +74,18 @@ ) from stateforge_agent._sfp._services.operation_lifecycle.coordinator import ( OperationLifecycleCoordinator, - OperationLifecycleLease, - OperationLifecyclePreconditionError, ) from stateforge_agent._sfp._services.operation_lifecycle.models import ( OperationStatus, ) -from stateforge_agent._sfp._services.trigger_publication.client import ( - SfpTriggerPublicationError, - SfpTriggerPublicationStaleError, - SfpTriggerPublicationUnavailableError, -) from stateforge_agent._sfp._services.trigger_publication.coordinator import ( TriggerPublicationCoordinator, - TriggerPublicationLease, ) from stateforge_agent._sfp.version import SfpVersion -_LOGGER = logging.getLogger(__name__) -_FAILED_CHANNEL_STATES = { - grpc.ChannelConnectivity.TRANSIENT_FAILURE, - grpc.ChannelConnectivity.SHUTDOWN, -} -_JITTER_STEPS = 1_000_000 ProviderDescriptorLoader = Callable[[UUID], ProviderEndpointDescriptor] -class ProviderConnectionState(StrEnum): - """Describe the Agent's local connection state for one Provider.""" - - UNAVAILABLE = "unavailable" - CONNECTING = "connecting" - HANDSHAKING = "handshaking" - RETRIEVING_MANIFEST = "retrieving_manifest" - CONNECTED = "connected" - INCOMPATIBLE = "incompatible" - MANIFEST_REJECTED = "manifest_rejected" - REPLACING_CONNECTING = "replacing_connecting" - REPLACING_HANDSHAKING = "replacing_handshaking" - REPLACING_RETRIEVING_MANIFEST = "replacing_retrieving_manifest" - - -class ProviderAvailabilityState(StrEnum): - """Describe whether one current Provider may accept new operations.""" - - UNAVAILABLE = "unavailable" - WATCHING = "watching" - AVAILABLE = "available" - UNKNOWN = "unknown" - NOT_SERVING = "not_serving" - SERVICE_UNKNOWN = "service_unknown" - WATCH_UNSUPPORTED = "watch_unsupported" - - -@dataclass(frozen=True, slots=True) -class ProviderSession: - """One negotiated SFP session bound to a Provider endpoint generation.""" - - provider_instance_id: UUID - endpoint_generation: int - socket_path: Path - selected_sfp_version: SfpVersion - provider_runtime_id: UUID - session_id: str - manifest: ManifestDeclaration - _channel: grpc.aio.Channel = field(repr=False, compare=False) - - -@dataclass(frozen=True, slots=True) -class ProviderConnectionStatus: - """Expose one immutable snapshot of local Provider connection state.""" - - provider_instance_id: UUID - state: ProviderConnectionState - active_endpoint_generation: int | None - candidate_endpoint_generation: int | None - - -@dataclass(frozen=True, slots=True) -class ProviderAvailabilityStatus: - """Expose one immutable operational-eligibility snapshot.""" - - provider_instance_id: UUID - state: ProviderAvailabilityState - endpoint_generation: int | None - session_id: str | None - manifest_revision: str | None - reason: str - - @property - def eligible(self) -> bool: - """Return whether new operation admission is currently allowed.""" - return self.state is ProviderAvailabilityState.AVAILABLE - - -@dataclass(slots=True) -class _ProviderRuntimeState: - """Keep all mutable connection state for one Provider together.""" - - desired_descriptor: ProviderEndpointDescriptor | None = None - active_session: ProviderSession | None = None - connection_status: ProviderConnectionStatus | None = None - availability_status: ProviderAvailabilityStatus | None = None - attempt_token: object | None = None - channel_watcher: asyncio.Task[None] | None = None - health_watcher: asyncio.Task[None] | None = None - trigger_watcher: asyncio.Task[None] | None = None - retry_task: asyncio.Task[None] | None = None - availability_generation: int = 0 - - class ProviderSessionManager: """Promote validated channels and replace them without a blind gap.""" @@ -239,11 +150,6 @@ def __init__( ) if health_retry_max_delay_seconds <= 0: raise ValueError("health_retry_max_delay_seconds must be positive") - self._agent_runtime_id = agent_runtime_id - self._agent_software_version = agent_software_version - self._supported_sfp_versions = supported_sfp_versions - self._channel_ready_timeout_seconds = channel_ready_timeout_seconds - self._handshake_timeout_seconds = handshake_timeout_seconds self._manifest_client = ( manifest_client if manifest_client is not None @@ -255,7 +161,6 @@ def __init__( self._reconnect_initial_delay_seconds = reconnect_initial_delay_seconds self._reconnect_max_delay_seconds = reconnect_max_delay_seconds self._health_retry_max_delay_seconds = health_retry_max_delay_seconds - self._descriptor_loader = descriptor_loader self._health_client = ( health_client if health_client is not None else HealthClient() ) @@ -268,12 +173,64 @@ def __init__( cache_entries=capability_assessment_cache_entries, client=capability_assessment_client, ) - self._trigger_publication = trigger_publication self._operation_admission = operation_admission self._operation_lifecycle = operation_lifecycle - self._providers: dict[UUID, _ProviderRuntimeState] = {} + self._providers: dict[UUID, ProviderRuntimeState] = {} self._closed = False self._lock = asyncio.Lock() + self._leases = ProviderSessionLeases( + lock=self._lock, + providers=self._providers, + is_closed=lambda: self._closed, + ) + self._availability = ProviderAvailabilityController( + lock=self._lock, + providers=self._providers, + capability_assessment=self._capability_assessment, + trigger_publication=trigger_publication, + retry_initial_delay_seconds=reconnect_initial_delay_seconds, + retry_max_delay_seconds=health_retry_max_delay_seconds, + ) + self._health_monitor = ProviderHealthMonitor( + client=self._health_client, + retry_initial_delay_seconds=reconnect_initial_delay_seconds, + retry_max_delay_seconds=health_retry_max_delay_seconds, + mark_watching=self._availability.mark_health_watching, + apply_status=self._availability.apply_health_status, + set_availability=self._availability.set_availability, + on_finished=self._availability.finish_health_watcher, + ) + self._operations = ProviderOperations( + sessions=self._leases, + capability_assessment=self._capability_assessment, + operation_admission=self._operation_admission, + operation_lifecycle=self._operation_lifecycle, + ) + self._connection_lifecycle = ProviderConnectionLifecycle( + owner=self, + agent_runtime_id=agent_runtime_id, + agent_software_version=agent_software_version, + supported_sfp_versions=supported_sfp_versions, + channel_ready_timeout_seconds=channel_ready_timeout_seconds, + handshake_timeout_seconds=handshake_timeout_seconds, + manifest_client=self._manifest_client, + ) + self._reconnect = ProviderReconnectController( + lock=self._lock, + providers=self._providers, + is_closed=lambda: self._closed, + descriptor_loader=descriptor_loader, + availability=self._availability, + ) + self._connection_watcher = ProviderConnectionWatcher( + retry_initial_delay_seconds=reconnect_initial_delay_seconds, + retry_max_delay_seconds=reconnect_max_delay_seconds, + load_retry_descriptor=self._reconnect.load_retry_descriptor, + establish_endpoint=self._connection_lifecycle.establish, + mark_incompatible=self._reconnect.mark_incompatible, + mark_manifest_rejected=self._reconnect.mark_manifest_rejected, + on_retry_finished=self._reconnect.finish_retry, + ) def get(self, provider_instance_id: UUID) -> ProviderSession | None: """Return the active session for one Provider, when present.""" @@ -309,12 +266,10 @@ async def assess_capabilities( execution_context: Mapping[str, object] | None = None, ) -> CapabilityAssessment: """Read current Provider evidence without authorizing execution.""" - lease = await self._capability_assessment_lease(provider_instance_id) - return await self._capability_assessment.assess( - lease, + return await self._operations.assess_capabilities( + provider_instance_id, operation_identifiers=operation_identifiers, execution_context=execution_context, - assert_current=self._assert_capability_assessment_is_current, ) async def cached_capability_assessment( @@ -325,12 +280,10 @@ async def cached_capability_assessment( execution_context: Mapping[str, object] | None = None, ) -> CapabilityAssessment | None: """Return exact-scope prior evidence, never an eligibility decision.""" - lease = await self._capability_assessment_lease(provider_instance_id) - return await self._capability_assessment.cached( - lease, + return await self._operations.cached_capability_assessment( + provider_instance_id, operation_identifiers=operation_identifiers, execution_context=execution_context, - assert_current=self._assert_capability_assessment_is_current, ) async def admit_operation( @@ -338,27 +291,7 @@ async def admit_operation( request: OperationAdmissionRequest, ) -> OperationAdmissionResult: """Preflight and submit one newly authorized generic operation.""" - coordinator = self._operation_admission - if coordinator is None: - raise OperationAdmissionPreconditionError( - "Operation Admission is not configured" - ) - assessment = await self.assess_capabilities( - request.provider_instance_id, - (request.operation_identifier,), - execution_context=request.execution_context, - ) - lease = await self._operation_admission_lease( - request.provider_instance_id - ) - result = await coordinator.admit( - lease, - request, - assessment, - assert_current=self._assert_operation_admission_is_current, - ) - await self._resume_operation_observation(request.provider_instance_id) - return result + return await self._operations.admit_operation(request) async def resolve_operation( self, @@ -366,31 +299,10 @@ async def resolve_operation( operation_id: UUID, ) -> OperationAdmissionResult: """Resolve one durable uncertain identity through a current session.""" - coordinator = self._operation_admission - if coordinator is None: - raise OperationAdmissionPreconditionError( - "Operation Admission is not configured" - ) - attempt = await coordinator.attempt(provider_instance_id, operation_id) - if attempt is None: - raise OperationAdmissionPreconditionError( - "Operation Admission identity is not durable" - ) - execution_context = MessageToDict(attempt.invocation.execution_context) - assessment = await self.assess_capabilities( + return await self._operations.resolve_operation( provider_instance_id, - (attempt.invocation.operation_identifier,), - execution_context=execution_context, - ) - lease = await self._operation_admission_lease(provider_instance_id) - result = await coordinator.submit( - lease, operation_id, - assessment, - assert_current=self._assert_operation_admission_is_current, ) - await self._resume_operation_observation(provider_instance_id) - return result async def operation_status( self, @@ -398,12 +310,10 @@ async def operation_status( operation_id: UUID, ) -> OperationStatus | None: """Return durable local knowledge of one operation.""" - coordinator = self._operation_lifecycle - if coordinator is None: - raise OperationLifecyclePreconditionError( - "Operation Lifecycle is not configured" - ) - return await coordinator.status(provider_instance_id, operation_id) + return await self._operations.operation_status( + provider_instance_id, + operation_id, + ) async def cancel_operation( self, @@ -411,167 +321,15 @@ async def cancel_operation( operation_id: UUID, ) -> OperationStatus: """Request cancellation for one admitted operation only.""" - coordinator = self._operation_lifecycle - if coordinator is None: - raise OperationLifecyclePreconditionError( - "Operation Lifecycle is not configured" - ) - lease = await self._operation_lifecycle_lease(provider_instance_id) - return await coordinator.cancel( - lease, + return await self._operations.cancel_operation( + provider_instance_id, operation_id, - assert_current=self._assert_operation_lifecycle_is_current, ) async def _resume_operation_observation( self, provider_instance_id: UUID ) -> None: - coordinator = self._operation_lifecycle - if coordinator is None: - return - try: - lease = await self._operation_lifecycle_lease(provider_instance_id) - except OperationLifecyclePreconditionError: - return - await coordinator.resume( - lease, - assert_current=self._assert_operation_lifecycle_is_current, - ) - - async def _operation_lifecycle_lease( - self, provider_instance_id: UUID - ) -> OperationLifecycleLease: - """Use the current channel even when Health blocks new admission.""" - async with self._lock: - provider = self._providers.get(provider_instance_id) - if provider is None or provider.active_session is None: - raise OperationLifecyclePreconditionError( - "Provider has no active session for Operation Lifecycle" - ) - session = provider.active_session - return OperationLifecycleLease( - provider_instance_id=session.provider_instance_id, - endpoint_generation=session.endpoint_generation, - session_id=session.session_id, - channel=session._channel, - session_token=session, - ) - - async def _assert_operation_lifecycle_is_current( - self, lease: OperationLifecycleLease - ) -> None: - async with self._lock: - provider = self._providers.get(lease.provider_instance_id) - if ( - self._closed - or provider is None - or provider.active_session is not lease.session_token - ): - raise OperationLifecyclePreconditionError( - "Operation Lifecycle session is no longer current" - ) - - async def _operation_admission_lease( - self, - provider_instance_id: UUID, - ) -> OperationAdmissionLease: - """Snapshot current operation declarations and availability fences.""" - async with self._lock: - provider = self._providers.get(provider_instance_id) - if provider is None or provider.active_session is None: - raise OperationAdmissionPreconditionError( - "Provider has no active SFP session" - ) - status = provider.availability_status - if status is None or not status.eligible: - raise OperationAdmissionPreconditionError( - "Provider is not available for Operation Admission" - ) - session = provider.active_session - return OperationAdmissionLease( - provider_instance_id=session.provider_instance_id, - endpoint_generation=session.endpoint_generation, - session_id=session.session_id, - manifest_revision=session.manifest.key.revision, - availability_generation=provider.availability_generation, - operation_declarations=( - session.manifest.operation_declarations - ), - channel=session._channel, - session_token=session, - ) - - async def _assert_operation_admission_is_current( - self, - lease: OperationAdmissionLease, - ) -> None: - """Fence one admission attempt to current session and readiness.""" - async with self._lock: - provider = self._providers.get(lease.provider_instance_id) - if ( - provider is None - or provider.active_session is not lease.session_token - or provider.availability_generation - != lease.availability_generation - ): - raise SfpOperationAdmissionStaleError( - "Operation Admission session is no longer current" - ) - status = provider.availability_status - if status is None or not status.eligible: - raise SfpOperationAdmissionStaleError( - "Provider is no longer available for Operation Admission" - ) - - async def _capability_assessment_lease( - self, - provider_instance_id: UUID, - ) -> CapabilityAssessmentLease: - """Snapshot one active session and its availability fence.""" - async with self._lock: - provider = self._providers.get(provider_instance_id) - if provider is None or provider.active_session is None: - raise SfpCapabilityAssessmentUnavailableError( - "Provider has no active SFP session" - ) - session = provider.active_session - return CapabilityAssessmentLease( - provider_instance_id=session.provider_instance_id, - endpoint_generation=session.endpoint_generation, - session_id=session.session_id, - manifest_revision=session.manifest.key.revision, - availability_generation=provider.availability_generation, - operation_identifiers=(session.manifest.operation_identifiers), - channel=session._channel, - session_token=session, - ) - - async def _assert_capability_assessment_is_current( - self, - lease: CapabilityAssessmentLease, - ) -> None: - """Fence one procedure lease to current session and availability.""" - async with self._lock: - provider = self._providers.get(lease.provider_instance_id) - if ( - provider is None - or provider.active_session is not lease.session_token - ): - raise SfpCapabilityAssessmentStaleError( - "Capability Assessment session is no longer current" - ) - if ( - provider.availability_generation - != lease.availability_generation - ): - raise SfpCapabilityAssessmentStaleError( - "Provider availability changed during assessment" - ) - status = provider.availability_status - if status is None or not status.eligible: - raise SfpCapabilityAssessmentUnavailableError( - "Provider is not currently available for assessment" - ) + await self._operations.resume_observation(provider_instance_id) async def establish( self, @@ -583,7 +341,7 @@ async def establish( raise RuntimeError("Provider session manager is closed") provider = self._providers.setdefault( descriptor.provider_instance_id, - _ProviderRuntimeState(), + ProviderRuntimeState(), ) desired = provider.desired_descriptor descriptor_is_stale = ( @@ -605,169 +363,20 @@ async def establish( ) await _cancel_task(retry_task) try: - return await self._establish_once(descriptor) + return await self._connection_lifecycle.establish(descriptor) except SfpIncompatibleError: - await self._mark_incompatible(descriptor) + await self._reconnect.mark_incompatible(descriptor) raise except SfpManifestError: - await self._mark_manifest_rejected(descriptor) + await self._reconnect.mark_manifest_rejected(descriptor) raise except SfpConnectionError: - await self._schedule_retry(descriptor.provider_instance_id) - raise - - async def _establish_once( - self, - descriptor: ProviderEndpointDescriptor, - ) -> ProviderSession: - """Connect, handshake, and atomically promote one endpoint.""" - current_session, attempt = await self._begin_attempt(descriptor) - if current_session is not None: - return current_session - if attempt is None: - raise RuntimeError("Connection attempt was not initialized") - - channel = create_provider_channel(f"unix:{descriptor.socket_path}") - try: - candidate = await self._connect_candidate( - channel=channel, - descriptor=descriptor, - attempt=attempt, - ) - if candidate is None: - await channel.close() - async with self._lock: - provider = self._providers.get( - descriptor.provider_instance_id, - ) - surviving_session = ( - provider.active_session - if provider is not None - else None - ) - if surviving_session is None: - raise SfpConnectionError( - "Provider connection attempt was superseded or removed" - ) - return surviving_session - ( - result, - replaced_session, - candidate_was_promoted, - ) = await self._promote_candidate( - candidate=candidate, - descriptor=descriptor, - attempt=attempt, + await self._reconnect.schedule_retry( + descriptor.provider_instance_id, + self._connection_watcher.retry, ) - except BaseException: - await channel.close() - async with self._lock: - provider = self._providers.get( - descriptor.provider_instance_id, - ) - if provider is not None and provider.attempt_token is attempt: - provider.attempt_token = None - current_session = provider.active_session - if current_session is None: - provider.connection_status = ProviderConnectionStatus( - provider_instance_id=descriptor.provider_instance_id, - state=ProviderConnectionState.UNAVAILABLE, - active_endpoint_generation=None, - candidate_endpoint_generation=None, - ) - self._record_availability( - provider, - _unavailable_status( - descriptor.provider_instance_id, - "Provider connection attempt failed", - ), - ) - else: - provider.connection_status = _connected_status( - current_session - ) raise - if not candidate_was_promoted: - await channel.close() - replaced_watcher: asyncio.Task[None] | None = None - replaced_health_watcher: asyncio.Task[None] | None = None - if candidate_was_promoted: - async with self._lock: - provider = self._providers.get(result.provider_instance_id) - if provider is not None and provider.active_session is result: - replaced_watcher = provider.channel_watcher - replaced_health_watcher = provider.health_watcher - provider.channel_watcher = asyncio.create_task( - self._watch_session(result), - name=( - "stateforge-sfp-channel-" - f"{result.provider_instance_id}" - ), - ) - provider.health_watcher = asyncio.create_task( - self._watch_health(result), - name=( - "stateforge-sfp-health-" - f"{result.provider_instance_id}" - ), - ) - await _cancel_task(replaced_watcher) - await _cancel_task(replaced_health_watcher) - if replaced_session is not None: - await replaced_session._channel.close() - await self._resume_operation_observation(result.provider_instance_id) - return result - - async def _connect_candidate( - self, - *, - channel: grpc.aio.Channel, - descriptor: ProviderEndpointDescriptor, - attempt: object, - ) -> ProviderSession | None: - """Open and handshake one provisional Provider channel.""" - try: - await asyncio.wait_for( - channel.channel_ready(), - timeout=self._channel_ready_timeout_seconds, - ) - except TimeoutError as error: - raise SfpConnectionError( - "Provider gRPC channel did not become ready before the " - f"deadline: {descriptor.socket_path}" - ) from error - - if not await self._mark_handshaking(descriptor, attempt): - return None - - result = await perform_handshake( - channel, - descriptor, - agent_runtime_id=self._agent_runtime_id, - agent_software_version=self._agent_software_version, - supported_sfp_versions=self._supported_sfp_versions, - timeout_seconds=self._handshake_timeout_seconds, - ) - if not await self._mark_retrieving_manifest(descriptor, attempt): - return None - manifest = await self._manifest_client.retrieve( - channel, - descriptor, - sfp_version=result.selected_sfp_version, - session_id=result.session_id, - ) - return ProviderSession( - provider_instance_id=descriptor.provider_instance_id, - endpoint_generation=descriptor.endpoint_generation, - socket_path=descriptor.socket_path, - selected_sfp_version=result.selected_sfp_version, - provider_runtime_id=result.provider_runtime_id, - session_id=result.session_id, - manifest=manifest, - _channel=channel, - ) - async def _promote_candidate( self, *, @@ -781,7 +390,7 @@ async def _promote_candidate( async with self._lock: provider = self._providers.setdefault( descriptor.provider_instance_id, - _ProviderRuntimeState(), + ProviderRuntimeState(), ) attempt_is_current = provider.attempt_token is attempt if not attempt_is_current: @@ -800,7 +409,7 @@ async def _promote_candidate( if attempt_is_current and superseded_session is None: provider.active_session = candidate provider.connection_status = _connected_status(candidate) - self._record_availability( + self._availability.record_availability( provider, _availability_status( candidate, @@ -819,12 +428,77 @@ async def _promote_candidate( return superseded_session, None, False return candidate, replaced_session, True + async def _connection_attempt_failed( + self, + descriptor: ProviderEndpointDescriptor, + attempt: object, + ) -> None: + """Clear the failed attempt while preserving any active session.""" + async with self._lock: + provider = self._providers.get(descriptor.provider_instance_id) + if provider is None or provider.attempt_token is not attempt: + return + provider.attempt_token = None + session = provider.active_session + if session is None: + provider.connection_status = ProviderConnectionStatus( + provider_instance_id=descriptor.provider_instance_id, + state=ProviderConnectionState.UNAVAILABLE, + active_endpoint_generation=None, + candidate_endpoint_generation=None, + ) + self._availability.record_availability( + provider, + _unavailable_status( + descriptor.provider_instance_id, + "Provider connection attempt failed", + ), + ) + else: + provider.connection_status = _connected_status(session) + + async def _activate_candidate( + self, + session: ProviderSession, + replaced_session: ProviderSession | None, + *, + promoted: bool, + ) -> None: + """Start current-session watchers and release replaced resources.""" + replaced_watcher: asyncio.Task[None] | None = None + replaced_health_watcher: asyncio.Task[None] | None = None + if promoted: + async with self._lock: + provider = self._providers.get(session.provider_instance_id) + if provider is not None and provider.active_session is session: + replaced_watcher = provider.channel_watcher + replaced_health_watcher = provider.health_watcher + provider.channel_watcher = asyncio.create_task( + self._watch_session(session), + name=( + "stateforge-sfp-channel-" + f"{session.provider_instance_id}" + ), + ) + provider.health_watcher = asyncio.create_task( + self._health_monitor.watch(session), + name=( + "stateforge-sfp-health-" + f"{session.provider_instance_id}" + ), + ) + await _cancel_task(replaced_watcher) + await _cancel_task(replaced_health_watcher) + if replaced_session is not None: + await replaced_session._channel.close() + await self._resume_operation_observation(session.provider_instance_id) + async def remove(self, provider_instance_id: UUID) -> None: """Remove and close the active session for one Provider.""" async with self._lock: provider = self._providers.setdefault( provider_instance_id, - _ProviderRuntimeState(), + ProviderRuntimeState(), ) session = provider.active_session watcher = provider.channel_watcher @@ -844,7 +518,7 @@ async def remove(self, provider_instance_id: UUID) -> None: active_endpoint_generation=None, candidate_endpoint_generation=None, ) - self._record_availability( + self._availability.record_availability( provider, ( _availability_status( @@ -917,7 +591,7 @@ async def close(self) -> None: active_endpoint_generation=None, candidate_endpoint_generation=None, ) - self._record_availability( + self._availability.record_availability( provider, ( _availability_status( @@ -955,516 +629,12 @@ async def close(self) -> None: await session._channel.close() self._capability_assessment.clear() - async def _schedule_retry(self, provider_instance_id: UUID) -> None: - """Ensure exactly one bounded retry loop exists for a Provider.""" - async with self._lock: - provider = self._providers.get(provider_instance_id) - if ( - self._closed - or provider is None - or provider.desired_descriptor is None - ): - return - if provider.attempt_token is not None: - return - existing = provider.retry_task - if existing is not None and not existing.done(): - return - provider.retry_task = asyncio.create_task( - self._retry_loop(provider_instance_id), - name=f"stateforge-sfp-retry-{provider_instance_id}", - ) - - async def _retry_loop(self, provider_instance_id: UUID) -> None: - """Retry a desired endpoint with capped exponential delay.""" - attempt_number = 0 - connected = False - try: - while True: - delay = self._retry_delay(attempt_number) - _LOGGER.info( - "Scheduling Provider reconnect: provider_instance_id=%s " - "attempt=%s delay_seconds=%.3f", - provider_instance_id, - attempt_number + 1, - delay, - ) - await asyncio.sleep(delay) - try: - descriptor = await self._load_retry_descriptor( - provider_instance_id - ) - except Exception as error: - _LOGGER.warning( - "Cannot reload Provider descriptor for reconnect: " - "provider_instance_id=%s error=%s", - provider_instance_id, - error, - ) - attempt_number += 1 - continue - if descriptor is None: - return - try: - await self._establish_once(descriptor) - except SfpIncompatibleError as error: - _LOGGER.warning( - "Provider is SFP-incompatible: " - "provider_instance_id=%s endpoint_generation=%s " - "error=%s", - provider_instance_id, - descriptor.endpoint_generation, - error, - ) - await self._mark_incompatible(descriptor) - return - except SfpManifestError as error: - _LOGGER.warning( - "Provider manifest rejected: " - "provider_instance_id=%s endpoint_generation=%s " - "error=%s", - provider_instance_id, - descriptor.endpoint_generation, - error, - ) - await self._mark_manifest_rejected(descriptor) - return - except SfpConnectionError as error: - _LOGGER.warning( - "Provider reconnect failed: provider_instance_id=%s " - "endpoint_generation=%s error=%s", - provider_instance_id, - descriptor.endpoint_generation, - error, - ) - attempt_number += 1 - continue - connected = True - return - finally: - reschedule = False - async with self._lock: - provider = self._providers.get(provider_instance_id) - if ( - provider is not None - and provider.retry_task is asyncio.current_task() - ): - provider.retry_task = None - desired = provider.desired_descriptor - active = provider.active_session - reschedule = ( - connected - and not self._closed - and desired is not None - and ( - active is None - or active.endpoint_generation - < desired.endpoint_generation - ) - ) - if reschedule: - await self._schedule_retry(provider_instance_id) - - async def _load_retry_descriptor( - self, - provider_instance_id: UUID, - ) -> ProviderEndpointDescriptor | None: - """Reload and retain the current validated retry target.""" - async with self._lock: - provider = self._providers.get(provider_instance_id) - if ( - self._closed - or provider is None - or provider.desired_descriptor is None - ): - return None - desired = provider.desired_descriptor - loader = self._descriptor_loader - if loader is None: - return desired - - loaded = loader(provider_instance_id) - if loaded.provider_instance_id != provider_instance_id: - raise SfpConnectionError( - "Reloaded descriptor returned a different Provider identity" - ) - async with self._lock: - provider = self._providers.get(provider_instance_id) - if ( - self._closed - or provider is None - or provider.desired_descriptor is None - ): - return None - current = provider.desired_descriptor - if loaded.endpoint_generation < current.endpoint_generation: - raise SfpConnectionError( - "Reloaded descriptor endpoint generation moved backwards" - ) - provider.desired_descriptor = loaded - return loaded - - def _retry_delay(self, attempt_number: int) -> float: - """Return capped exponential backoff with equal positive jitter.""" - ceiling = self._reconnect_initial_delay_seconds - for _ in range(attempt_number): - if ceiling >= self._reconnect_max_delay_seconds: - break - ceiling = min( - ceiling * 2, - self._reconnect_max_delay_seconds, - ) - half_ceiling = ceiling / 2 - jitter_fraction = secrets.randbelow(_JITTER_STEPS + 1) / _JITTER_STEPS - return half_ceiling + (half_ceiling * jitter_fraction) - - async def _mark_incompatible( - self, - descriptor: ProviderEndpointDescriptor, - ) -> None: - """Record incompatibility unless an older session still survives.""" - async with self._lock: - provider = self._providers.setdefault( - descriptor.provider_instance_id, - _ProviderRuntimeState(), - ) - current_session = provider.active_session - if current_session is not None: - provider.connection_status = _connected_status(current_session) - return - provider.connection_status = ProviderConnectionStatus( - provider_instance_id=descriptor.provider_instance_id, - state=ProviderConnectionState.INCOMPATIBLE, - active_endpoint_generation=None, - candidate_endpoint_generation=None, - ) - self._record_availability( - provider, - _unavailable_status( - descriptor.provider_instance_id, - "Provider is SFP-incompatible", - ), - ) - - async def _mark_manifest_rejected( - self, - descriptor: ProviderEndpointDescriptor, - ) -> None: - """Record rejection unless an older validated session survives.""" - async with self._lock: - provider = self._providers.setdefault( - descriptor.provider_instance_id, - _ProviderRuntimeState(), - ) - current_session = provider.active_session - if current_session is not None: - provider.connection_status = _connected_status(current_session) - return - provider.connection_status = ProviderConnectionStatus( - provider_instance_id=descriptor.provider_instance_id, - state=ProviderConnectionState.MANIFEST_REJECTED, - active_endpoint_generation=None, - candidate_endpoint_generation=None, - ) - self._record_availability( - provider, - _unavailable_status( - descriptor.provider_instance_id, - "Provider manifest was rejected", - ), - ) - - async def _watch_health(self, session: ProviderSession) -> None: - """Maintain one fenced Provider-wide Health Watch with backoff.""" - attempt_number = 0 - try: - while await self._mark_health_watching(session): - try: - async for status in self._health_client.watch( - session._channel - ): - attempt_number = 0 - if not await self._apply_health_status( - session, status - ): - return - reason = "Provider Health Watch ended" - except asyncio.CancelledError: - raise - except SfpHealthWatchError as error: - code = error.code - reason = f"Provider Health Watch failed: {code.name}" - if code is grpc.StatusCode.UNIMPLEMENTED: - await self._set_availability( - session, - ProviderAvailabilityState.WATCH_UNSUPPORTED, - reason, - ) - return - except SfpHealthError: - reason = "Provider Health Watch returned invalid data" - except Exception: - _LOGGER.exception( - "Provider Health Watch failed locally: " - "provider_instance_id=%s endpoint_generation=%s", - session.provider_instance_id, - session.endpoint_generation, - ) - reason = "Provider Health Watch failed locally" - - if not await self._set_availability( - session, - ProviderAvailabilityState.UNAVAILABLE, - reason, - ): - return - delay = self._health_retry_delay(attempt_number) - attempt_number += 1 - _LOGGER.info( - "Scheduling Provider Health Watch retry: " - "provider_instance_id=%s endpoint_generation=%s " - "attempt=%s delay_seconds=%.3f", - session.provider_instance_id, - session.endpoint_generation, - attempt_number, - delay, - ) - await asyncio.sleep(delay) - finally: - async with self._lock: - provider = self._providers.get(session.provider_instance_id) - if ( - provider is not None - and provider.health_watcher is asyncio.current_task() - ): - provider.health_watcher = None - - async def _mark_health_watching(self, session: ProviderSession) -> bool: - """Fail closed before opening or reopening a current Health Watch.""" - return await self._set_availability( - session, - ProviderAvailabilityState.WATCHING, - "awaiting Provider Health status", - ) - - async def _apply_health_status( - self, - session: ProviderSession, - status: int, - ) -> bool: - """Apply one ordered status only to its exact current session.""" - if status == health_pb2.HealthCheckResponse.SERVING: - state = ProviderAvailabilityState.AVAILABLE - reason = "Provider Health reports SERVING" - elif status == health_pb2.HealthCheckResponse.NOT_SERVING: - state = ProviderAvailabilityState.NOT_SERVING - reason = "Provider Health reports NOT_SERVING" - elif status == health_pb2.HealthCheckResponse.SERVICE_UNKNOWN: - state = ProviderAvailabilityState.SERVICE_UNKNOWN - reason = "Provider-wide Health service is unknown" - else: - state = ProviderAvailabilityState.UNKNOWN - reason = "Provider Health reports UNKNOWN" - return await self._set_availability(session, state, reason) - - async def _set_availability( - self, - session: ProviderSession, - state: ProviderAvailabilityState, - reason: str, - ) -> bool: - """Set health-derived eligibility when the session fence is current.""" - async with self._lock: - provider = self._providers.get(session.provider_instance_id) - if provider is None or provider.active_session is not session: - return False - status = _availability_status( - session, - state, - reason, - ) - self._record_availability(provider, status) - return True - - def _record_availability( - self, - provider: _ProviderRuntimeState, - status: ProviderAvailabilityStatus, - ) -> None: - """Store and log every operator-visible eligibility transition.""" - previous = provider.availability_status - provider.availability_status = status - if previous != status: - provider.availability_generation += 1 - self._capability_assessment.discard_provider( - status.provider_instance_id - ) - _LOGGER.info( - "Provider availability changed: " - "provider_instance_id=%s endpoint_generation=%s " - "state=%s eligible=%s reason=%s", - status.provider_instance_id, - status.endpoint_generation, - status.state, - status.eligible, - status.reason, - ) - self._synchronize_trigger_watcher(provider) - - def _synchronize_trigger_watcher( - self, - provider: _ProviderRuntimeState, - ) -> None: - """Start or fence Trigger Publication with current availability.""" - watcher = provider.trigger_watcher - status = provider.availability_status - session = provider.active_session - if ( - self._trigger_publication is None - or status is None - or not status.eligible - or session is None - or not session.manifest.trigger_schemas - ): - provider.trigger_watcher = None - if watcher is not None: - watcher.cancel() - return - if watcher is not None and not watcher.done(): - return - provider.trigger_watcher = asyncio.create_task( - self._watch_triggers( - session, - provider.availability_generation, - ), - name=f"stateforge-sfp-triggers-{session.provider_instance_id}", - ) - - async def _watch_triggers( - self, - session: ProviderSession, - availability_generation: int, - ) -> None: - """Maintain one fenced Trigger Publication stream with backoff.""" - coordinator = self._trigger_publication - if coordinator is None: - return - lease = TriggerPublicationLease( - provider_instance_id=session.provider_instance_id, - endpoint_generation=session.endpoint_generation, - session_id=session.session_id, - manifest_revision=session.manifest.key.revision, - availability_generation=availability_generation, - trigger_schemas=session.manifest.trigger_schemas, - channel=session._channel, - session_token=session, - ) - attempt_number = 0 - try: - while True: - try: - await coordinator.run( - lease, - assert_current=self._assert_trigger_stream_is_current, - ) - except asyncio.CancelledError: - raise - except SfpTriggerPublicationStaleError: - return - except SfpTriggerPublicationUnavailableError as error: - delay = self._health_retry_delay(attempt_number) - attempt_number += 1 - _LOGGER.info( - "Scheduling Trigger Publication retry: " - "provider_instance_id=%s endpoint_generation=%s " - "attempt=%s delay_seconds=%.3f reason=%s", - session.provider_instance_id, - session.endpoint_generation, - attempt_number, - delay, - error, - ) - await asyncio.sleep(delay) - continue - except SfpTriggerPublicationError as error: - _LOGGER.warning( - "Trigger Publication stream rejected: " - "provider_instance_id=%s endpoint_generation=%s " - "error=%s", - session.provider_instance_id, - session.endpoint_generation, - error, - ) - return - raise SfpTriggerPublicationUnavailableError( - "Trigger Publication stream ended" - ) - finally: - async with self._lock: - provider = self._providers.get(session.provider_instance_id) - if ( - provider is not None - and provider.trigger_watcher is asyncio.current_task() - ): - provider.trigger_watcher = None - - async def _assert_trigger_stream_is_current( - self, - lease: TriggerPublicationLease, - ) -> None: - """Fence Trigger Publication to one available current session.""" - async with self._lock: - provider = self._providers.get(lease.provider_instance_id) - if ( - provider is None - or provider.active_session is not lease.session_token - or provider.availability_generation - != lease.availability_generation - ): - raise SfpTriggerPublicationStaleError( - "Trigger Publication session is no longer current" - ) - status = provider.availability_status - if status is None or not status.eligible: - raise SfpTriggerPublicationStaleError( - "Provider is unavailable for Trigger Publication" - ) - - def _health_retry_delay(self, attempt_number: int) -> float: - """Return a capped jittered delay for a failed Health Watch.""" - ceiling = min( - self._reconnect_initial_delay_seconds, - self._health_retry_max_delay_seconds, - ) - for _ in range(attempt_number): - if ceiling >= self._health_retry_max_delay_seconds: - break - ceiling = min(ceiling * 2, self._health_retry_max_delay_seconds) - half_ceiling = ceiling / 2 - jitter_fraction = secrets.randbelow(_JITTER_STEPS + 1) / _JITTER_STEPS - return half_ceiling + (half_ceiling * jitter_fraction) - async def _watch_session(self, session: ProviderSession) -> None: """Invalidate one exact session when its gRPC channel is lost.""" - try: - state = session._channel.get_state(try_to_connect=False) - while state not in _FAILED_CHANNEL_STATES: - if state is grpc.ChannelConnectivity.IDLE: - state = session._channel.get_state(try_to_connect=True) - if state in _FAILED_CHANNEL_STATES: - break - await session._channel.wait_for_state_change(state) - state = session._channel.get_state(try_to_connect=False) - except asyncio.CancelledError: - raise - except Exception: - _LOGGER.exception( - "Provider channel watcher failed: provider_instance_id=%s " - "endpoint_generation=%s", - session.provider_instance_id, - session.endpoint_generation, - ) - await self._invalidate_lost_session(session) + await self._connection_watcher.watch_channel( + session, + on_lost=self._invalidate_lost_session, + ) async def _invalidate_lost_session( self, @@ -1517,7 +687,7 @@ async def _invalidate_lost_session( active_endpoint_generation=None, candidate_endpoint_generation=candidate_generation, ) - self._record_availability( + self._availability.record_availability( provider, _availability_status( session, @@ -1532,7 +702,10 @@ async def _invalidate_lost_session( session.provider_instance_id, session ) await session._channel.close() - await self._schedule_retry(session.provider_instance_id) + await self._reconnect.schedule_retry( + session.provider_instance_id, + self._connection_watcher.retry, + ) async def _begin_attempt( self, @@ -1542,7 +715,7 @@ async def _begin_attempt( async with self._lock: provider = self._providers.setdefault( descriptor.provider_instance_id, - _ProviderRuntimeState(), + ProviderRuntimeState(), ) current_session = provider.active_session if current_session is not None and ( @@ -1570,7 +743,7 @@ async def _begin_attempt( candidate_endpoint_generation=descriptor.endpoint_generation, ) if current_session is None: - self._record_availability( + self._availability.record_availability( provider, _unavailable_status( descriptor.provider_instance_id, @@ -1636,76 +809,6 @@ async def _mark_retrieving_manifest( return True -def _matches( - session: ProviderSession, - descriptor: ProviderEndpointDescriptor, -) -> bool: - """Return whether a session already represents one descriptor.""" - return ( - session.endpoint_generation == descriptor.endpoint_generation - and session.socket_path == descriptor.socket_path - ) - - -def _connected_status(session: ProviderSession) -> ProviderConnectionStatus: - """Build transport status for one manifest-accepted session.""" - return ProviderConnectionStatus( - provider_instance_id=session.provider_instance_id, - state=ProviderConnectionState.CONNECTED, - active_endpoint_generation=session.endpoint_generation, - candidate_endpoint_generation=None, - ) - - -def _availability_status( - session: ProviderSession, - state: ProviderAvailabilityState, - reason: str, -) -> ProviderAvailabilityStatus: - """Build an eligibility snapshot bound to one exact current session.""" - return ProviderAvailabilityStatus( - provider_instance_id=session.provider_instance_id, - state=state, - endpoint_generation=session.endpoint_generation, - session_id=session.session_id, - manifest_revision=session.manifest.key.revision, - reason=reason, - ) - - -def _unavailable_status( - provider_instance_id: UUID, - reason: str, -) -> ProviderAvailabilityStatus: - """Build an eligibility snapshot without an accepted current session.""" - return ProviderAvailabilityStatus( - provider_instance_id=provider_instance_id, - state=ProviderAvailabilityState.UNAVAILABLE, - endpoint_generation=None, - session_id=None, - manifest_revision=None, - reason=reason, - ) - - -def _unavailable_after( - previous: ProviderAvailabilityStatus | None, - provider_instance_id: UUID, - reason: str, -) -> ProviderAvailabilityStatus: - """Fail closed while retaining the last accepted session identity.""" - if previous is None or previous.endpoint_generation is None: - return _unavailable_status(provider_instance_id, reason) - return ProviderAvailabilityStatus( - provider_instance_id=provider_instance_id, - state=ProviderAvailabilityState.UNAVAILABLE, - endpoint_generation=previous.endpoint_generation, - session_id=previous.session_id, - manifest_revision=previous.manifest_revision, - reason=reason, - ) - - async def _cancel_task(task: asyncio.Task[None] | None) -> None: """Cancel and join one background lifecycle task.""" if task is None or task is asyncio.current_task(): diff --git a/src/stateforge_agent/_sfp/_runtime/state.py b/src/stateforge_agent/_sfp/_runtime/state.py new file mode 100644 index 0000000..91dcee7 --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/state.py @@ -0,0 +1,86 @@ +"""Build immutable snapshots for Provider session-state transitions.""" + +from __future__ import annotations + +from uuid import UUID + +from stateforge_agent._sfp._runtime.descriptor import ( + ProviderEndpointDescriptor, +) +from stateforge_agent._sfp._runtime.models import ( + ProviderAvailabilityState, + ProviderAvailabilityStatus, + ProviderConnectionState, + ProviderConnectionStatus, + ProviderSession, +) + + +def _matches( + session: ProviderSession, + descriptor: ProviderEndpointDescriptor, +) -> bool: + """Return whether a session already represents one descriptor.""" + return ( + session.endpoint_generation == descriptor.endpoint_generation + and session.socket_path == descriptor.socket_path + ) + + +def _connected_status(session: ProviderSession) -> ProviderConnectionStatus: + """Build transport status for one manifest-accepted session.""" + return ProviderConnectionStatus( + provider_instance_id=session.provider_instance_id, + state=ProviderConnectionState.CONNECTED, + active_endpoint_generation=session.endpoint_generation, + candidate_endpoint_generation=None, + ) + + +def _availability_status( + session: ProviderSession, + state: ProviderAvailabilityState, + reason: str, +) -> ProviderAvailabilityStatus: + """Build an eligibility snapshot bound to one exact current session.""" + return ProviderAvailabilityStatus( + provider_instance_id=session.provider_instance_id, + state=state, + endpoint_generation=session.endpoint_generation, + session_id=session.session_id, + manifest_revision=session.manifest.key.revision, + reason=reason, + ) + + +def _unavailable_status( + provider_instance_id: UUID, + reason: str, +) -> ProviderAvailabilityStatus: + """Build an eligibility snapshot without an accepted current session.""" + return ProviderAvailabilityStatus( + provider_instance_id=provider_instance_id, + state=ProviderAvailabilityState.UNAVAILABLE, + endpoint_generation=None, + session_id=None, + manifest_revision=None, + reason=reason, + ) + + +def _unavailable_after( + previous: ProviderAvailabilityStatus | None, + provider_instance_id: UUID, + reason: str, +) -> ProviderAvailabilityStatus: + """Fail closed while retaining the last accepted session identity.""" + if previous is None or previous.endpoint_generation is None: + return _unavailable_status(provider_instance_id, reason) + return ProviderAvailabilityStatus( + provider_instance_id=provider_instance_id, + state=ProviderAvailabilityState.UNAVAILABLE, + endpoint_generation=previous.endpoint_generation, + session_id=previous.session_id, + manifest_revision=previous.manifest_revision, + reason=reason, + ) diff --git a/src/stateforge_agent/_sfp/_runtime/watchers/__init__.py b/src/stateforge_agent/_sfp/_runtime/watchers/__init__.py new file mode 100644 index 0000000..2901701 --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/watchers/__init__.py @@ -0,0 +1 @@ +"""Run background tasks that observe Provider sessions.""" diff --git a/src/stateforge_agent/_sfp/_runtime/watchers/connection.py b/src/stateforge_agent/_sfp/_runtime/watchers/connection.py new file mode 100644 index 0000000..4ecb88e --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/watchers/connection.py @@ -0,0 +1,174 @@ +"""Monitor Provider channels and retry desired endpoint connections.""" + +from __future__ import annotations + +import asyncio +import logging +import secrets +from collections.abc import Awaitable, Callable, Coroutine +from typing import Any +from uuid import UUID + +import grpc + +from stateforge_agent._sfp._runtime.descriptor import ( + ProviderEndpointDescriptor, +) +from stateforge_agent._sfp._runtime.errors import ( + SfpConnectionError, + SfpIncompatibleError, + SfpManifestError, +) +from stateforge_agent._sfp._runtime.models import ProviderSession + +_LOGGER = logging.getLogger(__name__) +_FAILED_CHANNEL_STATES = { + grpc.ChannelConnectivity.TRANSIENT_FAILURE, + grpc.ChannelConnectivity.SHUTDOWN, +} +_JITTER_STEPS = 1_000_000 + +LoadRetryDescriptor = Callable[ + [UUID], Awaitable[ProviderEndpointDescriptor | None] +] +EstablishEndpoint = Callable[ + [ProviderEndpointDescriptor], Awaitable[ProviderSession] +] +MarkConnectionFailure = Callable[[ProviderEndpointDescriptor], Awaitable[None]] +ChannelLost = Callable[[ProviderSession], Awaitable[None]] +RetryLoop = Callable[[UUID], Coroutine[Any, Any, None]] + + +class ProviderConnectionWatcher: + """Run channel monitoring and retry loops without owning session state.""" + + def __init__( + self, + *, + retry_initial_delay_seconds: float, + retry_max_delay_seconds: float, + load_retry_descriptor: LoadRetryDescriptor, + establish_endpoint: EstablishEndpoint, + mark_incompatible: MarkConnectionFailure, + mark_manifest_rejected: MarkConnectionFailure, + on_retry_finished: Callable[[UUID, bool, RetryLoop], Awaitable[None]], + ) -> None: + """Bind reconnect work to manager-owned state transitions.""" + self._retry_initial_delay_seconds = retry_initial_delay_seconds + self._retry_max_delay_seconds = retry_max_delay_seconds + self._load_retry_descriptor = load_retry_descriptor + self._establish_endpoint = establish_endpoint + self._mark_incompatible = mark_incompatible + self._mark_manifest_rejected = mark_manifest_rejected + self._on_retry_finished = on_retry_finished + + async def watch_channel( + self, + session: ProviderSession, + *, + on_lost: ChannelLost, + ) -> None: + """Report loss of this exact session's gRPC channel.""" + try: + state = session._channel.get_state(try_to_connect=False) + while state not in _FAILED_CHANNEL_STATES: + if state is grpc.ChannelConnectivity.IDLE: + state = session._channel.get_state(try_to_connect=True) + if state in _FAILED_CHANNEL_STATES: + break + await session._channel.wait_for_state_change(state) + state = session._channel.get_state(try_to_connect=False) + except asyncio.CancelledError: + raise + except Exception: + _LOGGER.exception( + "Provider channel watcher failed: provider_instance_id=%s " + "endpoint_generation=%s", + session.provider_instance_id, + session.endpoint_generation, + ) + await on_lost(session) + + async def retry(self, provider_instance_id: UUID) -> None: + """Retry the desired endpoint with capped exponential backoff.""" + attempt_number = 0 + connected = False + try: + while True: + delay = self._retry_delay(attempt_number) + _LOGGER.info( + "Scheduling Provider reconnect: provider_instance_id=%s " + "attempt=%s delay_seconds=%.3f", + provider_instance_id, + attempt_number + 1, + delay, + ) + await asyncio.sleep(delay) + try: + descriptor = await self._load_retry_descriptor( + provider_instance_id + ) + except Exception as error: + _LOGGER.warning( + "Cannot reload Provider descriptor for reconnect: " + "provider_instance_id=%s error=%s", + provider_instance_id, + error, + ) + attempt_number += 1 + continue + if descriptor is None: + return + try: + await self._establish_endpoint(descriptor) + except SfpIncompatibleError as error: + _LOGGER.warning( + "Provider is SFP-incompatible: " + "provider_instance_id=%s endpoint_generation=%s " + "error=%s", + provider_instance_id, + descriptor.endpoint_generation, + error, + ) + await self._mark_incompatible(descriptor) + return + except SfpManifestError as error: + _LOGGER.warning( + "Provider manifest rejected: " + "provider_instance_id=%s endpoint_generation=%s " + "error=%s", + provider_instance_id, + descriptor.endpoint_generation, + error, + ) + await self._mark_manifest_rejected(descriptor) + return + except SfpConnectionError as error: + _LOGGER.warning( + "Provider reconnect failed: provider_instance_id=%s " + "endpoint_generation=%s error=%s", + provider_instance_id, + descriptor.endpoint_generation, + error, + ) + attempt_number += 1 + continue + connected = True + return + finally: + await self._on_retry_finished( + provider_instance_id, + connected, + self.retry, + ) + + def _retry_delay(self, attempt_number: int) -> float: + """Return capped exponential backoff with equal positive jitter.""" + ceiling = self._retry_initial_delay_seconds + for _ in range(attempt_number): + if ceiling >= self._retry_max_delay_seconds: + break + ceiling = min(ceiling * 2, self._retry_max_delay_seconds) + half_ceiling = ceiling / 2 + jitter_fraction = secrets.randbelow(_JITTER_STEPS + 1) / _JITTER_STEPS + return half_ceiling + half_ceiling * jitter_fraction diff --git a/src/stateforge_agent/_sfp/_runtime/watchers/health.py b/src/stateforge_agent/_sfp/_runtime/watchers/health.py new file mode 100644 index 0000000..05f0c27 --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/watchers/health.py @@ -0,0 +1,123 @@ +"""Observe Provider Health and report fenced availability changes.""" + +from __future__ import annotations + +import asyncio +import logging +import secrets +from collections.abc import Awaitable, Callable + +import grpc + +from stateforge_agent._sfp._runtime.models import ( + ProviderAvailabilityState, + ProviderSession, +) +from stateforge_agent._sfp._services.health.client import ( + HealthWatcher, + SfpHealthError, + SfpHealthWatchError, +) + +_LOGGER = logging.getLogger(__name__) +_JITTER_STEPS = 1_000_000 + +MarkHealthWatching = Callable[[ProviderSession], Awaitable[bool]] +ApplyHealthStatus = Callable[[ProviderSession, int], Awaitable[bool]] +SetAvailability = Callable[ + [ProviderSession, ProviderAvailabilityState, str], Awaitable[bool] +] +WatcherFinished = Callable[[ProviderSession], Awaitable[None]] + + +class ProviderHealthMonitor: + """Run a retrying Health Watch without owning session state.""" + + def __init__( + self, + *, + client: HealthWatcher, + retry_initial_delay_seconds: float, + retry_max_delay_seconds: float, + mark_watching: MarkHealthWatching, + apply_status: ApplyHealthStatus, + set_availability: SetAvailability, + on_finished: WatcherFinished, + ) -> None: + """Bind stream handling to manager-owned, fenced state callbacks.""" + self._client = client + self._retry_initial_delay_seconds = retry_initial_delay_seconds + self._retry_max_delay_seconds = retry_max_delay_seconds + self._mark_watching = mark_watching + self._apply_status = apply_status + self._set_availability = set_availability + self._on_finished = on_finished + + async def watch(self, session: ProviderSession) -> None: + """Maintain one fenced Provider-wide Health Watch with backoff.""" + attempt_number = 0 + try: + while await self._mark_watching(session): + try: + async for status in self._client.watch(session._channel): + attempt_number = 0 + if not await self._apply_status(session, status): + return + reason = "Provider Health Watch ended" + except asyncio.CancelledError: + raise + except SfpHealthWatchError as error: + code = error.code + reason = f"Provider Health Watch failed: {code.name}" + if code is grpc.StatusCode.UNIMPLEMENTED: + await self._set_availability( + session, + ProviderAvailabilityState.WATCH_UNSUPPORTED, + reason, + ) + return + except SfpHealthError: + reason = "Provider Health Watch returned invalid data" + except Exception: + _LOGGER.exception( + "Provider Health Watch failed locally: " + "provider_instance_id=%s endpoint_generation=%s", + session.provider_instance_id, + session.endpoint_generation, + ) + reason = "Provider Health Watch failed locally" + + if not await self._set_availability( + session, + ProviderAvailabilityState.UNAVAILABLE, + reason, + ): + return + delay = self._retry_delay(attempt_number) + attempt_number += 1 + _LOGGER.info( + "Scheduling Provider Health Watch retry: " + "provider_instance_id=%s endpoint_generation=%s " + "attempt=%s delay_seconds=%.3f", + session.provider_instance_id, + session.endpoint_generation, + attempt_number, + delay, + ) + await asyncio.sleep(delay) + finally: + await self._on_finished(session) + + def _retry_delay(self, attempt_number: int) -> float: + """Return a capped jittered delay for a failed Health Watch.""" + ceiling = min( + self._retry_initial_delay_seconds, + self._retry_max_delay_seconds, + ) + for _ in range(attempt_number): + if ceiling >= self._retry_max_delay_seconds: + break + ceiling = min(ceiling * 2, self._retry_max_delay_seconds) + half_ceiling = ceiling / 2 + jitter_fraction = secrets.randbelow(_JITTER_STEPS + 1) / _JITTER_STEPS + return half_ceiling + half_ceiling * jitter_fraction diff --git a/src/stateforge_agent/_sfp/_runtime/watchers/triggers.py b/src/stateforge_agent/_sfp/_runtime/watchers/triggers.py new file mode 100644 index 0000000..1e2c9b2 --- /dev/null +++ b/src/stateforge_agent/_sfp/_runtime/watchers/triggers.py @@ -0,0 +1,121 @@ +"""Consume Provider Trigger Publication streams for current sessions.""" + +from __future__ import annotations + +import asyncio +import logging +import secrets +from collections.abc import Awaitable, Callable + +from stateforge_agent._sfp._runtime.models import ProviderSession +from stateforge_agent._sfp._services.trigger_publication.client import ( + SfpTriggerPublicationError, + SfpTriggerPublicationStaleError, + SfpTriggerPublicationUnavailableError, +) +from stateforge_agent._sfp._services.trigger_publication.coordinator import ( + TriggerPublicationCoordinator, + TriggerPublicationLease, +) + +_LOGGER = logging.getLogger(__name__) +_JITTER_STEPS = 1_000_000 + +AssertCurrent = Callable[[TriggerPublicationLease], Awaitable[None]] +WatcherFinished = Callable[[ProviderSession], Awaitable[None]] + + +class ProviderTriggerWatcher: + """Maintain one retrying trigger stream without owning session state.""" + + def __init__( + self, + *, + coordinator: TriggerPublicationCoordinator | None, + retry_initial_delay_seconds: float, + retry_max_delay_seconds: float, + assert_current: AssertCurrent, + on_finished: WatcherFinished, + ) -> None: + """Bind the stream to manager-owned session fencing callbacks.""" + self._coordinator = coordinator + self._retry_initial_delay_seconds = retry_initial_delay_seconds + self._retry_max_delay_seconds = retry_max_delay_seconds + self._assert_current = assert_current + self._on_finished = on_finished + + async def watch( + self, + session: ProviderSession, + availability_generation: int, + ) -> None: + """Consume one current session's Trigger Publication stream.""" + coordinator = self._coordinator + if coordinator is None: + return + lease = TriggerPublicationLease( + provider_instance_id=session.provider_instance_id, + endpoint_generation=session.endpoint_generation, + session_id=session.session_id, + manifest_revision=session.manifest.key.revision, + availability_generation=availability_generation, + trigger_schemas=session.manifest.trigger_schemas, + channel=session._channel, + session_token=session, + ) + attempt_number = 0 + try: + while True: + try: + await coordinator.run( + lease, + assert_current=self._assert_current, + ) + except asyncio.CancelledError: + raise + except SfpTriggerPublicationStaleError: + return + except SfpTriggerPublicationUnavailableError as error: + delay = self._retry_delay(attempt_number) + attempt_number += 1 + _LOGGER.info( + "Scheduling Trigger Publication retry: " + "provider_instance_id=%s endpoint_generation=%s " + "attempt=%s delay_seconds=%.3f reason=%s", + session.provider_instance_id, + session.endpoint_generation, + attempt_number, + delay, + error, + ) + await asyncio.sleep(delay) + continue + except SfpTriggerPublicationError as error: + _LOGGER.warning( + "Trigger Publication stream rejected: " + "provider_instance_id=%s endpoint_generation=%s " + "error=%s", + session.provider_instance_id, + session.endpoint_generation, + error, + ) + return + raise SfpTriggerPublicationUnavailableError( + "Trigger Publication stream ended" + ) + finally: + await self._on_finished(session) + + def _retry_delay(self, attempt_number: int) -> float: + """Return a capped jittered retry delay for stream failures.""" + ceiling = min( + self._retry_initial_delay_seconds, + self._retry_max_delay_seconds, + ) + for _ in range(attempt_number): + if ceiling >= self._retry_max_delay_seconds: + break + ceiling = min(ceiling * 2, self._retry_max_delay_seconds) + half_ceiling = ceiling / 2 + jitter_fraction = secrets.randbelow(_JITTER_STEPS + 1) / _JITTER_STEPS + return half_ceiling + half_ceiling * jitter_fraction diff --git a/tests/test_sfp_session.py b/tests/test_sfp_session.py index a35a779..12d74c2 100644 --- a/tests/test_sfp_session.py +++ b/tests/test_sfp_session.py @@ -963,7 +963,7 @@ async def test_replacement_fences_stale_health_updates( assert status.state is ProviderAvailabilityState.WATCHING assert status.endpoint_generation == _REPLACEMENT_ENDPOINT_GENERATION - applied = await manager._apply_health_status( + applied = await manager._availability.apply_health_status( first_session, health_pb2.HealthCheckResponse.SERVING, ) @@ -1425,8 +1425,8 @@ def test_reconnect_delay_is_positive_and_capped() -> None: reconnect_max_delay_seconds=maximum_ceiling, ) - initial_delay = manager._retry_delay(0) - capped_delay = manager._retry_delay(100) + initial_delay = manager._connection_watcher._retry_delay(0) + capped_delay = manager._connection_watcher._retry_delay(100) assert initial_ceiling / 2 <= initial_delay <= initial_ceiling assert maximum_ceiling / 2 <= capped_delay <= maximum_ceiling From 07a3fc16b6f9b8fc17ad7ec4d08dac7845c2819a Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Wed, 4 Nov 2026 14:08:46 +0200 Subject: [PATCH 4/4] fix agent CI test assumptions --- tests/test_capability_assessment.py | 12 ++++++------ tests/test_operation_lifecycle.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/test_capability_assessment.py b/tests/test_capability_assessment.py index f3a2b38..6871b98 100644 --- a/tests/test_capability_assessment.py +++ b/tests/test_capability_assessment.py @@ -39,8 +39,8 @@ SfpCapabilityAssessmentUnavailableError, SfpVersion, ) -from stateforge_agent._sfp._runtime import sessions as sessions_module from stateforge_agent._sfp._runtime.channel import create_provider_channel +from stateforge_agent._sfp._runtime.models import ProviderRuntimeState from stateforge_agent._sfp._services.capability_assessment.cache import ( CapabilityAssessmentCache, ) @@ -589,7 +589,7 @@ def _manager_with_current_session( requester: _ScriptedRequester, ) -> tuple[ ProviderSessionManager, - sessions_module._ProviderRuntimeState, + ProviderRuntimeState, ]: manager = ProviderSessionManager( agent_runtime_id=uuid4(), @@ -608,7 +608,7 @@ def _manager_with_current_session( manifest_revision=session.manifest.key.revision, reason="test Provider is serving", ) - state = sessions_module._ProviderRuntimeState( + state = ProviderRuntimeState( active_session=session, availability_status=status, availability_generation=1, @@ -687,7 +687,7 @@ async def test_manager_requires_current_provider_availability() -> None: @pytest.mark.asyncio async def test_retry_stops_when_the_active_session_changes() -> None: - state_holder: list[sessions_module._ProviderRuntimeState] = [] + state_holder: list[ProviderRuntimeState] = [] def replace_session() -> None: state = state_holder[0] @@ -744,7 +744,7 @@ async def test_cached_evidence_is_scoped_and_state_invalidated() -> None: ) assert state.availability_status is not None - manager._record_availability( + manager._availability.record_availability( state, ProviderAvailabilityStatus( provider_instance_id=_PROVIDER_ID, @@ -755,7 +755,7 @@ async def test_cached_evidence_is_scoped_and_state_invalidated() -> None: reason="readiness changed", ), ) - manager._record_availability( + manager._availability.record_availability( state, ProviderAvailabilityStatus( provider_instance_id=_PROVIDER_ID, diff --git a/tests/test_operation_lifecycle.py b/tests/test_operation_lifecycle.py index f9bee99..699d3c3 100644 --- a/tests/test_operation_lifecycle.py +++ b/tests/test_operation_lifecycle.py @@ -44,6 +44,7 @@ ) _PROVIDER_ID = UUID("7f7db7d8-68b4-4f71-9769-065b5c4f941c") +_SHORT_SOCKET_DIRECTORY = Path("/tmp") # noqa: S108 _OPERATION_IDENTIFIER = "stateforge.test/inspect@1" _MANIFEST_REVISION = "sha256:" + "a" * 64 _RESULT_SCHEMA = { @@ -54,6 +55,11 @@ } +def _temporary_socket_path() -> Path: + """Use a short unique path that fits Unix socket address limits.""" + return _SHORT_SOCKET_DIRECTORY / f"sf-{uuid4().hex}.sock" + + def _timestamp(value: datetime) -> timestamp_pb2.Timestamp: timestamp = timestamp_pb2.Timestamp() timestamp.FromDatetime(value) @@ -644,7 +650,7 @@ async def cancel(request, _context): # type: ignore[no-untyped-def] ), ) ) - socket_path = Path("/private/tmp") / f"sf-{uuid4().hex[:12]}.sock" + socket_path = _temporary_socket_path() address = f"unix:{socket_path}" server.add_insecure_port(address) await server.start() @@ -723,7 +729,7 @@ async def watch(_request, _context): # type: ignore[no-untyped-def] ), ) ) - socket_path = Path("/private/tmp") / f"sf-{uuid4().hex[:12]}.sock" + socket_path = _temporary_socket_path() address = f"unix:{socket_path}" server.add_insecure_port(address) await server.start()