From 69c5d55ab5259b84edfa3365501abb06331b30e6 Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Wed, 28 Oct 2026 20:58:37 +0200 Subject: [PATCH 1/6] [TASK][MVP] Integrate Trigger Publication and Acknowledgement --- .../assert-trigger-publication/action.yml | 67 ++ .../assert-trigger-publication/assert.py | 133 ++++ .../create-fixture/fixtures/manifest.py | 57 +- .../triggers/publication/action.yml | 116 ++++ .github/workflows/rpm-installed-test.yml | 8 + README.md | 7 +- docs/ci-cd.md | 13 +- docs/configuration.md | 4 + docs/index.md | 11 +- docs/running.md | 15 + docs/sfp-trigger-publication.md | 88 +++ mkdocs.yml | 1 + packaging/rpm/SOURCES/stateforge-agent.toml | 2 + pyproject.toml | 4 +- src/stateforge_agent/_config/loading.py | 30 +- src/stateforge_agent/_config/models.py | 2 + src/stateforge_agent/_journal/__init__.py | 8 + src/stateforge_agent/_journal/sqlite.py | 103 ++++ src/stateforge_agent/_runtime/daemon.py | 29 + .../_sfp/_runtime/sessions.py | 149 +++++ .../_sfp/_services/manifest/declaration.py | 18 + .../_services/trigger_publication/__init__.py | 15 + .../_services/trigger_publication/client.py | 254 ++++++++ .../trigger_publication/coordinator.py | 252 ++++++++ .../_services/trigger_publication/store.py | 213 +++++++ tests/test_configuration.py | 12 + tests/test_daemon_provider_events.py | 2 + tests/test_rpm_automation.py | 43 ++ tests/test_sfp_session.py | 208 +++++++ tests/test_trigger_publication.py | 583 ++++++++++++++++++ tests/test_trigger_publication_store.py | 164 +++++ uv.lock | 20 +- 32 files changed, 2619 insertions(+), 12 deletions(-) create mode 100644 .github/actions/agent/assert-trigger-publication/action.yml create mode 100644 .github/actions/agent/assert-trigger-publication/assert.py create mode 100644 .github/actions/rpm/test-installed/triggers/publication/action.yml create mode 100644 docs/sfp-trigger-publication.md create mode 100644 src/stateforge_agent/_journal/__init__.py create mode 100644 src/stateforge_agent/_journal/sqlite.py create mode 100644 src/stateforge_agent/_sfp/_services/trigger_publication/__init__.py create mode 100644 src/stateforge_agent/_sfp/_services/trigger_publication/client.py create mode 100644 src/stateforge_agent/_sfp/_services/trigger_publication/coordinator.py create mode 100644 src/stateforge_agent/_sfp/_services/trigger_publication/store.py create mode 100644 tests/test_trigger_publication.py create mode 100644 tests/test_trigger_publication_store.py diff --git a/.github/actions/agent/assert-trigger-publication/action.yml b/.github/actions/agent/assert-trigger-publication/action.yml new file mode 100644 index 0000000..d03a83a --- /dev/null +++ b/.github/actions/agent/assert-trigger-publication/action.yml @@ -0,0 +1,67 @@ +name: Assert durable Agent trigger acceptance +description: Validate Provider publications in the installed Agent journal. + +inputs: + container-name: + description: Fedora container running the installed Agent. + required: true + provider-instance-id: + description: Provider identity whose durable publications are expected. + required: true + expected-triggers: + description: Space-separated trigger identifiers expected in the journal. + required: true + evidence-name: + description: Safe basename for the resulting JSON evidence. + required: true + evidence-directory: + description: Host directory receiving validated journal evidence. + required: true + timeout-seconds: + description: Maximum time to wait for the exact publication set. + required: false + default: "60" + +runs: + using: composite + steps: + - name: Assert durable trigger publications + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + CONTAINER_NAME: ${{ inputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + EVIDENCE_NAME: ${{ inputs.evidence-name }} + EXPECTED_TRIGGERS: ${{ inputs.expected-triggers }} + PROVIDER_INSTANCE_ID: ${{ inputs.provider-instance-id }} + TIMEOUT_SECONDS: ${{ inputs.timeout-seconds }} + run: | + set -euo pipefail + if [[ ! "${EVIDENCE_NAME}" =~ ^[a-z][a-z0-9-]{0,40}$ \ + || ! "${TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]; then + printf 'Trigger assertion inputs are invalid.\n' >&2 + exit 2 + fi + read -r -a expected_triggers <<< "${EXPECTED_TRIGGERS}" + if (( ${#expected_triggers[@]} == 0 )); then + printf 'At least one expected trigger is required.\n' >&2 + exit 2 + fi + + mkdir -p "${EVIDENCE_DIRECTORY}" + verifier_directory=/opt/stateforge-agent-tests/trigger-publication + verifier="${verifier_directory}/assert.py" + docker exec "${CONTAINER_NAME}" mkdir -p "${verifier_directory}" + docker cp "${ACTION_PATH}/assert.py" \ + "${CONTAINER_NAME}:${verifier}" + + arguments=() + for identifier in "${expected_triggers[@]}"; do + arguments+=(--expected-trigger "${identifier}") + done + docker exec "${CONTAINER_NAME}" /usr/bin/python3 "${verifier}" \ + --database /var/lib/stateforge-agent/journal/agent.db \ + --provider-instance-id "${PROVIDER_INSTANCE_ID}" \ + --timeout-seconds "${TIMEOUT_SECONDS}" \ + "${arguments[@]}" \ + > "${EVIDENCE_DIRECTORY}/${EVIDENCE_NAME}.json" diff --git a/.github/actions/agent/assert-trigger-publication/assert.py b/.github/actions/agent/assert-trigger-publication/assert.py new file mode 100644 index 0000000..bcb3296 --- /dev/null +++ b/.github/actions/agent/assert-trigger-publication/assert.py @@ -0,0 +1,133 @@ +"""Validate durable Trigger Publication evidence in an installed Agent.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sqlite3 +import sys +import time +from pathlib import Path +from uuid import UUID + +from stateforge_protocol.sfp.trigger_publication.v1 import ( + trigger_publication_pb2, + validation, +) + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--database", type=Path, required=True) + parser.add_argument("--provider-instance-id", type=UUID, required=True) + parser.add_argument("--expected-trigger", action="append", required=True) + parser.add_argument("--timeout-seconds", type=int, required=True) + return parser.parse_args() + + +def _rows( + database: Path, + provider_instance_id: UUID, +) -> list[sqlite3.Row]: + connection = sqlite3.connect(f"file:{database}?mode=ro", uri=True) + connection.row_factory = sqlite3.Row + try: + return list( + connection.execute( + """ + SELECT publication_id, content_fingerprint, publication, + trigger_identifier, observed_at, endpoint_generation, + session_id, manifest_revision, accepted_at, + forwarding_state + FROM trigger_publications + WHERE provider_instance_id = ? + ORDER BY accepted_at, publication_id + """, + (str(provider_instance_id),), + ) + ) + finally: + connection.close() + + +def _validated_evidence( + rows: list[sqlite3.Row], + expected_triggers: list[str], +) -> dict[str, object]: + actual = [str(row["trigger_identifier"]) for row in rows] + if sorted(actual) != sorted(expected_triggers): + raise ValueError( + "Durable publications do not match the expected trigger set" + ) + + publications: list[dict[str, object]] = [] + for row in rows: + publication = trigger_publication_pb2.TriggerPublication.FromString( + row["publication"] + ) + validation.validate_trigger_publication(publication) + fingerprint = validation.compute_publication_content_fingerprint( + publication + ) + if publication.publication_id != row["publication_id"]: + raise ValueError("Stored publication identity is inconsistent") + if publication.trigger_identifier != row["trigger_identifier"]: + raise ValueError("Stored trigger identifier is inconsistent") + if fingerprint != row["content_fingerprint"]: + raise ValueError("Stored publication fingerprint is inconsistent") + if row["forwarding_state"] != "pending": + raise ValueError( + "Accepted publication lost pending responsibility" + ) + publications.append( + { + "accepted_at": row["accepted_at"], + "content_fingerprint": fingerprint, + "endpoint_generation": row["endpoint_generation"], + "forwarding_state": row["forwarding_state"], + "manifest_revision": row["manifest_revision"], + "observed_at": row["observed_at"], + "publication_id": publication.publication_id, + "session_id_sha256": hashlib.sha256( + row["session_id"].encode() + ).hexdigest(), + "trigger_identifier": publication.trigger_identifier, + } + ) + return { + "publication_count": len(publications), + "publications": publications, + } + + +def main() -> None: + """Wait for, validate, and emit the expected durable evidence.""" + arguments = _arguments() + if arguments.timeout_seconds <= 0: + raise ValueError("timeout-seconds must be positive") + deadline = time.monotonic() + arguments.timeout_seconds + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + evidence = _validated_evidence( + _rows( + arguments.database, + arguments.provider_instance_id, + ), + arguments.expected_trigger, + ) + except (OSError, sqlite3.Error, ValueError) as error: + last_error = error + time.sleep(0.25) + continue + sys.stdout.write(json.dumps(evidence, indent=2, sort_keys=True)) + sys.stdout.write("\n") + return + raise RuntimeError( + "Installed Agent did not retain the expected trigger publications" + ) from last_error + + +if __name__ == "__main__": + main() diff --git a/.github/actions/provider/create-fixture/fixtures/manifest.py b/.github/actions/provider/create-fixture/fixtures/manifest.py index 54f7ddb..a314668 100644 --- a/.github/actions/provider/create-fixture/fixtures/manifest.py +++ b/.github/actions/provider/create-fixture/fixtures/manifest.py @@ -7,6 +7,7 @@ import hashlib import os import signal +from collections.abc import AsyncIterator from pathlib import Path import grpc @@ -19,8 +20,10 @@ Provider, ProviderModel, RequirementCheckResult, + Trigger, operation, requirement_check, + trigger, ) from stateforge_provider._services.capability_assessment.service import ( CapabilityAssessmentService, @@ -47,6 +50,19 @@ class FixtureResult(ProviderModel): value: str +class FixtureTriggerPayload(ProviderModel): + """Describe one deterministic runtime event for installed tests.""" + + sequence: int + source: str + + +_RUNTIME_REFRESHED = Trigger( + "stateforge.test/runtime.refreshed@1", + FixtureTriggerPayload, +) + + class ManifestFixtureProvider(Provider): """Expose a valid manifest and deterministic public Health controls.""" @@ -57,6 +73,7 @@ class ManifestFixtureProvider(Provider): ) version = "1.0.0" platforms = (Platform("linux", os.uname().machine),) + triggers = (_RUNTIME_REFRESHED,) def __init__( self, @@ -72,6 +89,10 @@ def __init__( self._health_status = HealthStatus.healthy() self._fail_health = False self._health_tasks: set[asyncio.Task[None]] = set() + self._trigger_tasks: set[asyncio.Task[None]] = set() + self._startup_events: asyncio.Queue[FixtureTriggerPayload] = ( + asyncio.Queue() + ) self._event_log = Path(os.environ["STATEFORGE_TEST_EVENT_LOG"]) self._target_path = Path(os.environ["STATEFORGE_TEST_TARGET_PATH"]) @@ -96,15 +117,26 @@ async def on_startup(self) -> None: HealthStatus.not_serving(), True, ) + loop.add_signal_handler(signal.SIGALRM, self._schedule_trigger) + self._startup_events.put_nowait( + FixtureTriggerPayload(sequence=1, source="decorated") + ) async def on_shutdown(self) -> None: """Remove fixture signal handlers and finish scheduled refreshes.""" loop = asyncio.get_running_loop() - for signal_number in (signal.SIGUSR1, signal.SIGUSR2, signal.SIGHUP): + for signal_number in ( + signal.SIGUSR1, + signal.SIGUSR2, + signal.SIGHUP, + signal.SIGALRM, + ): with contextlib.suppress(NotImplementedError, ValueError): loop.remove_signal_handler(signal_number) if self._health_tasks: await asyncio.gather(*self._health_tasks, return_exceptions=True) + if self._trigger_tasks: + await asyncio.gather(*self._trigger_tasks, return_exceptions=True) async def health(self) -> HealthStatus: """Return the scenario-controlled Provider-wide status.""" @@ -178,6 +210,29 @@ async def indeterminate(self, request: FixtureInput) -> FixtureResult: self._record("operation:indeterminate") return FixtureResult(value=request.value) + @trigger("stateforge.test/runtime.started@1") + async def runtime_started(self) -> AsyncIterator[FixtureTriggerPayload]: + """Publish one event through the decorated public Provider API.""" + yield await self._startup_events.get() + + def _schedule_trigger(self) -> None: + """Schedule callback publication from a process signal.""" + task = asyncio.create_task(self._publish_callback_trigger()) + self._trigger_tasks.add(task) + task.add_done_callback(self._trigger_tasks.discard) + + async def _publish_callback_trigger(self) -> None: + """Publish and record one callback-driven terminal outcome.""" + receipt = await self.publish_trigger( + _RUNTIME_REFRESHED, + FixtureTriggerPayload(sequence=2, source="callback"), + ) + outcome = await receipt.outcome() + self._record( + "trigger-outcome:" + f"{outcome.publication_id}:{outcome.disposition.value}" + ) + def _record(self, event: str) -> None: """Record fixture-only evidence outside the inspected target.""" with self._event_log.open("a", encoding="utf-8") as stream: diff --git a/.github/actions/rpm/test-installed/triggers/publication/action.yml b/.github/actions/rpm/test-installed/triggers/publication/action.yml new file mode 100644 index 0000000..ba4c5c3 --- /dev/null +++ b/.github/actions/rpm/test-installed/triggers/publication/action.yml @@ -0,0 +1,116 @@ +name: Test installed Trigger Publication +description: Verify real Provider events are durably accepted across Agent restart. + +inputs: + rpm-directory: + description: Directory containing the exact RPM artifact under test. + required: false + default: dist/rpm + evidence-directory: + description: Directory receiving this scenario's evidence. + required: false + default: dist/agent-system-test-evidence/trigger-publication + fixture-directory: + description: Directory containing the exact Provider fixture bundle. + required: false + default: dist/provider-fixture + +outputs: + evidence-directory: + description: Absolute directory containing scenario evidence. + value: ${{ steps.cleanup.outputs.evidence-directory }} + +runs: + using: composite + steps: + - name: Prepare installed Agent + id: setup + uses: ./.github/actions/rpm/test-installed/setup + with: + rpm-directory: ${{ inputs.rpm-directory }} + evidence-directory: ${{ inputs.evidence-directory }} + fixture-directory: ${{ inputs.fixture-directory }} + + - name: Start real Provider fixture + id: provider + uses: ./.github/actions/provider/start + with: + container-name: ${{ steps.setup.outputs.container-name }} + fixture-path: ${{ steps.setup.outputs.fixture-path }} + wheel-directory: ${{ steps.setup.outputs.wheel-directory }} + + - name: Assert decorated trigger acceptance + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + expected-triggers: stateforge.test/runtime.started@1 + evidence-name: before-agent-restart + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Stop the installed Agent + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + run: | + set -euo pipefail + docker exec "${CONTAINER_NAME}" \ + systemctl stop stateforge-agent.service + + - name: Start the same installed Agent again + uses: ./.github/actions/agent/start + with: + container-name: ${{ steps.setup.outputs.container-name }} + + - name: Publish through the Provider callback API + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + UNIT_NAME: ${{ steps.provider.outputs.unit-name }} + run: | + set -euo pipefail + docker exec "${CONTAINER_NAME}" \ + systemctl kill --kill-whom=main --signal=SIGALRM "${UNIT_NAME}" + + - name: Assert restart persistence and callback acceptance + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + expected-triggers: >- + stateforge.test/runtime.started@1 + stateforge.test/runtime.refreshed@1 + evidence-name: after-agent-restart + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Preserve Provider acknowledgement evidence + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + run: | + set -euo pipefail + event_log=/var/lib/stateforge-provider-fixtures/provider-a/events.log + deadline=$((SECONDS + 60)) + outcome="" + while (( SECONDS < deadline )); do + outcome="$(docker exec "${CONTAINER_NAME}" \ + grep -E '^trigger-outcome:[0-9a-f-]+:accepted_new$' \ + "${event_log}" || true)" + [[ -n "${outcome}" ]] && break + sleep 1 + done + if [[ -z "${outcome}" ]]; then + printf 'Provider did not receive the terminal trigger acknowledgement.\n' >&2 + exit 1 + fi + printf '%s\n' "${outcome}" \ + > "${EVIDENCE_DIRECTORY}/provider-trigger-outcome.txt" + + - name: Preserve evidence and clean up + id: cleanup + if: ${{ always() }} + uses: ./.github/actions/rpm/test-installed/cleanup + with: + container-name: stateforge-agent-system-test + evidence-directory: ${{ inputs.evidence-directory }} diff --git a/.github/workflows/rpm-installed-test.yml b/.github/workflows/rpm-installed-test.yml index c05e2ba..0997350 100644 --- a/.github/workflows/rpm-installed-test.yml +++ b/.github/workflows/rpm-installed-test.yml @@ -108,6 +108,7 @@ jobs: - capability-mismatch - capability-limits - provider-availability + - trigger-publication - multiple-providers - invalid-descriptor - uninstall-reinstall @@ -165,6 +166,13 @@ jobs: evidence-directory: dist/agent-system-test-evidence/${{ matrix.scenario }} fixture-directory: dist/provider-fixture + - name: Run Trigger Publication scenario + if: ${{ matrix.scenario == 'trigger-publication' }} + uses: ./.github/actions/rpm/test-installed/triggers/publication + with: + evidence-directory: dist/agent-system-test-evidence/${{ matrix.scenario }} + fixture-directory: dist/provider-fixture + - name: Run invalid-descriptor scenario if: ${{ matrix.scenario == 'invalid-descriptor' }} uses: ./.github/actions/rpm/test-installed/adversarial/invalid-descriptor diff --git a/README.md b/README.md index 4659512..8f7b32e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ The current implementation provides: - validated and reusable local gRPC channels; - SFP handshake and version negotiation; - bounded Provider manifest retrieval, validation, and declaration caching; -- standard gRPC Health Watch-based operational eligibility; and +- standard gRPC Health Watch-based operational eligibility; +- durable, idempotent acceptance of manifest-declared Provider triggers; and - bounded reconnection and endpoint replacement. Providers run as separate processes. They expose vendor-specific behavior over @@ -104,7 +105,9 @@ mkdocs build --strict The test suite includes integration coverage for Provider discovery, channel establishment, SFP negotiation, manifest exchange, endpoint replacement, Provider Health transitions, eligibility fencing, connection loss, and -recovery. +recovery. It also covers real Provider trigger publication, acknowledgement, +durable Agent restart persistence, idempotent replay, bounded capacity, and +stale-session fencing. ## Documentation diff --git a/docs/ci-cd.md b/docs/ci-cd.md index 1fbe918..3821a37 100644 --- a/docs/ci-cd.md +++ b/docs/ci-cd.md @@ -24,6 +24,8 @@ operation. │ │ │ └── multiple-providers/ │ │ ├── health/ │ │ │ └── provider-availability/ +│ │ ├── triggers/ +│ │ │ └── publication/ │ │ ├── adversarial/ │ │ │ └── invalid-descriptor/ │ │ └── destructive/ @@ -31,6 +33,7 @@ operation. │ ├── agent/ │ │ ├── assert-availability/ │ │ ├── assert-capability-assessment/ +│ │ ├── assert-trigger-publication/ │ │ ├── install/ │ │ ├── start/ │ │ └── collect-evidence/ @@ -116,7 +119,7 @@ Contents permission. The token is scoped to those two repositories, is not persisted by checkout, and expires automatically. A long-lived personal access token is neither required nor accepted by the workflow interface. -The installed-system workflow currently runs ten isolated scenarios: +The installed-system workflow currently runs eleven isolated scenarios: - smoke: one public-API Provider completes manifest exchange; - capability: the RPM-installed Agent retries one deterministic transient @@ -131,6 +134,9 @@ The installed-system workflow currently runs ten isolated scenarios: - lifecycle: two independent Provider processes are accepted concurrently; - health: real Providers drive initial replay, loss, recovery, concurrent Watch streams, shutdown, and same-installation generation replacement; +- triggers: a real Provider wheel publishes through both the decorated and + callback author APIs; the installed Agent validates and journals the events, + survives an Agent service restart, and returns a terminal acknowledgement; - adversarial: malformed discovery data is rejected without stopping the Agent; - destructive: the exact RPM is removed, installed again in a clean environment, and completes manifest exchange after reinstallation. @@ -155,6 +161,7 @@ digest of each generated wheel. | `agent/assert-availability` | Assert an exact Provider generation reached a local eligibility state. | | `agent/assert-capability-assessment` | Use the RPM-installed Agent package to collect and verify normalized evidence from a separate real Provider process. | | `agent/assert-capability-boundary` | Exercise timeout, malformed response, stale response-fence, and byte-limit behavior through the RPM-installed client. | +| `agent/assert-trigger-publication` | Validate canonical, durable Provider publication evidence in the installed Agent journal. | | `agent/collect-evidence` | Capture bounded evidence, uninstall the RPM, and clean up. | | `provider/create-fixture` | Invoke the selected Protocol and Provider checkouts' canonical package-build actions and stage their wheels with a selected fixture. | | `provider/start` | Run the fixture as a separate non-root systemd service. | @@ -191,8 +198,8 @@ assertions that distinguish that case. The reusable workflow gives every scenario its own job and clean container, so one failure cannot contaminate a later scenario. -Use one of the `smoke`, `capability`, `lifecycle`, `health`, `adversarial`, or -`destructive` tiers. A scenario action must always invoke cleanup with +Use one of the `smoke`, `capability`, `lifecycle`, `health`, `triggers`, +`adversarial`, or `destructive` tiers. A scenario action must always invoke cleanup with `always()` so failure evidence is retained. Adding a normal scenario requires one action and one thin workflow job; it must not duplicate RPM installation, systemd readiness, diff --git a/docs/configuration.md b/docs/configuration.md index 4f53e75..43c9376 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,9 +31,11 @@ request_timeout_seconds = 30 health_interval_seconds = 15 reconnect_initial_delay_seconds = 1 reconnect_max_delay_seconds = 30 +trigger_max_in_flight_publications = 32 [journal] backend = "sqlite" +maximum_pending_trigger_publications = 10000 [logging] level = "info" @@ -106,6 +108,7 @@ does not check whether the referenced file exists during `check-config`. | `health_interval_seconds` | Integer | `15` | Maximum bounded delay before reopening a failed Provider Health Watch. Must be greater than zero. The Agent does not poll healthy Providers. | | `reconnect_initial_delay_seconds` | Integer | `1` | Initial bounded-backoff delay after a failed Provider connection. Must be greater than zero. | | `reconnect_max_delay_seconds` | Integer | `30` | Maximum reconnect delay. Must be greater than zero and greater than or equal to the initial delay. | +| `trigger_max_in_flight_publications` | Integer | `32` | Requested SFP Trigger Publication receive window. Must be between 1 and the Protocol-owned maximum of 1,024. | The RPM creates the Provider discovery directory through `systemd-tmpfiles`. Provider processes publish their endpoint descriptors and Unix domain sockets @@ -116,6 +119,7 @@ there; the Agent does not use this directory for Provider-specific data. | Field | Type | Packaged value | Validation and purpose | | --- | --- | --- | --- | | `backend` | String | `sqlite` | Durable journal backend. `sqlite` is the only value supported by schema version 1. | +| `maximum_pending_trigger_publications` | Integer | `10000` | Maximum accepted trigger publications retained for later SF-AOP forwarding. Must be greater than zero. When full, new publications receive a retryable capacity rejection. | The journal database is stored at the package-owned path `/var/lib/stateforge-agent/journal/agent.db`. diff --git a/docs/index.md b/docs/index.md index 7550c3f..09eae73 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,7 +29,10 @@ The current runtime can: - cache accepted, immutable declarations independently of channel lifetime; and - collect bounded, read-only Capability Assessment evidence for declared - operations without converting that evidence into authorization. + operations without converting that evidence into authorization; and +- consume manifest-declared Provider triggers, validate their payloads, and + durably assume bounded local delivery responsibility before acknowledging + them. The runtime reports a Provider as connected only after both session setup and manifest validation succeed. Operational eligibility is stricter: the exact @@ -42,3 +45,9 @@ unsupported, and indeterminate outcomes together with their requirement states, reasons, and constraints. Assessment evidence is fenced to the exact session, manifest, availability generation, operation set, and execution context that produced it. + +Trigger Publication begins only for the exact current session while Provider +Health reports `SERVING`. The Agent validates each event against the canonical +P3 contract and accepted manifest schema, then commits its idempotency record +before returning an acknowledgement. Accepted events remain pending in the +Agent journal until the later SF-AOP forwarding boundary is implemented. diff --git a/docs/running.md b/docs/running.md index 1b6385f..b54cc5c 100644 --- a/docs/running.md +++ b/docs/running.md @@ -98,6 +98,21 @@ new-work eligibility; it neither cancels nor invents outcomes for work already admitted. Provider availability transitions and sanitized reasons are written to the local Agent journal for diagnostics and later internal consumers. +## Trigger acceptance + +Once a Provider is operationally eligible, the Agent opens its Trigger +Publication stream on the existing SFP channel. Incoming events must satisfy +the canonical Protocol contract, belong to the accepted manifest, and match +the declared payload schema. The Agent commits accepted content to its durable +journal before acknowledging the Provider. Exact replays receive a duplicate +acceptance; publication-ID reuse with different content is rejected. + +Health loss, endpoint replacement, descriptor removal, and channel failure +cancel the current trigger consumer. A stale stream cannot commit against a +new session or availability generation. Accepted records survive Agent restart +and remain bounded by `journal.maximum_pending_trigger_publications` while +they await the future SF-AOP forwarding implementation. + The packaged service owns the Agent filesystem layout. Persistent state is stored below `/var/lib/stateforge-agent`, and volatile runtime files are stored below `/run/stateforge-agent`. These locations are deliberately not diff --git a/docs/sfp-trigger-publication.md b/docs/sfp-trigger-publication.md new file mode 100644 index 0000000..827c91b --- /dev/null +++ b/docs/sfp-trigger-publication.md @@ -0,0 +1,88 @@ +# Trigger Publication + +The Agent consumes Provider-originated events through the StateForge Protocol +Trigger Publication procedure. A stream is opened only for a Provider session +that has an accepted manifest and currently reports `SERVING` through its +Provider-wide Health Watch. + +```mermaid +sequenceDiagram + participant P as Provider + participant A as Agent + participant J as Durable Agent journal + + A->>P: OpenStream(session fences, receive window) + P-->>A: Ready(exact session fences) + P-->>A: TriggerPublication + A->>A: Validate P3 and manifest payload schema + A->>J: Commit idempotency record and pending responsibility + J-->>A: Durable decision + A-->>P: Terminal acknowledgement +``` + +## Acceptance boundary + +For every publication, the Agent applies these checks in order: + +1. Validate the canonical P3 message, identifiers, timestamp, JSON value, and + serialized-message limits owned by `stateforge-protocol`. +2. Require the trigger identifier to exist in the exact accepted Provider + manifest. +3. Validate the payload against that declaration's Draft 2020-12 JSON Schema. +4. Commit the immutable publication and its content fingerprint to the Agent + journal. +5. Recheck that the Provider session, manifest, and availability generation + are still current. +6. Return one terminal acknowledgement to the Provider. + +The acknowledgement is never sent before the journal commit. A lost +acknowledgement therefore causes a safe Provider replay rather than a lost +event. + +## Durable decisions + +The idempotency key is the Provider installation identity together with the +Provider-minted publication ID. + +| Existing journal state | Agent decision | +| --- | --- | +| No matching key and capacity is available | Commit the publication and return `ACCEPTED_NEW`. | +| Same key and same canonical content fingerprint | Return `ACCEPTED_DUPLICATE`. | +| Same key with different content | Reject the publication as an idempotency conflict. | +| No matching key and the pending-responsibility limit is full | Return a retryable capacity rejection. | + +Accepted records remain in +`/var/lib/stateforge-agent/journal/agent.db`. They retain the deterministic +protobuf bytes, content fingerprint, trigger identifier, Provider/session +fences, acceptance time, and forwarding state. Payloads are not written to +logs or error messages. + +The current milestone stops at durable local acceptance. Accepted records are +marked `pending` for the future SF-AOP forwarding boundary; they are not +deleted or presented as delivered to the Control Plane. + +## Stream lifecycle + +Only one Trigger Publication consumer is active for an eligible Provider +session. The stream is cancelled when Health is lost, the endpoint is +replaced, the descriptor is removed, the channel fails, or the Agent shuts +down. Every callback is fenced to the exact Provider installation, endpoint +generation, session, manifest revision, and availability generation that +opened it, so stale streams cannot commit new responsibility. + +Transient transport failures are retried with the Agent's bounded reconnect +backoff. Canonical protocol violations and stale ready responses terminate the +attempt without accepting data. Provider replay across stream loss or Agent +restart is resolved by the durable idempotency record. + +## Bounds + +Two configuration values keep local responsibility explicit: + +- `providers.trigger_max_in_flight_publications` selects the receive window + advertised to each Provider stream; +- `journal.maximum_pending_trigger_publications` limits the number of + accepted publications awaiting the future forwarding boundary. + +Both values must be positive and must not exceed the canonical P3 limits +published by `stateforge-protocol`. diff --git a/mkdocs.yml b/mkdocs.yml index ea58a55..fb4f83a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,6 +19,7 @@ nav: - Running the Agent: running.md - Configuration: configuration.md - Capability Assessment: sfp-capability-assessment.md + - Trigger Publication: sfp-trigger-publication.md - RPM CI/CD: ci-cd.md - Internal reference: - Configuration models: reference/configuration.md diff --git a/packaging/rpm/SOURCES/stateforge-agent.toml b/packaging/rpm/SOURCES/stateforge-agent.toml index dcb19de..179db21 100644 --- a/packaging/rpm/SOURCES/stateforge-agent.toml +++ b/packaging/rpm/SOURCES/stateforge-agent.toml @@ -18,9 +18,11 @@ request_timeout_seconds = 30 health_interval_seconds = 15 reconnect_initial_delay_seconds = 1 reconnect_max_delay_seconds = 30 +trigger_max_in_flight_publications = 32 [journal] backend = "sqlite" +maximum_pending_trigger_publications = 10000 [logging] level = "info" diff --git a/pyproject.toml b/pyproject.toml index 54a61f9..9f92450 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ classifiers = [ dependencies = [ "grpcio>=1.83", "grpcio-health-checking>=1.83", + "jsonschema>=4.25,<5", "stateforge-protocol>=0.1.0,<0.2.0", "watchfiles>=1.2,<2", ] @@ -59,6 +60,7 @@ dev = [ "ruff>=0.12", "types-grpcio", "types-grpcio-health-checking", + "types-jsonschema", "types-protobuf", ] docs = [ @@ -74,7 +76,7 @@ environments = ["python_version == '3.13'"] [tool.uv.sources] -stateforge-protocol = { git = "https://github.com/stateforge-org/stateforge-protocol.git", rev = "a4c077a5b86757b1120b229c048aa0a4cbb9f147" } +stateforge-protocol = { git = "https://github.com/stateforge-org/stateforge-protocol.git", rev = "e4d67e8fd7abbb077e09c41664e6f2945760037e" } [tool.hatch.build.targets.wheel] diff --git a/src/stateforge_agent/_config/loading.py b/src/stateforge_agent/_config/loading.py index 06bcb81..95bf22e 100644 --- a/src/stateforge_agent/_config/loading.py +++ b/src/stateforge_agent/_config/loading.py @@ -12,6 +12,10 @@ from typing import TypeAlias, cast from urllib.parse import urlsplit +from stateforge_protocol.sfp.trigger_publication.v1 import ( + contract as trigger_publication_contract, +) + from stateforge_agent._config.errors import ( ConfigurationFileError, ConfigurationValidationError, @@ -330,17 +334,19 @@ def _validate_configuration( "health_interval_seconds", "reconnect_initial_delay_seconds", "reconnect_max_delay_seconds", + "trigger_max_in_flight_publications", }, required={ "health_interval_seconds", "reconnect_initial_delay_seconds", "reconnect_max_delay_seconds", + "trigger_max_in_flight_publications", }, ) validator.table( ("journal",), - allowed={"backend"}, - required={"backend"}, + allowed={"backend", "maximum_pending_trigger_publications"}, + required={"backend", "maximum_pending_trigger_publications"}, ) validator.table( ("logging",), @@ -377,6 +383,19 @@ def _validate_configuration( reconnect_max_delay = validator.positive_integer( ("providers", "reconnect_max_delay_seconds") ) + trigger_max_in_flight = validator.positive_integer( + ("providers", "trigger_max_in_flight_publications") + ) + if ( + trigger_max_in_flight + > trigger_publication_contract.MAX_IN_FLIGHT_PUBLICATIONS + ): + validator.add_issue( + ("providers", "trigger_max_in_flight_publications"), + "must not exceed the SFP Trigger Publication receive-window " + "limit of " + f"{trigger_publication_contract.MAX_IN_FLIGHT_PUBLICATIONS}", + ) if ( reconnect_initial_delay > 0 and reconnect_max_delay > 0 @@ -389,6 +408,9 @@ def _validate_configuration( ) journal_backend = validator.choice(("journal", "backend"), {"sqlite"}) + maximum_pending_trigger_publications = validator.positive_integer( + ("journal", "maximum_pending_trigger_publications") + ) log_level = validator.choice( ("logging", "level"), @@ -412,9 +434,13 @@ def _validate_configuration( health_interval_seconds=health_interval, reconnect_initial_delay_seconds=reconnect_initial_delay, reconnect_max_delay_seconds=reconnect_max_delay, + trigger_max_in_flight_publications=trigger_max_in_flight, ), journal=JournalConfiguration( backend=cast(JournalBackend, journal_backend), + maximum_pending_trigger_publications=( + maximum_pending_trigger_publications + ), ), logging=LoggingConfiguration( level=cast(LogLevel, log_level), diff --git a/src/stateforge_agent/_config/models.py b/src/stateforge_agent/_config/models.py index dd06c63..6e63ff3 100644 --- a/src/stateforge_agent/_config/models.py +++ b/src/stateforge_agent/_config/models.py @@ -33,6 +33,7 @@ class ProviderConfiguration: health_interval_seconds: int reconnect_initial_delay_seconds: int reconnect_max_delay_seconds: int + trigger_max_in_flight_publications: int @dataclass(frozen=True, slots=True) @@ -40,6 +41,7 @@ class JournalConfiguration: """Durable Agent journal settings.""" backend: JournalBackend + maximum_pending_trigger_publications: int @dataclass(frozen=True, slots=True) diff --git a/src/stateforge_agent/_journal/__init__.py b/src/stateforge_agent/_journal/__init__.py new file mode 100644 index 0000000..1348abe --- /dev/null +++ b/src/stateforge_agent/_journal/__init__.py @@ -0,0 +1,8 @@ +"""Agent-owned durable journal infrastructure.""" + +from stateforge_agent._journal.sqlite import ( + SqliteJournal, + SqliteJournalError, +) + +__all__ = ["SqliteJournal", "SqliteJournalError"] diff --git a/src/stateforge_agent/_journal/sqlite.py b/src/stateforge_agent/_journal/sqlite.py new file mode 100644 index 0000000..a7ae54e --- /dev/null +++ b/src/stateforge_agent/_journal/sqlite.py @@ -0,0 +1,103 @@ +"""Run serialized SQLite journal transactions outside the event loop.""" + +from __future__ import annotations + +import asyncio +import os +import sqlite3 +from collections.abc import Callable +from pathlib import Path +from typing import TypeVar + +_Result = TypeVar("_Result") + + +class SqliteJournalError(RuntimeError): + """Report that the Agent journal could not complete an operation.""" + + +class SqliteJournal: + """Own one bounded SQLite transaction boundary for Agent state.""" + + def __init__(self, path: Path) -> None: + """Bind the journal to its platform-owned persistent path.""" + self._path = path + self._lock = asyncio.Lock() + self._opened = False + + @property + def path(self) -> Path: + """Return the platform-owned database path.""" + return self._path + + async def open(self) -> None: + """Create and configure the journal without replacing existing data.""" + async with self._lock: + if self._opened: + return + try: + await asyncio.to_thread(self._prepare) + except (OSError, sqlite3.Error) as error: + raise SqliteJournalError( + "Agent journal could not be opened" + ) from error + self._opened = True + + async def transaction( + self, + operation: Callable[[sqlite3.Connection], _Result], + ) -> _Result: + """Run one immediate transaction with serialized local access.""" + async with self._lock: + if not self._opened: + raise SqliteJournalError("Agent journal is not open") + try: + return await asyncio.to_thread(self._transact, operation) + except (OSError, sqlite3.Error) as error: + raise SqliteJournalError( + "Agent journal transaction failed" + ) from error + + async def close(self) -> None: + """Reject future transactions after current work has finished.""" + async with self._lock: + self._opened = False + + def _prepare(self) -> None: + """Prepare the directory and durable SQLite mode synchronously.""" + self._path.parent.mkdir(mode=0o750, parents=True, exist_ok=True) + connection = self._connect() + try: + connection.execute("PRAGMA journal_mode = WAL") + finally: + connection.close() + os.chmod(self._path, 0o640) + + def _transact( + self, + operation: Callable[[sqlite3.Connection], _Result], + ) -> _Result: + """Execute and durably commit one synchronous transaction.""" + connection = self._connect() + try: + connection.execute("BEGIN IMMEDIATE") + result = operation(connection) + connection.commit() + return result + except BaseException: + connection.rollback() + raise + finally: + connection.close() + + def _connect(self) -> sqlite3.Connection: + """Open one fully configured local connection.""" + connection = sqlite3.connect( + self._path, + timeout=5, + isolation_level=None, + ) + connection.execute("PRAGMA busy_timeout = 5000") + connection.execute("PRAGMA foreign_keys = ON") + connection.execute("PRAGMA synchronous = FULL") + return connection diff --git a/src/stateforge_agent/_runtime/daemon.py b/src/stateforge_agent/_runtime/daemon.py index 6a5d103..4275b00 100644 --- a/src/stateforge_agent/_runtime/daemon.py +++ b/src/stateforge_agent/_runtime/daemon.py @@ -14,6 +14,7 @@ from stateforge_agent import __version__ from stateforge_agent._config.models import AgentConfiguration +from stateforge_agent._journal import SqliteJournal, SqliteJournalError from stateforge_agent._process import ProcessExitCode from stateforge_agent._runtime.layout import ( SYSTEM_RUNTIME_LAYOUT, @@ -38,6 +39,12 @@ ProviderDirectoryEventKind, ProviderDirectoryMonitor, ) +from stateforge_agent._sfp._services.trigger_publication.coordinator import ( + TriggerPublicationCoordinator, +) +from stateforge_agent._sfp._services.trigger_publication.store import ( + TriggerPublicationStore, +) _LOGGER = logging.getLogger(__name__) _SUPPORTED_SFP_VERSIONS = (SfpVersion(major=1, minor=0),) @@ -67,6 +74,21 @@ async def _run_agent( monitor = ProviderDirectoryMonitor( runtime_layout.provider_discovery_directory ) + journal = SqliteJournal(runtime_layout.journal_path) + trigger_store = TriggerPublicationStore( + journal, + maximum_pending_publications=( + configuration.journal.maximum_pending_trigger_publications + ), + ) + try: + await journal.open() + await trigger_store.initialize() + except SqliteJournalError: + _LOGGER.exception("Agent journal initialization failed") + for installed_signal in installed_signals: + loop.remove_signal_handler(installed_signal) + return ProcessExitCode.RUNTIME_FAILURE def reload_provider_descriptor( provider_instance_id: UUID, @@ -92,6 +114,12 @@ def reload_provider_descriptor( configuration.providers.health_interval_seconds ), descriptor_loader=reload_provider_descriptor, + trigger_publication=TriggerPublicationCoordinator( + trigger_store, + maximum_in_flight_publications=( + configuration.providers.trigger_max_in_flight_publications + ), + ), ) try: @@ -104,6 +132,7 @@ def reload_provider_descriptor( return ProcessExitCode.RUNTIME_FAILURE finally: await sessions.close() + await journal.close() for installed_signal in installed_signals: loop.remove_signal_handler(installed_signal) diff --git a/src/stateforge_agent/_sfp/_runtime/sessions.py b/src/stateforge_agent/_sfp/_runtime/sessions.py index d791275..1a26318 100644 --- a/src/stateforge_agent/_sfp/_runtime/sessions.py +++ b/src/stateforge_agent/_sfp/_runtime/sessions.py @@ -50,6 +50,15 @@ from stateforge_agent._sfp._services.manifest.declaration import ( ManifestDeclaration, ) +from stateforge_agent._sfp._services.trigger_publication.client import ( + SfpTriggerPublicationError, + SfpTriggerPublicationStaleError, + SfpTriggerPublicationUnavailableError, +) +from stateforge_agent._sfp._services.trigger_publication.coordinator import ( + TriggerPublicationCoordinator, + TriggerPublicationLease, +) from stateforge_agent._sfp.version import SfpVersion _LOGGER = logging.getLogger(__name__) @@ -140,6 +149,7 @@ class _ProviderRuntimeState: attempt_token: object | None = None channel_watcher: asyncio.Task[None] | None = None health_watcher: asyncio.Task[None] | None = None + trigger_watcher: asyncio.Task[None] | None = None retry_task: asyncio.Task[None] | None = None availability_generation: int = 0 @@ -170,6 +180,7 @@ def __init__( capability_assessment_client: ( CapabilityAssessmentRequester | None ) = None, + trigger_publication: TriggerPublicationCoordinator | None = None, ) -> None: """Create an empty Provider session registry.""" if not isinstance(agent_runtime_id, UUID): @@ -234,6 +245,7 @@ def __init__( cache_entries=capability_assessment_cache_entries, client=capability_assessment_client, ) + self._trigger_publication = trigger_publication self._providers: dict[UUID, _ProviderRuntimeState] = {} self._closed = False self._lock = asyncio.Lock() @@ -601,12 +613,14 @@ async def remove(self, provider_instance_id: UUID) -> None: session = provider.active_session watcher = provider.channel_watcher health_watcher = provider.health_watcher + trigger_watcher = provider.trigger_watcher retry_task = provider.retry_task provider.desired_descriptor = None provider.active_session = None provider.attempt_token = None provider.channel_watcher = None provider.health_watcher = None + provider.trigger_watcher = None provider.retry_task = None provider.connection_status = ProviderConnectionStatus( provider_instance_id=provider_instance_id, @@ -633,6 +647,7 @@ async def remove(self, provider_instance_id: UUID) -> None: await _cancel_task(retry_task) await _cancel_task(watcher) await _cancel_task(health_watcher) + await _cancel_task(trigger_watcher) if session is not None: await session._channel.close() @@ -655,6 +670,11 @@ async def close(self) -> None: for provider in self._providers.values() if provider.health_watcher is not None ) + trigger_watchers = tuple( + provider.trigger_watcher + for provider in self._providers.values() + if provider.trigger_watcher is not None + ) retry_tasks = tuple( provider.retry_task for provider in self._providers.values() @@ -667,6 +687,7 @@ async def close(self) -> None: provider.attempt_token = None provider.channel_watcher = None provider.health_watcher = None + provider.trigger_watcher = None provider.retry_task = None provider.connection_status = ProviderConnectionStatus( provider_instance_id=provider_instance_id, @@ -696,12 +717,16 @@ async def close(self) -> None: watcher.cancel() for health_watcher in health_watchers: health_watcher.cancel() + for trigger_watcher in trigger_watchers: + trigger_watcher.cancel() if retry_tasks: await asyncio.gather(*retry_tasks, return_exceptions=True) if watchers: await asyncio.gather(*watchers, return_exceptions=True) if health_watchers: await asyncio.gather(*health_watchers, return_exceptions=True) + if trigger_watchers: + await asyncio.gather(*trigger_watchers, return_exceptions=True) for session in sessions: await session._channel.close() self._capability_assessment.clear() @@ -1060,6 +1085,126 @@ def _record_availability( status.eligible, status.reason, ) + self._synchronize_trigger_watcher(provider) + + def _synchronize_trigger_watcher( + self, + provider: _ProviderRuntimeState, + ) -> None: + """Start or fence Trigger Publication with current availability.""" + watcher = provider.trigger_watcher + status = provider.availability_status + session = provider.active_session + if ( + self._trigger_publication is None + or status is None + or not status.eligible + or session is None + or not session.manifest.trigger_schemas + ): + provider.trigger_watcher = None + if watcher is not None: + watcher.cancel() + return + if watcher is not None and not watcher.done(): + return + provider.trigger_watcher = asyncio.create_task( + self._watch_triggers( + session, + provider.availability_generation, + ), + name=f"stateforge-sfp-triggers-{session.provider_instance_id}", + ) + + async def _watch_triggers( + self, + session: ProviderSession, + availability_generation: int, + ) -> None: + """Maintain one fenced Trigger Publication stream with backoff.""" + coordinator = self._trigger_publication + if coordinator is None: + return + lease = TriggerPublicationLease( + provider_instance_id=session.provider_instance_id, + endpoint_generation=session.endpoint_generation, + session_id=session.session_id, + manifest_revision=session.manifest.key.revision, + availability_generation=availability_generation, + trigger_schemas=session.manifest.trigger_schemas, + channel=session._channel, + session_token=session, + ) + attempt_number = 0 + try: + while True: + try: + await coordinator.run( + lease, + assert_current=self._assert_trigger_stream_is_current, + ) + except asyncio.CancelledError: + raise + except SfpTriggerPublicationStaleError: + return + except SfpTriggerPublicationUnavailableError as error: + delay = self._health_retry_delay(attempt_number) + attempt_number += 1 + _LOGGER.info( + "Scheduling Trigger Publication retry: " + "provider_instance_id=%s endpoint_generation=%s " + "attempt=%s delay_seconds=%.3f reason=%s", + session.provider_instance_id, + session.endpoint_generation, + attempt_number, + delay, + error, + ) + await asyncio.sleep(delay) + continue + except SfpTriggerPublicationError as error: + _LOGGER.warning( + "Trigger Publication stream rejected: " + "provider_instance_id=%s endpoint_generation=%s " + "error=%s", + session.provider_instance_id, + session.endpoint_generation, + error, + ) + return + raise SfpTriggerPublicationUnavailableError( + "Trigger Publication stream ended" + ) + finally: + async with self._lock: + provider = self._providers.get(session.provider_instance_id) + if ( + provider is not None + and provider.trigger_watcher is asyncio.current_task() + ): + provider.trigger_watcher = None + + async def _assert_trigger_stream_is_current( + self, + lease: TriggerPublicationLease, + ) -> None: + """Fence Trigger Publication to one available current session.""" + async with self._lock: + provider = self._providers.get(lease.provider_instance_id) + if ( + provider is None + or provider.active_session is not lease.session_token + or provider.availability_generation + != lease.availability_generation + ): + raise SfpTriggerPublicationStaleError( + "Trigger Publication session is no longer current" + ) + status = provider.availability_status + if status is None or not status.eligible: + raise SfpTriggerPublicationStaleError( + "Provider is unavailable for Trigger Publication" + ) def _health_retry_delay(self, attempt_number: int) -> float: """Return a capped jittered delay for a failed Health Watch.""" @@ -1103,6 +1248,7 @@ async def _invalidate_lost_session( ) -> None: """Remove a failed active session without disturbing replacements.""" health_watcher: asyncio.Task[None] | None = None + trigger_watcher: asyncio.Task[None] | None = None async with self._lock: provider = self._providers.get(session.provider_instance_id) if provider is None or provider.active_session is not session: @@ -1110,6 +1256,8 @@ async def _invalidate_lost_session( provider.active_session = None health_watcher = provider.health_watcher provider.health_watcher = None + trigger_watcher = provider.trigger_watcher + provider.trigger_watcher = None if provider.channel_watcher is asyncio.current_task(): provider.channel_watcher = None @@ -1154,6 +1302,7 @@ async def _invalidate_lost_session( ), ) await _cancel_task(health_watcher) + await _cancel_task(trigger_watcher) await session._channel.close() await self._schedule_retry(session.provider_instance_id) diff --git a/src/stateforge_agent/_sfp/_services/manifest/declaration.py b/src/stateforge_agent/_sfp/_services/manifest/declaration.py index 91dba8f..ea0dc50 100644 --- a/src/stateforge_agent/_sfp/_services/manifest/declaration.py +++ b/src/stateforge_agent/_sfp/_services/manifest/declaration.py @@ -64,6 +64,24 @@ def operation_identifiers(self) -> frozenset[str]: raise RuntimeError("Validated manifest cache entry is corrupt") return frozenset(operations) + @property + def trigger_schemas(self) -> dict[str, dict[str, Any]]: + """Return payload schemas keyed by declared trigger identifier.""" + triggers = self.as_dict().get("triggers") + if not isinstance(triggers, dict): + raise RuntimeError("Validated manifest cache entry is corrupt") + schemas: dict[str, dict[str, Any]] = {} + for identifier, declaration in triggers.items(): + if not isinstance(identifier, str) or not isinstance( + declaration, dict + ): + raise RuntimeError("Validated manifest cache entry is corrupt") + schema = declaration.get("payloadSchema") + if not isinstance(schema, dict): + raise RuntimeError("Validated manifest cache entry is corrupt") + schemas[identifier] = schema + return schemas + def cache_key( response: manifest_pb2.GetManifestResponse, diff --git a/src/stateforge_agent/_sfp/_services/trigger_publication/__init__.py b/src/stateforge_agent/_sfp/_services/trigger_publication/__init__.py new file mode 100644 index 0000000..a9b51be --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/trigger_publication/__init__.py @@ -0,0 +1,15 @@ +"""Agent-side Trigger Publication coordination.""" + +from stateforge_agent._sfp._services.trigger_publication.store import ( + TriggerAcceptanceContext, + TriggerAcceptanceDecision, + TriggerAcceptanceResult, + TriggerPublicationStore, +) + +__all__ = [ + "TriggerAcceptanceContext", + "TriggerAcceptanceDecision", + "TriggerAcceptanceResult", + "TriggerPublicationStore", +] diff --git a/src/stateforge_agent/_sfp/_services/trigger_publication/client.py b/src/stateforge_agent/_sfp/_services/trigger_publication/client.py new file mode 100644 index 0000000..9fa9788 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/trigger_publication/client.py @@ -0,0 +1,254 @@ +"""Consume one fenced SFP Trigger Publication stream.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Never, Protocol + +import grpc +from google.protobuf.message import DecodeError +from stateforge_protocol.sfp.trigger_publication.v1 import ( + contract, + trigger_publication_pb2, + validation, +) + + +class SfpTriggerPublicationError(RuntimeError): + """Report a sanitized Trigger Publication stream failure.""" + + +class SfpTriggerPublicationUnavailableError(SfpTriggerPublicationError): + """Report a stream that may be reopened while its lease remains current.""" + + +class SfpTriggerPublicationStaleError(SfpTriggerPublicationError): + """Report a stream rejected because its session fences are stale.""" + + +class SfpTriggerPublicationProtocolError(SfpTriggerPublicationError): + """Report invalid framing or Provider response data.""" + + +class SfpTriggerPublicationLimitError(SfpTriggerPublicationError): + """Report a Provider response exceeding the canonical byte limit.""" + + +@dataclass(frozen=True, slots=True) +class TriggerStreamBinding: + """Carry the exact Agent-owned fences used to open one stream.""" + + provider_instance_id: str + endpoint_generation: int + session_id: str + manifest_revision: str + maximum_in_flight_publications: int + + +TriggerPublicationHandler = Callable[ + [trigger_publication_pb2.TriggerPublication], + Awaitable[trigger_publication_pb2.TriggerAcknowledgement], +] + + +class TriggerPublicationConsumer(Protocol): + """Describe one finite ownership boundary for a stream attempt.""" + + async def consume( + self, + channel: grpc.aio.Channel, + binding: TriggerStreamBinding, + handler: TriggerPublicationHandler, + ) -> None: + """Consume until the stream closes, fails, or is cancelled.""" + ... + + +class _TriggerStreamCall(Protocol): + """Type the dynamic asynchronous bidirectional call used at runtime.""" + + async def write( + self, + request: trigger_publication_pb2.StreamTriggersRequest, + ) -> None: + """Write one request message.""" + ... + + async def read(self) -> object: + """Read one decoded response or end-of-stream marker.""" + ... + + def cancel(self) -> bool: + """Cancel the active call.""" + ... + + +@dataclass(frozen=True, slots=True) +class _DecodedTriggerResponse: + """Carry a decoded response or a safe wire rejection.""" + + response: trigger_publication_pb2.StreamTriggersResponse | None = None + malformed: bool = False + limit_exceeded: bool = False + + +_SERVICE_DESCRIPTOR = trigger_publication_pb2.DESCRIPTOR.services_by_name[ + "TriggerPublicationService" +] +_STREAM_DESCRIPTOR = _SERVICE_DESCRIPTOR.methods_by_name["StreamTriggers"] +_STREAM_METHOD_PATH = ( + f"/{_SERVICE_DESCRIPTOR.full_name}/{_STREAM_DESCRIPTOR.name}" +) + + +class TriggerPublicationClient: + """Open one Trigger Publication stream and acknowledge serially.""" + + async def consume( + self, + channel: grpc.aio.Channel, + binding: TriggerStreamBinding, + handler: TriggerPublicationHandler, + ) -> None: + """Pass validated publications to the durable handler.""" + open_message = trigger_publication_pb2.TriggerStreamOpen( + session_binding=trigger_publication_pb2.TriggerSessionBinding( + expected_provider_instance_id=binding.provider_instance_id, + expected_endpoint_generation=binding.endpoint_generation, + expected_session_id=binding.session_id, + expected_manifest_revision=binding.manifest_revision, + ), + requested_max_in_flight_publications=( + binding.maximum_in_flight_publications + ), + ) + opening_request = trigger_publication_pb2.StreamTriggersRequest( + open=open_message + ) + _validate_request(opening_request) + + callable_ = channel.stream_stream( + _STREAM_METHOD_PATH, + request_serializer=( + trigger_publication_pb2.StreamTriggersRequest.SerializeToString + ), + response_deserializer=_decode_response, + ) + call: _TriggerStreamCall = callable_() + try: + await call.write(opening_request) + first = await _read_response(call) + if first.WhichOneof("message") != "ready": + raise SfpTriggerPublicationProtocolError( + "Trigger Publication stream did not begin with ready" + ) + try: + validation.validate_stream_open_exchange( + open_message, + first.ready, + ) + except validation.TriggerPublicationValidationError as error: + raise SfpTriggerPublicationProtocolError( + "Trigger Publication ready response failed validation" + ) from error + + while True: + response = await _read_response(call) + if response.WhichOneof("message") != "publication": + raise SfpTriggerPublicationProtocolError( + "Trigger Publication stream returned invalid framing" + ) + acknowledgement = await handler(response.publication) + request = trigger_publication_pb2.StreamTriggersRequest( + acknowledgement=acknowledgement + ) + _validate_request(request) + await call.write(request) + except grpc.aio.AioRpcError as error: + _raise_rpc_error(error.code()) + finally: + call.cancel() + + +def _validate_request( + request: trigger_publication_pb2.StreamTriggersRequest, +) -> None: + """Apply canonical request limits and validation before transmission.""" + if request.ByteSize() > contract.MAX_REQUEST_BYTES: + raise SfpTriggerPublicationLimitError( + "Trigger Publication request limit exceeded" + ) + try: + validation.validate_stream_request(request) + except validation.TriggerPublicationValidationError as error: + raise SfpTriggerPublicationProtocolError( + "Trigger Publication request failed canonical validation" + ) from error + + +async def _read_response( + call: _TriggerStreamCall, +) -> trigger_publication_pb2.StreamTriggersResponse: + """Read and validate one bounded Provider response.""" + decoded = await call.read() + if decoded is grpc.aio.EOF: + raise SfpTriggerPublicationUnavailableError( + "Trigger Publication stream ended" + ) + if not isinstance(decoded, _DecodedTriggerResponse): + raise SfpTriggerPublicationProtocolError( + "Trigger Publication transport returned invalid data" + ) + if decoded.limit_exceeded: + raise SfpTriggerPublicationLimitError( + "Trigger Publication response limit exceeded" + ) + if decoded.malformed or decoded.response is None: + raise SfpTriggerPublicationProtocolError( + "Trigger Publication response protobuf is malformed" + ) + response = decoded.response + try: + validation.validate_stream_response(response) + except validation.TriggerPublicationValidationError as error: + raise SfpTriggerPublicationProtocolError( + "Trigger Publication response failed canonical validation" + ) from error + return response + + +def _decode_response(payload: bytes) -> _DecodedTriggerResponse: + """Reject oversized or malformed bytes before semantic handling.""" + if len(payload) > contract.MAX_RESPONSE_BYTES: + return _DecodedTriggerResponse(limit_exceeded=True) + try: + response = trigger_publication_pb2.StreamTriggersResponse.FromString( + payload + ) + except DecodeError: + return _DecodedTriggerResponse(malformed=True) + return _DecodedTriggerResponse(response=response) + + +def _raise_rpc_error(code: grpc.StatusCode) -> Never: + """Translate stable transport statuses into sanitized Agent errors.""" + if code in { + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.CANCELLED, + grpc.StatusCode.DEADLINE_EXCEEDED, + }: + raise SfpTriggerPublicationUnavailableError( + "Trigger Publication stream is unavailable" + ) + if code is grpc.StatusCode.FAILED_PRECONDITION: + raise SfpTriggerPublicationStaleError( + "Trigger Publication stream binding is stale" + ) + if code is grpc.StatusCode.RESOURCE_EXHAUSTED: + raise SfpTriggerPublicationLimitError( + "Trigger Publication stream limit exceeded" + ) + raise SfpTriggerPublicationProtocolError( + f"Trigger Publication stream failed: {code.name}" + ) diff --git a/src/stateforge_agent/_sfp/_services/trigger_publication/coordinator.py b/src/stateforge_agent/_sfp/_services/trigger_publication/coordinator.py new file mode 100644 index 0000000..ff65db2 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/trigger_publication/coordinator.py @@ -0,0 +1,252 @@ +"""Validate and durably accept Provider trigger publications.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from uuid import UUID + +import grpc +from google.protobuf.json_format import MessageToDict +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError, ValidationError +from stateforge_protocol.sfp import primitives +from stateforge_protocol.sfp.trigger_publication.v1 import ( + trigger_publication_pb2, + validation, +) + +from stateforge_agent._journal import SqliteJournalError +from stateforge_agent._sfp._services.trigger_publication.client import ( + SfpTriggerPublicationProtocolError, + TriggerPublicationClient, + TriggerPublicationConsumer, + TriggerStreamBinding, +) +from stateforge_agent._sfp._services.trigger_publication.store import ( + TriggerAcceptanceContext, + TriggerAcceptanceDecision, + TriggerPublicationStore, +) + + +@dataclass(frozen=True, slots=True) +class TriggerPublicationLease: + """Expose only current session facts required by Trigger Publication.""" + + provider_instance_id: UUID + endpoint_generation: int + session_id: str + manifest_revision: str + availability_generation: int + trigger_schemas: Mapping[str, Mapping[str, object]] + channel: grpc.aio.Channel = field(repr=False, compare=False) + session_token: object = field(repr=False, compare=False) + + +TriggerPublicationCurrentnessCheck = Callable[ + [TriggerPublicationLease], Awaitable[None] +] + + +class TriggerPublicationCoordinator: + """Own stream setup, manifest validation, and durable acknowledgement.""" + + def __init__( + self, + store: TriggerPublicationStore, + *, + maximum_in_flight_publications: int, + client: TriggerPublicationConsumer | None = None, + ) -> None: + """Bind the procedure to its durable responsibility store.""" + if maximum_in_flight_publications <= 0: + raise ValueError("maximum_in_flight_publications must be positive") + self._store = store + self._maximum_in_flight_publications = maximum_in_flight_publications + self._client = client or TriggerPublicationClient() + + async def run( + self, + lease: TriggerPublicationLease, + *, + assert_current: TriggerPublicationCurrentnessCheck, + ) -> None: + """Consume one stream attempt under an exact current-session lease.""" + await assert_current(lease) + + async def handle( + publication: trigger_publication_pb2.TriggerPublication, + ) -> trigger_publication_pb2.TriggerAcknowledgement: + return await self._accept( + lease, + publication, + assert_current=assert_current, + ) + + await self._client.consume( + lease.channel, + TriggerStreamBinding( + provider_instance_id=str(lease.provider_instance_id), + endpoint_generation=lease.endpoint_generation, + session_id=lease.session_id, + manifest_revision=lease.manifest_revision, + maximum_in_flight_publications=( + self._maximum_in_flight_publications + ), + ), + handle, + ) + + async def _accept( # noqa: PLR0911 + self, + lease: TriggerPublicationLease, + publication: trigger_publication_pb2.TriggerPublication, + *, + assert_current: TriggerPublicationCurrentnessCheck, + ) -> trigger_publication_pb2.TriggerAcknowledgement: + """Return a terminal decision only after all required boundaries.""" + await assert_current(lease) + invalid = _canonical_publication_error(publication) + if invalid is not None: + return _rejection( + publication, + trigger_publication_pb2.TRIGGER_REJECTION_REASON_INVALID_PUBLICATION, + retryable=False, + explanation=invalid, + ) + schema = lease.trigger_schemas.get(publication.trigger_identifier) + if schema is None: + return _rejection( + publication, + trigger_publication_pb2.TRIGGER_REJECTION_REASON_TRIGGER_NOT_DECLARED, + retryable=False, + explanation=( + "Trigger is not declared by the accepted manifest." + ), + ) + if not _payload_matches(schema, publication): + return _rejection( + publication, + trigger_publication_pb2.TRIGGER_REJECTION_REASON_PAYLOAD_SCHEMA_VIOLATION, + retryable=False, + explanation="Trigger payload does not match its declaration.", + ) + + try: + result = await self._store.accept( + TriggerAcceptanceContext( + provider_instance_id=lease.provider_instance_id, + endpoint_generation=lease.endpoint_generation, + session_id=lease.session_id, + manifest_revision=lease.manifest_revision, + ), + publication, + ) + except SqliteJournalError: + return _rejection( + publication, + trigger_publication_pb2.TRIGGER_REJECTION_REASON_DELIVERY_RESPONSIBILITY_UNAVAILABLE, + retryable=True, + explanation="Durable delivery responsibility is unavailable.", + ) + + await assert_current(lease) + if result.decision is TriggerAcceptanceDecision.ACCEPTED_NEW: + return _accepted( + publication, + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_NEW, + ) + if result.decision is TriggerAcceptanceDecision.ACCEPTED_DUPLICATE: + return _accepted( + publication, + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_DUPLICATE, + ) + if result.decision is TriggerAcceptanceDecision.IDEMPOTENCY_CONFLICT: + return _rejection( + publication, + trigger_publication_pb2.TRIGGER_REJECTION_REASON_IDEMPOTENCY_CONFLICT, + retryable=False, + explanation=( + "Publication ID was already accepted with different " + "content." + ), + ) + return _rejection( + publication, + trigger_publication_pb2.TRIGGER_REJECTION_REASON_AGENT_CAPACITY_EXHAUSTED, + retryable=True, + explanation="Agent trigger acceptance capacity is exhausted.", + ) + + +def _canonical_publication_error( + publication: trigger_publication_pb2.TriggerPublication, +) -> str | None: + """Return a safe publication-level error or fail invalid correlation.""" + try: + primitives.parse_canonical_uuid4(publication.publication_id) + except primitives.ProtocolScalarError as error: + raise SfpTriggerPublicationProtocolError( + "Trigger publication identity is invalid" + ) from error + try: + validation.validate_trigger_publication(publication) + except validation.TriggerPublicationValidationError: + return "Trigger publication failed canonical validation." + return None + + +def _payload_matches( + schema: Mapping[str, object], + publication: trigger_publication_pb2.TriggerPublication, +) -> bool: + """Validate payload against the exact accepted manifest declaration.""" + payload = MessageToDict(publication.payload) + try: + Draft202012Validator.check_schema(schema) + Draft202012Validator(schema).validate(payload) + except (SchemaError, ValidationError): + return False + return True + + +def _accepted( + publication: trigger_publication_pb2.TriggerPublication, + disposition: ( + trigger_publication_pb2.TriggerAcknowledgementDisposition.ValueType + ), +) -> trigger_publication_pb2.TriggerAcknowledgement: + """Construct and validate an accepted terminal acknowledgement.""" + acknowledgement = trigger_publication_pb2.TriggerAcknowledgement( + publication_id=publication.publication_id, + disposition=disposition, + ) + validation.validate_publication_acknowledgement( + publication, + acknowledgement, + ) + return acknowledgement + + +def _rejection( + publication: trigger_publication_pb2.TriggerPublication, + reason: trigger_publication_pb2.TriggerRejectionReason.ValueType, + *, + retryable: bool, + explanation: str, +) -> trigger_publication_pb2.TriggerAcknowledgement: + """Construct and validate one sanitized publication rejection.""" + acknowledgement = trigger_publication_pb2.TriggerAcknowledgement( + publication_id=publication.publication_id, + disposition=( + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_REJECTED + ), + rejection=trigger_publication_pb2.TriggerRejection( + reason=reason, + retryable=retryable, + explanation=explanation, + ), + ) + validation.validate_trigger_acknowledgement(acknowledgement) + return acknowledgement diff --git a/src/stateforge_agent/_sfp/_services/trigger_publication/store.py b/src/stateforge_agent/_sfp/_services/trigger_publication/store.py new file mode 100644 index 0000000..d9797f4 --- /dev/null +++ b/src/stateforge_agent/_sfp/_services/trigger_publication/store.py @@ -0,0 +1,213 @@ +"""Durably accept Provider trigger publications before acknowledgement.""" + +from __future__ import annotations + +import sqlite3 +from dataclasses import dataclass +from datetime import UTC, datetime +from enum import StrEnum +from uuid import UUID + +from stateforge_protocol.sfp.trigger_publication.v1 import ( + trigger_publication_pb2, + validation, +) + +from stateforge_agent._journal import SqliteJournal + +_SCHEMA_COMPONENT = "trigger_publication" +_SCHEMA_VERSION = 1 +_PENDING_STATE = "pending" + + +class TriggerAcceptanceDecision(StrEnum): + """Describe one durable idempotency decision.""" + + ACCEPTED_NEW = "accepted_new" + ACCEPTED_DUPLICATE = "accepted_duplicate" + IDEMPOTENCY_CONFLICT = "idempotency_conflict" + CAPACITY_EXHAUSTED = "capacity_exhausted" + + +@dataclass(frozen=True, slots=True) +class TriggerAcceptanceResult: + """Return a durable decision without exposing publication content.""" + + decision: TriggerAcceptanceDecision + content_fingerprint: str + pending_publications: int + + +@dataclass(frozen=True, slots=True) +class TriggerAcceptanceContext: + """Identify the current session that delivered one publication.""" + + provider_instance_id: UUID + endpoint_generation: int + session_id: str + manifest_revision: str + + +class TriggerPublicationStore: + """Own durable trigger idempotency and pending-forward responsibility.""" + + def __init__( + self, + journal: SqliteJournal, + *, + maximum_pending_publications: int, + ) -> None: + """Configure a bounded trigger responsibility store.""" + if maximum_pending_publications <= 0: + raise ValueError("maximum_pending_publications must be positive") + self._journal = journal + self._maximum_pending_publications = maximum_pending_publications + + async def initialize(self) -> None: + """Create the first trigger journal schema or verify its version.""" + await self._journal.transaction(self._initialize) + + async def accept( + self, + context: TriggerAcceptanceContext, + publication: trigger_publication_pb2.TriggerPublication, + ) -> TriggerAcceptanceResult: + """Atomically assume responsibility or classify an existing event.""" + fingerprint = validation.compute_publication_content_fingerprint( + publication + ) + serialized = publication.SerializeToString(deterministic=True) + accepted_at = datetime.now(UTC).isoformat() + + def accept_transaction( + connection: sqlite3.Connection, + ) -> TriggerAcceptanceResult: + existing = connection.execute( + """ + SELECT content_fingerprint + FROM trigger_publications + WHERE provider_instance_id = ? AND publication_id = ? + """, + ( + str(context.provider_instance_id), + publication.publication_id, + ), + ).fetchone() + pending = self._pending_count(connection) + if existing is not None: + decision = ( + TriggerAcceptanceDecision.ACCEPTED_DUPLICATE + if existing[0] == fingerprint + else TriggerAcceptanceDecision.IDEMPOTENCY_CONFLICT + ) + return TriggerAcceptanceResult(decision, fingerprint, pending) + if pending >= self._maximum_pending_publications: + return TriggerAcceptanceResult( + TriggerAcceptanceDecision.CAPACITY_EXHAUSTED, + fingerprint, + pending, + ) + + connection.execute( + """ + INSERT INTO trigger_publications ( + provider_instance_id, + publication_id, + content_fingerprint, + publication, + trigger_identifier, + observed_at, + endpoint_generation, + session_id, + manifest_revision, + accepted_at, + forwarding_state + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + str(context.provider_instance_id), + publication.publication_id, + fingerprint, + serialized, + publication.trigger_identifier, + publication.observed_at.ToJsonString(), + context.endpoint_generation, + context.session_id, + context.manifest_revision, + accepted_at, + _PENDING_STATE, + ), + ) + return TriggerAcceptanceResult( + TriggerAcceptanceDecision.ACCEPTED_NEW, + fingerprint, + pending + 1, + ) + + return await self._journal.transaction(accept_transaction) + + async def pending_count(self) -> int: + """Return the number of events awaiting the future SF-AOP boundary.""" + return await self._journal.transaction(self._pending_count) + + @staticmethod + def _initialize(connection: sqlite3.Connection) -> None: + """Create and verify the trigger-owned schema.""" + connection.execute( + """ + CREATE TABLE IF NOT EXISTS journal_schemas ( + component TEXT PRIMARY KEY, + version INTEGER NOT NULL CHECK (version > 0) + ) WITHOUT ROWID + """ + ) + row = connection.execute( + "SELECT version FROM journal_schemas WHERE component = ?", + (_SCHEMA_COMPONENT,), + ).fetchone() + if row is not None and row[0] != _SCHEMA_VERSION: + raise sqlite3.DatabaseError( + "Unsupported Trigger Publication journal schema" + ) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS trigger_publications ( + provider_instance_id TEXT NOT NULL, + publication_id TEXT NOT NULL, + content_fingerprint TEXT NOT NULL, + publication BLOB NOT NULL, + trigger_identifier TEXT NOT NULL, + observed_at TEXT NOT NULL, + endpoint_generation INTEGER NOT NULL + CHECK (endpoint_generation > 0), + session_id TEXT NOT NULL, + manifest_revision TEXT NOT NULL, + accepted_at TEXT NOT NULL, + forwarding_state TEXT NOT NULL + CHECK (forwarding_state = 'pending'), + PRIMARY KEY (provider_instance_id, publication_id) + ) WITHOUT ROWID + """ + ) + connection.execute( + """ + INSERT OR IGNORE INTO journal_schemas (component, version) + VALUES (?, ?) + """, + (_SCHEMA_COMPONENT, _SCHEMA_VERSION), + ) + + @staticmethod + def _pending_count(connection: sqlite3.Connection) -> int: + """Count pending rows without reading their serialized content.""" + row = connection.execute( + """ + SELECT COUNT(*) + FROM trigger_publications + WHERE forwarding_state = ? + """, + (_PENDING_STATE,), + ).fetchone() + if row is None: + raise sqlite3.DatabaseError("Trigger publication count is absent") + return int(row[0]) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 944da74..df293ab 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -25,9 +25,11 @@ health_interval_seconds = 15 reconnect_initial_delay_seconds = 1 reconnect_max_delay_seconds = 30 +trigger_max_in_flight_publications = 32 [journal] backend = "sqlite" +maximum_pending_trigger_publications = 10000 [logging] level = "info" @@ -39,6 +41,8 @@ _SUCCESS_EXIT_CODE = 0 _RUNTIME_FAILURE_EXIT_CODE = 1 _INVALID_INPUT_EXIT_CODE = 2 +_TRIGGER_RECEIVE_WINDOW = 32 +_TRIGGER_PENDING_CAPACITY = 10_000 def test_process_exit_codes_are_stable() -> None: @@ -63,6 +67,14 @@ def test_loads_complete_configuration(tmp_path: Path) -> None: "https://control.stateforge.example" ) assert loaded.config.journal.backend == "sqlite" + assert ( + loaded.config.providers.trigger_max_in_flight_publications + == _TRIGGER_RECEIVE_WINDOW + ) + assert ( + loaded.config.journal.maximum_pending_trigger_publications + == _TRIGGER_PENDING_CAPACITY + ) assert loaded.sources == (config_path,) assert loaded.fingerprint.startswith("sha256:") diff --git a/tests/test_daemon_provider_events.py b/tests/test_daemon_provider_events.py index fa16722..c2e16ff 100644 --- a/tests/test_daemon_provider_events.py +++ b/tests/test_daemon_provider_events.py @@ -78,9 +78,11 @@ def _configuration(directory: Path) -> AgentConfiguration: health_interval_seconds=10, reconnect_initial_delay_seconds=1, reconnect_max_delay_seconds=30, + trigger_max_in_flight_publications=32, ), journal=JournalConfiguration( backend="sqlite", + maximum_pending_trigger_publications=10000, ), logging=LoggingConfiguration( level="info", diff --git a/tests/test_rpm_automation.py b/tests/test_rpm_automation.py index df59d45..3fe3890 100644 --- a/tests/test_rpm_automation.py +++ b/tests/test_rpm_automation.py @@ -419,6 +419,7 @@ def test_action_library_has_expected_domain_boundaries() -> None: "agent/assert-availability", "agent/assert-capability-boundary", "agent/assert-capability-assessment", + "agent/assert-trigger-publication", "agent/collect-evidence", "agent/install", "agent/start", @@ -438,6 +439,7 @@ def test_action_library_has_expected_domain_boundaries() -> None: "rpm/test-installed/lifecycle/multiple-providers", "rpm/test-installed/setup", "rpm/test-installed/smoke/manifest-exchange", + "rpm/test-installed/triggers/publication", } observed = { str(path.parent.relative_to(ACTIONS)) @@ -493,6 +495,7 @@ def test_each_installed_scenario_is_an_independent_action() -> None: "adversarial", "capability", "destructive", + "triggers", } for scenario in scenarios: content = read(scenario) @@ -613,3 +616,43 @@ def test_installed_capability_scenario_uses_real_artifacts_and_processes() -> ( assert "session_id" in boundary_assertion assert "endpoint_generation" in boundary_assertion assert "manifest_revision" in boundary_assertion + + +def test_installed_trigger_scenario_uses_public_api_and_journal() -> None: + """Keep A4 evidence on exact packages and the durable Agent boundary.""" + workflow = read(WORKFLOWS / "rpm-installed-test.yml") + scenario = read( + ACTIONS + / "rpm" + / "test-installed" + / "triggers" + / "publication" + / "action.yml" + ) + assertion_action = read( + ACTIONS / "agent" / "assert-trigger-publication" / "action.yml" + ) + assertion = read( + ACTIONS / "agent" / "assert-trigger-publication" / "assert.py" + ) + fixture = read( + ACTIONS / "provider" / "create-fixture" / "fixtures" / "manifest.py" + ) + + assert "trigger-publication" in workflow + assert "./.github/actions/rpm/test-installed/setup" in scenario + assert "./.github/actions/provider/start" in scenario + assert "./.github/actions/agent/assert-trigger-publication" in scenario + assert "systemctl stop stateforge-agent.service" in scenario + assert "./.github/actions/agent/start" in scenario + assert "before-agent-restart" in scenario + assert "after-agent-restart" in scenario + assert "stateforge.test/runtime.started@1" in scenario + assert "stateforge.test/runtime.refreshed@1" in scenario + assert "@trigger(" in fixture + assert "publish_trigger(" in fixture + assert "/usr/bin/python3" in assertion_action + assert "mode=ro" in assertion + assert "validate_trigger_publication" in assertion + assert "compute_publication_content_fingerprint" in assertion + assert 'row["forwarding_state"] != "pending"' in assertion diff --git a/tests/test_sfp_session.py b/tests/test_sfp_session.py index 110f56c..a35a779 100644 --- a/tests/test_sfp_session.py +++ b/tests/test_sfp_session.py @@ -1,11 +1,13 @@ """Integration tests for Agent-side SFP session establishment.""" import asyncio +import json import shutil import tempfile from collections.abc import Iterator from dataclasses import dataclass from pathlib import Path +from typing import cast from uuid import UUID, uuid4 import grpc @@ -38,6 +40,13 @@ ManifestCacheKey, ManifestDeclaration, ) +from stateforge_agent._sfp._services.trigger_publication.client import ( + SfpTriggerPublicationUnavailableError, +) +from stateforge_agent._sfp._services.trigger_publication.coordinator import ( + TriggerPublicationCoordinator, + TriggerPublicationLease, +) _PROVIDER_INSTANCE_ID = UUID("7f7db7d8-68b4-4f71-9769-065b5c4f941c") _DIFFERENT_PROVIDER_INSTANCE_ID = "00000000-0000-4000-8000-000000000000" @@ -63,6 +72,34 @@ def _manifest_for( ) +def _trigger_manifest_for( + provider_instance_id: UUID = _PROVIDER_INSTANCE_ID, +) -> ManifestDeclaration: + """Return a validated-shape declaration containing one trigger.""" + document = { + "triggers": { + "example.systemd/unit.failed@1": { + "payloadSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"unit": {"type": "string"}}, + "required": ["unit"], + "additionalProperties": False, + } + } + } + } + return ManifestDeclaration( + key=ManifestCacheKey( + provider_instance_id=provider_instance_id, + sfp_version=_SFP_VERSION, + schema_version=1, + revision=_MANIFEST_REVISION, + ), + document=json.dumps(document).encode(), + ) + + class _StaticManifestClient: async def retrieve( self, @@ -76,6 +113,56 @@ async def retrieve( return _manifest_for(descriptor.provider_instance_id) +class _TriggerManifestClient(_StaticManifestClient): + async def retrieve( + self, + channel, + descriptor: ProviderEndpointDescriptor, + *, + sfp_version: SfpVersion, + session_id: str, + ) -> ManifestDeclaration: + del channel, sfp_version, session_id + return _trigger_manifest_for(descriptor.provider_instance_id) + + +class _RecordingTriggerCoordinator: + """Record trigger tasks and wait for session-owned cancellation.""" + + def __init__(self) -> None: + self.leases: list[TriggerPublicationLease] = [] + self.cancelled = 0 + self.changed = asyncio.Event() + + async def run(self, lease, *, assert_current) -> None: + await assert_current(lease) + self.leases.append(lease) + self.changed.set() + try: + await asyncio.Future() + except asyncio.CancelledError: + self.cancelled += 1 + self.changed.set() + raise + + +class _RetryingTriggerCoordinator(_RecordingTriggerCoordinator): + """Fail one stream attempt before retaining the replacement attempt.""" + + async def run(self, lease, *, assert_current) -> None: + await assert_current(lease) + self.leases.append(lease) + self.changed.set() + if len(self.leases) == 1: + raise SfpTriggerPublicationUnavailableError("injected disconnect") + try: + await asyncio.Future() + except asyncio.CancelledError: + self.cancelled += 1 + self.changed.set() + raise + + class _BlockingManifestClient(_StaticManifestClient): def __init__(self) -> None: self.started = asyncio.Event() @@ -428,6 +515,7 @@ def _manager( manifest_client=None, reconnect_initial_delay_seconds: float = 0.05, reconnect_max_delay_seconds: float = 0.2, + trigger_publication: _RecordingTriggerCoordinator | None = None, ) -> ProviderSessionManager: return ProviderSessionManager( agent_runtime_id=uuid4(), @@ -439,6 +527,11 @@ def _manager( reconnect_max_delay_seconds=reconnect_max_delay_seconds, descriptor_loader=descriptor_loader, manifest_client=manifest_client or _StaticManifestClient(), + trigger_publication=( + cast(TriggerPublicationCoordinator, trigger_publication) + if trigger_publication is not None + else None + ), ) @@ -469,6 +562,32 @@ async def _wait_for_availability( await asyncio.sleep(0) +async def _wait_for_trigger_runs( + coordinator: _RecordingTriggerCoordinator, + expected: int, +) -> None: + """Wait for a session lifecycle transition to start trigger work.""" + async with asyncio.timeout(3): + while len(coordinator.leases) < expected: + coordinator.changed.clear() + if len(coordinator.leases) >= expected: + return + await coordinator.changed.wait() + + +async def _wait_for_trigger_cancellations( + coordinator: _RecordingTriggerCoordinator, + expected: int, +) -> None: + """Wait for session fencing to cancel trigger work.""" + async with asyncio.timeout(3): + while coordinator.cancelled < expected: + coordinator.changed.clear() + if coordinator.cancelled >= expected: + return + await coordinator.changed.wait() + + def test_rejects_invalid_session_manager_configuration() -> None: with pytest.raises(ValueError, match="must not be empty"): ProviderSessionManager( @@ -626,6 +745,95 @@ async def test_health_watch_gates_new_operation_eligibility( await server.stop() +@pytest.mark.asyncio +async def test_trigger_stream_follows_exact_health_and_session_lease( + socket_directory: Path, +) -> None: + coordinator = _RecordingTriggerCoordinator() + server = _BootstrapServer( + socket_directory, + endpoint_generation=1, + health_status=health_pb2.HealthCheckResponse.SERVING, + ) + manager = _manager( + manifest_client=_TriggerManifestClient(), + trigger_publication=coordinator, + ) + await server.start() + try: + session = await manager.establish(server.descriptor) + await _wait_for_availability( + manager, + ProviderAvailabilityState.AVAILABLE, + ) + await _wait_for_trigger_runs(coordinator, 1) + first = coordinator.leases[0] + assert first.session_token is session + assert first.endpoint_generation == session.endpoint_generation + assert first.manifest_revision == session.manifest.key.revision + + server.health_servicer.publish( + health_pb2.HealthCheckResponse.NOT_SERVING + ) + await _wait_for_availability( + manager, + ProviderAvailabilityState.NOT_SERVING, + ) + await _wait_for_trigger_cancellations(coordinator, 1) + + server.health_servicer.publish(health_pb2.HealthCheckResponse.SERVING) + await _wait_for_availability( + manager, + ProviderAvailabilityState.AVAILABLE, + ) + await _wait_for_trigger_runs(coordinator, 2) + assert coordinator.leases[1].availability_generation != ( + first.availability_generation + ) + + await manager.remove(_PROVIDER_INSTANCE_ID) + await _wait_for_trigger_cancellations(coordinator, 2) + finally: + await manager.close() + await server.stop() + + +@pytest.mark.asyncio +async def test_trigger_stream_retries_transport_loss_and_stops_on_shutdown( + socket_directory: Path, +) -> None: + coordinator = _RetryingTriggerCoordinator() + server = _BootstrapServer( + socket_directory, + endpoint_generation=1, + health_status=health_pb2.HealthCheckResponse.SERVING, + ) + manager = _manager( + manifest_client=_TriggerManifestClient(), + trigger_publication=coordinator, + reconnect_initial_delay_seconds=0.01, + reconnect_max_delay_seconds=0.01, + ) + await server.start() + try: + await manager.establish(server.descriptor) + await _wait_for_availability( + manager, + ProviderAvailabilityState.AVAILABLE, + ) + await _wait_for_trigger_runs(coordinator, 2) + + first, retried = coordinator.leases + assert retried.session_token is first.session_token + assert retried.availability_generation == first.availability_generation + + await manager.close() + await _wait_for_trigger_cancellations(coordinator, 1) + finally: + await manager.close() + await server.stop() + + @pytest.mark.asyncio async def test_ended_health_watch_fails_closed_and_retries( socket_directory: Path, diff --git a/tests/test_trigger_publication.py b/tests/test_trigger_publication.py new file mode 100644 index 0000000..12e7483 --- /dev/null +++ b/tests/test_trigger_publication.py @@ -0,0 +1,583 @@ +"""Agent-side Trigger Publication validation and acknowledgement tests.""" + +import asyncio +import logging +from collections.abc import AsyncIterator, Awaitable, Callable +from dataclasses import replace +from pathlib import Path +from typing import cast +from uuid import uuid4 + +import grpc +import pytest +from google.protobuf.json_format import ParseDict +from google.protobuf.timestamp_pb2 import Timestamp +from stateforge_protocol.sfp.trigger_publication.v1 import ( + contract, + trigger_publication_pb2, + trigger_publication_pb2_grpc, +) + +from stateforge_agent._journal import SqliteJournal +from stateforge_agent._sfp._services.trigger_publication.client import ( + SfpTriggerPublicationLimitError, + SfpTriggerPublicationStaleError, + SfpTriggerPublicationUnavailableError, + TriggerPublicationClient, + TriggerPublicationHandler, + TriggerStreamBinding, +) +from stateforge_agent._sfp._services.trigger_publication.coordinator import ( + TriggerPublicationCoordinator, + TriggerPublicationLease, +) +from stateforge_agent._sfp._services.trigger_publication.store import ( + TriggerPublicationStore, +) + +_TRIGGER_IDENTIFIER = "example.systemd/unit.failed@1" +_STALE_BEFORE_DURABLE_COMMIT = 2 +_STALE_AFTER_DURABLE_COMMIT = 3 +_TRIGGER_SCHEMA = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"unit": {"type": "string"}}, + "required": ["unit"], + "additionalProperties": False, +} + + +class _TriggerPeer( + trigger_publication_pb2_grpc.TriggerPublicationServiceServicer +): + """Serve one publication through a real bidirectional gRPC stream.""" + + def __init__( + self, + publication: trigger_publication_pb2.TriggerPublication, + ) -> None: + self.publication = publication + self.requests: list[trigger_publication_pb2.StreamTriggersRequest] = [] + + async def StreamTriggers( # noqa: N802 + self, + request_iterator: AsyncIterator[ + trigger_publication_pb2.StreamTriggersRequest + ], + context, + ) -> AsyncIterator[trigger_publication_pb2.StreamTriggersResponse]: + del context + opening = await anext(request_iterator) + self.requests.append(opening) + binding = opening.open.session_binding + yield trigger_publication_pb2.StreamTriggersResponse( + ready=trigger_publication_pb2.TriggerStreamReady( + provider_instance_id=binding.expected_provider_instance_id, + endpoint_generation=binding.expected_endpoint_generation, + session_id=binding.expected_session_id, + manifest_revision=binding.expected_manifest_revision, + maximum_in_flight_publications=( + opening.open.requested_max_in_flight_publications + ), + ) + ) + yield trigger_publication_pb2.StreamTriggersResponse( + publication=self.publication + ) + self.requests.append(await anext(request_iterator)) + + +async def _start_peer( + peer: _TriggerPeer, +) -> tuple[grpc.aio.Server, grpc.aio.Channel]: + server = grpc.aio.server() + trigger_publication_pb2_grpc.add_TriggerPublicationServiceServicer_to_server( + peer, + server, + ) + port = server.add_insecure_port("127.0.0.1:0") + await server.start() + channel = grpc.aio.insecure_channel(f"127.0.0.1:{port}") + await channel.channel_ready() + return server, channel + + +class _ScriptedConsumer: + """Deliver scripted publications and retain their acknowledgements.""" + + def __init__( + self, + publications: list[trigger_publication_pb2.TriggerPublication], + ) -> None: + self.publications = publications + self.bindings: list[TriggerStreamBinding] = [] + self.acknowledgements: list[ + trigger_publication_pb2.TriggerAcknowledgement + ] = [] + + async def consume( + self, + channel: grpc.aio.Channel, + binding: TriggerStreamBinding, + handler: TriggerPublicationHandler, + ) -> None: + del channel + self.bindings.append(binding) + for publication in self.publications: + self.acknowledgements.append(await handler(publication)) + raise SfpTriggerPublicationStaleError("script complete") + + +def _publication( + *, + publication_id: str | None = None, + trigger_identifier: str = _TRIGGER_IDENTIFIER, + payload: dict[str, object] | None = None, +) -> trigger_publication_pb2.TriggerPublication: + payload_value = {"unit": "nginx.service"} if payload is None else payload + message = trigger_publication_pb2.TriggerPublication() + ParseDict(payload_value, message.payload) + return trigger_publication_pb2.TriggerPublication( + publication_id=publication_id or str(uuid4()), + trigger_identifier=trigger_identifier, + observed_at=Timestamp(seconds=1), + payload=message.payload, + ) + + +def _lease() -> TriggerPublicationLease: + return TriggerPublicationLease( + provider_instance_id=uuid4(), + endpoint_generation=1, + session_id="session-1", + manifest_revision="sha256:" + ("a" * 64), + availability_generation=2, + trigger_schemas={_TRIGGER_IDENTIFIER: _TRIGGER_SCHEMA}, + channel=cast(grpc.aio.Channel, object()), + session_token=object(), + ) + + +async def _coordinator( + path: Path, + consumer: _ScriptedConsumer, + *, + capacity: int = 10, +) -> tuple[ + SqliteJournal, + TriggerPublicationStore, + TriggerPublicationCoordinator, +]: + journal = SqliteJournal(path) + await journal.open() + store = TriggerPublicationStore( + journal, + maximum_pending_publications=capacity, + ) + await store.initialize() + return ( + journal, + store, + TriggerPublicationCoordinator( + store, + maximum_in_flight_publications=8, + client=consumer, + ), + ) + + +async def _current(lease: TriggerPublicationLease) -> None: + del lease + + +async def _run_script( + coordinator: TriggerPublicationCoordinator, + lease: TriggerPublicationLease, + *, + current: Callable[[TriggerPublicationLease], Awaitable[None]] = _current, +) -> None: + with pytest.raises( + SfpTriggerPublicationStaleError, match="script complete" + ): + await coordinator.run(lease, assert_current=current) + + +@pytest.mark.asyncio +async def test_client_opens_real_stream_and_returns_durable_ack() -> None: + publication = _publication() + peer = _TriggerPeer(publication) + server, channel = await _start_peer(peer) + lease = _lease() + + async def accept( + item: trigger_publication_pb2.TriggerPublication, + ) -> trigger_publication_pb2.TriggerAcknowledgement: + assert item == publication + return trigger_publication_pb2.TriggerAcknowledgement( + publication_id=item.publication_id, + disposition=( + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_NEW + ), + ) + + try: + async with asyncio.timeout(3): + with pytest.raises( + SfpTriggerPublicationUnavailableError, + match="ended", + ): + await TriggerPublicationClient().consume( + channel, + TriggerStreamBinding( + provider_instance_id=str(lease.provider_instance_id), + endpoint_generation=lease.endpoint_generation, + session_id=lease.session_id, + manifest_revision=lease.manifest_revision, + maximum_in_flight_publications=8, + ), + accept, + ) + assert peer.requests[0].WhichOneof("message") == "open" + acknowledgement = peer.requests[1].acknowledgement + assert acknowledgement.publication_id == publication.publication_id + assert acknowledgement.disposition == ( + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_NEW + ) + finally: + await channel.close() + await server.stop(None) + + +@pytest.mark.asyncio +async def test_accepts_new_and_duplicate_publications(tmp_path: Path) -> None: + publication = _publication() + consumer = _ScriptedConsumer([publication, publication]) + journal, _, coordinator = await _coordinator( + tmp_path / "agent.db", + consumer, + ) + + await _run_script(coordinator, _lease()) + + assert [item.disposition for item in consumer.acknowledgements] == [ + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_NEW, + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_DUPLICATE, + ] + await journal.close() + + +@pytest.mark.asyncio +async def test_rejects_undeclared_and_schema_invalid_publications( + tmp_path: Path, +) -> None: + consumer = _ScriptedConsumer( + [ + _publication(trigger_identifier="example.other/event@1"), + _publication(payload={"unit": 5}), + ] + ) + journal, _, coordinator = await _coordinator( + tmp_path / "agent.db", + consumer, + ) + + await _run_script(coordinator, _lease()) + + assert [item.rejection.reason for item in consumer.acknowledgements] == [ + trigger_publication_pb2.TRIGGER_REJECTION_REASON_TRIGGER_NOT_DECLARED, + trigger_publication_pb2.TRIGGER_REJECTION_REASON_PAYLOAD_SCHEMA_VIOLATION, + ] + await journal.close() + + +@pytest.mark.asyncio +async def test_rejects_conflicting_identity_and_exhausted_capacity( + tmp_path: Path, +) -> None: + publication_id = str(uuid4()) + consumer = _ScriptedConsumer( + [ + _publication(publication_id=publication_id), + _publication( + publication_id=publication_id, + payload={"unit": "postgresql.service"}, + ), + _publication(), + ] + ) + journal, _, coordinator = await _coordinator( + tmp_path / "agent.db", + consumer, + capacity=1, + ) + + await _run_script(coordinator, _lease()) + + acknowledgements = consumer.acknowledgements + assert acknowledgements[0].disposition == ( + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_NEW + ) + assert acknowledgements[1].rejection.reason == ( + trigger_publication_pb2.TRIGGER_REJECTION_REASON_IDEMPOTENCY_CONFLICT + ) + assert acknowledgements[2].rejection.reason == ( + trigger_publication_pb2.TRIGGER_REJECTION_REASON_AGENT_CAPACITY_EXHAUSTED + ) + assert acknowledgements[2].rejection.retryable + await journal.close() + + +@pytest.mark.asyncio +async def test_commit_before_lost_ack_replays_as_duplicate( + tmp_path: Path, +) -> None: + publication = _publication() + path = tmp_path / "agent.db" + interrupted = _ScriptedConsumer([publication]) + journal, store, coordinator = await _coordinator(path, interrupted) + calls = 0 + lease = _lease() + + async def become_stale(lease: TriggerPublicationLease) -> None: + nonlocal calls + del lease + calls += 1 + if calls == _STALE_AFTER_DURABLE_COMMIT: + raise SfpTriggerPublicationStaleError("availability changed") + + with pytest.raises( + SfpTriggerPublicationStaleError, + match="availability changed", + ): + await coordinator.run(lease, assert_current=become_stale) + assert interrupted.acknowledgements == [] + + replay = _ScriptedConsumer([publication]) + replay_coordinator = TriggerPublicationCoordinator( + store, + maximum_in_flight_publications=8, + client=replay, + ) + await _run_script(replay_coordinator, lease) + assert replay.acknowledgements[0].disposition == ( + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_DUPLICATE + ) + await journal.close() + + +@pytest.mark.asyncio +async def test_rejects_invalid_publication_without_assuming_responsibility( + tmp_path: Path, +) -> None: + publication = _publication() + publication.ClearField("observed_at") + consumer = _ScriptedConsumer([publication]) + journal, store, coordinator = await _coordinator( + tmp_path / "agent.db", + consumer, + ) + + await _run_script(coordinator, _lease()) + + acknowledgement = consumer.acknowledgements[0] + assert acknowledgement.disposition == ( + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_REJECTED + ) + assert acknowledgement.rejection.reason == ( + trigger_publication_pb2.TRIGGER_REJECTION_REASON_INVALID_PUBLICATION + ) + assert not acknowledgement.rejection.retryable + assert await store.pending_count() == 0 + await journal.close() + + +@pytest.mark.asyncio +async def test_storage_failure_before_commit_is_retryable_and_not_accepted( + tmp_path: Path, +) -> None: + publication = _publication() + consumer = _ScriptedConsumer([publication]) + path = tmp_path / "agent.db" + journal, _, coordinator = await _coordinator(path, consumer) + await journal.close() + + await _run_script(coordinator, _lease()) + + acknowledgement = consumer.acknowledgements[0] + assert acknowledgement.disposition == ( + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_REJECTED + ) + assert acknowledgement.rejection.reason == ( + trigger_publication_pb2.TRIGGER_REJECTION_REASON_DELIVERY_RESPONSIBILITY_UNAVAILABLE + ) + assert acknowledgement.rejection.retryable + restarted_journal = SqliteJournal(path) + await restarted_journal.open() + restarted_store = TriggerPublicationStore( + restarted_journal, + maximum_pending_publications=10, + ) + await restarted_store.initialize() + assert await restarted_store.pending_count() == 0 + await restarted_journal.close() + + +@pytest.mark.asyncio +async def test_stale_lease_before_commit_assumes_no_responsibility( + tmp_path: Path, +) -> None: + publication = _publication() + consumer = _ScriptedConsumer([publication]) + journal, store, coordinator = await _coordinator( + tmp_path / "agent.db", + consumer, + ) + checks = 0 + + async def become_stale(lease: TriggerPublicationLease) -> None: + nonlocal checks + del lease + checks += 1 + if checks == _STALE_BEFORE_DURABLE_COMMIT: + raise SfpTriggerPublicationStaleError("session replaced") + + with pytest.raises( + SfpTriggerPublicationStaleError, + match="session replaced", + ): + await coordinator.run(_lease(), assert_current=become_stale) + + assert consumer.acknowledgements == [] + assert await store.pending_count() == 0 + await journal.close() + + +@pytest.mark.asyncio +async def test_rejection_does_not_expose_sensitive_payload( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + sensitive_value = "never-emit-this-token" + consumer = _ScriptedConsumer( + [ + _publication( + payload={"unit": "nginx.service", "token": sensitive_value} + ) + ] + ) + journal, _, coordinator = await _coordinator( + tmp_path / "agent.db", + consumer, + ) + + with caplog.at_level(logging.DEBUG): + await _run_script(coordinator, _lease()) + + acknowledgement = consumer.acknowledgements[0] + assert acknowledgement.rejection.reason == ( + trigger_publication_pb2.TRIGGER_REJECTION_REASON_PAYLOAD_SCHEMA_VIOLATION + ) + assert sensitive_value not in str(acknowledgement) + assert sensitive_value not in caplog.text + await journal.close() + + +@pytest.mark.asyncio +async def test_source_and_target_payload_fields_remain_opaque( + tmp_path: Path, +) -> None: + identifier = "example.application/change.detected@1" + publication = _publication( + trigger_identifier=identifier, + payload={"source": "scheduler", "target": "database-primary"}, + ) + consumer = _ScriptedConsumer([publication]) + journal, store, coordinator = await _coordinator( + tmp_path / "agent.db", + consumer, + ) + lease = replace( + _lease(), + trigger_schemas={ + identifier: { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + }, + "required": ["source", "target"], + "additionalProperties": False, + } + }, + ) + + await _run_script(coordinator, lease) + + assert consumer.acknowledgements[0].disposition == ( + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_NEW + ) + assert await store.pending_count() == 1 + await journal.close() + + +@pytest.mark.asyncio +async def test_client_rejects_request_over_canonical_limit() -> None: + lease = _lease() + binding = TriggerStreamBinding( + provider_instance_id=str(lease.provider_instance_id), + endpoint_generation=lease.endpoint_generation, + session_id="x" * contract.MAX_REQUEST_BYTES, + manifest_revision=lease.manifest_revision, + maximum_in_flight_publications=8, + ) + + async def unreachable_handler( + publication: trigger_publication_pb2.TriggerPublication, + ) -> trigger_publication_pb2.TriggerAcknowledgement: + raise AssertionError(publication) + + with pytest.raises( + SfpTriggerPublicationLimitError, + match="request limit exceeded", + ): + await TriggerPublicationClient().consume( + cast(grpc.aio.Channel, object()), + binding, + unreachable_handler, + ) + + +@pytest.mark.asyncio +async def test_client_rejects_response_over_canonical_limit() -> None: + publication = _publication( + payload={"unit": "x" * contract.MAX_RESPONSE_BYTES} + ) + peer = _TriggerPeer(publication) + server, channel = await _start_peer(peer) + lease = _lease() + + async def unreachable_handler( + item: trigger_publication_pb2.TriggerPublication, + ) -> trigger_publication_pb2.TriggerAcknowledgement: + raise AssertionError(item) + + try: + with pytest.raises( + SfpTriggerPublicationLimitError, + match="response limit exceeded", + ): + await TriggerPublicationClient().consume( + channel, + TriggerStreamBinding( + provider_instance_id=str(lease.provider_instance_id), + endpoint_generation=lease.endpoint_generation, + session_id=lease.session_id, + manifest_revision=lease.manifest_revision, + maximum_in_flight_publications=8, + ), + unreachable_handler, + ) + finally: + await channel.close() + await server.stop(None) diff --git a/tests/test_trigger_publication_store.py b/tests/test_trigger_publication_store.py new file mode 100644 index 0000000..1d427fd --- /dev/null +++ b/tests/test_trigger_publication_store.py @@ -0,0 +1,164 @@ +"""Tests for durable Agent-side Trigger Publication acceptance.""" + +from pathlib import Path +from uuid import UUID, uuid4 + +import pytest +from google.protobuf.json_format import ParseDict +from google.protobuf.timestamp_pb2 import Timestamp +from stateforge_protocol.sfp.trigger_publication.v1 import ( + trigger_publication_pb2, +) + +from stateforge_agent._journal import SqliteJournal, SqliteJournalError +from stateforge_agent._sfp._services.trigger_publication import ( + TriggerAcceptanceContext, + TriggerAcceptanceDecision, + TriggerPublicationStore, +) + +_TRIGGER_IDENTIFIER = "example.systemd/unit.failed@1" +_TWO_PUBLICATIONS = 2 + + +def _context( + provider_instance_id: UUID | None = None, +) -> TriggerAcceptanceContext: + return TriggerAcceptanceContext( + provider_instance_id=provider_instance_id or uuid4(), + endpoint_generation=1, + session_id="session-1", + manifest_revision="sha256:" + ("a" * 64), + ) + + +def _publication( + publication_id: UUID, + *, + unit: str = "nginx.service", +) -> trigger_publication_pb2.TriggerPublication: + payload = ParseDict( + {"unit": unit, "state": "failed"}, + trigger_publication_pb2.TriggerPublication().payload, + ) + return trigger_publication_pb2.TriggerPublication( + publication_id=str(publication_id), + trigger_identifier=_TRIGGER_IDENTIFIER, + observed_at=Timestamp(seconds=1), + payload=payload, + ) + + +async def _store( + path: Path, + *, + capacity: int = 10, +) -> tuple[SqliteJournal, TriggerPublicationStore]: + journal = SqliteJournal(path) + await journal.open() + store = TriggerPublicationStore( + journal, + maximum_pending_publications=capacity, + ) + await store.initialize() + return journal, store + + +@pytest.mark.asyncio +async def test_accepts_new_publication_and_identical_replay_once( + tmp_path: Path, +) -> None: + journal, store = await _store(tmp_path / "agent.db") + context = _context() + publication = _publication(uuid4()) + + first = await store.accept(context, publication) + replay = await store.accept(context, publication) + + assert first.decision is TriggerAcceptanceDecision.ACCEPTED_NEW + assert replay.decision is TriggerAcceptanceDecision.ACCEPTED_DUPLICATE + assert replay.content_fingerprint == first.content_fingerprint + assert await store.pending_count() == 1 + await journal.close() + + +@pytest.mark.asyncio +async def test_detects_conflicting_publication_id_reuse( + tmp_path: Path, +) -> None: + journal, store = await _store(tmp_path / "agent.db") + context = _context() + publication_id = uuid4() + + first = await store.accept(context, _publication(publication_id)) + conflict = await store.accept( + context, + _publication(publication_id, unit="postgresql.service"), + ) + + assert first.decision is TriggerAcceptanceDecision.ACCEPTED_NEW + assert conflict.decision is TriggerAcceptanceDecision.IDEMPOTENCY_CONFLICT + assert await store.pending_count() == 1 + await journal.close() + + +@pytest.mark.asyncio +async def test_persists_acceptance_across_agent_restart( + tmp_path: Path, +) -> None: + path = tmp_path / "agent.db" + context = _context() + publication = _publication(uuid4()) + journal, store = await _store(path) + await store.accept(context, publication) + await journal.close() + + restarted_journal, restarted_store = await _store(path) + replay = await restarted_store.accept(context, publication) + + assert replay.decision is TriggerAcceptanceDecision.ACCEPTED_DUPLICATE + assert await restarted_store.pending_count() == 1 + await restarted_journal.close() + + +@pytest.mark.asyncio +async def test_scopes_idempotency_to_provider_installation( + tmp_path: Path, +) -> None: + journal, store = await _store(tmp_path / "agent.db") + publication = _publication(uuid4()) + + first = await store.accept(_context(), publication) + other_provider = await store.accept(_context(), publication) + + assert first.decision is TriggerAcceptanceDecision.ACCEPTED_NEW + assert other_provider.decision is TriggerAcceptanceDecision.ACCEPTED_NEW + assert await store.pending_count() == _TWO_PUBLICATIONS + await journal.close() + + +@pytest.mark.asyncio +async def test_capacity_rejects_only_new_responsibility( + tmp_path: Path, +) -> None: + journal, store = await _store(tmp_path / "agent.db", capacity=1) + context = _context() + accepted = _publication(uuid4()) + await store.accept(context, accepted) + + duplicate = await store.accept(context, accepted) + capacity = await store.accept(context, _publication(uuid4())) + + assert duplicate.decision is TriggerAcceptanceDecision.ACCEPTED_DUPLICATE + assert capacity.decision is TriggerAcceptanceDecision.CAPACITY_EXHAUSTED + assert await store.pending_count() == 1 + await journal.close() + + +@pytest.mark.asyncio +async def test_closed_journal_rejects_new_transactions(tmp_path: Path) -> None: + journal, store = await _store(tmp_path / "agent.db") + await journal.close() + + with pytest.raises(SqliteJournalError, match="not open"): + await store.pending_count() diff --git a/uv.lock b/uv.lock index d6c451a..76246f5 100644 --- a/uv.lock +++ b/uv.lock @@ -1447,6 +1447,7 @@ source = { editable = "." } dependencies = [ { name = "grpcio" }, { name = "grpcio-health-checking" }, + { name = "jsonschema" }, { name = "stateforge-protocol" }, { name = "watchfiles" }, ] @@ -1461,6 +1462,7 @@ dev = [ { name = "ruff" }, { name = "types-grpcio" }, { name = "types-grpcio-health-checking" }, + { name = "types-jsonschema" }, { name = "types-protobuf" }, ] docs = [ @@ -1474,7 +1476,8 @@ docs = [ requires-dist = [ { name = "grpcio", specifier = ">=1.83" }, { name = "grpcio-health-checking", specifier = ">=1.83" }, - { name = "stateforge-protocol", git = "https://github.com/stateforge-org/stateforge-protocol.git?rev=a4c077a5b86757b1120b229c048aa0a4cbb9f147" }, + { name = "jsonschema", specifier = ">=4.25,<5" }, + { name = "stateforge-protocol", git = "https://github.com/stateforge-org/stateforge-protocol.git?rev=e4d67e8fd7abbb077e09c41664e6f2945760037e" }, { name = "watchfiles", specifier = ">=1.2,<2" }, ] @@ -1488,6 +1491,7 @@ dev = [ { name = "ruff", specifier = ">=0.12" }, { name = "types-grpcio" }, { name = "types-grpcio-health-checking" }, + { name = "types-jsonschema" }, { name = "types-protobuf" }, ] docs = [ @@ -1500,7 +1504,7 @@ docs = [ [[package]] name = "stateforge-protocol" version = "0.1.0" -source = { git = "https://github.com/stateforge-org/stateforge-protocol.git?rev=a4c077a5b86757b1120b229c048aa0a4cbb9f147#a4c077a5b86757b1120b229c048aa0a4cbb9f147" } +source = { git = "https://github.com/stateforge-org/stateforge-protocol.git?rev=e4d67e8fd7abbb077e09c41664e6f2945760037e#e4d67e8fd7abbb077e09c41664e6f2945760037e" } dependencies = [ { name = "grpcio" }, { name = "grpcio-health-checking" }, @@ -1532,6 +1536,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/de/eca76e380f7e3b93bd7032847e4e9d43904e604b5782c83be9cf733be962/types_grpcio_health_checking-1.0.0.20261021-py3-none-any.whl", hash = "sha256:bac5319aa52e2012f745dba9c125f77326e60db14d60e15cff268dc8759387f5", size = 9828, upload-time = "2026-10-21T06:42:10.574Z" }, ] +[[package]] +name = "types-jsonschema" +version = "4.26.0.20261006" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/d2/1f742605f5a6d39f993134885b8de41c98af606871d3ebddaf3e776bd1eb/types_jsonschema-4.26.0.20261006.tar.gz", hash = "sha256:3eb7db61b6819d40addfdaac7173e749071a7d4a9a4394f0c844598ec84b2500", size = 16843, upload-time = "2026-10-06T08:16:07.318Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/a0/4f2e3c0dc3d5cad958006f2e0307065cc8fe4fbf0393ff0b69d55ee9bbe5/types_jsonschema-4.26.0.20261006-py3-none-any.whl", hash = "sha256:29301f4e65e3928540cdf5e23ad716e38bb0c0b416a48dada213edd3d70206ec", size = 16113, upload-time = "2026-10-06T08:16:06.355Z" }, +] + [[package]] name = "types-protobuf" version = "7.35.1.20260906" From 82eb8b600b96533107367a656a3fdf5fb584e728 Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Thu, 29 Oct 2026 11:00:45 +0200 Subject: [PATCH 2/6] test(sfp): harden trigger publication recovery coverage --- .../create-fixture/fixtures/manifest.py | 85 ++++- .github/actions/provider/start/action.yml | 29 ++ .../triggers/publication/action.yml | 263 ++++++++++++- .../triggers/storage-failure/action.yml | 188 ++++++++++ .github/workflows/rpm-installed-test.yml | 8 + src/stateforge_agent/_journal/sqlite.py | 7 +- .../_services/trigger_publication/client.py | 62 +++- tests/test_daemon_provider_events.py | 47 +++ tests/test_rpm_automation.py | 41 ++ tests/test_sqlite_journal.py | 150 ++++++++ tests/test_trigger_publication.py | 351 +++++++++++++++++- 11 files changed, 1216 insertions(+), 15 deletions(-) create mode 100644 .github/actions/rpm/test-installed/triggers/storage-failure/action.yml create mode 100644 tests/test_sqlite_journal.py diff --git a/.github/actions/provider/create-fixture/fixtures/manifest.py b/.github/actions/provider/create-fixture/fixtures/manifest.py index a314668..339d362 100644 --- a/.github/actions/provider/create-fixture/fixtures/manifest.py +++ b/.github/actions/provider/create-fixture/fixtures/manifest.py @@ -7,13 +7,17 @@ import hashlib import os import signal -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable from pathlib import Path +from uuid import UUID import grpc from stateforge_protocol.sfp.capability_assessment.v1 import ( capability_assessment_pb2, ) +from stateforge_protocol.sfp.trigger_publication.v1 import ( + trigger_publication_pb2, +) from stateforge_provider import ( HealthStatus, Platform, @@ -28,6 +32,13 @@ from stateforge_provider._services.capability_assessment.service import ( CapabilityAssessmentService, ) +from stateforge_provider._services.trigger_publication import ( + runtime as trigger_publication_runtime, +) +from stateforge_provider._services.trigger_publication.runtime import ( + TriggerPublicationRuntime, + TriggerStreamLease, +) _CAPABILITY_ATTEMPT_PREFIX = "rpc:capability-assessment:" @@ -217,6 +228,7 @@ async def runtime_started(self) -> AsyncIterator[FixtureTriggerPayload]: def _schedule_trigger(self) -> None: """Schedule callback publication from a process signal.""" + self._record(f"trigger-scheduled:{_RUNTIME_REFRESHED.identifier}") task = asyncio.create_task(self._publish_callback_trigger()) self._trigger_tasks.add(task) task.add_done_callback(self._trigger_tasks.discard) @@ -293,9 +305,80 @@ async def assess_capabilities( CapabilityAssessmentService.AssessCapabilities = assess_capabilities +def _install_trigger_acknowledgement_observer() -> None: + """Record terminal acknowledgements and optionally discard them.""" + drop_acknowledgements = ( + os.environ.get("STATEFORGE_TEST_DROP_TRIGGER_ACKS", "false") == "true" + ) + original = TriggerPublicationRuntime.acknowledge + marker = Path(os.environ["STATEFORGE_TEST_TRIGGER_ACK_MARKER"]) + event_log = Path(os.environ["STATEFORGE_TEST_EVENT_LOG"]) + + async def acknowledge( + runtime: TriggerPublicationRuntime, + lease: TriggerStreamLease, + acknowledgement: trigger_publication_pb2.TriggerAcknowledgement, + *, + is_session_current: Callable[[str], bool], + ) -> None: + disposition = ( + trigger_publication_pb2.TriggerAcknowledgementDisposition.Name( + acknowledgement.disposition + ) + .removeprefix("TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_") + .lower() + ) + if drop_acknowledgements and not await asyncio.to_thread( + marker.exists + ): + _record_event( + event_log, + "trigger-ack-dropped:" + f"{acknowledgement.publication_id}:{disposition}", + ) + return + await original( + runtime, + lease, + acknowledgement, + is_session_current=is_session_current, + ) + _record_event( + event_log, + "trigger-ack-observed:" + f"{acknowledgement.publication_id}:{disposition}", + ) + + TriggerPublicationRuntime.acknowledge = acknowledge + + +def _install_first_trigger_publication_id() -> None: + """Force only the first fixture publication to a deterministic UUID.""" + configured = os.environ.get( + "STATEFORGE_TEST_FIRST_TRIGGER_PUBLICATION_ID", + "", + ) + if not configured: + return + first_publication_id = UUID(configured) + original = trigger_publication_runtime.uuid4 + pending = True + + def next_publication_id() -> UUID: + nonlocal pending + if pending: + pending = False + return first_publication_id + return original() + + trigger_publication_runtime.uuid4 = next_publication_id + + def main() -> None: """Run the Provider fixture until its service manager stops it.""" _install_capability_unavailable_once() + _install_trigger_acknowledgement_observer() + _install_first_trigger_publication_id() ManifestFixtureProvider( discovery_directory=Path( os.environ["STATEFORGE_TEST_DISCOVERY_DIRECTORY"] diff --git a/.github/actions/provider/start/action.yml b/.github/actions/provider/start/action.yml index 89c87b1..8935d77 100644 --- a/.github/actions/provider/start/action.yml +++ b/.github/actions/provider/start/action.yml @@ -32,6 +32,14 @@ inputs: description: Make the first real Capability Assessment RPC unavailable. required: false default: "false" + drop-trigger-acks-until-restart: + description: Discard trigger acknowledgements until the restart marker exists. + required: false + default: "false" + first-trigger-publication-id: + description: Deterministic UUID for the fixture's first trigger publication. + required: false + default: "" outputs: provider-instance-id: @@ -57,6 +65,8 @@ runs: ACTION_PATH: ${{ github.action_path }} CAPABILITY_UNAVAILABLE_ONCE: ${{ inputs.capability-unavailable-once }} CONTAINER_NAME: ${{ inputs.container-name }} + DROP_TRIGGER_ACKS: ${{ inputs.drop-trigger-acks-until-restart }} + FIRST_TRIGGER_PUBLICATION_ID: ${{ inputs.first-trigger-publication-id }} FIXTURE_PATH: ${{ inputs.fixture-path }} PROVIDER_IDENTITY: ${{ inputs.provider-identity }} PROVIDER_NAME: ${{ inputs.provider-name }} @@ -80,6 +90,7 @@ runs: identity_path="${state_directory}/identity.json" event_log="${state_directory}/events.log" target_path="${state_directory}/target-state.txt" + trigger_ack_marker="${state_directory}/accept-trigger-acks" service_user="sf-${PROVIDER_NAME}" unit_name="stateforge-test-${PROVIDER_NAME}.service" @@ -87,6 +98,18 @@ runs: "${fixture_root}/wheels" "${state_directory}" docker exec "${CONTAINER_NAME}" bash -lc \ "printf 'fixture-target-state\\n' > '${target_path}'" + if [[ "${DROP_TRIGGER_ACKS}" == "true" ]]; then + docker exec "${CONTAINER_NAME}" rm -f "${trigger_ack_marker}" + elif [[ "${DROP_TRIGGER_ACKS}" != "false" ]]; then + printf 'drop-trigger-acks-until-restart must be true or false.\n' >&2 + exit 2 + fi + if [[ -n "${FIRST_TRIGGER_PUBLICATION_ID}" \ + && ! "${FIRST_TRIGGER_PUBLICATION_ID}" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ ]]; then + printf 'first-trigger-publication-id must be a lowercase UUIDv4.\n' \ + >&2 + exit 2 + fi docker cp "${FIXTURE_PATH}" \ "${CONTAINER_NAME}:${fixture_root}/provider.py" docker cp "${ACTION_PATH}/resolve-ready-endpoint.sh" \ @@ -140,6 +163,12 @@ runs: "STATEFORGE_TEST_TARGET_PATH=${target_path}" \ --setenv \ "STATEFORGE_TEST_CAPABILITY_UNAVAILABLE_ONCE=${CAPABILITY_UNAVAILABLE_ONCE}" \ + --setenv \ + "STATEFORGE_TEST_DROP_TRIGGER_ACKS=${DROP_TRIGGER_ACKS}" \ + --setenv \ + "STATEFORGE_TEST_TRIGGER_ACK_MARKER=${trigger_ack_marker}" \ + --setenv \ + "STATEFORGE_TEST_FIRST_TRIGGER_PUBLICATION_ID=${FIRST_TRIGGER_PUBLICATION_ID}" \ /usr/bin/python3 "${fixture_root}/provider.py" discovery_directory="/run/stateforge-agent/providers" diff --git a/.github/actions/rpm/test-installed/triggers/publication/action.yml b/.github/actions/rpm/test-installed/triggers/publication/action.yml index ba4c5c3..56d458f 100644 --- a/.github/actions/rpm/test-installed/triggers/publication/action.yml +++ b/.github/actions/rpm/test-installed/triggers/publication/action.yml @@ -38,6 +38,7 @@ runs: container-name: ${{ steps.setup.outputs.container-name }} fixture-path: ${{ steps.setup.outputs.fixture-path }} wheel-directory: ${{ steps.setup.outputs.wheel-directory }} + drop-trigger-acks-until-restart: "true" - name: Assert decorated trigger acceptance uses: ./.github/actions/agent/assert-trigger-publication @@ -48,20 +49,114 @@ runs: evidence-name: before-agent-restart evidence-directory: ${{ inputs.evidence-directory }} - - name: Stop the installed Agent + - name: Confirm durable commit preceded the lost acknowledgement shell: bash env: CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} run: | set -euo pipefail + event_log=/var/lib/stateforge-provider-fixtures/provider-a/events.log + deadline=$((SECONDS + 60)) + publication_id="" + while (( SECONDS < deadline )); do + publication_id="$(docker exec "${CONTAINER_NAME}" \ + sed -n \ + 's/^trigger-ack-dropped:\([0-9a-f-]*\):accepted_new$/\1/p' \ + "${event_log}" | tail -1)" + [[ -n "${publication_id}" ]] && break + sleep 1 + done + if [[ -z "${publication_id}" ]]; then + printf 'Provider did not discard the first terminal ACK.\n' >&2 + exit 1 + fi + jq -e --arg publication_id "${publication_id}" \ + '.publication_count == 1 + and .publications[0].publication_id == $publication_id' \ + "${EVIDENCE_DIRECTORY}/before-agent-restart.json" >/dev/null + printf '%s\n' "${publication_id}" \ + > "${EVIDENCE_DIRECTORY}/replayed-publication-id.txt" + + - name: Crash the installed Agent after durable acceptance + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + run: | + set -euo pipefail + docker exec "${CONTAINER_NAME}" \ + systemctl kill --kill-whom=main --signal=SIGKILL \ + stateforge-agent.service docker exec "${CONTAINER_NAME}" \ systemctl stop stateforge-agent.service + - name: Permit the replay acknowledgement + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + run: | + set -euo pipefail + docker exec "${CONTAINER_NAME}" \ + touch \ + /var/lib/stateforge-provider-fixtures/provider-a/accept-trigger-acks + - name: Start the same installed Agent again uses: ./.github/actions/agent/start with: container-name: ${{ steps.setup.outputs.container-name }} + - name: Confirm exact replay received the duplicate acknowledgement + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + run: | + set -euo pipefail + event_log=/var/lib/stateforge-provider-fixtures/provider-a/events.log + publication_id="$(cat \ + "${EVIDENCE_DIRECTORY}/replayed-publication-id.txt")" + expected="trigger-ack-observed:${publication_id}:accepted_duplicate" + deadline=$((SECONDS + 60)) + observed="" + while (( SECONDS < deadline )); do + observed="$(docker exec "${CONTAINER_NAME}" \ + grep -Fx "${expected}" "${event_log}" || true)" + [[ -n "${observed}" ]] && break + sleep 1 + done + if [[ -z "${observed}" ]]; then + printf 'Provider did not receive the duplicate replay ACK.\n' >&2 + exit 1 + fi + printf '%s\n' "${observed}" \ + > "${EVIDENCE_DIRECTORY}/provider-replay-outcome.txt" + + - name: Assert restart replay retained one immutable publication + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + expected-triggers: stateforge.test/runtime.started@1 + evidence-name: after-agent-restart + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Compare durable evidence across the replay + shell: bash + env: + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + run: | + set -euo pipefail + before="${EVIDENCE_DIRECTORY}/before-agent-restart.json" + after="${EVIDENCE_DIRECTORY}/after-agent-restart.json" + jq -e --slurp \ + '.[0].publication_count == 1 + and .[1].publication_count == 1 + and .[0].publications[0].publication_id + == .[1].publications[0].publication_id + and .[0].publications[0].content_fingerprint + == .[1].publications[0].content_fingerprint' \ + "${before}" "${after}" >/dev/null + - name: Publish through the Provider callback API shell: bash env: @@ -72,7 +167,7 @@ runs: docker exec "${CONTAINER_NAME}" \ systemctl kill --kill-whom=main --signal=SIGALRM "${UNIT_NAME}" - - name: Assert restart persistence and callback acceptance + - name: Assert callback acceptance after replay recovery uses: ./.github/actions/agent/assert-trigger-publication with: container-name: ${{ steps.setup.outputs.container-name }} @@ -80,7 +175,7 @@ runs: expected-triggers: >- stateforge.test/runtime.started@1 stateforge.test/runtime.refreshed@1 - evidence-name: after-agent-restart + evidence-name: after-callback evidence-directory: ${{ inputs.evidence-directory }} - name: Preserve Provider acknowledgement evidence @@ -107,6 +202,168 @@ runs: printf '%s\n' "${outcome}" \ > "${EVIDENCE_DIRECTORY}/provider-trigger-outcome.txt" + - name: Start isolation Provider A + id: isolation_a + uses: ./.github/actions/provider/start + with: + container-name: ${{ steps.setup.outputs.container-name }} + fixture-path: ${{ steps.setup.outputs.fixture-path }} + wheel-directory: ${{ steps.setup.outputs.wheel-directory }} + provider-name: trigger-a + provider-identity: stateforge.test.trigger.a + first-trigger-publication-id: 11111111-1111-4111-8111-111111111111 + + - name: Start isolation Provider B + id: isolation_b + uses: ./.github/actions/provider/start + with: + container-name: ${{ steps.setup.outputs.container-name }} + fixture-path: ${{ steps.setup.outputs.fixture-path }} + wheel-directory: ${{ steps.setup.outputs.wheel-directory }} + provider-name: trigger-b + provider-identity: stateforge.test.trigger.b + first-trigger-publication-id: 11111111-1111-4111-8111-111111111111 + + - name: Assert Provider A publication ownership + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.isolation_a.outputs.provider-instance-id }} + expected-triggers: stateforge.test/runtime.started@1 + evidence-name: isolation-provider-a + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Assert Provider B publication ownership + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.isolation_b.outputs.provider-instance-id }} + expected-triggers: stateforge.test/runtime.started@1 + evidence-name: isolation-provider-b + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Confirm identical IDs and acknowledgement routing remain isolated + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + PROVIDER_A_INSTANCE_ID: ${{ steps.isolation_a.outputs.provider-instance-id }} + PROVIDER_B_INSTANCE_ID: ${{ steps.isolation_b.outputs.provider-instance-id }} + run: | + set -euo pipefail + publication_id=11111111-1111-4111-8111-111111111111 + if [[ "${PROVIDER_A_INSTANCE_ID}" == "${PROVIDER_B_INSTANCE_ID}" ]]; then + printf 'Isolation Providers share one installation identity.\n' >&2 + exit 1 + fi + provider_a_session="$(jq -r \ + '.publications[0].session_id_sha256' \ + "${EVIDENCE_DIRECTORY}/isolation-provider-a.json")" + provider_b_session="$(jq -r \ + '.publications[0].session_id_sha256' \ + "${EVIDENCE_DIRECTORY}/isolation-provider-b.json")" + if [[ "${provider_a_session}" == "${provider_b_session}" ]]; then + printf 'Isolation Providers share one fenced SFP session.\n' >&2 + exit 1 + fi + for provider in a b; do + evidence="${EVIDENCE_DIRECTORY}/isolation-provider-${provider}.json" + jq -e --arg publication_id "${publication_id}" \ + '.publication_count == 1 + and .publications[0].publication_id == $publication_id + and .publications[0].forwarding_state == "pending"' \ + "${evidence}" >/dev/null + + event_log="/var/lib/stateforge-provider-fixtures/trigger-${provider}/events.log" + expected="trigger-ack-observed:${publication_id}:accepted_new" + deadline=$((SECONDS + 60)) + observed="" + while (( SECONDS < deadline )); do + observed="$(docker exec "${CONTAINER_NAME}" \ + grep -Fx "${expected}" "${event_log}" || true)" + [[ -n "${observed}" ]] && break + sleep 1 + done + if [[ -z "${observed}" ]]; then + printf 'Provider %s did not receive its terminal ACK.\n' \ + "${provider}" >&2 + exit 1 + fi + done + + - name: Disconnect Provider A while Provider B remains active + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + PROVIDER_A_UNIT: ${{ steps.isolation_a.outputs.unit-name }} + PROVIDER_B_UNIT: ${{ steps.isolation_b.outputs.unit-name }} + run: | + set -euo pipefail + docker exec "${CONTAINER_NAME}" \ + systemctl stop "${PROVIDER_A_UNIT}" + if docker exec "${CONTAINER_NAME}" \ + systemctl is-active --quiet "${PROVIDER_A_UNIT}"; then + printf 'Provider A remained active after its stop completed.\n' >&2 + exit 1 + fi + docker exec "${CONTAINER_NAME}" \ + systemctl is-active --quiet "${PROVIDER_B_UNIT}" + docker exec "${CONTAINER_NAME}" \ + systemctl kill --kill-whom=main --signal=SIGALRM \ + "${PROVIDER_B_UNIT}" + + - name: Assert Provider B continues after Provider A disconnects + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.isolation_b.outputs.provider-instance-id }} + expected-triggers: >- + stateforge.test/runtime.started@1 + stateforge.test/runtime.refreshed@1 + evidence-name: isolation-provider-b-after-a-disconnect + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Assert Provider A retained only its own publication + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.isolation_a.outputs.provider-instance-id }} + expected-triggers: stateforge.test/runtime.started@1 + evidence-name: isolation-provider-a-after-disconnect + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Confirm Provider B receives its post-fault acknowledgement + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + run: | + set -euo pipefail + event_log=/var/lib/stateforge-provider-fixtures/trigger-b/events.log + deadline=$((SECONDS + 60)) + outcome="" + while (( SECONDS < deadline )); do + outcome="$(docker exec "${CONTAINER_NAME}" \ + grep -E '^trigger-outcome:[0-9a-f-]+:accepted_new$' \ + "${event_log}" || true)" + [[ -n "${outcome}" ]] && break + sleep 1 + done + if [[ -z "${outcome}" ]]; then + printf 'Provider B did not receive its post-fault ACK.\n' >&2 + exit 1 + fi + printf '%s\n' "${outcome}" \ + > "${EVIDENCE_DIRECTORY}/isolation-provider-b-outcome.txt" + jq -e \ + '.publication_count == 2 + and any( + .publications[]; + .publication_id == "11111111-1111-4111-8111-111111111111" + )' \ + "${EVIDENCE_DIRECTORY}/isolation-provider-b-after-a-disconnect.json" \ + >/dev/null + - name: Preserve evidence and clean up id: cleanup if: ${{ always() }} diff --git a/.github/actions/rpm/test-installed/triggers/storage-failure/action.yml b/.github/actions/rpm/test-installed/triggers/storage-failure/action.yml new file mode 100644 index 0000000..0a0f26d --- /dev/null +++ b/.github/actions/rpm/test-installed/triggers/storage-failure/action.yml @@ -0,0 +1,188 @@ +name: Test installed Trigger Publication storage recovery +description: Verify retryable publication behavior across a real journal write failure. + +inputs: + rpm-directory: + description: Directory containing the exact RPM artifact under test. + required: false + default: dist/rpm + evidence-directory: + description: Directory receiving this scenario's evidence. + required: false + default: dist/agent-system-test-evidence/trigger-storage-failure + fixture-directory: + description: Directory containing the exact Provider fixture bundle. + required: false + default: dist/provider-fixture + +outputs: + evidence-directory: + description: Absolute directory containing scenario evidence. + value: ${{ steps.cleanup.outputs.evidence-directory }} + +runs: + using: composite + steps: + - name: Prepare installed Agent + id: setup + uses: ./.github/actions/rpm/test-installed/setup + with: + rpm-directory: ${{ inputs.rpm-directory }} + evidence-directory: ${{ inputs.evidence-directory }} + fixture-directory: ${{ inputs.fixture-directory }} + + - name: Start real Provider fixture + id: provider + uses: ./.github/actions/provider/start + with: + container-name: ${{ steps.setup.outputs.container-name }} + fixture-path: ${{ steps.setup.outputs.fixture-path }} + wheel-directory: ${{ steps.setup.outputs.wheel-directory }} + provider-name: storage-a + provider-identity: stateforge.test.trigger.storage + + - name: Assert baseline durable publication + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + expected-triggers: stateforge.test/runtime.started@1 + evidence-name: before-storage-failure + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Deny journal writes and request a callback publication + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + UNIT_NAME: ${{ steps.provider.outputs.unit-name }} + run: | + set -euo pipefail + journal_directory=/var/lib/stateforge-agent/journal + database="${journal_directory}/agent.db" + event_log=/var/lib/stateforge-provider-fixtures/storage-a/events.log + mkdir -p "${EVIDENCE_DIRECTORY}" + docker exec "${CONTAINER_NAME}" stat --format='%A %a %U:%G %n' \ + "${journal_directory}" "${database}" \ + > "${EVIDENCE_DIRECTORY}/journal-permissions-before.txt" + baseline_outcomes="$(docker exec "${CONTAINER_NAME}" \ + grep -Ec '^trigger-outcome:[0-9a-f-]+:accepted_new$' \ + "${event_log}" || true)" + printf '%s\n' "${baseline_outcomes}" \ + > "${EVIDENCE_DIRECTORY}/baseline-outcome-count.txt" + docker exec "${CONTAINER_NAME}" bash -lc \ + "chmod u-w '${journal_directory}' '${database}'*" + docker exec "${CONTAINER_NAME}" \ + systemctl kill --kill-whom=main --signal=SIGALRM "${UNIT_NAME}" + + scheduled='trigger-scheduled:stateforge.test/runtime.refreshed@1' + deadline=$((SECONDS + 60)) + while (( SECONDS < deadline )); do + if docker exec "${CONTAINER_NAME}" \ + grep -Fxq "${scheduled}" "${event_log}"; then + break + fi + sleep 1 + done + if ! docker exec "${CONTAINER_NAME}" \ + grep -Fxq "${scheduled}" "${event_log}"; then + printf 'Provider did not schedule the callback publication.\n' >&2 + exit 1 + fi + sleep 3 + + - name: Assert failed storage assumed no new responsibility + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + expected-triggers: stateforge.test/runtime.started@1 + evidence-name: during-storage-failure + evidence-directory: ${{ inputs.evidence-directory }} + timeout-seconds: "5" + + - name: Confirm no terminal acknowledgement escaped the failure + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + run: | + set -euo pipefail + event_log=/var/lib/stateforge-provider-fixtures/storage-a/events.log + before="$(cat \ + "${EVIDENCE_DIRECTORY}/baseline-outcome-count.txt")" + after="$(docker exec "${CONTAINER_NAME}" \ + grep -Ec '^trigger-outcome:[0-9a-f-]+:accepted_new$' \ + "${event_log}" || true)" + if [[ "${after}" != "${before}" ]]; then + printf 'Provider received a terminal ACK while storage failed.\n' \ + >&2 + exit 1 + fi + docker exec "${CONTAINER_NAME}" \ + systemctl is-active --quiet stateforge-agent.service + + - name: Restore journal writes + if: ${{ always() }} + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + run: | + set -euo pipefail + journal_directory=/var/lib/stateforge-agent/journal + database="${journal_directory}/agent.db" + if docker inspect "${CONTAINER_NAME}" >/dev/null 2>&1; then + docker exec "${CONTAINER_NAME}" bash -lc \ + "chmod u+w '${journal_directory}' '${database}'*" + docker exec "${CONTAINER_NAME}" \ + stat --format='%A %a %U:%G %n' \ + "${journal_directory}" "${database}" \ + > "${EVIDENCE_DIRECTORY}/journal-permissions-after.txt" + fi + + - name: Assert queued publication succeeds after storage recovery + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + expected-triggers: >- + stateforge.test/runtime.started@1 + stateforge.test/runtime.refreshed@1 + evidence-name: after-storage-recovery + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Confirm Provider receives the recovered acknowledgement + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + run: | + set -euo pipefail + event_log=/var/lib/stateforge-provider-fixtures/storage-a/events.log + before="$(cat \ + "${EVIDENCE_DIRECTORY}/baseline-outcome-count.txt")" + expected=$((before + 1)) + deadline=$((SECONDS + 60)) + after=0 + while (( SECONDS < deadline )); do + after="$(docker exec "${CONTAINER_NAME}" \ + grep -Ec '^trigger-outcome:[0-9a-f-]+:accepted_new$' \ + "${event_log}" || true)" + [[ "${after}" == "${expected}" ]] && break + sleep 1 + done + if [[ "${after}" != "${expected}" ]]; then + printf 'Provider did not receive the recovered terminal ACK.\n' >&2 + exit 1 + fi + docker exec "${CONTAINER_NAME}" tail -n 20 "${event_log}" \ + > "${EVIDENCE_DIRECTORY}/provider-storage-recovery-events.txt" + + - name: Preserve evidence and clean up + id: cleanup + if: ${{ always() }} + uses: ./.github/actions/rpm/test-installed/cleanup + with: + container-name: stateforge-agent-system-test + evidence-directory: ${{ inputs.evidence-directory }} diff --git a/.github/workflows/rpm-installed-test.yml b/.github/workflows/rpm-installed-test.yml index 0997350..8764b1b 100644 --- a/.github/workflows/rpm-installed-test.yml +++ b/.github/workflows/rpm-installed-test.yml @@ -109,6 +109,7 @@ jobs: - capability-limits - provider-availability - trigger-publication + - trigger-storage-failure - multiple-providers - invalid-descriptor - uninstall-reinstall @@ -173,6 +174,13 @@ jobs: evidence-directory: dist/agent-system-test-evidence/${{ matrix.scenario }} fixture-directory: dist/provider-fixture + - name: Run Trigger Publication storage-failure scenario + if: ${{ matrix.scenario == 'trigger-storage-failure' }} + uses: ./.github/actions/rpm/test-installed/triggers/storage-failure + with: + evidence-directory: dist/agent-system-test-evidence/${{ matrix.scenario }} + fixture-directory: dist/provider-fixture + - name: Run invalid-descriptor scenario if: ${{ matrix.scenario == 'invalid-descriptor' }} uses: ./.github/actions/rpm/test-installed/adversarial/invalid-descriptor diff --git a/src/stateforge_agent/_journal/sqlite.py b/src/stateforge_agent/_journal/sqlite.py index a7ae54e..9b4e23c 100644 --- a/src/stateforge_agent/_journal/sqlite.py +++ b/src/stateforge_agent/_journal/sqlite.py @@ -10,6 +10,7 @@ from typing import TypeVar _Result = TypeVar("_Result") +_BUSY_TIMEOUT_MILLISECONDS = 5000 class SqliteJournalError(RuntimeError): @@ -94,10 +95,12 @@ def _connect(self) -> sqlite3.Connection: """Open one fully configured local connection.""" connection = sqlite3.connect( self._path, - timeout=5, + timeout=_BUSY_TIMEOUT_MILLISECONDS / 1000, isolation_level=None, ) - connection.execute("PRAGMA busy_timeout = 5000") + connection.execute( + f"PRAGMA busy_timeout = {_BUSY_TIMEOUT_MILLISECONDS}" + ) connection.execute("PRAGMA foreign_keys = ON") connection.execute("PRAGMA synchronous = FULL") return connection diff --git a/src/stateforge_agent/_sfp/_services/trigger_publication/client.py b/src/stateforge_agent/_sfp/_services/trigger_publication/client.py index 9fa9788..13f136e 100644 --- a/src/stateforge_agent/_sfp/_services/trigger_publication/client.py +++ b/src/stateforge_agent/_sfp/_services/trigger_publication/client.py @@ -2,9 +2,10 @@ from __future__ import annotations +import asyncio from collections.abc import Awaitable, Callable from dataclasses import dataclass -from typing import Never, Protocol +from typing import Any, Never, Protocol import grpc from google.protobuf.message import DecodeError @@ -152,23 +153,68 @@ async def consume( raise SfpTriggerPublicationProtocolError( "Trigger Publication ready response failed validation" ) from error + await _consume_publications( + call, + maximum_in_flight=first.ready.maximum_in_flight_publications, + handler=handler, + ) + except grpc.aio.AioRpcError as error: + _raise_rpc_error(error.code()) + finally: + call.cancel() - while True: - response = await _read_response(call) + +async def _consume_publications( + call: _TriggerStreamCall, + *, + maximum_in_flight: int, + handler: TriggerPublicationHandler, +) -> None: + """Bound concurrent durable work by the Provider-selected wire window.""" + reader = asyncio.create_task(_read_response(call)) + handlers: set[ + asyncio.Future[trigger_publication_pb2.TriggerAcknowledgement] + ] = set() + try: + while True: + waiters: set[asyncio.Future[Any]] = {reader} + waiters.update(handlers) + done, _ = await asyncio.wait( + waiters, + return_when=asyncio.FIRST_COMPLETED, + ) + if reader in done: + response = reader.result() if response.WhichOneof("message") != "publication": raise SfpTriggerPublicationProtocolError( "Trigger Publication stream returned invalid framing" ) - acknowledgement = await handler(response.publication) + if len(handlers) >= maximum_in_flight: + raise SfpTriggerPublicationLimitError( + "Trigger Publication Provider exceeded the selected " + "receive window" + ) + handlers.add( + asyncio.ensure_future(handler(response.publication)) + ) + reader = asyncio.create_task(_read_response(call)) + continue + + for task in tuple(handlers): + if task not in done: + continue + handlers.remove(task) + acknowledgement = task.result() request = trigger_publication_pb2.StreamTriggersRequest( acknowledgement=acknowledgement ) _validate_request(request) await call.write(request) - except grpc.aio.AioRpcError as error: - _raise_rpc_error(error.code()) - finally: - call.cancel() + finally: + reader.cancel() + for task in handlers: + task.cancel() + await asyncio.gather(reader, *handlers, return_exceptions=True) def _validate_request( diff --git a/tests/test_daemon_provider_events.py b/tests/test_daemon_provider_events.py index c2e16ff..26ca3e7 100644 --- a/tests/test_daemon_provider_events.py +++ b/tests/test_daemon_provider_events.py @@ -3,6 +3,7 @@ import json import signal import socket +import sqlite3 from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock, Mock @@ -262,6 +263,52 @@ def fail_to_prepare(_monitor: ProviderDirectoryMonitor) -> None: assert result is ProcessExitCode.RUNTIME_FAILURE +@pytest.mark.asyncio +async def test_corrupt_journal_returns_runtime_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + layout = _runtime_layout(tmp_path) + layout.journal_path.parent.mkdir(parents=True) + layout.journal_path.write_bytes(b"not a SQLite database") + monkeypatch.setattr(daemon, "_install_signal_handlers", lambda *_: ()) + + result = await daemon._run_agent(_configuration(tmp_path), layout) + + assert result is ProcessExitCode.RUNTIME_FAILURE + + +@pytest.mark.asyncio +async def test_incompatible_trigger_schema_returns_runtime_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + layout = _runtime_layout(tmp_path) + layout.journal_path.parent.mkdir(parents=True) + connection = sqlite3.connect(layout.journal_path) + try: + connection.execute( + """ + CREATE TABLE journal_schemas ( + component TEXT PRIMARY KEY, + version INTEGER NOT NULL + ) WITHOUT ROWID + """ + ) + connection.execute( + "INSERT INTO journal_schemas (component, version) VALUES (?, ?)", + ("trigger_publication", 999), + ) + connection.commit() + finally: + connection.close() + monkeypatch.setattr(daemon, "_install_signal_handlers", lambda *_: ()) + + result = await daemon._run_agent(_configuration(tmp_path), layout) + + assert result is ProcessExitCode.RUNTIME_FAILURE + + def test_signal_handlers_are_registered_when_supported() -> None: loop = Mock() stop_event = Mock() diff --git a/tests/test_rpm_automation.py b/tests/test_rpm_automation.py index 3fe3890..48533d2 100644 --- a/tests/test_rpm_automation.py +++ b/tests/test_rpm_automation.py @@ -440,6 +440,7 @@ def test_action_library_has_expected_domain_boundaries() -> None: "rpm/test-installed/setup", "rpm/test-installed/smoke/manifest-exchange", "rpm/test-installed/triggers/publication", + "rpm/test-installed/triggers/storage-failure", } observed = { str(path.parent.relative_to(ACTIONS)) @@ -647,6 +648,20 @@ def test_installed_trigger_scenario_uses_public_api_and_journal() -> None: assert "./.github/actions/agent/start" in scenario assert "before-agent-restart" in scenario assert "after-agent-restart" in scenario + assert "after-callback" in scenario + assert "drop-trigger-acks-until-restart" in scenario + assert "accepted_duplicate" in scenario + assert "provider-replay-outcome.txt" in scenario + assert "content_fingerprint" in scenario + assert "systemctl kill --kill-whom=main --signal=SIGKILL" in scenario + assert "first-trigger-publication-id" in scenario + assert "isolation-provider-a" in scenario + assert "isolation-provider-b" in scenario + assert "PROVIDER_A_INSTANCE_ID" in scenario + assert "PROVIDER_B_INSTANCE_ID" in scenario + assert "isolation-provider-b-after-a-disconnect" in scenario + assert "isolation-provider-a-after-disconnect" in scenario + assert "isolation-provider-b-outcome.txt" in scenario assert "stateforge.test/runtime.started@1" in scenario assert "stateforge.test/runtime.refreshed@1" in scenario assert "@trigger(" in fixture @@ -656,3 +671,29 @@ def test_installed_trigger_scenario_uses_public_api_and_journal() -> None: assert "validate_trigger_publication" in assertion assert "compute_publication_content_fingerprint" in assertion assert 'row["forwarding_state"] != "pending"' in assertion + + +def test_installed_trigger_storage_failure_recovers_without_false_ack() -> ( + None +): + """Exercise a real journal write denial before retrying one event.""" + workflow = read(WORKFLOWS / "rpm-installed-test.yml") + scenario = read( + ACTIONS + / "rpm" + / "test-installed" + / "triggers" + / "storage-failure" + / "action.yml" + ) + + assert "trigger-storage-failure" in workflow + assert "./.github/actions/rpm/test-installed/setup" in scenario + assert "./.github/actions/provider/start" in scenario + assert "chmod u-w" in scenario + assert "during-storage-failure" in scenario + assert "chmod u+w" in scenario + assert "after-storage-recovery" in scenario + assert "Provider received a terminal ACK while storage failed" in scenario + assert "stateforge.test/runtime.refreshed@1" in scenario + assert "./.github/actions/rpm/test-installed/cleanup" in scenario diff --git a/tests/test_sqlite_journal.py b/tests/test_sqlite_journal.py new file mode 100644 index 0000000..6fb92e1 --- /dev/null +++ b/tests/test_sqlite_journal.py @@ -0,0 +1,150 @@ +"""Tests for SQLite journal failure and rollback boundaries.""" + +import errno +import os +import sqlite3 +from pathlib import Path + +import pytest + +from stateforge_agent._journal import SqliteJournal, SqliteJournalError +from stateforge_agent._journal import sqlite as sqlite_journal + +_LARGE_PAYLOAD_BYTES = 1024 * 1024 +_SHORT_BUSY_TIMEOUT_MILLISECONDS = 1 + + +async def _open_journal(path: Path) -> SqliteJournal: + journal = SqliteJournal(path) + await journal.open() + await journal.transaction(_create_payload_table) + return journal + + +def _create_payload_table(connection: sqlite3.Connection) -> None: + connection.execute( + "CREATE TABLE payloads (identifier INTEGER PRIMARY KEY, value BLOB)" + ) + + +def _payload_count(connection: sqlite3.Connection) -> int: + row = connection.execute("SELECT COUNT(*) FROM payloads").fetchone() + if row is None: + raise sqlite3.DatabaseError("Payload count is absent") + return int(row[0]) + + +@pytest.mark.asyncio +async def test_disk_full_rolls_back_the_complete_transaction( + tmp_path: Path, +) -> None: + journal = await _open_journal(tmp_path / "agent.db") + + def insert_without_available_pages( + connection: sqlite3.Connection, + ) -> None: + row = connection.execute("PRAGMA page_count").fetchone() + if row is None: + raise sqlite3.DatabaseError("Page count is absent") + connection.execute(f"PRAGMA max_page_count = {int(row[0])}") + connection.execute( + "INSERT INTO payloads (value) VALUES (?)", + (b"x" * _LARGE_PAYLOAD_BYTES,), + ) + + with pytest.raises(SqliteJournalError) as captured: + await journal.transaction(insert_without_available_pages) + + assert isinstance(captured.value.__cause__, sqlite3.DatabaseError) + assert await journal.transaction(_payload_count) == 0 + await journal.close() + + +@pytest.mark.asyncio +async def test_read_only_journal_rejects_writes_without_partial_state( + tmp_path: Path, +) -> None: + if os.geteuid() == 0: + pytest.skip("root bypasses filesystem write permission checks") + path = tmp_path / "journal" / "agent.db" + journal = await _open_journal(path) + + def insert_payload(connection: sqlite3.Connection) -> None: + connection.execute("INSERT INTO payloads (value) VALUES (X'01')") + + path.chmod(0o400) + path.parent.chmod(0o500) + try: + with pytest.raises(SqliteJournalError) as captured: + await journal.transaction(insert_payload) + assert isinstance(captured.value.__cause__, sqlite3.OperationalError) + finally: + path.parent.chmod(0o750) + path.chmod(0o640) + + assert await journal.transaction(_payload_count) == 0 + await journal.close() + + +@pytest.mark.asyncio +async def test_busy_database_times_out_without_partial_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "agent.db" + journal = await _open_journal(path) + monkeypatch.setattr( + sqlite_journal, + "_BUSY_TIMEOUT_MILLISECONDS", + _SHORT_BUSY_TIMEOUT_MILLISECONDS, + ) + blocker = sqlite3.connect(path, isolation_level=None) + blocker.execute("BEGIN IMMEDIATE") + + def insert_payload(connection: sqlite3.Connection) -> None: + connection.execute("INSERT INTO payloads (value) VALUES (X'02')") + + try: + with pytest.raises(SqliteJournalError) as captured: + await journal.transaction(insert_payload) + assert isinstance(captured.value.__cause__, sqlite3.OperationalError) + assert "locked" in str(captured.value.__cause__).lower() + finally: + blocker.rollback() + blocker.close() + + assert await journal.transaction(_payload_count) == 0 + await journal.close() + + +@pytest.mark.asyncio +async def test_filesystem_io_error_is_wrapped_as_a_journal_failure( + tmp_path: Path, +) -> None: + journal = await _open_journal(tmp_path / "agent.db") + injected_error = OSError(errno.EIO, "injected durable-write failure") + + def write_then_fail(connection: sqlite3.Connection) -> None: + connection.execute("INSERT INTO payloads (value) VALUES (X'03')") + raise injected_error + + with pytest.raises(SqliteJournalError) as captured: + await journal.transaction(write_then_fail) + + assert captured.value.__cause__ is injected_error + assert await journal.transaction(_payload_count) == 0 + await journal.close() + + +@pytest.mark.asyncio +async def test_corrupt_database_is_rejected_during_open( + tmp_path: Path, +) -> None: + path = tmp_path / "agent.db" + path.write_bytes(b"not a SQLite database") + journal = SqliteJournal(path) + + with pytest.raises(SqliteJournalError) as captured: + await journal.open() + + assert isinstance(captured.value.__cause__, sqlite3.DatabaseError) diff --git a/tests/test_trigger_publication.py b/tests/test_trigger_publication.py index 12e7483..bde20fa 100644 --- a/tests/test_trigger_publication.py +++ b/tests/test_trigger_publication.py @@ -20,24 +20,33 @@ from stateforge_agent._journal import SqliteJournal from stateforge_agent._sfp._services.trigger_publication.client import ( + SfpTriggerPublicationError, SfpTriggerPublicationLimitError, + SfpTriggerPublicationProtocolError, SfpTriggerPublicationStaleError, SfpTriggerPublicationUnavailableError, TriggerPublicationClient, TriggerPublicationHandler, TriggerStreamBinding, + _consume_publications, + _DecodedTriggerResponse, + _raise_rpc_error, ) from stateforge_agent._sfp._services.trigger_publication.coordinator import ( TriggerPublicationCoordinator, TriggerPublicationLease, ) from stateforge_agent._sfp._services.trigger_publication.store import ( + TriggerAcceptanceContext, + TriggerAcceptanceDecision, TriggerPublicationStore, ) _TRIGGER_IDENTIFIER = "example.systemd/unit.failed@1" _STALE_BEFORE_DURABLE_COMMIT = 2 _STALE_AFTER_DURABLE_COMMIT = 3 +_SELECTED_RECEIVE_WINDOW = 2 +_ISOLATED_PROVIDER_COUNT = 2 _TRIGGER_SCHEMA = { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", @@ -87,8 +96,127 @@ async def StreamTriggers( # noqa: N802 self.requests.append(await anext(request_iterator)) +class _ReadyScenarioPeer( + trigger_publication_pb2_grpc.TriggerPublicationServiceServicer +): + """Return one deliberately invalid stream-opening sequence.""" + + def __init__(self, scenario: str) -> None: + self.scenario = scenario + + async def StreamTriggers( # noqa: N802 + self, + request_iterator: AsyncIterator[ + trigger_publication_pb2.StreamTriggersRequest + ], + context, + ) -> AsyncIterator[trigger_publication_pb2.StreamTriggersResponse]: + del context + opening = await anext(request_iterator) + if self.scenario == "eof-before-ready": + return + if self.scenario == "publication-before-ready": + yield trigger_publication_pb2.StreamTriggersResponse( + publication=_publication() + ) + return + + binding = opening.open.session_binding + ready = trigger_publication_pb2.TriggerStreamReady( + provider_instance_id=binding.expected_provider_instance_id, + endpoint_generation=binding.expected_endpoint_generation, + session_id=binding.expected_session_id, + manifest_revision=binding.expected_manifest_revision, + maximum_in_flight_publications=( + opening.open.requested_max_in_flight_publications + ), + ) + if self.scenario == "mismatched-ready": + ready.session_id = "different-session" + elif self.scenario == "zero-window": + ready.maximum_in_flight_publications = 0 + elif self.scenario == "larger-window": + ready.maximum_in_flight_publications += 1 + yield trigger_publication_pb2.StreamTriggersResponse(ready=ready) + if self.scenario == "duplicate-ready": + yield trigger_publication_pb2.StreamTriggersResponse(ready=ready) + + +class _WindowPeer( + trigger_publication_pb2_grpc.TriggerPublicationServiceServicer +): + """Publish a fixed burst and retain acknowledgements for assertions.""" + + def __init__(self, publication_count: int, *, window: int) -> None: + self.publications = [_publication() for _ in range(publication_count)] + self.window = window + self.acknowledgements: list[ + trigger_publication_pb2.TriggerAcknowledgement + ] = [] + + async def StreamTriggers( # noqa: N802 + self, + request_iterator: AsyncIterator[ + trigger_publication_pb2.StreamTriggersRequest + ], + context, + ) -> AsyncIterator[trigger_publication_pb2.StreamTriggersResponse]: + del context + opening = await anext(request_iterator) + binding = opening.open.session_binding + yield trigger_publication_pb2.StreamTriggersResponse( + ready=trigger_publication_pb2.TriggerStreamReady( + provider_instance_id=binding.expected_provider_instance_id, + endpoint_generation=binding.expected_endpoint_generation, + session_id=binding.expected_session_id, + manifest_revision=binding.expected_manifest_revision, + maximum_in_flight_publications=self.window, + ) + ) + for publication in self.publications: + yield trigger_publication_pb2.StreamTriggersResponse( + publication=publication + ) + for _ in range(len(self.publications)): + request = await anext(request_iterator) + self.acknowledgements.append(request.acknowledgement) + + +class _ScriptedCall: + """Expose deterministic decoded responses without transport buffering.""" + + def __init__( + self, + publications: list[trigger_publication_pb2.TriggerPublication], + ) -> None: + self._responses = [ + _DecodedTriggerResponse( + response=trigger_publication_pb2.StreamTriggersResponse( + publication=publication + ) + ) + for publication in publications + ] + self.requests: list[trigger_publication_pb2.StreamTriggersRequest] = [] + self.cancelled = False + + async def write( + self, + request: trigger_publication_pb2.StreamTriggersRequest, + ) -> None: + self.requests.append(request) + + async def read(self) -> object: + await asyncio.sleep(0) + return self._responses.pop(0) + + def cancel(self) -> bool: + self.cancelled = True + return True + + async def _start_peer( - peer: _TriggerPeer, + peer: trigger_publication_pb2_grpc.TriggerPublicationServiceServicer, ) -> tuple[grpc.aio.Server, grpc.aio.Channel]: server = grpc.aio.server() trigger_publication_pb2_grpc.add_TriggerPublicationServiceServicer_to_server( @@ -248,6 +376,177 @@ async def accept( await server.stop(None) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("scenario", "error", "message"), + [ + ( + "eof-before-ready", + SfpTriggerPublicationUnavailableError, + "stream ended", + ), + ( + "publication-before-ready", + SfpTriggerPublicationProtocolError, + "did not begin with ready", + ), + ( + "mismatched-ready", + SfpTriggerPublicationProtocolError, + "ready response failed validation", + ), + ( + "zero-window", + SfpTriggerPublicationProtocolError, + "response failed canonical validation", + ), + ( + "larger-window", + SfpTriggerPublicationProtocolError, + "ready response failed validation", + ), + ( + "duplicate-ready", + SfpTriggerPublicationProtocolError, + "invalid framing", + ), + ], +) +async def test_client_rejects_invalid_stream_opening_sequences( + scenario: str, + error: type[SfpTriggerPublicationError], + message: str, +) -> None: + peer = _ReadyScenarioPeer(scenario) + server, channel = await _start_peer(peer) + lease = _lease() + + async def unreachable_handler( + publication: trigger_publication_pb2.TriggerPublication, + ) -> trigger_publication_pb2.TriggerAcknowledgement: + raise AssertionError(publication) + + try: + with pytest.raises(error, match=message): + await TriggerPublicationClient().consume( + channel, + TriggerStreamBinding( + provider_instance_id=str(lease.provider_instance_id), + endpoint_generation=lease.endpoint_generation, + session_id=lease.session_id, + manifest_revision=lease.manifest_revision, + maximum_in_flight_publications=2, + ), + unreachable_handler, + ) + finally: + await channel.close() + await server.stop(None) + + +@pytest.mark.asyncio +async def test_client_accepts_publications_at_selected_receive_window() -> ( + None +): + peer = _WindowPeer( + _SELECTED_RECEIVE_WINDOW, + window=_SELECTED_RECEIVE_WINDOW, + ) + server, channel = await _start_peer(peer) + lease = _lease() + + async def accept( + publication: trigger_publication_pb2.TriggerPublication, + ) -> trigger_publication_pb2.TriggerAcknowledgement: + return trigger_publication_pb2.TriggerAcknowledgement( + publication_id=publication.publication_id, + disposition=( + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_NEW + ), + ) + + try: + with pytest.raises( + SfpTriggerPublicationUnavailableError, + match="stream ended", + ): + await TriggerPublicationClient().consume( + channel, + TriggerStreamBinding( + provider_instance_id=str(lease.provider_instance_id), + endpoint_generation=lease.endpoint_generation, + session_id=lease.session_id, + manifest_revision=lease.manifest_revision, + maximum_in_flight_publications=_SELECTED_RECEIVE_WINDOW, + ), + accept, + ) + assert { + acknowledgement.publication_id + for acknowledgement in peer.acknowledgements + } == {publication.publication_id for publication in peer.publications} + finally: + await channel.close() + await server.stop(None) + + +@pytest.mark.asyncio +async def test_client_rejects_publication_over_selected_receive_window() -> ( + None +): + call = _ScriptedCall( + [_publication() for _ in range(_SELECTED_RECEIVE_WINDOW + 1)] + ) + release = asyncio.Event() + started = 0 + + async def blocked_accept( + publication: trigger_publication_pb2.TriggerPublication, + ) -> trigger_publication_pb2.TriggerAcknowledgement: + nonlocal started + started += 1 + await release.wait() + return trigger_publication_pb2.TriggerAcknowledgement( + publication_id=publication.publication_id, + disposition=( + trigger_publication_pb2.TRIGGER_ACKNOWLEDGEMENT_DISPOSITION_ACCEPTED_NEW + ), + ) + + with pytest.raises( + SfpTriggerPublicationLimitError, + match="exceeded the selected receive window", + ): + await _consume_publications( + call, + maximum_in_flight=_SELECTED_RECEIVE_WINDOW, + handler=blocked_accept, + ) + assert started == _SELECTED_RECEIVE_WINDOW + assert call.requests == [] + + +@pytest.mark.parametrize("status", list(grpc.StatusCode)) +def test_client_maps_every_grpc_status_to_stable_agent_error( + status: grpc.StatusCode, +) -> None: + if status in { + grpc.StatusCode.UNAVAILABLE, + grpc.StatusCode.CANCELLED, + grpc.StatusCode.DEADLINE_EXCEEDED, + }: + expected = SfpTriggerPublicationUnavailableError + elif status is grpc.StatusCode.FAILED_PRECONDITION: + expected = SfpTriggerPublicationStaleError + elif status is grpc.StatusCode.RESOURCE_EXHAUSTED: + expected = SfpTriggerPublicationLimitError + else: + expected = SfpTriggerPublicationProtocolError + + with pytest.raises(expected): + _raise_rpc_error(status) + + @pytest.mark.asyncio async def test_accepts_new_and_duplicate_publications(tmp_path: Path) -> None: publication = _publication() @@ -266,6 +565,56 @@ async def test_accepts_new_and_duplicate_publications(tmp_path: Path) -> None: await journal.close() +@pytest.mark.asyncio +async def test_publication_identity_is_scoped_to_provider( + tmp_path: Path, +) -> None: + publication = _publication( + publication_id="11111111-1111-4111-8111-111111111111" + ) + consumer = _ScriptedConsumer([]) + journal, store, _ = await _coordinator( + tmp_path / "agent.db", + consumer, + ) + first_lease = _lease() + second_lease = _lease() + first_context = TriggerAcceptanceContext( + provider_instance_id=first_lease.provider_instance_id, + endpoint_generation=first_lease.endpoint_generation, + session_id=first_lease.session_id, + manifest_revision=first_lease.manifest_revision, + ) + second_context = TriggerAcceptanceContext( + provider_instance_id=second_lease.provider_instance_id, + endpoint_generation=second_lease.endpoint_generation, + session_id=second_lease.session_id, + manifest_revision=second_lease.manifest_revision, + ) + + first = await store.accept(first_context, publication) + second = await store.accept(second_context, publication) + first_conflict = await store.accept( + first_context, + _publication( + publication_id=publication.publication_id, + payload={"unit": "postgresql.service"}, + ), + ) + second_replay = await store.accept(second_context, publication) + + assert first.decision is TriggerAcceptanceDecision.ACCEPTED_NEW + assert second.decision is TriggerAcceptanceDecision.ACCEPTED_NEW + assert first_conflict.decision is ( + TriggerAcceptanceDecision.IDEMPOTENCY_CONFLICT + ) + assert second_replay.decision is ( + TriggerAcceptanceDecision.ACCEPTED_DUPLICATE + ) + assert await store.pending_count() == _ISOLATED_PROVIDER_COUNT + await journal.close() + + @pytest.mark.asyncio async def test_rejects_undeclared_and_schema_invalid_publications( tmp_path: Path, From 84465e172a6a1a9288615a11978c592e6f66af9f Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Thu, 29 Oct 2026 11:24:23 +0200 Subject: [PATCH 3/6] fix(security): restrict agent journal permissions --- src/stateforge_agent/_journal/sqlite.py | 16 +++++++++- tests/test_sqlite_journal.py | 41 ++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/stateforge_agent/_journal/sqlite.py b/src/stateforge_agent/_journal/sqlite.py index 9b4e23c..b35b6fa 100644 --- a/src/stateforge_agent/_journal/sqlite.py +++ b/src/stateforge_agent/_journal/sqlite.py @@ -11,6 +11,7 @@ _Result = TypeVar("_Result") _BUSY_TIMEOUT_MILLISECONDS = 5000 +_JOURNAL_FILE_MODE = 0o600 class SqliteJournalError(RuntimeError): @@ -67,12 +68,25 @@ async def close(self) -> None: def _prepare(self) -> None: """Prepare the directory and durable SQLite mode synchronously.""" self._path.parent.mkdir(mode=0o750, parents=True, exist_ok=True) + self._secure_database_file() connection = self._connect() try: connection.execute("PRAGMA journal_mode = WAL") finally: connection.close() - os.chmod(self._path, 0o640) + + def _secure_database_file(self) -> None: + """Create or restrict the database before SQLite creates sidecars.""" + try: + descriptor = os.open( + self._path, + os.O_CREAT | os.O_EXCL | os.O_WRONLY, + _JOURNAL_FILE_MODE, + ) + except FileExistsError: + os.chmod(self._path, _JOURNAL_FILE_MODE) + else: + os.close(descriptor) def _transact( self, diff --git a/tests/test_sqlite_journal.py b/tests/test_sqlite_journal.py index 6fb92e1..4c6e9d0 100644 --- a/tests/test_sqlite_journal.py +++ b/tests/test_sqlite_journal.py @@ -3,6 +3,7 @@ import errno import os import sqlite3 +import stat from pathlib import Path import pytest @@ -12,6 +13,12 @@ _LARGE_PAYLOAD_BYTES = 1024 * 1024 _SHORT_BUSY_TIMEOUT_MILLISECONDS = 1 +_OWNER_ONLY_FILE_MODE = 0o600 + + +def _file_mode(path: Path) -> int: + """Return only the permission bits for a filesystem path.""" + return stat.S_IMODE(path.stat().st_mode) async def _open_journal(path: Path) -> SqliteJournal: @@ -34,6 +41,38 @@ def _payload_count(connection: sqlite3.Connection) -> int: return int(row[0]) +@pytest.mark.asyncio +async def test_open_creates_owner_only_journal(tmp_path: Path) -> None: + path = tmp_path / "journal" / "agent.db" + journal = SqliteJournal(path) + + await journal.open() + + assert _file_mode(path) == _OWNER_ONLY_FILE_MODE + assert await journal.transaction( + lambda _: ( + _file_mode(Path(f"{path}-wal")), + _file_mode(Path(f"{path}-shm")), + ) + ) == (_OWNER_ONLY_FILE_MODE, _OWNER_ONLY_FILE_MODE) + await journal.close() + + +@pytest.mark.asyncio +async def test_open_restricts_existing_permissive_journal( + tmp_path: Path, +) -> None: + path = tmp_path / "agent.db" + sqlite3.connect(path).close() + path.chmod(0o640) + journal = SqliteJournal(path) + + await journal.open() + + assert _file_mode(path) == _OWNER_ONLY_FILE_MODE + await journal.close() + + @pytest.mark.asyncio async def test_disk_full_rolls_back_the_complete_transaction( tmp_path: Path, @@ -80,7 +119,7 @@ def insert_payload(connection: sqlite3.Connection) -> None: assert isinstance(captured.value.__cause__, sqlite3.OperationalError) finally: path.parent.chmod(0o750) - path.chmod(0o640) + path.chmod(_OWNER_ONLY_FILE_MODE) assert await journal.transaction(_payload_count) == 0 await journal.close() From e1f49b820be4991563162833a339d385b79c2b94 Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Thu, 29 Oct 2026 11:28:35 +0200 Subject: [PATCH 4/6] fix(packaging): declare private agent journal directory --- packaging/rpm/SOURCES/stateforge-agent.tmpfiles | 2 ++ src/stateforge_agent/_journal/sqlite.py | 7 ++++++- tests/test_rpm_automation.py | 16 ++++++++++++++++ tests/test_sqlite_journal.py | 2 ++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packaging/rpm/SOURCES/stateforge-agent.tmpfiles b/packaging/rpm/SOURCES/stateforge-agent.tmpfiles index 085bf04..5683740 100644 --- a/packaging/rpm/SOURCES/stateforge-agent.tmpfiles +++ b/packaging/rpm/SOURCES/stateforge-agent.tmpfiles @@ -1,3 +1,5 @@ # Type Path Mode User Group Age Argument +d /var/lib/stateforge-agent 0750 stateforge-agent stateforge-agent - - +d /var/lib/stateforge-agent/journal 0700 stateforge-agent stateforge-agent - - d /run/stateforge-agent 0750 stateforge-agent stateforge-providers - - d /run/stateforge-agent/providers 3770 stateforge-agent stateforge-providers - - diff --git a/src/stateforge_agent/_journal/sqlite.py b/src/stateforge_agent/_journal/sqlite.py index b35b6fa..9b966ef 100644 --- a/src/stateforge_agent/_journal/sqlite.py +++ b/src/stateforge_agent/_journal/sqlite.py @@ -11,6 +11,7 @@ _Result = TypeVar("_Result") _BUSY_TIMEOUT_MILLISECONDS = 5000 +_JOURNAL_DIRECTORY_MODE = 0o700 _JOURNAL_FILE_MODE = 0o600 @@ -67,7 +68,11 @@ async def close(self) -> None: def _prepare(self) -> None: """Prepare the directory and durable SQLite mode synchronously.""" - self._path.parent.mkdir(mode=0o750, parents=True, exist_ok=True) + self._path.parent.mkdir( + mode=_JOURNAL_DIRECTORY_MODE, + parents=True, + exist_ok=True, + ) self._secure_database_file() connection = self._connect() try: diff --git a/tests/test_rpm_automation.py b/tests/test_rpm_automation.py index 48533d2..3f778fd 100644 --- a/tests/test_rpm_automation.py +++ b/tests/test_rpm_automation.py @@ -139,6 +139,22 @@ def test_packaged_service_restarts_only_recoverable_runtime_failures() -> None: assert "RestartPreventExitStatus=2" in service +def test_rpm_declares_private_persistent_journal_directory() -> None: + """Create the Agent journal with explicit package-owned permissions.""" + service = read( + ROOT / "packaging" / "rpm" / "SOURCES" / "stateforge-agent.service" + ) + tmpfiles = read( + ROOT / "packaging" / "rpm" / "SOURCES" / "stateforge-agent.tmpfiles" + ) + + assert "StateDirectory=stateforge-agent" in service + assert ( + "d /var/lib/stateforge-agent/journal 0700 " + "stateforge-agent stateforge-agent" + ) in tmpfiles + + def test_installed_test_consumes_rpm_and_packaged_provider_wheels() -> None: """Exercise the built RPM against an independently packaged Provider.""" workflow = read(WORKFLOWS / "rpm-installed-test.yml") diff --git a/tests/test_sqlite_journal.py b/tests/test_sqlite_journal.py index 4c6e9d0..024468c 100644 --- a/tests/test_sqlite_journal.py +++ b/tests/test_sqlite_journal.py @@ -13,6 +13,7 @@ _LARGE_PAYLOAD_BYTES = 1024 * 1024 _SHORT_BUSY_TIMEOUT_MILLISECONDS = 1 +_OWNER_ONLY_DIRECTORY_MODE = 0o700 _OWNER_ONLY_FILE_MODE = 0o600 @@ -48,6 +49,7 @@ async def test_open_creates_owner_only_journal(tmp_path: Path) -> None: await journal.open() + assert _file_mode(path.parent) == _OWNER_ONLY_DIRECTORY_MODE assert _file_mode(path) == _OWNER_ONLY_FILE_MODE assert await journal.transaction( lambda _: ( From 7e3b6159321a44d66fc96269ff8961762a21f4d4 Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Thu, 29 Oct 2026 11:49:52 +0200 Subject: [PATCH 5/6] refactor(journal): isolate storage backends --- docs/configuration.md | 6 +- src/stateforge_agent/_config/loading.py | 9 +- src/stateforge_agent/_config/models.py | 4 +- src/stateforge_agent/_journal/__init__.py | 16 +- src/stateforge_agent/_journal/base.py | 49 +++++ src/stateforge_agent/_journal/registry.py | 62 ++++++ src/stateforge_agent/_journal/sqlite.py | 44 +++- .../_journal/sqlite_trigger_publications.py | 200 ++++++++++++++++++ src/stateforge_agent/_runtime/daemon.py | 15 +- src/stateforge_agent/_runtime/layout.py | 5 - .../trigger_publication/coordinator.py | 4 +- .../_services/trigger_publication/store.py | 171 +-------------- tests/test_configuration.py | 5 + tests/test_daemon_provider_events.py | 33 ++- tests/test_journal_registry.py | 95 +++++++++ tests/test_runtime_layout.py | 3 - tests/test_trigger_publication.py | 6 +- tests/test_trigger_publication_store.py | 3 +- 18 files changed, 529 insertions(+), 201 deletions(-) create mode 100644 src/stateforge_agent/_journal/base.py create mode 100644 src/stateforge_agent/_journal/registry.py create mode 100644 src/stateforge_agent/_journal/sqlite_trigger_publications.py create mode 100644 tests/test_journal_registry.py diff --git a/docs/configuration.md b/docs/configuration.md index 43c9376..a80417b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -118,12 +118,16 @@ there; the Agent does not use this directory for Provider-specific data. | Field | Type | Packaged value | Validation and purpose | | --- | --- | --- | --- | -| `backend` | String | `sqlite` | Durable journal backend. `sqlite` is the only value supported by schema version 1. | +| `backend` | String | `sqlite` | Selects the durable journal implementation. `sqlite` selects the bundled SQLite implementation and is the only backend shipped in schema version 1. | | `maximum_pending_trigger_publications` | Integer | `10000` | Maximum accepted trigger publications retained for later SF-AOP forwarding. Must be greater than zero. When full, new publications receive a retryable capacity rejection. | The journal database is stored at the package-owned path `/var/lib/stateforge-agent/journal/agent.db`. +Journal implementations conform to storage-neutral Agent contracts. Runtime +services do not depend on SQLite classes or SQL; another backend can supply its +own journal and durable service stores under a different backend name. + The selected Backup Engine does not belong in this section. Backup Engines are operation tools selected through Control Plane policy rather than global Agent configuration. diff --git a/src/stateforge_agent/_config/loading.py b/src/stateforge_agent/_config/loading.py index 95bf22e..c1637ae 100644 --- a/src/stateforge_agent/_config/loading.py +++ b/src/stateforge_agent/_config/loading.py @@ -23,7 +23,6 @@ from stateforge_agent._config.models import ( AgentConfiguration, ControlPlaneConfiguration, - JournalBackend, JournalConfiguration, LogDestination, LogFormat, @@ -31,6 +30,7 @@ LogLevel, ProviderConfiguration, ) +from stateforge_agent._journal.registry import JOURNAL_BACKENDS ConfigValue: TypeAlias = ( str | int | float | bool | list["ConfigValue"] | dict[str, "ConfigValue"] @@ -407,7 +407,10 @@ def _validate_configuration( "providers.reconnect_initial_delay_seconds", ) - journal_backend = validator.choice(("journal", "backend"), {"sqlite"}) + journal_backend = validator.choice( + ("journal", "backend"), + set(JOURNAL_BACKENDS.backend_names), + ) maximum_pending_trigger_publications = validator.positive_integer( ("journal", "maximum_pending_trigger_publications") ) @@ -437,7 +440,7 @@ def _validate_configuration( trigger_max_in_flight_publications=trigger_max_in_flight, ), journal=JournalConfiguration( - backend=cast(JournalBackend, journal_backend), + backend=journal_backend, maximum_pending_trigger_publications=( maximum_pending_trigger_publications ), diff --git a/src/stateforge_agent/_config/models.py b/src/stateforge_agent/_config/models.py index 6e63ff3..a551929 100644 --- a/src/stateforge_agent/_config/models.py +++ b/src/stateforge_agent/_config/models.py @@ -13,7 +13,7 @@ ] LogFormat: TypeAlias = Literal["text", "json"] LogDestination: TypeAlias = Literal["stderr"] -JournalBackend: TypeAlias = Literal["sqlite"] +JournalBackendName: TypeAlias = str @dataclass(frozen=True, slots=True) @@ -40,7 +40,7 @@ class ProviderConfiguration: class JournalConfiguration: """Durable Agent journal settings.""" - backend: JournalBackend + backend: JournalBackendName maximum_pending_trigger_publications: int diff --git a/src/stateforge_agent/_journal/__init__.py b/src/stateforge_agent/_journal/__init__.py index 1348abe..1441543 100644 --- a/src/stateforge_agent/_journal/__init__.py +++ b/src/stateforge_agent/_journal/__init__.py @@ -1,8 +1,22 @@ """Agent-owned durable journal infrastructure.""" +from stateforge_agent._journal.base import Journal, JournalError +from stateforge_agent._journal.registry import ( + JOURNAL_BACKENDS, + JournalBackendRegistry, + create_journal, +) from stateforge_agent._journal.sqlite import ( SqliteJournal, SqliteJournalError, ) -__all__ = ["SqliteJournal", "SqliteJournalError"] +__all__ = [ + "JOURNAL_BACKENDS", + "Journal", + "JournalBackendRegistry", + "JournalError", + "SqliteJournal", + "SqliteJournalError", + "create_journal", +] diff --git a/src/stateforge_agent/_journal/base.py b/src/stateforge_agent/_journal/base.py new file mode 100644 index 0000000..1042c3a --- /dev/null +++ b/src/stateforge_agent/_journal/base.py @@ -0,0 +1,49 @@ +"""Define storage-neutral Agent journal contracts.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import TYPE_CHECKING, ClassVar, Self + +if TYPE_CHECKING: + from stateforge_agent._config.models import JournalConfiguration + from stateforge_agent._sfp._services.trigger_publication.store import ( + TriggerPublicationStore, + ) + + +class JournalError(RuntimeError): + """Report that durable Agent state could not complete an operation.""" + + +class Journal(ABC): + """Own durable Agent state without exposing a storage vendor.""" + + backend: ClassVar[str] + + @classmethod + @abstractmethod + def from_configuration( + cls, + configuration: JournalConfiguration, + *, + state_directory: Path, + ) -> Self: + """Construct this backend from validated application settings.""" + + @abstractmethod + async def open(self) -> None: + """Open the journal without replacing existing state.""" + + @abstractmethod + async def close(self) -> None: + """Close the journal after current work has completed.""" + + @abstractmethod + def trigger_publications( + self, + *, + maximum_pending_publications: int, + ) -> TriggerPublicationStore: + """Create the backend's durable Trigger Publication store.""" diff --git a/src/stateforge_agent/_journal/registry.py b/src/stateforge_agent/_journal/registry.py new file mode 100644 index 0000000..e8a153c --- /dev/null +++ b/src/stateforge_agent/_journal/registry.py @@ -0,0 +1,62 @@ +"""Resolve configured journal backend names to implementation classes.""" + +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path + +from stateforge_agent._config.models import JournalConfiguration +from stateforge_agent._journal.base import Journal +from stateforge_agent._journal.sqlite import SqliteJournal + + +class JournalBackendRegistry: + """Map stable configuration names to journal implementation classes.""" + + def __init__(self, journal_classes: Iterable[type[Journal]]) -> None: + """Register journal classes and reject ambiguous backend names.""" + self._journal_classes: dict[str, type[Journal]] = {} + for journal_class in journal_classes: + if journal_class.backend in self._journal_classes: + raise ValueError( + f"Duplicate journal backend: {journal_class.backend}" + ) + self._journal_classes[journal_class.backend] = journal_class + + @property + def backend_names(self) -> frozenset[str]: + """Return the configured names accepted by this registry.""" + return frozenset(self._journal_classes) + + def create( + self, + configuration: JournalConfiguration, + *, + state_directory: Path, + ) -> Journal: + """Construct the journal class selected by configuration.""" + try: + journal_class = self._journal_classes[configuration.backend] + except KeyError as error: + raise ValueError( + f"Unsupported journal backend: {configuration.backend}" + ) from error + return journal_class.from_configuration( + configuration, + state_directory=state_directory, + ) + + +JOURNAL_BACKENDS = JournalBackendRegistry((SqliteJournal,)) + + +def create_journal( + configuration: JournalConfiguration, + *, + state_directory: Path, +) -> Journal: + """Construct a journal through the application backend registry.""" + return JOURNAL_BACKENDS.create( + configuration, + state_directory=state_directory, + ) diff --git a/src/stateforge_agent/_journal/sqlite.py b/src/stateforge_agent/_journal/sqlite.py index 9b966ef..c76ddd5 100644 --- a/src/stateforge_agent/_journal/sqlite.py +++ b/src/stateforge_agent/_journal/sqlite.py @@ -7,7 +7,16 @@ import sqlite3 from collections.abc import Callable from pathlib import Path -from typing import TypeVar +from typing import Self, TypeVar + +from stateforge_agent._config.models import JournalConfiguration +from stateforge_agent._journal.base import Journal, JournalError +from stateforge_agent._journal.sqlite_trigger_publications import ( + SqliteTriggerPublicationStore, +) +from stateforge_agent._sfp._services.trigger_publication.store import ( + TriggerPublicationStore, +) _Result = TypeVar("_Result") _BUSY_TIMEOUT_MILLISECONDS = 5000 @@ -15,13 +24,31 @@ _JOURNAL_FILE_MODE = 0o600 -class SqliteJournalError(RuntimeError): +class SqliteJournalError(JournalError): """Report that the Agent journal could not complete an operation.""" -class SqliteJournal: +class SqliteJournal(Journal): """Own one bounded SQLite transaction boundary for Agent state.""" + backend = "sqlite" + + @classmethod + def from_configuration( + cls, + configuration: JournalConfiguration, + *, + state_directory: Path, + ) -> Self: + """Construct SQLite storage under the Agent-owned state directory.""" + del configuration + return cls(cls.database_path(state_directory)) + + @staticmethod + def database_path(state_directory: Path) -> Path: + """Return SQLite's backend-specific database path.""" + return state_directory / "journal" / "agent.db" + def __init__(self, path: Path) -> None: """Bind the journal to its platform-owned persistent path.""" self._path = path @@ -66,6 +93,17 @@ async def close(self) -> None: async with self._lock: self._opened = False + def trigger_publications( + self, + *, + maximum_pending_publications: int, + ) -> TriggerPublicationStore: + """Create the SQLite Trigger Publication adapter.""" + return SqliteTriggerPublicationStore( + self, + maximum_pending_publications=maximum_pending_publications, + ) + def _prepare(self) -> None: """Prepare the directory and durable SQLite mode synchronously.""" self._path.parent.mkdir( diff --git a/src/stateforge_agent/_journal/sqlite_trigger_publications.py b/src/stateforge_agent/_journal/sqlite_trigger_publications.py new file mode 100644 index 0000000..c9a78ea --- /dev/null +++ b/src/stateforge_agent/_journal/sqlite_trigger_publications.py @@ -0,0 +1,200 @@ +"""Persist Trigger Publication responsibility in SQLite.""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Protocol, TypeVar + +from stateforge_protocol.sfp.trigger_publication.v1 import ( + trigger_publication_pb2, + validation, +) + +from stateforge_agent._sfp._services.trigger_publication.store import ( + TriggerAcceptanceContext, + TriggerAcceptanceDecision, + TriggerAcceptanceResult, + TriggerPublicationStore, +) + +_SCHEMA_COMPONENT = "trigger_publication" +_SCHEMA_VERSION = 1 +_PENDING_STATE = "pending" +_Result = TypeVar("_Result") + + +class _SqliteTransactions(Protocol): + """Expose only the SQLite transaction boundary needed by this adapter.""" + + async def transaction( + self, + operation: Callable[[sqlite3.Connection], _Result], + ) -> _Result: + """Run one serialized SQLite transaction.""" + + +class SqliteTriggerPublicationStore(TriggerPublicationStore): + """Own SQLite trigger idempotency and pending-forward responsibility.""" + + def __init__( + self, + journal: _SqliteTransactions, + *, + maximum_pending_publications: int, + ) -> None: + """Configure a bounded SQLite trigger responsibility store.""" + if maximum_pending_publications <= 0: + raise ValueError("maximum_pending_publications must be positive") + self._journal = journal + self._maximum_pending_publications = maximum_pending_publications + + async def initialize(self) -> None: + """Create the first trigger journal schema or verify its version.""" + await self._journal.transaction(self._initialize) + + async def accept( + self, + context: TriggerAcceptanceContext, + publication: trigger_publication_pb2.TriggerPublication, + ) -> TriggerAcceptanceResult: + """Atomically assume responsibility or classify an existing event.""" + fingerprint = validation.compute_publication_content_fingerprint( + publication + ) + serialized = publication.SerializeToString(deterministic=True) + accepted_at = datetime.now(UTC).isoformat() + + def accept_transaction( + connection: sqlite3.Connection, + ) -> TriggerAcceptanceResult: + existing = connection.execute( + """ + SELECT content_fingerprint + FROM trigger_publications + WHERE provider_instance_id = ? AND publication_id = ? + """, + ( + str(context.provider_instance_id), + publication.publication_id, + ), + ).fetchone() + pending = self._pending_count(connection) + if existing is not None: + decision = ( + TriggerAcceptanceDecision.ACCEPTED_DUPLICATE + if existing[0] == fingerprint + else TriggerAcceptanceDecision.IDEMPOTENCY_CONFLICT + ) + return TriggerAcceptanceResult(decision, fingerprint, pending) + if pending >= self._maximum_pending_publications: + return TriggerAcceptanceResult( + TriggerAcceptanceDecision.CAPACITY_EXHAUSTED, + fingerprint, + pending, + ) + + connection.execute( + """ + INSERT INTO trigger_publications ( + provider_instance_id, + publication_id, + content_fingerprint, + publication, + trigger_identifier, + observed_at, + endpoint_generation, + session_id, + manifest_revision, + accepted_at, + forwarding_state + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + str(context.provider_instance_id), + publication.publication_id, + fingerprint, + serialized, + publication.trigger_identifier, + publication.observed_at.ToJsonString(), + context.endpoint_generation, + context.session_id, + context.manifest_revision, + accepted_at, + _PENDING_STATE, + ), + ) + return TriggerAcceptanceResult( + TriggerAcceptanceDecision.ACCEPTED_NEW, + fingerprint, + pending + 1, + ) + + return await self._journal.transaction(accept_transaction) + + async def pending_count(self) -> int: + """Return the number of events awaiting the future SF-AOP boundary.""" + return await self._journal.transaction(self._pending_count) + + @staticmethod + def _initialize(connection: sqlite3.Connection) -> None: + """Create and verify the trigger-owned schema.""" + connection.execute( + """ + CREATE TABLE IF NOT EXISTS journal_schemas ( + component TEXT PRIMARY KEY, + version INTEGER NOT NULL CHECK (version > 0) + ) WITHOUT ROWID + """ + ) + row = connection.execute( + "SELECT version FROM journal_schemas WHERE component = ?", + (_SCHEMA_COMPONENT,), + ).fetchone() + if row is not None and row[0] != _SCHEMA_VERSION: + raise sqlite3.DatabaseError( + "Unsupported Trigger Publication journal schema" + ) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS trigger_publications ( + provider_instance_id TEXT NOT NULL, + publication_id TEXT NOT NULL, + content_fingerprint TEXT NOT NULL, + publication BLOB NOT NULL, + trigger_identifier TEXT NOT NULL, + observed_at TEXT NOT NULL, + endpoint_generation INTEGER NOT NULL + CHECK (endpoint_generation > 0), + session_id TEXT NOT NULL, + manifest_revision TEXT NOT NULL, + accepted_at TEXT NOT NULL, + forwarding_state TEXT NOT NULL + CHECK (forwarding_state = 'pending'), + PRIMARY KEY (provider_instance_id, publication_id) + ) WITHOUT ROWID + """ + ) + connection.execute( + """ + INSERT OR IGNORE INTO journal_schemas (component, version) + VALUES (?, ?) + """, + (_SCHEMA_COMPONENT, _SCHEMA_VERSION), + ) + + @staticmethod + def _pending_count(connection: sqlite3.Connection) -> int: + """Count pending rows without reading their serialized content.""" + row = connection.execute( + """ + SELECT COUNT(*) + FROM trigger_publications + WHERE forwarding_state = ? + """, + (_PENDING_STATE,), + ).fetchone() + if row is None: + raise sqlite3.DatabaseError("Trigger publication count is absent") + return int(row[0]) diff --git a/src/stateforge_agent/_runtime/daemon.py b/src/stateforge_agent/_runtime/daemon.py index 4275b00..f424800 100644 --- a/src/stateforge_agent/_runtime/daemon.py +++ b/src/stateforge_agent/_runtime/daemon.py @@ -14,7 +14,7 @@ from stateforge_agent import __version__ from stateforge_agent._config.models import AgentConfiguration -from stateforge_agent._journal import SqliteJournal, SqliteJournalError +from stateforge_agent._journal import JournalError, create_journal from stateforge_agent._process import ProcessExitCode from stateforge_agent._runtime.layout import ( SYSTEM_RUNTIME_LAYOUT, @@ -42,9 +42,6 @@ from stateforge_agent._sfp._services.trigger_publication.coordinator import ( TriggerPublicationCoordinator, ) -from stateforge_agent._sfp._services.trigger_publication.store import ( - TriggerPublicationStore, -) _LOGGER = logging.getLogger(__name__) _SUPPORTED_SFP_VERSIONS = (SfpVersion(major=1, minor=0),) @@ -74,9 +71,11 @@ async def _run_agent( monitor = ProviderDirectoryMonitor( runtime_layout.provider_discovery_directory ) - journal = SqliteJournal(runtime_layout.journal_path) - trigger_store = TriggerPublicationStore( - journal, + journal = create_journal( + configuration.journal, + state_directory=runtime_layout.state_directory, + ) + trigger_store = journal.trigger_publications( maximum_pending_publications=( configuration.journal.maximum_pending_trigger_publications ), @@ -84,7 +83,7 @@ async def _run_agent( try: await journal.open() await trigger_store.initialize() - except SqliteJournalError: + except JournalError: _LOGGER.exception("Agent journal initialization failed") for installed_signal in installed_signals: loop.remove_signal_handler(installed_signal) diff --git a/src/stateforge_agent/_runtime/layout.py b/src/stateforge_agent/_runtime/layout.py index 416fd55..831bd79 100644 --- a/src/stateforge_agent/_runtime/layout.py +++ b/src/stateforge_agent/_runtime/layout.py @@ -16,11 +16,6 @@ def staging_directory(self) -> Path: """Return the SDK-managed operation staging directory.""" return self.state_directory / "staging" - @property - def journal_path(self) -> Path: - """Return the path of the durable Agent journal database.""" - return self.state_directory / "journal" / "agent.db" - @property def control_socket_path(self) -> Path: """Return the path of the local Agent control socket.""" diff --git a/src/stateforge_agent/_sfp/_services/trigger_publication/coordinator.py b/src/stateforge_agent/_sfp/_services/trigger_publication/coordinator.py index ff65db2..5de0062 100644 --- a/src/stateforge_agent/_sfp/_services/trigger_publication/coordinator.py +++ b/src/stateforge_agent/_sfp/_services/trigger_publication/coordinator.py @@ -16,7 +16,7 @@ validation, ) -from stateforge_agent._journal import SqliteJournalError +from stateforge_agent._journal import JournalError from stateforge_agent._sfp._services.trigger_publication.client import ( SfpTriggerPublicationProtocolError, TriggerPublicationClient, @@ -143,7 +143,7 @@ async def _accept( # noqa: PLR0911 ), publication, ) - except SqliteJournalError: + except JournalError: return _rejection( publication, trigger_publication_pb2.TRIGGER_REJECTION_REASON_DELIVERY_RESPONSIBILITY_UNAVAILABLE, diff --git a/src/stateforge_agent/_sfp/_services/trigger_publication/store.py b/src/stateforge_agent/_sfp/_services/trigger_publication/store.py index d9797f4..2593049 100644 --- a/src/stateforge_agent/_sfp/_services/trigger_publication/store.py +++ b/src/stateforge_agent/_sfp/_services/trigger_publication/store.py @@ -1,24 +1,16 @@ -"""Durably accept Provider trigger publications before acknowledgement.""" +"""Define storage-neutral durable Trigger Publication contracts.""" from __future__ import annotations -import sqlite3 +from abc import ABC, abstractmethod from dataclasses import dataclass -from datetime import UTC, datetime from enum import StrEnum from uuid import UUID from stateforge_protocol.sfp.trigger_publication.v1 import ( trigger_publication_pb2, - validation, ) -from stateforge_agent._journal import SqliteJournal - -_SCHEMA_COMPONENT = "trigger_publication" -_SCHEMA_VERSION = 1 -_PENDING_STATE = "pending" - class TriggerAcceptanceDecision(StrEnum): """Describe one durable idempotency decision.""" @@ -48,166 +40,21 @@ class TriggerAcceptanceContext: manifest_revision: str -class TriggerPublicationStore: - """Own durable trigger idempotency and pending-forward responsibility.""" - - def __init__( - self, - journal: SqliteJournal, - *, - maximum_pending_publications: int, - ) -> None: - """Configure a bounded trigger responsibility store.""" - if maximum_pending_publications <= 0: - raise ValueError("maximum_pending_publications must be positive") - self._journal = journal - self._maximum_pending_publications = maximum_pending_publications +class TriggerPublicationStore(ABC): + """Persist trigger responsibility without exposing a storage vendor.""" + @abstractmethod async def initialize(self) -> None: - """Create the first trigger journal schema or verify its version.""" - await self._journal.transaction(self._initialize) + """Prepare durable Trigger Publication state.""" + @abstractmethod async def accept( self, context: TriggerAcceptanceContext, publication: trigger_publication_pb2.TriggerPublication, ) -> TriggerAcceptanceResult: - """Atomically assume responsibility or classify an existing event.""" - fingerprint = validation.compute_publication_content_fingerprint( - publication - ) - serialized = publication.SerializeToString(deterministic=True) - accepted_at = datetime.now(UTC).isoformat() - - def accept_transaction( - connection: sqlite3.Connection, - ) -> TriggerAcceptanceResult: - existing = connection.execute( - """ - SELECT content_fingerprint - FROM trigger_publications - WHERE provider_instance_id = ? AND publication_id = ? - """, - ( - str(context.provider_instance_id), - publication.publication_id, - ), - ).fetchone() - pending = self._pending_count(connection) - if existing is not None: - decision = ( - TriggerAcceptanceDecision.ACCEPTED_DUPLICATE - if existing[0] == fingerprint - else TriggerAcceptanceDecision.IDEMPOTENCY_CONFLICT - ) - return TriggerAcceptanceResult(decision, fingerprint, pending) - if pending >= self._maximum_pending_publications: - return TriggerAcceptanceResult( - TriggerAcceptanceDecision.CAPACITY_EXHAUSTED, - fingerprint, - pending, - ) - - connection.execute( - """ - INSERT INTO trigger_publications ( - provider_instance_id, - publication_id, - content_fingerprint, - publication, - trigger_identifier, - observed_at, - endpoint_generation, - session_id, - manifest_revision, - accepted_at, - forwarding_state - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, - ( - str(context.provider_instance_id), - publication.publication_id, - fingerprint, - serialized, - publication.trigger_identifier, - publication.observed_at.ToJsonString(), - context.endpoint_generation, - context.session_id, - context.manifest_revision, - accepted_at, - _PENDING_STATE, - ), - ) - return TriggerAcceptanceResult( - TriggerAcceptanceDecision.ACCEPTED_NEW, - fingerprint, - pending + 1, - ) - - return await self._journal.transaction(accept_transaction) + """Atomically assume responsibility or classify existing content.""" + @abstractmethod async def pending_count(self) -> int: """Return the number of events awaiting the future SF-AOP boundary.""" - return await self._journal.transaction(self._pending_count) - - @staticmethod - def _initialize(connection: sqlite3.Connection) -> None: - """Create and verify the trigger-owned schema.""" - connection.execute( - """ - CREATE TABLE IF NOT EXISTS journal_schemas ( - component TEXT PRIMARY KEY, - version INTEGER NOT NULL CHECK (version > 0) - ) WITHOUT ROWID - """ - ) - row = connection.execute( - "SELECT version FROM journal_schemas WHERE component = ?", - (_SCHEMA_COMPONENT,), - ).fetchone() - if row is not None and row[0] != _SCHEMA_VERSION: - raise sqlite3.DatabaseError( - "Unsupported Trigger Publication journal schema" - ) - connection.execute( - """ - CREATE TABLE IF NOT EXISTS trigger_publications ( - provider_instance_id TEXT NOT NULL, - publication_id TEXT NOT NULL, - content_fingerprint TEXT NOT NULL, - publication BLOB NOT NULL, - trigger_identifier TEXT NOT NULL, - observed_at TEXT NOT NULL, - endpoint_generation INTEGER NOT NULL - CHECK (endpoint_generation > 0), - session_id TEXT NOT NULL, - manifest_revision TEXT NOT NULL, - accepted_at TEXT NOT NULL, - forwarding_state TEXT NOT NULL - CHECK (forwarding_state = 'pending'), - PRIMARY KEY (provider_instance_id, publication_id) - ) WITHOUT ROWID - """ - ) - connection.execute( - """ - INSERT OR IGNORE INTO journal_schemas (component, version) - VALUES (?, ?) - """, - (_SCHEMA_COMPONENT, _SCHEMA_VERSION), - ) - - @staticmethod - def _pending_count(connection: sqlite3.Connection) -> int: - """Count pending rows without reading their serialized content.""" - row = connection.execute( - """ - SELECT COUNT(*) - FROM trigger_publications - WHERE forwarding_state = ? - """, - (_PENDING_STATE,), - ).fetchone() - if row is None: - raise sqlite3.DatabaseError("Trigger publication count is absent") - return int(row[0]) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index df293ab..9ca3285 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -146,6 +146,11 @@ def test_rejects_unknown_key_with_defining_source(tmp_path: Path) -> None: 'ca_certificate = "etc/pki/tls/certs/ca-bundle.crt"', "must be an absolute path", ), + ( + 'backend = "sqlite"', + 'backend = "unknown"', + "must be one of: sqlite", + ), ], ) def test_rejects_invalid_values( diff --git a/tests/test_daemon_provider_events.py b/tests/test_daemon_provider_events.py index 26ca3e7..c8491d7 100644 --- a/tests/test_daemon_provider_events.py +++ b/tests/test_daemon_provider_events.py @@ -18,6 +18,7 @@ LoggingConfiguration, ProviderConfiguration, ) +from stateforge_agent._journal import SqliteJournal from stateforge_agent._process import ProcessExitCode from stateforge_agent._runtime import daemon from stateforge_agent._runtime.daemon import ( @@ -198,19 +199,39 @@ async def test_agent_runtime_starts_and_stops_cleanly( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: + selected_backends: list[tuple[JournalConfiguration, Path]] = [] + create_journal = daemon.create_journal + def stop_after_start(loop, stop_event): loop.call_soon(stop_event.set) return () + def record_backend( + configuration: JournalConfiguration, + *, + state_directory: Path, + ): + selected_backends.append((configuration, state_directory)) + return create_journal( + configuration, + state_directory=state_directory, + ) + monkeypatch.setattr(daemon, "_install_signal_handlers", stop_after_start) + monkeypatch.setattr(daemon, "create_journal", record_backend) + + runtime_layout = _runtime_layout(tmp_path) assert ( await daemon._run_agent( _configuration(tmp_path), - _runtime_layout(tmp_path), + runtime_layout, ) is ProcessExitCode.SUCCESS ) + assert selected_backends == [ + (_configuration(tmp_path).journal, runtime_layout.state_directory) + ] def test_run_agent_returns_daemon_result( @@ -269,8 +290,9 @@ async def test_corrupt_journal_returns_runtime_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: layout = _runtime_layout(tmp_path) - layout.journal_path.parent.mkdir(parents=True) - layout.journal_path.write_bytes(b"not a SQLite database") + journal_path = SqliteJournal.database_path(layout.state_directory) + journal_path.parent.mkdir(parents=True) + journal_path.write_bytes(b"not a SQLite database") monkeypatch.setattr(daemon, "_install_signal_handlers", lambda *_: ()) result = await daemon._run_agent(_configuration(tmp_path), layout) @@ -284,8 +306,9 @@ async def test_incompatible_trigger_schema_returns_runtime_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: layout = _runtime_layout(tmp_path) - layout.journal_path.parent.mkdir(parents=True) - connection = sqlite3.connect(layout.journal_path) + journal_path = SqliteJournal.database_path(layout.state_directory) + journal_path.parent.mkdir(parents=True) + connection = sqlite3.connect(journal_path) try: connection.execute( """ diff --git a/tests/test_journal_registry.py b/tests/test_journal_registry.py new file mode 100644 index 0000000..fa16fca --- /dev/null +++ b/tests/test_journal_registry.py @@ -0,0 +1,95 @@ +"""Tests for class-based journal backend selection.""" + +from pathlib import Path + +import pytest + +from stateforge_agent._config.models import JournalConfiguration +from stateforge_agent._journal import ( + JOURNAL_BACKENDS, + Journal, + JournalBackendRegistry, + SqliteJournal, + create_journal, +) +from stateforge_agent._sfp._services.trigger_publication.store import ( + TriggerPublicationStore, +) + + +class _AlternativeJournal(Journal): + """Prove another backend can satisfy the class boundary.""" + + backend = "alternative" + + def __init__(self, state_directory: Path) -> None: + self.state_directory = state_directory + + @classmethod + def from_configuration( + cls, + configuration: JournalConfiguration, + *, + state_directory: Path, + ) -> "_AlternativeJournal": + assert configuration.backend == cls.backend + return cls(state_directory) + + async def open(self) -> None: + return None + + async def close(self) -> None: + return None + + def trigger_publications( + self, + *, + maximum_pending_publications: int, + ) -> TriggerPublicationStore: + del maximum_pending_publications + raise NotImplementedError + + +def _configuration(backend: str = "sqlite") -> JournalConfiguration: + return JournalConfiguration( + backend=backend, + maximum_pending_trigger_publications=10, + ) + + +def test_configured_sqlite_backend_resolves_to_its_journal_class( + tmp_path: Path, +) -> None: + journal = create_journal( + _configuration(), + state_directory=tmp_path, + ) + + assert isinstance(journal, SqliteJournal) + assert journal.path == tmp_path / "journal" / "agent.db" + + +def test_registry_publishes_supported_backend_names() -> None: + assert JOURNAL_BACKENDS.backend_names == frozenset({"sqlite"}) + + +def test_registry_constructs_an_independent_backend_class( + tmp_path: Path, +) -> None: + registry = JournalBackendRegistry((_AlternativeJournal,)) + + journal = registry.create( + _configuration("alternative"), + state_directory=tmp_path, + ) + + assert isinstance(journal, _AlternativeJournal) + assert journal.state_directory == tmp_path + + +def test_registry_rejects_unknown_backend(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="Unsupported journal backend"): + create_journal( + _configuration("unknown"), + state_directory=tmp_path, + ) diff --git a/tests/test_runtime_layout.py b/tests/test_runtime_layout.py index 6a9fa18..93a68d3 100644 --- a/tests/test_runtime_layout.py +++ b/tests/test_runtime_layout.py @@ -14,9 +14,6 @@ def test_derives_agent_owned_paths_from_platform_directories() -> None: assert layout.staging_directory == Path( "/var/lib/stateforge-agent/staging" ) - assert layout.journal_path == Path( - "/var/lib/stateforge-agent/journal/agent.db" - ) assert layout.control_socket_path == Path( "/run/stateforge-agent/control.sock" ) diff --git a/tests/test_trigger_publication.py b/tests/test_trigger_publication.py index bde20fa..473652e 100644 --- a/tests/test_trigger_publication.py +++ b/tests/test_trigger_publication.py @@ -298,8 +298,7 @@ async def _coordinator( ]: journal = SqliteJournal(path) await journal.open() - store = TriggerPublicationStore( - journal, + store = journal.trigger_publications( maximum_pending_publications=capacity, ) await store.initialize() @@ -762,8 +761,7 @@ async def test_storage_failure_before_commit_is_retryable_and_not_accepted( assert acknowledgement.rejection.retryable restarted_journal = SqliteJournal(path) await restarted_journal.open() - restarted_store = TriggerPublicationStore( - restarted_journal, + restarted_store = restarted_journal.trigger_publications( maximum_pending_publications=10, ) await restarted_store.initialize() diff --git a/tests/test_trigger_publication_store.py b/tests/test_trigger_publication_store.py index 1d427fd..76b44a9 100644 --- a/tests/test_trigger_publication_store.py +++ b/tests/test_trigger_publication_store.py @@ -56,8 +56,7 @@ async def _store( ) -> tuple[SqliteJournal, TriggerPublicationStore]: journal = SqliteJournal(path) await journal.open() - store = TriggerPublicationStore( - journal, + store = journal.trigger_publications( maximum_pending_publications=capacity, ) await store.initialize() From 996f01e01deaa8056f1beb89630b854a45584179 Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Thu, 29 Oct 2026 12:00:45 +0200 Subject: [PATCH 6/6] fix(imports): break journal configuration cycle --- src/stateforge_agent/_journal/registry.py | 5 +++- src/stateforge_agent/_journal/sqlite.py | 6 +++-- tests/test_import_boundaries.py | 28 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 tests/test_import_boundaries.py diff --git a/src/stateforge_agent/_journal/registry.py b/src/stateforge_agent/_journal/registry.py index e8a153c..9f4955a 100644 --- a/src/stateforge_agent/_journal/registry.py +++ b/src/stateforge_agent/_journal/registry.py @@ -4,11 +4,14 @@ from collections.abc import Iterable from pathlib import Path +from typing import TYPE_CHECKING -from stateforge_agent._config.models import JournalConfiguration from stateforge_agent._journal.base import Journal from stateforge_agent._journal.sqlite import SqliteJournal +if TYPE_CHECKING: + from stateforge_agent._config.models import JournalConfiguration + class JournalBackendRegistry: """Map stable configuration names to journal implementation classes.""" diff --git a/src/stateforge_agent/_journal/sqlite.py b/src/stateforge_agent/_journal/sqlite.py index c76ddd5..0d01728 100644 --- a/src/stateforge_agent/_journal/sqlite.py +++ b/src/stateforge_agent/_journal/sqlite.py @@ -7,9 +7,8 @@ import sqlite3 from collections.abc import Callable from pathlib import Path -from typing import Self, TypeVar +from typing import TYPE_CHECKING, Self, TypeVar -from stateforge_agent._config.models import JournalConfiguration from stateforge_agent._journal.base import Journal, JournalError from stateforge_agent._journal.sqlite_trigger_publications import ( SqliteTriggerPublicationStore, @@ -18,6 +17,9 @@ TriggerPublicationStore, ) +if TYPE_CHECKING: + from stateforge_agent._config.models import JournalConfiguration + _Result = TypeVar("_Result") _BUSY_TIMEOUT_MILLISECONDS = 5000 _JOURNAL_DIRECTORY_MODE = 0o700 diff --git a/tests/test_import_boundaries.py b/tests/test_import_boundaries.py new file mode 100644 index 0000000..8af47c8 --- /dev/null +++ b/tests/test_import_boundaries.py @@ -0,0 +1,28 @@ +"""Verify application packages import from a fresh interpreter.""" + +import os +import subprocess +import sys +from pathlib import Path + + +def test_sfp_and_configuration_import_without_a_cycle() -> None: + repository = Path(__file__).resolve().parents[1] + environment = os.environ.copy() + environment["PYTHONPATH"] = str(repository / "src") + + result = subprocess.run( + ( + sys.executable, + "-c", + "from stateforge_agent._sfp import ProviderSessionManager; " + "from stateforge_agent._config import load_configuration", + ), + cwd=repository, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr