From e87488d1e2d33338918ce5af41e75f6221a4c29f Mon Sep 17 00:00:00 2001 From: Noam Alum Date: Thu, 5 Nov 2026 12:28:34 +0200 Subject: [PATCH] Add installed SFP recovery coverage --- .../provider/start/resolve-ready-endpoint.sh | 40 ++-- .../rpm/test-installed/recovery/action.yml | 187 ++++++++++++++++ .../recovery/assert-provider-restart.py | 202 ++++++++++++++++++ .github/workflows/rpm-installed-test.yml | 8 + .../operation_admission/coordinator.py | 5 + tests/test_operation_admission.py | 50 +++++ tests/test_rpm_automation.py | 40 +++- 7 files changed, 517 insertions(+), 15 deletions(-) create mode 100644 .github/actions/rpm/test-installed/recovery/action.yml create mode 100644 .github/actions/rpm/test-installed/recovery/assert-provider-restart.py diff --git a/.github/actions/provider/start/resolve-ready-endpoint.sh b/.github/actions/provider/start/resolve-ready-endpoint.sh index 3bf8c18..21b7bb8 100644 --- a/.github/actions/provider/start/resolve-ready-endpoint.sh +++ b/.github/actions/provider/start/resolve-ready-endpoint.sh @@ -18,8 +18,12 @@ if [[ ! "${PREVIOUS_GENERATION}" =~ ^[0-9]+$ ]]; then exit 2 fi -systemctl is-active --quiet "${UNIT_NAME}" -[[ -f "${IDENTITY_PATH}" ]] +if ! systemctl is-active --quiet "${UNIT_NAME}"; then + exit 1 +fi +if [[ ! -f "${IDENTITY_PATH}" ]]; then + exit 1 +fi identity_before="$(cat -- "${IDENTITY_PATH}")" provider_instance_id="$( @@ -34,10 +38,14 @@ identity_generation="$( | select(. == floor and . > $previous_generation)' \ <<< "${identity_before}" 2>/dev/null )" -[[ "${provider_instance_id}" =~ ${UUID_PATTERN} ]] +if [[ ! "${provider_instance_id}" =~ ${UUID_PATTERN} ]]; then + exit 1 +fi descriptor_path="${DISCOVERY_DIRECTORY}/${provider_instance_id}.sfp.json" -[[ -f "${descriptor_path}" ]] +if [[ ! -f "${descriptor_path}" ]]; then + exit 1 +fi descriptor_before="$(cat -- "${descriptor_path}")" descriptor_provider_instance_id="$( jq -er '.provider_instance_id | strings' <<< "${descriptor_before}" \ @@ -56,18 +64,24 @@ socket_path="$( 2>/dev/null )" -[[ "${descriptor_provider_instance_id}" == "${provider_instance_id}" ]] -[[ "${descriptor_generation}" == "${identity_generation}" ]] -[[ "${transport_type}" == "unix" ]] -[[ "${socket_path}" == \ - "${DISCOVERY_DIRECTORY}/${provider_instance_id}.${identity_generation}.sock" ]] -test -S "${socket_path}" +if [[ "${descriptor_provider_instance_id}" != "${provider_instance_id}" \ + || "${descriptor_generation}" != "${identity_generation}" \ + || "${transport_type}" != "unix" \ + || "${socket_path}" != \ + "${DISCOVERY_DIRECTORY}/${provider_instance_id}.${identity_generation}.sock" \ + || ! -S "${socket_path}" ]]; then + exit 1 +fi # Reject a mixed observation if either atomically published file changed while # its peer was being inspected. -[[ "$(cat -- "${IDENTITY_PATH}")" == "${identity_before}" ]] -[[ "$(cat -- "${descriptor_path}")" == "${descriptor_before}" ]] -systemctl is-active --quiet "${UNIT_NAME}" +if [[ "$(cat -- "${IDENTITY_PATH}")" != "${identity_before}" \ + || "$(cat -- "${descriptor_path}")" != "${descriptor_before}" ]]; then + exit 1 +fi +if ! systemctl is-active --quiet "${UNIT_NAME}"; then + exit 1 +fi printf '%s\t%s\t%s\n' \ "${provider_instance_id}" "${identity_generation}" "${socket_path}" diff --git a/.github/actions/rpm/test-installed/recovery/action.yml b/.github/actions/rpm/test-installed/recovery/action.yml new file mode 100644 index 0000000..497f166 --- /dev/null +++ b/.github/actions/rpm/test-installed/recovery/action.yml @@ -0,0 +1,187 @@ +name: Test installed SFP recovery +description: Recover a held operation and an unacknowledged trigger after Agent restart. + +inputs: + rpm-directory: + description: Directory containing the exact RPM artifact under test. + required: false + default: dist/rpm + evidence-directory: + description: Directory receiving this scenario's evidence. + required: false + default: dist/agent-system-test-evidence/sfp-recovery + fixture-directory: + description: Directory containing the exact Provider fixture bundle. + required: false + default: dist/provider-fixture + +runs: + using: composite + steps: + - name: Prepare installed Agent and exact dependency artifacts + id: setup + uses: ./.github/actions/rpm/test-installed/setup + with: + rpm-directory: ${{ inputs.rpm-directory }} + evidence-directory: ${{ inputs.evidence-directory }} + fixture-directory: ${{ inputs.fixture-directory }} + + - name: Start real Provider and withhold its trigger acknowledgement + id: provider + uses: ./.github/actions/provider/start + with: + container-name: ${{ steps.setup.outputs.container-name }} + fixture-path: ${{ steps.setup.outputs.fixture-path }} + wheel-directory: ${{ steps.setup.outputs.wheel-directory }} + provider-name: provider-a + provider-identity: stateforge.test.sfp_recovery + drop-trigger-acks-until-restart: "true" + + - name: Confirm Agent durably accepted the unacknowledged trigger + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + run: | + set -euo pipefail + event_log=/var/lib/stateforge-provider-fixtures/provider-a/events.log + deadline=$((SECONDS + 60)) + publication_id="" + while (( SECONDS < deadline )); do + publication_id="$(docker exec "${CONTAINER_NAME}" \ + sed -n \ + 's/^trigger-ack-dropped:\([0-9a-f-]*\):accepted_new$/\1/p' \ + "${event_log}" | tail -1)" + [[ -n "${publication_id}" ]] && break + sleep 1 + done + if [[ -z "${publication_id}" ]]; then + printf 'Provider did not observe the intentionally lost ACK.\n' >&2 + exit 1 + fi + printf '%s\n' "${publication_id}" \ + > "${EVIDENCE_DIRECTORY}/publication-id.txt" + + - name: Capture durable trigger evidence before recovery + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + expected-triggers: stateforge.test/runtime.started@1 + evidence-name: before-agent-restart + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Permit acknowledgement of the replayed publication + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + run: | + set -euo pipefail + docker exec "${CONTAINER_NAME}" \ + touch \ + /var/lib/stateforge-provider-fixtures/provider-a/accept-trigger-acks + + - name: Recover held operation through the installed Agent + uses: ./.github/actions/agent/assert-operation-lifecycle + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + provider-state-name: provider-a + evidence-directory: ${{ inputs.evidence-directory }}/operation-lifecycle + + - name: Confirm the same trigger was replayed and durably retained + uses: ./.github/actions/agent/assert-trigger-publication + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + expected-triggers: stateforge.test/runtime.started@1 + evidence-name: after-agent-restart + evidence-directory: ${{ inputs.evidence-directory }} + + - name: Verify replay identity and terminal acknowledgement + shell: bash + env: + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + run: | + set -euo pipefail + publication_id="$(cat "${EVIDENCE_DIRECTORY}/publication-id.txt")" + expected="trigger-ack-observed:${publication_id}:accepted_duplicate" + event_log=/var/lib/stateforge-provider-fixtures/provider-a/events.log + deadline=$((SECONDS + 60)) + observed="" + while (( SECONDS < deadline )); do + observed="$(docker exec "${CONTAINER_NAME}" \ + grep -Fx "${expected}" "${event_log}" || true)" + [[ -n "${observed}" ]] && break + sleep 1 + done + if [[ -z "${observed}" ]]; then + printf 'Provider did not receive the duplicate replay ACK.\n' >&2 + exit 1 + fi + jq -e --arg publication_id "${publication_id}" \ + '.publication_count == 1 + and .publications[0].publication_id == $publication_id' \ + "${EVIDENCE_DIRECTORY}/before-agent-restart.json" \ + "${EVIDENCE_DIRECTORY}/after-agent-restart.json" >/dev/null + jq -e --slurp \ + '.[0].publication_count == 1 + and .[1].publication_count == 1 + and .[0].publications[0].publication_id + == .[1].publications[0].publication_id + and .[0].publications[0].content_fingerprint + == .[1].publications[0].content_fingerprint' \ + "${EVIDENCE_DIRECTORY}/before-agent-restart.json" \ + "${EVIDENCE_DIRECTORY}/after-agent-restart.json" >/dev/null + printf '%s\n' "${observed}" \ + > "${EVIDENCE_DIRECTORY}/provider-replay-outcome.txt" + + - name: Lose Provider execution state and fence redispatch + id: provider-restart + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + CONTAINER_NAME: ${{ steps.setup.outputs.container-name }} + EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }} + PROVIDER_INSTANCE_ID: ${{ steps.provider.outputs.provider-instance-id }} + PROVIDER_STATE_NAME: provider-a + PROVIDER_UNIT_NAME: ${{ steps.provider.outputs.unit-name }} + run: | + set -euo pipefail + mkdir -p "${EVIDENCE_DIRECTORY}" + state_directory="/var/lib/stateforge-provider-fixtures/${PROVIDER_STATE_NAME}" + docker exec "${CONTAINER_NAME}" rm -f \ + "${state_directory}/lifecycle-release" + verifier_directory="/opt/stateforge-agent-tests/recovery" + docker exec "${CONTAINER_NAME}" mkdir -p "${verifier_directory}" + docker cp "${ACTION_PATH}/assert-provider-restart.py" \ + "${CONTAINER_NAME}:${verifier_directory}/assert.py" + docker exec "${CONTAINER_NAME}" /usr/bin/python3 \ + "${verifier_directory}/assert.py" \ + --provider-instance-id "${PROVIDER_INSTANCE_ID}" \ + --provider-state-name "${PROVIDER_STATE_NAME}" \ + --provider-unit-name "${PROVIDER_UNIT_NAME}" \ + > "${EVIDENCE_DIRECTORY}/provider-restart.json" + replacement_generation="$(jq -er \ + '.replacement_endpoint_generation' \ + "${EVIDENCE_DIRECTORY}/provider-restart.json")" + printf 'endpoint-generation=%s\n' "${replacement_generation}" \ + >> "${GITHUB_OUTPUT}" + + - name: Require Agent readiness on the replacement generation + uses: ./.github/actions/agent/assert-availability + with: + container-name: ${{ steps.setup.outputs.container-name }} + provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }} + endpoint-generation: ${{ steps.provider-restart.outputs.endpoint-generation }} + state: available + journal-cursor: ${{ steps.provider.outputs.agent-journal-cursor }} + evidence-directory: ${{ inputs.evidence-directory }}/provider-restart-availability + + - name: Preserve evidence and clean up + if: ${{ always() }} + uses: ./.github/actions/rpm/test-installed/cleanup + with: + container-name: ${{ steps.setup.outputs.container-name }} + evidence-directory: ${{ inputs.evidence-directory }} diff --git a/.github/actions/rpm/test-installed/recovery/assert-provider-restart.py b/.github/actions/rpm/test-installed/recovery/assert-provider-restart.py new file mode 100644 index 0000000..ce71c1c --- /dev/null +++ b/.github/actions/rpm/test-installed/recovery/assert-provider-restart.py @@ -0,0 +1,202 @@ +"""Verify Provider restart cannot redispatch work whose state was lost.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from datetime import UTC, datetime, timedelta +from importlib.metadata import version +from pathlib import Path +from uuid import UUID, uuid4 + +from stateforge_agent._journal import SqliteJournal +from stateforge_agent._sfp import ( + OperationAdmissionRequest, + ProviderAvailabilityState, + ProviderSessionManager, + SfpVersion, +) +from stateforge_agent._sfp._runtime.descriptor import ( + ProviderEndpointDescriptor, + load_provider_descriptor, +) +from stateforge_agent._sfp._services.operation_admission.coordinator import ( + OperationAdmissionCoordinator, +) +from stateforge_agent._sfp._services.operation_lifecycle.coordinator import ( + OperationLifecycleCoordinator, +) + +_AGENT_JOURNAL = Path("/var/lib/stateforge-agent/journal/agent.db") +_DISCOVERY_DIRECTORY = Path("/run/stateforge-agent/providers") +_STATE_DIRECTORY = Path("/var/lib/stateforge-provider-fixtures") +_OPERATION = "stateforge.test/lifecycle@1" + + +def _arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--provider-instance-id", type=UUID, required=True) + parser.add_argument("--provider-state-name", required=True) + parser.add_argument("--provider-unit-name", required=True) + return parser.parse_args() + + +async def _run( # noqa: PLR0915 - one focused installed recovery scenario + arguments: argparse.Namespace, +) -> dict[str, object]: + provider_id: UUID = arguments.provider_instance_id + descriptor_path = _DISCOVERY_DIRECTORY / f"{provider_id}.sfp.json" + + def descriptor_loader( + requested_provider_id: UUID, + ) -> ProviderEndpointDescriptor: + if requested_provider_id != provider_id: + raise AssertionError("Reconnect requested a different Provider") + return load_provider_descriptor( + descriptor_path, + discovery_directory=_DISCOVERY_DIRECTORY, + ) + + journal = SqliteJournal(_AGENT_JOURNAL) + await journal.open() + store = journal.operation_admissions(maximum_unresolved_attempts=32) + await store.initialize() + manager = ProviderSessionManager( + agent_runtime_id=uuid4(), + agent_software_version=version("stateforge-agent"), + supported_sfp_versions=(SfpVersion(major=1, minor=0),), + descriptor_loader=descriptor_loader, + reconnect_initial_delay_seconds=0.05, + reconnect_max_delay_seconds=0.2, + health_retry_max_delay_seconds=0.2, + capability_assessment_retry_initial_delay_seconds=0.05, + operation_admission=OperationAdmissionCoordinator(store), + operation_lifecycle=OperationLifecycleCoordinator( + store, + retry_initial_delay_seconds=0.05, + retry_max_delay_seconds=0.2, + ), + ) + state_directory = _STATE_DIRECTORY / arguments.provider_state_name + event_log = state_directory / "events.log" + try: + await manager.establish(descriptor_loader(provider_id)) + async with asyncio.timeout(15): + # Availability is observed through the asynchronous Health watcher. + while not manager.is_eligible(provider_id): # noqa: ASYNC110 + await asyncio.sleep(0.05) + + previous_session = manager.get(provider_id) + if previous_session is None: + raise AssertionError("Initial Provider session is absent") + operation = await manager.admit_operation( + OperationAdmissionRequest( + provider_instance_id=provider_id, + operation_identifier=_OPERATION, + payload={"value": "held"}, + execution_deadline=datetime.now(UTC) + timedelta(minutes=2), + execution_context={"scenario": "provider-process-restart"}, + ) + ) + async with asyncio.timeout(15): + while True: + status = await manager.operation_status( + provider_id, + operation.operation_id, + ) + if status is not None and status.state == ( + "OPERATION_STATE_RUNNING" + ): + break + await asyncio.sleep(0.05) + + restart = await asyncio.create_subprocess_exec( + "/usr/bin/systemctl", + "restart", + arguments.provider_unit_name, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _, restart_error = await restart.communicate() + if restart.returncode != 0: + raise AssertionError( + "Provider systemd restart failed: " + f"{restart_error.decode(errors='replace')}" + ) + + async with asyncio.timeout(20): + while True: + current_session = manager.get(provider_id) + availability = manager.availability_status(provider_id) + if ( + current_session is not None + and current_session.endpoint_generation + > previous_session.endpoint_generation + and current_session.provider_runtime_id + != previous_session.provider_runtime_id + and availability is not None + and availability.state + is ProviderAvailabilityState.AVAILABLE + ): + break + await asyncio.sleep(0.05) + + await asyncio.sleep(0.25) + final_status = await manager.operation_status( + provider_id, + operation.operation_id, + ) + dispatches = sum( + line.endswith(str(operation.operation_id)) + and line.startswith("operation:lifecycle:") + for line in event_log.read_text(encoding="utf-8").splitlines() + ) + if final_status is None or final_status.state != ( + "OPERATION_STATE_RUNNING" + ): + raise AssertionError( + "Agent did not preserve the last proven running state after " + "Provider state loss" + ) + if dispatches != 1: + raise AssertionError( + "Provider replacement redispatched previously admitted work" + ) + current_session = manager.get(provider_id) + if current_session is None: + raise AssertionError("Replacement Provider session disappeared") + return { + "agent_version": version("stateforge-agent"), + "protocol_version": version("stateforge-protocol"), + "provider_version": version("stateforge-provider"), + "operation_id": str(operation.operation_id), + "operation_state_after_provider_restart": final_status.state, + "provider_instance_id": str(provider_id), + "previous_endpoint_generation": ( + previous_session.endpoint_generation + ), + "replacement_endpoint_generation": ( + current_session.endpoint_generation + ), + "provider_runtime_changed": True, + "dispatch_count": dispatches, + "provider_state_retained": False, + "no_redispatch": True, + } + finally: + await manager.close() + await journal.close() + + +def main() -> None: + """Emit sanitized evidence from the installed packages.""" + sys.stdout.write( + json.dumps(asyncio.run(_run(_arguments())), indent=2, sort_keys=True) + + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/rpm-installed-test.yml b/.github/workflows/rpm-installed-test.yml index fe9e0cd..948e48a 100644 --- a/.github/workflows/rpm-installed-test.yml +++ b/.github/workflows/rpm-installed-test.yml @@ -109,6 +109,7 @@ jobs: - capability-limits - operation-admission - operation-lifecycle + - sfp-recovery - provider-availability - trigger-publication - trigger-storage-failure @@ -169,6 +170,13 @@ jobs: evidence-directory: dist/agent-system-test-evidence/${{ matrix.scenario }} fixture-directory: dist/provider-fixture + - name: Run SFP recovery scenario + if: ${{ matrix.scenario == 'sfp-recovery' }} + uses: ./.github/actions/rpm/test-installed/recovery + with: + evidence-directory: dist/agent-system-test-evidence/${{ matrix.scenario }} + fixture-directory: dist/provider-fixture + - name: Run multiple-Providers scenario if: ${{ matrix.scenario == 'multiple-providers' }} uses: ./.github/actions/rpm/test-installed/lifecycle/multiple-providers diff --git a/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py b/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py index 4c21495..6a03cf5 100644 --- a/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py +++ b/src/stateforge_agent/_sfp/_services/operation_admission/coordinator.py @@ -343,6 +343,11 @@ async def _require_live_deadline( ) -> None: if _deadline(attempt.invocation) > datetime.now(UTC): return + if attempt.attempt_count > 0: + raise OperationAdmissionUnresolvedError( + attempt.operation_id, + "OPERATION_DEADLINE_EXPIRED_WITHOUT_REMOTE_OUTCOME", + ) rejection = OperationRejection( operation_id=attempt.operation_id, reason="ADMISSION_ERROR_REASON_OPERATION_DEADLINE_EXPIRED", diff --git a/tests/test_operation_admission.py b/tests/test_operation_admission.py index f519fe5..78f7036 100644 --- a/tests/test_operation_admission.py +++ b/tests/test_operation_admission.py @@ -722,6 +722,56 @@ async def test_expired_execution_deadline_never_contacts_provider( assert client.requests == [] +@pytest.mark.asyncio +async def test_expired_deadline_after_transport_remains_indeterminate( + tmp_path: Path, +) -> None: + client = _ScriptedClient([]) + journal, coordinator, store = await _coordinator( + tmp_path / "agent.db", client + ) + lease = _lease() + invocation = _wire_request().invocation + invocation.execution_deadline.FromDatetime( + datetime.now(UTC) - timedelta(seconds=1) + ) + operation_id = UUID(invocation.operation_id) + fingerprint = validation.compute_request_fingerprint(invocation) + try: + await store.create_intent( + _PROVIDER_ID, + invocation, + fingerprint, + result_schema_json=b'{"type":"object"}', + ) + await store.record_transport_attempt( + _PROVIDER_ID, + operation_id, + endpoint_generation=lease.endpoint_generation, + session_id=lease.session_id, + ) + + with pytest.raises(OperationAdmissionUnresolvedError) as captured: + await coordinator.submit( + lease, + operation_id, + _assessment(lease), + assert_current=_current, + ) + persisted = await store.get(_PROVIDER_ID, operation_id) + finally: + await journal.close() + + assert captured.value.reason == ( + "OPERATION_DEADLINE_EXPIRED_WITHOUT_REMOTE_OUTCOME" + ) + assert persisted is not None + assert persisted.state is OperationAttemptState.UNRESOLVED + assert persisted.rejection is None + assert persisted.attempt_count == 1 + assert client.requests == [] + + @pytest.mark.asyncio async def test_post_accept_storage_failure_leaves_identity_unresolved( tmp_path: Path, diff --git a/tests/test_rpm_automation.py b/tests/test_rpm_automation.py index af5d00d..e1a17d2 100644 --- a/tests/test_rpm_automation.py +++ b/tests/test_rpm_automation.py @@ -10,7 +10,7 @@ DEPENDENCY_COUNT = 2 HEALTH_SCENARIO_PROVIDER_COUNT = 3 MANIFEST_ASSERTION_COUNT = 6 -AVAILABILITY_ASSERTION_COUNT = 13 +AVAILABILITY_ASSERTION_COUNT = 14 OPERATION_PROVIDER_COUNT = 2 @@ -458,6 +458,7 @@ def test_action_library_has_expected_domain_boundaries() -> None: "rpm/test-installed/lifecycle/multiple-providers", "rpm/test-installed/operations/admission", "rpm/test-installed/operations/lifecycle", + "rpm/test-installed/recovery", "rpm/test-installed/setup", "rpm/test-installed/smoke/manifest-exchange", "rpm/test-installed/triggers/publication", @@ -505,7 +506,12 @@ def test_installed_test_always_collects_and_cleans_up() -> None: def test_each_installed_scenario_is_an_independent_action() -> None: """Keep scenario behavior outside shared environment orchestration.""" scenario_root = ACTIONS / "rpm" / "test-installed" - scenarios = sorted(scenario_root.glob("*/*/action.yml")) + scenarios = sorted( + [ + *scenario_root.glob("*/*/action.yml"), + *scenario_root.glob("recovery/action.yml"), + ] + ) tiers = { scenario.relative_to(scenario_root).parts[0] for scenario in scenarios @@ -519,6 +525,7 @@ def test_each_installed_scenario_is_an_independent_action() -> None: "capability", "destructive", "triggers", + "recovery", } for scenario in scenarios: content = read(scenario) @@ -527,6 +534,35 @@ def test_each_installed_scenario_is_an_independent_action() -> None: assert "if: ${{ always() }}" in content +def test_installed_recovery_composes_operation_and_trigger_replay() -> None: + """Exercise both durable recovery responsibilities in one system run.""" + workflow = read(WORKFLOWS / "rpm-installed-test.yml") + scenario = read( + ACTIONS / "rpm" / "test-installed" / "recovery" / "action.yml" + ) + + assert "sfp-recovery" in workflow + assert "./.github/actions/rpm/test-installed/recovery" in workflow + assert 'drop-trigger-acks-until-restart: "true"' in scenario + assert "before-agent-restart" in scenario + assert "assert-operation-lifecycle" in scenario + assert "after-agent-restart" in scenario + assert "accepted_duplicate" in scenario + assert "content_fingerprint" in scenario + assert "assert-provider-restart.py" in scenario + assert "agent/assert-availability" in scenario + + provider_restart = read( + ACTIONS + / "rpm" + / "test-installed" + / "recovery" + / "assert-provider-restart.py" + ) + assert "provider_state_retained" in provider_restart + assert "no_redispatch" in provider_restart + + def test_installed_health_scenario_uses_real_artifacts_and_processes() -> None: """Keep A2 evidence on the RPM, Provider wheel, and shared SFP channel.""" workflow = read(WORKFLOWS / "rpm-installed-test.yml")