Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 27 additions & 13 deletions .github/actions/provider/start/resolve-ready-endpoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,12 @@ if [[ ! "${PREVIOUS_GENERATION}" =~ ^[0-9]+$ ]]; then
exit 2
fi

systemctl is-active --quiet "${UNIT_NAME}"
[[ -f "${IDENTITY_PATH}" ]]
if ! systemctl is-active --quiet "${UNIT_NAME}"; then
exit 1
fi
if [[ ! -f "${IDENTITY_PATH}" ]]; then
exit 1
fi

identity_before="$(cat -- "${IDENTITY_PATH}")"
provider_instance_id="$(
Expand All @@ -34,10 +38,14 @@ identity_generation="$(
| select(. == floor and . > $previous_generation)' \
<<< "${identity_before}" 2>/dev/null
)"
[[ "${provider_instance_id}" =~ ${UUID_PATTERN} ]]
if [[ ! "${provider_instance_id}" =~ ${UUID_PATTERN} ]]; then
exit 1
fi

descriptor_path="${DISCOVERY_DIRECTORY}/${provider_instance_id}.sfp.json"
[[ -f "${descriptor_path}" ]]
if [[ ! -f "${descriptor_path}" ]]; then
exit 1
fi
descriptor_before="$(cat -- "${descriptor_path}")"
descriptor_provider_instance_id="$(
jq -er '.provider_instance_id | strings' <<< "${descriptor_before}" \
Expand All @@ -56,18 +64,24 @@ socket_path="$(
2>/dev/null
)"

[[ "${descriptor_provider_instance_id}" == "${provider_instance_id}" ]]
[[ "${descriptor_generation}" == "${identity_generation}" ]]
[[ "${transport_type}" == "unix" ]]
[[ "${socket_path}" == \
"${DISCOVERY_DIRECTORY}/${provider_instance_id}.${identity_generation}.sock" ]]
test -S "${socket_path}"
if [[ "${descriptor_provider_instance_id}" != "${provider_instance_id}" \
|| "${descriptor_generation}" != "${identity_generation}" \
|| "${transport_type}" != "unix" \
|| "${socket_path}" != \
"${DISCOVERY_DIRECTORY}/${provider_instance_id}.${identity_generation}.sock" \
|| ! -S "${socket_path}" ]]; then
exit 1
fi

# Reject a mixed observation if either atomically published file changed while
# its peer was being inspected.
[[ "$(cat -- "${IDENTITY_PATH}")" == "${identity_before}" ]]
[[ "$(cat -- "${descriptor_path}")" == "${descriptor_before}" ]]
systemctl is-active --quiet "${UNIT_NAME}"
if [[ "$(cat -- "${IDENTITY_PATH}")" != "${identity_before}" \
|| "$(cat -- "${descriptor_path}")" != "${descriptor_before}" ]]; then
exit 1
fi
if ! systemctl is-active --quiet "${UNIT_NAME}"; then
exit 1
fi

printf '%s\t%s\t%s\n' \
"${provider_instance_id}" "${identity_generation}" "${socket_path}"
187 changes: 187 additions & 0 deletions .github/actions/rpm/test-installed/recovery/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
name: Test installed SFP recovery
description: Recover a held operation and an unacknowledged trigger after Agent restart.

inputs:
rpm-directory:
description: Directory containing the exact RPM artifact under test.
required: false
default: dist/rpm
evidence-directory:
description: Directory receiving this scenario's evidence.
required: false
default: dist/agent-system-test-evidence/sfp-recovery
fixture-directory:
description: Directory containing the exact Provider fixture bundle.
required: false
default: dist/provider-fixture

runs:
using: composite
steps:
- name: Prepare installed Agent and exact dependency artifacts
id: setup
uses: ./.github/actions/rpm/test-installed/setup
with:
rpm-directory: ${{ inputs.rpm-directory }}
evidence-directory: ${{ inputs.evidence-directory }}
fixture-directory: ${{ inputs.fixture-directory }}

- name: Start real Provider and withhold its trigger acknowledgement
id: provider
uses: ./.github/actions/provider/start
with:
container-name: ${{ steps.setup.outputs.container-name }}
fixture-path: ${{ steps.setup.outputs.fixture-path }}
wheel-directory: ${{ steps.setup.outputs.wheel-directory }}
provider-name: provider-a
provider-identity: stateforge.test.sfp_recovery
drop-trigger-acks-until-restart: "true"

- name: Confirm Agent durably accepted the unacknowledged trigger
shell: bash
env:
CONTAINER_NAME: ${{ steps.setup.outputs.container-name }}
EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }}
run: |
set -euo pipefail
event_log=/var/lib/stateforge-provider-fixtures/provider-a/events.log
deadline=$((SECONDS + 60))
publication_id=""
while (( SECONDS < deadline )); do
publication_id="$(docker exec "${CONTAINER_NAME}" \
sed -n \
's/^trigger-ack-dropped:\([0-9a-f-]*\):accepted_new$/\1/p' \
"${event_log}" | tail -1)"
[[ -n "${publication_id}" ]] && break
sleep 1
done
if [[ -z "${publication_id}" ]]; then
printf 'Provider did not observe the intentionally lost ACK.\n' >&2
exit 1
fi
printf '%s\n' "${publication_id}" \
> "${EVIDENCE_DIRECTORY}/publication-id.txt"

- name: Capture durable trigger evidence before recovery
uses: ./.github/actions/agent/assert-trigger-publication
with:
container-name: ${{ steps.setup.outputs.container-name }}
provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }}
expected-triggers: stateforge.test/runtime.started@1
evidence-name: before-agent-restart
evidence-directory: ${{ inputs.evidence-directory }}

- name: Permit acknowledgement of the replayed publication
shell: bash
env:
CONTAINER_NAME: ${{ steps.setup.outputs.container-name }}
run: |
set -euo pipefail
docker exec "${CONTAINER_NAME}" \
touch \
/var/lib/stateforge-provider-fixtures/provider-a/accept-trigger-acks

- name: Recover held operation through the installed Agent
uses: ./.github/actions/agent/assert-operation-lifecycle
with:
container-name: ${{ steps.setup.outputs.container-name }}
provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }}
provider-state-name: provider-a
evidence-directory: ${{ inputs.evidence-directory }}/operation-lifecycle

- name: Confirm the same trigger was replayed and durably retained
uses: ./.github/actions/agent/assert-trigger-publication
with:
container-name: ${{ steps.setup.outputs.container-name }}
provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }}
expected-triggers: stateforge.test/runtime.started@1
evidence-name: after-agent-restart
evidence-directory: ${{ inputs.evidence-directory }}

- name: Verify replay identity and terminal acknowledgement
shell: bash
env:
CONTAINER_NAME: ${{ steps.setup.outputs.container-name }}
EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }}
run: |
set -euo pipefail
publication_id="$(cat "${EVIDENCE_DIRECTORY}/publication-id.txt")"
expected="trigger-ack-observed:${publication_id}:accepted_duplicate"
event_log=/var/lib/stateforge-provider-fixtures/provider-a/events.log
deadline=$((SECONDS + 60))
observed=""
while (( SECONDS < deadline )); do
observed="$(docker exec "${CONTAINER_NAME}" \
grep -Fx "${expected}" "${event_log}" || true)"
[[ -n "${observed}" ]] && break
sleep 1
done
if [[ -z "${observed}" ]]; then
printf 'Provider did not receive the duplicate replay ACK.\n' >&2
exit 1
fi
jq -e --arg publication_id "${publication_id}" \
'.publication_count == 1
and .publications[0].publication_id == $publication_id' \
"${EVIDENCE_DIRECTORY}/before-agent-restart.json" \
"${EVIDENCE_DIRECTORY}/after-agent-restart.json" >/dev/null
jq -e --slurp \
'.[0].publication_count == 1
and .[1].publication_count == 1
and .[0].publications[0].publication_id
== .[1].publications[0].publication_id
and .[0].publications[0].content_fingerprint
== .[1].publications[0].content_fingerprint' \
"${EVIDENCE_DIRECTORY}/before-agent-restart.json" \
"${EVIDENCE_DIRECTORY}/after-agent-restart.json" >/dev/null
printf '%s\n' "${observed}" \
> "${EVIDENCE_DIRECTORY}/provider-replay-outcome.txt"

- name: Lose Provider execution state and fence redispatch
id: provider-restart
shell: bash
env:
ACTION_PATH: ${{ github.action_path }}
CONTAINER_NAME: ${{ steps.setup.outputs.container-name }}
EVIDENCE_DIRECTORY: ${{ inputs.evidence-directory }}
PROVIDER_INSTANCE_ID: ${{ steps.provider.outputs.provider-instance-id }}
PROVIDER_STATE_NAME: provider-a
PROVIDER_UNIT_NAME: ${{ steps.provider.outputs.unit-name }}
run: |
set -euo pipefail
mkdir -p "${EVIDENCE_DIRECTORY}"
state_directory="/var/lib/stateforge-provider-fixtures/${PROVIDER_STATE_NAME}"
docker exec "${CONTAINER_NAME}" rm -f \
"${state_directory}/lifecycle-release"
verifier_directory="/opt/stateforge-agent-tests/recovery"
docker exec "${CONTAINER_NAME}" mkdir -p "${verifier_directory}"
docker cp "${ACTION_PATH}/assert-provider-restart.py" \
"${CONTAINER_NAME}:${verifier_directory}/assert.py"
docker exec "${CONTAINER_NAME}" /usr/bin/python3 \
"${verifier_directory}/assert.py" \
--provider-instance-id "${PROVIDER_INSTANCE_ID}" \
--provider-state-name "${PROVIDER_STATE_NAME}" \
--provider-unit-name "${PROVIDER_UNIT_NAME}" \
> "${EVIDENCE_DIRECTORY}/provider-restart.json"
replacement_generation="$(jq -er \
'.replacement_endpoint_generation' \
"${EVIDENCE_DIRECTORY}/provider-restart.json")"
printf 'endpoint-generation=%s\n' "${replacement_generation}" \
>> "${GITHUB_OUTPUT}"

- name: Require Agent readiness on the replacement generation
uses: ./.github/actions/agent/assert-availability
with:
container-name: ${{ steps.setup.outputs.container-name }}
provider-instance-id: ${{ steps.provider.outputs.provider-instance-id }}
endpoint-generation: ${{ steps.provider-restart.outputs.endpoint-generation }}
state: available
journal-cursor: ${{ steps.provider.outputs.agent-journal-cursor }}
evidence-directory: ${{ inputs.evidence-directory }}/provider-restart-availability

- name: Preserve evidence and clean up
if: ${{ always() }}
uses: ./.github/actions/rpm/test-installed/cleanup
with:
container-name: ${{ steps.setup.outputs.container-name }}
evidence-directory: ${{ inputs.evidence-directory }}
Loading
Loading