From 5043225e6037aaa74b0739b9e44ab80ddb0baa50 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:07:10 +0000 Subject: [PATCH 1/2] Give request-ID refusals their own exit codes and qualify Server 3.13.0 A signal or update refused with HTTP 409 request_id_conflict now exits 9 (REQUEST_ID_CONFLICT) and request_id_ambiguous exits 10 (REQUEST_ID_AMBIGUOUS), with a request_id.ambiguous next step. Both are definite refusals that recorded nothing; other 409s keep exit 2. A legacy receipt (retained, intent not verified) is no longer shown as a plain "Intent Verified: no" match: the human output says the Server did not compare the command's intent. The PHP source reference mirrors the change so php-main parity holds, with new ambiguity parity cases. scripts/acceptance/command-keys/published_cohort.py is a self-contained installed cohort against a published native Server by digest: response loss with generated and explicit keys, concurrency, changed-intent conflicts, Unicode limits, closed/continued/fenced runs, namespaces, and synthetic legacy readback and ambiguity through the Rust SDK held-FD helper. published-command-keys.yml runs it for the released CLI and this checkout (durable-workflow/.github#216). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/published-command-keys.yml | 127 ++++ docs/cli-reference.md | 41 +- rust/parity/cases.json | 87 +++ rust/src/commands/workflow_message.rs | 11 +- rust/src/error.rs | 49 ++ rust/tests/workflow_message.rs | 185 ++++++ scripts/acceptance/command-keys/README.md | 34 + .../command-keys/published_cohort.py | 612 ++++++++++++++++++ src/Commands/BaseCommand.php | 25 +- src/Support/ExitCode.php | 6 + src/Support/ServerHttpException.php | 8 + tests/Commands/ExitCodePolicyTest.php | 10 + .../Commands/WorkflowCommandRequestIdTest.php | 79 ++- 13 files changed, 1250 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/published-command-keys.yml create mode 100644 scripts/acceptance/command-keys/published_cohort.py diff --git a/.github/workflows/published-command-keys.yml b/.github/workflows/published-command-keys.yml new file mode 100644 index 00000000..77fb2877 --- /dev/null +++ b/.github/workflows/published-command-keys.yml @@ -0,0 +1,127 @@ +name: Published command keys + +# Installed signal/update command-key cohort (durable-workflow/.github#216) +# against the published native Server image, by digest. The published job +# runs the released CLI binary; the source job runs this checkout's build +# and expects the request-ID exit codes (9 conflict, 10 ambiguous). + +on: + pull_request: + paths: + - '.github/workflows/published-command-keys.yml' + - 'scripts/acceptance/command-keys/published_cohort.py' + - 'rust/src/error.rs' + - 'rust/src/commands/workflow_message.rs' + workflow_dispatch: + +permissions: + contents: read + +env: + SERVER_INDEX: sha256:30ccaf2cea6810a92915a9eb389f1a63ea870818a74c9cc1a321379b07b2d25c + SERVER_SOURCE: 2d1244be4611e1bc4aa4f0d856c708bed17937b4 + SERVER_VERSION: 3.13.0 + CLI_VERSION: 2.10.0 + CLI_SHA256: dfd1d0417c5767c95a30b25296558b90e4ad12e7319487449b303076a2fd0a58 + PROXY_SOURCE: 117118c83093822469b3824ba9587ff4068ed29f + HELPER_SOURCE: 6280877bd8e4ebf5db0ff523239c4246230571db + +jobs: + cohort: + name: ${{ matrix.cli }} CLI, ${{ matrix.phase }} cells + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04 + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + cli: [published, source] + phase: [modern, legacy] + defaults: + run: + shell: bash --noprofile --norc -e -o pipefail {0} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.13' + - name: Bootstrap verified dependency trust + uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust + with: + clients: pip,python,cargo,curl,git + - name: Download the accepted shared inputs + run: | + umask 077 + inputs="$RUNNER_TEMP/command-key-inputs" + mkdir -m 700 "$inputs" + echo "INPUTS=$inputs" >> "$GITHUB_ENV" + fetch() { curl --fail --silent --show-error --location --proto '=https' --max-time 60 --max-filesize "$3" "$1" --output "$inputs/$2"; } + fetch "https://raw.githubusercontent.com/durable-workflow/.github/$PROXY_SOURCE/conformance/command-keys/committed_response_loss.py" committed_response_loss.py 65536 + fetch "https://raw.githubusercontent.com/durable-workflow/sdk-rust/$HELPER_SOURCE/scripts/acceptance/command-keys/sqlite_legacy.py" sqlite_legacy.py 131072 + fetch "https://raw.githubusercontent.com/durable-workflow/sdk-rust/$HELPER_SOURCE/scripts/acceptance/command-keys/command-key-legacy-v1-vectors.json" command-key-legacy-v1-vectors.json 131072 + # The cohort verifies each file against its accepted SHA256 before use. + - name: Download the published CLI + if: matrix.cli == 'published' + run: | + curl --fail --silent --show-error --location --proto '=https' --max-time 120 --max-filesize 67108864 \ + "https://github.com/durable-workflow/cli/releases/download/$CLI_VERSION/dw-linux-x86_64" --output "$INPUTS/dw" + echo "$CLI_SHA256 $INPUTS/dw" | sha256sum -c - + { + echo "CLI_BINARY=$INPUTS/dw" + echo "CLI_BINARY_SHA256=$CLI_SHA256" + echo "TYPED_EXIT=" + } >> "$GITHUB_ENV" + - name: Install Rust 1.99 + if: matrix.cli == 'source' + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable + with: + toolchain: 1.99.0 + - name: Build this checkout's CLI + if: matrix.cli == 'source' + working-directory: rust + run: | + cargo build --locked + cp target/debug/dw "$INPUTS/dw" + { + echo "CLI_BINARY=$INPUTS/dw" + echo "CLI_BINARY_SHA256=$(sha256sum "$INPUTS/dw" | cut -d' ' -f1)" + echo "TYPED_EXIT=--typed-exit-codes" + } >> "$GITHUB_ENV" + - name: Pull the published Server by digest + run: docker pull "ghcr.io/durable-workflow/server@$SERVER_INDEX" + - name: Run the ${{ matrix.phase }} cells + run: | + # TYPED_EXIT is empty for the published CLI. + # shellcheck disable=SC2086 + timeout --kill-after=30s 15m python3 -I scripts/acceptance/command-keys/published_cohort.py \ + --phase "${{ matrix.phase }}" $TYPED_EXIT \ + --cli "$CLI_BINARY" --cli-sha256 "$CLI_BINARY_SHA256" --inputs "$INPUTS" \ + --server-index "$SERVER_INDEX" --server-source "$SERVER_SOURCE" --server-version "$SERVER_VERSION" \ + --work-dir "$RUNNER_TEMP/cohort" --container-prefix "cli-command-keys-$GITHUB_RUN_ID" \ + --owner "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" | tee command-key-result.json + python3 -I -c 'import json,sys; r=json.loads(open("command-key-result.json").read().splitlines()[-1]); print("::notice::", r["phase"], r["outcome"], r["checks"], "checks; cleanup", r["cleanup"]); sys.exit(r["outcome"] != "pass")' + { + echo "### ${{ matrix.cli }} CLI, ${{ matrix.phase }} cells" + echo '```json' + tail -n 1 command-key-result.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: Remove any container this run created + if: always() + run: | + ids="$(docker ps --all --quiet --filter "label=durable-workflow.cli-command-keys=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT")" + if [ -n "$ids" ]; then docker rm --force --volumes $ids; fi + test -z "$(docker ps --all --quiet --filter "label=durable-workflow.cli-command-keys=$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT")" + - name: Keep the sanitized observations + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: command-keys-${{ matrix.cli }}-${{ matrix.phase }} + path: | + command-key-result.json + ${{ runner.temp }}/cohort/checks.json + ${{ runner.temp }}/cohort/invocations.json + retention-days: 7 + if-no-files-found: warn diff --git a/docs/cli-reference.md b/docs/cli-reference.md index ebbca450..4aa0db48 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -538,22 +538,27 @@ Explicit empty or whitespace-only keys are refused before POST. Keys contain truncation or case conversion. The key is shown safely in human output and preserved exactly in JSON/JSONL, including after a timeout or network failure. -After a lost response the admission outcome is unknown. Inspect workflow -history and command receipts before retrying; a key alone does not establish -retry safety. Rely on the complete retained-admission and canonical-intent -guarantee only when the Server advertises that published capability. Ordinary -request-key storage and partial update receipt lookup do not supply the full -signal/update guarantee. No currently qualified published Server tuple -advertises that complete capability. - -When the Server reports receipt metadata, `retained: true` identifies an -original receipt and `intent_verified` reports whether its intent was -verified. Preserve its original status and exact target. A qualified -retained-admission contract may refuse a different typed intent with HTTP 409 -and `reason: request_id_conflict`; the response identifies the retained -command and conflict parts. Legacy receipts with unverified intent never -authorize a new command. Deduplicated admission does not make external effects -exactly once. +After a lost response the admission outcome is unknown. The CLI sends each +signal or update once and never resends it; the key it used is printed, and is +in the JSON error envelope's `request_id`. On a Server that advertises +`capabilities.command_request_keys` (native Server 3.9.2 and later), the key is +a durable command key on the workflow instance: repeat the same command with +the same `--request-id`, name, input and `--run-id` and the Server returns the +original receipt (`retained: true`, `intent_verified: true`) without recording +a second command. On other Servers, inspect workflow history and command +receipts before retrying; a key alone does not establish retry safety. + +The same key with a different name, input or run target is refused with HTTP +409 `reason: request_id_conflict`, and the CLI exits `9` +(`REQUEST_ID_CONFLICT`). The response identifies the retained command and the +conflicting parts. A key that matches more than one retained command is refused +with HTTP 409 `reason: request_id_ambiguous` and exits `10` +(`REQUEST_ID_AMBIGUOUS`); nothing is recorded and no receipt is selected. +Neither refusal records anything. A legacy receipt, admitted before the Server +recorded command intent, reads back with `retained: true` and +`intent_verified: false`: the Server returned it without comparing the retried +command with it, and the human output says so. It never authorizes a new +command. Deduplicated admission does not make external effects exactly once. `workflow:update-outcome` reads an update that was sent earlier by its update ID, with one `GET /api/workflows/{workflow_id}/updates/{update_id}`, and never @@ -802,13 +807,15 @@ from there: |------|------|---------| | 0 | `SUCCESS` | Operation completed successfully. | | 1 | `FAILURE` | Generic failure — command ran but did not succeed. | -| 2 | `INVALID` | Invalid usage — bad arguments, unknown options, or local validation. Also returned for HTTP 4xx responses that are not covered below (e.g. 400, 422). | +| 2 | `INVALID` | Invalid usage — bad arguments, unknown options, or local validation. Also returned for HTTP 4xx responses that are not covered below (e.g. 400, 422, and a `409` other than a request-ID refusal). | | 3 | `NETWORK` | Could not reach the server (connection refused, DNS failure, TLS handshake failure, transport error). | | 4 | `AUTH` | Authentication or authorization failure. Returned for HTTP `401` and `403`. | | 5 | `NOT_FOUND` | Resource not found. Returned for HTTP `404`. | | 6 | `SERVER` | Server error. Returned for HTTP `5xx`. | | 7 | `TIMEOUT` | Request timed out before the server responded. Also returned for HTTP `408`. | | 8 | `COMPATIBILITY` | Compatibility failure. The CLI/server protocol window cannot be used safely, so the command refused before the requested operation. | +| 9 | `REQUEST_ID_CONFLICT` | HTTP `409` with reason `request_id_conflict`. The signal or update's `--request-id` already names a retained command with a different name, input or run target. Nothing was recorded. Repeat the original command unchanged to read its receipt, or use a new `--request-id`. | +| 10 | `REQUEST_ID_AMBIGUOUS` | HTTP `409` with reason `request_id_ambiguous`. The `--request-id` matches more than one retained command, so the Server recorded nothing and selected no receipt. Retrying it unchanged fails the same way. | Example: diff --git a/rust/parity/cases.json b/rust/parity/cases.json index dfa330a1..5c37d9a3 100644 --- a/rust/parity/cases.json +++ b/rust/parity/cases.json @@ -15596,6 +15596,93 @@ }, "reference": "php-main" }, + { + "name": "workflow signal request id ambiguous JSON", + "argv": [ + "workflow:signal", + "chk-42", + "approve", + "--server={mock}", + "--request-id=approve-1", + "--json" + ], + "mock": [], + "mock_routes": { + "/api/workflows/chk-42/signal/approve": { + "method": "POST", + "status": 409, + "headers": { + "X-Durable-Workflow-Control-Plane-Version": "2" + }, + "body": { + "message": "The request_id matches more than one retained command.", + "reason": "request_id_ambiguous", + "retained": true, + "intent_verified": false, + "retryable": false + } + } + }, + "reference": "php-main" + }, + { + "name": "workflow signal request id ambiguous human", + "argv": [ + "workflow:signal", + "chk-42", + "approve", + "--server={mock}", + "--request-id=approve-1" + ], + "mock": [], + "mock_routes": { + "/api/workflows/chk-42/signal/approve": { + "method": "POST", + "status": 409, + "headers": { + "X-Durable-Workflow-Control-Plane-Version": "2" + }, + "body": { + "message": "The request_id matches more than one retained command.", + "reason": "request_id_ambiguous", + "retained": true, + "intent_verified": false, + "retryable": false + } + } + }, + "reference": "php-main" + }, + { + "name": "workflow update request id ambiguous JSON", + "argv": [ + "workflow:update", + "chk-42", + "quota.adjust", + "--server={mock}", + "--request-id=approve-1", + "--wait=completed", + "--json" + ], + "mock": [], + "mock_routes": { + "/api/workflows/chk-42/update/quota.adjust": { + "method": "POST", + "status": 409, + "headers": { + "X-Durable-Workflow-Control-Plane-Version": "2" + }, + "body": { + "message": "The request_id matches more than one retained command.", + "reason": "request_id_ambiguous", + "retained": true, + "intent_verified": false, + "retryable": false + } + } + }, + "reference": "php-main" + }, { "name": "workflow update server error JSON", "argv": [ diff --git a/rust/src/commands/workflow_message.rs b/rust/src/commands/workflow_message.rs index 22b9ebf1..bd4928d4 100644 --- a/rust/src/commands/workflow_message.rs +++ b/rust/src/commands/workflow_message.rs @@ -143,9 +143,18 @@ fn receipt_lines(result: &Value) { )); } if let Some(Value::Bool(verified)) = result.get("intent_verified") { + // A legacy receipt is retained without a recorded intent, so the + // Server returned it without comparing this command to it. + let legacy = !*verified && result.get("retained") == Some(&Value::Bool(true)); formatter::writeln_out(&format!( " Intent Verified: {}", - if *verified { "yes" } else { "no" } + if *verified { + "yes" + } else if legacy { + "no (legacy receipt; the Server did not compare this command's intent with it)" + } else { + "no" + } )); } } diff --git a/rust/src/error.rs b/rust/src/error.rs index accd9129..fd3016db 100644 --- a/rust/src/error.rs +++ b/rust/src/error.rs @@ -13,6 +13,12 @@ pub mod exit { pub const SERVER: i32 = 6; pub const TIMEOUT: i32 = 7; pub const COMPATIBILITY: i32 = 8; + /// HTTP 409 `request_id_conflict`: the command key already names a + /// retained command with a different intent. Nothing was recorded. + pub const REQUEST_ID_CONFLICT: i32 = 9; + /// HTTP 409 `request_id_ambiguous`: the command key matches more than one + /// retained command. Nothing was recorded and no receipt was selected. + pub const REQUEST_ID_AMBIGUOUS: i32 = 10; pub fn from_http_status(status: u16) -> i32 { match status { @@ -62,6 +68,16 @@ impl CliError { CliError::Invalid(_) => exit::INVALID, CliError::Network(_) => exit::NETWORK, CliError::Timeout(_) => exit::TIMEOUT, + CliError::Http { status: 409, .. } + if self.reason() == Some(REQUEST_ID_CONFLICT_REASON) => + { + exit::REQUEST_ID_CONFLICT + } + CliError::Http { status: 409, .. } + if self.reason() == Some(REQUEST_ID_AMBIGUOUS_REASON) => + { + exit::REQUEST_ID_AMBIGUOUS + } CliError::Http { status, .. } => exit::from_http_status(*status), CliError::Compatibility(_) => exit::COMPATIBILITY, CliError::Failure(_) => exit::FAILURE, @@ -183,6 +199,10 @@ impl ErrorContext<'_> { /// different command. const REQUEST_ID_CONFLICT_REASON: &str = "request_id_conflict"; +/// The Server's refusal reason for a request ID that matches more than one +/// retained command, so no receipt can be selected. +const REQUEST_ID_AMBIGUOUS_REASON: &str = "request_id_ambiguous"; + /// 2.x `errorRecommendations()`: the error's next step, then, for a keyed /// command whose outcome is unknown, the inspect-before-retry warning. pub fn recommendations(error: &CliError, context: &ErrorContext) -> Vec { @@ -267,6 +287,20 @@ fn base_recommendations(error: &CliError, context: &ErrorContext) -> Vec ), })]; } + if error.reason() == Some(REQUEST_ID_AMBIGUOUS_REASON) { + let empty = Map::new(); + let body = body.as_ref().unwrap_or(&empty); + let request_id = scalar_text(body.get("request_id")) + .or_else(|| context.request_id.clone()) + .unwrap_or_else(|| "given".into()); + return vec![json!({ + "id": "request_id.ambiguous", + "severity": "error", + "message": format!( + "Request id [{request_id}] matches more than one retained command, so the Server recorded nothing and selected no receipt. Retrying it unchanged fails the same way. Inspect workflow history to see which commands were admitted, and pass a new --request-id to record a new command.", + ), + })]; + } } // A failed update read later is final, not a request to fix and retry. if context.command == "workflow:update-outcome" @@ -663,6 +697,21 @@ mod tests { assert_eq!(exit::from_http_status(503), exit::SERVER); } + #[test] + fn request_id_refusals_have_their_own_exit_codes() { + let conflict = CliError::from_http(409, r#"{"reason":"request_id_conflict"}"#); + assert_eq!(conflict.exit_code(), exit::REQUEST_ID_CONFLICT); + assert_eq!(exit::REQUEST_ID_CONFLICT, 9); + let ambiguous = CliError::from_http(409, r#"{"reason":"request_id_ambiguous"}"#); + assert_eq!(ambiguous.exit_code(), exit::REQUEST_ID_AMBIGUOUS); + assert_eq!(exit::REQUEST_ID_AMBIGUOUS, 10); + // Only the Server's 409 refusal is typed; other statuses keep their category. + let other = CliError::from_http(422, r#"{"reason":"request_id_conflict"}"#); + assert_eq!(other.exit_code(), exit::INVALID); + let closed = CliError::from_http(409, r#"{"reason":"run_not_active"}"#); + assert_eq!(closed.exit_code(), exit::INVALID); + } + #[test] fn refusal_without_message_keeps_reason_and_server_response() { let error = CliError::from_http( diff --git a/rust/tests/workflow_message.rs b/rust/tests/workflow_message.rs index 9e4fd66e..86b6d100 100644 --- a/rust/tests/workflow_message.rs +++ b/rust/tests/workflow_message.rs @@ -986,3 +986,188 @@ fn update_points_an_unfinished_update_at_update_outcome_only_when_the_server_can let stdout = String::from_utf8(output.stdout).unwrap(); assert!(!stdout.contains("workflow:update-outcome"), "{stdout}"); } + +fn refusal_ids(error: &Value) -> Vec { + error["recommendations"] + .as_array() + .unwrap() + .iter() + .map(|item| item["id"].as_str().unwrap().to_owned()) + .collect() +} + +#[test] +fn request_id_conflict_exits_9_names_the_retained_command_and_is_not_resent() { + let scratch = Scratch::new(); + let server = Server::new(vec![Reply::Json { + status: 409, + body: json!({ + "message": "The request_id was already used for a different signal command.", + "reason": "request_id_conflict", + "outcome": "rejected_request_id_conflict", + "retained": true, + "intent_verified": true, + "retained_command_id": "cmd-1", + "retained_outcome": "signal_received", + "conflict": ["arguments"], + }), + }]); + let output = scratch.run( + &server.url, + &[ + "workflow:signal", + "chk-42", + "approve", + "--request-id=approve-1", + "--json", + ], + ); + assert_eq!(output.status.code(), Some(9), "{output:?}"); + assert_eq!(server.lines().len(), 1, "a conflict is never resent"); + let error = stdout_json(&output); + assert_eq!(error["exit_code"], 9); + assert_eq!(error["reason"], "request_id_conflict"); + assert_eq!(error["request_id"], "approve-1"); + assert_eq!(error["retained_command_id"], "cmd-1"); + assert_eq!(refusal_ids(&error), ["request_id.conflict"]); + + let human = scratch.run( + &server.url, + &["workflow:update", "chk-42", "adjust", "--input=[2]"], + ); + assert_eq!(human.status.code(), Some(9), "{human:?}"); + let stderr = String::from_utf8(human.stderr).unwrap(); + assert!(stderr.contains("Reason: request_id_conflict"), "{stderr}"); + assert!(stderr.contains("Retained command: id cmd-1"), "{stderr}"); + assert!(stderr.contains("Conflicting parts: arguments"), "{stderr}"); + let generated = stderr + .lines() + .find_map(|line| line.strip_prefix("Request ID: ")) + .unwrap_or_else(|| panic!("{stderr}")); + assert!(is_generated(generated, "update"), "{stderr}"); + assert!( + stderr.contains(&format!("Request id [{generated}] already identifies")), + "{stderr}" + ); + assert_eq!(server.lines().len(), 2); +} + +#[test] +fn request_id_ambiguous_exits_10_selects_no_receipt_and_is_not_resent() { + let scratch = Scratch::new(); + let server = Server::new(vec![Reply::Json { + status: 409, + body: json!({ + "message": "The request_id matches more than one retained command.", + "reason": "request_id_ambiguous", + "retained": true, + "intent_verified": false, + "retryable": false, + }), + }]); + for (kind, name) in [ + ("workflow:signal", "approve"), + ("workflow:update", "adjust"), + ] { + let output = scratch.run( + &server.url, + &[kind, "chk-42", name, "--request-id=shared-key", "--json"], + ); + assert_eq!(output.status.code(), Some(10), "{output:?}"); + let error = stdout_json(&output); + assert_eq!(error["exit_code"], 10); + assert_eq!(error["reason"], "request_id_ambiguous"); + assert_eq!(error["request_id"], "shared-key"); + assert!( + error["command_id"].is_null(), + "no receipt is selected: {error}" + ); + assert!( + error["update_id"].is_null(), + "no receipt is selected: {error}" + ); + assert_eq!(refusal_ids(&error), ["request_id.ambiguous"]); + } + assert_eq!(server.lines().len(), 2, "each ambiguity is sent once"); + + let human = scratch.run( + &server.url, + &[ + "workflow:signal", + "chk-42", + "approve", + "--request-id=shared-key", + ], + ); + assert_eq!(human.status.code(), Some(10)); + let stderr = String::from_utf8(human.stderr).unwrap(); + assert!(stderr.contains("Reason: request_id_ambiguous"), "{stderr}"); + assert!( + stderr.contains("Request id [shared-key] matches more than one retained command"), + "{stderr}" + ); + assert!( + !stderr.contains("The outcome of this command is unknown."), + "an ambiguity is a definite refusal: {stderr}" + ); +} + +#[test] +fn other_conflicts_keep_exit_2() { + let scratch = Scratch::new(); + let server = Server::new(vec![Reply::Json { + status: 409, + body: json!({"message": "Run is closed.", "reason": "run_not_active"}), + }]); + let output = scratch.run( + &server.url, + &["workflow:signal", "chk-42", "approve", "--json"], + ); + assert_eq!(output.status.code(), Some(2), "{output:?}"); + assert_eq!(refusal_ids(&stdout_json(&output)), ["request.invalid"]); +} + +#[test] +fn a_legacy_unverified_receipt_is_not_shown_as_a_verified_match() { + let scratch = Scratch::new(); + let server = Server::new(vec![ok(json!({ + "workflow_id": "chk-42", + "signal_name": "approve", + "outcome": "signal_received", + "command_id": "cmd-legacy", + "request_id": "legacy-key", + "retained": true, + "intent_verified": false, + "control_plane": contract("signal", json!({})), + }))]); + let human = scratch.run( + &server.url, + &[ + "workflow:signal", + "chk-42", + "approve", + "--request-id=legacy-key", + ], + ); + assert_eq!(human.status.code(), Some(0), "{human:?}"); + let stdout = String::from_utf8(human.stdout).unwrap(); + assert!( + stdout.contains( + "Intent Verified: no (legacy receipt; the Server did not compare this command's intent with it)" + ), + "{stdout}" + ); + let json = scratch.run( + &server.url, + &[ + "workflow:signal", + "chk-42", + "approve", + "--request-id=legacy-key", + "--json", + ], + ); + let receipt = stdout_json(&json); + assert_eq!(receipt["retained"], true); + assert_eq!(receipt["intent_verified"], false); +} diff --git a/scripts/acceptance/command-keys/README.md b/scripts/acceptance/command-keys/README.md index 2ac8c034..c28d7bb8 100644 --- a/scripts/acceptance/command-keys/README.md +++ b/scripts/acceptance/command-keys/README.md @@ -137,6 +137,40 @@ schema-only mutations that the predecessor's row-only comparison accepted. Modern and legacy zero-write checks use the same bounded typed logical snapshot, including catalog SQL, columns, indexes, triggers, views and all table rows. +## Published Server 3.13.0 cohort + +`published_cohort.py` is the current, self-contained installed cohort. It runs +one phase per invocation against a published native Server image by digest, +with one CLI binary, and builds nothing: + +- `--phase modern`: committed-response loss for signals and updates with a + generated key (recovered from the JSON error envelope) and an explicit key, + four simultaneous first sends and four concurrent identical retries, + changed-argument, changed-name and changed-run-target conflicts with no + database change, 255-character ASCII, two-byte and four-byte keys (each + retried four ways at once), local refusal of 256 characters before any + POST, case-sensitive keys, completed and continued runs (retained readback, + conflicts, and a fresh key refused on the closed or historical run with no + admission), an explicitly selected run, and namespace isolation. +- `--phase legacy`: real API admissions, then a stopped copy rewritten by the + accepted Rust SDK `sqlite_legacy.py` held-descriptor helper. On restart it + checks legacy readback (`retained: true`, `intent_verified: false`, never + shown as verified in human output) for open, completed and continued runs, + name/run conflicts, and the three ambiguity shapes, each with an unchanged + complete logical snapshot. The synthetic transform is not a physical 3.8 + upgrade. + +Pass `--typed-exit-codes` for a CLI with the request-ID exit codes (9 +`REQUEST_ID_CONFLICT`, 10 `REQUEST_ID_AMBIGUOUS`); without it every refusal +must exit 2, as CLI 2.10.0 and earlier do. `--inputs` names a directory with +the shared proxy (`.github` `117118c8`), `sqlite_legacy.py` and +`command-key-legacy-v1-vectors.json` (`sdk-rust` `6280877b`); each is checked +against its SHA256 in the script before use. The Server runs as the invoking +user with one CPU, 1 GiB, 256 PIDs, a read-only root and a numeric loopback +port, and every created container is removed and its absence checked. +`.github/workflows/published-command-keys.yml` runs both phases for the +published CLI and for this checkout's build. + ## Physical 3.8 upgrade `run_cli_216.py` verifies and holds literal-hash modern, legacy or physical implementation bytes diff --git a/scripts/acceptance/command-keys/published_cohort.py b/scripts/acceptance/command-keys/published_cohort.py new file mode 100644 index 00000000..45d01ae0 --- /dev/null +++ b/scripts/acceptance/command-keys/published_cohort.py @@ -0,0 +1,612 @@ +#!/usr/bin/env python3 +"""Installed CLI command-key cohort against one published native Server image. + +One run is one phase (`modern` or `legacy`) with one CLI binary, one Server +image and owned synthetic SQLite state. It builds nothing. The modern phase +covers committed-response loss, explicit and generated-key recovery, +concurrency, changed-intent conflicts, Unicode limits, closed, continued and +fenced runs and namespace isolation. The legacy phase admits real commands, +stops the Server, rewrites a stopped copy through the accepted Rust SDK +`sqlite_legacy.py` held-descriptor helper, restarts on that copy, and checks +legacy readback and ambiguity refusals. The synthetic transform is not a +physical 3.8 upgrade. + +`--typed-exit-codes` expects the request-ID exit codes (9 conflict, 10 +ambiguous); without it a refusal must exit 2 (CLI 2.10.0 and earlier). +""" +import argparse +import concurrent.futures +import datetime +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import sqlite3 +import subprocess +import sys +import threading +import time +import types +import urllib.error +import urllib.request + +PROXY_SHA256 = 'bd1e8ccbbf76f63fc2eafa278a023eb181d405b68d9c30016392e395a1eff725' +HELPER_SHA256 = '4446843f79252f23cc58898debf63920c041a1391c9f3c5248e5bea04b481fbf' +MANIFEST_SHA256 = 'da1990000c611d7377690c7526487d6415fa4f9cb42d299f59585a7c2eebf769' +# Official Avro single-object envelopes of null and 1, as the Python SDK +# 2.17.1 serializer with fastavro 1.13.1 produced them for cli#152. +EMPTY = {'codec': 'avro', 'blob': 'wwHioz3/VYAiNwA='} +ONE = {'codec': 'avro', 'blob': 'wwHioz3/VYAiNwQC'} +NIL_RUN = '00000000000000000000000000' + +CHECKS, INVOCATIONS = [], [] +STATE = {} + + +def require(condition, label, detail=None): + if not condition: + raise AssertionError(label + (': ' + json.dumps(detail, default=str)[:4000] if detail is not None else '')) + CHECKS.append(label) + + +def verified(path, digest, name): + data = Path(path).read_bytes() + if hashlib.sha256(data).hexdigest() != digest: + raise SystemExit(name + ' differs from its accepted SHA256') + return data + + +def module(data, name, path): + value = types.ModuleType(name) + value.__file__ = str(path) + exec(compile(data, '<' + name + '>', 'exec'), value.__dict__) + return value + + +class Runtime: + """One owned Server container on a numeric loopback port.""" + + def __init__(self, options, work, token): + self.options, self.work, self.token = options, work, token + self.ids, self.current, self.base = [], None, None + + def docker(self, *args, check=True): + result = subprocess.run(['docker', *args], capture_output=True, text=True, timeout=60) + if check and result.returncode: + raise RuntimeError('docker ' + args[0] + ' failed: ' + result.stderr.replace(self.token, '[redacted]')[:2000]) + return result.stdout.strip() + + def start(self, database): + name = self.options.container_prefix + '-' + self.options.phase + '-' + secrets.token_hex(4) + envfile = self.work / 'runtime.env' + with open(os.open(envfile, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600), 'w') as output: + output.write('DW_AUTH_TOKEN=' + self.token + '\n') + try: + self.current = self.docker( + 'create', '--init', '--read-only', '--name', name, '--cpus', '1', '--memory', '1g', + '--pids-limit', '256', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', + '--label', 'durable-workflow.cli-command-keys=' + self.options.owner, + # Run as the invoking user so the owned private data + # directory needs no wider permissions on any host. + '--user', '%d:%d' % (os.getuid(), os.getgid()), + '--tmpfs', '/tmp:mode=1777,size=64m', + '--tmpfs', '/app/storage:uid=%d,gid=%d,mode=0700,size=64m' % (os.getuid(), os.getgid()), + '-v', str(self.work / 'data') + ':/cohort:rw', '-p', '127.0.0.1::8080', + '--env-file', str(envfile), '-e', 'DW_BIND_ADDRESS=0.0.0.0:8080', + '-e', 'DW_RUST_MAINTENANCE_LOOP=0', '-e', 'DB_CONNECTION=sqlite', + '-e', 'DB_DATABASE=/cohort/' + database, self.options.image) + self.ids.append(self.current) + self.docker('start', self.current) + finally: + envfile.unlink() + host, port = self.docker('port', self.current, '8080/tcp').splitlines()[0].rsplit(':', 1) + require(host == '127.0.0.1' and 0 < int(port) < 65536, 'runtime is bound to numeric loopback only') + self.base = 'http://127.0.0.1:' + port + deadline = time.monotonic() + 60 + while True: + try: + status, info = request('GET', '/api/cluster/info') + if status == 200: + return info + except (OSError, ValueError): + pass + if time.monotonic() > deadline: + raise RuntimeError('runtime did not become ready') + time.sleep(.1) + + def stop(self): + if self.current is None: + return + self.docker('stop', '--time', '10', self.current) + self.docker('rm', '-v', self.current) + self.current = None + + def remove_all(self): + for identity in self.ids: + self.docker('rm', '-fv', identity, check=False) + for identity in self.ids: + result = subprocess.run(['docker', 'inspect', identity], capture_output=True, text=True, timeout=30) + if result.returncode == 0: + raise RuntimeError('owned container ' + identity + ' is still present') + self.current = None + + +HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def request(method, path, body=None, namespace='default'): + headers = {'Authorization': 'Bearer ' + STATE['token'], 'X-Namespace': namespace, + 'Content-Type': 'application/json'} + worker = path.startswith('/api/worker/') + headers['X-Durable-Workflow-Protocol-Version' if worker else 'X-Durable-Workflow-Control-Plane-Version'] = '1.20' if worker else '2' + raw = urllib.request.Request(STATE['runtime'].base + path, method=method, headers=headers, + data=None if body is None else json.dumps(body).encode()) + try: + with HTTP.open(raw, timeout=15) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, json.load(error) + + +def cli(*args, namespace='default', exit_code=0): + result = subprocess.run([STATE['cli'], *args, '--server=' + STATE['proxy'].url, '--namespace=' + namespace, '--json'], + env=STATE['env'], capture_output=True, text=True, timeout=45) + try: + body = json.loads(result.stdout) + except ValueError: + body = {'stdout': result.stdout[:2000], 'stderr': result.stderr[:2000]} + INVOCATIONS.append({'arguments': args, 'namespace': namespace, 'exit': result.returncode, 'body': body}) + if result.returncode != exit_code: + raise AssertionError(json.dumps({'args': args, 'expected_exit': exit_code, 'exit': result.returncode, + 'body': body, 'stderr': result.stderr[:2000]})) + return body + + +def message(kind, workflow, key=None, value=1, name=None, run=None, namespace='default', exit_code=0): + args = ['workflow:' + kind, workflow, name or ('finish' if kind == 'signal' else 'set'), '--input=' + json.dumps([value])] + if key is not None: + args.append('--request-id=' + key) + if run is not None: + args.append('--run-id=' + run) + return cli(*args, namespace=namespace, exit_code=exit_code) + + +def path_for(kind, fixture, run=None): + path = '/api/workflows/' + fixture['workflow'] + if run: + path += '/runs/' + run + return path + '/' + kind + '/' + ('finish' if kind == 'signal' else 'set') + + +def poll(f): + status, body = request('POST', '/api/worker/workflow-tasks/poll', + {'worker_id': f['worker'], 'task_queue': f['queue'], 'timeout_seconds': 0}, f['namespace']) + require(status == 200, 'worker poll answered', body) + return body.get('task') + + +def finish(f, task, commands): + require(bool(task), 'a workflow task was leased') + status, body = request('POST', '/api/worker/workflow-tasks/' + task['task_id'] + '/complete', + {'lease_owner': task['lease_owner'], 'workflow_task_attempt': task['workflow_task_attempt'], + 'commands': commands}, f['namespace']) + require(status == 200, 'workflow task completed', body) + + +def fixture(kind, tag, namespace='default', workflow=None): + wf = workflow or 'cli216b-' + tag + '-' + secrets.token_hex(4) + f = {'workflow': wf, 'worker': wf + '-w', 'queue': wf + '-q', 'type': wf + '-t', 'namespace': namespace} + param = {'name': 'value', 'type': 'int', 'required': True, 'variadic': False, + 'default_available': False, 'allows_null': False} + contracts = {'signals': ['finish'], 'signal_contracts': [{'name': 'finish', 'parameters': [param]}], + 'updates': ['set'], 'update_validators': [], 'update_contracts': [{'name': 'set', 'parameters': [param]}]} + status, body = request('POST', '/api/worker/register', { + 'worker_id': f['worker'], 'task_queue': f['queue'], 'runtime': 'python', + 'supported_workflow_types': [f['type']], 'supported_activity_types': [], 'capabilities': ['workflow_updates'], + 'capability_manifest': {name: {'supported': False, 'minimum_protocol_version': '1.18'} + for name in ('local_activities', 'worker_sessions', 'sticky_execution')}, + 'workflow_command_contracts': {f['type']: contracts}}, namespace) + require(status == 201, 'synthetic worker registered', body) + started = cli('workflow:start', '--workflow-id=' + wf, '--type=' + f['type'], '--task-queue=' + f['queue'], + '--input=[]', namespace=namespace) + f['run'] = started['run_id'] + if kind == 'update': + finish(f, poll(f), [{'type': 'open_signal_wait', 'signal_name': 'finish'}]) + return f + + +def history(f, run=None): + status, body = request('GET', '/api/workflows/' + f['workflow'] + '/runs/' + (run or f['run']) + '/history', + namespace=f['namespace']) + require(status == 200, 'history read', body) + return body['events'] + + +def snapshot(): + path = STATE['work'] / 'data' / STATE['database'] + with sqlite3.connect(path.as_uri() + '?mode=ro', uri=True, timeout=2) as connection: + connection.execute('BEGIN') + return STATE['helper'].snapshot(connection) + + +def typed(answer, reason): + # Prefer the Server's own refusal body; the envelope copies only some fields. + candidates = [answer.get('server_response') or {}, answer] + [value for value in answer.values() if isinstance(value, dict)] + return next((value for value in candidates if value.get('reason') == reason), None) + + +def refusal_exit(reason): + if not STATE['typed']: + return 2 + return {'request_id_conflict': 9, 'request_id_ambiguous': 10}[reason] + + +def original(kind, f, key=None, lost=False, run=None): + path = path_for(kind, f, run=run) + before = len(STATE['proxy'].records) + if lost: + STATE['proxy'].arm('POST', path) + answer = message(kind, f['workflow'], key, run=run, namespace=f['namespace'], exit_code=3 if lost else 0) + sent = [r for r in STATE['proxy'].records[before:] if r['method'] == 'POST' and r['path'] == path] + require(len(sent) == 1, kind + ': exactly one admission POST; no implicit resend') + wire = sent[0] + require(wire['upstream_status'] == 202 and wire['upstream_complete'], kind + ': upstream admission completed') + chosen = answer.get('request_id') + require(isinstance(chosen, str) and chosen != '', kind + ': CLI reports the command key it sent', answer) + if key is None: + require(re.fullmatch('cli-' + kind + '-[0-9a-f]{32}', chosen) is not None, kind + ': generated key shape') + else: + require(chosen == key, kind + ': CLI reports the explicit key') + if lost: + require(wire['dropped'] and wire['upstream_body_bytes'] > 0, kind + ': committed upstream receipt was dropped') + require(any(item.get('id') == 'request_id.retry' for item in answer.get('recommendations', [])), + kind + ': lost response is reported as an unknown outcome', answer) + admitted = snapshot() + start = len(STATE['proxy'].records) + receipt = message(kind, f['workflow'], chosen, run=run, namespace=f['namespace']) + recovered = [r for r in STATE['proxy'].records[start:] if r['method'] == 'POST' and r['path'] == path] + require(len(recovered) == 1 and recovered[0]['request_body_sha256'] == wire['request_body_sha256'], + kind + ': recovery with the reported key sends identical bytes') + require(receipt.get('retained') is True and receipt.get('intent_verified') is True, + kind + ': recovery returns the verified retained receipt', receipt) + require(snapshot() == admitted, kind + ': recovery writes nothing') + else: + receipt = answer + require(receipt.get('retained') is False and receipt.get('intent_verified') is True, + kind + ': fresh receipt is verified and not retained', receipt) + fields = ('command_id', 'run_id', 'command_sequence', 'accepted_at', 'update_id', 'signal_name') + return {'request_id': chosen, 'receipt': {k: receipt[k] for k in fields if k in receipt}} + + +def retained(kind, f, record, run=None): + before = snapshot() + answer = message(kind, f['workflow'], record['request_id'], run=run, namespace=f['namespace']) + for key, value in record['receipt'].items(): + require(answer.get(key) == value, kind + ': retained original ' + key, answer) + require(answer.get('retained') is True and answer.get('intent_verified') is True, kind + ': verified retained intent') + require(snapshot() == before, kind + ': retained retry writes nothing') + return answer + + +def conflict(kind, f, record, part, **options): + before = snapshot() + answer = message(kind, f['workflow'], record['request_id'], namespace=f['namespace'], + exit_code=refusal_exit('request_id_conflict'), **options) + body = typed(answer, 'request_id_conflict') + require(body is not None, kind + ': typed conflict for ' + part, answer) + expected = part + if part == 'target': + expected = 'target_or_arguments' if options.get('run') is not None else 'arguments' + elif kind == 'update' and part == 'name': + expected = 'arguments' + require(body.get('conflict') == [expected] and body.get('retained_command_id') == record['receipt']['command_id'], + kind + ': conflict names the retained command and ' + part, body) + require(answer.get('request_id') == record['request_id'], kind + ': conflict envelope keeps the key') + if STATE['typed']: + require([item.get('id') for item in answer.get('recommendations', [])] == ['request_id.conflict'], + kind + ': conflict recommendation', answer) + require(snapshot() == before, kind + ': conflict writes nothing') + + +def terminal(kind, f, record, action): + commands = [] + if kind == 'update': + commands.append({'type': 'complete_update', 'update_id': record['receipt']['update_id'], + 'result': STATE['update_result']}) + commands.append({'type': action, 'result': EMPTY}) + event = 'WorkflowCompleted' if action == 'complete_workflow' else 'WorkflowContinuedAsNew' + for _ in range(10): + finish(f, poll(f), commands) + if any(row['event_type'] == event for row in history(f)): + break + require(any(row['event_type'] == event for row in history(f)), kind + ': original run recorded ' + action) + + +def fenced(kind, f, key): + """A fresh key cannot admit a command on a closed or historical run. + + The Server durably records the refused command (`command_status: + rejected`), so this checks admissions in history rather than zero writes. + """ + event_type = 'SignalReceived' if kind == 'signal' else 'UpdateAccepted' + admitted = sum(row['event_type'] == event_type for row in history(f)) + answer = message(kind, f['workflow'], key, run=f['run'], namespace=f['namespace'], exit_code=2) + require(answer.get('accepted') is not True and answer.get('command_status') in (None, 'rejected') + and answer.get('update_status') in (None, 'rejected'), kind + ': fenced run refuses the fresh command', answer) + require(sum(row['event_type'] == event_type for row in history(f)) == admitted, + kind + ': fenced refusal admits nothing to the run') + return answer.get('reason') + + +def run_modern(info): + require(info['capabilities']['command_request_keys'] is True, 'Server advertises command_request_keys') + STATE['update_result'] = ONE + for kind in ('signal', 'update'): + event_type = 'SignalReceived' if kind == 'signal' else 'UpdateAccepted' + f = fixture(kind, kind + '-loss') + record = original(kind, f, lost=True) # generated key recovered from the error envelope + retained(kind, f, record) + conflict(kind, f, record, 'arguments', value=2) + conflict(kind, f, record, 'name', name='other') + conflict(kind, f, record, 'target', run=f['run']) + explicit = original(kind, f, key=kind + '-explicit-loss', lost=True) + retained(kind, f, explicit) + key = kind + '-concurrent' + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + replies = list(pool.map(lambda _: message(kind, f['workflow'], key), range(4))) + require(len({r['command_id'] for r in replies}) == 1 and sum(r.get('retained') is True for r in replies) == 3, + kind + ': four simultaneous first sends admit exactly one command', replies) + before = snapshot() + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + again = list(pool.map(lambda _: message(kind, f['workflow'], key), range(4))) + require(all(r.get('retained') is True and r['command_id'] == replies[0]['command_id'] for r in again), + kind + ': four concurrent identical retries return the original receipt') + require(snapshot() == before, kind + ': concurrent retries write nothing') + require(sum(e['event_type'] == event_type for e in history(f)) == 3, + kind + ': history has one admission per key') + for label, unicode_key in (('ascii', 'k' * 255), ('two-byte', 'é' * 255), ('four-byte', '🙂' * 255)): + unicode_record = original(kind, f, key=unicode_key) + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(lambda _: retained(kind, f, unicode_record), range(4))) + require(len(unicode_key) == 255, kind + ': 255-character ' + label + ' key retained') + before, wire = snapshot(), len(STATE['proxy'].records) + message(kind, f['workflow'], key='é' * 256, namespace=f['namespace'], exit_code=2) + require(not any(r['method'] == 'POST' for r in STATE['proxy'].records[wire:]), + kind + ': 256-character key refused before any POST') + require(snapshot() == before, kind + ': overlength refusal writes nothing') + a = message(kind, f['workflow'], key='CaseSensitive') + b = message(kind, f['workflow'], key='casesensitive') + require(a['command_id'] != b['command_id'], kind + ': keys are case sensitive') + for action in ('complete_workflow', 'continue_as_new'): + f = fixture(kind, kind + '-' + action) + record = original(kind, f, key=kind + '-' + action, lost=True) + terminal(kind, f, record, action) + answer = retained(kind, f, record) + if kind == 'update': + require(answer.get('update_status') == 'completed' and answer.get('result_envelope') == ONE, + 'update: completed typed result survives ' + action, answer) + conflict(kind, f, record, 'arguments', value=2) + conflict(kind, f, record, 'target', run=NIL_RUN) + reason = fenced(kind, f, kind + '-' + action + '-fresh') + require(reason in ('run_not_active', 'run_closed', 'workflow_closed', 'historical_run_command_rejected', + 'workflow_not_running', 'run_not_current'), kind + ': ' + action + ' fence reason', reason) + if action == 'continue_as_new': + status, described = request('GET', '/api/workflows/' + f['workflow']) + successor = described.get('run_id') + require(status == 200 and successor and successor != f['run'], kind + ': continued run has a successor') + successor_events = [e['event_type'] for e in history(f, successor)] + require(event_type not in successor_events, kind + ': retained retry admitted nothing to the successor') + f = fixture(kind, kind + '-selected') + record = original(kind, f, key=kind + '-explicit-run', run=f['run'], lost=True) + retained(kind, f, record, run=f['run']) + conflict(kind, f, record, 'target') + status, _ = request('POST', '/api/namespaces', {'name': 'cli216b-other', 'description': 'synthetic', 'retention_days': 30}) + require(status in (200, 201), 'synthetic second namespace created') + for kind in ('signal', 'update'): + common = 'cli216b-ns-' + kind + a = fixture(kind, common, workflow=common) + b = fixture(kind, common, namespace='cli216b-other', workflow=common + '-other') + ra = message(kind, a['workflow'], key='same-key') + rb = message(kind, b['workflow'], key='same-key', namespace=b['namespace']) + require(ra['command_id'] != rb['command_id'], kind + ': one key admits independently per namespace') + before = snapshot() + foreign = message(kind, a['workflow'], key='same-key', namespace=b['namespace'], exit_code=5) + require(foreign.get('command_id') is None and foreign.get('update_id') is None, + kind + ': another namespace cannot read the retained receipt', foreign) + require(snapshot() == before, kind + ': foreign refusal writes nothing') + + +def run_legacy(manifest): + STATE['update_result'] = EMPTY + selections, cases = [], [] + + def select(kind, f, record, stored): + selections.append({'command_id': record['receipt']['command_id'], 'command_type': kind, + 'workflow_instance_id': f['workflow'], 'workflow_run_id': f['run'], + 'caller_key': record['request_id'], 'legacy_request_id': stored}) + + for kind in manifest['legacy_readback']['types']: + for state, action in [('open', None), ('completed', 'complete_workflow'), ('continued', 'continue_as_new')]: + f = fixture(kind, 'legacy-' + kind + '-' + state) + record = original(kind, f, key=manifest['keys']['original']) + if action: + terminal(kind, f, record, action) + select(kind, f, record, record['request_id']) + cases.append({'kind': kind, 'fixture': f, 'record': record, 'state': state, 'shape': 'readback'}) + for shape in ['two-rows', 'legacy-token', 'modern-raw']: + f = fixture(kind, 'ambiguity-' + kind + '-' + shape) + record = original(kind, f, key=manifest['keys']['original']) + token = manifest['keys']['tokens'][kind] + if shape == 'two-rows': + second = original(kind, f, key=manifest['keys']['second']) + select(kind, f, second, manifest['keys']['original']) + elif shape == 'legacy-token': + select(kind, f, record, token) + cases.append({'kind': kind, 'fixture': f, 'record': record, 'shape': shape, + 'retry_key': token if shape == 'modern-raw' else manifest['keys']['original']}) + STATE['proxy'].shutdown() + STATE['proxy'].server_close() + STATE['runtime'].stop() + source = STATE['work'] / 'data' / 'seed.sqlite' + with sqlite3.connect(source) as connection: + require(connection.execute('PRAGMA wal_checkpoint(TRUNCATE)').fetchone()[0] == 0, 'stopped seed checkpointed') + transform = STATE['helper'].transform_copy(source, STATE['work'] / 'data' / 'legacy.sqlite', selections, stopped=True) + require(transform['transformed'] == len(selections), 'helper rewrote exactly the selected admissions', transform) + STATE['database'] = 'legacy.sqlite' + info = STATE['runtime'].start('legacy.sqlite') + require(info['version'] == STATE['options'].server_version, 'same Server version on the legacy copy') + STATE['proxy'] = STATE['proxy_type'](STATE['runtime'].base) + threading.Thread(target=STATE['proxy'].serve_forever, kwargs={'poll_interval': .01}, daemon=True).start() + retry = manifest['legacy_readback']['retry_input'] + + def readback(case, run=None): + kind, f, record = case['kind'], case['fixture'], case['record'] + answer = message(kind, f['workflow'], record['request_id'], value=retry, run=run) + for field, value in record['receipt'].items(): + require(answer.get(field) == value, kind + ': legacy original ' + field, answer) + require(answer.get('retained') is True and answer.get('intent_verified') is False, + kind + ': legacy readback is retained and explicitly unverified', answer) + if kind == 'update' and case['state'] != 'open': + require(answer.get('update_status') == 'completed', 'legacy update stays completed after ' + case['state']) + return answer + + for case in cases: + kind, f, record = case['kind'], case['fixture'], case['record'] + label = kind + ':' + case['shape'] + ':' + case.get('state', 'open') + before = snapshot() + if case['shape'] == 'readback': + with concurrent.futures.ThreadPoolExecutor(max_workers=manifest['legacy_readback']['concurrency']) as pool: + replies = list(pool.map(lambda _: readback(case), range(manifest['legacy_readback']['concurrency']))) + require(len({r['command_id'] for r in replies}) == 1, label + ': concurrent readers get one original identity') + readback(case, run=f['run']) + human = subprocess.run([STATE['cli'], 'workflow:' + kind, f['workflow'], + 'finish' if kind == 'signal' else 'set', '--input=' + json.dumps([retry]), + '--request-id=' + record['request_id'], '--server=' + STATE['proxy'].url, + '--namespace=default'], env=STATE['env'], capture_output=True, text=True, timeout=45) + require(human.returncode == 0 and 'Intent Verified: no' in human.stdout + and 'Intent Verified: yes' not in human.stdout, label + ': human output does not claim verified intent', + human.stdout[-2000:]) + if STATE['typed']: + require('legacy receipt' in human.stdout, label + ': human output names the legacy receipt') + for options in [{'name': 'other'}, {'run': NIL_RUN}]: + reply = message(kind, f['workflow'], record['request_id'], value=999, + exit_code=refusal_exit('request_id_conflict'), **options) + body = typed(reply, 'request_id_conflict') + require(body is not None and body.get('retained_command_id') == record['receipt']['command_id'], + label + ': changed name/run refusal names the original command', reply) + else: + reply = message(kind, f['workflow'], case['retry_key'], value=retry, + exit_code=refusal_exit('request_id_ambiguous')) + body = typed(reply, 'request_id_ambiguous') + require(body is not None and body.get('retained') is True and body.get('intent_verified') is False + and body.get('retryable') is False, label + ': ambiguity fails closed and unverified', reply) + require(all(body.get(field) is None for field in ('command_id', 'update_id', 'retained_command_id')), + label + ': ambiguity selects no receipt') + require(reply.get('request_id') == case['retry_key'], label + ': ambiguity envelope keeps the key') + if STATE['typed']: + require([item.get('id') for item in reply.get('recommendations', [])] == ['request_id.ambiguous'], + label + ': ambiguity recommendation', reply) + require(snapshot() == before, label + ': complete logical rows unchanged') + return transform + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--phase', choices=['modern', 'legacy'], required=True) + parser.add_argument('--cli', type=Path, required=True) + parser.add_argument('--cli-sha256', required=True) + parser.add_argument('--typed-exit-codes', action='store_true') + parser.add_argument('--inputs', type=Path, required=True, + help='directory with committed_response_loss.py, sqlite_legacy.py and command-key-legacy-v1-vectors.json') + parser.add_argument('--server-index', required=True) + parser.add_argument('--server-source', required=True) + parser.add_argument('--server-version', required=True) + parser.add_argument('--work-dir', type=Path, required=True) + parser.add_argument('--container-prefix', default='cli-command-keys') + parser.add_argument('--owner', default='manual') + options = parser.parse_args() + if not __debug__: + parser.error('run without Python optimization') + if (re.fullmatch(r'sha256:[0-9a-f]{64}', options.server_index) is None + or re.fullmatch(r'[0-9a-f]{40}', options.server_source) is None + or re.fullmatch(r'[0-9a-f]{64}', options.cli_sha256) is None + or re.fullmatch(r'[a-z0-9][a-z0-9.-]{0,40}', options.container_prefix) is None): + parser.error('exact immutable identities and a plain container prefix are required') + if not options.work_dir.is_absolute() or options.work_dir.exists(): + parser.error('the work directory must be a new absolute path') + options.image = 'ghcr.io/durable-workflow/server@' + options.server_index + cli_bytes = verified(options.cli, options.cli_sha256, 'CLI binary') + proxy = module(verified(options.inputs / 'committed_response_loss.py', PROXY_SHA256, 'response-loss proxy'), 'shared_proxy', + options.inputs / 'committed_response_loss.py') + helper = module(verified(options.inputs / 'sqlite_legacy.py', HELPER_SHA256, 'SQLite helper'), 'sqlite_legacy', + options.inputs / 'sqlite_legacy.py') + manifest_bytes = verified(options.inputs / 'command-key-legacy-v1-vectors.json', MANIFEST_SHA256, 'legacy manifest') + work = options.work_dir + work.mkdir(mode=0o700) + for name in ('data', 'home', 'config', 'tmp', 'bin'): + (work / name).mkdir(mode=0o700) + (work / 'command-key-legacy-v1-vectors.json').write_bytes(manifest_bytes) + helper.MANIFEST = work / 'command-key-legacy-v1-vectors.json' + binary = work / 'bin' / 'dw' + with open(os.open(binary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o500), 'wb') as output: + output.write(cli_bytes) + token = secrets.token_hex(32) + STATE.update(token=token, work=work, helper=helper, cli=str(binary), typed=options.typed_exit_codes, + options=options, proxy_type=proxy.CommittedResponseLoss, + database='database.sqlite' if options.phase == 'modern' else 'seed.sqlite', + env={'PATH': '/usr/bin:/bin', 'HOME': str(work / 'home'), 'DW_CONFIG_HOME': str(work / 'config'), + 'TMPDIR': str(work / 'tmp'), 'DURABLE_WORKFLOW_AUTH_TOKEN': token}) + runtime = Runtime(options, work, token) + STATE['runtime'] = runtime + started = datetime.datetime.now(datetime.timezone.utc).isoformat() + outcome, error, transform, cleanup = 'fail', None, None, None + version = subprocess.run([str(binary), '--version'], capture_output=True, text=True, timeout=20, + env=STATE['env']).stdout.strip() + try: + image = json.loads(runtime.docker('image', 'inspect', options.image))[0] + labels = image['Config']['Labels'] + require(image['Os'] == 'linux' and image['Architecture'] == 'amd64', 'Server image is linux/amd64') + require(labels['org.opencontainers.image.revision'] == options.server_source, 'Server image source revision') + require(labels['dev.durable-workflow.release.tag'] == options.server_version, 'Server image release tag') + info = runtime.start(STATE['database']) + require(info['version'] == options.server_version, 'Server reports the published version') + STATE['proxy'] = proxy.CommittedResponseLoss(runtime.base) + threading.Thread(target=STATE['proxy'].serve_forever, kwargs={'poll_interval': .01}, daemon=True).start() + if options.phase == 'modern': + run_modern(info) + else: + transform = run_legacy(json.loads(manifest_bytes)) + outcome = 'pass' + except Exception as failure: # report the first failure, then clean up + error = (type(failure).__name__ + ': ' + str(failure)).replace(token, '[redacted]') + finally: + if STATE.get('proxy'): + STATE['proxy'].shutdown() + STATE['proxy'].server_close() + try: + runtime.remove_all() + for path in (work / 'data').iterdir(): + path.unlink() + cleanup = 'pass' + except Exception as failure: + cleanup = 'fail: ' + str(failure) + outcome = 'fail' + report = {'phase': options.phase, 'outcome': outcome, 'error': error, 'checks': len(CHECKS), + 'invocations': len(INVOCATIONS), 'started_at': started, + 'finished_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'cli_version': version, 'cli_sha256': options.cli_sha256, 'typed_exit_codes': options.typed_exit_codes, + 'server_index': options.server_index, 'server_source': options.server_source, + 'server_version': options.server_version, 'proxy_sha256': PROXY_SHA256, + 'helper_sha256': HELPER_SHA256, 'manifest_sha256': MANIFEST_SHA256, + 'runner_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + 'container_ids': runtime.ids, 'cleanup': cleanup, 'transform': transform} + (work / 'checks.json').write_text(json.dumps(CHECKS, indent=1)) + (work / 'invocations.json').write_text(json.dumps(INVOCATIONS, indent=1).replace(token, '[redacted]')) + print(json.dumps(report), flush=True) + return 0 if outcome == 'pass' else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/Commands/BaseCommand.php b/src/Commands/BaseCommand.php index 885f1d31..fc0ab146 100644 --- a/src/Commands/BaseCommand.php +++ b/src/Commands/BaseCommand.php @@ -34,6 +34,8 @@ abstract class BaseCommand extends Command private const REQUEST_ID_CONFLICT_REASON = 'request_id_conflict'; + private const REQUEST_ID_AMBIGUOUS_REASON = 'request_id_ambiguous'; + private const NAMESPACE_SCOPED_COMMAND_NAMES = [ 'activity:complete', 'activity:describe', @@ -739,7 +741,12 @@ protected function writeCommandReceiptLines(OutputInterface $output, array $resu } if (is_bool($result['intent_verified'] ?? null)) { - $output->writeln(' Intent Verified: '.($result['intent_verified'] ? 'yes' : 'no')); + // A legacy receipt is retained without a recorded intent, so the + // Server returned it without comparing this command to it. + $legacy = $result['intent_verified'] === false && ($result['retained'] ?? null) === true; + $output->writeln(' Intent Verified: '.($result['intent_verified'] + ? 'yes' + : ($legacy ? "no (legacy receipt; the Server did not compare this command's intent with it)" : 'no'))); } } @@ -1862,6 +1869,22 @@ private function httpErrorRecommendations(ServerHttpException $e, ?InputInterfac ]]; } + if ($e->statusCode === 409 && $e->reason() === self::REQUEST_ID_AMBIGUOUS_REASON) { + $body = $e->body ?? []; + $requestId = $this->nonEmptyScalarString($body['request_id'] ?? null) + ?? ($input !== null ? $this->errorRequestIdContext($input) : null) + ?? 'given'; + + return [[ + 'id' => 'request_id.ambiguous', + 'severity' => 'error', + 'message' => sprintf( + 'Request id [%s] matches more than one retained command, so the Server recorded nothing and selected no receipt. Retrying it unchanged fails the same way. Inspect workflow history to see which commands were admitted, and pass a new --request-id to record a new command.', + $requestId, + ), + ]]; + } + if ($e->statusCode >= 400 && $e->statusCode <= 499) { return [[ 'id' => 'request.invalid', diff --git a/src/Support/ExitCode.php b/src/Support/ExitCode.php index 0edb5090..f5f51602 100644 --- a/src/Support/ExitCode.php +++ b/src/Support/ExitCode.php @@ -42,6 +42,12 @@ final class ExitCode /** Compatibility failure — the CLI/server protocol window cannot be used safely. */ public const COMPATIBILITY = 8; + /** HTTP 409 `request_id_conflict` — the command key names a retained command with a different intent; nothing was recorded. */ + public const REQUEST_ID_CONFLICT = 9; + + /** HTTP 409 `request_id_ambiguous` — the command key matches more than one retained command; nothing was recorded and no receipt was selected. */ + public const REQUEST_ID_AMBIGUOUS = 10; + private function __construct() {} /** diff --git a/src/Support/ServerHttpException.php b/src/Support/ServerHttpException.php index 12a5420d..dfd09518 100644 --- a/src/Support/ServerHttpException.php +++ b/src/Support/ServerHttpException.php @@ -45,6 +45,14 @@ public function validationErrors(): ?array public function exitCode(): int { + if ($this->statusCode === 409 && $this->reason() === 'request_id_conflict') { + return ExitCode::REQUEST_ID_CONFLICT; + } + + if ($this->statusCode === 409 && $this->reason() === 'request_id_ambiguous') { + return ExitCode::REQUEST_ID_AMBIGUOUS; + } + return ExitCode::fromHttpStatus($this->statusCode); } } diff --git a/tests/Commands/ExitCodePolicyTest.php b/tests/Commands/ExitCodePolicyTest.php index 0a64f9d8..d27de93c 100644 --- a/tests/Commands/ExitCodePolicyTest.php +++ b/tests/Commands/ExitCodePolicyTest.php @@ -49,6 +49,16 @@ public static function httpStatusCases(): iterable yield '200 ok → FAILURE (non-error status)' => [200, ExitCode::FAILURE]; } + public function test_request_id_refusals_have_their_own_exit_codes(): void + { + self::assertSame(9, ExitCode::REQUEST_ID_CONFLICT); + self::assertSame(10, ExitCode::REQUEST_ID_AMBIGUOUS); + self::assertSame(ExitCode::REQUEST_ID_CONFLICT, (new ServerHttpException('x', 409, body: ['reason' => 'request_id_conflict']))->exitCode()); + self::assertSame(ExitCode::REQUEST_ID_AMBIGUOUS, (new ServerHttpException('x', 409, body: ['reason' => 'request_id_ambiguous']))->exitCode()); + self::assertSame(ExitCode::INVALID, (new ServerHttpException('x', 409, body: ['reason' => 'run_not_active']))->exitCode()); + self::assertSame(ExitCode::INVALID, (new ServerHttpException('x', 422, body: ['reason' => 'request_id_conflict']))->exitCode()); + } + public function test_base_command_translates_network_exception_to_network_exit_code(): void { $command = new ThrowingBaseCommand(new NetworkException('Connection refused')); diff --git a/tests/Commands/WorkflowCommandRequestIdTest.php b/tests/Commands/WorkflowCommandRequestIdTest.php index 3615c686..3e7d6dee 100644 --- a/tests/Commands/WorkflowCommandRequestIdTest.php +++ b/tests/Commands/WorkflowCommandRequestIdTest.php @@ -57,7 +57,7 @@ public function test_opaque_command_values_are_safe_in_human_output_and_exact_in : $this->runUpdate($client, $arguments, $decorated); self::assertSame($value, $client->lastPostBody['request_id']); self::assertSame(match ($scenario) { - 'conflict' => ExitCode::INVALID, 'uncertain' => ExitCode::NETWORK, default => Command::SUCCESS, + 'conflict' => ExitCode::REQUEST_ID_CONFLICT, 'uncertain' => ExitCode::NETWORK, default => Command::SUCCESS, }, $tester->getStatusCode()); if ($json) { $decoded = $this->json($tester); @@ -343,7 +343,7 @@ public function test_signal_request_id_conflict_names_the_key_and_the_retained_c '--input' => '[{"approved":false}]', ]); - self::assertSame(ExitCode::INVALID, $tester->getStatusCode()); + self::assertSame(ExitCode::REQUEST_ID_CONFLICT, $tester->getStatusCode()); self::assertNotSame(Command::SUCCESS, $tester->getStatusCode()); self::assertSame('approve-1', $client->lastPostBody['request_id']); @@ -375,12 +375,12 @@ public function test_signal_request_id_conflict_json_error_is_structured(): void '--json' => true, ]); - self::assertSame(ExitCode::INVALID, $tester->getStatusCode()); + self::assertSame(ExitCode::REQUEST_ID_CONFLICT, $tester->getStatusCode()); $decoded = $this->json($tester); self::assertSame('Server error: Request id already identifies a different signal.', $decoded['error']); - self::assertSame(ExitCode::INVALID, $decoded['exit_code']); + self::assertSame(ExitCode::REQUEST_ID_CONFLICT, $decoded['exit_code']); self::assertSame('workflow:signal', $decoded['command']); self::assertSame(409, $decoded['status_code']); self::assertSame('request_id_conflict', $decoded['reason']); @@ -413,7 +413,7 @@ public function test_update_request_id_conflict_json_error_is_structured(): void '--json' => true, ]); - self::assertSame(ExitCode::INVALID, $tester->getStatusCode()); + self::assertSame(ExitCode::REQUEST_ID_CONFLICT, $tester->getStatusCode()); self::assertSame('/workflows/wf-1/runs/run-2/update/approve', $client->lastPostPath); $decoded = $this->json($tester); @@ -439,6 +439,75 @@ public function test_update_request_id_conflict_json_error_is_structured(): void self::assertSame(['request_id.conflict'], array_column($decoded['recommendations'], 'id')); } + public function test_request_id_ambiguous_has_its_own_exit_code_and_selects_no_receipt(): void + { + $body = [ + 'message' => 'The request_id matches more than one retained command.', + 'reason' => 'request_id_ambiguous', + 'retained' => true, + 'intent_verified' => false, + 'retryable' => false, + ]; + foreach (['signal', 'update'] as $operation) { + $client = new CommandRequestIdFakeClient([], exception: new ServerHttpException( + 'Server error: The request_id matches more than one retained command.', + 409, + body: $body, + ), contract: self::requestContract()); + $arguments = ['workflow-id' => 'wf-1', $operation.'-name' => 'approve', '--request-id' => 'shared-key', '--json' => true]; + $tester = $operation === 'signal' ? $this->runSignal($client, $arguments) : $this->runUpdate($client, $arguments); + + self::assertSame(ExitCode::REQUEST_ID_AMBIGUOUS, $tester->getStatusCode()); + self::assertSame(10, ExitCode::REQUEST_ID_AMBIGUOUS); + $decoded = $this->json($tester); + self::assertSame(ExitCode::REQUEST_ID_AMBIGUOUS, $decoded['exit_code']); + self::assertSame('request_id_ambiguous', $decoded['reason']); + self::assertSame('shared-key', $decoded['request_id']); + self::assertArrayNotHasKey('command_id', $decoded); + self::assertSame(['request_id.ambiguous'], array_column($decoded['recommendations'], 'id')); + self::assertStringContainsString('Request id [shared-key] matches more than one retained command', $decoded['recommendations'][0]['message']); + } + + $client = new CommandRequestIdFakeClient([], exception: new ServerHttpException('Server error: ambiguous', 409, body: $body)); + $human = $this->runSignal($client, ['workflow-id' => 'wf-1', 'signal-name' => 'approve', '--request-id' => 'shared-key']); + self::assertSame(ExitCode::REQUEST_ID_AMBIGUOUS, $human->getStatusCode()); + self::assertStringContainsString('Reason: request_id_ambiguous', $human->getDisplay()); + self::assertStringNotContainsString('The outcome of this command is unknown.', $human->getDisplay()); + } + + public function test_other_409_refusals_keep_the_invalid_exit_code(): void + { + $client = new CommandRequestIdFakeClient([], exception: new ServerHttpException( + 'Server error: Run is closed.', + 409, + body: ['message' => 'Run is closed.', 'reason' => 'run_not_active'], + )); + $tester = $this->runSignal($client, ['workflow-id' => 'wf-1', 'signal-name' => 'approve', '--json' => true]); + + self::assertSame(ExitCode::INVALID, $tester->getStatusCode()); + self::assertSame(['request.invalid'], array_column($this->json($tester)['recommendations'], 'id')); + } + + public function test_a_legacy_unverified_receipt_is_not_shown_as_a_verified_match(): void + { + $receipt = self::signalReceipt(['request_id' => 'legacy-key', 'retained' => true, 'intent_verified' => false]); + $human = $this->runSignal( + new CommandRequestIdFakeClient($receipt, contract: self::requestContract()), + ['workflow-id' => 'wf-1', 'signal-name' => 'approve', '--request-id' => 'legacy-key'], + ); + + self::assertSame(Command::SUCCESS, $human->getStatusCode()); + self::assertStringContainsString("Intent Verified: no (legacy receipt; the Server did not compare this command's intent with it)", $human->getDisplay()); + + $json = $this->runSignal( + new CommandRequestIdFakeClient($receipt, contract: self::requestContract()), + ['workflow-id' => 'wf-1', 'signal-name' => 'approve', '--request-id' => 'legacy-key', '--json' => true], + ); + $decoded = $this->json($json); + self::assertTrue($decoded['retained']); + self::assertFalse($decoded['intent_verified']); + } + public function test_signal_transport_failure_prints_the_key_that_was_sent(): void { $client = new CommandRequestIdFakeClient([], exception: new NetworkException('Server unreachable: connection reset')); From 524c37f0c5bc612df87aab8cdd5329d567ae0b6e Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:14:09 +0000 Subject: [PATCH 2/2] Address analysis findings in the command-key cohort Extract the ambiguity next step and the legacy receipt line out of nested branches in the PHP reference, give the cohort's repeated literals names, bind loop variables in its concurrent lambdas, replace the generated-key regex with a plain hex check, and fix the legacy readback concurrency to the accepted vector's four. Classify scripts/acceptance like scripts/conformance as test code for analysis: these are operator-run harnesses whose inputs are hash-checked before use. Co-Authored-By: Claude Opus 5.5 --- .../command-keys/published_cohort.py | 50 +++++++++++-------- sonar-project.properties | 3 +- src/Commands/BaseCommand.php | 44 +++++++++------- 3 files changed, 58 insertions(+), 39 deletions(-) diff --git a/scripts/acceptance/command-keys/published_cohort.py b/scripts/acceptance/command-keys/published_cohort.py index 45d01ae0..7b2aa0ae 100644 --- a/scripts/acceptance/command-keys/published_cohort.py +++ b/scripts/acceptance/command-keys/published_cohort.py @@ -40,6 +40,12 @@ EMPTY = {'codec': 'avro', 'blob': 'wwHioz3/VYAiNwA='} ONE = {'codec': 'avro', 'blob': 'wwHioz3/VYAiNwQC'} NIL_RUN = '00000000000000000000000000' +REDACTED = '[redacted]' +WORKFLOWS = '/api/workflows/' +LEGACY_DB = 'legacy.sqlite' +MANIFEST_NAME = 'command-key-legacy-v1-vectors.json' +HEX = frozenset('0123456789abcdef') +CONCURRENCY = 4 CHECKS, INVOCATIONS = [], [] STATE = {} @@ -75,7 +81,7 @@ def __init__(self, options, work, token): def docker(self, *args, check=True): result = subprocess.run(['docker', *args], capture_output=True, text=True, timeout=60) if check and result.returncode: - raise RuntimeError('docker ' + args[0] + ' failed: ' + result.stderr.replace(self.token, '[redacted]')[:2000]) + raise RuntimeError('docker ' + args[0] + ' failed: ' + result.stderr.replace(self.token, REDACTED)[:2000]) return result.stdout.strip() def start(self, database): @@ -174,7 +180,7 @@ def message(kind, workflow, key=None, value=1, name=None, run=None, namespace='d def path_for(kind, fixture, run=None): - path = '/api/workflows/' + fixture['workflow'] + path = WORKFLOWS + fixture['workflow'] if run: path += '/runs/' + run return path + '/' + kind + '/' + ('finish' if kind == 'signal' else 'set') @@ -218,7 +224,7 @@ def fixture(kind, tag, namespace='default', workflow=None): def history(f, run=None): - status, body = request('GET', '/api/workflows/' + f['workflow'] + '/runs/' + (run or f['run']) + '/history', + status, body = request('GET', WORKFLOWS + f['workflow'] + '/runs/' + (run or f['run']) + '/history', namespace=f['namespace']) require(status == 200, 'history read', body) return body['events'] @@ -256,7 +262,8 @@ def original(kind, f, key=None, lost=False, run=None): chosen = answer.get('request_id') require(isinstance(chosen, str) and chosen != '', kind + ': CLI reports the command key it sent', answer) if key is None: - require(re.fullmatch('cli-' + kind + '-[0-9a-f]{32}', chosen) is not None, kind + ': generated key shape') + suffix = chosen.removeprefix('cli-' + kind + '-') + require(suffix != chosen and len(suffix) == 32 and set(suffix) <= HEX, kind + ': generated key shape', chosen) else: require(chosen == key, kind + ': CLI reports the explicit key') if lost: @@ -354,13 +361,13 @@ def run_modern(info): explicit = original(kind, f, key=kind + '-explicit-loss', lost=True) retained(kind, f, explicit) key = kind + '-concurrent' - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: - replies = list(pool.map(lambda _: message(kind, f['workflow'], key), range(4))) + with concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + replies = list(pool.map(lambda _, k=kind, w=f['workflow'], r=key: message(k, w, r), range(CONCURRENCY))) require(len({r['command_id'] for r in replies}) == 1 and sum(r.get('retained') is True for r in replies) == 3, kind + ': four simultaneous first sends admit exactly one command', replies) before = snapshot() - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: - again = list(pool.map(lambda _: message(kind, f['workflow'], key), range(4))) + with concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + again = list(pool.map(lambda _, k=kind, w=f['workflow'], r=key: message(k, w, r), range(CONCURRENCY))) require(all(r.get('retained') is True and r['command_id'] == replies[0]['command_id'] for r in again), kind + ': four concurrent identical retries return the original receipt') require(snapshot() == before, kind + ': concurrent retries write nothing') @@ -368,8 +375,8 @@ def run_modern(info): kind + ': history has one admission per key') for label, unicode_key in (('ascii', 'k' * 255), ('two-byte', 'é' * 255), ('four-byte', '🙂' * 255)): unicode_record = original(kind, f, key=unicode_key) - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: - list(pool.map(lambda _: retained(kind, f, unicode_record), range(4))) + with concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + list(pool.map(lambda _, k=kind, x=f, r=unicode_record: retained(k, x, r), range(CONCURRENCY))) require(len(unicode_key) == 255, kind + ': 255-character ' + label + ' key retained') before, wire = snapshot(), len(STATE['proxy'].records) message(kind, f['workflow'], key='é' * 256, namespace=f['namespace'], exit_code=2) @@ -393,7 +400,7 @@ def run_modern(info): require(reason in ('run_not_active', 'run_closed', 'workflow_closed', 'historical_run_command_rejected', 'workflow_not_running', 'run_not_current'), kind + ': ' + action + ' fence reason', reason) if action == 'continue_as_new': - status, described = request('GET', '/api/workflows/' + f['workflow']) + status, described = request('GET', WORKFLOWS + f['workflow']) successor = described.get('run_id') require(status == 200 and successor and successor != f['run'], kind + ': continued run has a successor') successor_events = [e['event_type'] for e in history(f, successor)] @@ -452,10 +459,10 @@ def select(kind, f, record, stored): source = STATE['work'] / 'data' / 'seed.sqlite' with sqlite3.connect(source) as connection: require(connection.execute('PRAGMA wal_checkpoint(TRUNCATE)').fetchone()[0] == 0, 'stopped seed checkpointed') - transform = STATE['helper'].transform_copy(source, STATE['work'] / 'data' / 'legacy.sqlite', selections, stopped=True) + transform = STATE['helper'].transform_copy(source, STATE['work'] / 'data' / LEGACY_DB, selections, stopped=True) require(transform['transformed'] == len(selections), 'helper rewrote exactly the selected admissions', transform) - STATE['database'] = 'legacy.sqlite' - info = STATE['runtime'].start('legacy.sqlite') + STATE['database'] = LEGACY_DB + info = STATE['runtime'].start(LEGACY_DB) require(info['version'] == STATE['options'].server_version, 'same Server version on the legacy copy') STATE['proxy'] = STATE['proxy_type'](STATE['runtime'].base) threading.Thread(target=STATE['proxy'].serve_forever, kwargs={'poll_interval': .01}, daemon=True).start() @@ -477,8 +484,9 @@ def readback(case, run=None): label = kind + ':' + case['shape'] + ':' + case.get('state', 'open') before = snapshot() if case['shape'] == 'readback': - with concurrent.futures.ThreadPoolExecutor(max_workers=manifest['legacy_readback']['concurrency']) as pool: - replies = list(pool.map(lambda _: readback(case), range(manifest['legacy_readback']['concurrency']))) + require(manifest['legacy_readback']['concurrency'] == CONCURRENCY, 'accepted vector concurrency') + with concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + replies = list(pool.map(lambda _, c=case: readback(c), range(CONCURRENCY))) require(len({r['command_id'] for r in replies}) == 1, label + ': concurrent readers get one original identity') readback(case, run=f['run']) human = subprocess.run([STATE['cli'], 'workflow:' + kind, f['workflow'], @@ -542,13 +550,13 @@ def main(): options.inputs / 'committed_response_loss.py') helper = module(verified(options.inputs / 'sqlite_legacy.py', HELPER_SHA256, 'SQLite helper'), 'sqlite_legacy', options.inputs / 'sqlite_legacy.py') - manifest_bytes = verified(options.inputs / 'command-key-legacy-v1-vectors.json', MANIFEST_SHA256, 'legacy manifest') + manifest_bytes = verified(options.inputs / MANIFEST_NAME, MANIFEST_SHA256, 'legacy manifest') work = options.work_dir work.mkdir(mode=0o700) for name in ('data', 'home', 'config', 'tmp', 'bin'): (work / name).mkdir(mode=0o700) - (work / 'command-key-legacy-v1-vectors.json').write_bytes(manifest_bytes) - helper.MANIFEST = work / 'command-key-legacy-v1-vectors.json' + (work / MANIFEST_NAME).write_bytes(manifest_bytes) + helper.MANIFEST = work / MANIFEST_NAME binary = work / 'bin' / 'dw' with open(os.open(binary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o500), 'wb') as output: output.write(cli_bytes) @@ -580,7 +588,7 @@ def main(): transform = run_legacy(json.loads(manifest_bytes)) outcome = 'pass' except Exception as failure: # report the first failure, then clean up - error = (type(failure).__name__ + ': ' + str(failure)).replace(token, '[redacted]') + error = (type(failure).__name__ + ': ' + str(failure)).replace(token, REDACTED) finally: if STATE.get('proxy'): STATE['proxy'].shutdown() @@ -603,7 +611,7 @@ def main(): 'runner_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), 'container_ids': runtime.ids, 'cleanup': cleanup, 'transform': transform} (work / 'checks.json').write_text(json.dumps(CHECKS, indent=1)) - (work / 'invocations.json').write_text(json.dumps(INVOCATIONS, indent=1).replace(token, '[redacted]')) + (work / 'invocations.json').write_text(json.dumps(INVOCATIONS, indent=1).replace(token, REDACTED)) print(json.dumps(report), flush=True) return 0 if outcome == 'pass' else 1 diff --git a/sonar-project.properties b/sonar-project.properties index f9a46039..9c691db1 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -4,7 +4,8 @@ sonar.projectName=cli sonar.sourceEncoding=UTF-8 sonar.sources=. sonar.tests=. -sonar.test.inclusions=tests/**,rust/parity/**,rust/src/compat/fixtures/**,**/tests/**,**/*.test.*,**/*.spec.*,**/test_*.py,benchmarks/**,scripts/conformance/** +# Operator-run acceptance harnesses (scripts/acceptance) are test code, like scripts/conformance. +sonar.test.inclusions=tests/**,rust/parity/**,rust/src/compat/fixtures/**,**/tests/**,**/*.test.*,**/*.spec.*,**/test_*.py,benchmarks/**,scripts/conformance/**,scripts/acceptance/** # Dependencies, generated assets and disposable runtime/build output are not source. sonar.exclusions=**/vendor/**,**/node_modules/**,**/target/**,**/dist/**,**/build/**,**/storage/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/.scannerwork/**,**/.git/**,static/vendor/**,public/vendor/**,resources/js/vendor/** sonar.test.exclusions=**/vendor/**,**/node_modules/**,**/target/**,**/.venv/** diff --git a/src/Commands/BaseCommand.php b/src/Commands/BaseCommand.php index fc0ab146..a13ce78f 100644 --- a/src/Commands/BaseCommand.php +++ b/src/Commands/BaseCommand.php @@ -743,10 +743,13 @@ protected function writeCommandReceiptLines(OutputInterface $output, array $resu if (is_bool($result['intent_verified'] ?? null)) { // A legacy receipt is retained without a recorded intent, so the // Server returned it without comparing this command to it. - $legacy = $result['intent_verified'] === false && ($result['retained'] ?? null) === true; - $output->writeln(' Intent Verified: '.($result['intent_verified'] - ? 'yes' - : ($legacy ? "no (legacy receipt; the Server did not compare this command's intent with it)" : 'no'))); + $verified = 'no'; + if ($result['intent_verified']) { + $verified = 'yes'; + } elseif (($result['retained'] ?? null) === true) { + $verified = "no (legacy receipt; the Server did not compare this command's intent with it)"; + } + $output->writeln(' Intent Verified: '.$verified); } } @@ -1827,6 +1830,25 @@ private function baseErrorRecommendations(\Throwable $e, ?InputInterface $input) }; } + /** + * @return array{id: string, severity: string, message: string} + */ + private function requestIdAmbiguousRecommendation(ServerHttpException $e, ?InputInterface $input): array + { + $requestId = $this->nonEmptyScalarString(($e->body ?? [])['request_id'] ?? null) + ?? ($input !== null ? $this->errorRequestIdContext($input) : null) + ?? 'given'; + + return [ + 'id' => 'request_id.ambiguous', + 'severity' => 'error', + 'message' => sprintf( + 'Request id [%s] matches more than one retained command, so the Server recorded nothing and selected no receipt. Retrying it unchanged fails the same way. Inspect workflow history to see which commands were admitted, and pass a new --request-id to record a new command.', + $requestId, + ), + ]; + } + /** * @return list */ @@ -1870,19 +1892,7 @@ private function httpErrorRecommendations(ServerHttpException $e, ?InputInterfac } if ($e->statusCode === 409 && $e->reason() === self::REQUEST_ID_AMBIGUOUS_REASON) { - $body = $e->body ?? []; - $requestId = $this->nonEmptyScalarString($body['request_id'] ?? null) - ?? ($input !== null ? $this->errorRequestIdContext($input) : null) - ?? 'given'; - - return [[ - 'id' => 'request_id.ambiguous', - 'severity' => 'error', - 'message' => sprintf( - 'Request id [%s] matches more than one retained command, so the Server recorded nothing and selected no receipt. Retrying it unchanged fails the same way. Inspect workflow history to see which commands were admitted, and pass a new --request-id to record a new command.', - $requestId, - ), - ]]; + return [$this->requestIdAmbiguousRecommendation($e, $input)]; } if ($e->statusCode >= 400 && $e->statusCode <= 499) {