diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 8a842537..ff26e111 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -74,10 +74,19 @@ jobs: include: - adapter: cloudflare target: wasm32-unknown-unknown + features: cloudflare - adapter: fastly target: wasm32-wasip1 + features: fastly + # The `cli` feature is a valid wasip1 build combination (the CLI host + # code compiled for the guest target); gate it so a dead-code or + # portability regression there is caught, not only in the fastly-only build. + - adapter: fastly + target: wasm32-wasip1 + features: fastly cli - adapter: spin target: wasm32-wasip2 + features: spin steps: - uses: actions/checkout@v6 @@ -110,8 +119,8 @@ jobs: - name: Fetch dependencies (locked) run: cargo fetch --locked - - name: Clippy ${{ matrix.adapter }} on ${{ matrix.target }} - run: cargo clippy -p edgezero-adapter-${{ matrix.adapter }} --features ${{ matrix.adapter }} --target ${{ matrix.target }} --all-targets -- -D warnings + - name: Clippy ${{ matrix.adapter }} (${{ matrix.features }}) on ${{ matrix.target }} + run: cargo clippy -p edgezero-adapter-${{ matrix.adapter }} --features "${{ matrix.features }}" --target ${{ matrix.target }} --all-targets -- -D warnings format-docs: runs-on: ubuntu-latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a390532..b07d4907 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -224,5 +224,15 @@ jobs: ${{ matrix.runner_env }}: ${{ matrix.runner_value }} run: cargo test -p edgezero-adapter-${{ matrix.adapter }} --features ${{ matrix.adapter }} --target ${{ matrix.target }} --test contract + # The colocated runtime unit tests (config-store resolver, chunk + # remediation) need the `fastly` feature and only RUN under Viceroy — the + # `--test contract` line above never exercises them. Run the lib target so + # the runtime fail-closed/remediation behaviour is actually verified in CI. + - name: Run fastly wasm runtime unit tests + if: matrix.adapter == 'fastly' + env: + ${{ matrix.runner_env }}: ${{ matrix.runner_value }} + run: cargo test -p edgezero-adapter-fastly --features fastly --target wasm32-wasip1 --lib + - name: Check ${{ matrix.adapter }} wasm target run: cargo check -p edgezero-adapter-${{ matrix.adapter }} --features ${{ matrix.adapter }} --target ${{ matrix.target }} diff --git a/Cargo.lock b/Cargo.lock index 93e63b90..6966641c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -719,6 +719,7 @@ dependencies = [ "flate2", "futures", "futures-util", + "handlebars", "log", "log-fastly", "serde", diff --git a/crates/edgezero-adapter-fastly/Cargo.toml b/crates/edgezero-adapter-fastly/Cargo.toml index 08c4832d..02ae5a83 100644 --- a/crates/edgezero-adapter-fastly/Cargo.toml +++ b/crates/edgezero-adapter-fastly/Cargo.toml @@ -49,4 +49,5 @@ walkdir = { workspace = true, optional = true } [dev-dependencies] edgezero-core = { path = "../edgezero-core", features = ["test-utils"] } +handlebars = { workspace = true } tempfile = { workspace = true } diff --git a/crates/edgezero-adapter-fastly/src/chunked_config.rs b/crates/edgezero-adapter-fastly/src/chunked_config.rs index 6c8b0937..55c3e63a 100644 --- a/crates/edgezero-adapter-fastly/src/chunked_config.rs +++ b/crates/edgezero-adapter-fastly/src/chunked_config.rs @@ -21,19 +21,27 @@ use sha2::{Digest as _, Sha256}; -/// Per-entry value limit enforced by Fastly Config Store. Used by the -/// CLI writer to gate direct-vs-chunked storage; the runtime resolver -/// reads chunk lengths from the pointer struct, not this constant. -#[cfg(any(feature = "cli", test))] +/// Per-entry value limit enforced by Fastly Config Store. Used by the CLI writer +/// to gate direct-vs-chunked storage, and by the pointer validator (which both +/// the CLI and the runtime resolver call) — a chunked envelope is always larger +/// than this, so a pointer claiming otherwise is not one this writer emitted. pub(crate) const FASTLY_CONFIG_ENTRY_LIMIT: usize = 8_000; -/// Target payload size per chunk (kept under the entry limit to leave -/// room for the key and any protocol overhead). CLI writer only. -#[cfg(any(feature = "cli", test))] +/// Maximum length of a Config Store KEY, in CHARACTERS. Fastly's docs disagree +/// (the guide says 255, the API reference says 256); we take the STRICTER 255 so +/// a key we emit is valid under either. The writer must not emit a physical key +/// longer than this — a derived chunk key adds ~85 characters to the root, so a +/// near-limit root would otherwise fail only mid-write. Also used by the pointer +/// validator (CLI and runtime): a pointer referencing an over-limit key is not +/// one this writer could have produced. +pub(crate) const FASTLY_CONFIG_KEY_LIMIT: usize = 255; +/// Target payload size per chunk (kept under the entry limit to leave room for +/// the key and any protocol overhead). The writer never emits a chunk larger +/// than this, so the pointer validator (CLI and runtime) uses it as an upper +/// bound on any single chunk's declared length. pub(crate) const CHUNK_PAYLOAD_TARGET: usize = 7_000; -/// Infix inserted between the root key and the content-address in a -/// chunk key: `.__edgezero_chunks..`. CLI writer -/// only; the resolver reads chunk keys from the pointer struct. -#[cfg(any(feature = "cli", test))] +/// Infix inserted between the root key and the content-address in a chunk key: +/// `.__edgezero_chunks..`. Used by the writer and by the +/// pointer validator (CLI and runtime) to recognise a canonical chunk key. pub(crate) const CHUNK_KEY_INFIX: &str = ".__edgezero_chunks."; /// `edgezero_kind` discriminant stored in the pointer JSON. Used by /// BOTH the writer (when serialising the pointer) AND the resolver @@ -54,6 +62,61 @@ struct FastlyChunkPointer { version: u8, } +/// What a stored root value announces itself to be, via `edgezero_kind`. +enum RootValueKind { + /// Definitely not ours — a plain string, a scalar, a `BlobEnvelope`, or a + /// COMPLETE, parseable JSON object that carries no `edgezero_kind`. The + /// Config Store holds arbitrary values, so the runtime returns these + /// verbatim, and GC can safely treat them as inert zero-reference roots. + Foreign, + /// Starts like a JSON object (`{`) but does NOT parse. The runtime still + /// returns it verbatim, but GC must NOT assume it is inert: it could be a + /// TRUNCATED or corrupted chunk pointer whose (unreadable) chunk references + /// would be orphaned if we treated it as a zero-reference root. GC fails + /// closed on it. Chunk PAYLOADS also land here (a raw envelope fragment + /// starting `{"data"...` does not parse), which is why the chunk-shaped + /// delete-candidate path is decided before this ever matters. + MalformedObject, + /// One of our v1 chunk pointers. + Pointer, + /// Carries our reserved `edgezero_kind` field but names a kind this build + /// does not understand — corrupt or future-versioned state. + UnknownKind, +} + +/// Why resolving a stored root value failed. Keeping the future-format case +/// DISTINCT from corruption is the whole point: a newer format must never be +/// overwritten (CLI) or read as repairable corruption, whereas genuinely corrupt +/// or incomplete state IS repairable by a push. The distinction has to survive +/// pointer resolution — a v2 envelope reassembled from v1 chunks is only knowable +/// AFTER the chunks are fetched — so it cannot be recovered from the raw value +/// alone. +pub(crate) enum ResolveFailure { + /// Corrupt or incomplete state a push can repair by overwriting. Carries a + /// redacted, human-readable message. + Corrupt(String), + /// The value (or the envelope it reconstructs to) uses a config format this + /// build does not recognise: an unknown/newer `edgezero_kind`, or a bumped + /// envelope/pointer `version`. Carries a redacted, human-readable message. + FutureFormat(String), +} + +impl ResolveFailure { + /// The redacted, human-readable message, discarding the category. + pub(crate) fn into_message(self) -> String { + match self { + Self::FutureFormat(message) | Self::Corrupt(message) => message, + } + } + + /// Is this a newer format this build must not overwrite (vs. repairable + /// corruption)? + #[cfg(any(feature = "cli", feature = "fastly", test))] + pub(crate) fn is_future_format(&self) -> bool { + matches!(self, Self::FutureFormat(_)) + } +} + #[derive(serde::Deserialize, serde::Serialize)] struct FastlyChunkRef { key: String, @@ -61,19 +124,46 @@ struct FastlyChunkRef { sha256: String, } +/// A chunk reference from a validated pointer, for `config gc`. +#[cfg(any(feature = "cli", test))] +pub(crate) struct GcChunkRef { + pub key: String, + pub len: usize, + pub sha256: String, +} + +/// A validated v1 pointer's contents, for `config gc`. +#[cfg(any(feature = "cli", test))] +pub(crate) struct GcPointer { + /// Validated: non-empty, one generation, dense `0..n-1` in order. + pub chunks: Vec, + pub envelope_len: usize, + pub envelope_sha256: String, +} + +/// What a root's value IS, once classified for `config gc`. +#[cfg(any(feature = "cli", test))] +pub(crate) enum GcRootValue { + /// A valid v1 chunk pointer. Its METADATA is validated; its CONTENT must + /// still be verified against the store -- see [`gc_verify_generation`]. + Chunked(GcPointer), + /// A valid, integrity-checked envelope stored inline. References no chunks. + Direct, +} + // --------------------------------------------------------------------------- // Public helpers // --------------------------------------------------------------------------- /// Compute the lowercase-hex SHA-256 of `bytes`. -fn sha256_hex(bytes: &[u8]) -> String { +pub(crate) fn sha256_hex(bytes: &[u8]) -> String { format!("{:x}", Sha256::digest(bytes)) } /// Prepare the physical Config Store entries for a single logical /// `(root_key, envelope_json)` pair. /// -/// - If `envelope_json.len() <= 8_000`: returns one `(root_key, envelope_json)` tuple unchanged. +/// - If `envelope_json` is at most 8 000 BYTES (a conservative UTF-8 byte check): returns one `(root_key, envelope_json)` tuple unchanged. /// - Otherwise: splits into UTF-8-safe 7 000-byte chunks, returns chunk /// entries first and the root pointer entry last. /// @@ -83,11 +173,54 @@ fn sha256_hex(bytes: &[u8]) -> String { /// /// Returns an error string if the pointer JSON itself would exceed 8 000 /// characters (extremely unlikely in practice; recommends restructuring). +/// Reject a physical Config Store key that exceeds the store's key limit, +/// before any write is attempted. +#[cfg(any(feature = "cli", test))] +fn check_config_key_len(key: &str) -> Result<(), String> { + // CHARACTERS, not bytes: Fastly's limit is a character count, so a non-ASCII + // `--key` must be measured by `chars().count()`, not `len()` (UTF-8 bytes). + let char_len = key.chars().count(); + if char_len > FASTLY_CONFIG_KEY_LIMIT { + return Err(format!( + "config-store key is {char_len} characters, over Fastly's \ + {FASTLY_CONFIG_KEY_LIMIT}-character limit. Chunked storage derives keys of the form \ + `.__edgezero_chunks.<64-char-sha>.`, adding ~85 characters to the root, \ + so a long store id or `--key` override can push the derived key past the limit. Use a \ + shorter store id / `--key`." + )); + } + Ok(()) +} + #[cfg(any(feature = "cli", test))] pub(crate) fn prepare_fastly_config_entries( root_key: &str, envelope_json: &str, ) -> Result, String> { + // An EMPTY root key is writer-valid but resolver-INVALID: canonical chunk + // parsing rejects an empty root, so a chunked empty-key push would write + // chunks that can never be reassembled (and a partial cloud push could commit + // them before the final pointer write fails). Reject it before any I/O. + if root_key.is_empty() { + return Err( + "config key is empty; an empty key cannot be resolved at runtime. Provide a store id \ + or a non-empty `--key`." + .to_owned(), + ); + } + // The root key itself is a physical key on both paths (the direct entry and + // the pointer entry), so it must fit the store's key limit before anything + // is written. + check_config_key_len(root_key)?; + + // Direct-vs-chunked uses a conservative UTF-8 BYTE-count check (spec + // 2026-06-16 permits this and the merge-base writer used it). Byte length is + // >= character length, so this chunks whenever a stricter character check + // would AND for a few multibyte values that fit the character cap — which is + // safe (it never over-stores) and, crucially, is what EXISTING v1 pointers + // were written against. A character-based threshold would leave a value that + // is <= 8000 chars but > 8000 bytes stored directly, and REJECT the existing + // chunked pointer for it on read (an HTTP 500) — a v1 read-compat break. if envelope_json.len() <= FASTLY_CONFIG_ENTRY_LIMIT { return Ok(vec![(root_key.to_owned(), envelope_json.to_owned())]); } @@ -105,16 +238,10 @@ pub(crate) fn prepare_fastly_config_entries( }) .unwrap_or_default(); - // Split env_bytes into UTF-8-safe chunks of at most CHUNK_PAYLOAD_TARGET bytes. + // Split env_bytes into UTF-8-safe chunks of at most CHUNK_PAYLOAD_TARGET + // bytes, using the shared span computation the resolver replays. let mut chunks: Vec<(String, String, usize)> = Vec::new(); // (key, value, len) - let mut start = 0_usize; - let mut idx = 0_usize; - while start < env_bytes.len() { - // Walk forward by CHUNK_PAYLOAD_TARGET bytes, then retreat to the - // last codepoint boundary so we never split a multi-byte codepoint. - let end_raw = (start.saturating_add(CHUNK_PAYLOAD_TARGET)).min(env_bytes.len()); - // Find the largest valid UTF-8 boundary <= end_raw. - let end = find_utf8_boundary(envelope_json, start, end_raw); + for (idx, (start, end)) in writer_chunk_spans(envelope_json).into_iter().enumerate() { let chunk_bytes = env_bytes.get(start..end).ok_or_else(|| { format!("chunk boundary [{start}..{end}] out of range for `{root_key}`") })?; @@ -122,9 +249,12 @@ pub(crate) fn prepare_fastly_config_entries( format!("chunk [{start}..{end}] for `{root_key}` is not valid UTF-8: {err}") })?; let chunk_key = format!("{root_key}{CHUNK_KEY_INFIX}{envelope_sha256}.{idx}"); + // A chunk key adds the infix + a 64-char SHA + an index to the root key + // (~85 chars). A root that is itself valid can push the derived key past + // the store limit; reject it here rather than fail mid-write with some + // chunks already committed. + check_config_key_len(&chunk_key)?; chunks.push((chunk_key, chunk_str.to_owned(), chunk_bytes.len())); - start = end; - idx = idx.saturating_add(1); } // Build chunk refs with per-chunk SHAs. @@ -165,152 +295,1942 @@ pub(crate) fn prepare_fastly_config_entries( for (key, value, _) in chunks { entries.push((key, value)); } - entries.push((root_key.to_owned(), pointer_json)); - Ok(entries) -} + entries.push((root_key.to_owned(), pointer_json)); + Ok(entries) +} + +/// Find the largest byte offset `<= end_raw` that is a valid UTF-8 +/// codepoint boundary within `src`. `start` is used as a hint so we +/// don't scan the entire string from zero each time. +fn find_utf8_boundary(src: &str, start: usize, end_raw: usize) -> usize { + if end_raw >= src.len() { + return src.len(); + } + // Walk backwards from end_raw until we land on a valid char boundary. + let mut boundary = end_raw; + while boundary > start && !src.is_char_boundary(boundary) { + boundary = boundary.saturating_sub(1); + } + boundary +} + +/// The exact byte spans this writer splits an over-limit envelope into: walk +/// forward `CHUNK_PAYLOAD_TARGET` bytes, then retreat to the previous UTF-8 +/// codepoint boundary so no chunk ever splits a codepoint. +/// +/// SINGLE SOURCE OF TRUTH for the split layout. `prepare_fastly_config_entries` +/// builds its chunks from these spans, and the resolver replays them against a +/// reconstructed envelope to prove the pointer's boundaries are exactly the ones +/// this writer emits — so the two can never drift. Un-gated on purpose: the +/// runtime resolver (no `cli` feature) needs it for that check. +fn writer_chunk_spans(envelope_json: &str) -> Vec<(usize, usize)> { + let len = envelope_json.len(); + let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut start = 0_usize; + while start < len { + let end_raw = (start.saturating_add(CHUNK_PAYLOAD_TARGET)).min(len); + let end = find_utf8_boundary(envelope_json, start, end_raw); + spans.push((start, end)); + start = end; + } + spans +} + +/// Prove a pointer's chunk boundaries are EXACTLY the ones this writer emits for +/// the reconstructed envelope. +/// +/// Called after the per-chunk and whole-envelope SHAs have verified, so the +/// reconstructed BYTES are already proven correct; this pins the LAYOUT. Replay +/// the writer's span computation: the chunk count and every chunk's byte length +/// must match the pointer. That rejects a hand-authored pointer which reassembles +/// to the same bytes along boundaries the writer would never choose, so the +/// resolver accepts only writer-produced layouts -- the same guarantee +/// `prove_generation` enforces before a GC delete. +/// +/// Comparing LENGTHS is sufficient: the SHAs already pinned the bytes, so equal +/// boundaries pin each chunk to the writer's own chunk. +/// +/// Diagnostics name a chunk's POSITION, never its key: `chunks[].key` is +/// pointer-controlled and these messages are logged verbatim. +pub(crate) fn verify_writer_split_layout( + root_key: &str, + reconstructed: &str, + chunk_lens: &[usize], +) -> Result<(), String> { + let spans = writer_chunk_spans(reconstructed); + if spans.len() != chunk_lens.len() { + return Err(format!( + "chunk pointer at `{root_key}` names {} chunk(s), but this writer would split its \ + own reconstructed envelope into {}", + chunk_lens.len(), + spans.len() + )); + } + for (position, (&(start, end), &declared_len)) in + spans.iter().zip(chunk_lens.iter()).enumerate() + { + let expected_len = end.saturating_sub(start); + if expected_len != declared_len { + return Err(format!( + "chunk {position} referenced by pointer at `{root_key}` records {declared_len} \ + bytes, but this writer splits the reconstructed envelope into {expected_len} \ + bytes at that boundary" + )); + } + } + Ok(()) +} + +/// The declared byte length of each chunk a pointer references, in order — the +/// input [`verify_writer_split_layout`] compares against the writer's own split. +/// Only the GC path (`cli.rs`) needs it; the runtime resolver inlines the same +/// mapping, so this is gated to the `cli` feature to stay dead-code-free in the +/// guest build. +#[cfg(feature = "cli")] +pub(crate) fn chunk_lengths(chunks: &[GcChunkRef]) -> Vec { + chunks.iter().map(|chunk| chunk.len).collect() +} + +/// Resolve a logical `(root_key, root_value)` pair from the store, transparently +/// reassembling one of OUR chunk pointers and passing every other value through. +/// +/// A Config Store holds ARBITRARY values — the shared `ConfigStore` contract +/// requires `get` to return whatever bytes are stored. So the ONLY values this +/// touches are the ones that announce themselves via `edgezero_kind`: +/// +/// 1. Not ours (a plain string, a direct `BlobEnvelope`, unrelated JSON, or +/// anything that is not a JSON object) → returned VERBATIM. Envelope +/// integrity is the typed app-config layer's job, which parses and verifies +/// it there; doing it here would reject ordinary entries. +/// 2. `edgezero_kind == POINTER_KIND` → resolved: parse and validate the pointer +/// metadata, fetch each chunk, verify each chunk's length and SHA, +/// concatenate in pointer order, verify `envelope_len` + `envelope_sha256` +/// and the exact writer split layout, then return the reconstructed envelope. +/// 3. `edgezero_kind` present but naming a kind this build does not understand → +/// a [`ResolveFailure::FutureFormat`] error. That field is our reserved +/// namespace, so an unrecognised value in it is future-versioned state. +/// +/// # Errors +/// +/// Returns a typed [`ResolveFailure`] naming the root key or the failing chunk +/// POSITION when any integrity check fails, distinguishing a NEWER format (must +/// not be overwritten) from repairable corruption. [`resolve_fastly_config_value`] +/// is the string-erased wrapper for callers that do not need the distinction. +pub(crate) fn resolve_fastly_config_value_typed( + root_key: &str, + root_value: String, + mut fetch: F, +) -> Result +where + F: FnMut(&str) -> Result, String>, +{ + match classify_root_value(&root_value) { + // Not ours (a foreign value, or an object-shaped value that does not + // parse): the Config Store contract returns stored bytes untouched. + // Envelope integrity is the typed app-config layer's job downstream. + RootValueKind::Foreign | RootValueKind::MalformedObject => return Ok(root_value), + RootValueKind::UnknownKind => { + // An unrecognised `edgezero_kind` is a NEWER/foreign format, not + // repairable corruption. The stored kind is not echoed: it is a + // value-controlled string on a path whose diagnostics are logged. + return Err(ResolveFailure::FutureFormat(format!( + "value at key `{root_key}` has an unknown `edgezero_kind` (value redacted); \ + expected `{POINTER_KIND}`" + ))); + } + RootValueKind::Pointer => {} + } + + // --- It is one of ours: resolve it. --- + // Detect a NEWER pointer version BEFORE deserializing the exact v1 struct: an + // incomplete future pointer (e.g. one that renamed/dropped fields) would + // otherwise fail the struct parse and read as repairable corruption a + // downgrade push could overwrite. Keys on `version` alone (schema-agnostic). + // This inspects OUR reserved namespace (a pointer), never an arbitrary direct + // value -- those are already returned verbatim above. + if value_is_future_format(&root_value) { + return Err(ResolveFailure::FutureFormat(format!( + "chunk pointer at `{root_key}` uses a pointer version this build does not recognise (a \ + newer format)" + ))); + } + // The pointer entry itself is a single Config Store value, so the writer can + // never emit one larger than the entry limit. Reject an over-limit pointer up + // front — this bounds how many chunk refs it can carry (hence the fetch + // fan-out) before any further parsing. + if root_value.len() > FASTLY_CONFIG_ENTRY_LIMIT { + return Err(ResolveFailure::Corrupt(format!( + "chunk pointer at `{root_key}` is larger than the {FASTLY_CONFIG_ENTRY_LIMIT}-character \ + entry limit and so is not one this writer could have stored" + ))); + } + let pointer: FastlyChunkPointer = serde_json::from_str(&root_value).map_err(|err| { + ResolveFailure::Corrupt(format!( + "value at key `{root_key}` announces itself as a chunk pointer but is not a valid \ + one ({})", + redact_json_err(&err) + )) + })?; + + if pointer.edgezero_kind != POINTER_KIND { + // An unrecognised `edgezero_kind` is a NEWER/foreign format, not + // repairable corruption. The stored kind is not echoed: it is a + // value-controlled string on a path whose diagnostics are logged. + return Err(ResolveFailure::FutureFormat(format!( + "chunk pointer at `{root_key}` has an unknown `edgezero_kind` (value redacted); \ + expected `{POINTER_KIND}`" + ))); + } + if pointer.version != 1 { + // A bumped POINTER version is a newer format, not corruption. + return Err(ResolveFailure::FutureFormat(format!( + "chunk pointer at `{root_key}` has unsupported version \ + {}; expected 1", + pointer.version + ))); + } + // Validate the pointer METADATA before fetching anything: a shape the writer + // could never emit (non-canonical keys, mixed generations, gaps, per-chunk + // lengths over the payload target, a sum that disagrees with `envelope_len`, + // or an `envelope_len` small enough to have been stored directly) is + // rejected up front rather than driving a fan-out of chunk fetches whose + // hashes could only fail at the end. Same validator the CLI/GC path uses. + validate_pointer_chunks(root_key, &pointer).map_err(ResolveFailure::Corrupt)?; + + // Fetch, verify, and concatenate all chunks. + // + // NOT `with_capacity(pointer.envelope_len)`: that length is untrusted stored + // metadata, and this runs in the edge guest. A pointer declaring `usize::MAX` + // would abort the worker on a capacity overflow before any of the checks + // below could reject it. Grow from the bytes we actually fetch instead. + let mut reconstructed = String::new(); + for (position, chunk_ref) in pointer.chunks.iter().enumerate() { + let chunk_value = fetch(&chunk_ref.key) + .map_err(|_err| { + // The callback's error carries the chunk KEY (pointer-controlled), + // so it is not propagated verbatim. Position locates the fault. + ResolveFailure::Corrupt(format!( + "chunk {position} referenced by pointer at `{root_key}` could not be fetched \ + (details redacted)" + )) + })? + .ok_or_else(|| { + ResolveFailure::Corrupt(format!( + "missing chunk {position} referenced by pointer at `{root_key}`" + )) + })?; + + // Verify length. + if chunk_value.len() != chunk_ref.len { + return Err(ResolveFailure::Corrupt(format!( + "chunk {position} (referenced by `{root_key}`) has length {} but the pointer \ + records {}", + chunk_value.len(), + chunk_ref.len, + ))); + } + + // Verify SHA. Neither hash is echoed: the expected one comes from the + // stored pointer, so it is value-controlled. + if sha256_hex(chunk_value.as_bytes()) != chunk_ref.sha256 { + return Err(ResolveFailure::Corrupt(format!( + "chunk {position} (referenced by `{root_key}`) does not match the SHA-256 the \ + pointer records for it (hashes redacted)" + ))); + } + + reconstructed.push_str(&chunk_value); + } + + // Verify total envelope length. + if reconstructed.len() != pointer.envelope_len { + return Err(ResolveFailure::Corrupt(format!( + "reconstructed envelope for `{root_key}` has length {} but pointer \ + records {}", + reconstructed.len(), + pointer.envelope_len, + ))); + } + + // Verify total envelope SHA. + let actual_env_sha = sha256_hex(reconstructed.as_bytes()); + if actual_env_sha != pointer.envelope_sha256 { + // Neither SHA is echoed: the expected one is `pointer.envelope_sha256`, + // a stored, value-controlled string that a malformed pointer can set to + // anything (a secret), and this runs on the read path where diagnostics + // are logged. + return Err(ResolveFailure::Corrupt(format!( + "reconstructed envelope for `{root_key}` does not match the pointer's \ + `envelope_sha256` (hashes redacted)" + ))); + } + + let declared_lens: Vec = pointer.chunks.iter().map(|chunk| chunk.len).collect(); + verify_writer_split_layout(root_key, &reconstructed, &declared_lens) + .map_err(ResolveFailure::Corrupt)?; + + finalize_reconstructed_envelope(root_key, reconstructed) +} + +/// The final gate the chunked path shares with the direct one: reject a NEWER +/// inner envelope, then parse+verify the exact v1 `BlobEnvelope`. +/// +/// Version detection runs BEFORE v1 deserialization: the outer pointer is a valid +/// v1 pointer, so the raw value alone never reveals a bumped INNER envelope +/// version, and a v2 envelope may not even deserialize as v1 -- which would +/// otherwise erase into repairable corruption and let a downgrade push overwrite +/// a newer format. Parsing+verifying then guarantees the resolver only ever +/// returns a verified envelope, so no stored (possibly-secret) hash escapes into +/// core's own `verify()` and out through an HTTP 500. +fn finalize_reconstructed_envelope( + root_key: &str, + reconstructed: String, +) -> Result { + use edgezero_core::blob_envelope::BlobEnvelope; + + if value_is_future_format(&reconstructed) { + return Err(ResolveFailure::FutureFormat(format!( + "reconstructed envelope at key `{root_key}` uses an envelope version this build does \ + not recognise (a newer format); redeploy an updated build rather than overwrite it" + ))); + } + let envelope: BlobEnvelope = serde_json::from_str(&reconstructed).map_err(|err| { + ResolveFailure::Corrupt(format!( + "reconstructed value at key `{root_key}` is not a valid config envelope ({})", + redact_json_err(&err) + )) + })?; + envelope.verify().map_err(|_err| { + ResolveFailure::Corrupt(format!( + "reconstructed envelope at key `{root_key}` failed its integrity check (details \ + redacted -- the stored hashes are value-controlled)" + )) + })?; + Ok(reconstructed) +} + +/// String-erased wrapper over `resolve_fastly_config_value_typed` for the tests +/// that do not need to tell a NEWER format from repairable corruption. Production +/// callers use the typed form directly. +/// +/// # Errors +/// +/// Returns the redacted error message when any integrity check fails. +#[cfg(test)] +pub(crate) fn resolve_fastly_config_value( + root_key: &str, + root_value: String, + fetch: F, +) -> Result +where + F: FnMut(&str) -> Result, String>, +{ + resolve_fastly_config_value_typed(root_key, root_value, fetch) + .map_err(ResolveFailure::into_message) +} + +/// Validate a prior root value and return the chunk keys it referenced, +/// scoped to `root_key`'s own chunk namespace. Used only for chunk GC on +/// re-push (Stage 7 writeback). +/// +/// Parse as `serde_json::Value` FIRST so a pointer-kind value with +/// missing/invalid fields still reaches the warning path instead of being +/// silently dropped by a failed struct deserialize. +/// +/// Returns: +/// - `Ok(keys)` -- value is a valid v1 chunk pointer; `keys` are its +/// `chunks[].key` entries (all confirmed to match +/// `"{root_key}{CHUNK_KEY_INFIX}"`). +/// - `Ok(vec![])` -- value is a direct `BlobEnvelope`, absent, or not +/// pointer-shaped at all (normal: first push, or was-direct). Silent. +/// - `Err(msg)` -- value IS pointer-kind (`edgezero_kind == POINTER_KIND`) +/// but fails validation: malformed, unsupported `version`, or a +/// referenced key falls outside this root's chunk prefix. Callers log +/// `msg` as a warning and skip GC for this root (delete nothing). +#[cfg(any(feature = "cli", test))] +pub(crate) fn prior_chunk_keys(root_key: &str, raw: &str) -> Result, String> { + // 1. Parse loosely. Not-JSON, or not our pointer kind => silent. + let value: serde_json::Value = match serde_json::from_str(raw) { + Ok(value) => value, + Err(_) => return Ok(Vec::new()), + }; + if value + .get("edgezero_kind") + .and_then(serde_json::Value::as_str) + != Some(POINTER_KIND) + { + // Direct BlobEnvelope, unrelated JSON, or first push. + return Ok(Vec::new()); + } + + // 2. It IS pointer-kind: from here every failure WARNS (Err), never silent. + // The serde error is REDACTED -- its Display quotes the offending value + // ("invalid type: string \"hunter2\", expected u8"), and a stored config + // may hold credentials. Position + category only. + let pointer: FastlyChunkPointer = serde_json::from_value(value).map_err(|err| { + format!( + "prior chunk pointer at `{root_key}` is malformed ({}); skipping chunk GC", + redact_json_err(&err) + ) + })?; + if pointer.version != 1 { + return Err(format!( + "prior chunk pointer at `{root_key}` has unsupported version {}; skipping chunk GC", + pointer.version + )); + } + + // 3. The pointer must be INTERNALLY CONSISTENT, not merely well-typed. + // `chunks` is the authoritative live set on the GC path, so a pointer + // that parses but under-reports its chunks (empty list, an omitted + // index, a stale generation mixed in, lengths that cannot add up to the + // envelope) would leave real live chunks looking orphaned. Only accept a + // chunk list this writer could actually have emitted. + validate_pointer_chunks(root_key, &pointer) + .map_err(|err| format!("{err}; skipping chunk GC"))?; + + Ok(pointer.chunks.into_iter().map(|chunk| chunk.key).collect()) +} + +/// Describe a `serde_json` failure WITHOUT quoting the offending value. +/// +/// `serde_json::Error`'s `Display` embeds the input that failed to parse, and +/// these diagnostics are logged verbatim. Stored app config may hold secrets, so +/// only the position and category ever escape. +/// +/// Unconditional: the runtime resolver needs it too. A guest reading a malformed +/// pointer must not log the value either — that path runs on every request. +fn redact_json_err(err: &serde_json::Error) -> String { + use serde_json::error::Category; + + let category = match err.classify() { + Category::Io => "io", + Category::Syntax => "syntax", + Category::Data => "schema", + Category::Eof => "unexpected end of input", + }; + format!( + "{category} error at line {} column {} -- value redacted", + err.line(), + err.column() + ) +} + +/// Is this pointer's chunk list one `prepare_fastly_config_entries` could have +/// produced? Anything else is treated as unreadable rather than authoritative. +/// +/// Checks, in order: non-empty; every key canonical for this root; a SINGLE +/// generation matching `envelope_sha256`; indexes exactly `0..n-1`, each once, +/// in order; per-chunk lengths within what the writer emits; and +/// `sum(len) == envelope_len`, computed with CHECKED arithmetic. +/// +/// Called on BOTH the CLI/GC path and the runtime resolver, so a pointer shape +/// the writer could never emit is rejected everywhere, not just during GC. +/// +/// **Diagnostics name a chunk's POSITION, never its key.** `chunks[].key` is +/// pointer-controlled and, on this path, not yet validated -- a malformed +/// pointer can carry any string at all there (`prod-db-password=hunter2`), and +/// these messages are logged verbatim. +/// +/// **Lengths are untrusted input.** They are attacker-supplied `usize`s, so they +/// are bounded against what the writer can actually emit and summed with +/// `checked_add`. Saturating arithmetic would let a metadata-consistent pointer +/// declare `usize::MAX` and survive validation. +fn validate_pointer_chunks(root_key: &str, pointer: &FastlyChunkPointer) -> Result<(), String> { + let bad = |detail: &str| format!("chunk pointer at `{root_key}` {detail}"); + + if pointer.chunks.is_empty() { + // An oversized envelope always splits into >= 2 chunks, so an empty + // list is never something we wrote -- and it would report "references + // nothing", orphaning the real chunks. + return Err(bad("references no chunks at all")); + } + + let mut total_len = 0_usize; + for (position, chunk_ref) in pointer.chunks.iter().enumerate() { + // Every referenced key must be a CANONICAL chunk key of this root -- + // the same validator that gates deletion. + let Some((generation, index)) = chunk_key_parts(root_key, &chunk_ref.key) else { + return Err(bad(&format!( + "references a non-canonical chunk key at position {position}" + ))); + }; + // The physical key must fit the store's key limit — the writer never + // emits one that does not, so an over-limit key is not ours. + if chunk_ref.key.chars().count() > FASTLY_CONFIG_KEY_LIMIT { + return Err(bad(&format!( + "references a chunk key at position {position} longer than the \ + {FASTLY_CONFIG_KEY_LIMIT}-character store limit" + ))); + } + // One pointer names exactly one generation. A mixed list means some + // other generation's chunks are being reported as this one's. + if generation != pointer.envelope_sha256 { + return Err(bad(&format!( + "references a chunk at position {position} belonging to a different generation \ + than the pointer's own `envelope_sha256`" + ))); + } + // Indexes are dense, unique and ordered: 0, 1, ... n-1. This is what + // rejects an omitted, duplicated or reordered index -- each of which + // would silently shrink the live set. + if index != position { + return Err(bad(&format!( + "references chunk index {index} at position {position}, but a chunk list must be \ + indexed 0..n-1 with no gaps, duplicates or reordering" + ))); + } + if chunk_ref.len == 0 { + return Err(bad(&format!( + "references a zero-length chunk at position {position}" + ))); + } + // The writer never emits a chunk larger than its payload target, so a + // bigger one is not ours -- and this is what stops an absurd declared + // length ever reaching an allocation. + if chunk_ref.len > CHUNK_PAYLOAD_TARGET { + return Err(bad(&format!( + "references a chunk at position {position} declaring {} bytes, more than the \ + {CHUNK_PAYLOAD_TARGET}-byte maximum this writer emits", + chunk_ref.len + ))); + } + // Every chunk EXCEPT the last is a full payload: the writer walks + // `CHUNK_PAYLOAD_TARGET` bytes then retreats at most 3 to the previous + // UTF-8 boundary (the longest codepoint is 4 bytes), so a non-last chunk + // is always >= CHUNK_PAYLOAD_TARGET - 3. This pins the split layout to + // what the writer emits and bounds the chunk count (hence the runtime + // fetch fan-out) to about `envelope_len / CHUNK_PAYLOAD_TARGET`; a + // hand-authored pointer of many tiny chunks is rejected. + let is_last = position == pointer.chunks.len().saturating_sub(1); + if !is_last && chunk_ref.len < CHUNK_PAYLOAD_TARGET.saturating_sub(3) { + return Err(bad(&format!( + "references a non-final chunk at position {position} of only {} bytes; this writer \ + fills every chunk but the last to within 3 bytes of {CHUNK_PAYLOAD_TARGET}", + chunk_ref.len + ))); + } + total_len = total_len.checked_add(chunk_ref.len).ok_or_else(|| { + bad("declares chunk lengths that overflow when summed (not a real envelope)") + })?; + } + + // The parts must add up to the whole. If they don't, the pointer does not + // describe the envelope it claims to, so its chunk list is not trustworthy. + if total_len != pointer.envelope_len { + return Err(bad(&format!( + "declares `envelope_len` {} but its chunk lengths sum to {total_len}", + pointer.envelope_len + ))); + } + // We only chunk what does not fit directly, so a pointer describing an + // envelope that WOULD have fit is not one we wrote. + if pointer.envelope_len <= FASTLY_CONFIG_ENTRY_LIMIT { + return Err(bad(&format!( + "declares an envelope of {} bytes, which fits the {FASTLY_CONFIG_ENTRY_LIMIT}-byte \ + entry limit and so would never have been chunked by this writer", + pointer.envelope_len + ))); + } + + Ok(()) +} + +/// Classify a stored value by its `edgezero_kind`, without trusting it further. +/// +/// This is the single discriminant for "is this entry ours?", shared by the +/// runtime resolver and GC so the two can never disagree about what counts as a +/// pointer. Chunk payloads are raw envelope fragments and essentially never +/// parse as JSON, let alone carry `edgezero_kind`. +fn classify_root_value(raw: &str) -> RootValueKind { + // One of our pointers is always a JSON OBJECT. A value that does not even + // start like one (a scalar, a string, `greeting = "hello"`) can never be a + // pointer, corrupt or otherwise, so it is unambiguously foreign -- and this + // keeps ordinary string values off the JSON parser entirely. + if !raw.trim_start().starts_with('{') { + return RootValueKind::Foreign; + } + // It LOOKS like an object. If it does not parse, it is a malformed object -- + // possibly a truncated/corrupt pointer -- NOT a value GC may assume is inert. + let Ok(value) = serde_json::from_str::(raw) else { + return RootValueKind::MalformedObject; + }; + // Discriminate on the PRESENCE of `edgezero_kind`, not just a string match. A + // field of ANY type claims our reserved namespace, so a non-string value + // (`edgezero_kind: 7`, an object, null) is `UnknownKind`, never `Foreign` -- + // otherwise GC would wrongly treat it as an inert value and reclaim chunks a + // future/foreign format still references. + match value.get("edgezero_kind") { + None => RootValueKind::Foreign, + Some(serde_json::Value::String(kind)) if kind == POINTER_KIND => RootValueKind::Pointer, + Some(_) => RootValueKind::UnknownKind, + } +} + +/// Does this value announce itself as a chunk pointer? +/// +/// A cheap discriminant for "this entry is a ROOT, wherever it happens to live". +/// GC must not decide root-vs-chunk by KEY SHAPE: the runtime resolver follows +/// whatever pointer it is handed, so a pointer parked at a chunk-shaped key +/// still makes its references live. +#[cfg(test)] +pub(crate) fn value_is_pointer_kind(raw: &str) -> bool { + matches!(classify_root_value(raw), RootValueKind::Pointer) +} + +/// Is this value one GC may safely treat as an INERT, zero-reference root? +/// +/// True ONLY for a definitively foreign value — a scalar, a plain string, or a +/// complete JSON object without our discriminator. It is deliberately FALSE for +/// a malformed object (a possible truncated/corrupt pointer whose chunk +/// references we cannot read), an unknown-kind value, and a pointer: those must +/// fail closed, because assuming they reference no chunks could orphan live ones. +#[cfg(any(feature = "cli", test))] +pub(crate) fn value_is_inert_foreign(raw: &str) -> bool { + matches!(classify_root_value(raw), RootValueKind::Foreign) +} + +/// Does this value claim our reserved `edgezero_kind` namespace at all — the +/// recognised pointer kind, an unknown kind, or a non-string kind? +/// +/// GC uses this to keep a chunk-shaped key that holds a namespace-claiming value +/// OUT of the plain chunk-payload delete path: a real chunk payload is a raw +/// envelope fragment that announces nothing, so anything that DOES announce is +/// suspicious (a parked pointer, a future-format value) and must be classified, +/// not blindly treated as a deletable fragment. +#[cfg(any(feature = "cli", test))] +pub(crate) fn value_announces_our_kind(raw: &str) -> bool { + matches!( + classify_root_value(raw), + RootValueKind::Pointer | RootValueKind::UnknownKind + ) +} + +/// Does this value announce an `edgezero_kind` this build does NOT recognise — an +/// unknown string kind, or a non-string kind (a NEWER or foreign format)? +/// +/// The CLI read path refuses to OVERWRITE such a value: an older CLI clobbering a +/// format it cannot read would lose the newer config. It is a hard read error +/// (upgrade the CLI), never the repairable `Corrupt`. +#[cfg(test)] +pub(crate) fn value_is_unknown_kind(raw: &str) -> bool { + matches!(classify_root_value(raw), RootValueKind::UnknownKind) +} + +/// Was this value written by a NEWER format this v1 reader cannot handle? +/// +/// True for any of: an unknown/non-string `edgezero_kind`; OUR pointer kind with +/// a bumped pointer `version`; a JSON object carrying a `version` field that is +/// present but NOT 1 (a newer format from a future writer). Per the blob spec's +/// v1-reader rule, these must FAIL CLOSED everywhere destructive -- the read path +/// refuses to overwrite them, and GC / local prune never delete them -- so an +/// older CLI cannot clobber config a newer writer produced. A MISSING version (a +/// malformed v1 value) is NOT future: it stays repairable corruption. +/// +/// Version detection is deliberately SCHEMA-AGNOSTIC: it keys on the `version` +/// field alone, NOT the presence of the four v1 fields. A future shape such as +/// `{"version":2,"payload":...}` may drop or rename v1 fields, so requiring them +/// would let it slip through as repairable corruption. This mirrors the generic +/// classifier's `body_is_future_envelope`. +/// +/// Also compiled for the RUNTIME (`fastly`): the guest uses it to give the right +/// remediation (redeploy an updated build, not re-push the same config). +/// +/// Always compiled: the resolver consults it on the reconstructed envelope, and +/// the resolver is compiled regardless of feature set. +pub(crate) fn value_is_future_format(raw: &str) -> bool { + use edgezero_core::blob_envelope::ENVELOPE_VERSION_V1; + let Ok(serde_json::Value::Object(obj)) = serde_json::from_str::(raw) else { + return false; + }; + let version_is_not_one = obj + .get("version") + .and_then(serde_json::Value::as_u64) + .is_some_and(|version| version != u64::from(ENVELOPE_VERSION_V1)); + match obj.get("edgezero_kind") { + // Our pointer kind, but a newer POINTER version. + Some(serde_json::Value::String(kind)) if kind == POINTER_KIND => version_is_not_one, + // A discriminator we do not recognise (unknown string, or non-string). + Some(_) => true, + // No discriminator: any object carrying a bumped `version` is a newer + // format, whatever else its shape (schema-agnostic). + None => version_is_not_one, + } +} + +/// Classify a ROOT value for `config gc` — the live-set input on a DESTRUCTIVE +/// path, so it is fail-closed. Unlike [`prior_chunk_keys`] (which is lenient for +/// the push path and treats unrelated/empty JSON as "references nothing"), this +/// accepts ONLY: +/// +/// - a valid, integrity-checked direct `BlobEnvelope` → `Direct`; +/// - a valid v1 chunk pointer → `Chunked` (metadata validated). +/// +/// Anything else — an empty string, a truncated/partial value, unrelated JSON, +/// or a pointer that fails validation — is `Err`. A root we cannot classify has +/// unknown references, so we must reclaim NOTHING rather than treat its live +/// chunks as orphaned. +/// +/// **Metadata is not proof.** A `Chunked` result says the pointer is +/// self-consistent, NOT that it honestly describes its generation: a pointer can +/// drop its last chunk ref AND restate `envelope_len` as the remaining sum, and +/// every metadata check still passes while the dropped chunk silently leaves the +/// live set. Callers MUST reconstruct the referenced chunks and put them through +/// [`gc_verify_generation`] before trusting the live set. +#[cfg(any(feature = "cli", test))] +pub(crate) fn gc_classify_root(root_key: &str, raw: &str) -> Result { + use edgezero_core::blob_envelope::BlobEnvelope; + + if raw.trim().is_empty() { + return Err(format!( + "root `{root_key}` has an empty value; refusing to reclaim (cannot tell what it references)" + )); + } + // Diagnostics are REDACTED: serde's Display quotes the offending input, + // `verify()`'s names the stored hashes, and BOTH are attacker- or + // config-controlled strings that may hold credentials. + // + // Discriminate on the SAME classifier the runtime uses, so GC and the runtime + // never disagree about what a value is. Critically, a value carrying an + // `edgezero_kind` this build does not recognise is NOT parsed as a direct + // envelope: `BlobEnvelope` ignores unknown fields, so a future-format pointer + // with envelope-shaped fields would otherwise classify as `Direct` (zero + // references) and its canonical chunks would be reclaimed. Fail closed. + match classify_root_value(raw) { + RootValueKind::Pointer => { + let pointer: FastlyChunkPointer = serde_json::from_str(raw).map_err(|err| { + format!( + "root `{root_key}` is a chunk pointer but is malformed ({}); refusing to reclaim", + redact_json_err(&err) + ) + })?; + if pointer.version != 1 { + return Err(format!( + "root `{root_key}` chunk pointer has unsupported version {}; refusing to reclaim", + pointer.version + )); + } + validate_pointer_chunks(root_key, &pointer)?; + Ok(GcRootValue::Chunked(GcPointer { + chunks: pointer + .chunks + .into_iter() + .map(|chunk| GcChunkRef { + key: chunk.key, + len: chunk.len, + sha256: chunk.sha256, + }) + .collect(), + envelope_len: pointer.envelope_len, + envelope_sha256: pointer.envelope_sha256, + })) + } + RootValueKind::UnknownKind => Err(format!( + "root `{root_key}` carries an `edgezero_kind` this build does not recognise (value \ + redacted); refusing to reclaim -- its chunks may belong to a newer format" + )), + RootValueKind::MalformedObject => Err(format!( + "root `{root_key}` is an object-shaped value that does not parse (possibly a truncated \ + pointer); refusing to reclaim" + )), + // No `edgezero_kind`: it counts as a root only if it is a valid, + // integrity-checked direct envelope; anything else is left to the + // caller's inert-foreign handling. + RootValueKind::Foreign => { + let envelope: BlobEnvelope = serde_json::from_str(raw).map_err(|err| { + format!( + "root `{root_key}` is neither a valid chunk pointer nor a valid config envelope \ + ({}); refusing to reclaim", + redact_json_err(&err) + ) + })?; + envelope.verify().map_err(|_err| { + format!( + "root `{root_key}` envelope failed its integrity check (details redacted -- the \ + stored hashes are config-controlled); refusing to reclaim" + ) + })?; + Ok(GcRootValue::Direct) + } + } +} + +/// Check that `assembled` really is the generation named by `generation_sha`. +/// +/// A chunk key embeds the SHA-256 of the WHOLE envelope it belongs to, so +/// reassembling a chunk set either reproduces the content-address its own keys +/// name, or it does not. Nothing an inconsistent store says about lengths, +/// indexes or counts survives this. +/// +/// **This is a consistency check, NOT proof of authorship — do not mistake it +/// for one.** Content-addressing is not a signature: anyone can pick an envelope +/// E, compute `H = sha256(E)`, and store the pieces under `H`. No preimage +/// attack is involved, so passing here says the bytes are internally consistent, +/// not that `EdgeZero` wrote them. Callers deciding to DELETE must additionally +/// require the entries to be byte-identical to this writer's own output — see +/// `prove_generation` in `cli.rs`. +/// +/// Used for BOTH destructive decisions: +/// - a live pointer's chunks must reconstruct the envelope it claims (else its +/// chunk list is not the true live set); +/// - a delete candidate's generation must reconstruct a real envelope (a +/// necessary, but not sufficient, condition for reclaiming it). +#[cfg(any(feature = "cli", test))] +pub(crate) fn gc_verify_generation(generation_sha: &str, assembled: &str) -> Result<(), String> { + use edgezero_core::blob_envelope::BlobEnvelope; + + // Neither hash is echoed: `generation_sha` comes from the chunk KEYS + // (pointer-controlled) and `actual` is derived from the reassembled config + // bytes. Both are stored/derived strings, and this message is logged. + if sha256_hex(assembled.as_bytes()) != generation_sha { + return Err( + "the chunk set does not reassemble to the generation its own keys name (hashes \ + redacted), so these chunks are not the generation they claim" + .to_owned(), + ); + } + // Content-addressing proves the bytes are SELF-CONSISTENT, not that EdgeZero + // wrote them (a forger can content-address their own envelope -- see this + // function's doc comment). Parsing here just confirms the reassembled bytes + // are a config envelope at all; the authorship-adjacent decision (delete or + // not) is `prove_generation`'s round-trip against the writer, in cli.rs. + let envelope: BlobEnvelope = serde_json::from_str(assembled).map_err(|err| { + format!( + "the chunk set reassembles to its content-address but does not parse as a config \ + envelope ({})", + redact_json_err(&err) + ) + })?; + envelope.verify().map_err(|_err| { + "the chunk set reassembles to a config envelope that fails its own integrity check \ + (details redacted -- the stored hashes are config-controlled)" + .to_owned() + })?; + Ok(()) +} + +/// Extract the content-address (envelope SHA) of the generation a chunk key +/// belongs to, for `root_key`. Returns `None` when `key` is not a well-formed +/// chunk key of that root. +/// +/// This is how reclamation groups the store's ACTUAL keys into generations. It +/// validates the shape (`.__edgezero_chunks..`) rather +/// than trusting it: a hand-edited or foreign key never becomes a delete target. +#[cfg(any(feature = "cli", test))] +pub(crate) fn chunk_key_generation(root_key: &str, key: &str) -> Option { + chunk_key_parts(root_key, key).map(|(generation, _)| generation) +} + +/// Split a canonical chunk key into its `(generation, index)`. +/// +/// The validating half of [`chunk_key_generation`], which discards the index. +/// Pointer validation needs both: the generation to prove a chunk list names one +/// generation, the index to prove the list is dense and ordered. +fn chunk_key_parts(root_key: &str, key: &str) -> Option<(String, usize)> { + if root_key.is_empty() { + return None; + } + let prefix = format!("{root_key}{CHUNK_KEY_INFIX}"); + let rest = key.strip_prefix(&prefix)?; + let (sha, index) = rest.rsplit_once('.')?; + // Canonical shape ONLY — this gates a destructive delete, so it must match + // exactly what `prepare_fastly_config_entries` emits and nothing else: + // - a 64-char LOWERCASE hex SHA-256 (`format!("{:x}", Sha256::digest(..))`), + // - a canonical decimal index (no leading zeros; `usize` `Display`). + // Anything else (short/uppercase hash, `00`, `007`) is foreign or + // hand-edited and must never become a delete candidate. + if !is_canonical_sha256_hex(sha) { + return None; + } + let parsed = canonical_index(index)?; + Some((sha.to_owned(), parsed)) +} + +/// Exactly 64 lowercase hex characters. +fn is_canonical_sha256_hex(sha: &str) -> bool { + sha.len() == 64 + && sha + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Parse a canonical decimal index: all ASCII digits, no leading zero unless the +/// value is exactly `0`, and it must fit `usize`. +/// +/// Must be an index this writer could actually have emitted: the chunk loop +/// counts in `usize`, so a digit run that overflows it (or is absurdly long) is +/// not ours -- accept only what `prepare_fastly_config_entries` can produce. +fn canonical_index(index: &str) -> Option { + if index.is_empty() || !index.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + if index != "0" && index.starts_with('0') { + return None; + } + index.parse::().ok() +} + +// --------------------------------------------------------------------------- +// Unit tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + + use std::collections::HashMap; + + use super::*; + + /// the index must be one this writer could emit. The + /// chunk loop counts in `usize`, so a digit run that overflows it is not + /// ours -- and this gate authorises DELETES. + #[test] + fn canonical_index_must_fit_usize() { + let sha = "a".repeat(64); + let root = "app_config"; + assert!(chunk_key_generation(root, &format!("{root}.__edgezero_chunks.{sha}.0")).is_some()); + assert!(chunk_key_generation(root, &format!("{root}.__edgezero_chunks.{sha}.7")).is_some()); + // 40 digits: all-ASCII-digit and no leading zero, but not a usize. + let overflow = "9".repeat(40); + assert!( + chunk_key_generation(root, &format!("{root}.__edgezero_chunks.{sha}.{overflow}")) + .is_none(), + "an index that overflows usize is not a key we wrote" + ); + } + + // ---- gc_classify_root / pointer consistency () ---- + + /// A real pointer + its real chunks, for mutation in the tests below. + fn pointer_fixture(root: &str) -> (String, FastlyChunkPointer) { + let envelope = make_envelope_json(20_000); + let entries = prepare_fastly_config_entries(root, &envelope).expect("expand"); + let (_, pointer_json) = entries.last().expect("pointer").clone(); + let pointer: FastlyChunkPointer = serde_json::from_str(&pointer_json).expect("parse"); + (pointer_json, pointer) + } + + fn reserialise(pointer: &FastlyChunkPointer) -> String { + serde_json::to_string(pointer).expect("serialise") + } + + /// A valid pointer whose chunks are scoped to a CHUNK-SHAPED holder key + /// classifies successfully (returns `Chunked` with those keys), not `Err`. + /// The cross-root GC test only shows an unclassifiable pointer failing + /// closed; this pins that a well-formed self-scoped pointer is accepted. + #[test] + fn gc_classify_root_accepts_a_pointer_rooted_at_a_chunk_shaped_key() { + let holder = format!("app_config{CHUNK_KEY_INFIX}{}.0", "e".repeat(64)); + let (pointer_json, pointer) = pointer_fixture(&holder); + let classified = gc_classify_root(&holder, &pointer_json) + .expect("a valid self-scoped pointer classifies"); + let GcRootValue::Chunked(gc_pointer) = classified else { + panic!("a pointer value must classify as Chunked"); + }; + assert_eq!(gc_pointer.chunks.len(), pointer.chunks.len()); + for chunk in &gc_pointer.chunks { + assert!( + chunk.key.starts_with(&format!("{holder}{CHUNK_KEY_INFIX}")), + "each referenced key must be scoped to the holder: `{}`", + chunk.key + ); + } + } + + /// The happy path: a real pointer classifies as its own chunk keys. + #[test] + fn gc_classify_root_accepts_a_real_pointer() { + let root = "app_config"; + let (pointer_json, pointer) = pointer_fixture(root); + let classified = + gc_classify_root(root, &pointer_json).expect("a real pointer must classify"); + let GcRootValue::Chunked(gc_pointer) = classified else { + panic!("a pointer value must classify as Chunked"); + }; + assert!( + gc_pointer.chunks.len() >= 2, + "an oversized envelope always splits into >= 2" + ); + // The classifier must carry the pointer's refs VERBATIM: `config gc` + // checks each stored chunk against these `len`/`sha256` values before + // reassembling, so a lossy or reordered mapping here would silently + // weaken every downstream content check. + assert_eq!(gc_pointer.envelope_len, pointer.envelope_len); + assert_eq!(gc_pointer.envelope_sha256, pointer.envelope_sha256); + assert_eq!(gc_pointer.chunks.len(), pointer.chunks.len()); + for (got, want) in gc_pointer.chunks.iter().zip(pointer.chunks.iter()) { + assert_eq!(got.key, want.key); + assert_eq!(got.len, want.len); + assert_eq!(got.sha256, want.sha256); + } + // ...and those refs actually describe the stored chunks. + let entries = + prepare_fastly_config_entries(root, &make_envelope_json(20_000)).expect("expand"); + for (chunk, (_, value)) in gc_pointer.chunks.iter().zip(entries.iter()) { + assert_eq!(chunk.len, value.len(), "ref len must match the real chunk"); + assert_eq!( + chunk.sha256, + sha256_hex(value.as_bytes()), + "ref sha256 must match the real chunk" + ); + } + } + + /// A direct envelope is a root that references no chunks. + #[test] + fn gc_classify_root_accepts_a_direct_envelope() { + let envelope = make_envelope_json(200); + let classified = + gc_classify_root("app_config", &envelope).expect("a valid envelope must classify"); + assert!( + matches!(classified, GcRootValue::Direct), + "an inline envelope must classify as Direct (it references no chunks)" + ); + } + + /// values that are NOT a valid envelope or pointer must + /// fail closed. `Ok([])` here would mean "references nothing" and would make + /// the root's own live chunks look orphaned. + #[test] + fn gc_classify_root_fails_closed_on_unclassifiable_values() { + let root = "app_config"; + let (pointer_json, _) = pointer_fixture(root); + // A VALID-JSON partial pointer: `chunks` truncated to one entry. This is + // the case the earlier truncated-JSON test missed. + let mut partial: FastlyChunkPointer = serde_json::from_str(&pointer_json).unwrap(); + partial.chunks.truncate(1); + let cases: Vec<(&str, String)> = vec![ + ("empty", String::new()), + ("whitespace", " ".to_owned()), + ("not json", "not-json-at-all".to_owned()), + ("unrelated json", r#"{"some":"value"}"#.to_owned()), + ("json scalar", "42".to_owned()), + ("valid-JSON partial pointer", reserialise(&partial)), + ]; + for (label, raw) in cases { + assert!( + gc_classify_root(root, &raw).is_err(), + "a {label} root value must fail closed, not classify as \"references nothing\"" + ); + } + } + + /// a pointer that PARSES but under-reports its chunks + /// would shrink the live set and get real live chunks deleted. Every way a + /// chunk list can lie must be rejected. + #[test] + fn pointer_chunk_list_must_be_internally_consistent() { + let root = "app_config"; + let (_, base) = pointer_fixture(root); + + // 1. Empty list: never emitted (an oversized envelope splits into >= 2). + let mut empty = clone_pointer(&base); + empty.chunks.clear(); + // 2. Omitted index: chunks [0, 2] -- 1 is live but unreferenced. + let mut omitted = clone_pointer(&base); + omitted.chunks.remove(1); + // 3. Duplicate index: [0, 0] -- reports fewer distinct keys than exist. + let mut duplicated = clone_pointer(&base); + let first = clone_ref(&base.chunks[0]); + duplicated.chunks = vec![clone_ref(&first), first]; + // 4. Reordered: [1, 0] -- not a shape the writer emits. + let mut reordered = clone_pointer(&base); + reordered.chunks.reverse(); + // 5. Mixed generation: a key from another envelope's content-address. + let mut mixed = clone_pointer(&base); + let other = make_envelope_json(19_000); + let other_entries = prepare_fastly_config_entries(root, &other).expect("expand"); + mixed.chunks[1] = FastlyChunkRef { + key: other_entries[1].0.clone(), + len: base.chunks[1].len, + sha256: base.chunks[1].sha256.clone(), + }; + // 6. Lengths that cannot add up to the declared envelope. + let mut bad_len = clone_pointer(&base); + bad_len.envelope_len = base.envelope_len.saturating_add(999); + // 7. Zero-length chunk. + let mut zero_len = clone_pointer(&base); + zero_len.chunks[0].len = 0; + + for (label, pointer) in [ + ("an empty chunk list", empty), + ("an omitted index", omitted), + ("a duplicated index", duplicated), + ("a reordered list", reordered), + ("a mixed generation", mixed), + ("an inconsistent envelope_len", bad_len), + ("a zero-length chunk", zero_len), + ] { + let raw = reserialise(&pointer); + assert!( + gc_classify_root(root, &raw).is_err(), + "a pointer with {label} must be rejected, not treated as the authoritative live set" + ); + assert!( + prior_chunk_keys(root, &raw).is_err(), + "a pointer with {label} must also warn on the push path, not prune from a bad list" + ); + } + + // Control: the unmutated fixture still passes, so the assertions above + // are rejecting the mutation and not something incidental. + gc_classify_root(root, &reserialise(&base)) + .expect("the unmutated fixture must still classify"); + } + + /// a parse diagnostic must never quote the stored + /// value -- app config may hold credentials and these lines are logged. + #[test] + fn pointer_parse_errors_do_not_leak_stored_values() { + const SENTINEL: &str = "s3cr3t-do-not-log"; + let root = "app_config"; + // `version` is typed `u8`; a string there makes serde quote it: + // `invalid type: string "s3cr3t-do-not-log", expected u8`. + let malformed = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":"{SENTINEL}","chunks":[],"data_sha256":"","envelope_len":0,"envelope_sha256":""}}"# + ); + let Err(gc_err) = gc_classify_root(root, &malformed) else { + panic!("a malformed pointer must fail closed on the gc path"); + }; + for err in [ + prior_chunk_keys(root, &malformed).expect_err("must warn"), + gc_err, + ] { + assert!( + !err.contains(SENTINEL), + "diagnostic leaked a stored value: {err}" + ); + assert!( + err.contains("redacted"), + "diagnostic should say the value was redacted: {err}" + ); + } + } + + fn clone_pointer(pointer: &FastlyChunkPointer) -> FastlyChunkPointer { + FastlyChunkPointer { + chunks: pointer.chunks.iter().map(clone_ref).collect(), + data_sha256: pointer.data_sha256.clone(), + edgezero_kind: pointer.edgezero_kind.clone(), + envelope_len: pointer.envelope_len, + envelope_sha256: pointer.envelope_sha256.clone(), + version: pointer.version, + } + } + + fn clone_ref(chunk: &FastlyChunkRef) -> FastlyChunkRef { + FastlyChunkRef { + key: chunk.key.clone(), + len: chunk.len, + sha256: chunk.sha256.clone(), + } + } + + /// Round-7 [P1]: a COORDINATED partial pointer -- drop the last chunk ref AND + /// restate `envelope_len` as the remaining sum. Generation, indexes, lengths + /// and the sum all agree; only the CONTENT disagrees. Metadata validation + /// cannot see it, so the dropped chunk would silently leave the live set and + /// become deletable. Only reassembling and hashing catches this. + #[test] + fn coordinated_partial_pointer_fails_content_verification() { + let root = "app_config"; + let envelope = make_envelope_json(20_000); + let entries = prepare_fastly_config_entries(root, &envelope).expect("expand"); + let (_, pointer_json) = entries.last().expect("pointer").clone(); + let mut pointer: FastlyChunkPointer = serde_json::from_str(&pointer_json).unwrap(); + assert!(pointer.chunks.len() >= 3, "need >= 3 chunks for this case"); + + let dropped = pointer.chunks.pop().expect("last chunk"); + pointer.envelope_len = pointer.chunks.iter().map(|chunk| chunk.len).sum(); + let doctored = serde_json::to_string(&pointer).expect("serialise"); + + // METADATA validation passes -- that is the whole point of the attack. + let classified = gc_classify_root(root, &doctored) + .expect("the doctored pointer is metadata-consistent by construction"); + let GcRootValue::Chunked(gc_pointer) = classified else { + panic!("expected a chunked root"); + }; + assert!( + !gc_pointer + .chunks + .iter() + .any(|chunk| chunk.key == dropped.key), + "fixture check: the dropped chunk is absent from the pointer's list" + ); + + // CONTENT verification is what rejects it: the surviving chunks cannot + // hash to the envelope the generation names. + let assembled: String = entries[..entries.len().saturating_sub(2)] + .iter() + .map(|(_, value)| value.as_str()) + .collect(); + let err = gc_verify_generation(&gc_pointer.envelope_sha256, &assembled) + .expect_err("a partial chunk set must not verify as its generation"); + assert!( + err.contains("not the generation they claim"), + "expected a content-address mismatch, got: {err}" + ); + + // And the honest, complete set still verifies -- so the assertion above + // is rejecting the omission, not something incidental. + let whole: String = entries[..entries.len().saturating_sub(1)] + .iter() + .map(|(_, value)| value.as_str()) + .collect(); + gc_verify_generation(&gc_pointer.envelope_sha256, &whole) + .expect("the complete chunk set must verify"); + } + + /// Round-7/9 [P1]: an entry that merely LOOKS like a chunk key is not a chunk. + /// Reassembling to the content-address its keys name is NECESSARY but not + /// sufficient (a forger can content-address their own data with no preimage); + /// `prove_generation` adds the writer round-trip. Here we check the necessary + /// half: a value that does not even hash to its generation is never ours. + #[test] + fn only_a_real_generation_verifies() { + let root = "app_config"; + let envelope = make_envelope_json(20_000); + let entries = prepare_fastly_config_entries(root, &envelope).expect("expand"); + let generation = chunk_key_generation(root, &entries[0].0).expect("canonical key"); + + for (label, value) in [ + ("plain text", "just some plain text"), + ("unrelated json", r#"{"some":"value"}"#), + ("someone's real config", make_envelope_json(200).as_str()), + ] { + assert!( + gc_verify_generation(&generation, value).is_err(), + "a {label} value must never verify as a generation we wrote" + ); + } + + // The genuine article does verify. + let whole: String = entries[..entries.len().saturating_sub(1)] + .iter() + .map(|(_, value)| value.as_str()) + .collect(); + gc_verify_generation(&generation, &whole).expect("the real generation must verify"); + } + + /// `chunks[].key` is pointer-controlled and NOT yet + /// validated where we report it, so a malformed pointer can smuggle a secret + /// into a log line. Diagnostics must name a position, never the key. + #[test] + fn pointer_key_does_not_leak_into_diagnostics() { + const SENTINEL: &str = "prod-db-password=hunter2"; + let root = "app_config"; + let malformed = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{{"key":"{SENTINEL}","len":10,"sha256":"x"}}],"data_sha256":"","envelope_len":10,"envelope_sha256":"{}"}}"#, + "a".repeat(64) + ); + let err = prior_chunk_keys(root, &malformed).expect_err("must warn"); + assert!( + !err.contains(SENTINEL), + "a pointer-controlled key must never reach a diagnostic: {err}" + ); + } + + /// `envelope_len` and the per-chunk lengths are + /// untrusted `usize`s that later size allocations. Saturating arithmetic let + /// a metadata-consistent pointer declare `usize::MAX` and pass validation, + /// which then reached `String::with_capacity` and aborted the process. + #[test] + fn absurd_pointer_lengths_are_rejected_before_allocating() { + let root = "app_config"; + let sha = "a".repeat(64); + let huge = usize::MAX; + let overflowing = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.0","len":{huge},"sha256":"x"}},{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.1","len":{huge},"sha256":"y"}}],"data_sha256":"","envelope_len":{huge},"envelope_sha256":"{sha}"}}"# + ); + assert!( + prior_chunk_keys(root, &overflowing).is_err(), + "chunk lengths that overflow when summed must be rejected" + ); + + // A single absurd chunk: no overflow, but far beyond what we emit. + let oversized = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.0","len":{huge},"sha256":"x"}}],"data_sha256":"","envelope_len":{huge},"envelope_sha256":"{sha}"}}"# + ); + assert!( + prior_chunk_keys(root, &oversized).is_err(), + "a chunk larger than the writer's payload target must be rejected" + ); + } + + /// root-vs-chunk is decided by VALUE. A pointer value is a + /// root wherever it lives; a chunk payload (raw envelope fragment) is not. + #[test] + fn value_is_pointer_kind_detects_pointers_only() { + let root = "app_config"; + let (pointer_json, _) = pointer_fixture(root); + assert!( + value_is_pointer_kind(&pointer_json), + "a real pointer value must be recognised as pointer-kind" + ); + + // A genuine chunk PAYLOAD: a raw fragment of envelope JSON. + let entries = + prepare_fastly_config_entries(root, &make_envelope_json(20_000)).expect("expand"); + assert!( + !value_is_pointer_kind(&entries[0].1), + "a chunk payload must NOT be mistaken for a pointer" + ); + // Direct envelopes, unrelated JSON and non-JSON are not pointers either. + assert!(!value_is_pointer_kind(&make_envelope_json(200))); + assert!(!value_is_pointer_kind(r#"{"some":"value"}"#)); + assert!(!value_is_pointer_kind("not json at all")); + } + + /// The three GC-facing classifier predicates agree on every category, and -- + /// the crux of the unknown-kind data-loss bug -- a value that claims our + /// namespace with an UNKNOWN or NON-STRING `edgezero_kind` is neither inert + /// foreign nor a pointer: it announces our kind, so GC classifies it (and + /// fails closed) rather than treating it as a deletable/ignorable value. + #[test] + fn classifier_predicates_agree_on_every_category() { + // (raw, is_pointer, is_inert_foreign, announces_our_kind, is_unknown_kind) + let cases: &[(&str, bool, bool, bool, bool)] = &[ + ( + r#"{"edgezero_kind":"fastly_config_chunks"}"#, + true, + false, + true, + false, + ), + // Unknown STRING kind: claims our namespace, not inert, not our pointer. + ( + r#"{"edgezero_kind":"fastly_config_chunks_v2"}"#, + false, + false, + true, + true, + ), + // NON-STRING kind: still claims the namespace -- must NOT be inert. + (r#"{"edgezero_kind":7}"#, false, false, true, true), + (r#"{"edgezero_kind":null}"#, false, false, true, true), + ( + r#"{"edgezero_kind":{"nested":true}}"#, + false, + false, + true, + true, + ), + // Genuinely foreign: no discriminator. + (r#"{"unrelated":"json"}"#, false, true, false, false), + ("value_a", false, true, false, false), + ("", false, true, false, false), + // Object-shaped but unparseable (a possible truncated pointer): none. + ( + r#"{"edgezero_kind":"fastly_config"#, + false, + false, + false, + false, + ), + ]; + for &(raw, is_ptr, inert, announces, unknown) in cases { + assert_eq!(value_is_pointer_kind(raw), is_ptr, "pointer_kind: {raw:?}"); + assert_eq!(value_is_inert_foreign(raw), inert, "inert_foreign: {raw:?}"); + assert_eq!( + value_announces_our_kind(raw), + announces, + "announces_our_kind: {raw:?}" + ); + assert_eq!(value_is_unknown_kind(raw), unknown, "unknown_kind: {raw:?}"); + // No value is ever BOTH inert-foreign and namespace-claiming. + assert!(!(inert && announces), "mutually exclusive: {raw:?}"); + } + } + + /// `value_is_future_format` catches a NEWER writer's output (an unknown kind, + /// or a bumped envelope/pointer version) but NOT ordinary v1 corruption -- so + /// a future format fails closed while a malformed v1 value stays repairable. + #[test] + fn future_format_detects_bumped_versions_but_not_v1_corruption() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let v1 = serde_json::to_string(&BlobEnvelope::new( + json!({ "k": "v" }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .unwrap(); + + // A v2 DIRECT envelope: envelope-shaped, version bumped -> future. + let mut v2_env: serde_json::Value = serde_json::from_str(&v1).unwrap(); + v2_env["version"] = json!(2_u32); + assert!(value_is_future_format(&v2_env.to_string()), "v2 envelope"); + + // SCHEMA-CHANGING v2: a future shape that DROPS the v1 fields must still be + // future -- detection keys on `version` alone, not the four v1 fields. + assert!( + value_is_future_format(r#"{"version":2,"payload":{"k":"v"}}"#), + "schema-changing v2 (no v1 fields)" + ); + + // A v2 POINTER (our kind, bumped pointer version) -> future. + assert!( + value_is_future_format( + r#"{"edgezero_kind":"fastly_config_chunks","version":2,"chunks":[]}"# + ), + "v2 pointer" + ); + // Unknown/non-string kind -> future. + assert!(value_is_future_format(r#"{"edgezero_kind":"future"}"#)); + assert!(value_is_future_format(r#"{"edgezero_kind":9}"#)); + + // A valid v1 envelope is NOT future. + assert!(!value_is_future_format(&v1), "v1 envelope"); + // A v1 envelope with a WRONG sha (corrupt, version 1) is NOT future -- + // it stays repairable corruption. + let mut bad_sha: serde_json::Value = serde_json::from_str(&v1).unwrap(); + bad_sha["sha256"] = json!("0".repeat(64)); + assert!(!value_is_future_format(&bad_sha.to_string()), "v1 bad sha"); + // A malformed v1 pointer MISSING its version is NOT future (repairable). + assert!(!value_is_future_format( + r#"{"edgezero_kind":"fastly_config_chunks","chunks":[]}"# + )); + // A plain foreign value is not future. + assert!(!value_is_future_format(r#"{"unrelated":"json"}"#)); + assert!(!value_is_future_format("not json")); + } + + /// GC data-loss guard: a value carrying a FUTURE `edgezero_kind` but shaped + /// like a valid direct envelope must NOT classify as `Direct`. `BlobEnvelope` + /// ignores unknown fields, so without the discriminator check GC would call + /// it a zero-reference root and reclaim canonical chunks a newer format still + /// references. It must fail closed instead. + #[test] + fn gc_classify_root_fails_closed_on_a_future_kind_envelope() { + use edgezero_core::blob_envelope::BlobEnvelope; + + // A real, integrity-valid envelope... + let envelope = BlobEnvelope::new( + serde_json::json!({ "k": "v" }), + "2026-01-01T00:00:00Z".to_owned(), + ); + let mut object = serde_json::to_value(&envelope).expect("to value"); + // ...with a future discriminator grafted on. Valid envelope fields remain. + object.as_object_mut().expect("object").insert( + "edgezero_kind".to_owned(), + serde_json::json!("fastly_config_chunks_v2"), + ); + let raw = serde_json::to_string(&object).expect("serialise"); + + let Err(err) = gc_classify_root("app_config", &raw) else { + panic!("a future-kind value must not classify as a reclaimable Direct root") + }; + assert!( + err.contains("does not recognise") && err.contains("refusing to reclaim"), + "must fail closed naming the unknown kind: {err}" + ); + } + + /// The runtime resolver rejects a pointer shape the writer could never emit + /// BEFORE fetching any chunk -- the same metadata validation the CLI/GC path + /// runs, so invariant 14 holds on the read path too. + #[test] + fn runtime_resolver_rejects_non_writer_pointer_shapes() { + let root = "app_config"; + let sha = "a".repeat(64); + // A metadata-consistent-looking pointer whose per-chunk length is far + // over the writer's payload target (and whose envelope is too small to + // have been chunked at all). + let bogus = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.0","len":99999,"sha256":"x"}},{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.1","len":1,"sha256":"y"}}],"data_sha256":"","envelope_len":100000,"envelope_sha256":"{sha}"}}"# + ); + // No fetch should even be attempted: fail on metadata alone. + let mut fetched = false; + let err = resolve_fastly_config_value(root, bogus, |_key| { + fetched = true; + Ok(None) + }) + .expect_err("a non-writer pointer shape must be rejected"); + assert!( + !fetched, + "validation must reject before any chunk fetch: {err}" + ); + + // The real thing still resolves. + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries(root, &envelope).unwrap(); + let (root_key, pointer_json) = entries.last().unwrap().clone(); + let chunk_map: HashMap = entries[..entries.len().saturating_sub(1)] + .iter() + .cloned() + .collect(); + let resolved = resolve_fastly_config_value(&root_key, pointer_json, |key| { + Ok(chunk_map.get(key).cloned()) + }) + .expect("a real chunked value must still resolve"); + assert_eq!(resolved, envelope); + } + + /// A hand-authored pointer of MANY tiny chunks (metadata-consistent: dense, + /// summing to `envelope_len`) is rejected — its non-final chunks are far + /// below a full payload, so it is not writer output. This bounds the runtime + /// fetch fan-out. + #[test] + fn pointer_with_many_tiny_chunks_is_rejected() { + use std::fmt::Write as _; + let root = "app_config"; + let sha = "a".repeat(64); + // 100 chunks of 100 bytes each = 10_000 bytes (> entry limit, dense). + let mut chunks = String::new(); + for idx in 0..100_u32 { + if idx > 0 { + chunks.push(','); + } + write!( + chunks, + r#"{{"key":"{root}{CHUNK_KEY_INFIX}{sha}.{idx}","len":100,"sha256":"x"}}"# + ) + .expect("write to String"); + } + let bogus = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{chunks}],"data_sha256":"","envelope_len":10000,"envelope_sha256":"{sha}"}}"# + ); + let mut fetched = false; + let err = resolve_fastly_config_value(root, bogus, |_key| { + fetched = true; + Ok(None) + }) + .expect_err("a tiny-chunk fan-out must be rejected"); + assert!(!fetched, "must reject before fetching 100 chunks: {err}"); + } -/// Find the largest byte offset `<= end_raw` that is a valid UTF-8 -/// codepoint boundary within `src`. `start` is used as a hint so we -/// don't scan the entire string from zero each time. CLI writer only. -#[cfg(any(feature = "cli", test))] -fn find_utf8_boundary(src: &str, start: usize, end_raw: usize) -> usize { - if end_raw >= src.len() { - return src.len(); + /// RESOLVER-TO-WRITER ROUND TRIP: for a spread of over-limit sizes, the + /// writer's own output must resolve back to the exact input. This is the + /// positive half of the exact split-boundary contract — whatever the writer + /// emits, the resolver (including its exact-boundary replay) accepts. + #[test] + fn writer_output_round_trips_through_the_resolver() { + let root = "app_config"; + // Sizes that land on, just past, and well past chunk boundaries. + for target in [ + FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1), + CHUNK_PAYLOAD_TARGET.saturating_add(50), + CHUNK_PAYLOAD_TARGET.saturating_mul(2).saturating_add(10), + 20_000, + 35_000, + ] { + let envelope = make_envelope_json(target); + let entries = prepare_fastly_config_entries(root, &envelope).expect("expand"); + let (root_key, pointer_json) = entries.last().expect("pointer").clone(); + let chunk_map: HashMap = entries[..entries.len().saturating_sub(1)] + .iter() + .cloned() + .collect(); + let resolved = resolve_fastly_config_value(&root_key, pointer_json, |key| { + Ok(chunk_map.get(key).cloned()) + }) + .unwrap_or_else(|err| panic!("size {target} must round-trip: {err}")); + assert_eq!(resolved, envelope, "size {target} must reconstruct exactly"); + } } - // Walk backwards from end_raw until we land on a valid char boundary. - let mut boundary = end_raw; - while boundary > start && !src.is_char_boundary(boundary) { - boundary = boundary.saturating_sub(1); + + /// EXACT SPLIT BOUNDARIES: a pointer whose chunks reassemble to the correct + /// bytes but along boundaries this writer would never choose is rejected. Here + /// two bytes are moved from the first (full) chunk into the last (short) one: + /// every metadata check still passes (dense indexes, single generation, both + /// lengths in range, sum == `envelope_len`, per-chunk and whole-envelope SHAs + /// correct), yet the writer splits the first chunk at 7 000 bytes, not 6 998, + /// so the resolver's boundary replay must reject it. + #[test] + fn resolver_rejects_a_non_writer_split_that_reassembles_correctly() { + let root = "app_config"; + // Just over the entry limit: chunk 0 is a full CHUNK_PAYLOAD_TARGET, chunk + // 1 is a short remainder with room to absorb the shifted bytes. + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(100)); + let entries = prepare_fastly_config_entries(root, &envelope).expect("expand"); + assert_eq!(entries.len(), 3, "expected exactly 2 chunks + pointer"); + let (_, pointer_json) = entries.last().expect("pointer").clone(); + let real: FastlyChunkPointer = serde_json::from_str(&pointer_json).expect("parse"); + + // Re-split the SAME bytes 2 bytes earlier: chunk 0 shrinks to 6 998 + // (still >= CHUNK_PAYLOAD_TARGET - 3, so the pre-fetch bound passes), + // chunk 1 grows correspondingly (still <= CHUNK_PAYLOAD_TARGET). + let cut = CHUNK_PAYLOAD_TARGET.saturating_sub(2); + let head = envelope.get(..cut).expect("cut is a char boundary"); + let tail = envelope.get(cut..).expect("cut is a char boundary"); + let key0 = format!("{root}{CHUNK_KEY_INFIX}{}.0", real.envelope_sha256); + let key1 = format!("{root}{CHUNK_KEY_INFIX}{}.1", real.envelope_sha256); + let shifted = FastlyChunkPointer { + chunks: vec![ + FastlyChunkRef { + key: key0.clone(), + len: head.len(), + sha256: sha256_hex(head.as_bytes()), + }, + FastlyChunkRef { + key: key1.clone(), + len: tail.len(), + sha256: sha256_hex(tail.as_bytes()), + }, + ], + data_sha256: real.data_sha256.clone(), + edgezero_kind: real.edgezero_kind.clone(), + envelope_len: real.envelope_len, + envelope_sha256: real.envelope_sha256.clone(), + version: real.version, + }; + let chunk_map: HashMap = + HashMap::from([(key0, head.to_owned()), (key1, tail.to_owned())]); + let err = resolve_fastly_config_value(root, reserialise(&shifted), |key| { + Ok(chunk_map.get(key).cloned()) + }) + .expect_err("a non-writer split must be rejected even if it reassembles correctly"); + assert!( + err.contains("writer splits the reconstructed envelope"), + "must fail on the exact-boundary check, got: {err}" + ); } - boundary -} -/// Resolve a logical `(root_key, root_value)` pair from the store, -/// handling both direct `BlobEnvelope` values and chunk-pointer values. -/// -/// Algorithm: -/// 1. Try to parse `root_value` as a `BlobEnvelope`. If it parses, -/// verify it. A valid direct envelope is returned unchanged; a parsed -/// envelope whose SHA/version verification fails is an error. -/// 2. Only when `root_value` does not parse as a `BlobEnvelope`, try to -/// parse it as a `FastlyChunkPointer`. Unknown `edgezero_kind` / version -/// is an error. -/// 3. Fetch each chunk via the `fetch` callback. -/// 4. Verify each chunk's length and SHA against the pointer. -/// 5. Concatenate in pointer order, verify `envelope_len` + `envelope_sha256`, -/// then return the reconstructed envelope JSON string. -/// -/// # Errors -/// -/// Returns a descriptive error string naming the root key or the failing -/// chunk key when any integrity check fails. -pub(crate) fn resolve_fastly_config_value( - root_key: &str, - root_value: String, - mut fetch: F, -) -> Result -where - F: FnMut(&str) -> Result, String>, -{ - use edgezero_core::blob_envelope::BlobEnvelope; + /// FROZEN v1 WIRE FORMAT. Every other resolver test builds its fixture with + /// the CURRENT writer, so a writer and resolver that drift TOGETHER would + /// keep them green while silently breaking values an earlier release stored. + /// + /// This one hand-builds the v1 layout from literal constants — the 7 000-byte + /// split boundary, the `.__edgezero_chunks.` infix, the `.` key + /// suffix, and the pointer's field names — and never calls + /// `prepare_fastly_config_entries`. It therefore fails if the writer's split + /// or key format changes, which is exactly the v1 read-compatibility break + /// that needs to be caught deliberately rather than discovered in production. + #[test] + fn resolver_reads_a_frozen_v1_pointer_built_without_the_writer() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; - // --- Path 1: direct BlobEnvelope --- - if let Ok(envelope) = serde_json::from_str::(&root_value) { - envelope.verify().map_err(|err| { - format!("BlobEnvelope at key `{root_key}` failed integrity check: {err}") - })?; - return Ok(root_value); + /// v1 chunk payload boundary, in BYTES. Frozen. + const V1_SPLIT: usize = 7_000; + /// v1 chunk-key infix. Frozen. + const V1_INFIX: &str = ".__edgezero_chunks."; + // PRECOMPUTED wire hashes for this exact fixture, captured once. They are + // asserted below and fed into the pointer as LITERALS -- not recomputed -- + // so a change to the envelope serialisation OR the SHA helper makes the + // live bytes disagree with these constants and the test fails, catching a + // v1 wire break that a self-recomputing fixture would hide. + const FROZEN_LEN: usize = 8_348; + const FROZEN_ENVELOPE_SHA: &str = + "a340975d23554e600cfb14fc5455da8ad83127440f3e11d7b526c140efb76c75"; + const FROZEN_INNER_SHA: &str = + "24cc7f56c47fb2428171bc4fbf1ad2270488b8c5fb06c617050b68182eae6747"; + const FROZEN_CHUNK0_SHA: &str = + "f1f1d19798b57ed4ebf2415ffec8d1a9fac6af372aa0e2592620d4f9e790bca5"; + const FROZEN_CHUNK1_SHA: &str = + "f4c8a4d91bb29a6d395692591cb4b714336d83d0f95ca1688a49d6ad760a77ff"; + const FROZEN_CHUNK1_LEN: usize = FROZEN_LEN - V1_SPLIT; + + // ASCII content only, so the v1 split lands on 7 000 exactly and this + // fixture stays independent of the codepoint-boundary retreat. + let envelope = serde_json::to_string(&BlobEnvelope::new( + json!({ "frozen": "v".repeat(8_200) }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .expect("envelope"); + // Freeze the wire: any drift in serialisation or hashing trips these. + assert_eq!(envelope.len(), FROZEN_LEN, "envelope byte length drifted"); + assert_eq!( + sha256_hex(envelope.as_bytes()), + FROZEN_ENVELOPE_SHA, + "envelope wire bytes or the SHA helper drifted from the frozen v1 value" + ); + let head = envelope.get(..V1_SPLIT).expect("ascii boundary"); + let tail = envelope.get(V1_SPLIT..).expect("ascii boundary"); + assert_eq!( + sha256_hex(head.as_bytes()), + FROZEN_CHUNK0_SHA, + "chunk 0 drifted" + ); + assert_eq!( + sha256_hex(tail.as_bytes()), + FROZEN_CHUNK1_SHA, + "chunk 1 drifted" + ); + assert_eq!(tail.len(), FROZEN_CHUNK1_LEN); + + // The frozen pointer: field names, indexes, lengths, and the LITERAL + // precomputed hashes, all written out. The resolver checks the actual + // chunk bytes against these literals, so a drift is rejected on read. + let key0 = format!("root{V1_INFIX}{FROZEN_ENVELOPE_SHA}.0"); + let key1 = format!("root{V1_INFIX}{FROZEN_ENVELOPE_SHA}.1"); + let pointer = json!({ + "chunks": [ + { "key": key0, "len": V1_SPLIT, "sha256": FROZEN_CHUNK0_SHA }, + { "key": key1, "len": FROZEN_CHUNK1_LEN, "sha256": FROZEN_CHUNK1_SHA }, + ], + "data_sha256": FROZEN_INNER_SHA, + "edgezero_kind": "fastly_config_chunks", + "envelope_len": FROZEN_LEN, + "envelope_sha256": FROZEN_ENVELOPE_SHA, + "version": 1_u8, + }) + .to_string(); + + let chunk_map: HashMap = + HashMap::from([(key0, head.to_owned()), (key1, tail.to_owned())]); + let resolved = + resolve_fastly_config_value("root", pointer, |key| Ok(chunk_map.get(key).cloned())) + .expect("a value stored by v1 must still resolve"); + assert_eq!(resolved, envelope, "must reconstruct the original bytes"); } - // --- Path 2: chunk pointer --- - let pointer: FastlyChunkPointer = serde_json::from_str(&root_value).map_err(|err| { - format!( - "value at key `{root_key}` is neither a valid BlobEnvelope nor a valid \ - chunk pointer: {err}" + /// A valid v1 pointer whose chunks reassemble to a NEWER (v2) envelope is only + /// knowable AFTER reassembly -- the raw outer pointer is a healthy v1 value. + /// The resolver must type this `FutureFormat`, never erase it into an untyped + /// error the CLI would read as repairable corruption and OVERWRITE (a + /// downgrade clobbering a newer format). + #[test] + fn resolver_types_a_future_inner_envelope_as_future_format() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + // A LARGE v2 envelope, so the v1 writer chunks it (a small value stores + // direct). The writer chunks arbitrary bytes; the newer VERSION lives in + // the reassembled payload, not the (v1) pointer. + let mut v2: serde_json::Value = serde_json::from_str( + &serde_json::to_string(&BlobEnvelope::new( + json!({ "x": "z".repeat(9_000) }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .expect("envelope"), ) - })?; + .expect("parse"); + v2["version"] = json!(2_u32); + let body = v2.to_string(); + + let entries = prepare_fastly_config_entries("root", &body).expect("chunk the v2 body"); + assert!(entries.len() > 1, "the v2 body must be chunked, not direct"); + let mut chunk_map: HashMap = HashMap::new(); + let mut found_pointer = None; + for (key, value) in entries { + if key == "root" { + found_pointer = Some(value); + } else { + chunk_map.insert(key, value); + } + } + let pointer_body = found_pointer.expect("a pointer entry at the root key"); - if pointer.edgezero_kind != POINTER_KIND { - return Err(format!( - "chunk pointer at `{root_key}` has unknown edgezero_kind \ - `{}`; expected `{POINTER_KIND}`", - pointer.edgezero_kind - )); - } - if pointer.version != 1 { - return Err(format!( - "chunk pointer at `{root_key}` has unsupported version \ - {}; expected 1", - pointer.version - )); + let err = resolve_fastly_config_value_typed("root", pointer_body, |key| { + Ok(chunk_map.get(key).cloned()) + }) + .expect_err("a future inner envelope must not resolve to Ok"); + assert!( + err.is_future_format(), + "a future INNER envelope must be typed FutureFormat, not corruption" + ); + assert!( + err.into_message().contains("newer format"), + "the message must name the newer format" + ); } - // Fetch, verify, and concatenate all chunks. - let mut reconstructed = String::with_capacity(pointer.envelope_len); - for chunk_ref in &pointer.chunks { - let chunk_value = fetch(&chunk_ref.key)?.ok_or_else(|| { - format!( - "missing chunk `{}` referenced by pointer at `{root_key}`", - chunk_ref.key - ) - })?; + /// FROZEN v1 MULTIBYTE WIRE FORMAT. Pins the codepoint-boundary RETREAT: a + /// 4-byte codepoint is placed so it straddles byte 7 000, and chunk 0 is + /// hand-split at the retreat boundary (the codepoint's start, 6 998) using a + /// frozen rule written out in the test -- never the production writer. If the + /// writer's retreat logic ever changes to split elsewhere, the resolver's own + /// exact-split replay no longer matches this frozen boundary and the test + /// fails, catching a wire-incompatible change that ASCII fixtures cannot. + #[test] + fn resolver_reads_a_frozen_v1_multibyte_pointer_split_mid_codepoint() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; - // Verify length. - if chunk_value.len() != chunk_ref.len { - return Err(format!( - "chunk `{}` (referenced by `{root_key}`) has length {} but pointer \ - records {}", - chunk_ref.key, - chunk_value.len(), - chunk_ref.len, - )); - } + /// v1 chunk payload boundary, in BYTES. Frozen. + const V1_SPLIT: usize = 7_000; + const V1_INFIX: &str = ".__edgezero_chunks."; + /// A 4-byte codepoint (U+1F980, the crab). + const CRAB: &str = "\u{1F980}"; + /// The crab starts here: the largest codepoint boundary <= 7 000. + const V1_BOUNDARY: usize = 6_998; + // PRECOMPUTED wire hashes for this exact fixture, captured once and fed in + // as LITERALS. A drift in serialisation, the SHA helper, or the retreat + // boundary makes the live bytes disagree with these and the test fails. + const FROZEN_LEN: usize = 9_131; + const FROZEN_ENVELOPE_SHA: &str = + "769140bf1a0c17cf64074ca182e9c07869d4100f1eb99d2b68a847022c1bbe13"; + const FROZEN_INNER_SHA: &str = + "457e339ad691524cc35a69c0d2d947e9338b09f9e0d841945b184807279ab22f"; + const FROZEN_CHUNK0_SHA: &str = + "edff7b12241b5b5640a5d9b2af8af43e3ff4ce95a45ef6f31a59c7631282b7c3"; + const FROZEN_CHUNK1_SHA: &str = + "e4031d9aecd7374bb79ee535bc481d2e1b1b9d40385d9ee57fda7132c4dcb015"; + const FROZEN_CHUNK1_LEN: usize = FROZEN_LEN - V1_BOUNDARY; + + // Locate where the `frozen` string VALUE begins in the serialised + // envelope. The prefix up to it is constant, so a probe measures it. + let marker = "\"frozen\":\""; + let probe = serde_json::to_string(&BlobEnvelope::new( + json!({ "frozen": "" }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .expect("probe"); + let content_start = probe.find(marker).expect("frozen marker") + marker.len(); + + // Place the crab so it occupies absolute bytes 6 998..7 002 -- byte 7 000 + // falls INSIDE it. Pad after it so the envelope exceeds the entry limit. + let lead = V1_SPLIT.saturating_sub(2).saturating_sub(content_start); + let frozen = format!("{}{CRAB}{}", "v".repeat(lead), "v".repeat(2_000)); + let envelope = serde_json::to_string(&BlobEnvelope::new( + json!({ "frozen": frozen }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .expect("envelope"); + // Freeze the wire. + assert_eq!(envelope.len(), FROZEN_LEN, "envelope byte length drifted"); + assert_eq!( + sha256_hex(envelope.as_bytes()), + FROZEN_ENVELOPE_SHA, + "envelope wire bytes or the SHA helper drifted from the frozen v1 value" + ); + assert!( + !envelope.is_char_boundary(V1_SPLIT), + "byte 7000 must fall mid-codepoint so retreat is exercised" + ); - // Verify SHA. - let actual_sha = sha256_hex(chunk_value.as_bytes()); - if actual_sha != chunk_ref.sha256 { - return Err(format!( - "chunk `{}` (referenced by `{root_key}`) SHA mismatch: \ - expected `{}`, got `{actual_sha}`", - chunk_ref.key, chunk_ref.sha256, - )); + // Frozen retreat rule, hand-written: the largest codepoint boundary + // <= 7000. This is what a v1 writer did; it is NOT the production fn. + let mut boundary = V1_SPLIT; + while boundary > 0 && !envelope.is_char_boundary(boundary) { + boundary = boundary.saturating_sub(1); } + assert_eq!(boundary, V1_BOUNDARY, "the crab starts at byte 6998"); - reconstructed.push_str(&chunk_value); + let head = envelope.get(..boundary).expect("codepoint boundary"); + let tail = envelope.get(boundary..).expect("codepoint boundary"); + assert_eq!( + sha256_hex(head.as_bytes()), + FROZEN_CHUNK0_SHA, + "chunk 0 drifted" + ); + assert_eq!( + sha256_hex(tail.as_bytes()), + FROZEN_CHUNK1_SHA, + "chunk 1 drifted" + ); + assert_eq!(tail.len(), FROZEN_CHUNK1_LEN); + + let key0 = format!("root{V1_INFIX}{FROZEN_ENVELOPE_SHA}.0"); + let key1 = format!("root{V1_INFIX}{FROZEN_ENVELOPE_SHA}.1"); + let pointer = json!({ + "chunks": [ + { "key": key0, "len": V1_BOUNDARY, "sha256": FROZEN_CHUNK0_SHA }, + { "key": key1, "len": FROZEN_CHUNK1_LEN, "sha256": FROZEN_CHUNK1_SHA }, + ], + "data_sha256": FROZEN_INNER_SHA, + "edgezero_kind": "fastly_config_chunks", + "envelope_len": FROZEN_LEN, + "envelope_sha256": FROZEN_ENVELOPE_SHA, + "version": 1_u8, + }) + .to_string(); + + let chunk_map: HashMap = + HashMap::from([(key0, head.to_owned()), (key1, tail.to_owned())]); + let resolved = + resolve_fastly_config_value("root", pointer, |key| Ok(chunk_map.get(key).cloned())) + .expect("a v1 multibyte value split mid-codepoint must resolve"); + assert_eq!(resolved, envelope, "must reconstruct the original bytes"); } - // Verify total envelope length. - if reconstructed.len() != pointer.envelope_len { - return Err(format!( - "reconstructed envelope for `{root_key}` has length {} but pointer \ - records {}", - reconstructed.len(), - pointer.envelope_len, - )); + /// v1 READ-COMPATIBILITY: an envelope UNDER 8 000 characters but OVER 8 000 + /// UTF-8 BYTES is chunked by this writer's conservative byte-count threshold + /// (the merge-base behaviour), and MUST reconstruct and resolve — a + /// character-based threshold would leave it stored directly and reject the + /// existing chunked pointer on read (an HTTP 500 for previously stored data). + #[test] + fn chunked_value_under_char_limit_but_over_byte_limit_resolves() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + // ~2001 crabs of data: the serialised envelope is > 8000 BYTES but + // < 8000 CHARACTERS. + let data = json!({ "crabs": "\u{1F980}".repeat(2_001) }); + let envelope = + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".into())).unwrap(); + assert!( + envelope.len() > FASTLY_CONFIG_ENTRY_LIMIT, + "must exceed the byte limit" + ); + assert!( + envelope.chars().count() <= FASTLY_CONFIG_ENTRY_LIMIT, + "must fit the character limit -- this is the compatibility gap" + ); + + // The writer chunks it (byte-based), exactly as the merge-base did. + let entries = prepare_fastly_config_entries("app_config", &envelope).unwrap(); + assert!( + entries.len() >= 3, + "must have chunked (>= 2 chunks + pointer)" + ); + let (root_key, pointer_json) = entries.last().unwrap().clone(); + let chunk_map: HashMap = entries[..entries.len().saturating_sub(1)] + .iter() + .cloned() + .collect(); + + // The resolver must ACCEPT it and reconstruct the original. + let resolved = resolve_fastly_config_value(&root_key, pointer_json, |key| { + Ok(chunk_map.get(key).cloned()) + }) + .expect("an existing v1 byte-chunked value must still resolve"); + assert_eq!(resolved, envelope); } - // Verify total envelope SHA. - let actual_env_sha = sha256_hex(reconstructed.as_bytes()); - if actual_env_sha != pointer.envelope_sha256 { - return Err(format!( - "reconstructed envelope for `{root_key}` SHA mismatch: \ - expected `{}`, got `{actual_env_sha}`", - pointer.envelope_sha256, - )); + /// An empty root key is rejected before any output: writer-valid but + /// resolver-invalid (canonical chunk parsing rejects an empty root). + #[test] + fn empty_root_key_is_rejected() { + assert!( + prepare_fastly_config_entries("", "{}").is_err(), + "an empty key must be rejected on the direct path" + ); + let big = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + assert!( + prepare_fastly_config_entries("", &big).is_err(), + "an empty key must be rejected on the chunked path too" + ); } - Ok(reconstructed) -} + /// A root that is itself valid but long enough that its DERIVED chunk keys + /// exceed the store limit must be rejected up front, not fail mid-write. + #[test] + fn oversized_derived_chunk_keys_are_rejected_before_write() { + // A root ~200 chars: valid as a key on its own, but + ~85 for the chunk + // suffix exceeds the limit. + let long_root = "r".repeat(200); + let big_envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let err = prepare_fastly_config_entries(&long_root, &big_envelope) + .expect_err("derived chunk keys over the limit must be rejected"); + assert!( + err.contains(&FASTLY_CONFIG_KEY_LIMIT.to_string()) && err.contains("limit"), + "error must name the key limit: {err}" + ); -// --------------------------------------------------------------------------- -// Unit tests -// --------------------------------------------------------------------------- + // A root over the limit is rejected even for a DIRECT (unchunked) value. + let over_limit_root = "r".repeat(FASTLY_CONFIG_KEY_LIMIT.saturating_add(1)); + assert!( + prepare_fastly_config_entries(&over_limit_root, "{}").is_err(), + "a root key over the limit must be rejected on the direct path too" + ); -#[cfg(test)] -mod tests { - use std::collections::HashMap; + // A normal root is unaffected. + prepare_fastly_config_entries("app_config", &big_envelope) + .expect("a normal root must still expand"); - use super::*; + // The limit is CHARACTERS, not bytes: a multi-byte key at exactly the + // character limit must be accepted even though its byte length exceeds + // it. U+00E9 ('é') is 2 bytes, so 255 of them is 510 bytes, 255 chars. + let multibyte_root = "\u{e9}".repeat(FASTLY_CONFIG_KEY_LIMIT); + assert!( + multibyte_root.len() > FASTLY_CONFIG_KEY_LIMIT, + "fixture must be multi-byte" + ); + prepare_fastly_config_entries(&multibyte_root, "{}") + .expect("a key at the CHARACTER limit must be accepted regardless of byte length"); + } // ---- helpers ---- @@ -387,14 +2307,15 @@ mod tests { use edgezero_core::blob_envelope::BlobEnvelope; use serde_json::json; - // crab emoji: 4 bytes each; 3000 crabs = 12 000 bytes of payload + // crab emoji: 4 bytes each. The direct-vs-chunked threshold is a + // conservative BYTE count, so 3 000 crabs (12 000 bytes) chunks. let emoji_block = "\u{1F980}".repeat(3_000); let data = json!({ "crabs": emoji_block }); let envelope = serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".into())).unwrap(); assert!( envelope.len() > FASTLY_CONFIG_ENTRY_LIMIT, - "emoji envelope must be oversized" + "emoji envelope must exceed the byte limit so it chunks" ); let entries = prepare_fastly_config_entries("emoji_key", &envelope).unwrap(); @@ -456,6 +2377,50 @@ mod tests { // ---- resolve tests ---- + /// GENERIC CONTRACT: a Config Store holds ARBITRARY values, not only our + /// envelopes. Anything that does not announce itself as one of our chunk + /// pointers must come back VERBATIM. Routing every value through + /// envelope/pointer parsing turned ordinary entries (`"value_a"`, the + /// documented `greeting = "hello from config store"`) into corruption + /// errors and broke the shared `ConfigStore` contract. + #[test] + fn resolver_passes_through_values_that_are_not_our_pointers() { + for raw in [ + "value_a", + "hello from config store", + "", + "42", + " ", + r#"{"unrelated":"json"}"#, + "{not even valid json", + ] { + let out = resolve_fastly_config_value("k", raw.to_owned(), |_chunk_key| { + Err("fetch must not be called for a non-pointer value".to_owned()) + }) + .unwrap_or_else(|err| panic!("raw value {raw:?} must pass through: {err}")); + assert_eq!(out, raw, "value must be returned verbatim"); + } + + // A value that CLAIMS our namespace but is not our pointer kind -- an + // unknown string OR a non-string `edgezero_kind` -- is NOT passed through: + // it is our reserved field, so an unrecognised value in it is an error, + // never an ordinary entry. + for raw in [ + r#"{"edgezero_kind":"fastly_config_chunks_v2"}"#, + r#"{"edgezero_kind":null}"#, + r#"{"edgezero_kind":7}"#, + ] { + let err = resolve_fastly_config_value("k", raw.to_owned(), |_chunk_key| { + Err("fetch must not be called".to_owned()) + }) + .expect_err(&format!("a namespace-claiming value must error: {raw}")); + assert!( + err.contains("unknown `edgezero_kind`"), + "must report an unknown kind: {err}" + ); + } + } + #[test] fn resolver_returns_direct_envelope_unchanged() { use edgezero_core::blob_envelope::BlobEnvelope; @@ -533,12 +2498,20 @@ mod tests { }) .expect_err("corrupt chunk must error"); assert!( - err.contains("SHA mismatch") || err.contains("sha") || err.contains("mismatch"), - "error must mention hash mismatch: {err}" + err.contains("does not match the SHA-256"), + "error must say what failed: {err}" + ); + // the diagnostic identifies the chunk by POSITION, not by + // key. The key comes from the stored pointer, so echoing it would let a + // malformed pointer smuggle a secret into a log line. Same for the + // hashes it records. + assert!( + err.contains("chunk 0"), + "error must locate the failing chunk by position: {err}" ); assert!( - err.contains(&first_chunk_key), - "error must name the failing chunk key: {err}" + !err.contains(&first_chunk_key), + "a pointer-controlled key must not be echoed into a log line: {err}" ); } @@ -568,13 +2541,14 @@ mod tests { } }) .expect_err("length-mismatched chunk must error"); + assert!(err.contains("length"), "error must mention length: {err}"); assert!( - err.contains("length") || err.contains("len"), - "error must mention length: {err}" + err.contains("chunk 0"), + "error must locate the failing chunk by position: {err}" ); assert!( - err.contains(&first_chunk_key), - "error must name the failing chunk key: {err}" + !err.contains(&first_chunk_key), + "a pointer-controlled key must not be echoed into a log line: {err}" ); } @@ -591,26 +2565,99 @@ mod tests { .cloned() .collect(); - // Tamper with envelope_sha256 in the pointer JSON. + // `envelope_sha256` is a stored, value-controlled string. + // Set it to a SENTINEL secret and require the diagnostic not to echo it. + let sentinel = "SUPER_SECRET_ENVELOPE_HASH"; let mut pointer: FastlyChunkPointer = serde_json::from_str(&pointer_json).unwrap(); - pointer.envelope_sha256 = "ff".repeat(32); + pointer.envelope_sha256 = sentinel.to_owned(); let tampered_pointer_json = serde_json::to_string(&pointer).unwrap(); let err = resolve_fastly_config_value(&root_key, tampered_pointer_json, |chunk_key| { Ok(chunk_map.get(chunk_key).cloned()) }) .expect_err("envelope hash mismatch must error"); + // Whether metadata validation (the chunk keys name a different + // generation than the tampered `envelope_sha256`) or the full-envelope + // check catches it, the stored hash must never reach the diagnostic. assert!( - err.contains("SHA mismatch") || err.contains("mismatch"), - "error must mention hash mismatch: {err}" + !err.contains(sentinel), + "the stored envelope_sha256 must never reach a diagnostic: {err}" ); assert!(err.contains("root"), "error must name the root key: {err}"); } + /// the CHUNKED read path returns reconstructed bytes + /// after checking only the OUTER pointer hash -- it never parses/verifies the + /// inner `BlobEnvelope` the way the DIRECT path does. So an envelope whose + /// embedded `sha256` is a secret passes the resolver, and core's later + /// `verify()` formats that stored hash into an HTTP 500. The resolver must + /// reject it here, redacted. + #[test] + fn chunked_read_verifies_inner_envelope() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + const SENTINEL: &str = "SUPER_SECRET_INNER_SHA"; + + // A structurally valid envelope whose inner `sha256` is a sentinel that + // does NOT match its data -- so `verify()` fails and would name it. + let mut envelope = BlobEnvelope::new(json!({"k":"v"}), "2026-06-22T00:00:00Z".into()); + envelope.sha256 = SENTINEL.to_owned(); + // Pad the SERIALISED envelope past the entry limit so it chunks. + let mut envelope_json = serde_json::to_string(&envelope).unwrap(); + envelope_json.push_str(&" ".repeat(FASTLY_CONFIG_ENTRY_LIMIT)); + let entries = prepare_fastly_config_entries("root", &envelope_json).unwrap(); + let (root_key, pointer_json) = entries.last().unwrap().clone(); + let chunk_map: HashMap = entries[..entries.len().saturating_sub(1)] + .iter() + .cloned() + .collect(); + + let err = resolve_fastly_config_value(&root_key, pointer_json, |chunk_key| { + Ok(chunk_map.get(chunk_key).cloned()) + }) + .expect_err("a reconstructed envelope with a bad inner sha must be rejected"); + assert!( + !err.contains(SENTINEL), + "the inner envelope hash must never reach a diagnostic: {err}" + ); + } + + /// a fetch-callback failure carries the pointer-controlled + /// chunk KEY. The resolver must locate the fault by position, not echo it. + #[test] + fn resolver_fetch_error_does_not_leak_chunk_key() { + const SENTINEL: &str = "SUPER_SECRET_IN_A_CHUNK_KEY"; + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries("root", &envelope).unwrap(); + let (root_key, pointer_json) = entries.last().unwrap().clone(); + // A VALID pointer (so metadata validation passes), whose first chunk key + // the callback names in its failure — as the real store lookup would. + let pointer: FastlyChunkPointer = serde_json::from_str(&pointer_json).unwrap(); + let secret_key = pointer.chunks[0].key.clone(); + + let err = resolve_fastly_config_value(&root_key, pointer_json, |chunk_key| { + Err(format!( + "config store lookup failed for `{chunk_key}` -- {SENTINEL}" + )) + }) + .expect_err("a fetch failure must error"); + assert!( + !err.contains(SENTINEL) && !err.contains(&secret_key), + "a fetch failure must not echo the pointer-controlled chunk key: {err}" + ); + assert!( + err.contains("chunk 0") && err.contains("could not be fetched"), + "the error must locate the fault by position: {err}" + ); + } + #[test] fn resolver_errors_on_malformed_pointer() { - // A value that is neither a valid BlobEnvelope nor a valid pointer. - let err = resolve_fastly_config_value("my_key", "not json at all".to_owned(), |_| Ok(None)) + // A value that ANNOUNCES our pointer kind but is missing every other + // required field. It claims to be ours, so it must error rather than + // pass through as an ordinary value. + let malformed = format!(r#"{{"edgezero_kind":"{POINTER_KIND}"}}"#); + let err = resolve_fastly_config_value("my_key", malformed, |_| Ok(None)) .expect_err("malformed pointer must error"); assert!( err.contains("my_key"), @@ -657,4 +2704,144 @@ mod tests { ); assert!(err.contains("my_key"), "error must name root key: {err}"); } + + // ---- prior_chunk_keys tests ---- + + #[test] + fn prior_chunk_keys_returns_valid_v1_pointer_keys() { + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries("app_config", &envelope).unwrap(); + let (_, pointer_json) = entries.last().unwrap(); + + let keys = prior_chunk_keys("app_config", pointer_json).expect("valid pointer"); + + let expected: Vec = entries[..entries.len().saturating_sub(1)] + .iter() + .map(|(key, _)| key.clone()) + .collect(); + assert_eq!(keys, expected); + } + + #[test] + fn prior_chunk_keys_returns_empty_for_direct_envelope() { + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT); + assert_eq!( + prior_chunk_keys("app_config", &envelope).unwrap(), + Vec::::new() + ); + } + + #[test] + fn prior_chunk_keys_returns_empty_for_unrelated_json() { + assert_eq!( + prior_chunk_keys("app_config", r#"{"hello":"world"}"#).unwrap(), + Vec::::new() + ); + } + + #[test] + fn prior_chunk_keys_returns_empty_for_wrong_kind() { + let raw = r#"{"edgezero_kind":"other","version":1,"chunks":[],"data_sha256":"","envelope_len":0,"envelope_sha256":""}"#; + assert_eq!( + prior_chunk_keys("app_config", raw).unwrap(), + Vec::::new() + ); + } + + #[test] + fn prior_chunk_keys_rejects_unsupported_pointer_version() { + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries("app_config", &envelope).unwrap(); + let (_, pointer_json) = entries.last().unwrap(); + let mut pointer: FastlyChunkPointer = serde_json::from_str(pointer_json).unwrap(); + pointer.version = 2; + let raw = serde_json::to_string(&pointer).unwrap(); + + let err = prior_chunk_keys("app_config", &raw).expect_err("version 2 should warn"); + assert!(err.contains("unsupported version"), "{err}"); + } + + #[test] + fn prior_chunk_keys_rejects_foreign_chunk_prefix() { + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries("app_config", &envelope).unwrap(); + let (_, pointer_json) = entries.last().unwrap(); + let mut pointer: FastlyChunkPointer = serde_json::from_str(pointer_json).unwrap(); + pointer.chunks[0].key = + pointer.chunks[0] + .key + .replacen("app_config", "app_config_staging", 1); + let raw = serde_json::to_string(&pointer).unwrap(); + + let err = prior_chunk_keys("app_config", &raw).expect_err("foreign chunk should warn"); + assert!(err.contains("non-canonical"), "{err}"); + } + + // ---- chunk_key_generation (canonical only) ---- + + #[test] + fn chunk_key_generation_extracts_the_sha() { + let envelope = make_envelope_json(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = prepare_fastly_config_entries("app_config", &envelope).unwrap(); + let (chunk_key, _) = &entries[0]; + let sha = chunk_key_generation("app_config", chunk_key).expect("a chunk key"); + assert_eq!(sha.len(), 64, "64-char sha256 hex"); + assert!(chunk_key.contains(&sha)); + } + + #[test] + fn chunk_key_generation_requires_canonical_shape() { + let good = "a".repeat(64); + let base = format!("app_config.__edgezero_chunks.{good}"); + // Canonical. + assert!(chunk_key_generation("app_config", &format!("{base}.0")).is_some()); + assert!(chunk_key_generation("app_config", &format!("{base}.17")).is_some()); + + // Different root / the root itself. + assert!( + chunk_key_generation("app_config", &format!("other.__edgezero_chunks.{good}.0")) + .is_none() + ); + assert!(chunk_key_generation("app_config", "app_config").is_none()); + // Empty root. + assert!(chunk_key_generation("", &format!(".__edgezero_chunks.{good}.0")).is_none()); + // Short SHA (< 64). + assert!( + chunk_key_generation("app_config", "app_config.__edgezero_chunks.abc123.0").is_none() + ); + // Uppercase SHA. + let upper = "A".repeat(64); + assert!( + chunk_key_generation( + "app_config", + &format!("app_config.__edgezero_chunks.{upper}.0") + ) + .is_none() + ); + // Non-hex SHA of the right length. + let nonhex = "z".repeat(64); + assert!( + chunk_key_generation( + "app_config", + &format!("app_config.__edgezero_chunks.{nonhex}.0") + ) + .is_none() + ); + // Leading-zero / non-numeric / missing index. + assert!(chunk_key_generation("app_config", &format!("{base}.00")).is_none()); + assert!(chunk_key_generation("app_config", &format!("{base}.007")).is_none()); + assert!(chunk_key_generation("app_config", &format!("{base}.x")).is_none()); + assert!(chunk_key_generation("app_config", &base).is_none()); + } + + // Regression for the Value-first rule: a value that IS pointer-kind but + // is missing required fields (`chunks`, `data_sha256`, …) must WARN, not + // be silently dropped by a failed struct deserialize. + #[test] + fn prior_chunk_keys_warns_on_pointer_kind_with_missing_fields() { + let raw = r#"{"edgezero_kind":"fastly_config_chunks","version":2}"#; + let err = prior_chunk_keys("app_config", raw) + .expect_err("pointer-kind but malformed must warn, not Ok([])"); + assert!(!err.is_empty(), "{err}"); + } } diff --git a/crates/edgezero-adapter-fastly/src/cli.rs b/crates/edgezero-adapter-fastly/src/cli.rs index 678f7b3f..f3e64b31 100644 --- a/crates/edgezero-adapter-fastly/src/cli.rs +++ b/crates/edgezero-adapter-fastly/src/cli.rs @@ -1,11 +1,22 @@ +use std::cell::{Cell, RefCell}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::env; +use std::fmt::Write as _; use std::fs; use std::io::{ErrorKind, Write as _}; use std::path::{Path, PathBuf}; +use std::process::ChildStdin; use std::process::Command; use std::process::Stdio; - -use crate::chunked_config::{prepare_fastly_config_entries, resolve_fastly_config_value}; +use std::process::id as process_id; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::chunked_config::{ + CHUNK_KEY_INFIX, GcPointer, GcRootValue, ResolveFailure, chunk_key_generation, chunk_lengths, + gc_classify_root, gc_verify_generation, prepare_fastly_config_entries, prior_chunk_keys, + resolve_fastly_config_value_typed, sha256_hex, value_announces_our_kind, + value_is_future_format, value_is_inert_foreign, verify_writer_split_layout, +}; use ctor::ctor; use edgezero_adapter::cli_support::{ find_manifest_upwards, find_workspace_root, path_distance, read_package_name, run_native_cli, @@ -117,6 +128,12 @@ static FASTLY_TEMPLATE_REGISTRATIONS: &[TemplateRegistration] = &[ const FASTLY_INSTALL_HINT: &str = "install the Fastly CLI (https://www.fastly.com/documentation/reference/tools/cli/) and try again"; +/// Hard-error message for a value written by a NEWER format this v1 CLI must not +/// overwrite. Shared by the read path so the wording stays consistent. +const FUTURE_FORMAT_READ_ERROR: &str = "the remote value uses a config format this CLI version does not recognise (a newer \ + `edgezero_kind` or envelope/pointer version); UPGRADE the CLI to push to this store rather \ + than overwrite a newer format."; + struct FastlyCliAdapter; /// Outcome of scanning `fastly config-store list --json` for a @@ -141,6 +158,108 @@ enum ConfigStoreLookup { SchemaDrift(String), } +/// The reclamation plan for `config gc`: the orphan chunk entries to delete +/// (with their ages) plus the counts for the summary line. Produced by +/// `plan_gc_reclamation` (which owns every safety guard); consumed by +/// `gc_fastly_config_store` (which reports and deletes). +struct GcPlan { + /// Whole generations to reclaim, each a list of `(key, age_secs)`. Grouped, + /// not flat: a generation is provable only as a UNIT (see + /// `prove_generation`), so deleting part of one destroys the very evidence + /// that licenses deleting the rest. + doomed: Vec>, + live_count: usize, + retained_recent: usize, + roots: usize, + /// Chunk-shaped entries we could NOT prove our writer produced, so left + /// untouched. Surfaced so an operator can see we declined to judge them. + unprovable: usize, + /// Non-fatal problems to print — see `GcClassification::warnings`. + warnings: Vec, +} + +/// What one pass of `config gc`'s delete loop actually did. +struct GcDeleteOutcome { + /// Entries whose delete returned success. + deleted: usize, + /// Keys whose delete returned non-zero. + failed: Vec, + /// Survivors of a generation in which an earlier sibling's delete had + /// ALREADY succeeded before a later one failed. These are definitely an + /// incomplete generation now, so they can never be proved (or reclaimed) + /// again -- manual removal only. + stranded: Vec, + /// Members of a generation whose ONLY failure was on a delete with no + /// confirmed prior sibling success. A failed remote delete has UNKNOWN + /// outcome (Fastly may have committed it before returning an error), so we + /// cannot say whether the generation is still whole. A re-run reclaims it if + /// it is, or reports it as an unprovable fragment if it is not. + uncertain: Vec, +} + +/// The result of classifying a store's entries for reclamation. +struct GcClassification { + /// Chunk keys a live root pointer references, each verified against its + /// content-address. Never deletable. + live: HashSet, + /// Keys whose OWN value is a runtime-readable root — a valid direct envelope + /// or a pointer — regardless of what their key looks like. Never deletable. + protected: HashSet, + /// Count of entries classified as roots, for the summary line. + roots: usize, + /// Non-fatal problems the operator should see — currently roots that are + /// not runtime-readable and so can never be reclaimed automatically. + warnings: Vec, +} + +/// One `config-store-entry list` item. +/// +/// `item_value` IS captured — `config gc` must parse root pointers to learn +/// which chunks are live, and one listing avoids a `describe` per root. It is +/// the config payload: it may be read in memory but must NEVER be logged or +/// surfaced (see `redact_describe_response` / `redact_stderr`). +struct ConfigStoreItem { + created_at: String, + item_key: String, + item_value: String, +} + +/// Per-root plan for the LOCAL path's eager prune. +/// +/// Local reclamation is safe to do immediately: `fastly.toml` is a single +/// file that Viceroy reads at startup — there is no propagation window and no +/// POP that could still be serving the previous pointer. (The cloud path +/// cannot do this; see `reclaim_orphan_generations`.) +struct FastlyConfigGcPlan { + /// Exact keep-set this push writes for the root (chunk keys + root key). + new_keys: HashSet, + /// Prior chunk keys to consider deleting, or a warning to surface + /// (suspicious prior pointer) that skips GC for this root. + prior_keys: Result, String>, +} + +/// An exclusive, cross-process advisory lock covering a local `fastly.toml` +/// rewrite. Serialises concurrent pushes so their read-modify-write cycles +/// cannot interleave and lose each other's edits. +/// +/// The lock is a persistent sidecar file next to the manifest. It is never +/// unlinked — deleting it would reintroduce a create/lock race between two +/// processes each making their own lock file. Dropping the guard releases the +/// OS lock (closing the file descriptor). `File::lock` is advisory, so it only +/// coordinates other lockers, which is exactly the pushes we control. +struct ManifestLock { + _file: fs::File, + /// The REAL file the lock guards, resolved through any symlink. Callers read + /// and replace THIS path, so every alias operates on one target. + target: PathBuf, +} + +/// Removes a staging temp file on drop unless disarmed — so every early return +/// (permission failure, write failure, rename failure) cleans up after itself. +struct TempFileGuard { + path: Option, +} + // The three `validate_*` trait methods exist on `Adapter` because // spin requires them (variable-name regex, `[component.*]` // discovery, flat-namespace collision). The trait surface is typed @@ -191,10 +310,46 @@ impl Adapter for FastlyCliAdapter { } } + fn gc_config_entries( + &self, + _manifest_root: &Path, + _adapter_manifest_path: Option<&str>, + _component_selector: Option<&str>, + store: &ResolvedStoreId, + _push_ctx: &AdapterPushContext<'_>, + older_than_secs: u64, + dry_run: bool, + ) -> Result, String> { + gc_fastly_config_store(store.platform.as_str(), older_than_secs, dry_run) + } + fn name(&self) -> &'static str { "fastly" } + fn preflight_config_write(&self, key: &str, body: &str) -> Result<(), String> { + // Reject an infeasible push here, BEFORE the CLI's remote read, so it + // fails offline rather than after a list/describe. The write path + // re-checks, so this is a strict early gate, not the only one. + // + // An empty key is writer-valid but resolver-invalid (canonical chunk + // parsing rejects an empty root); reject it before any I/O. + if key.is_empty() { + return Err( + "config key is empty; provide a store id or a non-empty `--key`".to_owned(), + ); + } + let entry = [(key.to_owned(), String::new())]; + reject_reserved_root_keys(&entry)?; + // Run the full chunk expansion OFFLINE (no I/O): exactly what the write + // path does, so every body-dependent feasibility failure — the root key + // over the store limit, a DERIVED chunk key over it once the value + // chunks, or a pointer that would not fit the entry limit — is caught + // here, before the remote read, instead of after it. + prepare_fastly_config_entries(key, body)?; + Ok(()) + } + fn provision( &self, manifest_root: &Path, @@ -368,21 +523,25 @@ impl Adapter for FastlyCliAdapter { "no config entries to push to fastly config-store `{name}` (logical id `{logical}`)" )]); } - // Expand each logical (key, envelope_json) into physical entries - // via the chunk-pointer helper. Entries ≤ 8 000 chars go through - // as a single direct entry; larger envelopes are split into - // content-addressed chunks with a root pointer written LAST. - // Collect all physical entries before any writes so pointer-too- - // large errors surface before touching the remote store. + // Reject reserved keys before any expansion or I/O. + reject_reserved_root_keys(entries)?; + reject_duplicate_root_keys(entries)?; + // Expand each logical root once: flatten for the commit, and keep + // the exact per-root keep-set + the value written at the root key + // for GC (no prefix scan of the flattened set). Collecting all + // physical entries first also surfaces pointer-too-large errors + // before touching the remote store. let mut physical_entries: Vec<(String, String)> = Vec::new(); + let mut roots: Vec<(String, HashSet, String)> = Vec::with_capacity(entries.len()); for (key, body) in entries { - let expanded = prepare_fastly_config_entries(key, body)?; + let (expanded, new_keys, new_root_value) = expand_root(key, body)?; physical_entries.extend(expanded); + roots.push((key.clone(), new_keys, new_root_value)); } if dry_run { - // Report intent without shelling out. One line per logical key - // noting whether it would be direct or chunked, plus chunk count. - let mut out = Vec::with_capacity(entries.len().saturating_add(1)); + // Report intent without shelling out. Stays fully offline: no + // store-id resolution, no remote read (so no GC count). + let mut out = Vec::with_capacity(entries.len().saturating_mul(2).saturating_add(1)); out.push(format!( "would resolve fastly config-store `{name}` (logical id `{logical}`) via `fastly config-store list --json` and push entries:" )); @@ -404,7 +563,23 @@ impl Adapter for FastlyCliAdapter { } return Ok(out); } - let resolved_id = resolve_remote_config_store_id(name)?; + let resolved_id = + resolve_remote_config_store_id(name)?.ok_or_else(|| no_matching_store_error(name))?; + // NOTE: a cloud push does NOT reclaim orphaned chunks. + // + // Fastly's config store is eventually consistent, so a generation may + // only be deleted once the pointer that referenced it has stopped being + // served everywhere. Fastly records no pointer-supersession time + // (`updated_at` is NOT bumped by `update --upsert` -- verified against + // the live API), offers no compare-and-swap with which to record one + // safely, and chunk `created_at` is NOT a proxy for it (a chunked -> + // direct -> direct transition leaves the old generation with no + // "successor" at all). Every attempt to synthesise that fact is unsound. + // + // So reclamation is an explicit, operator-invoked `config gc`: the + // operator supplies the one fact the platform cannot -- that the current + // config has been live long enough that nothing is serving the old + // pointers. See the spec's "Cloud reclamation". push_entries_with_committer(&physical_entries, |key, value| { create_config_store_entry(&resolved_id, key, value) })?; @@ -447,19 +622,26 @@ impl Adapter for FastlyCliAdapter { fastly_path.display() )]); } - // Expand logical entries into physical entries (chunks + pointer). + // Reject reserved keys before any expansion or I/O. + reject_reserved_root_keys(entries)?; + reject_duplicate_root_keys(entries)?; + // Expand each logical root once: flatten for the write, keep the + // exact per-root keep-set for GC (no prefix scan of the flattened set). let mut physical_entries: Vec<(String, String)> = Vec::new(); + let mut gc_roots: Vec<(String, HashSet)> = Vec::with_capacity(entries.len()); for (key, body) in entries { - let expanded = prepare_fastly_config_entries(key, body)?; + let (expanded, new_keys, _new_root) = expand_root(key, body)?; physical_entries.extend(expanded); + gc_roots.push((key.clone(), new_keys)); } if dry_run { - let mut out = Vec::with_capacity(entries.len().saturating_add(1)); + let counts = local_orphan_counts_for_dry_run(&fastly_path, name, entries); + let mut out = Vec::with_capacity(entries.len().saturating_mul(2).saturating_add(1)); out.push(format!( "would edit `[local_server.config_stores.{name}.contents]` in {} (logical id `{logical}`) with entries:", fastly_path.display(), )); - for (key, body) in entries { + for (idx, (key, body)) in entries.iter().enumerate() { let expanded = prepare_fastly_config_entries(key, body) .unwrap_or_else(|_| vec![(key.clone(), body.clone())]); if expanded.len() == 1 { @@ -474,16 +656,28 @@ impl Adapter for FastlyCliAdapter { body.len() )); } + match counts.get(idx).map(|(_, count)| count) { + Some(Ok(n)) => out.push(format!( + " would delete {n} orphan chunks from the previous generation of `{key}`" + )), + Some(Err(reason)) => out.push(format!( + " would delete an unknown number of orphan chunks from the previous generation of `{key}` (unknown: {reason})" + )), + None => {} + } } return Ok(out); } - write_fastly_local_config_store(&fastly_path, name, &physical_entries)?; - Ok(vec![format!( + let warnings = + write_fastly_local_config_store(&fastly_path, name, &physical_entries, &gc_roots)?; + let mut out = vec![format!( "wrote {} physical entries ({} logical) to `[local_server.config_stores.{name}.contents]` in {} (logical id `{logical}`); restart `fastly compute serve` to pick up changes", physical_entries.len(), entries.len(), fastly_path.display() - )]) + )]; + out.extend(warnings); + Ok(out) } fn read_config_entry( @@ -500,19 +694,12 @@ impl Adapter for FastlyCliAdapter { // demand via `fastly config-store list --json`, then parse the // JSON response. let name = store.platform.as_str(); - let store_id = match resolve_remote_config_store_id(name) { - Ok(id) => id, - Err(err) => { - // "not found" from resolve means the store doesn't exist. - let lower = err.to_ascii_lowercase(); - if lower.contains("not found") - || lower.contains("did you run") - || lower.contains("no fastly config-store matches") - { - return Ok(ReadConfigEntry::MissingStore); - } - return Err(err); - } + // A TYPED absence: `Ok(None)` (list succeeded, no store matched) is the + // only path to MissingStore. Any operational failure stays `Err` and fails + // closed -- an incomplete read must never read as absence and authorise an + // overwrite of healthy remote state. + let Some(store_id) = resolve_remote_config_store_id(name)? else { + return Ok(ReadConfigEntry::MissingStore); }; let store_arg = format!("--store-id={store_id}"); let key_arg = format!("--key={key}"); @@ -533,41 +720,77 @@ impl Adapter for FastlyCliAdapter { } })?; if output.status.success() { - let stdout = String::from_utf8_lossy(&output.stdout); + let stdout = strict_stdout(output.stdout, "config-store-entry describe --json")?; // Parse the JSON and extract the `item_value` field. - let parsed: serde_json::Value = - serde_json::from_str(&stdout).map_err(|err| { - format!( - "failed to parse `fastly config-store-entry describe` JSON: {err}\nraw stdout: {stdout}" - ) - })?; + let parsed: serde_json::Value = serde_json::from_str(&stdout).map_err(|_err| { + format!( + "failed to parse `fastly config-store-entry describe` JSON (parse error \ + redacted; response: {})", + redact_describe_response(&stdout) + ) + })?; let value = parsed .get("item_value") .and_then(serde_json::Value::as_str) .ok_or_else(|| { format!( "`fastly config-store-entry describe` JSON has no string `item_value` field; \ - fastly CLI may have changed its output schema. Raw stdout: {stdout}" + fastly CLI may have changed its output schema. (response: {})", + redact_describe_response(&stdout) ) })?; // Resolve chunk pointers: if `value` is a direct BlobEnvelope it // passes through unchanged; if it is a chunk pointer the chunks // are fetched from the same store and reconstructed. - let resolved = resolve_fastly_config_value(key, value.to_owned(), |chunk_key| { - fetch_remote_config_store_entry(&store_id, chunk_key) - })?; - return Ok(ReadConfigEntry::Present(resolved)); + // + // A chunk describe that fails could not be FULLY read. Confirm whether + // the chunk is genuinely ABSENT against the complete store listing + // (authoritative), never the describe 404: + // - CONFIRMED absent → resolve to a repairable `Corrupt`. The blob + // spec makes persistent chunk loss repairable by re-pushing, so a + // push can overwrite to fix it. + // - present-but-unreadable, or the listing itself failed → + // `fetch_failed`: an incomplete read that must be a HARD error, + // never an overwritable value. + let store_keys: RefCell, String>>> = RefCell::new(None); + let fetch_failed: Cell = Cell::new(false); + let resolved = resolve_fastly_config_value_typed(key, value.to_owned(), |chunk_key| { + match fetch_remote_config_store_entry(&store_id, chunk_key) { + Ok(found) => Ok(Some(found)), + Err(_describe_err) => { + match confirm_key_absent_cached(&store_keys, &store_id, chunk_key) { + Ok(true) => Ok(None), // genuinely gone → repairable Corrupt + Ok(false) => { + fetch_failed.set(true); + Err("a referenced chunk is present in the store but its value \ + could not be read (incomplete read)" + .to_owned()) + } + Err(list_err) => { + fetch_failed.set(true); + Err(list_err) + } + } + } + } + }); + return classify_resolved_read(resolved, value, fetch_failed.get()); } - let stderr = String::from_utf8_lossy(&output.stderr); - let lower = stderr.to_ascii_lowercase(); - if lower.contains("not found") || lower.contains("does not exist") || lower.contains("404") - { + // The describe failed. Absence is CONFIRMED only by a complete listing + // that omits the key -- never by a describe 404, which a proxy/endpoint or + // auth failure produces just the same. A present key (or a listing that + // itself fails) is a hard error, so two such incomplete reads can never + // pass the pre-write recheck and authorise an overwrite. + if confirm_entry_absent(&store_id, key)? { return Ok(ReadConfigEntry::MissingKey); } + let stderr = String::from_utf8_lossy(&output.stderr); Err(format!( - "`fastly config-store-entry describe --store-id={store_id} --key={key} --json` exited with status {}\nstderr: {}", + "`fastly config-store-entry describe --store-id={store_id} --key={key} --json` exited \ + with status {} but the key IS present in the store listing (an operational failure, \ + not absence); nothing was changed.\nstderr: {}", output.status, - stderr.trim() + redact_stderr(&stderr) )) } @@ -590,54 +813,105 @@ impl Adapter for FastlyCliAdapter { }; let fastly_path = manifest_root.join(rel); let name = store.platform.as_str(); + // A prior-state read failure must never BLOCK the command: the diff just + // cannot be computed, so it degrades to `Unsupported` ("cannot diff"). + // Downstream, a dry-run then reaches the writer's orphan-count + // degradation (spec 12.x) and a real push reaches the writer, which + // fails fatally on malformed TOML or overwrites otherwise. Erroring here + // would newly fail a dry-run that reads nothing today. let raw = match fs::read_to_string(&fastly_path) { Ok(text) => text, Err(err) if err.kind() == ErrorKind::NotFound => { return Ok(ReadConfigEntry::MissingStore); } - Err(err) => { - return Err(format!("failed to read {}: {err}", fastly_path.display())); + Err(_err) => { + return Ok(ReadConfigEntry::Unsupported( + "local fastly.toml could not be read; cannot diff the prior value", + )); } }; - let doc: toml_edit::DocumentMut = raw - .parse() - .map_err(|err| format!("failed to parse {}: {err}", fastly_path.display()))?; - // Probe `[local_server.config_stores.]` — if absent, the store - // has not been seeded locally yet. - let Some(contents) = doc - .get("local_server") - .and_then(|ls| ls.get("config_stores")) - .and_then(|cs| cs.get(name)) - .and_then(|store_tbl| store_tbl.get("contents")) - else { - return Ok(ReadConfigEntry::MissingStore); + let Ok(doc) = raw.parse::() else { + return Ok(ReadConfigEntry::Unsupported( + "local fastly.toml is not valid TOML; cannot diff the prior value", + )); + }; + // Descend `[local_server.config_stores..contents]` level by level. + // At each level an ABSENT key means the store isn't seeded yet + // (MissingStore), but a key that is PRESENT yet not a table is malformed + // store state — distinct outcomes. Collapsing the malformed case into + // MissingStore (as a plain `.get().and_then()` chain does) would render an + // inaccurate "all values added" diff, so it degrades to "cannot diff". + // + // `descend` returns Ok(None) for absent (-> MissingStore) and + // Err(Unsupported) for present-but-not-a-table. + let descend = |parent: &'_ toml_edit::Item, + child: &str| + -> Result, ReadConfigEntry> { + match parent.get(child) { + None => Ok(None), + Some(item) if item.is_table_like() => Ok(Some(item.clone())), + Some(_) => Err(ReadConfigEntry::Unsupported( + "a local config-store parent table is not a table; cannot diff the prior value", + )), + } + }; + let root_item = toml_edit::Item::Table(doc.as_table().clone()); + let contents_item = (|| { + let Some(local_server) = descend(&root_item, "local_server")? else { + return Ok(None); + }; + let Some(config_stores) = descend(&local_server, "config_stores")? else { + return Ok(None); + }; + let Some(store_tbl) = descend(&config_stores, name)? else { + return Ok(None); + }; + descend(&store_tbl, "contents") + })(); + let contents = match contents_item { + Ok(Some(item)) => item, + Ok(None) => return Ok(ReadConfigEntry::MissingStore), + Err(unsupported) => return Ok(unsupported), + }; + // `contents` MUST be a table of `key = "value"` pairs. (Guaranteed by + // `descend` above, but re-borrow as a table to index it.) + let Some(contents_tbl) = contents.as_table_like() else { + return Ok(ReadConfigEntry::Unsupported( + "local config-store `contents` is not a table; cannot diff the prior value", + )); }; // The contents table is `key = "value"` pairs. - match contents.get(key) { + match contents_tbl.get(key) { Some(item) => { - let value = item.as_str().ok_or_else(|| { - format!( - "`[local_server.config_stores.{name}.contents].{key}` in {} is not a string", - fastly_path.display() - ) - })?; + let Some(value) = item.as_str() else { + return Ok(ReadConfigEntry::Unsupported( + "the local prior value is not a string; cannot diff the prior value", + )); + }; // Resolve chunk pointers using the same toml contents table. let resolved = - resolve_fastly_config_value(key, value.to_owned(), |chunk_key| match contents - .get(chunk_key) - { - Some(chunk_item) => { - let chunk_val = chunk_item.as_str().ok_or_else(|| { - format!( - "chunk key `{chunk_key}` in {} is not a string", - fastly_path.display() - ) - })?; - Ok(Some(chunk_val.to_owned())) + resolve_fastly_config_value_typed(key, value.to_owned(), |chunk_key| { + match contents_tbl.get(chunk_key) { + Some(chunk_item) => { + let chunk_val = chunk_item.as_str().ok_or_else(|| { + format!( + "chunk key `{chunk_key}` in {} is not a string", + fastly_path.display() + ) + })?; + Ok(Some(chunk_val.to_owned())) + } + None => Ok(None), } - None => Ok(None), - })?; - Ok(ReadConfigEntry::Present(resolved)) + }); + // Same taxonomy as the cloud read, so recovery is uniform across + // targets: a valid envelope is `Present`; a non-envelope or + // corrupt/incomplete value is `Corrupt` (the local writer's + // fail-soft then overwrites it); an unknown/future kind is a hard + // error (do not clobber a newer format). There is no + // infrastructure fetch here -- the chunks are read from the local + // TOML table -- so `fetch_failed` is always false. + classify_resolved_read(resolved, value, false) } None => Ok(ReadConfigEntry::MissingKey), } @@ -653,19 +927,181 @@ impl Adapter for FastlyCliAdapter { } } +impl ManifestLock { + fn acquire(manifest_path: &Path) -> Result { + // Key the lock on the REAL target, so a symlinked manifest and a direct + // path to the same file acquire the SAME lock rather than two different + // sidecars. Every manifest writer (config push AND provision) takes this + // lock, so their read-modify-writes serialise instead of clobbering. + let target = canonical_manifest_target(manifest_path)?; + // A hard-linked manifest cannot be safely replaced: two hard links share + // one inode but have distinct pathnames, so they key DIFFERENT sidecar + // locks (no mutual exclusion), and the atomic rename swaps in a NEW inode, + // breaking the link. We cannot detect the other names, so fail closed + // rather than silently diverge or break the link. + reject_hard_linked_manifest(&target)?; + let dir = target.parent().unwrap_or_else(|| Path::new(".")); + let file_name = target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("fastly.toml"); + let lock_path = dir.join(format!(".{file_name}.edgezero-lock")); + let file = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&lock_path) + .map_err(|err| format!("failed to open lock file {}: {err}", lock_path.display()))?; + // Blocks until any other writer holding the lock releases it. + file.lock() + .map_err(|err| format!("failed to lock {}: {err}", lock_path.display()))?; + Ok(Self { + _file: file, + target, + }) + } + + /// The real file this lock guards. Callers read and replace THIS path. + fn target(&self) -> &Path { + &self.target + } +} + +impl TempFileGuard { + fn disarm(&mut self) { + self.path = None; + } +} + +impl Drop for TempFileGuard { + fn drop(&mut self) { + if let Some(path) = &self.path { + let _cleanup = fs::remove_file(path); + } + } +} + +/// Resolve a manifest path to the REAL file every alias shares, so a symlink and +/// a direct path lock and replace the SAME target. An existing file (or symlink) +/// canonicalizes directly; a not-yet-created file canonicalizes via its parent +/// so a fresh `fastly.toml` still keys on a stable location. +/// +/// FAILS CLOSED on an ambiguous chain: a symlink whose target cannot be read, or +/// a chain too deep / cyclic, returns `Err` rather than falling back to a writable +/// path that could replace an intermediate link. +fn canonical_manifest_target(path: &Path) -> Result { + // Follow the WHOLE symlink chain to the final target -- each hop may itself be + // a dangling symlink (fastly.toml -> middle.toml -> missing.toml). We write at + // the final target, preserving every intermediate link, and a direct writer to + // that same target keys on the same lock. + let mut current = path.to_owned(); + // Bounded to avoid spinning on a symlink cycle (canonicalize would ELOOP). + for _ in 0..40_u32 { + // Fully resolvable => the real existing file. + if let Ok(real) = fs::canonicalize(¤t) { + return Ok(real); + } + // Otherwise, if this hop is a symlink, follow one link and continue. + match fs::symlink_metadata(¤t) { + Ok(meta) if meta.file_type().is_symlink() => match fs::read_link(¤t) { + Ok(link) => { + current = if link.is_absolute() { + link + } else { + // A relative link resolves against the DIRECTORY holding it. + current + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(link) + }; + } + // A symlink we cannot read: refuse rather than guess a target. + Err(err) => { + return Err(format!( + "could not read the manifest symlink `{}` ({err}); refusing to write", + current.display() + )); + } + }, + // Not a symlink -- a plain not-yet-created file, or the final dangling + // target: this is where the write should land. + _ => return Ok(canonicalize_parent_join(¤t)), + } + } + // Exhausted the hop budget: a cyclic or absurdly deep chain. Fail closed. + Err(format!( + "the manifest symlink chain starting at `{}` is too deep or cyclic; refusing to write", + path.display() + )) +} + +/// Canonicalize `path`'s PARENT (which should exist) and rejoin the file name, +/// so a not-yet-created file still resolves to a stable absolute location. +fn canonicalize_parent_join(path: &Path) -> PathBuf { + let parent = match path.parent() { + Some(parent) if !parent.as_os_str().is_empty() => parent, + _ => Path::new("."), + }; + let file_name = path.file_name().unwrap_or(path.as_os_str()); + match fs::canonicalize(parent) { + Ok(real_parent) => real_parent.join(file_name), + Err(_) => path.to_owned(), + } +} + +/// Refuse to operate on a manifest that has MORE THAN ONE hard link. Such a file +/// cannot be replaced safely: the atomic rename installs a new inode (breaking +/// the link), and the path-based lock cannot serialise writers arriving via the +/// other names. Fail closed with a fix. A not-yet-created file, or a filesystem +/// that does not report a link count, is left alone. +/// +/// The link count is read via the platform `MetadataExt` -- `nlink()` on Unix, +/// `number_of_links()` on Windows (both stable, no extra deps) -- so Windows +/// hard-link aliases are caught too, not just Unix ones. On any other target the +/// count is unknown and the file is left alone. +fn reject_hard_linked_manifest(target: &Path) -> Result<(), String> { + #[cfg(unix)] + let link_count: Option = { + use std::os::unix::fs::MetadataExt as _; + fs::metadata(target).ok().map(|meta| meta.nlink()) + }; + #[cfg(windows)] + let link_count: Option = { + use std::os::windows::fs::MetadataExt as _; + fs::metadata(target) + .ok() + .and_then(|meta| meta.number_of_links()) + .map(u64::from) + }; + #[cfg(not(any(unix, windows)))] + let link_count: Option = None; + + if let Some(count) = link_count + && count > 1 + { + return Err(format!( + "{} has multiple hard links (link count {count}); refusing to replace it -- an atomic \ + rename would break the link and concurrent writers via the other names could \ + diverge. Remove the extra hard link(s), or use a symlink instead.", + target.display(), + )); + } + Ok(()) +} + /// Fetch a single entry value from a remote Fastly Config Store entry by /// key, using `fastly config-store-entry describe --store-id= --key= /// --json`. Used by the chunk-pointer resolver to fan out to chunk entries. /// -/// Returns: -/// - `Ok(Some(value))` when the entry exists. -/// - `Ok(None)` when the entry is absent (not-found / 404 / does not exist). -/// - `Err(...)` on adapter or parse errors. +/// `Ok(value)` when the entry exists; `Err` on ANY failure, INCLUDING a +/// not-found. Absence is NOT decided here (a describe 404 is not proof) -- the +/// caller confirms it against the complete store listing. /// /// # Errors /// Returns an error if `fastly` isn't on `PATH`, spawning fails, the JSON -/// cannot be parsed, or the CLI exits with an unexpected non-zero status. -fn fetch_remote_config_store_entry(store_id: &str, key: &str) -> Result, String> { +/// cannot be parsed, or the CLI exits with a non-zero status (not-found included). +fn fetch_remote_config_store_entry(store_id: &str, key: &str) -> Result { let store_arg = format!("--store-id={store_id}"); let key_arg = format!("--key={key}"); let output = Command::new("fastly") @@ -685,11 +1121,12 @@ fn fetch_remote_config_store_entry(store_id: &str, key: &str) -> Result Result Result, String> { + let store_arg = format!("--store-id={store_id}"); + let output = Command::new("fastly") + .args(["config-store-entry", "list", store_arg.as_str(), "--json"]) + .output() + .map_err(|err| { + if err.kind() == ErrorKind::NotFound { + format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") + } else { + format!("failed to spawn `fastly`: {err}") + } + })?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!( + "`fastly config-store-entry list --store-id={store_id} --json` exited with status {}\nstderr: {}", + output.status, + redact_stderr(&stderr) + )); + } + let stdout = strict_stdout(output.stdout, "config-store-entry list --json")?; + let parsed: serde_json::Value = serde_json::from_str(&stdout).map_err(|_err| { + format!( + "failed to parse `fastly config-store-entry list` JSON (parse error redacted; \ + response: {})", + redact_describe_response(&stdout) + ) + })?; + let array = parsed.as_array().ok_or_else(|| { + format!( + "refusing to confirm absence: `fastly config-store-entry list --json` did not return a \ + bare array (response: {}). A paginated or partial view could hide a present key and \ + turn it into a false absence that authorises an overwrite.", + redact_describe_response(&stdout) + ) + })?; + let mut keys = HashSet::with_capacity(array.len()); + for (idx, entry) in array.iter().enumerate() { + let key = entry + .get("item_key") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + format!( + "`fastly config-store-entry list` entry #{idx} is missing a string `item_key`; \ + refusing to confirm absence on an unreadable listing" + ) + })?; + if key.is_empty() { + return Err(format!( + "`fastly config-store-entry list` entry #{idx} has an empty `item_key`; refusing \ + to confirm absence on an unreadable listing" + )); + } + if !keys.insert(key.to_owned()) { + return Err(format!( + "`fastly config-store-entry list` returned duplicate key `{key}`; refusing to \ + confirm absence on an ambiguous listing" + )); + } + } + Ok(keys) +} + +/// Confirm `key` is ABSENT from the store via a complete listing (authoritative). +/// `Ok(true)` = the listing succeeded and omits the key. `Ok(false)` = the key IS +/// present (so a describe failure on it was operational, not absence). `Err` = the +/// listing itself failed. All three fail closed for the caller: only `Ok(true)` +/// is a genuine absence. +fn confirm_entry_absent(store_id: &str, key: &str) -> Result { + Ok(!list_config_store_keys(store_id)?.contains(key)) +} + +/// Cached form of [`confirm_entry_absent`] for chunk fetches: lists the store at +/// most ONCE per read (a whole lost generation would otherwise list per chunk). +fn confirm_key_absent_cached( + cache: &RefCell, String>>>, + store_id: &str, + key: &str, +) -> Result { + let mut slot = cache.borrow_mut(); + if slot.is_none() { + *slot = Some(list_config_store_keys(store_id)); + } + match slot.as_ref() { + Some(Ok(keys)) => Ok(!keys.contains(key)), + Some(Err(err)) => Err(err.clone()), + // Unreachable: populated just above. Fail closed rather than unwrap. + None => Err("internal error: store listing cache was not populated".to_owned()), + } +} + +/// Convert `fastly` stdout to a `String`, FAILING CLOSED on invalid UTF-8 rather +/// than substituting U+FFFD. A lossy replacement inside a JSON string could +/// mutate a stored root value or chunk and yield parseable-but-WRONG data on a +/// path that drives an overwrite or a deletion, violating the exact-read +/// invariant. Diagnostics only ever see redacted output, so stderr stays lossy. +fn strict_stdout(stdout: Vec, command: &str) -> Result { + String::from_utf8(stdout).map_err(|_err| { + format!( + "`fastly {command}` returned non-UTF-8 output; refusing to act on it -- a lossy \ + conversion could mutate a stored value. Nothing was changed." + ) + }) +} + +/// Does `body` parse AND integrity-verify as a `BlobEnvelope`? +/// +/// The typed-config key must hold a valid envelope. A resolved chunk pointer +/// already reconstructs and verifies one; a DIRECT or foreign value is checked +/// here. A value that is not a verifying envelope (invalid JSON, missing fields, +/// or a SHA mismatch) is corrupt FOR THE PUSH -- something to overwrite, not to +/// diff against. +fn body_is_valid_envelope(body: &str) -> bool { + use edgezero_core::blob_envelope::BlobEnvelope; + serde_json::from_str::(body).is_ok_and(|envelope| envelope.verify().is_ok()) +} + +/// Map a `resolve_fastly_config_value` result to a read outcome, distinguishing +/// the cases that must NOT be treated as overwritable corruption: +/// +/// - a FUTURE format (unknown/newer `edgezero_kind`, or a bumped envelope/pointer +/// `version`) → a hard error: overwriting a newer format with this v1 CLI would +/// lose it. Checked FIRST. Detected two ways: on the raw stored value (a direct +/// future envelope, or a future pointer version), AND via a typed +/// [`ResolveFailure::FutureFormat`] from the resolver -- the ONLY signal for a +/// newer INNER envelope reassembled from v1 chunks, which the raw value alone +/// cannot reveal. +/// - a resolve error where a chunk FETCH failed for infrastructure reasons +/// (`fetch_failed`) → a hard error: the read was incomplete, so a push must not +/// overwrite healthy remote state. +/// - `Ok(body)` that verifies as an envelope → `Present`. +/// - `Ok(body)` that is NOT a valid envelope (a malformed direct value, a SHA +/// mismatch, a foreign non-envelope) → `Corrupt` (repairable by overwrite). +/// - any other resolve error (bad/missing chunk, malformed pointer) → `Corrupt`. +fn classify_resolved_read( + resolved: Result, + raw_value: &str, + fetch_failed: bool, +) -> Result { + // A newer format is refused BEFORE anything else: on the raw value (direct + // future envelope or future pointer version) OR when the resolver typed the + // failure as a newer format (a future inner envelope only knowable after the + // chunks are reassembled). Overwriting a newer format with this v1 CLI would + // lose it. + if value_is_future_format(raw_value) + || resolved + .as_ref() + .err() + .is_some_and(ResolveFailure::is_future_format) + { + return Err(FUTURE_FORMAT_READ_ERROR.to_owned()); + } + match resolved { + // An INFRASTRUCTURE fetch failure: the read was incomplete, so a push must + // not overwrite. The resolver's message is already redacted (it names only + // a chunk POSITION, never a value), so surface it for diagnostics. + Err(err) if fetch_failed => Err(format!( + "a chunk fetch failed while reading the remote value ({}); the remote was not fully \ + read, so nothing was changed. Fix connectivity/auth and retry.", + err.into_message() + )), + Ok(body) if body_is_valid_envelope(&body) => Ok(ReadConfigEntry::Present(body)), + Ok(_) => Ok(ReadConfigEntry::Corrupt( + "remote value is not a valid config envelope; a push will overwrite it", + )), + // A confirmed-absent chunk, a hash mismatch, or a malformed pointer: the + // value was fully read and is provably unusable, so a push repairs it. + Err(_) => Ok(ReadConfigEntry::Corrupt( + "remote prior value could not be resolved (corrupt or incomplete chunk state); a push \ + will overwrite it", + )), + } +} + /// Shell out to `fastly -store create --name=`. The /// caller resolves `` from `EDGEZERO__STORES______NAME` /// (falling back to the logical id), so this helper takes whatever the @@ -803,9 +1421,12 @@ fn read_fastly_service_id(path: &Path) -> Result, String> { Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), Err(err) => return Err(format!("failed to read {}: {err}", path.display())), }; - let doc: toml_edit::DocumentMut = raw - .parse() - .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + let doc: toml_edit::DocumentMut = raw.parse().map_err(|_err| { + format!( + "failed to parse {} as TOML (details redacted: the error can quote a stored value)", + path.display() + ) + })?; let svc = doc .get("service_id") .and_then(|item| item.as_str()) @@ -852,9 +1473,12 @@ fn setup_block_present(path: &Path, kind: &str, id: &str) -> Result return Ok(false), Err(err) => return Err(format!("failed to read {}: {err}", path.display())), }; - let doc: toml_edit::DocumentMut = raw - .parse() - .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + let doc: toml_edit::DocumentMut = raw.parse().map_err(|_err| { + format!( + "failed to parse {} as TOML (details redacted: the error can quote a stored value)", + path.display() + ) + })?; let plural = format!("{kind}_stores"); Ok(doc .get("setup") @@ -878,14 +1502,23 @@ fn setup_block_present(path: &Path, kind: &str, id: &str) -> Result Result<(), String> { use toml_edit::{DocumentMut, Item, table}; - let raw = match fs::read_to_string(path) { + // Provision writes the SAME manifest as `config push --local`; take the same + // lock so a concurrent provision and push serialise instead of clobbering + // each other's edit, and operate on the real target the lock resolved. + let lock = ManifestLock::acquire(path)?; + let target = lock.target(); + + let raw = match fs::read_to_string(target) { Ok(text) => text, Err(err) if err.kind() == ErrorKind::NotFound => String::new(), - Err(err) => return Err(format!("failed to read {}: {err}", path.display())), + Err(err) => return Err(format!("failed to read {}: {err}", target.display())), }; - let mut doc: DocumentMut = raw - .parse() - .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + let mut doc: DocumentMut = raw.parse().map_err(|_err| { + format!( + "failed to parse {} as TOML (details redacted: the error can quote a stored value)", + target.display() + ) + })?; let plural = format!("{kind}_stores"); let parent_entry = doc.entry("setup").or_insert_with(table); @@ -908,8 +1541,7 @@ fn append_fastly_setup(path: &Path, kind: &str, id: &str) -> Result<(), String> kind_tbl.insert(id, Item::Table(toml_edit::Table::new())); } - fs::write(path, doc.to_string()) - .map_err(|err| format!("failed to write {}: {err}", path.display()))?; + atomically_replace_file(target, &raw, &doc.to_string())?; Ok(()) } @@ -920,21 +1552,90 @@ fn append_fastly_setup(path: &Path, kind: &str, id: &str) -> Result<(), String> /// values). Idempotent — re-running just rewrites `contents`. Other /// blocks in `fastly.toml` (setup, scripts, the actual `[local_server]` /// secret stores, etc.) are preserved via `toml_edit`. +/// Force `format = "inline-toml"` on a local config-store entry: the inline +/// `contents` this writer emits REQUIRE it. A pre-existing `format = "json"` / +/// `"file"` (pointing at an external file) would otherwise SURVIVE next to the +/// inline contents, leaving Viceroy with a contradictory store the command still +/// reports as written. Returns a warning when it REPLACED an incompatible value. +fn ensure_inline_toml_format( + store_tbl: &mut toml_edit::Table, + platform_name: &str, +) -> Option { + let existing = store_tbl + .get("format") + .and_then(toml_edit::Item::as_str) + .map(str::to_owned); + if existing.as_deref() == Some("inline-toml") { + return None; + } + let warning = existing.filter(|fmt| fmt != "inline-toml").map(|prev| { + format!( + "warning: `local_server.config_stores.{platform_name}` declared `format = \"{prev}\"`, \ + which is incompatible with the inline `contents` written here; it was changed to \ + `format = \"inline-toml\"`" + ) + }); + store_tbl.insert("format", toml_edit::value("inline-toml")); + warning +} + +/// TOCTOU guard for the LOCAL writer: refuse to overwrite a root that now holds a +/// NEWER format, classified HERE under the write lock. The generic push's +/// pre-push future-format check ran BEFORE the lock, so a newer writer could have +/// installed a v2 value in between; without this the old writer would clobber it. +fn reject_future_local_roots( + contents_tbl: &toml_edit::Table, + gc_roots: &[(String, HashSet)], +) -> Result<(), String> { + for (root_key, _) in gc_roots { + if contents_tbl + .get(root_key) + .and_then(toml_edit::Item::as_str) + .is_some_and(value_is_future_format) + { + return Err(format!( + "refusing to overwrite `{root_key}`: the local store now holds a value in a newer \ + format this CLI does not recognise (installed since the pre-push check). Upgrade \ + the CLI rather than clobber a newer format. Nothing was changed." + )); + } + } + Ok(()) +} + fn write_fastly_local_config_store( path: &Path, platform_name: &str, entries: &[(String, String)], -) -> Result<(), String> { + gc_roots: &[(String, HashSet)], +) -> Result, String> { use toml_edit::{DocumentMut, Item, Table, Value, table}; - let raw = match fs::read_to_string(path) { + // Hold a cross-process advisory lock for the WHOLE read-modify-write. Two + // concurrent local pushes would otherwise both read the file, each apply + // their own edit, and the later rename would discard the earlier push's + // change. Serialising here makes each push read what the previous one wrote + // and build on it, so both edits survive. Released when `_lock` drops. + let lock = ManifestLock::acquire(path)?; + // Read and replace the REAL target the lock guards, so a symlinked manifest + // and a direct path never diverge between the read, the compare, and the + // rename. + let target = lock.target(); + + let raw = match fs::read_to_string(target) { Ok(text) => text, Err(err) if err.kind() == ErrorKind::NotFound => String::new(), - Err(err) => return Err(format!("failed to read {}: {err}", path.display())), + Err(err) => return Err(format!("failed to read {}: {err}", target.display())), }; - let mut doc: DocumentMut = raw - .parse() - .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + // Redacted: `toml_edit`'s parse error quotes the offending source LINE, which + // in a config-store `contents` table is a stored (possibly secret-bearing) + // value. The diff read redacts the same failure; the writer must too. + let mut doc: DocumentMut = raw.parse().map_err(|_err| { + format!( + "failed to parse {} as TOML (details redacted: the error can quote a stored value)", + target.display() + ) + })?; let local_server_entry = doc.entry("local_server").or_insert_with(table); let local_server_tbl = local_server_entry.as_table_mut().ok_or_else(|| { @@ -975,11 +1676,7 @@ fn write_fastly_local_config_store( path.display() ) })?; - // Ensure the `format` key is present even on a pre-existing - // entry that omitted it. - if !store_tbl.contains_key("format") { - store_tbl.insert("format", toml_edit::value("inline-toml")); - } + let format_change_warning = ensure_inline_toml_format(store_tbl, platform_name); let contents_entry = store_tbl .entry("contents") .or_insert_with(|| Item::Table(Table::new())); @@ -989,200 +1686,409 @@ fn write_fastly_local_config_store( path.display() ) })?; + reject_future_local_roots(contents_tbl, gc_roots)?; + // Snapshot prior chunk keys per GC root BEFORE the upsert, using the + // exact keep-set the caller computed for each root (no prefix scan). + let mut plans: Vec = Vec::with_capacity(gc_roots.len()); + for (root_key, new_keys) in gc_roots { + let prior_keys = contents_tbl + .get(root_key) + .and_then(toml_edit::Item::as_str) + .map_or_else(|| Ok(Vec::new()), |value| prior_chunk_keys(root_key, value)); + plans.push(FastlyConfigGcPlan { + new_keys: new_keys.clone(), + prior_keys, + }); + } + + // Upsert the new physical entries. for (key, value) in entries { contents_tbl.insert(key, Item::Value(Value::from(value.clone()))); } - fs::write(path, doc.to_string()) - .map_err(|err| format!("failed to write {}: {err}", path.display()))?; - Ok(()) -} - -// ------------------------------------------------------------------- -// `config push` helpers -// ------------------------------------------------------------------- - -/// Shell out to `fastly config-store-entry create --store-id= -/// --key= --value=` for a single entry. Surfaces fastly's -/// stderr verbatim on failure — including the "entry already -/// exists" error, which is the operator's signal to delete the -/// entry (or use `config-store-entry update` manually) before -/// re-running push. -/// Drive a sequential per-entry commit loop and produce the -/// partial-failure diagnostic when the committer fails mid-way. -/// Pure (no I/O) so the diagnostic shape is unit-testable without -/// the fastly CLI on PATH; production calls it with a closure that -/// shells out via `create_config_store_entry`. On success returns -/// the count of committed entries; on failure returns an error -/// string naming committed / failed / not-attempted keys so the -/// operator can resume from a known boundary. -fn push_entries_with_committer( - entries: &[(String, String)], - mut committer: F, -) -> Result -where - F: FnMut(&str, &str) -> Result<(), String>, -{ - let mut pushed: Vec = Vec::with_capacity(entries.len()); - for (key, value) in entries { - if let Err(err) = committer(key, value) { - let remaining: Vec<&str> = entries - .iter() - .skip(pushed.len().saturating_add(1)) - .map(|(remaining_key, _)| remaining_key.as_str()) - .collect(); - return Err(format!( - "fastly push failed at entry `{key}` after committing {committed} of {total} entries; the remaining {remaining_count} entries were NOT pushed.\n Committed (safe to skip on retry): {pushed:?}\n Failed: `{key}` — {err}\n Not attempted (re-push these): {remaining:?}", - committed = pushed.len(), - total = entries.len(), - remaining_count = remaining.len() - )); + // Prune orphans in the same in-memory rewrite; a suspicious prior + // pointer (Err) warns and deletes nothing. + let mut warnings = Vec::new(); + warnings.extend(format_change_warning); + for plan in &plans { + match orphan_chunk_keys(plan) { + Ok(orphans) => { + for key in orphans { + // Never remove an orphan whose VALUE is itself a root, claims + // our namespace, or was written by a NEWER format: a valid + // direct envelope, a pointer, an unknown/future `edgezero_kind`, + // or a bumped envelope/pointer version (all of which the cloud + // GC path also fails closed on). Deleting one would drop live + // or newer-format config. + let value_protected = contents_tbl + .get(&key) + .and_then(toml_edit::Item::as_str) + .is_some_and(|value| { + value_announces_our_kind(value) + || value_is_future_format(value) + || gc_classify_root(&key, value).is_ok() + }); + // Symmetric with cloud GC's fail-closed rule: a key that has + // any canonical chunk nested BENEATH it is a (possibly + // truncated/unreadable) NESTED ROOT. Deleting it would orphan + // that nested generation, so keep it even when its own value + // does not classify. Only a raw leaf PAYLOAD -- announcing no + // kind, not an envelope, and with nothing nested under it -- + // prunes normally. + let has_nested_generation = contents_tbl.iter().any(|(other, _)| { + other != key && chunk_key_generation(&key, other).is_some() + }); + if value_protected || has_nested_generation { + warnings.push(format!( + "warning: kept `{key}` -- it is a runtime-readable root, claims the \ + `edgezero_kind` namespace, or is a nested root with chunks beneath it; \ + not a prunable chunk payload" + )); + continue; + } + contents_tbl.remove(&key); + } + } + Err(err) => warnings.push(format!("warning: {err}")), } - pushed.push(key.clone()); } - Ok(pushed.len()) + + atomically_replace_file(target, &raw, &doc.to_string())?; + Ok(warnings) } -/// Shell `fastly config-store-entry update --upsert --stdin` with -/// the value piped through stdin instead of `--value=` on -/// argv. +/// Replace an already-canonical `target`'s contents ATOMICALLY. Callers pass +/// [`ManifestLock::target`] and hold the lock across the surrounding +/// read-modify-write, so this is not racing another writer; the re-read + compare +/// is a defence-in-depth corruption check, not the concurrency guard. /// -/// Two reasons for this exact invocation: +/// In order: /// -/// 1. `--upsert` (vs. the original `create` subcommand): the prior -/// `create` form errored on any key that already existed in the -/// config store, which made `config push` non-repeatable — -/// after the first push, every follow-up push triggered by a -/// config edit would fail at the first unchanged key. -/// `update --upsert` is documented as "insert or update", which -/// matches the convergent semantic the other config-push paths -/// already have (axum overwrites the JSON, cloudflare's -/// `wrangler kv bulk put` overwrites, spin's -/// `cloud key-value set` overwrites). +/// 1. Re-read `target` and require it to still hold the bytes this rewrite +/// started from (`expected_before`). A mismatch means something OUTSIDE our +/// writers mutated it, so fail rather than overwrite. +/// 2. Create a FRESH temp file in the target's directory with `create_new` +/// (`O_EXCL`): this never follows a file or symlink someone pre-planted at the +/// temp path, and successive names avoid collisions. The rename stays within +/// one directory so it cannot cross a filesystem boundary. +/// 3. Copy the target's existing permissions onto the temp BEFORE writing, so the +/// config bytes are never briefly readable under wider permissions than the +/// manifest allows, then write, then `rename` over the target. `rename` is +/// atomic on POSIX, so a concurrent reader sees either the old file or the new. /// -/// 2. `--stdin` (vs. `--value=`): `--value=` exposed every -/// config entry's bytes in `ps`/`/proc//cmdline` listings -/// AND was bounded by the host's `ARG_MAX` (4 KiB to 256 KiB -/// depending on platform — easy to trip with a JSON blob). -/// `--stdin` reads the value from stdin instead — keeps value -/// bytes out of argv and lifts the size cap to whatever the OS -/// pipe buffer + the CLI's read accept (megabytes in practice). -fn create_config_store_entry(store_id: &str, key: &str, value: &str) -> Result<(), String> { - let store_arg = format!("--store-id={store_id}"); - let key_arg = format!("--key={key}"); - let mut child = Command::new("fastly") - .args([ - "config-store-entry", - "update", - store_arg.as_str(), - key_arg.as_str(), - "--upsert", - "--stdin", - ]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|err| { - if err.kind() == ErrorKind::NotFound { - format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") - } else { - format!("failed to spawn `fastly`: {err}") - } - })?; - // Move stdin OUT of child via `take` so the ChildStdin drops at - // end of scope — that closes the pipe and lets the CLI see EOF. - // `child.wait_with_output()` then consumes child cleanly. - let mut stdin = child - .stdin - .take() - .ok_or_else(|| "failed to open stdin pipe to `fastly`".to_owned())?; - stdin - .write_all(value.as_bytes()) - .map_err(|err| format!("failed to write value to `fastly` stdin: {err}"))?; - drop(stdin); - let output = child - .wait_with_output() - .map_err(|err| format!("failed to wait on `fastly`: {err}"))?; - if output.status.success() { - return Ok(()); +/// A [`TempFileGuard`] removes the temp on any failure after it is created. +fn atomically_replace_file( + target: &Path, + expected_before: &str, + contents: &str, +) -> Result<(), String> { + let current = match fs::read_to_string(target) { + Ok(text) => text, + Err(err) if err.kind() == ErrorKind::NotFound => String::new(), + Err(err) => return Err(format!("failed to re-read {}: {err}", target.display())), + }; + if current != expected_before { + return Err(format!( + "{} changed on disk while this write was preparing its rewrite; nothing was written. \ + Re-run to pick up the other change.", + target.display() + )); } - Err(format!( - "`fastly config-store-entry update --store-id={store_id} --key={key} --upsert --stdin` exited with status {}\nstderr: {}", - output.status, - String::from_utf8_lossy(&output.stderr).trim() - )) -} -/// Parse `fastly config-store list --json` output and return the -/// platform `id` of the store whose `name` matches `name`. Accepts -/// both a bare array (`[ {"id": "...", "name": "..."}, ... ]`) -/// and an `{"items": [...]}` envelope so this stays compatible -/// across fastly CLI versions. -fn find_config_store_id(stdout: &str, name: &str) -> ConfigStoreLookup { - let parsed: serde_json::Value = match serde_json::from_str(stdout) { - Ok(value) => value, - Err(err) => { - return ConfigStoreLookup::SchemaDrift(format!("stdout did not parse as JSON: {err}")); + let dir = target.parent().unwrap_or_else(|| Path::new(".")); + let file_name = target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("fastly.toml"); + // Create a staging file that CANNOT be an attacker's pre-planted symlink: + // `create_new` fails if the path already exists (regular file or symlink), so + // we retry successive names until we own a fresh inode. + let mut attempt = 0_u32; + let (tmp_path, mut tmp_file) = loop { + let candidate = dir.join(format!( + ".{file_name}.edgezero-{}-{attempt}.tmp", + process_id() + )); + match fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&candidate) + { + Ok(file) => break (candidate, file), + Err(err) if err.kind() == ErrorKind::AlreadyExists => { + attempt = attempt.saturating_add(1); + if attempt > 1_024 { + return Err(format!( + "could not create a staging temp file next to {}", + target.display() + )); + } + } + Err(err) => return Err(format!("failed to create staging temp file: {err}")), } }; - let Some(array) = parsed - .as_array() - .or_else(|| parsed.get("items").and_then(serde_json::Value::as_array)) - else { - return ConfigStoreLookup::SchemaDrift(format!( - "expected a bare array `[...]` or an `{{\"items\": [...]}}` envelope; got JSON of shape `{}`", - shape_summary(&parsed) - )); + let mut guard = TempFileGuard { + path: Some(tmp_path.clone()), }; - let mut any_well_formed = false; - for entry in array { - let entry_name = entry.get("name").and_then(serde_json::Value::as_str); - let entry_id = entry.get("id").and_then(serde_json::Value::as_str); - if entry_name.is_some() && entry_id.is_some() { - any_well_formed = true; - } - if entry_name == Some(name) { - return entry_id.map_or_else( - || { - ConfigStoreLookup::SchemaDrift(format!( - "entry matched name `{name}` but is missing a string `id` field" - )) - }, - |id| ConfigStoreLookup::Found(id.to_owned()), - ); + + // Match the target's permissions BEFORE writing any bytes, so config content + // never lands under wider permissions than the manifest already had. A brand + // NEW manifest (NotFound) keeps the create default -- nothing to preserve -- + // but any OTHER metadata error means the target EXISTS yet we cannot read its + // mode, so we must NOT silently widen: fail rather than guess. + match fs::metadata(target) { + Ok(meta) => tmp_file + .set_permissions(meta.permissions()) + .map_err(|err| format!("failed to set permissions on the staging temp file: {err}"))?, + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => { + return Err(format!( + "failed to read the permissions of {} (refusing to widen access): {err}", + target.display() + )); } } - if array.is_empty() || any_well_formed { - ConfigStoreLookup::NotFound - } else { - ConfigStoreLookup::SchemaDrift( - "no entry has both string `name` and `id` fields -- fastly CLI may have changed its output schema" - .to_owned(), - ) + tmp_file + .write_all(contents.as_bytes()) + .map_err(|err| format!("failed to write the staging temp file: {err}"))?; + // Flush to disk BEFORE the rename. A writeback error (ENOSPC/EIO) must surface + // HERE, while the known-good manifest is still untouched -- NOT be swallowed + // so the command "succeeds" after installing content that never reached disk. + // The guard removes the temp on this error. + tmp_file + .sync_all() + .map_err(|err| format!("failed to flush the staging temp file to disk: {err}"))?; + drop(tmp_file); + + fs::rename(&tmp_path, target) + .map_err(|err| format!("failed to replace {}: {err}", target.display()))?; + guard.disarm(); + // Sync the containing directory so the rename entry itself survives a crash. + // Best-effort: opening a directory as a file is not portable (Windows), and + // the critical durability -- the file's contents -- is already flushed above. + if let Ok(dir_handle) = fs::File::open(dir) { + let _dir_sync = dir_handle.sync_all(); } + Ok(()) } -/// One-line type label for a `serde_json::Value` (for diagnostic -/// error messages — not a canonical JSON-schema description). -fn shape_summary(value: &serde_json::Value) -> &'static str { - match value { - serde_json::Value::Null => "null", - serde_json::Value::Bool(_) => "bool", - serde_json::Value::Number(_) => "number", - serde_json::Value::String(_) => "string", - serde_json::Value::Array(_) => "array", - serde_json::Value::Object(_) => "object", +// ------------------------------------------------------------------- +// chunk GC helpers (Stage 7 re-push reclamation) +// ------------------------------------------------------------------- + +/// Expand ONE logical `(root_key, body)` into its physical entries, the +/// exact keep-set for that root, and the value written at the root key. +/// No cross-root prefix scanning (a free-form `--key` can't mislead it). +#[expect( + clippy::type_complexity, + reason = "one-off internal return; a named type would not aid readability" +)] +fn expand_root( + root_key: &str, + body: &str, +) -> Result<(Vec<(String, String)>, HashSet, String), String> { + let expanded = prepare_fastly_config_entries(root_key, body)?; + let new_keys: HashSet = expanded.iter().map(|(key, _)| key.clone()).collect(); + // prepare_* always emits the root entry LAST (root pointer or direct + // value). Make the invariant explicit rather than silently defaulting. + let new_root_value = expanded + .last() + .map(|(_, value)| value.clone()) + .ok_or_else(|| format!("internal: no physical entries produced for root `{root_key}`"))?; + Ok((expanded, new_keys, new_root_value)) +} + +/// Orphans = prior chunk keys not in the new keep-set. Propagates a +/// suspicious-pointer `Err` so the caller can warn and skip GC. +fn orphan_chunk_keys(plan: &FastlyConfigGcPlan) -> Result, String> { + match &plan.prior_keys { + Ok(prior) => Ok(prior + .iter() + .filter(|key| !plan.new_keys.contains(*key)) + .cloned() + .collect()), + Err(err) => Err(err.clone()), } } -/// Resolve the platform config-store id on demand: shell out to -/// `fastly config-store list --json`, parse the JSON, match by -/// `name`. The provision flow doesn't persist this id, so push -/// has to re-fetch every time. -fn resolve_remote_config_store_id(name: &str) -> Result { +/// Reject logical keys that collide with the reserved chunk namespace. +/// `--key` is free-form, so this is enforced at the Fastly adapter +/// boundary: such a key would let a push write into another key's chunk +/// space, and could not be reclaimed correctly. +fn reject_reserved_root_keys(entries: &[(String, String)]) -> Result<(), String> { + for (key, _) in entries { + if key.contains(CHUNK_KEY_INFIX) { + return Err(format!( + "config key `{key}` contains the reserved infix `{CHUNK_KEY_INFIX}`, which collides with Fastly chunk storage; choose a different config key (or --key override)" + )); + } + } + Ok(()) +} + +/// Unix epoch seconds. Push-time only (the `cli` feature is native). +fn unix_now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) +} + +/// Reject a batch that names the same logical root key more than once. +/// +/// The adapter trait takes an entry slice and does not enforce uniqueness, +/// but GC builds one plan per entry and snapshots every plan against the +/// SAME prior generation. With `[(root, A), (root, B)]` the last tuple wins +/// the upsert (root = B), yet A's plan would still reclaim `prior - A_keys` +/// — which includes B's freshly-written chunks — leaving the final pointer +/// referencing missing chunks. Rejecting is safer than silently coalescing: +/// a duplicated key is a caller bug, and picking a winner would hide it. +fn reject_duplicate_root_keys(entries: &[(String, String)]) -> Result<(), String> { + let mut seen: HashSet<&str> = HashSet::with_capacity(entries.len()); + for (key, _) in entries { + if !seen.insert(key.as_str()) { + return Err(format!( + "config key `{key}` appears more than once in a single push; each logical key must be pushed exactly once" + )); + } + } + Ok(()) +} + +/// Best-effort per-root orphan count for `config push --local --dry-run`. +/// Navigate to `[local_server.config_stores..contents]` for the +/// dry-run counter. `Ok(None)` when any level is absent (no prior state); +/// `Err` when a level is present but the wrong type — prior state the real +/// writer would reject, so the count must degrade to "unknown", not 0. +fn local_contents_table<'doc>( + doc: &'doc toml_edit::DocumentMut, + platform_name: &str, +) -> Result, String> { + let malformed = || "could not read prior state".to_owned(); + let Some(server_item) = doc.get("local_server") else { + return Ok(None); + }; + let Some(server) = server_item.as_table() else { + return Err(malformed()); + }; + let Some(stores_item) = server.get("config_stores") else { + return Ok(None); + }; + let Some(stores) = stores_item.as_table() else { + return Err(malformed()); + }; + let Some(store_item) = stores.get(platform_name) else { + return Ok(None); + }; + let Some(store) = store_item.as_table() else { + return Err(malformed()); + }; + let Some(contents_item) = store.get("contents") else { + return Ok(None); + }; + contents_item + .as_table() + .map_or_else(|| Err(malformed()), |table| Ok(Some(table))) +} + +/// Reads the current `fastly.toml` (offline) and, for each logical +/// `(root_key, body)`, counts `prior_chunk_keys(root, old) - new_keys` +/// where `new_keys` is the root's OWN expansion. Never fails the dry-run: +/// on a missing file / no prior pointer / direct prior value it reports +/// `Ok(0)`; on unreadable or malformed prior state it reports `Err(reason)` +/// which the caller renders as an "unknown" line. +fn local_orphan_counts_for_dry_run( + path: &Path, + platform_name: &str, + entries: &[(String, String)], +) -> Vec<(String, Result)> { + use toml_edit::DocumentMut; + + // Parse the current file once (best-effort). Absent file => no prior. + let parsed: Result, String> = match fs::read_to_string(path) { + Ok(text) => text + .parse::() + .map(Some) + .map_err(|_err| "could not read prior state".to_owned()), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(None), + Err(_) => Err("could not read prior state".to_owned()), + }; + + entries + .iter() + .map(|(root_key, body)| { + let new_keys = match expand_root(root_key, body) { + Ok((_, keys, _)) => keys, + Err(err) => return (root_key.clone(), Err(err)), + }; + let count = match &parsed { + Err(reason) => Err(reason.clone()), + Ok(None) => Ok(0), + Ok(Some(doc)) => match local_contents_table(doc, platform_name) { + Err(reason) => Err(reason), + Ok(None) => Ok(0), + Ok(Some(contents)) => match contents.get(root_key) { + None => Ok(0), // no prior value for this root + Some(item) => match item.as_str() { + None => Err("could not read prior state".to_owned()), + Some(raw) => match prior_chunk_keys(root_key, raw) { + Ok(prior) => Ok(prior + .iter() + .filter(|key| !new_keys.contains(*key)) + // Match the real prune exactly, so the count + // equals what would actually be deleted: + .filter(|key| { + let Some(orphan_item) = contents.get(key) else { + // Already ABSENT: the real prune's + // remove() is a no-op, so this is not a + // deletion and must not be counted. + return false; + }; + // KEPT (not counted) if the real prune + // would protect it: its value is a + // runtime-readable root / claims our + // namespace, OR it is a nested root with + // chunks beneath it. Mirror both, so the + // count equals actual deletions. + let value_protected = + orphan_item.as_str().is_some_and(|text| { + value_announces_our_kind(text) + || value_is_future_format(text) + || gc_classify_root(key, text).is_ok() + }); + let has_nested = contents.iter().any(|(other, _)| { + other != key.as_str() + && chunk_key_generation(key, other).is_some() + }); + !(value_protected || has_nested) + }) + .count()), + Err(_) => Err("suspicious prior pointer".to_owned()), + }, + }, + }, + }, + }; + (root_key.clone(), count) + }) + .collect() +} + +// ------------------------------------------------------------------- +// `config push` helpers +// ------------------------------------------------------------------- + +/// Run `fastly config-store-entry list --store-id= --json` and return each +/// item's `item_key`, `item_value`, and `created_at`. +/// +/// The item VALUE is KEPT (not discarded): `config gc` classifies each root by +/// its value (`gc_classify_root`) and reconstructs live generations from the +/// chunk values, so all three fields are required. The value is used internally +/// only and is NEVER echoed into a diagnostic — parse failures redact it via +/// `redact_describe_response`. +fn list_config_store_entries(store_id: &str) -> Result, String> { + let store_arg = format!("--store-id={store_id}"); let output = Command::new("fastly") - .args(["config-store", "list", "--json"]) + .args(["config-store-entry", "list", store_arg.as_str(), "--json"]) .output() .map_err(|err| { if err.kind() == ErrorKind::NotFound { @@ -1192,2145 +2098,6969 @@ fn resolve_remote_config_store_id(name: &str) -> Result { } })?; if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); return Err(format!( - "`fastly config-store list --json` exited with status {}\nstderr: {}", + "`fastly config-store-entry list --store-id={store_id} --json` exited with status {}\nstderr: {}", output.status, - String::from_utf8_lossy(&output.stderr).trim() + redact_stderr(&stderr) )); } - let stdout = String::from_utf8_lossy(&output.stdout); - match find_config_store_id(&stdout, name) { - ConfigStoreLookup::Found(id) => Ok(id), - ConfigStoreLookup::NotFound => Err(format!( - "no fastly config-store matches `{name}` (did you run `edgezero provision --adapter fastly`?)" - )), - ConfigStoreLookup::SchemaDrift(detail) => Err(format!( - "could not parse `fastly config-store list --json` output: {detail}.\n The fastly CLI may have changed its JSON schema in a recent version. Please file a bug report at https://github.com/stackpop/edgezero/issues with the fastly CLI version (`fastly version`) and the raw stdout. Workaround: pin to a known-compatible fastly CLI version." - )), + let stdout = strict_stdout(output.stdout, "config-store-entry list --json")?; + let parsed: serde_json::Value = serde_json::from_str(&stdout).map_err(|_err| { + format!( + "failed to parse `fastly config-store-entry list` JSON (parse error redacted; \ + response: {})", + redact_describe_response(&stdout) + ) + })?; + // A BARE ARRAY ONLY. The installed Fastly CLI returns the complete store as + // a top-level array with no cursor/paging flags. Any other shape (e.g. an + // `{"items":[...], ...}` envelope) may carry pagination metadata we do not + // follow -- and a page that omitted a ROOT while listing its chunks would + // make live chunks look orphaned. The completeness guard cannot see a root + // that isn't there, so we refuse rather than reclaim from a partial view. + let array = parsed.as_array().ok_or_else(|| { + format!( + "refusing to reclaim: `fastly config-store-entry list --json` did not return a bare \ + array (response: {}). This build only supports an unpaginated listing; a partial view \ + could hide a root and orphan its live chunks. Nothing was deleted.", + redact_describe_response(&stdout) + ) + })?; + // FAIL CLOSED on any malformed entry. A missing/non-string field on a + // reclamation input must NEVER be silently skipped or defaulted to empty: + // skipping a root hides the chunks it references (they'd look orphaned and + // get deleted while live), and an empty `item_value` makes a real root + // parse as "references nothing" — same catastrophe. If we can't read the + // listing exactly, we delete nothing. + let mut items = Vec::with_capacity(array.len()); + for (idx, entry) in array.iter().enumerate() { + let field = |name: &str| -> Result { + let raw = entry + .get(name) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + format!( + "`fastly config-store-entry list` entry #{idx} is missing a string `{name}` \ + field; refusing to reclaim on an unreadable listing (nothing deleted)" + ) + })?; + // An EMPTY field is as dangerous as a missing one: an empty root + // value would classify as "references nothing" and orphan its live + // chunks. Reject it here rather than reason about it later. + if raw.is_empty() { + return Err(format!( + "`fastly config-store-entry list` entry #{idx} has an empty `{name}` field; \ + refusing to reclaim on an unreadable listing (nothing deleted)" + )); + } + Ok(raw.to_owned()) + }; + items.push(ConfigStoreItem { + created_at: field("created_at")?, + item_key: field("item_key")?, + item_value: field("item_value")?, + }); } -} - -/// # Errors -/// Returns an error if the Fastly CLI build command fails. -#[inline] -pub fn build(extra_args: &[String]) -> Result { - let manifest = - find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; - let manifest_dir = manifest - .parent() - .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; - let cargo_manifest = manifest_dir.join("Cargo.toml"); - let crate_name = read_package_name(&cargo_manifest)?; - let status = Command::new("cargo") - .args([ - "build", - "--release", - "--target", - "wasm32-wasip1", - "--manifest-path", - cargo_manifest - .to_str() - .ok_or("invalid Cargo manifest path")?, - ]) - .args(extra_args) - .status() - .map_err(|err| format!("failed to run cargo build: {err}"))?; - if !status.success() { - return Err(format!("cargo build failed with status {status}")); + // DUPLICATE KEYS => fail closed. A key must appear once; a store cannot + // really hold two entries under one key, so duplicate rows mean we are not + // reading the store we think we are (a merged/paginated view, or a CLI + // change). Left alone, the last row silently wins for BOTH the live-set + // lookup and `created_at`, so conflicting rows could age a recent key into + // eligibility and schedule the same key for two deletes. + let mut seen: HashSet<&str> = HashSet::with_capacity(items.len()); + if let Some(duplicate) = items + .iter() + .find(|item| !seen.insert(item.item_key.as_str())) + { + return Err(format!( + "refusing to reclaim: `fastly config-store-entry list` returned key `{}` more than \ + once. A key is unique in a config store, so this listing does not describe one \ + consistent view of it (nothing was deleted).", + duplicate.item_key + )); } - let workspace_root = find_workspace_root(manifest_dir); - let artifact = locate_artifact(&workspace_root, manifest_dir, &crate_name)?; - let pkg_dir = workspace_root.join("pkg"); - fs::create_dir_all(&pkg_dir) - .map_err(|err| format!("failed to create {}: {err}", pkg_dir.display()))?; - let dest = pkg_dir.join(format!("{}.wasm", crate_name.replace('-', "_"))); - fs::copy(&artifact, &dest) - .map_err(|err| format!("failed to copy artifact to {}: {err}", dest.display()))?; - - Ok(dest) + Ok(items) } -/// # Errors -/// Returns an error if the Fastly CLI deploy command fails. -#[inline] -pub fn deploy(extra_args: &[String]) -> Result<(), String> { - let manifest = - find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; - let manifest_dir = manifest - .parent() - .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; +/// RFC 3339 (`2026-07-13T03:27:42Z`) -> unix seconds. +fn parse_rfc3339_secs(raw: &str) -> Option { + let stamp = chrono::DateTime::parse_from_rfc3339(raw).ok()?; + u64::try_from(stamp.timestamp()).ok() +} - let status = Command::new("fastly") - .args(["compute", "deploy"]) - .args(extra_args) - .current_dir(manifest_dir) - .status() - .map_err(|err| format!("failed to run fastly CLI: {err}"))?; - if !status.success() { - return Err(format!("fastly compute deploy failed with status {status}")); +/// `config gc` for Fastly: delete chunk entries that no LIVE root pointer +/// references and that are older than the operator's `older_than_secs`. +/// +/// Why this is a separate, operator-invoked command rather than part of `config +/// push`: see `Adapter::gc_config_entries`. The operator's `--older-than` is the +/// safety assertion the platform cannot make. A dry-run prints exactly which +/// keys would go, with ages, so the assertion is reviewable. +/// +/// Fails CLOSED: if the listing is unreadable, or a root's value cannot be +/// classified, nothing is deleted. +fn gc_fastly_config_store( + store_name: &str, + older_than_secs: u64, + dry_run: bool, +) -> Result, String> { + // THE destructive boundary enforces its own precondition. The CLI rejects a + // zero window too, but `gc_config_entries` is a public trait method any + // caller can reach directly -- a safety rule that lives only in the CLI is + // not a safety rule. A zero window asserts nothing: it makes every orphan + // eligible, including one superseded a second ago whose pointer POPs are + // still serving. (A dry-run may preview at zero; it deletes nothing.) + if !dry_run && older_than_secs == 0 { + return Err( + "refusing to reclaim: a destructive `config gc` requires a non-zero `--older-than` \ + window. Zero asserts nothing -- it would make every orphan eligible, including \ + chunks a pointer POPs are still serving. Nothing was deleted." + .to_owned(), + ); } - - Ok(()) -} - -fn find_fastly_manifest(start: &Path) -> Result { - if let Some(found) = find_manifest_upwards(start, "fastly.toml") { - return Ok(found); + let resolved_id = resolve_remote_config_store_id(store_name)? + .ok_or_else(|| no_matching_store_error(store_name))?; + let items = list_config_store_entries(&resolved_id)?; + let plan = plan_gc_reclamation(&items, unix_now_secs(), older_than_secs)?; + let GcPlan { + doomed, + live_count, + retained_recent, + roots, + unprovable, + warnings, + } = plan; + + let doomed_count: usize = doomed.iter().map(Vec::len).sum(); + let mut out = vec![format!( + "fastly config-store `{store_name}` (id={resolved_id}): {} entries, {roots} root(s), {live_count} live chunk(s), {doomed_count} orphan(s) in {} generation(s) older than {older_than_secs}s, {retained_recent} orphan(s) too recent", + items.len(), + doomed.len(), + )]; + out.extend(warnings); + if unprovable > 0 { + // NEVER silent: these entries look like chunk keys but we could not + // prove our writer produced them, so we left them alone. Say so, or the + // summary reads as "everything reclaimable was reclaimed". + out.push(format!( + " {unprovable} chunk-shaped entr(ies) left untouched: they are not byte-identical to what this writer would produce (wrong content-address, a split this writer would not choose, an incomplete generation, or a count it would never emit), so EdgeZero cannot claim them" + )); } - - let root = find_workspace_root(start); - let mut candidates: Vec = WalkDir::new(&root) - .follow_links(true) - .max_depth(8) - .into_iter() - .filter_map(Result::ok) - .map(|entry| entry.path().to_path_buf()) - .filter(|path| { - path.file_name().is_some_and(|n| n == "fastly.toml") - && path - .parent() - .is_some_and(|dir| dir.join("Cargo.toml").exists()) - }) - .collect(); - - if candidates.is_empty() { - return Err("could not locate fastly.toml".to_owned()); + if doomed_count == 0 { + out.push("nothing to reclaim".to_owned()); + return Ok(out); } - - candidates.sort_by_key(|path| { - let parent = path.parent().unwrap_or(Path::new("")); - path_distance(start, parent) - }); - - Ok(candidates.remove(0)) -} - -fn locate_artifact( - workspace_root: &Path, - manifest_dir: &Path, - crate_name: &str, -) -> Result { - let target_triple = "wasm32-wasip1"; - let release_name = format!("{}.wasm", crate_name.replace('-', "_")); - - if let Some(custom) = env::var_os("CARGO_TARGET_DIR") { - let candidate = PathBuf::from(custom) - .join(target_triple) - .join("release") - .join(&release_name); - if candidate.exists() { - return Ok(candidate); + if dry_run { + // A dry-run only PLANS: list every candidate and stop. Nothing is + // attempted, so there is no confirmed/failed/skipped distinction yet. + for (key, age) in doomed.iter().flatten() { + out.push(format!(" would delete `{key}` (age {age}s)")); } + // `--yes` ALWAYS requires an explicit non-zero `--older-than` (a + // destructive run must not guess the window), so the apply instruction + // names both -- "re-run with --yes" alone would be rejected. + out.push(format!( + "dry-run: {doomed_count} orphan chunk(s) planned for deletion; re-run with \ + `--yes --older-than ` (a non-zero window is required) to apply" + )); + return Ok(out); } - - let manifest_target = manifest_dir - .join("target") - .join(target_triple) - .join("release") - .join(&release_name); - if manifest_target.exists() { - return Ok(manifest_target); + // Real run: `doomed_count` is the PLANNED count. Do NOT pre-print each key as + // "deleting" -- execution stops at a generation's first failure, so some + // planned keys are never attempted. `execute_gc_deletes` reports the real + // per-key outcome (deleted / FAILED / skipped) as it happens. + out.push(format!( + "reclaiming {doomed_count} planned orphan chunk(s) across {} generation(s)", + doomed.len() + )); + + let GcDeleteOutcome { + deleted, + failed, + stranded, + uncertain, + } = execute_gc_deletes(&resolved_id, &doomed, &mut out); + out.push(format!( + "reclaimed {deleted} of {doomed_count} orphan chunk entries" + )); + if failed.is_empty() { + return Ok(out); } - - let workspace_target = workspace_root - .join("target") - .join(target_triple) - .join("release") - .join(&release_name); - if workspace_target.exists() { - return Ok(workspace_target); + // Partial/total failure must be a non-zero exit so automation can see it. + let mut diagnostic = format!( + "{}\nconfig gc: {} of {doomed_count} deletes FAILED ({})", + out.join("\n"), + failed.len(), + failed.join(", ") + ); + // A generation whose only failure was on an unconfirmed delete: the outcome + // is UNKNOWN (Fastly may have committed it), so a re-run is worth trying but + // may find a fragment. + if !uncertain.is_empty() { + write!( + diagnostic, + ".\nNOTE: a failed remote delete has an unknown outcome -- Fastly may have applied it \ + before returning an error. Re-run `config gc`: it reclaims each affected generation \ + if it is still whole, or reports it as an unprovable fragment (\"left untouched\") if \ + a delete did commit. If reported as a fragment, remove the survivors by hand:\n{}", + recovery_commands(&resolved_id, &uncertain) + ) + .map_err(|err| format!("failed to format the gc diagnostic: {err}"))?; } - - Err(format!( - "compiled artifact not found (looked in {} and workspace target)", - manifest_dir.display() - )) + // A generation with a CONFIRMED prior delete: definitely a fragment now. + if !stranded.is_empty() { + write!( + diagnostic, + ".\nWARNING: {} entr(ies) are now an INCOMPLETE generation because a sibling was \ + already deleted before the failure: {}. `config gc` proves a generation by \ + reassembling it, so it can no longer prove these and will never reclaim them -- \ + re-running will NOT help. They are inert (no pointer references them). Remove them \ + by hand once you are satisfied they are unreferenced:\n{}", + stranded.len(), + stranded.join(", "), + recovery_commands(&resolved_id, &stranded), + ) + .map_err(|err| format!("failed to format the gc diagnostic: {err}"))?; + } + Err(diagnostic) } -#[inline] -pub fn register() { - register_adapter(&FASTLY_ADAPTER); - register_adapter_blueprint(&FASTLY_BLUEPRINT); +/// Render copy-pasteable `fastly config-store-entry delete` commands, one per +/// key, with EVERY interpolated value single-quoted for POSIX shells. +/// +/// Root keys are free-form (`--key `), and a chunk key preserves its +/// root, so a key can contain `$(...)`, spaces, or `;`. Pasting an unquoted +/// command could execute or misparse it, so this is not cosmetic. +/// +/// The escaping is POSIX/bash (Linux/macOS). A leading note makes that explicit, +/// because Windows `cmd` and PowerShell quote differently — an operator on those +/// shells must adapt the quoting rather than paste verbatim. +fn recovery_commands(store_id: &str, keys: &[String]) -> String { + let commands = keys + .iter() + .map(|key| { + format!( + " fastly config-store-entry delete --store-id={} --key={} --auto-yes", + shell_single_quote(store_id), + shell_single_quote(key), + ) + }) + .collect::>() + .join("\n"); + format!( + " # POSIX/bash (Linux/macOS). On Windows cmd/PowerShell the quoting \ + differs -- adapt it for your shell.\n{commands}" + ) } -#[ctor(unsafe)] -fn register_ctor() { - register(); +/// Single-quote a value for a POSIX shell: wrap in `'...'` and rewrite each +/// embedded `'` as `'\''`. Inside single quotes every other byte -- `$`, spaces, +/// `;`, `$(...)`, backticks -- is literal, so this neutralises any hostile key. +fn shell_single_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) } -/// # Errors -/// Returns an error if the Fastly CLI serve command (Viceroy) fails. -#[inline] -pub fn serve(extra_args: &[String]) -> Result<(), String> { - let manifest = - find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; - let manifest_dir = manifest - .parent() - .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; - - let status = Command::new("fastly") - .args(["compute", "serve"]) - .args(extra_args) - .current_dir(manifest_dir) - .status() - .map_err(|err| format!("failed to run fastly CLI: {err}"))?; - if !status.success() { - return Err(format!("fastly compute serve failed with status {status}")); +/// Delete each doomed generation, stopping a generation at its FIRST failure. +/// +/// A generation is provable only as a whole (`prove_generation` reassembles it), +/// so a half-deleted one can never be proved again: the next run sees a fragment, +/// cannot verify it, and correctly refuses to touch it — forever. Ploughing on +/// after a failure is therefore the one thing that turns a possibly-recoverable +/// error into permanent, unreclaimable litter. +/// +/// A failed remote delete has an UNKNOWN outcome — Fastly may commit it before +/// returning an error — so nothing here is promised as cleanly retryable. The +/// caller distinguishes two cases: a failure with a CONFIRMED prior sibling +/// delete strands the survivors for good (manual recovery), and a failure with +/// no confirmed prior delete leaves the generation in an UNCERTAIN state (a +/// re-run may reclaim it, or surface it as an unprovable fragment). Generations +/// are independent, so a failure in one does not stop the others. +fn execute_gc_deletes( + resolved_id: &str, + doomed: &[Vec<(String, u64)>], + out: &mut Vec, +) -> GcDeleteOutcome { + let mut outcome = GcDeleteOutcome { + deleted: 0, + failed: Vec::new(), + stranded: Vec::new(), + uncertain: Vec::new(), + }; + for generation in doomed { + let mut deleted_here: Vec<&str> = Vec::new(); + for (key, _) in generation { + match delete_config_store_entry(resolved_id, key) { + Ok(()) => { + outcome.deleted = outcome.deleted.saturating_add(1); + deleted_here.push(key.as_str()); + // CONFIRMED gone, per key, as it happens. + out.push(format!(" deleted `{key}`")); + } + Err(err) => { + out.push(format!(" FAILED to delete `{key}` ({err})")); + outcome.failed.push(key.clone()); + // Everything in this generation we have NOT confirmed deleted + // -- the failed key itself, plus the ones we never reached. + let unconfirmed: Vec = generation + .iter() + .map(|(member, _)| member.clone()) + .filter(|member| !deleted_here.contains(&member.as_str())) + .collect(); + // Distinguish the ones we NEVER ATTEMPTED (after the stop) + // from the failed key itself, so the report is not read as + // "all of these were tried and failed". + for skipped in unconfirmed.iter().filter(|member| *member != key) { + out.push(format!( + " skipped `{skipped}` (not attempted: this generation's delete stopped at the failure above)" + )); + } + if deleted_here.is_empty() { + // No sibling is CONFIRMED gone. The failed delete's + // outcome is unknown: if it did not commit, the + // generation is whole and a re-run reclaims it; if it + // did, the re-run finds a fragment and reports it. Either + // way we must not claim clean retryability. + outcome.uncertain.extend(unconfirmed); + } else { + // A sibling is CONFIRMED gone, so this generation is + // definitely a fragment no future run can prove. + outcome.stranded.extend(unconfirmed); + } + break; // stop THIS generation; the others are independent + } + } + } } - - Ok(()) + outcome } -#[cfg(test)] -mod tests { - use super::*; - use edgezero_adapter::cli_support::read_package_name; - #[cfg(unix)] - use edgezero_core::test_env::PathPrepend; - - #[cfg(unix)] - use std::sync::Mutex; - use tempfile::tempdir; - - // Shared fixture names. Pinning these as consts (instead of - // inline `"sessions"` / `"app_config"` per call site) keeps the - // setup-vs-assertion pair in sync -- a typo in one place no - // longer silently divorces from the other, because both reference - // the same const. Also names the intent: these are the LOGICAL - // store ids the fastly adapter operates on, not arbitrary strings. - const TEST_KV_ID: &str = "sessions"; - const TEST_CONFIG_ID: &str = "app_config"; - const TEST_SECRET_ID: &str = "default"; - - #[test] - fn finds_closest_manifest_when_multiple_exist() { - let dir = tempdir().unwrap(); - let root = dir.path(); - fs::write(root.join("Cargo.toml"), "[workspace]").unwrap(); +/// Classify a store's entries: the live chunk set, the protected root keys, and +/// the root count. +/// +/// Root-vs-chunk is decided by VALUE, not key shape. The runtime resolver reads +/// whatever value sits at a key, so ANY entry whose value is a valid direct +/// envelope or a chunk pointer is a runtime-readable root and must never be +/// deleted — even at a chunk-shaped key. Two ways that happens: +/// +/// - a pointer parked at a chunk-shaped key makes its references LIVE; +/// - a value that is itself a valid direct envelope (e.g. a small envelope whose +/// first 7 000-byte chunk is the whole envelope plus trailing whitespace, and +/// so still parses and verifies) is a root in its own right. +/// +/// Only a value that is NEITHER — a raw envelope fragment, which does not parse — +/// is a delete candidate. In normal operation a chunk payload is exactly such a +/// fragment, so this protects the pathological cases at no cost to real GC. +fn classify_store_entries( + items: &[ConfigStoreItem], + value_by_key: &HashMap<&str, &str>, +) -> Result { + let mut live: HashSet = HashSet::new(); + let mut protected: HashSet = HashSet::new(); + let mut roots = 0_usize; + let mut warnings: Vec = Vec::new(); + for item in items { + let is_chunk_shaped = chunk_key_generation_any(&item.item_key).is_some(); + let classified = match gc_classify_root(&item.item_key, &item.item_value) { + Ok(classified) => classified, + // A chunk-shaped key whose value we cannot classify is a genuine + // chunk fragment (a candidate) ONLY if BOTH hold: + // - the value ANNOUNCES no kind. A real chunk payload is a raw + // envelope fragment (no `edgezero_kind`); anything that DOES claim + // our namespace -- a parked pointer, an unknown/future kind -- is + // root-like or suspicious and must fail closed below. + // - NOTHING is nested beneath this key. A truncated/corrupt pointer + // at a chunk-shaped key is ALSO an unparseable fragment, but if it + // is a nested ROOT with its own generation, those nested chunks are + // proven independently and would be deleted while their (unreadable) + // root can no longer name them -- silent loss of a whole nested + // generation. If any canonical chunk of THIS key exists, treat the + // key as an unreadable nested root and FAIL CLOSED. A real leaf + // payload never has nested chunks, so normal GC is unaffected. + Err(_) + if is_chunk_shaped + && !value_announces_our_kind(&item.item_value) + && !value_is_future_format(&item.item_value) + && !items.iter().any(|other| { + other.item_key != item.item_key + && chunk_key_generation(&item.item_key, &other.item_key).is_some() + }) => + { + continue; // a leaf chunk payload: a delete candidate + } + // A definitively FOREIGN entry at an ORDINARY key — a plain string + // like `greeting = "hello"`, a scalar, or a complete JSON object + // without our discriminator. The runtime returns it verbatim and it + // references no chunks, so protect it as a zero-reference root. + // Aborting here would let one ordinary sibling block reclamation of + // every generation in the store. + // + // Three guards keep this from ever masking corruption: + // - the value must be provably inert (NOT a malformed object that + // could be a truncated/corrupt pointer, NOT a value claiming our + // namespace) -- otherwise we might orphan chunks a broken root + // still references; + // - it must NOT be a future format. A direct envelope from a newer + // writer classifies as `Foreign` (no `edgezero_kind`), so without + // this guard it would be waved through as a zero-reference root -- + // yet a newer format may reference chunks under a scheme this build + // cannot read, and GC would plan them for deletion. Fail closed; + // - the KEY must be outside our reserved `.__edgezero_chunks.` + // namespace. A non-canonical key that still lives in that + // namespace is not an ordinary sibling; we cannot say what it is, + // so it fails closed below rather than being waved through. + Err(_) + if value_is_inert_foreign(&item.item_value) + && !value_is_future_format(&item.item_value) + && !item.item_key.contains(CHUNK_KEY_INFIX) => + { + roots = roots.saturating_add(1); + protected.insert(item.item_key.clone()); + continue; + } + Err(err) => { + return Err(format!( + "refusing to reclaim: could not classify root `{}` ({err}); nothing was deleted", + item.item_key + )); + } + }; + // A runtime-readable root, wherever it lives: never a delete candidate. + roots = roots.saturating_add(1); + protected.insert(item.item_key.clone()); + let GcRootValue::Chunked(pointer) = classified else { + continue; // A direct envelope references no chunks. + }; + // The pointer's METADATA is self-consistent by here. That is not proof + // that it honestly describes its generation: a pointer can drop its last + // chunk ref AND restate `envelope_len` as the remaining sum, and every + // metadata check still passes while the dropped chunk silently leaves + // the live set and becomes deletable. So reassemble what it references + // and hold the bytes against its content-address. + let assembled = assemble_pointer_chunks(&item.item_key, &pointer, value_by_key)?; + // The reassembled value may be a NEWER inner format (a bumped envelope + // version, or an unknown `edgezero_kind`) that `BlobEnvelope` deserialize + // silently ignores. Such a format can reference ADDITIONAL generations this + // build cannot see, so trusting only the outer pointer's chunks as the live + // set would let GC delete those as orphans. The runtime resolver rejects + // this case; GC must too. Fail closed. + if value_is_future_format(&assembled) { + return Err(format!( + "refusing to reclaim: root `{}` reconstructs to a value in a newer format this \ + build does not recognise. It may reference generations this build cannot see, so \ + treating its outer chunks as the whole live set could delete live data. Nothing \ + was deleted.", + item.item_key + )); + } + gc_verify_generation(&pointer.envelope_sha256, &assembled).map_err(|err| { + format!( + "refusing to reclaim: root `{}` names a chunk set that does not reconstruct the \ + envelope it claims ({err}). Its chunk list is therefore not a trustworthy live \ + set, and treating it as one could delete a live chunk. Nothing was deleted.", + item.item_key + ) + })?; + // Same exact-split predicate the RUNTIME resolver applies. The content + // checks above only prove the bytes; a pointer whose boundaries are not + // the ones this writer emits reassembles correctly here but is REJECTED + // at runtime -- so GC would otherwise call it a healthy live root while + // the guest 500s on it, and its generation can never satisfy + // `prove_generation` either, making it permanently unreclaimable. + // + // We still protect it (fail-closed: never delete on a judgement we are + // unsure of), but we no longer call it healthy silently -- the operator + // gets told it is unreadable and will not be reclaimed automatically. + if let Err(err) = + verify_writer_split_layout(&item.item_key, &assembled, &chunk_lengths(&pointer.chunks)) + { + warnings.push(format!( + "warning: root `{}` is NOT runtime-readable ({err}). Its chunks are kept, but this \ + generation can never be proven writer-produced, so `config gc` will never reclaim \ + it. Re-run `config push` for this key to rewrite it, then re-run `config gc`.", + item.item_key + )); + } + live.extend(pointer.chunks.into_iter().map(|chunk| chunk.key)); + } + Ok(GcClassification { + live, + protected, + roots, + warnings, + }) +} - let first = root.join("crates/first"); - fs::create_dir_all(&first).unwrap(); - fs::write(first.join("Cargo.toml"), "[package]\nname=\"first\"").unwrap(); - fs::write(first.join("fastly.toml"), "name=\"first\"").unwrap(); +/// The reclamation plan for one store: which orphan chunk entries to delete, and +/// the counts for the summary line. Deriving it is where every safety guard +/// lives, so it is fail-closed throughout — any unreadable/incomplete state +/// returns `Err` and the caller deletes nothing. +/// +/// The organising idea is that **content-addressing makes a chunk set +/// self-proving**: a chunk key embeds the SHA-256 of the whole envelope it +/// belongs to, so reassembling a generation either reproduces the +/// content-address its own keys name, or it does not. Every destructive decision +/// here rests on that hash — never on what the store's metadata claims about +/// itself, which is exactly what an inconsistent store gets wrong. +fn plan_gc_reclamation( + items: &[ConfigStoreItem], + now: u64, + older_than_secs: u64, +) -> Result { + let mut value_by_key: HashMap<&str, &str> = HashMap::with_capacity(items.len()); + let mut created_by_key: HashMap<&str, u64> = HashMap::with_capacity(items.len()); + for item in items { + let Some(created) = parse_rfc3339_secs(&item.created_at) else { + // Unparseable timestamp anywhere in the listing -> fail closed. On a + // DELETE path we will not guess an age. + return Err(format!( + "refusing to reclaim: entry `{}` has an unreadable `created_at`; nothing was deleted", + item.item_key + )); + }; + created_by_key.insert(item.item_key.as_str(), created); + value_by_key.insert(item.item_key.as_str(), item.item_value.as_str()); + } - let second = root.join("examples/second"); - fs::create_dir_all(&second).unwrap(); - fs::write(second.join("Cargo.toml"), "[package]\nname=\"second\"").unwrap(); - fs::write(second.join("fastly.toml"), "name=\"second\"").unwrap(); + // ---- 1. Classify entries: live chunks, protected roots, root count ---- + let GcClassification { + live, + protected, + roots, + warnings, + } = classify_store_entries(items, &value_by_key)?; + + // ---- 2. Per-root live-config age (best-effort; see the guard below) ---- + // rsplit_once (the LAST infix): a chunk of a chunk-shaped root nests the infix + // twice, and its root is everything before the LAST one. Splitting on the + // first would attribute a nested chunk's age to the wrong (outer) root. + let root_live_since: HashMap<&str, u64> = live.iter().fold(HashMap::new(), |mut acc, key| { + if let Some((root, _)) = key.rsplit_once(CHUNK_KEY_INFIX) { + let created = *created_by_key.get(key.as_str()).unwrap_or(&0); + let slot = acc.entry(root).or_insert(0); + *slot = (*slot).max(created); + } + acc + }); - let found = find_fastly_manifest(&second).unwrap(); - assert_eq!(found, second.join("fastly.toml")); + // ---- 3. Candidates, grouped by GENERATION and proven writer-produced ---- + // A per-key decision cannot be safe: an entry is only ours if the whole + // generation it belongs to reassembles to the content-address its keys name. + // So group first, prove second, and delete whole generations or none -- a + // partial delete would leave a corrupt generation behind. + let mut groups: BTreeMap<(&str, String), Vec<&ConfigStoreItem>> = BTreeMap::new(); + for item in items { + if live.contains(&item.item_key) { + continue; + } + // A key whose own value is a runtime-readable root is never a candidate, + // even when its key is chunk-shaped (a valid direct envelope can sit at + // one). Excluding it here also means any real chunk sharing that + // generation drops to an incomplete group, which prove_generation then + // leaves untouched — safe: we leak rather than delete a possible root. + if protected.contains(&item.item_key) { + continue; + } + // rsplit_once (the LAST infix): the same nested-chunk correctness the + // live-set scan and classification use — a chunk of a chunk-shaped root + // is grouped under THAT root, not the outer one, so nested orphans are + // grouped (and thus reclaimed or reported), not silently dropped. + let Some((root, _)) = item.item_key.rsplit_once(CHUNK_KEY_INFIX) else { + continue; // a root + }; + let Some(generation) = chunk_key_generation(root, &item.item_key) else { + continue; // chunk-shaped but NOT canonical => never a key we emit + }; + groups.entry((root, generation)).or_default().push(item); } - #[test] - fn finds_manifest_in_current_directory() { - let dir = tempdir().unwrap(); - let root = dir.path(); - fs::write(root.join("Cargo.toml"), "[workspace]").unwrap(); - fs::write(root.join("fastly.toml"), "name = \"demo\"").unwrap(); + let mut doomed: Vec> = Vec::new(); + let mut retained_recent = 0_usize; + let mut unprovable = 0_usize; + for ((root, generation), group) in groups { + if prove_generation(root, &generation, &group).is_err() { + // We cannot prove we wrote this, so we do not touch it. It may be an + // ordinary entry that merely LOOKS like a chunk key (a store can + // predate this feature or be shared, and push-time reserved-key + // rejection cannot protect what already exists), or a half-written + // generation. Skipped rather than fatal: one foreign entry must not + // block reclamation of the store forever. Reported in the summary. + unprovable = unprovable.saturating_add(group.len()); + continue; + } - let manifest = find_fastly_manifest(root).expect("should find manifest"); - assert_eq!(manifest, root.join("fastly.toml")); + // Age the generation as a UNIT, by its youngest member: deleting a + // generation is one decision, so its most restrictive age governs. + let group_age = group + .iter() + .map(|item| { + now.saturating_sub(*created_by_key.get(item.item_key.as_str()).unwrap_or(&0)) + }) + .min() + .unwrap_or(0); + // BOTH ages must clear the operator's window; neither substitutes for + // the other, so take the more restrictive (the MINIMUM). + // + // - The chunks' OWN age is mandatory: a generation written seconds ago + // is inside the propagation window whatever its root looks like (e.g. + // a concurrent push wrote it and has not committed its pointer yet), + // so an old-looking root must never license deleting it. + // - The root's live-config age (when known) is an EXTRA restriction: it + // catches an old generation superseded recently, which its own age + // cannot see. + let effective_age = root_live_since.get(root).map_or(group_age, |live_since| { + group_age.min(now.saturating_sub(*live_since)) + }); + if effective_age < older_than_secs { + retained_recent = retained_recent.saturating_add(group.len()); + continue; + } + doomed.push( + group + .iter() + .map(|item| { + let age = now + .saturating_sub(*created_by_key.get(item.item_key.as_str()).unwrap_or(&0)); + (item.item_key.clone(), age) + }) + .collect(), + ); } - #[test] - fn locate_artifact_considers_workspace_target() { - let dir = tempdir().unwrap(); - let workspace = dir.path(); - let manifest_dir = workspace.join("service"); - fs::create_dir_all(manifest_dir.join("target/wasm32-wasip1/release")).unwrap(); - let artifact = workspace.join("target/wasm32-wasip1/release/demo.wasm"); - fs::create_dir_all(artifact.parent().unwrap()).unwrap(); - fs::write(&artifact, "wasm").unwrap(); + Ok(GcPlan { + doomed, + live_count: live.len(), + retained_recent, + roots, + unprovable, + warnings, + }) +} - let located = locate_artifact(workspace, &manifest_dir, "demo").unwrap(); - assert_eq!(located, artifact); +/// Reassemble the chunks a live pointer references, in index order, checking each +/// against the pointer's own per-chunk `len`/`sha256` along the way. +/// +/// Fails closed when a referenced key is absent from the listing. This subsumes +/// the old standalone completeness guard: an incomplete or paginated listing +/// cannot produce the bytes, so it can never reach a passing verification. +fn assemble_pointer_chunks( + root_key: &str, + pointer: &GcPointer, + value_by_key: &HashMap<&str, &str>, +) -> Result { + // NOT `with_capacity(pointer.envelope_len)`: that length is untrusted stored + // metadata. `validate_pointer_chunks` bounds it, but this is a destructive + // path -- do not reserve from a number the store supplied when growing from + // the bytes we actually read costs nothing. + let mut assembled = String::new(); + // The chunk KEY is pointer-controlled (a malformed pointer can carry any + // string there), so diagnostics name a POSITION, not the key. `root_key` is + // the operator's own logical entry key and is named for context, as the rest + // of the GC diagnostics do. + for (position, chunk) in pointer.chunks.iter().enumerate() { + let Some(value) = value_by_key.get(chunk.key.as_str()) else { + return Err(format!( + "refusing to reclaim: root `{root_key}` references chunk {position}, which is \ + absent from the store listing (the listing may be incomplete/paginated, or the \ + store is already inconsistent); nothing was deleted" + )); + }; + if value.len() != chunk.len { + return Err(format!( + "refusing to reclaim: root `{root_key}` says chunk {position} is {} bytes but the \ + store holds {}; nothing was deleted", + chunk.len, + value.len() + )); + } + if sha256_hex(value.as_bytes()) != chunk.sha256 { + return Err(format!( + "refusing to reclaim: the stored value of chunk {position} does not match the \ + SHA-256 that root `{root_key}` records for it; nothing was deleted" + )); + } + assembled.push_str(value); } - - #[test] - fn read_package_falls_back_to_name() { - let dir = tempdir().unwrap(); - let manifest = dir.path().join("Cargo.toml"); - fs::write(&manifest, "name = \"demo\"").unwrap(); - let name = read_package_name(&manifest).unwrap(); - assert_eq!(name, "demo"); + if assembled.len() != pointer.envelope_len { + return Err(format!( + "refusing to reclaim: root `{root_key}` declares an envelope of {} bytes but its \ + chunks reassemble to {}; nothing was deleted", + pointer.envelope_len, + assembled.len() + )); } + Ok(assembled) +} - #[test] - fn read_package_prefers_package_table() { - let dir = tempdir().unwrap(); - let manifest = dir.path().join("Cargo.toml"); - fs::write(&manifest, "[package]\nname = \"demo\"\n").unwrap(); - let name = read_package_name(&manifest).unwrap(); - assert_eq!(name, "demo"); +/// Is this candidate generation byte-identical to what THIS writer would have +/// produced for the bytes it contains? +/// +/// The gate on every delete. `group` is every listed entry sharing one +/// `(root, generation)`. +/// +/// **What this proves, precisely.** We reassemble the group in index order and +/// re-run `prepare_fastly_config_entries` over the result. If the writer, given +/// those exact bytes, would emit exactly these keys and these values, the entries +/// are indistinguishable from our own output: same direct-vs-chunked threshold, +/// same UTF-8-safe 7 000-byte boundaries, same content-addressed keys, same +/// count. A lone chunk fails automatically (an envelope small enough to store +/// directly round-trips to a single ROOT-keyed entry, and a large one to >= 2 +/// chunks), as does any set split at boundaries we would not choose. +/// +/// **What this does NOT prove: authorship.** Content-addressing is not a +/// signature. A foreign writer can pick envelope E, compute `H = sha256(E)`, +/// split E exactly as we would, and store the parts under our reserved +/// `.__edgezero_chunks.` namespace; that group is byte-identical to ours and we +/// will reclaim it. No preimage attack is needed, and no check over the stored +/// bytes alone can separate the two — telling them apart needs trusted +/// generation metadata or an authenticated marker, and the store offers neither +/// (any writer with store access could forge either). +/// +/// We accept that residual: the namespace is reserved by convention, push-time +/// validation rejects logical keys inside it, and anything passing this gate is +/// a faithful reproduction of our format. The spec documents it as a limitation +/// rather than claiming a guarantee we cannot make. +fn prove_generation( + root: &str, + generation: &str, + group: &[&ConfigStoreItem], +) -> Result<(), String> { + let mut ordered: Vec<(usize, &str)> = Vec::with_capacity(group.len()); + for item in group { + let index = item + .item_key + .rsplit_once('.') + .and_then(|(_, index)| index.parse::().ok()) + .ok_or_else(|| format!("`{}` has no readable index", item.item_key))?; + ordered.push((index, item.item_value.as_str())); } - - // ---------- push_entries_with_committer ---------- - - #[test] - fn push_entries_with_committer_returns_count_when_all_succeed() { + ordered.sort_by_key(|&(index, _)| index); + for (position, &(index, _)) in ordered.iter().enumerate() { + if index != position { + return Err(format!( + "indexes are not dense 0..n-1 (found {index} at position {position})" + )); + } + } + let assembled: String = ordered.iter().map(|&(_, value)| value).collect(); + + // 1. The bytes must be the generation the keys name, and a real envelope. + gc_verify_generation(generation, &assembled)?; + + // 2. ...and the writer, given those bytes, must produce EXACTLY these + // entries. This is what pins the split boundaries and the chunked-vs- + // direct threshold, so a set assembled by anything that does not + // reproduce our writer's output byte-for-byte is left alone. + let expected = prepare_fastly_config_entries(root, &assembled) + .map_err(|err| format!("this writer could not re-derive the generation ({err})"))?; + let Some(expected_chunks) = expected.get(..expected.len().saturating_sub(1)) else { + return Err("this writer produced no chunk entries for these bytes".to_owned()); + }; + if expected_chunks.is_empty() { + // The envelope fits directly, so the writer would never have chunked it: + // whatever these entries are, they are not ours. + return Err( + "these bytes fit the entry limit, so this writer would have stored them directly \ + rather than in chunks" + .to_owned(), + ); + } + if expected_chunks.len() != ordered.len() { + return Err(format!( + "this writer would split these bytes into {} chunk(s), not {}", + expected_chunks.len(), + ordered.len() + )); + } + for ((expected_key, expected_value), item) in + expected_chunks.iter().zip(group_in_index_order(group)) + { + if *expected_key != item.item_key { + return Err(format!( + "this writer would not have produced the key `{}`", + item.item_key + )); + } + if *expected_value != item.item_value { + return Err(format!( + "the stored value of `{}` is not the chunk this writer would have written at that \ + index", + item.item_key + )); + } + } + Ok(()) +} + +/// `group` sorted by chunk index, so it lines up with the writer's output order. +fn group_in_index_order<'item>(group: &[&'item ConfigStoreItem]) -> Vec<&'item ConfigStoreItem> { + let mut ordered: Vec<&ConfigStoreItem> = group.to_vec(); + ordered.sort_by_key(|item| { + item.item_key + .rsplit_once('.') + .and_then(|(_, index)| index.parse::().ok()) + .unwrap_or(usize::MAX) + }); + ordered +} + +/// Is this key a chunk key of ANY root? (`config gc` scans the whole store, so +/// it cannot scope to one root up front.) Validates the canonical shape. +fn chunk_key_generation_any(key: &str) -> Option { + // Split on the LAST infix, not the first: a chunk of a root that ITSELF + // contains the infix (a pointer parked at a chunk-shaped key with self-scoped + // chunks) has the infix twice, and its chunk suffix is after the LAST one. + // Splitting on the first would misread the doubly-nested chunk as a + // non-chunk, get it classified as an unclassifiable root, and abort the whole + // store's GC. For an ordinary single-infix key the root has no infix, so the + // last infix IS the first — this only changes the nested case. + let (root, _rest) = key.rsplit_once(CHUNK_KEY_INFIX)?; + chunk_key_generation(root, key) +} + +/// Drive a sequential per-entry commit loop and produce the +/// partial-failure diagnostic when the committer fails mid-way. +/// Pure (no I/O) so the diagnostic shape is unit-testable without +/// the fastly CLI on PATH; production calls it with a closure that +/// shells out via `create_config_store_entry`. On success returns +/// the count of committed entries; on failure returns an error +/// string. The FAILED entry's outcome is UNKNOWN — Fastly may have +/// committed it before returning the error — so the message does not +/// claim a clean boundary; it directs the operator to re-run the whole +/// idempotent push rather than hand-resume from a supposed cut point. +fn push_entries_with_committer( + entries: &[(String, String)], + mut committer: F, +) -> Result +where + F: FnMut(&str, &str) -> Result<(), String>, +{ + let mut pushed: Vec = Vec::with_capacity(entries.len()); + for (key, value) in entries { + if let Err(err) = committer(key, value) { + let remaining: Vec<&str> = entries + .iter() + .skip(pushed.len().saturating_add(1)) + .map(|(remaining_key, _)| remaining_key.as_str()) + .collect(); + return Err(format!( + "fastly push failed at entry `{key}` while committing {committed} of {total} entries.\n \ + The failed entry's outcome is UNKNOWN: Fastly may have committed it before the error \ + (a timeout can arrive after the write lands), including when it is the root pointer.\n \ + Recovery: re-run the SAME `config push`. It is idempotent -- chunk keys are content-addressed \ + and writes use `--upsert` -- so entries already written are rewritten harmlessly and any \ + missing ones are filled. Do NOT hand-delete the failed key.\n \ + Already written (a retry rewrites them): {pushed:?}\n \ + Failed: `{key}` (outcome unknown) -- {err}\n \ + Not attempted: {remaining:?}", + committed = pushed.len(), + total = entries.len(), + )); + } + pushed.push(key.clone()); + } + Ok(pushed.len()) +} + +/// Shell `fastly config-store-entry update --upsert --stdin` with +/// the value piped through stdin instead of `--value=` on +/// argv. +/// +/// Two reasons for this exact invocation: +/// +/// 1. `--upsert` (vs. the original `create` subcommand): the prior +/// `create` form errored on any key that already existed in the +/// config store, which made `config push` non-repeatable — +/// after the first push, every follow-up push triggered by a +/// config edit would fail at the first unchanged key. +/// `update --upsert` is documented as "insert or update", which +/// matches the convergent semantic the other config-push paths +/// already have (axum overwrites the JSON, cloudflare's +/// `wrangler kv bulk put` overwrites, spin's +/// `cloud key-value set` overwrites). +/// +/// 2. `--stdin` (vs. `--value=`): `--value=` exposed every +/// config entry's bytes in `ps`/`/proc//cmdline` listings +/// AND was bounded by the host's `ARG_MAX` (4 KiB to 256 KiB +/// depending on platform — easy to trip with a JSON blob). +/// `--stdin` reads the value from stdin instead — keeps value +/// bytes out of argv and lifts the size cap to whatever the OS +/// pipe buffer + the CLI's read accept (megabytes in practice). +fn create_config_store_entry(store_id: &str, key: &str, value: &str) -> Result<(), String> { + let store_arg = format!("--store-id={store_id}"); + let key_arg = format!("--key={key}"); + let mut child = Command::new("fastly") + .args([ + "config-store-entry", + "update", + store_arg.as_str(), + key_arg.as_str(), + "--upsert", + "--stdin", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|err| { + if err.kind() == ErrorKind::NotFound { + format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") + } else { + format!("failed to spawn `fastly`: {err}") + } + })?; + // Take stdin OUT of the child and hand it to a helper that writes the value + // and drops the handle on return — closing the pipe so the CLI sees EOF. + // Dropping on scope-exit rather than via an explicit `drop()` keeps this + // valid on targets where `ChildStdin` is a non-Drop stub. + // `child.wait_with_output()` then consumes child cleanly. + let stdin = child + .stdin + .take() + .ok_or_else(|| "failed to open stdin pipe to `fastly`".to_owned())?; + write_value_to_fastly_stdin(stdin, value)?; + let output = child + .wait_with_output() + .map_err(|err| format!("failed to wait on `fastly`: {err}"))?; + if output.status.success() { + return Ok(()); + } + Err(format!( + "`fastly config-store-entry update --store-id={store_id} --key={key} --upsert --stdin` exited with status {}\nstderr: {}", + output.status, + redact_stderr(&String::from_utf8_lossy(&output.stderr)) + )) +} + +/// Write `value` to the child's stdin, then drop the handle as it falls out of +/// scope on return — closing the pipe so the `fastly` CLI sees EOF. Taking +/// `stdin` by value gives a natural scope-end drop rather than an explicit +/// `drop()`, which also keeps this valid on targets where `ChildStdin` is a +/// non-Drop stub. +fn write_value_to_fastly_stdin(mut stdin: ChildStdin, value: &str) -> Result<(), String> { + stdin + .write_all(value.as_bytes()) + .map_err(|err| format!("failed to write value to `fastly` stdin: {err}")) +} + +fn delete_config_store_entry(store_id: &str, key: &str) -> Result<(), String> { + let store_arg = format!("--store-id={store_id}"); + let key_arg = format!("--key={key}"); + let output = Command::new("fastly") + .args([ + "config-store-entry", + "delete", + store_arg.as_str(), + key_arg.as_str(), + "--auto-yes", + ]) + .output() + .map_err(|err| { + if err.kind() == ErrorKind::NotFound { + format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") + } else { + format!("failed to spawn `fastly`: {err}") + } + })?; + if output.status.success() { + return Ok(()); + } + // EVERY non-zero delete is a failure -- no "already gone" special case. + // Pattern-matching stderr for "not found"/"404" cannot reliably tell "this + // key is already gone" from "the store does not exist", an auth failure, or + // a 500: messages like `config store abc does not exist while deleting key + // ` name the key AND say "does not exist". Reporting those as a + // successful reclamation is strictly worse than a retry, and a retry is + // free: `config gc` re-lists the store, so a key that really is gone simply + // will not appear as a candidate next run. + // Redact stderr: a Fastly error can quote the entry value back, which on the + // delete path would put a stored config value into CI logs. + let stderr = String::from_utf8_lossy(&output.stderr); + Err(format!( + "`fastly config-store-entry delete --store-id={store_id} --key={key} --auto-yes` exited with status {}\n{}", + output.status, + redact_stderr(&stderr) + )) +} + +/// Parse `fastly config-store list --json` output and return the +/// platform `id` of the store whose `name` matches `name`. Accepts +/// both a bare array (`[ {"id": "...", "name": "..."}, ... ]`) +/// and an `{"items": [...]}` envelope so this stays compatible +/// across fastly CLI versions. +fn find_config_store_id(stdout: &str, name: &str) -> ConfigStoreLookup { + let parsed: serde_json::Value = match serde_json::from_str(stdout) { + Ok(value) => value, + Err(err) => { + return ConfigStoreLookup::SchemaDrift(format!("stdout did not parse as JSON: {err}")); + } + }; + let Some(array) = parsed + .as_array() + .or_else(|| parsed.get("items").and_then(serde_json::Value::as_array)) + else { + return ConfigStoreLookup::SchemaDrift(format!( + "expected a bare array `[...]` or an `{{\"items\": [...]}}` envelope; got JSON of shape `{}`", + shape_summary(&parsed) + )); + }; + let mut any_well_formed = false; + for entry in array { + let entry_name = entry.get("name").and_then(serde_json::Value::as_str); + let entry_id = entry.get("id").and_then(serde_json::Value::as_str); + if entry_name.is_some() && entry_id.is_some() { + any_well_formed = true; + } + if entry_name == Some(name) { + return entry_id.map_or_else( + || { + ConfigStoreLookup::SchemaDrift(format!( + "entry matched name `{name}` but is missing a string `id` field" + )) + }, + |id| ConfigStoreLookup::Found(id.to_owned()), + ); + } + } + if array.is_empty() || any_well_formed { + ConfigStoreLookup::NotFound + } else { + ConfigStoreLookup::SchemaDrift( + "no entry has both string `name` and `id` fields -- fastly CLI may have changed its output schema" + .to_owned(), + ) + } +} + +/// Summarise a `fastly ... describe` response for diagnostics WITHOUT +/// leaking its contents. +/// +/// The response body is the stored config value. App config may hold +/// credentials, internal endpoints, or security policy, and this adapter +/// performs no secret stripping — while CLI status lines are logged +/// verbatim and CI logs are commonly retained and shared. So a schema-drift +/// diagnostic must never echo the payload: report only its size and its +/// top-level *shape* (field names for an object, type otherwise), never a +/// value. +fn redact_describe_response(stdout: &str) -> String { + let len = stdout.len(); + serde_json::from_str::(stdout).map_or_else( + |_err| format!("{len} bytes, not valid JSON"), + |value| match value { + serde_json::Value::Object(map) => { + // Object KEYS are stored/provider-controlled data (a wrong-shape + // response could be `{"": ...}`), so only the COUNT is + // reported, never the key names. + format!("{len} bytes, JSON object with {} field(s)", map.len()) + } + other @ (serde_json::Value::Null + | serde_json::Value::Bool(_) + | serde_json::Value::Number(_) + | serde_json::Value::String(_) + | serde_json::Value::Array(_)) => { + format!("{len} bytes, JSON {}", shape_summary(&other)) + } + }, + ) +} + +/// Summarise a failing `fastly` invocation's stderr WITHOUT echoing it. +/// +/// The `describe` and `update --stdin` paths carry the stored config value, so +/// a Fastly error that quotes the payload back would put credentials straight +/// into CI logs — the same exposure as the stdout leak, via the failure branch. +/// Not-found *classification* still inspects stderr internally; only the +/// user-facing string is redacted. +fn redact_stderr(stderr: &str) -> String { + let len = stderr.trim().len(); + format!( + "{len} bytes suppressed (may echo the stored config value); re-run the `fastly` command directly to inspect it" + ) +} + +/// One-line type label for a `serde_json::Value` (for diagnostic +/// error messages — not a canonical JSON-schema description). +fn shape_summary(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "bool", + serde_json::Value::Number(_) => "number", + serde_json::Value::String(_) => "string", + serde_json::Value::Array(_) => "array", + serde_json::Value::Object(_) => "object", + } +} + +/// Resolve the platform config-store id on demand: shell out to +/// `fastly config-store list --json`, parse the JSON, match by +/// `name`. The provision flow doesn't persist this id, so push +/// has to re-fetch every time. +/// +/// Returns a TYPED absence: `Ok(None)` ONLY when the list call SUCCEEDS and no +/// store matches (a genuine absence). An operational failure (missing binary, +/// spawn/list failure, schema drift) stays `Err` -- callers that read for a diff +/// must not treat an operational failure as "store absent" and overwrite. +fn resolve_remote_config_store_id(name: &str) -> Result, String> { + let output = Command::new("fastly") + .args(["config-store", "list", "--json"]) + .output() + .map_err(|err| { + if err.kind() == ErrorKind::NotFound { + format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") + } else { + format!("failed to spawn `fastly`: {err}") + } + })?; + if !output.status.success() { + return Err(format!( + "`fastly config-store list --json` exited with status {}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + let stdout = strict_stdout(output.stdout, "config-store list --json")?; + match find_config_store_id(&stdout, name) { + ConfigStoreLookup::Found(id) => Ok(Some(id)), + ConfigStoreLookup::NotFound => Ok(None), + ConfigStoreLookup::SchemaDrift(detail) => Err(format!( + "could not parse `fastly config-store list --json` output: {detail}.\n The fastly CLI may have changed its JSON schema in a recent version. Please file a bug report at https://github.com/stackpop/edgezero/issues with the fastly CLI version (`fastly version`) and the raw stdout. Workaround: pin to a known-compatible fastly CLI version." + )), + } +} + +/// Message for a genuinely-absent store, for the write/GC callers that treat +/// absence as a hard error (they cannot operate on a store that does not exist). +fn no_matching_store_error(name: &str) -> String { + format!( + "no fastly config-store matches `{name}` (did you run `edgezero provision --adapter fastly`?)" + ) +} + +/// # Errors +/// Returns an error if the Fastly CLI build command fails. +#[inline] +pub fn build(extra_args: &[String]) -> Result { + let manifest = + find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; + let manifest_dir = manifest + .parent() + .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; + let cargo_manifest = manifest_dir.join("Cargo.toml"); + let crate_name = read_package_name(&cargo_manifest)?; + + let status = Command::new("cargo") + .args([ + "build", + "--release", + "--target", + "wasm32-wasip1", + "--manifest-path", + cargo_manifest + .to_str() + .ok_or("invalid Cargo manifest path")?, + ]) + .args(extra_args) + .status() + .map_err(|err| format!("failed to run cargo build: {err}"))?; + if !status.success() { + return Err(format!("cargo build failed with status {status}")); + } + + let workspace_root = find_workspace_root(manifest_dir); + let artifact = locate_artifact(&workspace_root, manifest_dir, &crate_name)?; + let pkg_dir = workspace_root.join("pkg"); + fs::create_dir_all(&pkg_dir) + .map_err(|err| format!("failed to create {}: {err}", pkg_dir.display()))?; + let dest = pkg_dir.join(format!("{}.wasm", crate_name.replace('-', "_"))); + fs::copy(&artifact, &dest) + .map_err(|err| format!("failed to copy artifact to {}: {err}", dest.display()))?; + + Ok(dest) +} + +/// # Errors +/// Returns an error if the Fastly CLI deploy command fails. +#[inline] +pub fn deploy(extra_args: &[String]) -> Result<(), String> { + let manifest = + find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; + let manifest_dir = manifest + .parent() + .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; + + let status = Command::new("fastly") + .args(["compute", "deploy"]) + .args(extra_args) + .current_dir(manifest_dir) + .status() + .map_err(|err| format!("failed to run fastly CLI: {err}"))?; + if !status.success() { + return Err(format!("fastly compute deploy failed with status {status}")); + } + + Ok(()) +} + +fn find_fastly_manifest(start: &Path) -> Result { + if let Some(found) = find_manifest_upwards(start, "fastly.toml") { + return Ok(found); + } + + let root = find_workspace_root(start); + let mut candidates: Vec = WalkDir::new(&root) + .follow_links(true) + .max_depth(8) + .into_iter() + .filter_map(Result::ok) + .map(|entry| entry.path().to_path_buf()) + .filter(|path| { + path.file_name().is_some_and(|n| n == "fastly.toml") + && path + .parent() + .is_some_and(|dir| dir.join("Cargo.toml").exists()) + }) + .collect(); + + if candidates.is_empty() { + return Err("could not locate fastly.toml".to_owned()); + } + + candidates.sort_by_key(|path| { + let parent = path.parent().unwrap_or(Path::new("")); + path_distance(start, parent) + }); + + Ok(candidates.remove(0)) +} + +fn locate_artifact( + workspace_root: &Path, + manifest_dir: &Path, + crate_name: &str, +) -> Result { + let target_triple = "wasm32-wasip1"; + let release_name = format!("{}.wasm", crate_name.replace('-', "_")); + + if let Some(custom) = env::var_os("CARGO_TARGET_DIR") { + let candidate = PathBuf::from(custom) + .join(target_triple) + .join("release") + .join(&release_name); + if candidate.exists() { + return Ok(candidate); + } + } + + let manifest_target = manifest_dir + .join("target") + .join(target_triple) + .join("release") + .join(&release_name); + if manifest_target.exists() { + return Ok(manifest_target); + } + + let workspace_target = workspace_root + .join("target") + .join(target_triple) + .join("release") + .join(&release_name); + if workspace_target.exists() { + return Ok(workspace_target); + } + + Err(format!( + "compiled artifact not found (looked in {} and workspace target)", + manifest_dir.display() + )) +} + +#[inline] +pub fn register() { + register_adapter(&FASTLY_ADAPTER); + register_adapter_blueprint(&FASTLY_BLUEPRINT); +} + +#[ctor(unsafe)] +fn register_ctor() { + register(); +} + +/// # Errors +/// Returns an error if the Fastly CLI serve command (Viceroy) fails. +#[inline] +pub fn serve(extra_args: &[String]) -> Result<(), String> { + let manifest = + find_fastly_manifest(env::current_dir().map_err(|err| err.to_string())?.as_path())?; + let manifest_dir = manifest + .parent() + .ok_or_else(|| "fastly manifest has no parent directory".to_owned())?; + + let status = Command::new("fastly") + .args(["compute", "serve"]) + .args(extra_args) + .current_dir(manifest_dir) + .status() + .map_err(|err| format!("failed to run fastly CLI: {err}"))?; + if !status.success() { + return Err(format!("fastly compute serve failed with status {status}")); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use edgezero_adapter::cli_support::read_package_name; + #[cfg(unix)] + use edgezero_core::test_env::PathPrepend; + use std::collections::HashSet; + + #[cfg(unix)] + use std::sync::Mutex; + use tempfile::tempdir; + + // Shared fixture names. Pinning these as consts (instead of + // inline `"sessions"` / `"app_config"` per call site) keeps the + // setup-vs-assertion pair in sync -- a typo in one place no + // longer silently divorces from the other, because both reference + // the same const. Also names the intent: these are the LOGICAL + // store ids the fastly adapter operates on, not arbitrary strings. + const TEST_KV_ID: &str = "sessions"; + const TEST_CONFIG_ID: &str = "app_config"; + const TEST_SECRET_ID: &str = "default"; + + #[test] + fn finds_closest_manifest_when_multiple_exist() { + let dir = tempdir().unwrap(); + let root = dir.path(); + fs::write(root.join("Cargo.toml"), "[workspace]").unwrap(); + + let first = root.join("crates/first"); + fs::create_dir_all(&first).unwrap(); + fs::write(first.join("Cargo.toml"), "[package]\nname=\"first\"").unwrap(); + fs::write(first.join("fastly.toml"), "name=\"first\"").unwrap(); + + let second = root.join("examples/second"); + fs::create_dir_all(&second).unwrap(); + fs::write(second.join("Cargo.toml"), "[package]\nname=\"second\"").unwrap(); + fs::write(second.join("fastly.toml"), "name=\"second\"").unwrap(); + + let found = find_fastly_manifest(&second).unwrap(); + assert_eq!(found, second.join("fastly.toml")); + } + + #[test] + fn finds_manifest_in_current_directory() { + let dir = tempdir().unwrap(); + let root = dir.path(); + fs::write(root.join("Cargo.toml"), "[workspace]").unwrap(); + fs::write(root.join("fastly.toml"), "name = \"demo\"").unwrap(); + + let manifest = find_fastly_manifest(root).expect("should find manifest"); + assert_eq!(manifest, root.join("fastly.toml")); + } + + #[test] + fn locate_artifact_considers_workspace_target() { + let dir = tempdir().unwrap(); + let workspace = dir.path(); + let manifest_dir = workspace.join("service"); + fs::create_dir_all(manifest_dir.join("target/wasm32-wasip1/release")).unwrap(); + let artifact = workspace.join("target/wasm32-wasip1/release/demo.wasm"); + fs::create_dir_all(artifact.parent().unwrap()).unwrap(); + fs::write(&artifact, "wasm").unwrap(); + + let located = locate_artifact(workspace, &manifest_dir, "demo").unwrap(); + assert_eq!(located, artifact); + } + + #[test] + fn read_package_falls_back_to_name() { + let dir = tempdir().unwrap(); + let manifest = dir.path().join("Cargo.toml"); + fs::write(&manifest, "name = \"demo\"").unwrap(); + let name = read_package_name(&manifest).unwrap(); + assert_eq!(name, "demo"); + } + + #[test] + fn read_package_prefers_package_table() { + let dir = tempdir().unwrap(); + let manifest = dir.path().join("Cargo.toml"); + fs::write(&manifest, "[package]\nname = \"demo\"\n").unwrap(); + let name = read_package_name(&manifest).unwrap(); + assert_eq!(name, "demo"); + } + + // ---------- push_entries_with_committer ---------- + + #[test] + fn push_entries_with_committer_returns_count_when_all_succeed() { + let entries = vec![ + ("a".to_owned(), "1".to_owned()), + ("b".to_owned(), "2".to_owned()), + ("c".to_owned(), "3".to_owned()), + ]; + let pushed = push_entries_with_committer(&entries, |_, _| Ok(())).expect("all succeed"); + assert_eq!(pushed, 3); + } + + #[test] + fn push_entries_with_committer_zero_entries_is_ok() { + let pushed = push_entries_with_committer(&[], |_, _| Ok(())).expect("empty is fine"); + assert_eq!(pushed, 0); + } + + #[test] + fn push_entries_with_committer_failure_surfaces_committed_failed_not_attempted() { + // Mock committer: succeed for first 2 keys, fail at third. + let entries = vec![ + ("k1".to_owned(), "v1".to_owned()), + ("k2".to_owned(), "v2".to_owned()), + ("k3".to_owned(), "v3".to_owned()), + ("k4".to_owned(), "v4".to_owned()), + ("k5".to_owned(), "v5".to_owned()), + ]; + let mut calls: usize = 0; + let err = push_entries_with_committer(&entries, |key, _| { + calls = calls.saturating_add(1); + if key == "k3" { + Err("simulated fastly stderr".to_owned()) + } else { + Ok(()) + } + }) + .expect_err("middle failure must error"); + // Committer was invoked for k1, k2, k3 and stopped. + assert_eq!(calls, 3_usize, "no retries beyond failure point"); + // Error names all three categories. + assert!(err.contains("k1") && err.contains("k2"), "committed: {err}"); + assert!( + err.contains("Failed: `k3`"), + "failed entry named exactly: {err}" + ); + assert!( + err.contains("k4") && err.contains("k5"), + "not-attempted: {err}" + ); + assert!(err.contains("simulated fastly stderr"), "inner err: {err}"); + // Counts are sane. + assert!( + err.contains("committing 2 of 5 entries"), + "committed/total count: {err}" + ); + // The failed entry's outcome is UNKNOWN and recovery is a full idempotent + // re-run, not a hand-resume from a claimed boundary. + assert!( + err.contains("UNKNOWN") && err.contains("outcome unknown"), + "failed outcome must be stated unknown: {err}" + ); + assert!( + err.contains("re-run the SAME") && err.contains("idempotent"), + "recovery must be a full idempotent re-run: {err}" + ); + assert!( + !err.contains("safe to skip on retry"), + "must not claim committed entries can be skipped from a known boundary: {err}" + ); + } + + #[test] + fn push_entries_with_committer_first_entry_failure_reports_zero_committed() { + let entries = vec![ + ("only".to_owned(), "val".to_owned()), + ("never".to_owned(), "tried".to_owned()), + ]; + let err = push_entries_with_committer(&entries, |_, _| Err("nope".to_owned())) + .expect_err("first-entry failure"); + assert!(err.contains("committing 0 of 2"), "zero committed: {err}"); + assert!( + err.contains("Failed: `only`"), + "first-entry failure named: {err}" + ); + assert!( + err.contains("never"), + "second entry as not-attempted: {err}" + ); + } + + #[test] + fn push_entries_with_committer_last_entry_failure_reports_n_minus_one_committed() { + let entries = vec![ + ("a".to_owned(), "1".to_owned()), + ("b".to_owned(), "2".to_owned()), + ("c".to_owned(), "3".to_owned()), + ]; + let err = push_entries_with_committer(&entries, |key, _| { + if key == "c" { + Err("late failure".to_owned()) + } else { + Ok(()) + } + }) + .expect_err("last-entry failure"); + assert!(err.contains("committing 2 of 3"), "n-1 committed: {err}"); + assert!( + err.contains("Not attempted: []"), + "zero not-attempted when the last entry fails: {err}" + ); + } + + // ---------- looks_like_already_exists ---------- + + #[test] + fn looks_like_already_exists_recognises_common_phrasings() { + // Real-shaped fastly CLI error strings (paraphrased; the + // CLI varies across versions). Each must be detected so + // create_fastly_store can treat it as idempotent success. + assert!(looks_like_already_exists( + "Error: a kv-store with that name already exists", + "kv", + )); + assert!(looks_like_already_exists( + "ERROR: Conflict (409): duplicate kv_store name", + "kv", + )); + assert!(looks_like_already_exists( + "A config-store with this name already exists", + "config", + )); + // Spaced form: some fastly CLI versions emit prose + // ("kv store"); accept it alongside the punctuated forms. + assert!(looks_like_already_exists( + "Error: kv store conflict: name already in use", + "kv", + )); + } + + #[test] + fn looks_like_already_exists_rejects_unrelated_errors() { + assert!(!looks_like_already_exists( + "Error: unauthenticated; run `fastly profile create`", + "kv", + )); + assert!(!looks_like_already_exists( + "Error: network unreachable", + "kv", + )); + assert!(!looks_like_already_exists("", "kv")); + } + + #[test] + fn looks_like_already_exists_rejects_unrelated_conflict_errors() { + // The earlier wider heuristic swallowed ANY stderr + // containing "conflict" or "already exists", which would + // misread an unrelated 409 from a different fastly + // subcommand (e.g. a service-version conflict during a + // parallel deploy) as idempotent store-create success. + // Now we require the kind context too, so unrelated + // conflicts surface as failures. + assert!( + !looks_like_already_exists( + "Error: 409 Conflict on /service/abc/version/42 -- already exists", + "kv", + ), + "service-version conflict must NOT be misread as kv-store idempotency" + ); + assert!( + !looks_like_already_exists( + "Error: invalid duplicate request; check name resolution", + "kv", + ), + "unrelated `duplicate ... name` AND-match must NOT trigger" + ); + // And the kind must match: a config-store conflict must + // not look-like-already-exists for a kv-store create call. + assert!( + !looks_like_already_exists("Error: a config-store with that name already exists", "kv",), + "wrong-kind conflict must NOT trigger" + ); + } + + // ---------- setup_block_present ---------- + + #[test] + fn setup_block_present_true_when_table_exists() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write( + &path, + "name = \"demo\"\n[setup.kv_stores.sessions]\n[local_server.kv_stores.sessions]\n", + ) + .expect("write"); + assert!(setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); + } + + /// The three provisioning parsers must NOT echo a malformed fastly.toml's + /// source text (which can contain a stored secret) on a parse failure. + #[test] + fn provisioning_parsers_redact_malformed_toml() { + const SENTINEL: &str = "SUPER_SECRET_IN_A_BROKEN_LINE"; + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // Malformed TOML whose offending line carries a secret. + fs::write(&path, format!("service_id = \"{SENTINEL}\" = broken\n")).expect("write"); + + let errs = [ + read_fastly_service_id(&path).expect_err("malformed toml must error"), + setup_block_present(&path, "kv", TEST_KV_ID).expect_err("malformed toml must error"), + append_fastly_setup(&path, "kv", TEST_KV_ID).expect_err("malformed toml must error"), + ]; + for err in &errs { + assert!( + !err.contains(SENTINEL), + "a parse error must not echo the stored value: {err}" + ); + assert!( + err.contains("redacted"), + "error should say it redacted: {err}" + ); + } + } + + #[test] + fn setup_block_present_false_when_id_missing() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n[setup.kv_stores.other]\n").expect("write"); + assert!(!setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); + } + + #[test] + fn setup_block_present_false_for_missing_file() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("does-not-exist.toml"); + assert!(!setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); + } + + #[test] + fn setup_block_present_true_when_only_setup_exists() { + // Post-F6 (PR #269 round 2): `setup_block_present` only + // checks `[setup._stores.]`. The pre-fix check + // ALSO required `[local_server._stores.]`, but + // writing an empty `[local_server.*]` table didn't match + // fastly's local-server schema (config-stores need + // `format` + contents, kv/secret stores need a JSON file + // or `{key, data}` entries). Local-server seeding moved + // to `config push --adapter fastly --local`, so probe + // only cares about `[setup]` now. + let dir = tempdir().expect("tempdir"); + let only_setup = dir.path().join("only_setup.toml"); + fs::write(&only_setup, "name = \"demo\"\n[setup.kv_stores.sessions]\n").expect("write"); + assert!( + setup_block_present(&only_setup, "kv", TEST_KV_ID).expect("probe"), + "[setup.*] alone is now sufficient: {only_setup:?}" + ); + + let only_local = dir.path().join("only_local.toml"); + fs::write( + &only_local, + "name = \"demo\"\n[local_server.kv_stores.sessions]\n", + ) + .expect("write"); + assert!( + !setup_block_present(&only_local, "kv", TEST_KV_ID).expect("probe"), + "[local_server.*] alone is NOT a provisioned-setup signal" + ); + } + + // ---------- append_fastly_setup ---------- + + #[test] + fn append_fastly_setup_creates_setup_table_in_minimal_file() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + append_fastly_setup(&path, "kv", TEST_KV_ID).expect("append"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("[setup.kv_stores.sessions]"), + "setup table added: {after}" + ); + // Post-F6: no `[local_server.*]` write — that empty stanza + // didn't satisfy fastly's local-server schema and made + // `fastly compute serve` error or skip the store. Local- + // server seeding is now `config push --adapter fastly + // --local`'s job. + assert!( + !after.contains("[local_server.kv_stores.sessions]"), + "[local_server.*] empty table no longer written by provision: {after}" + ); + assert!( + after.contains("name = \"demo\""), + "preserved original keys: {after}" + ); + } + + #[test] + fn append_fastly_setup_appends_alongside_existing_kind_tables() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "[setup.kv_stores.cache]\n").expect("write"); + append_fastly_setup(&path, "kv", TEST_KV_ID).expect("append"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("[setup.kv_stores.cache]"), + "existing entry kept: {after}" + ); + assert!( + after.contains("[setup.kv_stores.sessions]"), + "new entry added: {after}" + ); + } + + #[test] + fn append_fastly_setup_is_idempotent_on_duplicate_id() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "[setup.kv_stores.sessions]\nfoo = \"keep\"\n").expect("write"); + append_fastly_setup(&path, "kv", TEST_KV_ID).expect("idempotent append"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("foo = \"keep\""), + "did not stomp existing key: {after}" + ); + } + + #[test] + fn append_fastly_setup_creates_file_when_missing() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // Note: no fs::write — file starts absent. + append_fastly_setup(&path, "config", TEST_CONFIG_ID).expect("create"); + let after = fs::read_to_string(&path).expect("read back"); + assert!(after.contains("[setup.config_stores.app_config]")); + assert!( + !after.contains("[local_server.config_stores.app_config]"), + "[local_server.*] no longer written by provision: {after}" + ); + } + + #[test] + fn append_fastly_setup_preserves_top_comments() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write( + &path, + "# managed by hand -- please keep this line\nname = \"demo\"\n", + ) + .expect("write"); + append_fastly_setup(&path, "secret", TEST_SECRET_ID).expect("append"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("# managed by hand"), + "preserved comment: {after}" + ); + } + + // ---------- write_fastly_local_config_store (config push --local) ---------- + + #[test] + fn write_fastly_local_config_store_creates_inline_block_in_minimal_file() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + let entries = vec![ + ("greeting".to_owned(), "hello".to_owned()), + ("service.timeout_ms".to_owned(), "1500".to_owned()), + ]; + write_fastly_local_config_store(&path, TEST_CONFIG_ID, &entries, &[]).expect("write"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains(&format!("[local_server.config_stores.{TEST_CONFIG_ID}]")), + "store table: {after}" + ); + assert!( + after.contains("format = \"inline-toml\""), + "format field: {after}" + ); + assert!( + after.contains(&format!( + "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" + )), + "contents table: {after}" + ); + assert!(after.contains("greeting = \"hello\""), "key 1: {after}"); + assert!( + after.contains("\"service.timeout_ms\" = \"1500\""), + "dotted key quoted: {after}" + ); + assert!(after.contains("name = \"demo\""), "preserved: {after}"); + } + + /// An existing `format = "json"` / `"file"` is INCOMPATIBLE with the inline + /// `contents` this writer emits. It must be overwritten to `inline-toml` (with + /// a warning), not left to survive and produce a contradictory store the + /// command still reports as written. + #[test] + fn write_fastly_local_config_store_replaces_incompatible_format() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write( + &path, + format!( + "name = \"demo\"\n\n[local_server.config_stores.{TEST_CONFIG_ID}]\nformat = \"json\"\nfile = \"cfg.json\"\n", + ), + ) + .expect("write"); + let warnings = write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "hello".to_owned())], + &[], + ) + .expect("write"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("format = \"inline-toml\""), + "incompatible format must be replaced: {after}" + ); + assert!( + !after.contains("format = \"json\""), + "the old incompatible format must not survive: {after}" + ); + assert!( + warnings + .iter() + .any(|w| w.contains("incompatible") && w.contains("inline-toml")), + "must warn that an incompatible format was replaced: {warnings:?}" + ); + } + + #[test] + fn write_fastly_local_config_store_replaces_existing_block_on_re_push() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "stale".to_owned())], + &[], + ) + .expect("first write"); + write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "fresh".to_owned())], + &[], + ) + .expect("second write"); + let after = fs::read_to_string(&path).expect("read back"); + assert!(after.contains("greeting = \"fresh\""), "new value: {after}"); + assert!( + !after.contains("greeting = \"stale\""), + "stale value dropped: {after}" + ); + } + + #[test] + fn write_fastly_local_config_store_preserves_unrelated_blocks() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + let original = "\ +[setup.kv_stores.sessions] + +[[local_server.kv_stores.sessions]] +key = \"__init__\" +data = \"\" + +[scripts] +build = \"cargo build --release\" +"; + fs::write(&path, original).expect("write"); + write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "hi".to_owned())], + &[], + ) + .expect("write"); + let after = fs::read_to_string(&path).expect("read back"); + assert!( + after.contains("[setup.kv_stores.sessions]"), + "setup KV kept: {after}" + ); + assert!(after.contains("[scripts]"), "scripts table kept: {after}"); + assert!( + after.contains("build = \"cargo build --release\""), + "scripts value kept: {after}" + ); + assert!( + after.contains(&format!( + "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" + )), + "new config_stores block added: {after}" + ); + } + + #[test] + fn write_fastly_local_config_store_creates_file_when_missing() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // No fs::write — file absent. + write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "hi".to_owned())], + &[], + ) + .expect("write"); + let after = fs::read_to_string(&path).expect("read back"); + assert!(after.contains(&format!( + "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" + ))); + assert!(after.contains("greeting = \"hi\"")); + } + + // ---------- provision (dry-run + error path) ---------- + + #[test] + fn provision_dry_run_does_not_invoke_fastly() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); + let config_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_CONFIG_ID]); + let secret_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_SECRET_ID]); + let stores = ProvisionStores { + config: &config_ids, + kv: &kv_ids, + secrets: &secret_ids, + }; + let out = FastlyCliAdapter + .provision(dir.path(), Some("fastly.toml"), None, &stores, true) + .expect("dry-run succeeds"); + // 1 KV + 1 config + 1 secret + 1 runtime-env = 4 status lines. + assert_eq!(out.len(), 4); + assert!(out[0].contains("would run `fastly kv-store create --name=sessions`")); + assert!(out[1].contains("would run `fastly config-store create --name=app_config`")); + assert!(out[2].contains("would run `fastly secret-store create --name=default`")); + assert!( + out[3].contains("would run `fastly config-store create --name=edgezero_runtime_env`"), + "runtime-env store row: {out:?}", + ); + // Manifest untouched. + let after = fs::read_to_string(&path).expect("read"); + assert_eq!(after, "name = \"demo\"\n", "dry-run mutated fastly.toml"); + } + + #[test] + fn provision_errors_when_adapter_manifest_path_missing() { + let dir = tempdir().expect("tempdir"); + let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); + let stores = ProvisionStores { + config: &[], + kv: &kv_ids, + secrets: &[], + }; + let err = FastlyCliAdapter + .provision(dir.path(), None, None, &stores, true) + .expect_err("missing adapter manifest path must error"); + assert!( + err.contains("fastly.toml"), + "error names what's missing: {err}" + ); + } + + #[test] + fn provision_with_no_declared_stores_says_so() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // Pre-populate the runtime-env block so the provision flow's + // unconditional runtime-env step skips (otherwise it would + // shell out to real `fastly` to create the store). + fs::write( + &path, + "name = \"demo\"\n[setup.config_stores.edgezero_runtime_env]\n", + ) + .expect("write"); + let stores = ProvisionStores { + config: &[], + kv: &[], + secrets: &[], + }; + let out = FastlyCliAdapter + .provision(dir.path(), Some("fastly.toml"), None, &stores, false) + .expect("no-store provision is fine"); + assert_eq!(out, vec!["fastly has no declared stores to provision"]); + } + + #[test] + fn provision_skips_id_when_setup_block_already_present() { + // setup_block_present's role in the flow: re-running + // provision after the user already declared a store in + // fastly.toml must be a no-op (no shell-out to fastly). + // We can verify this in a real (non-dry-run) call because + // the skip path bypasses create_fastly_store entirely. + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write( + &path, + "[setup.kv_stores.sessions]\n[local_server.kv_stores.sessions]\n\ + [setup.config_stores.edgezero_runtime_env]\n", + ) + .expect("write"); + let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); + let stores = ProvisionStores { + config: &[], + kv: &kv_ids, + secrets: &[], + }; + let out = FastlyCliAdapter + .provision(dir.path(), Some("fastly.toml"), None, &stores, false) + .expect("skip path succeeds without invoking fastly"); + assert_eq!(out.len(), 1); + assert!(out[0].contains("already declared"), "got: {out:?}"); + } + + /// When `fastly.toml` declares `service_id`, the next + /// `fastly compute deploy` skips `[setup]` entirely. provision + /// must emit the `fastly resource-link create` remediation for + /// every store it creates -- including the implicit + /// `edgezero_runtime_env` store the runtime override path + /// depends on. Without this, a freshly-provisioned override + /// store would not be linked to the already-deployed service + /// and the runtime would silently fall back to baked defaults. + #[test] + fn provision_emits_resource_link_note_for_runtime_env_on_existing_service() { + // Dry-run only -- we just want to drive the resource_link_note + // helper for the runtime-env store branch. The real-create + // path can't run in tests (would shell out to `fastly`). + // The dry-run output line for runtime-env doesn't include the + // note (the helper only fires on real create), so we test the + // helper directly here. + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\nservice_id = \"abc123svc\"\n").expect("write"); + let note = resource_link_note(&path, "config", "edgezero_runtime_env") + .expect("read service_id") + .expect("note present when service_id set"); + assert!( + note.contains("service_id = \"abc123svc\""), + "note quotes the service id: {note}" + ); + assert!( + note.contains("fastly config-store list --json"), + "note tells operator how to find the store id: {note}" + ); + assert!( + note.contains("name=`edgezero_runtime_env`"), + "note names the runtime override store: {note}" + ); + assert!( + note.contains( + "fastly resource-link create --service-id=abc123svc --resource-id= --version=latest --autoclone --name=edgezero_runtime_env" + ), + "note carries the full resource-link command: {note}" + ); + } + + /// And the inverse: no `service_id` (a service that hasn't been + /// deployed yet) means `[setup]` will be applied on the next + /// `compute deploy`, so no manual resource-link step is needed. + /// The helper must return `None` to avoid noisy false-positive + /// guidance. + #[test] + fn provision_skips_resource_link_note_when_service_undeployed() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + let note = + resource_link_note(&path, "config", "edgezero_runtime_env").expect("read service_id"); + assert!( + note.is_none(), + "no service_id => no resource-link prompt: {note:?}" + ); + } + + // ---------- find_config_store_id ---------- + + #[test] + fn find_config_store_id_matches_bare_array_by_name() { + let stdout = format!( + r#"[ + {{"id": "abc123", "name": "{TEST_CONFIG_ID}"}}, + {{"id": "def456", "name": "other_store"}} + ]"# + ); + match find_config_store_id(&stdout, TEST_CONFIG_ID) { + ConfigStoreLookup::Found(id) => assert_eq!(id, "abc123"), + ConfigStoreLookup::NotFound => panic!("expected Found, got NotFound"), + ConfigStoreLookup::SchemaDrift(detail) => { + panic!("expected Found, got SchemaDrift({detail})") + } + } + } + + #[test] + fn find_config_store_id_tolerates_items_envelope() { + let stdout = format!( + r#"{{"items": [ + {{"id": "xyz789", "name": "{TEST_CONFIG_ID}"}} + ]}}"# + ); + match find_config_store_id(&stdout, TEST_CONFIG_ID) { + ConfigStoreLookup::Found(id) => assert_eq!(id, "xyz789"), + ConfigStoreLookup::NotFound => panic!("expected Found, got NotFound"), + ConfigStoreLookup::SchemaDrift(detail) => { + panic!("expected Found, got SchemaDrift({detail})") + } + } + } + + #[test] + fn find_config_store_id_distinguishes_not_found_from_match_failure() { + // JSON parses cleanly, entries are well-formed + // (`name` + `id` strings present), but no entry matches + // → NotFound. Operator likely needs to run `provision`. + let stdout = r#"[{"id": "abc", "name": "other"}]"#; + assert!(matches!( + find_config_store_id(stdout, "missing"), + ConfigStoreLookup::NotFound + )); + } + + #[test] + fn find_config_store_id_flags_schema_drift_on_malformed_json() { + // Unparseable bytes are NOT a "store not found" — they're + // a "fastly CLI output format changed" signal. Operator + // needs different recovery (file a bug, pin CLI version) + // than for the "store doesn't exist yet" case. + let drift = find_config_store_id("not json", "anything"); + assert!( + matches!(drift, ConfigStoreLookup::SchemaDrift(_)), + "non-JSON stdout must be schema drift, got {drift:?}" + ); + let empty = find_config_store_id("", "anything"); + assert!( + matches!(empty, ConfigStoreLookup::SchemaDrift(_)), + "empty stdout must be schema drift, got {empty:?}" + ); + } + + #[test] + fn find_config_store_id_flags_schema_drift_when_shape_unexpected() { + // JSON parses but the top-level is neither a bare array + // nor an `{items: [...]}` envelope. + let stdout = r#"{"namespace": "fastly", "list": []}"#; + match find_config_store_id(stdout, "any") { + ConfigStoreLookup::SchemaDrift(detail) => { + assert!( + detail.contains("bare array") || detail.contains("items"), + "schema-drift detail names the expected shapes: {detail}" + ); + } + ConfigStoreLookup::Found(id) => panic!("expected SchemaDrift, got Found({id})"), + ConfigStoreLookup::NotFound => panic!("expected SchemaDrift, got NotFound"), + } + } + + #[test] + fn find_config_store_id_flags_schema_drift_when_entries_lack_name_id() { + // Array of objects but none have BOTH string `name` and + // string `id` fields — suggests schema rename (e.g. + // fastly renamed `name` → `title`). + let stdout = format!(r#"[{{"title": "{TEST_CONFIG_ID}", "uid": "abc"}}]"#); + let drift = find_config_store_id(&stdout, TEST_CONFIG_ID); + assert!( + matches!(drift, ConfigStoreLookup::SchemaDrift(_)), + "entries lacking name/id must be schema drift, got {drift:?}" + ); + } + + #[test] + fn find_config_store_id_returns_not_found_for_empty_array() { + // Empty array IS a valid "store doesn't exist yet" signal, + // not schema drift — fastly CLI legitimately returns `[]` + // when no config-stores exist. + let drift = find_config_store_id("[]", "any"); + assert!( + matches!(drift, ConfigStoreLookup::NotFound), + "empty array must be NotFound, got {drift:?}" + ); + } + + // ---------- push_config_entries (dry-run + error paths) ---------- + + #[test] + fn push_dry_run_does_not_invoke_fastly() { + let dir = tempdir().expect("tempdir"); + let entries = vec![ + ("greeting".to_owned(), "hello".to_owned()), + ("feature.new_checkout".to_owned(), "false".to_owned()), + ]; + let out = FastlyCliAdapter + .push_config_entries( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + true, + ) + .expect("dry-run succeeds"); + // First line names the resolve+publish flow; then one preview line per + // key. A push no longer reclaims anything (see `config gc`), so there is + // no GC-intent line. + assert_eq!(out.len(), 1 + entries.len(), "header + per-entry preview"); + assert!( + out[0].contains("would resolve fastly config-store `app_config`") + && out[0].contains("push entries"), + "dry-run header describes the would-be flow: {out:?}" + ); + assert!( + out.iter().any(|line| line.contains("`greeting`")), + "dry-run lists `greeting`: {out:?}" + ); + assert!( + out.iter() + .any(|line| line.contains("`feature.new_checkout`")), + "dry-run lists `feature.new_checkout`: {out:?}" + ); + } + + #[test] + fn push_with_no_entries_reports_no_op_without_invoking_fastly() { + let dir = tempdir().expect("tempdir"); + let out = FastlyCliAdapter + .push_config_entries( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[], + &AdapterPushContext::new(), + false, + ) + .expect("zero-entry push is fine"); + assert_eq!(out.len(), 1); + assert!( + out[0].contains("no config entries"), + "status line names the no-op: {out:?}" + ); + } + + // ---------- read_config_entry_local ---------- + + #[test] + fn read_local_returns_missing_store_when_fastly_toml_absent() { + let dir = tempdir().expect("tempdir"); + // No fastly.toml written — file missing. + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("missing file is not an error"); + assert!( + matches!(result, ReadConfigEntry::MissingStore), + "absent fastly.toml => MissingStore" + ); + } + + #[test] + fn read_local_returns_missing_store_when_no_local_server_contents() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // fastly.toml exists but has no [local_server.config_stores.*] block. + fs::write(&path, "name = \"demo\"\n[setup.config_stores.app_config]\n").expect("write"); + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("missing local_server block is not an error"); + assert!( + matches!(result, ReadConfigEntry::MissingStore), + "no local_server stanza => MissingStore" + ); + } + + #[test] + fn read_local_returns_missing_key_when_key_absent_from_contents() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + // Write a local_server block with a different key so the store exists + // but the requested key is absent. + fs::write( + &path, + format!( + "name = \"demo\"\n\ + [local_server.config_stores.{TEST_CONFIG_ID}]\n\ + format = \"inline-toml\"\n\ + [local_server.config_stores.{TEST_CONFIG_ID}.contents]\n\ + other_key = \"other_value\"\n" + ), + ) + .expect("write"); + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("missing key is not an error"); + assert!( + matches!(result, ReadConfigEntry::MissingKey), + "key absent from contents => MissingKey" + ); + } + + #[test] + fn read_local_returns_present_when_key_exists_in_contents() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write initial toml"); + + // Use a valid BlobEnvelope value — the resolver requires BlobEnvelope + // or chunk-pointer JSON; raw strings are not accepted post-chunking. + let envelope_json = serde_json::to_string(&BlobEnvelope::new( + json!({"hello": "fastly"}), + "2026-06-22T00:00:00Z".into(), + )) + .expect("serialize"); + write_fastly_local_config_store( + &path, + TEST_CONFIG_ID, + &[("greeting".to_owned(), envelope_json.clone())], + &[], + ) + .expect("setup write"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("key present"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present variant"); + }; + assert_eq!(value, envelope_json, "value matches what was written"); + } + + #[test] + fn read_local_roundtrips_with_push_local() { + // Write via push_config_entries_local, then read via + // read_config_entry_local — the two must agree on the value. + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + + // push_config_entries_local passes the value through the chunk-pointer + // helper which stores it verbatim when ≤ 8 000 chars. The reader then + // resolves it through the same resolver that requires BlobEnvelope JSON. + let envelope_json = serde_json::to_string(&BlobEnvelope::new( + json!({"hello": "roundtrip"}), + "2026-06-22T00:00:00Z".into(), + )) + .expect("serialize"); + let entries = vec![("greeting".to_owned(), envelope_json.clone())]; + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + false, + ) + .expect("push succeeds"); + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("read succeeds"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present after push+read roundtrip"); + }; + assert_eq!(value, envelope_json, "roundtrip value matches"); + } + + #[test] + fn read_local_requires_adapter_manifest_path() { + let dir = tempdir().expect("tempdir"); + let result = FastlyCliAdapter.read_config_entry_local( + dir.path(), + None, // adapter_manifest_path missing + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ); + match result { + Err(err) => assert!( + err.contains("[adapters.fastly.adapter].manifest"), + "error names the missing field: {err}" + ), + Ok(_) => panic!("expected Err when adapter_manifest_path is None"), + } + } + + // ---------- read_config_entry (fake fastly, remote shell-out) ---------- + + /// Build a tempdir containing a `fastly` shim script that: + /// - Responds to `config-store list --json` with a store-list JSON containing + /// `TEST_CONFIG_ID` mapped to `store-abc123`. + /// - Responds to `config-store-entry describe ...` with `stdout_body` on + /// stdout and `stderr_body` on stderr, exiting with `exit_code`. + /// + /// Payloads are written to separate sibling files so shell-active chars + /// in the content don't get re-interpreted by the script. + #[cfg(unix)] + fn fake_fastly_returning( + stdout_body: &str, + stderr_body: &str, + exit_code: i32, + ) -> tempfile::TempDir { + fake_fastly_returning_with_keys(stdout_body, stderr_body, exit_code, &[]) + } + + /// As [`fake_fastly_returning`], but also serves `config-store-entry list` + /// with a bare array of the `entry_list_keys` as `item_key` entries. A + /// describe FAILURE is confirmed against this listing: keys present here read + /// as a present-but-unreadable hard error, keys absent read as `MissingKey`. + #[cfg(unix)] + fn fake_fastly_returning_with_keys( + stdout_body: &str, + stderr_body: &str, + exit_code: i32, + entry_list_keys: &[&str], + ) -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt as _; + let dir = tempdir().expect("tempdir"); + let script_path = dir.path().join("fastly"); + let stdout_file = dir.path().join("stdout_payload.txt"); + let stderr_file = dir.path().join("stderr_payload.txt"); + let list_file = dir.path().join("list_payload.txt"); + let entry_list_file = dir.path().join("entry_list_payload.txt"); + // Store-list JSON: bare array with one entry matching TEST_CONFIG_ID. + let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); + let entry_list_json = { + let items: Vec = entry_list_keys + .iter() + .map(|key| format!(r#"{{"item_key":{}}}"#, serde_json::to_string(key).unwrap())) + .collect(); + format!("[{}]", items.join(",")) + }; + fs::write(&stdout_file, stdout_body).expect("write stdout payload"); + fs::write(&stderr_file, stderr_body).expect("write stderr payload"); + fs::write(&list_file, list_json).expect("write list payload"); + fs::write(&entry_list_file, entry_list_json).expect("write entry list payload"); + let script = format!( + "#!/bin/sh\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\nif [ \"$1\" = \"config-store-entry\" ] && [ \"$2\" = \"list\" ]; then\n cat '{}'\n exit 0\nfi\ncat '{}'\ncat '{}' >&2\nexit {exit_code}\n", + list_file.display(), + entry_list_file.display(), + stdout_file.display(), + stderr_file.display(), + ); + fs::write(&script_path, script).expect("write fastly script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod +x"); + dir + } + + /// Build a fake `fastly` that logs each argv token (one per line) to + /// `out_path`, handles the list call correctly, and exits 0 for both calls. + #[cfg(unix)] + fn fake_fastly_argv_log(out_path: &Path) -> tempfile::TempDir { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + use std::os::unix::fs::PermissionsExt as _; + let dir = tempdir().expect("tempdir"); + let script_path = dir.path().join("fastly"); + let list_file = dir.path().join("list_payload.txt"); + let entry_file = dir.path().join("entry_payload.txt"); + let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); + // item_value must be a valid BlobEnvelope JSON so the resolver accepts it. + let envelope_json = serde_json::to_string(&BlobEnvelope::new( + json!({"v": "logged"}), + "2026-06-22T00:00:00Z".into(), + )) + .expect("serialize"); + let entry_json = format!( + r#"{{"item_value":{},"store_id":"store-abc123"}}"#, + serde_json::to_string(&envelope_json).expect("escape") + ); + fs::write(&list_file, list_json).expect("write list payload"); + fs::write(&entry_file, &entry_json).expect("write entry payload"); + let script = format!( + "#!/bin/sh\nfor arg in \"$@\"; do printf '%s\\n' \"$arg\" >> '{}'; done\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\ncat '{}'\nexit 0\n", + out_path.display(), + list_file.display(), + entry_file.display(), + ); + fs::write(&script_path, script).expect("write script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod +x"); + dir + } + + /// Process-wide mutex serialising PATH-mutating tests so parallel + /// test threads don't race on the environment variable. + #[cfg(unix)] + fn path_mutation_guard() -> &'static Mutex<()> { + use std::sync::OnceLock; + static GUARD: OnceLock> = OnceLock::new(); + GUARD.get_or_init(|| Mutex::new(())) + } + + #[cfg(unix)] + #[test] + fn read_remote_returns_present_on_success() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // Fake fastly: list succeeds with app_config → store-abc123; + // describe returns valid JSON with item_value that is a BlobEnvelope. + let envelope = serde_json::to_string(&BlobEnvelope::new( + json!({"hello": "fastly"}), + "2026-06-22T00:00:00Z".into(), + )) + .expect("serialize"); + let entry_json = format!( + r#"{{"item_value":{},"store_id":"store-abc123"}}"#, + serde_json::to_string(&envelope).expect("escape") + ); + let fake = fake_fastly_returning(&entry_json, "", 0); + let _path = PathPrepend::new(fake.path()); + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("fake fastly exit-0 must succeed"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present"); + }; + assert_eq!(value, envelope); + } + + #[cfg(unix)] + #[test] + fn read_remote_returns_missing_key_when_confirmed_absent() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // describe exits non-zero, and the complete store listing (empty here) + // CONFIRMS the key is absent → MissingKey (not decided by the 404 alone). + let fake = fake_fastly_returning("", "Error: item not found", 1); + let _path = PathPrepend::new(fake.path()); + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("not-found maps to MissingKey (not Err)"); + assert!( + matches!(result, ReadConfigEntry::MissingKey), + "not-found stderr => MissingKey" + ); + } + + /// The Fastly impl distinguishes store-not-found from key-not-found via + /// `resolve_remote_config_store_id`: when the list call exits non-zero and + /// the error string contains "not found", `read_config_entry` returns + /// `MissingStore` without ever calling the describe subcommand. + #[cfg(unix)] + #[test] + fn read_remote_fails_closed_when_the_list_call_itself_errors() { + use std::os::unix::fs::PermissionsExt as _; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // The list call EXITS NON-ZERO with "not found"-shaped stderr. That is an + // OPERATIONAL failure (auth/network/server), not proof the store is absent + // -- the absence signal is a SUCCESSFUL list that omits the store. So this + // must fail closed (a hard error the operator retries), NEVER MissingStore: + // reading it as absence could authorise an overwrite of a store we never + // actually queried. + let fake_dir = tempdir().expect("tempdir"); + let stderr_file = fake_dir.path().join("stderr_payload.txt"); + fs::write(&stderr_file, "Error: config store not found for service").expect("write stderr"); + let script_path = fake_dir.path().join("fastly"); + let script = format!( + "#!/bin/sh\ncat '{stderr}' >&2\nexit 1\n", + stderr = stderr_file.display(), + ); + fs::write(&script_path, script).expect("write script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod +x"); + let _path = PathPrepend::new(fake_dir.path()); + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ); + assert!( + result.is_err(), + "a failed list call must fail closed, not read as MissingStore" + ); + } + + #[cfg(unix)] + #[test] + fn read_remote_returns_missing_store_when_the_store_is_genuinely_absent() { + use std::os::unix::fs::PermissionsExt as _; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // The list call SUCCEEDS and returns a valid, empty store array. The store + // is genuinely absent -> `no fastly config-store matches` -> MissingStore. + let fake_dir = tempdir().expect("tempdir"); + let script_path = fake_dir.path().join("fastly"); + fs::write(&script_path, "#!/bin/sh\necho '[]'\nexit 0\n").expect("write script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod +x"); + let _path = PathPrepend::new(fake_dir.path()); + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("a successful list that omits the store maps to MissingStore"); + assert!( + matches!(result, ReadConfigEntry::MissingStore), + "store absent from a successful list => MissingStore" + ); + } + + /// Verify that `read_config_entry` invokes + /// `fastly config-store-entry describe --store-id= --key= --json` + /// (after the resolve step that calls `fastly config-store list --json`). + #[cfg(unix)] + #[test] + fn read_remote_invokes_correct_argv() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let argv_log = dir.path().join("argv.txt"); + let fake = fake_fastly_argv_log(&argv_log); + let _path = PathPrepend::new(fake.path()); + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "greeting", + &AdapterPushContext::new(), + ) + .expect("argv-log fake must succeed"); + assert!( + matches!(result, ReadConfigEntry::Present(_)), + "expected Present from argv-log fake" + ); + let captured = fs::read_to_string(&argv_log).expect("argv log"); + // The describe call must include these args (resolve call args + // are also captured but we only assert the describe shape here). + assert!( + captured.contains("config-store-entry"), + "must invoke config-store-entry; got:\n{captured}" + ); + assert!( + captured.contains("describe"), + "must pass describe subcommand; got:\n{captured}" + ); + assert!( + captured.contains("--store-id=store-abc123"), + "must pass resolved store id; got:\n{captured}" + ); + assert!( + captured.contains("--key=greeting"), + "must pass --key=; got:\n{captured}" + ); + assert!( + captured.contains("--json"), + "must pass --json flag; got:\n{captured}" + ); + } + + // ---------- chunked push integration tests ---------- + + /// Build a valid `BlobEnvelope` JSON string of approximately `target_len` bytes. + fn make_test_envelope(target_len: usize) -> String { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let pad = "x".repeat(target_len.saturating_add(64)); + let data = json!({ "pad": pad }); + let raw = + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".into())).unwrap(); + if raw.len() >= target_len { + let overhead = raw.len().saturating_sub(pad.len()); + let adjusted = "x".repeat(target_len.saturating_sub(overhead)); + let data2 = json!({ "pad": adjusted }); + serde_json::to_string(&BlobEnvelope::new(data2, "2026-06-22T00:00:00Z".into())).unwrap() + } else { + raw + } + } + + /// Build a fake `fastly` script whose describe response depends on + /// the `--key=` argument: `key_responses` maps key names to JSON + /// item-value responses. Falls back to exit 1 "not found" for unknown keys. + #[cfg(unix)] + fn fake_fastly_with_key_dispatch( + _dir: &Path, + key_responses: &[(String, String)], + ) -> tempfile::TempDir { + use std::fmt::Write as _; + use std::os::unix::fs::PermissionsExt as _; + let fake_dir = tempdir().expect("tempdir"); + let list_file = fake_dir.path().join("list.json"); + let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); + fs::write(&list_file, list_json).expect("write list"); + // The `config-store-entry list` response: a bare array of the keys present + // in `key_responses`. Absence confirmation lists the store and checks + // membership, so a key omitted here reads as CONFIRMED absent. Only + // `item_key` is needed (the keys-only listing is value-tolerant). + let entry_list_file = fake_dir.path().join("entry_list.json"); + let entries_json = { + let items: Vec = key_responses + .iter() + .map(|(key, _)| { + format!(r#"{{"item_key":{}}}"#, serde_json::to_string(key).unwrap()) + }) + .collect(); + format!("[{}]", items.join(",")) + }; + fs::write(&entry_list_file, entries_json).expect("write entry list"); + // Write each key response to a named file. + let mut dispatch_lines = String::new(); + for (key, response) in key_responses { + let resp_file = fake_dir.path().join(format!("resp_{key}.json")); + fs::write(&resp_file, response).expect("write resp"); + // Use exact-match: iterate argv and compare each token literally + // so that a root key like "app_config" does NOT match a chunk key + // like "app_config.__edgezero_chunks.abc.0". + writeln!( + dispatch_lines, + " for arg in \"$@\"; do if [ \"$arg\" = \"--key={key}\" ]; then cat '{}'; exit 0; fi; done", + resp_file.display() + ) + .expect("write to String is infallible"); + } + // `config-store` (store list) and `config-store-entry list` are served + // from their files; a `describe` for an unknown key exits 1 "not found", + // which the caller then CONFIRMS against the entry list. + let script = format!( + "#!/bin/sh\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\nif [ \"$1\" = \"config-store-entry\" ] && [ \"$2\" = \"list\" ]; then\n cat '{}'\n exit 0\nfi\n{dispatch_lines}echo 'Error: item not found' >&2\nexit 1\n", + list_file.display(), + entry_list_file.display() + ); + let script_path = fake_dir.path().join("fastly"); + fs::write(&script_path, &script).expect("write script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod"); + fake_dir + } + + /// Fake `fastly` for cloud chunk-GC tests. Logs each + /// `config-store-entry` op ("describe " / "update " / + /// "delete ", plus "delete-argv ") to `oplog`. + /// + /// `root_describe_seq` gives the successive raw `item_value`s returned when + /// the ROOT key is described (call 1 = the pre-commit prior read, call 2 = + /// the post-commit read-back). `entry_list` is served for + /// `config-store-entry list` and is what reclamation derives generations + /// and supersession times from. `fail_delete_key` makes that one delete + /// exit non-zero. `describe_hard_error` makes the FIRST describe of each key + /// fail hard (so the prior read errors while the read-back still works). + #[cfg(unix)] + fn fake_fastly_gc( + root_key: &str, + root_describe_seq: &[String], + entry_list: &[(String, String, String)], + fail_delete_key: Option<&str>, + describe_hard_error: bool, + oplog: &Path, + ) -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt as _; + // Rendered with handlebars. Triple-stache `{{{ }}}` disables HTML + // escaping (paths are not markup); the shell's own `${var}` / + // `$(( ))` use single braces so they are literal text to handlebars. + const TEMPLATE: &str = r#"#!/bin/sh +if [ "$1" = "config-store" ]; then cat '{{{list}}}'; exit 0; fi +sub="$2" +key="" +for arg in "$@"; do case "$arg" in --key=*) key="${arg#--key=}";; esac; done +if [ "$sub" = "list" ]; then printf 'list\n' >> '{{{oplog}}}'; cat '{{{entries}}}'; exit 0; fi +if [ "$sub" = "update" ]; then cat >/dev/null; printf 'update %s\n' "$key" >> '{{{oplog}}}'; exit 0; fi +if [ "$sub" = "delete" ]; then printf 'delete %s\n' "$key" >> '{{{oplog}}}'; printf 'delete-argv %s\n' "$*" >> '{{{oplog}}}'; if [ "$key" = "{{{fail}}}" ]; then echo 'Error: 404 item not found' >&2; exit 1; fi; exit 0; fi +if [ "$sub" = "describe" ]; then + printf 'describe %s\n' "$key" >> '{{{oplog}}}' + cfile='{{{dir}}}/count_'"$key" + n=0; [ -f "$cfile" ] && n=$(cat "$cfile"); n=$((n+1)); printf '%s' "$n" > "$cfile" + {{#if hard_error}}if [ "$n" = "1" ]; then echo 'Error: internal server error' >&2; exit 1; fi{{/if}} + rf='{{{dir}}}/resp_'"$key"'_'"$n"'.json' + if [ -f "$rf" ]; then cat "$rf"; exit 0; fi + echo 'Error: item not found' >&2; exit 1 +fi +echo 'unexpected' >&2; exit 1 +"#; + let dir = tempdir().expect("tempdir"); + let list_file = dir.path().join("list.json"); + fs::write( + &list_file, + format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#), + ) + .expect("list"); + let entries_file = dir.path().join("entries.json"); + fs::write(&entries_file, entry_list_json(entry_list)).expect("entries"); + for (index, value) in root_describe_seq.iter().enumerate() { + let wrapped = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(value).expect("escape") + ); + let nth = index.saturating_add(1); + fs::write( + dir.path().join(format!("resp_{root_key}_{nth}.json")), + wrapped, + ) + .expect("resp"); + } + let data = serde_json::json!({ + "list": list_file.display().to_string(), + "entries": entries_file.display().to_string(), + "oplog": oplog.display().to_string(), + "dir": dir.path().display().to_string(), + "fail": fail_delete_key.unwrap_or(""), + "hard_error": describe_hard_error, + }); + let script = handlebars::Handlebars::new() + .render_template(TEMPLATE, &data) + .expect("render fake fastly script"); + let script_path = dir.path().join("fastly"); + fs::write(&script_path, script).expect("script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod"); + dir + } + + /// Like `fake_fastly_gc`, but serves a VERBATIM `config-store-entry list` + /// payload so a test can present a shape `entry_list_json` cannot build + /// (e.g. a paginated envelope). + #[cfg(unix)] + fn fake_fastly_gc_raw_list( + root_key: &str, + raw_listing: &str, + oplog: &Path, + ) -> tempfile::TempDir { + let dir = fake_fastly_gc(root_key, &[], &[], None, false, oplog); + fs::write(dir.path().join("entries.json"), raw_listing).expect("raw entries"); + dir + } + + /// A `config-store-entry list --json` payload. The item VALUE is a + /// placeholder: reclamation must only ever use keys and timestamps. + #[cfg(unix)] + fn entry_list_json(items: &[(String, String, String)]) -> String { + let entries: Vec = items + .iter() + .map(|(key, created, value)| { + serde_json::json!({ + "item_key": key, + "created_at": created, + "item_value": value, + }) + }) + .collect(); + serde_json::to_string(&entries).expect("entry list json") + } + + /// An RFC-3339 stamp `secs` in the past (the shape Fastly returns). + #[cfg(unix)] + fn stamp_secs_ago(secs: u64) -> String { + let delta = chrono::Duration::seconds(i64::try_from(secs).unwrap_or(0)); + let now = chrono::Utc::now(); + now.checked_sub_signed(delta) + .unwrap_or(now) + .to_rfc3339_opts(chrono::SecondsFormat::Secs, true) + } + + /// Every chunk of `envelope` as the listing would return it: REAL keys and + /// REAL payload bytes. + /// + /// The values are not decorative. `config gc` proves a generation is ours by + /// reassembling it and hashing the result against the content-address its + /// keys name, so a placeholder value would (correctly) fail verification and + /// never be reclaimed. Fixtures must be honest for these tests to mean + /// anything. + #[cfg(unix)] + fn listed_generation( + root_key: &str, + envelope: &str, + secs_ago: u64, + ) -> Vec<(String, String, String)> { + let (chunks, _) = chunked_parts(root_key, envelope); + let stamp = stamp_secs_ago(secs_ago); + chunks + .into_iter() + .map(|(key, value)| (key, stamp.clone(), value)) + .collect() + } + + /// The ROOT entry as the listing would return it: its value is the pointer, + /// which is how `config gc` learns which chunks are live. + #[cfg(unix)] + fn listed_root(root_key: &str, envelope: &str, secs_ago: u64) -> (String, String, String) { + let (_, pointer) = chunked_parts(root_key, envelope); + (root_key.to_owned(), stamp_secs_ago(secs_ago), pointer) + } + + /// A chunked envelope with a distinct payload per tag, padded to `pad` + /// characters so a caller can force a given number of chunks (7 000 bytes + /// each). + #[cfg(unix)] + fn gen_envelope_padded(tag: &str, pad: usize) -> String { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let data = json!({ tag: "x".repeat(pad) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".to_owned())) + .expect("envelope") + } + + /// A chunked envelope with a distinct payload per tag. + #[cfg(unix)] + fn gen_envelope(tag: &str) -> String { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let data = json!({ tag: "x".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".to_owned())) + .expect("envelope") + } + + /// Split a chunked envelope into (chunk `(key, value)` pairs, root pointer). + #[cfg(unix)] + fn chunked_parts(root_key: &str, envelope: &str) -> (Vec<(String, String)>, String) { + let entries = prepare_fastly_config_entries(root_key, envelope).expect("expand"); + let (_, pointer) = entries.last().expect("pointer").clone(); + let chunks = entries[..entries.len().saturating_sub(1)].to_vec(); + (chunks, pointer) + } + + /// Just the chunk KEYS of a generation (for delete assertions). + #[cfg(unix)] + fn chunk_keys_of(root_key: &str, envelope: &str) -> Vec { + let (chunks, _) = chunked_parts(root_key, envelope); + chunks.into_iter().map(|(key, _)| key).collect() + } + + #[cfg(unix)] + fn oplog_has(oplog: &Path, line: &str) -> bool { + fs::read_to_string(oplog) + .unwrap_or_default() + .lines() + .any(|entry| entry == line) + } + + #[cfg(unix)] + #[test] + fn push_config_entries_rejects_reserved_key() { + let dir = tempdir().expect("tempdir"); + let bad_key = format!("app_config{CHUNK_KEY_INFIX}deadbeef.0"); + let err = FastlyCliAdapter + .push_config_entries( + dir.path(), + None, + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(bad_key.clone(), "{}".to_owned())], + &AdapterPushContext::new(), + false, + ) + .expect_err("reserved key must be rejected"); + assert!(err.contains(&bad_key), "names the key: {err}"); + } + + /// Schema drift must never echo the config payload — including OBJECT KEYS, + /// which are provider/stored data. App config can hold credentials; CLI + /// status lines are logged verbatim and CI logs are retained/shared. Only a + /// size + field COUNT may be reported. + #[cfg(unix)] + #[test] + fn read_config_entry_schema_drift_does_not_leak_payload() { + const SENTINEL: &str = "SUPER_SECRET_TOKEN_abc123"; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // The sentinel is an OBJECT KEY (not a value): the earlier redactor joined + // keys into the diagnostic, so this is what pins the key-disclosure fix. + let drift = format!(r#"{{"{SENTINEL}":"x"}}"#); + let fake = fake_fastly_returning(&drift, "", 0); + let _path = PathPrepend::new(fake.path()); + + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ); + let Err(err) = result else { + panic!("schema drift must error") + }; + assert!( + !err.contains(SENTINEL), + "error must not leak an object KEY from the config payload: {err}" + ); + assert!( + err.contains("bytes") && err.contains("field(s)"), + "error should carry a redacted size + field COUNT: {err}" + ); + } + + /// The FAILURE branch leaks too: a Fastly error that quotes the stored + /// value back in stderr must not reach the user-facing error. + #[cfg(unix)] + #[test] + fn read_config_entry_stderr_failure_does_not_leak_payload() { + const SENTINEL: &str = "SUPER_SECRET_TOKEN_stderr1"; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // A hard failure that echoes the value. The key IS present in the store + // listing, so absence confirmation fails and the read surfaces the + // (redacted) describe stderr on the hard-error path. + let stderr = format!("Error: internal failure processing value {SENTINEL}"); + let fake = fake_fastly_returning_with_keys("", &stderr, 1, &["cfg"]); + let _path = PathPrepend::new(fake.path()); + + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ); + let Err(err) = result else { + panic!("hard stderr failure must error") + }; + assert!( + !err.contains(SENTINEL), + "stderr must be redacted, not echoed: {err}" + ); + assert!( + err.contains("suppressed"), + "error should say the stderr was suppressed: {err}" + ); + } + + /// The WRITE path leaks too: a failing `config-store-entry update --upsert` + /// whose stderr quotes the value being written must be redacted. + #[cfg(unix)] + #[test] + fn upsert_stderr_failure_does_not_leak_payload() { + const SENTINEL: &str = "SUPER_SECRET_TOKEN_upsert1"; + let _lock = path_mutation_guard().lock().expect("guard"); + // A fake `fastly` that fails every call, echoing the value in stderr. + let stderr = format!("Error: rejected value {SENTINEL}"); + let fake = fake_fastly_returning("", &stderr, 1); + let _path = PathPrepend::new(fake.path()); + + let err = create_config_store_entry("store-abc", "cfg", SENTINEL) + .expect_err("a failing upsert must error"); + assert!( + !err.contains(SENTINEL), + "upsert stderr must be redacted, not echoed: {err}" + ); + assert!( + err.contains("suppressed"), + "error should say the stderr was suppressed: {err}" + ); + } + + /// `config gc` reads `item_value` for every entry (to classify roots). A + /// malformed listing whose values carry secrets must fail closed WITHOUT + /// echoing any value. (Replaces the old push prior-read redaction tests, + /// which are now vacuous: a cloud push performs no pre-commit read.) + #[cfg(unix)] + #[test] + fn gc_list_failure_does_not_leak_payload() { + const SENTINEL: &str = "SUPER_SECRET_TOKEN_gc_list"; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + let good = entry_list_json(&listing); + // A valid entry whose VALUE contains the sentinel, plus a malformed + // sibling (no created_at) to trip the fail-closed path. + let mut array: serde_json::Value = serde_json::from_str(&good).unwrap(); + let arr = array.as_array_mut().unwrap(); + arr.push(serde_json::json!({ + "item_key": "some.__edgezero_chunks.deadbeef.0", + "item_value": SENTINEL, + })); + let fake = fake_fastly_gc( + TEST_CONFIG_ID, + &[], + &listing, + None, + false, + &dir.path().join("ops.log"), + ); + fs::write( + fake.path().join("entries.json"), + serde_json::to_string(&array).unwrap(), + ) + .expect("overwrite entries"); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); + assert!( + !err.contains(SENTINEL), + "the fail-closed error must not echo a stored value: {err}" + ); + } + + #[cfg(unix)] + #[test] + fn push_config_entries_writes_direct_entry_at_exactly_8000_chars() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let argv_log = dir.path().join("argv.txt"); + let fake = fake_fastly_argv_log(&argv_log); + let _path = PathPrepend::new(fake.path()); + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + assert_eq!(envelope.len(), FASTLY_CONFIG_ENTRY_LIMIT); + + let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + let out = FastlyCliAdapter + .push_config_entries( + dir.path(), + None, + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + false, + ) + .expect("push must succeed"); + // One physical entry written (direct). + let captured = fs::read_to_string(&argv_log).expect("argv log"); + assert!( + captured.contains(&format!("--key={TEST_CONFIG_ID}")), + "must write root key directly: {captured}" + ); + assert!( + out[0].contains("1 physical entries (1 logical)"), + "summary reports 1 physical entry: {out:?}" + ); + } + + #[cfg(unix)] + #[test] + fn push_config_entries_writes_chunks_and_root_pointer_for_8001_chars() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let argv_log = dir.path().join("argv.txt"); + let fake = fake_fastly_argv_log(&argv_log); + let _path = PathPrepend::new(fake.path()); + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + assert!(envelope.len() > FASTLY_CONFIG_ENTRY_LIMIT); + + let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + let out = FastlyCliAdapter + .push_config_entries( + dir.path(), + None, + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + false, + ) + .expect("push must succeed"); + let captured = fs::read_to_string(&argv_log).expect("argv log"); + // At least one chunk key must appear before the root key. + assert!( + captured.contains(".__edgezero_chunks."), + "chunk keys must be written: {captured}" + ); + // Root pointer must also be written. + assert!( + captured.contains(&format!("--key={TEST_CONFIG_ID}")), + "root pointer must be written: {captured}" + ); + // Root key must be LAST in the log (chunk lines come before it). + let root_pos = captured.rfind(&format!("--key={TEST_CONFIG_ID}")).unwrap(); + let chunk_pos = captured.find(".__edgezero_chunks.").unwrap(); + assert!( + chunk_pos < root_pos, + "chunk writes must precede root pointer write: chunk_pos={chunk_pos} root_pos={root_pos}" + ); + assert!(out[0].contains("logical"), "summary line present: {out:?}"); + } + + #[cfg(unix)] + #[test] + fn push_config_entries_dry_run_reports_direct_vs_chunked() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + + let direct_envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let chunked_envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = vec![ - ("a".to_owned(), "1".to_owned()), - ("b".to_owned(), "2".to_owned()), - ("c".to_owned(), "3".to_owned()), + ("cfg_direct".to_owned(), direct_envelope), + ("cfg_chunked".to_owned(), chunked_envelope), ]; - let pushed = push_entries_with_committer(&entries, |_, _| Ok(())).expect("all succeed"); - assert_eq!(pushed, 3); + let out = FastlyCliAdapter + .push_config_entries( + dir.path(), + None, + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + true, // dry_run + ) + .expect("dry-run must not error"); + + // No shellout happens; output must describe intent. + let combined = out.join("\n"); + assert!( + combined.contains("would push `cfg_direct` as direct entry"), + "must report direct: {combined}" + ); + assert!( + combined.contains("would push `cfg_chunked` as chunked"), + "must report chunked: {combined}" + ); + } + + /// Spec 12.7: pushing two blobs under different root keys + /// (e.g. `app_config` + `app_config_staging`) must leave both + /// keys readable from the local fastly.toml so the runtime + /// `EDGEZERO__STORES__CONFIG__APP_CONFIG__KEY` override can + /// switch between them. Prior to the upsert fix the second + /// push wholesale-replaced the per-store contents table. + #[cfg(unix)] + #[test] + fn push_config_entries_local_preserves_sibling_keys() { + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + let store = ResolvedStoreId::from_logical(TEST_CONFIG_ID); + let ctx = AdapterPushContext::new(); + + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &store, + &[("app_config".to_owned(), "{\"envelope\":\"A\"}".to_owned())], + &ctx, + false, + ) + .expect("first push"); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &store, + &[( + "app_config_staging".to_owned(), + "{\"envelope\":\"B\"}".to_owned(), + )], + &ctx, + false, + ) + .expect("second push (sibling key)"); + + let raw = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = raw.parse().expect("parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents after sibling push"); + let app_config = contents + .get("app_config") + .and_then(toml_edit::Item::as_str) + .expect("default key must survive sibling push"); + assert_eq!( + app_config, "{\"envelope\":\"A\"}", + "default key value: {raw}" + ); + let staging = contents + .get("app_config_staging") + .and_then(toml_edit::Item::as_str) + .expect("staging key must be present"); + assert_eq!(staging, "{\"envelope\":\"B\"}", "staging key value: {raw}"); + } + + #[cfg(unix)] + #[test] + fn push_config_entries_local_writes_literal_dotted_chunk_keys() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("write"); + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + false, + ) + .expect("local push must succeed"); + + let after = fs::read_to_string(&fastly_toml).expect("read back"); + // Chunk keys contain '.' and must appear as quoted string keys, + // not as TOML nested tables (which would look like [table.sub]). + assert!( + after.contains(".__edgezero_chunks."), + "chunk keys written to fastly.toml: {after}" + ); + // Parse with toml_edit and confirm chunk keys are string-keyed entries. + let doc: toml_edit::DocumentMut = after.parse().expect("must parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .expect("contents table must exist"); + // At least one chunk key must be present as a string value (not a table). + let has_chunk_string = contents.as_table().is_some_and(|tbl| { + tbl.iter() + .any(|(key, val)| key.contains(".__edgezero_chunks.") && val.as_value().is_some()) + }); + assert!( + has_chunk_string, + "chunk keys must be literal string-valued entries, not nested tables: {after}" + ); + } + + #[cfg(unix)] + #[test] + fn push_config_entries_local_dry_run_reports_chunking_and_does_not_edit_fastly_toml() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + let original = "name = \"demo\"\n"; + fs::write(&fastly_toml, original).expect("write"); + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + let out = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &entries, + &AdapterPushContext::new(), + true, // dry_run + ) + .expect("local dry-run must not error"); + + // File must be untouched. + let after = fs::read_to_string(&fastly_toml).expect("read back"); + assert_eq!(after, original, "dry-run must not edit fastly.toml"); + + // Output must describe chunking intent. + let combined = out.join("\n"); + assert!( + combined.contains("would set") && combined.contains("chunked"), + "must report chunked intent: {combined}" + ); + } + + // ---------- chunked read integration tests ---------- + + #[cfg(unix)] + #[test] + fn read_config_entry_resolves_direct_value_unchanged() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + + let envelope = BlobEnvelope::new(json!({"hello": "world"}), "2026-06-22T00:00:00Z".into()); + let json_str = serde_json::to_string(&envelope).unwrap(); + let item_json = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(&json_str).unwrap() + ); + let fake = fake_fastly_returning(&item_json, "", 0); + let _path = PathPrepend::new(fake.path()); + + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ) + .expect("read must succeed"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present"); + }; + assert_eq!(value, json_str, "direct envelope passes through unchanged"); + } + + #[cfg(unix)] + #[test] + fn read_config_entry_reconstructs_chunked_envelope() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); + let (_, pointer_json) = physical.last().unwrap(); + // Build a key→response map for every physical entry. + let mut key_responses: Vec<(String, String)> = Vec::new(); + for (pk, pv) in &physical { + let resp = format!(r#"{{"item_value":{}}}"#, serde_json::to_string(pv).unwrap()); + key_responses.push((pk.clone(), resp)); + } + // The root key should return the pointer. + let ptr_resp = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(pointer_json).unwrap() + ); + key_responses.push((TEST_CONFIG_ID.to_owned(), ptr_resp)); + + let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); + let _path = PathPrepend::new(fake.path()); + + let result = FastlyCliAdapter + .read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + TEST_CONFIG_ID, + &AdapterPushContext::new(), + ) + .expect("chunked read must succeed"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present"); + }; + assert_eq!( + value, envelope, + "reconstructed envelope must equal original" + ); } + #[cfg(unix)] #[test] - fn push_entries_with_committer_zero_entries_is_ok() { - let pushed = push_entries_with_committer(&[], |_, _| Ok(())).expect("empty is fine"); - assert_eq!(pushed, 0); + fn read_config_entry_reports_corrupt_on_a_confirmed_absent_chunk() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); + let (_, pointer_json) = physical.last().unwrap(); + // Only provide the root pointer; omit chunk responses so the chunk fetch + // gets a CLEAN not-found (`Error: item not found`, no operational marker). + let ptr_resp = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(pointer_json).unwrap() + ); + let key_responses = vec![(TEST_CONFIG_ID.to_owned(), ptr_resp)]; + let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); + let _path = PathPrepend::new(fake.path()); + + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + TEST_CONFIG_ID, + &AdapterPushContext::new(), + ); + // The chunk describe fails, and the complete store listing (which holds + // only the root pointer) CONFIRMS the chunk is absent. The blob spec makes + // persistent chunk loss REPAIRABLE by re-pushing, so the read reports + // `Corrupt` (a push overwrites to repair), NOT a hard error -- otherwise + // `config push` could never fix it. Absence is confirmed by the listing, + // never by the describe 404 alone, so a proxy/auth failure (where the + // listing also fails, or shows the chunk present) stays a hard error. + assert!( + matches!(result, Ok(ReadConfigEntry::Corrupt(_))), + "a confirmed-absent chunk must be repairable Corrupt, not a hard error" + ); } + #[cfg(unix)] #[test] - fn push_entries_with_committer_failure_surfaces_committed_failed_not_attempted() { - // Mock committer: succeed for first 2 keys, fail at third. - let entries = vec![ - ("k1".to_owned(), "v1".to_owned()), - ("k2".to_owned(), "v2".to_owned()), - ("k3".to_owned(), "v3".to_owned()), - ("k4".to_owned(), "v4".to_owned()), - ("k5".to_owned(), "v5".to_owned()), - ]; - let mut calls: usize = 0; - let err = push_entries_with_committer(&entries, |key, _| { - calls = calls.saturating_add(1); - if key == "k3" { - Err("simulated fastly stderr".to_owned()) - } else { - Ok(()) - } - }) - .expect_err("middle failure must error"); - // Committer was invoked for k1, k2, k3 and stopped. - assert_eq!(calls, 3_usize, "no retries beyond failure point"); - // Error names all three categories. - assert!(err.contains("k1") && err.contains("k2"), "committed: {err}"); + fn read_config_entry_reports_corrupt_on_chunk_hash_mismatch() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); + let (_, pointer_json) = physical.last().unwrap(); + let mut key_responses: Vec<(String, String)> = Vec::new(); + // Corrupt first chunk's content. + let (first_chunk_key, first_chunk_val) = &physical[0]; + let corrupted: String = first_chunk_val.chars().map(|_| 'Z').collect(); + let corrupt_resp = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(&corrupted).unwrap() + ); + key_responses.push((first_chunk_key.clone(), corrupt_resp)); + // Remaining chunks as normal. + for (pk, pv) in physical + .iter() + .take(physical.len().saturating_sub(1)) + .skip(1) + { + key_responses.push(( + pk.clone(), + format!(r#"{{"item_value":{}}}"#, serde_json::to_string(pv).unwrap()), + )); + } + key_responses.push(( + TEST_CONFIG_ID.to_owned(), + format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(pointer_json).unwrap() + ), + )); + let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); + let _path = PathPrepend::new(fake.path()); + + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + TEST_CONFIG_ID, + &AdapterPushContext::new(), + ); + // A chunk-hash mismatch at an EXISTING entry is corrupt stored state the + // push repairs by overwriting, so the CLI read reports `Corrupt`. (The + // RUNTIME path keeps a hash mismatch as Internal — see config_store.rs.) assert!( - err.contains("Failed: `k3`"), - "failed entry named exactly: {err}" + matches!(result, Ok(ReadConfigEntry::Corrupt(_))), + "a chunk-hash mismatch at an existing entry must be Corrupt (repairable), not an error" + ); + } + + #[cfg(unix)] + #[test] + fn read_config_entry_reports_corrupt_for_malformed_pointer() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + // Root value ANNOUNCES our chunk-pointer kind but is malformed. The + // `describe` SUCCEEDS (the entry exists), so a resolve failure is CORRUPT + // stored state, not an IO error: the read must report `Corrupt` so a push + // can overwrite it (in-band repair), NOT hard-error and block recovery. + let bad_json = r#"{"edgezero_kind":"fastly_config_chunks","some_field":"x"}"#; + let item_json = format!( + r#"{{"item_value":{}}}"#, + serde_json::to_string(bad_json).unwrap() + ); + let fake = fake_fastly_returning(&item_json, "", 0); + let _path = PathPrepend::new(fake.path()); + + let result = FastlyCliAdapter.read_config_entry( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), ); assert!( - err.contains("k4") && err.contains("k5"), - "not-attempted: {err}" + matches!(result, Ok(ReadConfigEntry::Corrupt(_))), + "a malformed pointer at an EXISTING entry must be Corrupt (repairable), not an error" ); - assert!(err.contains("simulated fastly stderr"), "inner err: {err}"); - // Counts are sane. + } + + /// The read taxonomy distinguishes repairable corruption from cases a push + /// must NOT overwrite: an infrastructure fetch failure (incomplete read) and + /// an unknown/future format both stay hard errors, while a malformed direct + /// value, a SHA mismatch, and a resolve error are repairable `Corrupt`. + #[test] + fn classify_resolved_read_separates_corrupt_from_infra_and_unknown() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let envelope = serde_json::to_string(&BlobEnvelope::new( + json!({ "k": "v" }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .expect("envelope"); + + // A valid envelope resolves to Present. + assert!(matches!( + classify_resolved_read(Ok(envelope.clone()), &envelope, false), + Ok(ReadConfigEntry::Present(_)) + )); + + // A direct value with a wrong `sha256` is NOT a valid envelope -> Corrupt. + let mut tampered_value: serde_json::Value = serde_json::from_str(&envelope).expect("parse"); + tampered_value["sha256"] = json!("0".repeat(64)); + let tampered = tampered_value.to_string(); + assert!(matches!( + classify_resolved_read(Ok(tampered.clone()), &tampered, false), + Ok(ReadConfigEntry::Corrupt(_)) + )); + + // Invalid JSON / a plain non-envelope value -> Corrupt (not Present). + assert!(matches!( + classify_resolved_read(Ok("not an envelope".to_owned()), "not an envelope", false), + Ok(ReadConfigEntry::Corrupt(_)) + )); + + // A resolve error caused by an INFRASTRUCTURE fetch failure stays a HARD + // error: the read was incomplete, so a push must not overwrite. + let infra = classify_resolved_read( + Err(ResolveFailure::Corrupt("boom".to_owned())), + "{\"edgezero_kind\":\"fastly_config_chunks\"}", + true, + ); + assert!( + infra + .as_ref() + .is_err_and(|err| err.contains("not fully read")), + "an infra fetch failure must be a hard error, not Corrupt" + ); + + // A value announcing an UNKNOWN/future kind is a HARD error (upgrade CLI), + // never offered for overwrite. + let unknown = classify_resolved_read( + Err(ResolveFailure::FutureFormat("x".to_owned())), + r#"{"edgezero_kind":"fastly_config_chunks_v2"}"#, + false, + ); + assert!( + unknown + .as_ref() + .is_err_and(|err| err.contains("does not recognise")), + "an unknown/future kind must be a hard error" + ); + + // A NEWER INNER envelope (a valid v1 pointer wrapping a v2 envelope) is + // only knowable AFTER reassembly: the raw value is a healthy v1 pointer, + // so the typed `FutureFormat` failure is the ONLY signal. It must be a + // hard error, never repairable Corrupt -- a downgrade push must not + // overwrite it. + let inner_future = classify_resolved_read( + Err(ResolveFailure::FutureFormat( + "newer inner envelope".to_owned(), + )), + r#"{"edgezero_kind":"fastly_config_chunks","version":1,"chunks":[]}"#, + false, + ); + assert!( + inner_future + .as_ref() + .is_err_and(|err| err.contains("UPGRADE")), + "a future INNER envelope (typed FutureFormat) must be a hard error, not Corrupt" + ); + + // An ordinary resolve error (bad/missing chunk) is repairable Corrupt. + assert!(matches!( + classify_resolved_read( + Err(ResolveFailure::Corrupt("bad chunk".to_owned())), + r#"{"edgezero_kind":"fastly_config_chunks","chunks":[]}"#, + false + ), + Ok(ReadConfigEntry::Corrupt(_)) + )); + + // A future ENVELOPE version (passed through as Ok) is a hard error, NOT + // the repairable Corrupt -- an older CLI must not overwrite it. + let mut v2_value: serde_json::Value = serde_json::from_str(&envelope).expect("parse"); + v2_value["version"] = json!(2_u32); + let v2_env = v2_value.to_string(); + assert!( + classify_resolved_read(Ok(v2_env.clone()), &v2_env, false) + .as_ref() + .is_err_and(|err| err.contains("UPGRADE")), + "a v2 direct envelope must be a hard error, not Corrupt" + ); + + // A future POINTER version (resolve fails on the version check) is a hard + // error too -- the pointer kind is ours, but the version is newer. + let v2_ptr = r#"{"edgezero_kind":"fastly_config_chunks","version":2,"chunks":[]}"#; + assert!( + classify_resolved_read( + Err(ResolveFailure::FutureFormat( + "unsupported version".to_owned() + )), + v2_ptr, + false + ) + .as_ref() + .is_err_and(|err| err.contains("UPGRADE")), + "a v2 pointer must be a hard error, not Corrupt" + ); + } + + // ---------- local read integration tests ---------- + + #[test] + fn read_config_entry_local_resolves_direct_value() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + + let envelope = BlobEnvelope::new(json!({"x": 1_i32}), "2026-06-22T00:00:00Z".into()); + let json_str = serde_json::to_string(&envelope).unwrap(); + // Write directly as a single entry (not via push_config_entries_local so we + // control the exact TOML content). + write_fastly_local_config_store( + &fastly_toml, + TEST_CONFIG_ID, + &[("cfg".to_owned(), json_str.clone())], + &[], + ) + .expect("write"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ) + .expect("local read must succeed"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present"); + }; + assert_eq!(value, json_str, "direct envelope passes through unchanged"); + } + + #[test] + fn read_config_entry_local_reconstructs_chunked_envelope() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); + // Write all physical entries (chunks + pointer) to the local store. + write_fastly_local_config_store(&fastly_toml, TEST_CONFIG_ID, &physical, &[]) + .expect("write"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + TEST_CONFIG_ID, + &AdapterPushContext::new(), + ) + .expect("local chunked read must succeed"); + let ReadConfigEntry::Present(value) = result else { + panic!("expected Present"); + }; + assert_eq!( + value, envelope, + "reconstructed envelope must equal original" + ); + } + + /// a corrupt/invalid prior value must NOT abort the + /// local read, or the CLI push aborts on the diff read before the writer's + /// fail-soft ("overwrite, warn, prune nothing") can repair the state. + /// `config push` is how an operator recovers, so the read reports `Corrupt` + /// ("cannot diff; will overwrite") and lets the write proceed. + #[test] + fn read_config_entry_local_degrades_corrupt_prior_to_corrupt() { + use crate::chunked_config::{CHUNK_KEY_INFIX, POINTER_KIND}; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + + // A pointer-KIND value that is invalid (missing the chunks it needs). + // The resolver would error on this; the local read must NOT propagate + // that as `Err`. + let broken_pointer = format!( + r#"{{"edgezero_kind":"{POINTER_KIND}","version":1,"chunks":[{{"key":"cfg{CHUNK_KEY_INFIX}{sha}.0","len":10,"sha256":"x"}}],"data_sha256":"","envelope_len":10,"envelope_sha256":"{sha}"}}"#, + sha = "a".repeat(64), + ); + write_fastly_local_config_store( + &fastly_toml, + TEST_CONFIG_ID, + &[("cfg".to_owned(), broken_pointer)], + &[], + ) + .expect("write"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ) + .expect("a corrupt local prior must NOT abort the read"); assert!( - err.contains("committing 2 of 5 entries"), - "committed/total count: {err}" + matches!(result, ReadConfigEntry::Corrupt(_)), + "a corrupt prior value must degrade to Corrupt so the push can overwrite it" ); } + /// A `contents` that is not a table (a scalar or array) is malformed store + /// state. It must degrade to `Unsupported`, not fall through to `MissingKey` + /// (which would render an inaccurate "all values added" diff). #[test] - fn push_entries_with_committer_first_entry_failure_reports_zero_committed() { - let entries = vec![ - ("only".to_owned(), "val".to_owned()), - ("never".to_owned(), "tried".to_owned()), - ]; - let err = push_entries_with_committer(&entries, |_, _| Err("nope".to_owned())) - .expect_err("first-entry failure"); - assert!(err.contains("committing 0 of 2"), "zero committed: {err}"); - assert!( - err.contains("Failed: `only`"), - "first-entry failure named: {err}" - ); + fn read_config_entry_local_non_table_contents_is_unsupported() { + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write( + &fastly_toml, + format!("[local_server.config_stores.{TEST_CONFIG_ID}]\ncontents = 42\n"), + ) + .expect("seed"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ) + .expect("a non-table contents must NOT abort the read"); assert!( - err.contains("never"), - "second entry as not-attempted: {err}" + matches!(result, ReadConfigEntry::Unsupported(_)), + "a non-table `contents` must degrade to Unsupported, not MissingKey" ); } + /// A malformed PARENT table (`local_server` etc. as a scalar) must degrade to + /// Unsupported, not collapse to `MissingStore`'s "all values added" diff. #[test] - fn push_entries_with_committer_last_entry_failure_reports_n_minus_one_committed() { - let entries = vec![ - ("a".to_owned(), "1".to_owned()), - ("b".to_owned(), "2".to_owned()), - ("c".to_owned(), "3".to_owned()), - ]; - let err = push_entries_with_committer(&entries, |key, _| { - if key == "c" { - Err("late failure".to_owned()) - } else { - Ok(()) - } - }) - .expect_err("last-entry failure"); - assert!(err.contains("committing 2 of 3"), "n-1 committed: {err}"); + fn read_config_entry_local_non_table_parent_is_unsupported() { + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + // `local_server` is a scalar, not a table. + fs::write(&fastly_toml, "local_server = 42\n").expect("seed"); + + let result = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + "cfg", + &AdapterPushContext::new(), + ) + .expect("a non-table parent must NOT abort the read"); assert!( - err.contains("the remaining 0 entries"), - "zero not-attempted when last fails: {err}" + matches!(result, ReadConfigEntry::Unsupported(_)), + "a non-table parent must degrade to Unsupported, not MissingStore" ); } - // ---------- looks_like_already_exists ---------- - + /// Spec 12.3 + 9.3: a second oversized push must converge the + /// runtime on the NEW envelope — chunk keys are content-addressed + /// by the full-envelope SHA, so push B writes a new chunk-set and + /// installs a new root pointer. + /// + /// The local fastly.toml writer upserts per-key (so a sibling + /// `--key app_config_staging` push leaves `app_config` intact per + /// spec 12.7). Within the SAME root key, GC on re-push prunes the + /// prior generation: after envelope B's push, envelope A's chunks — + /// now unreferenced by the `app_config` pointer — are removed from + /// the contents table. A read after push B follows the active + /// pointer and reconstructs envelope B, not A. + #[cfg(unix)] #[test] - fn looks_like_already_exists_recognises_common_phrasings() { - // Real-shaped fastly CLI error strings (paraphrased; the - // CLI varies across versions). Each must be detected so - // create_fastly_store can treat it as idempotent success. - assert!(looks_like_already_exists( - "Error: a kv-store with that name already exists", - "kv", - )); - assert!(looks_like_already_exists( - "ERROR: Conflict (409): duplicate kv_store name", - "kv", - )); - assert!(looks_like_already_exists( - "A config-store with this name already exists", - "config", - )); - // Spaced form: some fastly CLI versions emit prose - // ("kv store"); accept it alongside the punctuated forms. - assert!(looks_like_already_exists( - "Error: kv store conflict: name already in use", - "kv", - )); - } + #[expect( + clippy::too_many_lines, + reason = "linear test scenario: push A, inspect, push B, inspect, read; splitting would obscure the chunk-set comparison" + )] + fn second_oversized_push_converges_runtime_on_new_envelope() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); - #[test] - fn looks_like_already_exists_rejects_unrelated_errors() { - assert!(!looks_like_already_exists( - "Error: unauthenticated; run `fastly profile create`", - "kv", - )); - assert!(!looks_like_already_exists( - "Error: network unreachable", - "kv", - )); - assert!(!looks_like_already_exists("", "kv")); - } + // First push: envelope A. Records the chunk-key set so we can + // confirm they are pruned by the second push's GC. + let envelope_a = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), envelope_a.clone())], + &AdapterPushContext::new(), + false, + ) + .expect("first push must succeed"); - #[test] - fn looks_like_already_exists_rejects_unrelated_conflict_errors() { - // The earlier wider heuristic swallowed ANY stderr - // containing "conflict" or "already exists", which would - // misread an unrelated 409 from a different fastly - // subcommand (e.g. a service-version conflict during a - // parallel deploy) as idempotent store-create success. - // Now we require the kind context too, so unrelated - // conflicts surface as failures. + let after_a = fs::read_to_string(&fastly_toml).expect("read"); + let doc_a: toml_edit::DocumentMut = after_a.parse().expect("parse"); + let contents_a = doc_a + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents table after push A"); + let chunks_a: Vec = contents_a + .iter() + .map(|(key, _)| key.to_owned()) + .filter(|key| key.contains(".__edgezero_chunks.")) + .collect(); assert!( - !looks_like_already_exists( - "Error: 409 Conflict on /service/abc/version/42 -- already exists", - "kv", - ), - "service-version conflict must NOT be misread as kv-store idempotency" + !chunks_a.is_empty(), + "push A must have produced chunk entries: {after_a}" ); + + // Second push: a DIFFERENT oversized envelope B. The + // content-addressed chunk keys must shift to B's sha; GC then + // prunes the old A-chunks. Build envelope B with a distinct + // payload key so its SHA differs from A's even at the same + // total length. + let envelope_b = { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let data = json!({ "alt": "x".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:01Z".to_owned())) + .expect("envelope B serialises") + }; + assert_ne!(envelope_a, envelope_b, "test fixtures must differ"); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), envelope_b.clone())], + &AdapterPushContext::new(), + false, + ) + .expect("second push must succeed"); + + let after_b = fs::read_to_string(&fastly_toml).expect("read"); + let doc_b: toml_edit::DocumentMut = after_b.parse().expect("parse"); + let contents_b = doc_b + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents table after push B"); + let chunks_b: Vec = contents_b + .iter() + .map(|(key, _)| key.to_owned()) + .filter(|key| key.contains(".__edgezero_chunks.")) + .collect(); assert!( - !looks_like_already_exists( - "Error: invalid duplicate request; check name resolution", - "kv", - ), - "unrelated `duplicate ... name` AND-match must NOT trigger" + !chunks_b.is_empty(), + "push B must have produced chunk entries: {after_b}" ); - // And the kind must match: a config-store conflict must - // not look-like-already-exists for a kv-store create call. + + // Chunk keys are content-addressed by envelope SHA, so the B + // push installs a fresh chunk-set whose keys are all distinct + // from A's. GC on re-push prunes the now-unreferenced A-chunks. + let new_b_chunks: Vec<&String> = chunks_b + .iter() + .filter(|key| !chunks_a.contains(*key)) + .collect(); assert!( - !looks_like_already_exists("Error: a config-store with that name already exists", "kv",), - "wrong-kind conflict must NOT trigger" + !new_b_chunks.is_empty(), + "push B must have added at least one new content-addressed chunk: A-set={chunks_a:?} B-set={chunks_b:?}" + ); + // Old A-chunks are pruned: GC deletes the prior generation the + // old pointer referenced once B's pointer supersedes it. + for chunk_key in &chunks_a { + assert!( + !chunks_b.contains(chunk_key), + "old A-chunk `{chunk_key}` must be pruned from the local table after push B; B-set={chunks_b:?}" + ); + } + + // Runtime-correctness property: a fresh read after push B + // reconstructs envelope B (NOT envelope A). + let read = FastlyCliAdapter + .read_config_entry_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + TEST_CONFIG_ID, + &AdapterPushContext::new(), + ) + .expect("local read after push B"); + let ReadConfigEntry::Present(value) = read else { + panic!("expected Present after push B"); + }; + assert_eq!( + value, envelope_b, + "read after second push must reconstruct envelope B, not A" + ); + assert_ne!( + value, envelope_a, + "old envelope A's chunks must be inert -- read must NOT return A" ); } - // ---------- setup_block_present ---------- - + // ---------- config gc (operator-invoked reclamation) ---------- + + #[cfg(unix)] + fn run_gc(dir: &Path, older_than_secs: u64, dry_run: bool) -> Result, String> { + FastlyCliAdapter.gc_config_entries( + dir, + None, + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &AdapterPushContext::new(), + older_than_secs, + dry_run, + ) + } + + /// gc never deletes a chunk the LIVE root pointer references, however old. + #[cfg(unix)] #[test] - fn setup_block_present_true_when_table_exists() { + fn gc_never_deletes_live_chunks() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write( - &path, - "name = \"demo\"\n[setup.kv_stores.sessions]\n[local_server.kv_stores.sessions]\n", - ) - .expect("write"); - assert!(setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let live_chunks = chunk_keys_of(TEST_CONFIG_ID, &live); + // The live generation is ANCIENT, but it is referenced by the root. + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = run_gc(dir.path(), 1, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &live_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "live chunk `{key}` must never be reclaimed; log:\n{log}\nout: {out:?}" + ); + } } + /// gc reclaims unreferenced chunks older than the operator's threshold. + #[cfg(unix)] #[test] - fn setup_block_present_false_when_id_missing() { + fn gc_reclaims_unreferenced_chunks_older_than_threshold() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n[setup.kv_stores.other]\n").expect("write"); - assert!(!setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let live_chunks = chunk_keys_of(TEST_CONFIG_ID, &live); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + + // The live config has been stable for 2 days; the operator asserts a 1-day + // window. So everything superseded (<= when live went live, i.e. >= 2 + // days ago) is safely reclaimable. + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); // a week old + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + for key in &dead_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "orphan `{key}` older than the threshold must be reclaimed; out: {out:?}" + ); + } + for key in &live_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "live chunk `{key}` must survive" + ); + } } + /// The soundness test (design-3 counterexample): a root whose + /// current config was deployed seconds ago must NOT have its prior generation + /// reclaimed, even if that generation's chunks are ANCIENT. The clock is the + /// live config's age, not the orphan chunk's own creation time. + #[cfg(unix)] #[test] - fn setup_block_present_false_for_missing_file() { + fn gc_protects_recently_superseded_generation_with_old_chunks() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("does-not-exist.toml"); - assert!(!setup_block_present(&path, "kv", TEST_KV_ID).expect("probe")); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let prior = gen_envelope("prior"); + let prior_chunks = chunk_keys_of(TEST_CONFIG_ID, &prior); + + // Live config went live 30s ago; the prior generation's chunks are a year + // old but were superseded only 30s ago -> POPs may still serve them. + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 30)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 30)); + listing.extend(listed_generation(TEST_CONFIG_ID, &prior, 31_536_000)); // ~1 year + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // Even a generous 1-day threshold must NOT delete the prior generation, + // because the live config has only been stable for 30 seconds. + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &prior_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a generation superseded 30s ago must be retained despite old chunks: `{key}`; log:\n{log}" + ); + } } + /// a live root whose pointer drops its + /// last chunk ref AND restates `envelope_len` as the remaining sum passes + /// every metadata check. The dropped chunk is then absent from the live set + /// and looks like a deletable orphan -- while the config still needs it. + /// + /// Guards the PLANNER's content verification (a unit test on + /// `gc_verify_generation` alone does not prove the planner calls it). + #[cfg(unix)] #[test] - fn setup_block_present_true_when_only_setup_exists() { - // Post-F6 (PR #269 round 2): `setup_block_present` only - // checks `[setup._stores.]`. The pre-fix check - // ALSO required `[local_server._stores.]`, but - // writing an empty `[local_server.*]` table didn't match - // fastly's local-server schema (config-stores need - // `format` + contents, kv/secret stores need a JSON file - // or `{key, data}` entries). Local-server seeding moved - // to `config push --adapter fastly --local`, so probe - // only cares about `[setup]` now. + fn gc_fails_closed_when_a_live_pointer_underreports_its_chunks() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let only_setup = dir.path().join("only_setup.toml"); - fs::write(&only_setup, "name = \"demo\"\n[setup.kv_stores.sessions]\n").expect("write"); + let oplog = dir.path().join("ops.log"); + + // Padded so the generation is >= 3 chunks: this case needs a ref to + // drop that still leaves a plausible multi-chunk set behind. + let live = gen_envelope_padded("live", 20_000); + let (chunks, pointer_json) = chunked_parts(TEST_CONFIG_ID, &live); + assert!(chunks.len() >= 3, "need >= 3 chunks for this case"); + + // Doctor the pointer: drop the last ref, restate envelope_len to match + // the survivors. Generation, indexes, per-chunk lens and the sum all + // still agree -- only the CONTENT does not. + let mut pointer: serde_json::Value = serde_json::from_str(&pointer_json).expect("parse"); + let refs = pointer + .get_mut("chunks") + .and_then(serde_json::Value::as_array_mut) + .expect("chunks array"); + refs.pop().expect("drop the last chunk ref"); + let surviving_len: u64 = refs + .iter() + .filter_map(|chunk| chunk.get("len").and_then(serde_json::Value::as_u64)) + .sum(); + pointer["envelope_len"] = serde_json::json!(surviving_len); + let doctored = serde_json::to_string(&pointer).expect("serialise"); + + // The store still physically holds ALL the chunks, including the one the + // doctored pointer no longer names. + let orphaned_by_omission = chunks.last().expect("last chunk").0.clone(); + let stamp = stamp_secs_ago(999_999); + let mut listing = vec![(TEST_CONFIG_ID.to_owned(), stamp.clone(), doctored)]; + for (key, value) in chunks { + listing.push((key, stamp.clone(), value)); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 1, false).expect_err("must fail closed"); assert!( - setup_block_present(&only_setup, "kv", TEST_KV_ID).expect("probe"), - "[setup.*] alone is now sufficient: {only_setup:?}" + err.contains("does not reconstruct the envelope it claims"), + "expected a content-address mismatch on the live pointer, got: {err}" ); - - let only_local = dir.path().join("only_local.toml"); - fs::write( - &only_local, - "name = \"demo\"\n[local_server.kv_stores.sessions]\n", - ) - .expect("write"); assert!( - !setup_block_present(&only_local, "kv", TEST_KV_ID).expect("probe"), - "[local_server.*] alone is NOT a provisioned-setup signal" + !oplog_has(&oplog, &format!("delete {orphaned_by_omission}")), + "a chunk the live config still needs must never be deleted because its pointer \ + under-reported it: `{orphaned_by_omission}`" ); } - // ---------- append_fastly_setup ---------- - + /// a LONE entry whose value hashes to the generation + /// its own key names would otherwise "prove" itself and be deleted. But our + /// writer never emits a one-chunk generation (an oversized envelope always + /// splits into >= 2), so a group of one is never ours -- it is a root-like + /// value sitting at a chunk-shaped key. This is the case a pure hash check + /// cannot catch on its own. + #[cfg(unix)] #[test] - fn append_fastly_setup_creates_setup_table_in_minimal_file() { + fn gc_never_reclaims_a_lone_self_consistent_chunk() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); - append_fastly_setup(&path, "kv", TEST_KV_ID).expect("append"); - let after = fs::read_to_string(&path).expect("read back"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + + // A complete envelope stored at a chunk-shaped key whose generation IS + // that envelope's own SHA -- so it reassembles to its content-address. + let squatter_value = gen_envelope("someones-real-config"); + let self_sha = sha256_hex(squatter_value.as_bytes()); + let squatter_key = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{self_sha}.0"); + listing.push(( + squatter_key.clone(), + stamp_secs_ago(31_536_000), + squatter_value, + )); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); assert!( - after.contains("[setup.kv_stores.sessions]"), - "setup table added: {after}" + !oplog_has(&oplog, &format!("delete {squatter_key}")), + "a one-chunk 'generation' is never something this writer emitted, so it must not be \ + reclaimed even though it hashes to its own key: `{squatter_key}`; log:\n{log}" ); - // Post-F6: no `[local_server.*]` write — that empty stanza - // didn't satisfy fastly's local-server schema and made - // `fastly compute serve` error or skip the store. Local- - // server seeding is now `config push --adapter fastly - // --local`'s job. + } + + /// a delete that fails on a generation's FIRST key has + /// an UNKNOWN outcome -- Fastly may have committed it before returning an + /// error. called this "whole and retryable", which is unsound: if the + /// failed delete did commit, a re-run finds a fragment. The honest report is + /// a NOTE that the outcome is uncertain, NOT a clean-retry promise. We still + /// stop the generation so a CONFIRMED partial delete cannot happen. + #[cfg(unix)] + #[test] + fn gc_first_delete_failure_is_reported_as_uncertain_not_clean_retry() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + assert!(dead_chunks.len() >= 2, "need a multi-chunk generation"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + // The FIRST chunk of the doomed generation fails to delete. + let fake = fake_fastly_gc( + TEST_CONFIG_ID, + &[], + &listing, + Some(&dead_chunks[0]), + false, + &oplog, + ); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("a failed delete is a failure"); assert!( - !after.contains("[local_server.kv_stores.sessions]"), - "[local_server.*] empty table no longer written by provision: {after}" + err.contains("unknown outcome"), + "a failed delete's outcome is unknown and must be reported as such: {err}" ); assert!( - after.contains("name = \"demo\""), - "preserved original keys: {after}" + !err.contains("will retry them"), + "the disproven clean-retry promise must be gone: {err}" ); + // The siblings must NOT have been ATTEMPTED -- stopping is what prevents a + // CONFIRMED partial delete. + for key in dead_chunks.iter().skip(1) { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "after the first failure the generation must be left alone: `{key}`" + ); + } } + /// the stateful case. A remote delete that COMMITS but + /// still reports failure leaves a real fragment. On the SECOND run that + /// missing key makes the generation unprovable, so it must be reported as + /// left-untouched (surfaced), never silently dropped. + #[cfg(unix)] #[test] - fn append_fastly_setup_appends_alongside_existing_kind_tables() { + fn gc_committed_but_failed_delete_surfaces_as_unprovable_next_run() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "[setup.kv_stores.cache]\n").expect("write"); - append_fastly_setup(&path, "kv", TEST_KV_ID).expect("append"); - let after = fs::read_to_string(&path).expect("read back"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + assert!(dead_chunks.len() >= 2, "need a multi-chunk generation"); + + // SECOND run's world: the first chunk's delete committed last time, so it + // is gone. The generation is now a fragment. + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + let mut dead_gen = listed_generation(TEST_CONFIG_ID, &dead, 604_800); + let survivor = dead_gen[1].0.clone(); + dead_gen.remove(0); // the committed-deleted chunk is absent now + listing.extend(dead_gen); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = run_gc(dir.path(), 86_400, false).expect("gc succeeds"); assert!( - after.contains("[setup.kv_stores.cache]"), - "existing entry kept: {after}" + !oplog_has(&oplog, &format!("delete {survivor}")), + "an unprovable fragment survivor must not be deleted: `{survivor}`" ); assert!( - after.contains("[setup.kv_stores.sessions]"), - "new entry added: {after}" + out.iter() + .any(|line| line.contains("not byte-identical to what this writer would produce")), + "the surviving fragment must be SURFACED as left-untouched, not silently dropped: {out:?}" ); } + /// if a delete fails PART-WAY through a generation, the + /// survivors are an incomplete generation that `prove_generation` can never + /// verify again -- so `gc` will never reclaim them. Claiming "re-run to + /// retry" there was false. Say plainly that recovery is manual. + #[cfg(unix)] #[test] - fn append_fastly_setup_is_idempotent_on_duplicate_id() { + fn gc_reports_stranded_survivors_as_manual_recovery() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "[setup.kv_stores.sessions]\nfoo = \"keep\"\n").expect("write"); - append_fastly_setup(&path, "kv", TEST_KV_ID).expect("idempotent append"); - let after = fs::read_to_string(&path).expect("read back"); + let oplog = dir.path().join("ops.log"); + + // Padded to >= 3 chunks so a mid-generation failure leaves survivors. + let live = gen_envelope("live"); + let dead = gen_envelope_padded("dead", 20_000); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + assert!(dead_chunks.len() >= 3, "need >= 3 chunks"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + // The SECOND chunk fails: the first is already gone by then. + let fake = fake_fastly_gc( + TEST_CONFIG_ID, + &[], + &listing, + Some(&dead_chunks[1]), + false, + &oplog, + ); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("a failed delete is a failure"); assert!( - after.contains("foo = \"keep\""), - "did not stomp existing key: {after}" + err.contains("INCOMPLETE generation") && err.contains("re-running will NOT help"), + "a stranded fragment must not be described as retryable: {err}" + ); + // It must name the survivors and how to remove them by hand. + for key in dead_chunks.iter().skip(2) { + assert!( + err.contains(key.as_str()), + "the operator needs the exact surviving keys: `{key}` missing from: {err}" + ); + } + assert!( + err.contains("fastly config-store-entry delete"), + "give the operator the recovery command: {err}" ); + // And we stopped rather than deleting the rest. + for key in dead_chunks.iter().skip(2) { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "deletion must stop at the first failure in a generation: `{key}`" + ); + } } + /// root keys are free-form, so a chunk key can hold + /// shell metacharacters. Manual-recovery commands must render them so that + /// pasting cannot execute or misparse -- single-quoted, with embedded quotes + /// escaped. #[test] - fn append_fastly_setup_creates_file_when_missing() { - let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - // Note: no fs::write — file starts absent. - append_fastly_setup(&path, "config", TEST_CONFIG_ID).expect("create"); - let after = fs::read_to_string(&path).expect("read back"); - assert!(after.contains("[setup.config_stores.app_config]")); + fn recovery_commands_are_shell_safe() { + // A key crafted to run `id` and to break argument parsing if unquoted. + let hostile = "app$(id).__edgezero_chunks.'; rm -rf /'.0".to_owned(); + let keys = [hostile.clone()]; + let rendered = recovery_commands("store-abc", &keys); + + // The dangerous substring is not sitting there unquoted. assert!( - !after.contains("[local_server.config_stores.app_config]"), - "[local_server.*] no longer written by provision: {after}" + !rendered.contains("$(id)") || rendered.contains("'app$(id)"), + "shell-active text must be inside single quotes: {rendered}" + ); + // Every embedded single quote is closed-escaped-reopened, so no quote + // context leaks. + assert!( + rendered.contains(r"'\''"), + "embedded single quotes must be escaped as '\\'': {rendered}" + ); + // Sanity: what a POSIX shell would parse back out of our --key argument + // is EXACTLY the original key (round-trip through `sh`). + let key_arg = rendered + .split("--key=") + .nth(1) + .and_then(|rest| rest.split(" --auto-yes").next()) + .expect("a --key argument"); + let out = Command::new("sh") + .arg("-c") + .arg(format!("printf '%s' {key_arg}")) + .output() + .expect("run sh"); + assert_eq!( + String::from_utf8_lossy(&out.stdout), + hostile, + "the shell must parse the quoted argument back to the exact key" ); } + /// a valid DIRECT envelope at a chunk-shaped key is a + /// runtime-readable root, but round 9 only protected POINTER values there. + /// + /// Construction: pad a small valid envelope with trailing JSON whitespace + /// past the entry limit. The writer chunks it; chunk 0 (the first 7 000 + /// bytes) is the whole envelope plus trailing spaces, which STILL parses and + /// verifies as that envelope. So chunk 0's key holds a valid direct envelope + /// -- a root -- yet the generation round-trips through the writer and passes + /// every proof, so GC deletes chunk 0. + #[cfg(unix)] #[test] - fn append_fastly_setup_preserves_top_comments() { + fn valid_envelope_at_chunk_shaped_key_is_a_protected_root() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write( - &path, - "# managed by hand -- please keep this line\nname = \"demo\"\n", - ) - .expect("write"); - append_fastly_setup(&path, "secret", TEST_SECRET_ID).expect("append"); - let after = fs::read_to_string(&path).expect("read back"); + let oplog = dir.path().join("ops.log"); + + // A small valid envelope + trailing whitespace over the entry limit. + let envelope = BlobEnvelope::new(json!({"k":"v"}), "2026-06-22T00:00:00Z".into()); + let mut padded = serde_json::to_string(&envelope).unwrap(); + padded.push_str(&" ".repeat(8_200)); + let entries = prepare_fastly_config_entries(TEST_CONFIG_ID, &padded).expect("expand"); + assert!(entries.len() >= 3, "need >= 2 chunks + pointer"); + let holder_key = entries[0].0.clone(); + // Sanity: chunk 0's value IS a standalone valid envelope. + let parsed: BlobEnvelope = + serde_json::from_str(&entries[0].1).expect("chunk 0 must parse as an envelope"); + parsed.verify().expect("chunk 0 must verify as an envelope"); + + // Seed the store with the chunk entries only -- NO live pointer refers + // to them, so this generation looks orphaned. Aged old. + let stamp = stamp_secs_ago(604_800); + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + for (key, value) in &entries[..entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp.clone(), value.clone())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + drop(run_gc(dir.path(), 86_400, false)); + let log = fs::read_to_string(&oplog).unwrap_or_default(); assert!( - after.contains("# managed by hand"), - "preserved comment: {after}" + !oplog_has(&oplog, &format!("delete {holder_key}")), + "an entry whose value is a valid direct envelope is a runtime-readable root and must \ + never be deleted, whatever its key looks like: `{holder_key}`; log:\n{log}" ); + // The SIBLING chunks must survive too: protecting the holder drops the + // generation to an incomplete group, which is left unprovable — so + // nothing in this generation is deleted, not just the holder. + for (key, _) in &entries[1..entries.len().saturating_sub(1)] { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a sibling of a protected root must also survive (the group is left \ + unprovable): `{key}`; log:\n{log}" + ); + } } - // ---------- write_fastly_local_config_store (config push --local) ---------- - + /// A self-scoped pointer at a chunk-shaped holder key (its chunks nest the + /// infix twice) must NOT abort store-wide GC: the doubly-nested chunks are + /// recognised as chunks (via the LAST infix), so the holder classifies as a + /// root, its references are counted live, and other roots still reclaim. + #[cfg(unix)] #[test] - fn write_fastly_local_config_store_creates_inline_block_in_minimal_file() { + fn gc_tolerates_a_self_scoped_pointer_at_a_chunk_shaped_root() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); - let entries = vec![ - ("greeting".to_owned(), "hello".to_owned()), - ("service.timeout_ms".to_owned(), "1500".to_owned()), - ]; - write_fastly_local_config_store(&path, TEST_CONFIG_ID, &entries).expect("write"); - let after = fs::read_to_string(&path).expect("read back"); - assert!( - after.contains(&format!("[local_server.config_stores.{TEST_CONFIG_ID}]")), - "store table: {after}" - ); - assert!( - after.contains("format = \"inline-toml\""), - "format field: {after}" - ); - assert!( - after.contains(&format!( - "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" - )), - "contents table: {after}" - ); - assert!(after.contains("greeting = \"hello\""), "key 1: {after}"); + let oplog = dir.path().join("ops.log"); + + // A pointer parked at a chunk-shaped key, with chunks scoped to itself. + let holder_key = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "e".repeat(64)); + let nested = gen_envelope("nested"); + let nested_entries = prepare_fastly_config_entries(&holder_key, &nested).expect("expand"); + let (_, holder_pointer) = nested_entries.last().expect("pointer").clone(); + + // A normal live root, and a normal orphan generation that SHOULD reclaim. + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + let stamp = stamp_secs_ago(604_800); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + listing.push((holder_key.clone(), stamp.clone(), holder_pointer)); + for (key, value) in &nested_entries[..nested_entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp.clone(), value.clone())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // The run must SUCCEED (not abort) and still reclaim the ordinary orphan. + run_gc(dir.path(), 86_400, false).expect("store-wide GC must not abort"); + for key in &dead_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "an ordinary orphan must still be reclaimed despite the self-scoped pointer: `{key}`" + ); + } assert!( - after.contains("\"service.timeout_ms\" = \"1500\""), - "dotted key quoted: {after}" + !oplog_has(&oplog, &format!("delete {holder_key}")), + "the chunk-shaped holder root must never be deleted" ); - assert!(after.contains("name = \"demo\""), "preserved: {after}"); } + /// A nested ORPHAN generation (chunks scoped to a chunk-shaped root, with NO + /// live pointer referencing them) must be grouped and reclaimed, not silently + /// dropped. Age and grouping split on the LAST infix, so the nested chunks are + /// attributed to their real (nested) root. + #[cfg(unix)] #[test] - fn write_fastly_local_config_store_replaces_existing_block_on_re_push() { + fn gc_reclaims_a_nested_orphan_generation() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); - write_fastly_local_config_store( - &path, - TEST_CONFIG_ID, - &[("greeting".to_owned(), "stale".to_owned())], - ) - .expect("first write"); - write_fastly_local_config_store( - &path, - TEST_CONFIG_ID, - &[("greeting".to_owned(), "fresh".to_owned())], - ) - .expect("second write"); - let after = fs::read_to_string(&path).expect("read back"); - assert!(after.contains("greeting = \"fresh\""), "new value: {after}"); - assert!( - !after.contains("greeting = \"stale\""), - "stale value dropped: {after}" - ); + let oplog = dir.path().join("ops.log"); + + // A chunk-shaped root, and a full generation of chunks SCOPED to it — but + // no pointer references them, so they are orphaned. + let nested_root = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "f".repeat(64)); + let nested = gen_envelope("nested-orphan"); + let nested_entries = prepare_fastly_config_entries(&nested_root, &nested).expect("expand"); + let nested_chunks: Vec = nested_entries[..nested_entries.len().saturating_sub(1)] + .iter() + .map(|(key, _)| key.clone()) + .collect(); + + let live = gen_envelope("live"); + let stamp = stamp_secs_ago(604_800); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + for (key, value) in &nested_entries[..nested_entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp.clone(), value.clone())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &nested_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "a nested orphan generation must be reclaimed, not silently dropped: `{key}`; \ + log:\n{log}" + ); + } } + /// FAIL CLOSED: a MALFORMED pointer sitting at a chunk-shaped root that HAS a + /// nested generation beneath it must abort GC, not let that nested generation + /// be reclaimed. The truncated pointer cannot announce its discriminator, so + /// it looks like a chunk fragment -- but its nested chunks are proven + /// independently and would be deleted while their (unreadable) root can no + /// longer name them. That is exactly the truncated-pointer data loss the + /// spec forbids, so the whole run must refuse. + #[cfg(unix)] #[test] - fn write_fastly_local_config_store_preserves_unrelated_blocks() { + fn gc_fails_closed_on_a_malformed_pointer_at_a_chunk_shaped_root_with_nested_chunks() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - let original = "\ -[setup.kv_stores.sessions] + let oplog = dir.path().join("ops.log"); -[[local_server.kv_stores.sessions]] -key = \"__init__\" -data = \"\" + let nested_root = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "f".repeat(64)); + let nested = gen_envelope("nested"); + let nested_entries = prepare_fastly_config_entries(&nested_root, &nested).expect("expand"); + let nested_chunks: Vec = nested_entries[..nested_entries.len().saturating_sub(1)] + .iter() + .map(|(key, _)| key.clone()) + .collect(); -[scripts] -build = \"cargo build --release\" -"; - fs::write(&path, original).expect("write"); - write_fastly_local_config_store( - &path, - TEST_CONFIG_ID, - &[("greeting".to_owned(), "hi".to_owned())], - ) - .expect("write"); - let after = fs::read_to_string(&path).expect("read back"); - assert!( - after.contains("[setup.kv_stores.sessions]"), - "setup KV kept: {after}" - ); - assert!(after.contains("[scripts]"), "scripts table kept: {after}"); - assert!( - after.contains("build = \"cargo build --release\""), - "scripts value kept: {after}" - ); + let live = gen_envelope("live"); + let stamp = stamp_secs_ago(604_800); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + // A truncated pointer at the chunk-shaped nested root: it WAS a pointer, + // now cut off, so it cannot announce its `edgezero_kind`. + listing.push(( + nested_root.clone(), + stamp.clone(), + r#"{"chunks":[{"key":"#.to_owned(), + )); + // ...its aged, independently-provable nested generation. + for (key, value) in &nested_entries[..nested_entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp.clone(), value.clone())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false) + .expect_err("an unreadable nested root must fail closed, not be reclaimed"); assert!( - after.contains(&format!( - "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" - )), - "new config_stores block added: {after}" + err.contains("refusing to reclaim"), + "must fail closed, not delete: {err}" ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &nested_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a nested generation under an unreadable root must NOT be deleted: `{key}`; \ + log:\n{log}" + ); + } } + /// Age attribution works per NESTED root: a nested orphan generation whose + /// nested root's live config went live RECENTLY must be RETAINED (POPs may + /// still serve the superseded generation), even though the orphan's own + /// chunks are old. This pins that `root_live_since` splits on the last infix. + #[cfg(unix)] #[test] - fn write_fastly_local_config_store_creates_file_when_missing() { + fn gc_retains_a_nested_orphan_under_a_recently_changed_nested_root() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - // No fs::write — file absent. - write_fastly_local_config_store( - &path, - TEST_CONFIG_ID, - &[("greeting".to_owned(), "hi".to_owned())], - ) - .expect("write"); - let after = fs::read_to_string(&path).expect("read back"); - assert!(after.contains(&format!( - "[local_server.config_stores.{TEST_CONFIG_ID}.contents]" - ))); - assert!(after.contains("greeting = \"hi\"")); + let oplog = dir.path().join("ops.log"); + + let nested_root = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "a".repeat(64)); + + // The nested root's CURRENT (live) generation, created 30s ago. + let live_nested = gen_envelope("live-nested"); + let live_entries = prepare_fastly_config_entries(&nested_root, &live_nested).expect("exp"); + let (_, live_pointer) = live_entries.last().expect("pointer").clone(); + + // An OLD orphan generation under the SAME nested root (a week old). + let old_nested = gen_envelope("old-nested-orphan"); + let old_entries = prepare_fastly_config_entries(&nested_root, &old_nested).expect("exp"); + let old_chunks: Vec = old_entries[..old_entries.len().saturating_sub(1)] + .iter() + .map(|(key, _)| key.clone()) + .collect(); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + // The nested root holds its live pointer; its live chunks are 30s old. + listing.push((nested_root.clone(), stamp_secs_ago(30), live_pointer)); + for (key, value) in &live_entries[..live_entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp_secs_ago(30), value.clone())); + } + // The old orphan chunks are a week old. + for (key, value) in &old_entries[..old_entries.len().saturating_sub(1)] { + listing.push((key.clone(), stamp_secs_ago(604_800), value.clone())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // A generous 1-day window: the orphan's OWN chunks are older, but the + // nested root's live config is only 30s old, so its orphan is retained. + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &old_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a nested orphan under a recently-changed nested root must be retained: `{key}`; \ + log:\n{log}" + ); + } } - // ---------- provision (dry-run + error path) ---------- + /// A generation is aged by its YOUNGEST member, so a generation with one + /// recent chunk is retained whole even if its other chunks are ancient. + #[cfg(unix)] + #[test] + fn gc_ages_a_generation_by_its_youngest_member() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + // `app_config` live is direct, so there is no live-config age signal — + // aging falls to the generation's own chunks. + let live_direct = gen_envelope_padded("live-direct", 100); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + assert!(dead_chunks.len() >= 2, "need a multi-chunk generation"); + + let mut listing = vec![( + TEST_CONFIG_ID.to_owned(), + stamp_secs_ago(999_999), + live_direct, + )]; + // The doomed generation: chunk 0 written 30s ago (YOUNG), the rest a week + // ago. Its youngest-member age (30s) is under the 1-day window. + let dead_parts = chunked_parts(TEST_CONFIG_ID, &dead).0; + for (idx, (key, value)) in dead_parts.iter().enumerate() { + let age = if idx == 0 { 30 } else { 604_800 }; + listing.push((key.clone(), stamp_secs_ago(age), value.clone())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &dead_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a generation with a recent member must be retained WHOLE (aged by its youngest): \ + `{key}`; log:\n{log}" + ); + } + } + /// A delete failure in one generation must not stop an INDEPENDENT + /// generation's deletes. + #[cfg(unix)] #[test] - fn provision_dry_run_does_not_invoke_fastly() { + fn gc_failure_in_one_generation_does_not_stop_another() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); - let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); - let config_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_CONFIG_ID]); - let secret_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_SECRET_ID]); - let stores = ProvisionStores { - config: &config_ids, - kv: &kv_ids, - secrets: &secret_ids, - }; - let out = FastlyCliAdapter - .provision(dir.path(), Some("fastly.toml"), None, &stores, true) - .expect("dry-run succeeds"); - // 1 KV + 1 config + 1 secret + 1 runtime-env = 4 status lines. - assert_eq!(out.len(), 4); - assert!(out[0].contains("would run `fastly kv-store create --name=sessions`")); - assert!(out[1].contains("would run `fastly config-store create --name=app_config`")); - assert!(out[2].contains("would run `fastly secret-store create --name=default`")); - assert!( - out[3].contains("would run `fastly config-store create --name=edgezero_runtime_env`"), - "runtime-env store row: {out:?}", + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead_a = gen_envelope("dead-a"); + let dead_b = gen_envelope("dead-b"); + let a_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead_a); + let b_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead_b); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead_a, 604_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead_b, 604_800)); + + // Generation A's first delete fails. + let fake = fake_fastly_gc( + TEST_CONFIG_ID, + &[], + &listing, + Some(&a_chunks[0]), + false, + &oplog, ); - // Manifest untouched. - let after = fs::read_to_string(&path).expect("read"); - assert_eq!(after, "name = \"demo\"\n", "dry-run mutated fastly.toml"); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("a failed delete is a failure"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + // Generation B must still have been reclaimed despite A's failure. + for key in &b_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "an independent generation must still be reclaimed after another one fails: \ + `{key}`; err: {err}; log:\n{log}" + ); + } } + /// key shape is not authoritative for ROOTS either. + /// + /// A valid pointer stored at a chunk-SHAPED key (`shadow.__edgezero_chunks. + /// .0`) is skipped by the live-set scan, which excludes chunk-shaped + /// keys up front. The runtime resolver follows any pointer it is given, so + /// that pointer's references ARE live -- but GC never sees them, calls the + /// generation orphaned, and deletes it. + #[cfg(unix)] #[test] - fn provision_errors_when_adapter_manifest_path_missing() { + fn pointer_at_chunk_shaped_key_keeps_its_references_live() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); - let stores = ProvisionStores { - config: &[], - kv: &kv_ids, - secrets: &[], - }; - let err = FastlyCliAdapter - .provision(dir.path(), None, None, &stores, true) - .expect_err("missing adapter manifest path must error"); - assert!( - err.contains("fastly.toml"), - "error names what's missing: {err}" - ); + let oplog = dir.path().join("ops.log"); + + // `app_config`'s CURRENT config is small enough to store directly, so + // its own root references no chunks at all. + let live_direct = gen_envelope_padded("live-direct", 100); + let mut listing = vec![( + TEST_CONFIG_ID.to_owned(), + stamp_secs_ago(999_999), + live_direct, + )]; + + // An older chunked generation of `app_config` still exists... + let referenced = gen_envelope("still-referenced"); + let referenced_chunks = chunk_keys_of(TEST_CONFIG_ID, &referenced); + listing.extend(listed_generation(TEST_CONFIG_ID, &referenced, 604_800)); + + // ...and a pointer at a CHUNK-SHAPED key references it. The resolver + // would happily follow this, so those chunks are LIVE. + let (_, referenced_pointer) = chunked_parts(TEST_CONFIG_ID, &referenced); + let shadow_key = format!("shadow{CHUNK_KEY_INFIX}{}.0", "d".repeat(64)); + listing.push((shadow_key, stamp_secs_ago(604_800), referenced_pointer)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // The RESULT does not matter here (it may Err after the fix if the + // shadow pointer's own chunks are incomplete); the invariant is purely + // that no LIVE-referenced chunk is deleted, which the oplog proves. + drop(run_gc(dir.path(), 86_400, false)); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &referenced_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a chunk a live pointer references must never be deleted, whatever the KEY of \ + the entry holding that pointer looks like: `{key}`; log:\n{log}" + ); + } } + /// a FOREIGN writer needs NO preimage to satisfy a + /// content-address. Pick envelope E, compute H = sha256(E), split E however + /// you like, store the parts as `.__edgezero_chunks.H.0` / `.1`. Under + /// hash-only checking that group "proved" itself and was deleted. + /// + /// The round-trip closes it: the writer, given those same bytes, must emit + /// exactly these keys and values. A split at boundaries we would never + /// choose is not our output, so it is left alone. + #[cfg(unix)] #[test] - fn provision_with_no_declared_stores_says_so() { + fn gc_never_reclaims_a_foreign_content_addressed_group() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - // Pre-populate the runtime-env block so the provision flow's - // unconditional runtime-env step skips (otherwise it would - // shell out to real `fastly` to create the store). - fs::write( - &path, - "name = \"demo\"\n[setup.config_stores.edgezero_runtime_env]\n", - ) - .expect("write"); - let stores = ProvisionStores { - config: &[], - kv: &[], - secrets: &[], - }; - let out = FastlyCliAdapter - .provision(dir.path(), Some("fastly.toml"), None, &stores, false) - .expect("no-store provision is fine"); - assert_eq!(out, vec!["fastly has no declared stores to provision"]); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + + // A foreign writer's data: a valid envelope, content-addressed under our + // reserved namespace, but split at ITS OWN boundary (not our 7 000-byte + // UTF-8-safe one). Everything hashes correctly -- no preimage needed. + let foreign = gen_envelope_padded("foreign-tool", 20_000); + let generation = sha256_hex(foreign.as_bytes()); + let (head, tail) = foreign.split_at(1_234); + let foreign_keys = [ + format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{generation}.0"), + format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{generation}.1"), + ]; + listing.push(( + foreign_keys[0].clone(), + stamp_secs_ago(31_536_000), + head.to_owned(), + )); + listing.push(( + foreign_keys[1].clone(), + stamp_secs_ago(31_536_000), + tail.to_owned(), + )); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &foreign_keys { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a group this writer would never have produced must not be reclaimed, however \ + well it hashes: `{key}`; log:\n{log}" + ); + } } + /// an entry can be chunk-SHAPED without being a chunk + /// -- a store may predate this feature or be shared, and push-time + /// reserved-key rejection cannot protect what already exists. Deleting one + /// would destroy live config. + /// + /// proof is CONTENT, not shape. A candidate generation is ours only + /// if it reassembles to the content-address its own keys name. Unprovable + /// entries are left UNTOUCHED and reported -- not fatal, because one foreign + /// entry must not block reclaiming the rest of the store forever. + #[cfg(unix)] #[test] - fn provision_skips_id_when_setup_block_already_present() { - // setup_block_present's role in the flow: re-running - // provision after the user already declared a store in - // fastly.toml must be a no-op (no shell-out to fastly). - // We can verify this in a real (non-dry-run) call because - // the skip path bypasses create_fastly_store entirely. + fn gc_leaves_unprovable_chunk_shaped_entries_untouched() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write( - &path, - "[setup.kv_stores.sessions]\n[local_server.kv_stores.sessions]\n\ - [setup.config_stores.edgezero_runtime_env]\n", - ) - .expect("write"); - let kv_ids: Vec = ResolvedStoreId::from_logicals(&[TEST_KV_ID]); - let stores = ProvisionStores { - config: &[], - kv: &kv_ids, - secrets: &[], - }; - let out = FastlyCliAdapter - .provision(dir.path(), Some("fastly.toml"), None, &stores, false) - .expect("skip path succeeds without invoking fastly"); - assert_eq!(out.len(), 1); - assert!(out[0].contains("already declared"), "got: {out:?}"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + // A real orphan generation: provable, old -> must still be reclaimed. + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + // Pre-existing entries at chunk-shaped keys that we did NOT write: one + // holding somebody's real config envelope, one holding plain text. + // Both are old enough to look "eligible" on age alone. + let envelope_squatter = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "b".repeat(64)); + let text_squatter = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{}.0", "c".repeat(64)); + listing.push(( + envelope_squatter.clone(), + stamp_secs_ago(31_536_000), + gen_envelope("someones-real-config"), + )); + listing.push(( + text_squatter.clone(), + stamp_secs_ago(31_536_000), + "just some plain text".to_owned(), + )); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + + for key in [&envelope_squatter, &text_squatter] { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "an entry we cannot prove we wrote must never be deleted: `{key}`; log:\n{log}" + ); + } + // Left untouched must not mean silently ignored. The wording must not + // over-claim either: these two entries fail for DIFFERENT reasons (a + // wrong content-address vs a count this writer never emits), so the + // summary says "not byte-identical to what this writer would produce" + // rather than naming one specific check. + assert!( + out.iter() + .any(|line| line.contains("not byte-identical to what this writer would produce")), + "the summary must report what it declined to judge; out: {out:?}" + ); + // ...and a genuine orphan generation is still reclaimed, so one foreign + // entry does not block the store. + for key in &dead_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "a provable orphan generation must still be reclaimed: `{key}`; log:\n{log}" + ); + } } - /// When `fastly.toml` declares `service_id`, the next - /// `fastly compute deploy` skips `[setup]` entirely. provision - /// must emit the `fastly resource-link create` remediation for - /// every store it creates -- including the implicit - /// `edgezero_runtime_env` store the runtime override path - /// depends on. Without this, a freshly-provisioned override - /// store would not be linked to the already-deployed service - /// and the runtime would silently fall back to baked defaults. + /// a key is unique in a config store, so duplicate rows + /// mean the listing is not one consistent view. Left alone, last-row-wins on + /// `created_at` could age a recent key into eligibility. + #[cfg(unix)] #[test] - fn provision_emits_resource_link_note_for_runtime_env_on_existing_service() { - // Dry-run only -- we just want to drive the resource_link_note - // helper for the runtime-env store branch. The real-create - // path can't run in tests (would shell out to `fastly`). - // The dry-run output line for runtime-env doesn't include the - // note (the helper only fires on real create), so we test the - // helper directly here. + fn gc_fails_closed_on_duplicate_listing_keys() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\nservice_id = \"abc123svc\"\n").expect("write"); - let note = resource_link_note(&path, "config", "edgezero_runtime_env") - .expect("read service_id") - .expect("note present when service_id set"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + let mut orphans = listed_generation(TEST_CONFIG_ID, &dead, 30); + // The same key twice, with conflicting ages: young (real) then ancient. + let (dup_key, _, dup_value) = orphans[0].clone(); + orphans.push((dup_key.clone(), stamp_secs_ago(31_536_000), dup_value)); + listing.extend(orphans); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); assert!( - note.contains("service_id = \"abc123svc\""), - "note quotes the service id: {note}" + err.contains("more than once"), + "expected a refusal naming the duplicate key, got: {err}" ); assert!( - note.contains("fastly config-store list --json"), - "note tells operator how to find the store id: {note}" + !oplog_has(&oplog, &format!("delete {dup_key}")), + "a duplicated row must not let a recent key be aged into eligibility" ); + } + + /// `gc_config_entries` is a public trait method, so the + /// zero-window rule must live at the DESTRUCTIVE boundary, not only in the + /// CLI that usually calls it. Rejected before any `fastly` invocation. + #[cfg(unix)] + #[test] + fn gc_adapter_boundary_rejects_a_zero_window() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // Straight at the adapter, bypassing the CLI's own gate. + let err = run_gc(dir.path(), 0, false).expect_err("a destructive zero window must fail"); assert!( - note.contains("name=`edgezero_runtime_env`"), - "note names the runtime override store: {note}" + err.contains("non-zero `--older-than`"), + "expected the boundary itself to reject zero, got: {err}" ); assert!( - note.contains( - "fastly resource-link create --service-id=abc123svc --resource-id= --version=latest --autoclone --name=edgezero_runtime_env" - ), - "note carries the full resource-link command: {note}" + !fs::read_to_string(&oplog) + .unwrap_or_default() + .contains("delete "), + "nothing may be deleted under a zero window" ); + // A DRY-RUN at zero is still allowed: it previews and deletes nothing. + run_gc(dir.path(), 0, true).expect("a dry-run may preview at zero"); } - /// And the inverse: no `service_id` (a service that hasn't been - /// deployed yet) means `[setup]` will be applied on the next - /// `compute deploy`, so no manual resource-link step is needed. - /// The helper must return `None` to avoid noisy false-positive - /// guidance. + /// a root whose value is TRUNCATED/unparseable must fail + /// closed. It is pointer-shaped garbage -- we cannot tell what it references, + /// so its (live!) chunks must not be judged orphaned. Regression guard: the + /// push-path helper returns `Ok([])` for a non-pointer value, which on THIS + /// path would read as "references nothing" and reclaim the whole store. + #[cfg(unix)] #[test] - fn provision_skips_resource_link_note_when_service_undeployed() { + fn gc_fails_closed_on_truncated_root_pointer() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); - let note = - resource_link_note(&path, "config", "edgezero_runtime_env").expect("read service_id"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let live_chunks = chunk_keys_of(TEST_CONFIG_ID, &live); + let (_, pointer) = chunked_parts(TEST_CONFIG_ID, &live); + // A write that landed half-way: a valid PREFIX of the real pointer that + // is no longer valid JSON. (Chars, not a byte slice -- never split a + // codepoint.) + let truncated: String = pointer.chars().take(40).collect(); assert!( - note.is_none(), - "no service_id => no resource-link prompt: {note:?}" + serde_json::from_str::(&truncated).is_err(), + "fixture must be unparseable to exercise the classifier: {truncated}" ); - } - // ---------- find_config_store_id ---------- + let mut listing = vec![( + TEST_CONFIG_ID.to_owned(), + stamp_secs_ago(999_999), + truncated, + )]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); - #[test] - fn find_config_store_id_matches_bare_array_by_name() { - let stdout = format!( - r#"[ - {{"id": "abc123", "name": "{TEST_CONFIG_ID}"}}, - {{"id": "def456", "name": "other_store"}} - ]"# + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 1, false).expect_err("must fail closed"); + assert!( + err.contains("refusing to reclaim"), + "expected a fail-closed refusal, got: {err}" ); - match find_config_store_id(&stdout, TEST_CONFIG_ID) { - ConfigStoreLookup::Found(id) => assert_eq!(id, "abc123"), - ConfigStoreLookup::NotFound => panic!("expected Found, got NotFound"), - ConfigStoreLookup::SchemaDrift(detail) => { - panic!("expected Found, got SchemaDrift({detail})") - } + for key in &live_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "nothing may be deleted when a root is unclassifiable: `{key}`" + ); } } + /// an ENVELOPED listing (`{"items":[...]}`) may carry + /// pagination we do not follow. A page that omitted a root would make that + /// root's live chunks look orphaned -- and the completeness guard cannot see + /// a root that isn't there. Refuse the shape outright. + #[cfg(unix)] #[test] - fn find_config_store_id_tolerates_items_envelope() { - let stdout = format!( - r#"{{"items": [ - {{"id": "xyz789", "name": "{TEST_CONFIG_ID}"}} - ]}}"# + fn gc_fails_closed_on_enveloped_listing() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 999_999)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + let enveloped = format!( + r#"{{"items":{},"next_cursor":"abc"}}"#, + entry_list_json(&listing) ); - match find_config_store_id(&stdout, TEST_CONFIG_ID) { - ConfigStoreLookup::Found(id) => assert_eq!(id, "xyz789"), - ConfigStoreLookup::NotFound => panic!("expected Found, got NotFound"), - ConfigStoreLookup::SchemaDrift(detail) => { - panic!("expected Found, got SchemaDrift({detail})") - } - } - } - #[test] - fn find_config_store_id_distinguishes_not_found_from_match_failure() { - // JSON parses cleanly, entries are well-formed - // (`name` + `id` strings present), but no entry matches - // → NotFound. Operator likely needs to run `provision`. - let stdout = r#"[{"id": "abc", "name": "other"}]"#; - assert!(matches!( - find_config_store_id(stdout, "missing"), - ConfigStoreLookup::NotFound - )); - } + let fake = fake_fastly_gc_raw_list(TEST_CONFIG_ID, &enveloped, &oplog); + let _path = PathPrepend::new(fake.path()); - #[test] - fn find_config_store_id_flags_schema_drift_on_malformed_json() { - // Unparseable bytes are NOT a "store not found" — they're - // a "fastly CLI output format changed" signal. Operator - // needs different recovery (file a bug, pin CLI version) - // than for the "store doesn't exist yet" case. - let drift = find_config_store_id("not json", "anything"); + let err = run_gc(dir.path(), 1, false).expect_err("must fail closed"); assert!( - matches!(drift, ConfigStoreLookup::SchemaDrift(_)), - "non-JSON stdout must be schema drift, got {drift:?}" + err.contains("bare array") && err.contains("Nothing was deleted"), + "expected a refusal naming the unsupported listing shape, got: {err}" ); - let empty = find_config_store_id("", "anything"); assert!( - matches!(empty, ConfigStoreLookup::SchemaDrift(_)), - "empty stdout must be schema drift, got {empty:?}" + !fs::read_to_string(&oplog) + .unwrap_or_default() + .contains("delete "), + "an unsupported listing shape must delete nothing" ); } + /// a root with an EMPTY value is as dangerous as a + /// missing one -- it would classify as "references nothing" and orphan its + /// live chunks. The listing parser rejects it before any reasoning. + #[cfg(unix)] #[test] - fn find_config_store_id_flags_schema_drift_when_shape_unexpected() { - // JSON parses but the top-level is neither a bare array - // nor an `{items: [...]}` envelope. - let stdout = r#"{"namespace": "fastly", "list": []}"#; - match find_config_store_id(stdout, "any") { - ConfigStoreLookup::SchemaDrift(detail) => { - assert!( - detail.contains("bare array") || detail.contains("items"), - "schema-drift detail names the expected shapes: {detail}" - ); - } - ConfigStoreLookup::Found(id) => panic!("expected SchemaDrift, got Found({id})"), - ConfigStoreLookup::NotFound => panic!("expected SchemaDrift, got NotFound"), + fn gc_fails_closed_on_empty_root_value() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let live_chunks = chunk_keys_of(TEST_CONFIG_ID, &live); + let mut listing = vec![( + TEST_CONFIG_ID.to_owned(), + stamp_secs_ago(999_999), + String::new(), + )]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 999_999)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 1, false).expect_err("must fail closed"); + assert!( + err.contains("empty `item_value`"), + "expected a refusal naming the empty field, got: {err}" + ); + for key in &live_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "nothing may be deleted on an unreadable listing: `{key}`" + ); } } + /// the orphan's OWN age is mandatory -- an old root does + /// not license deleting a chunk written seconds ago (e.g. by a concurrent + /// push that has not committed its pointer yet). Both ages must clear the + /// window; the more restrictive wins. + #[cfg(unix)] #[test] - fn find_config_store_id_flags_schema_drift_when_entries_lack_name_id() { - // Array of objects but none have BOTH string `name` and - // string `id` fields — suggests schema rename (e.g. - // fastly renamed `name` → `title`). - let stdout = format!(r#"[{{"title": "{TEST_CONFIG_ID}", "uid": "abc"}}]"#); - let drift = find_config_store_id(&stdout, TEST_CONFIG_ID); - assert!( - matches!(drift, ConfigStoreLookup::SchemaDrift(_)), - "entries lacking name/id must be schema drift, got {drift:?}" - ); + fn gc_retains_young_orphan_under_long_stable_root() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let fresh = gen_envelope("fresh"); + let fresh_chunks = chunk_keys_of(TEST_CONFIG_ID, &fresh); + + // The root's live config has been stable for a year -- so the live-config + // clock alone would happily reclaim. But these chunks were written 10s + // ago and no pointer names them yet. + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 31_536_000)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 31_536_000)); + listing.extend(listed_generation(TEST_CONFIG_ID, &fresh, 10)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &fresh_chunks { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a chunk written 10s ago must be retained under a 1-day window regardless of \ + how stable its root is: `{key}`; log:\n{log}" + ); + } } + /// GC and the RUNTIME must agree on what a readable pointer is. A pointer + /// whose chunks reassemble to the correct bytes along boundaries this writer + /// would never choose is REJECTED by the runtime resolver, so GC must not + /// silently report it as a healthy root: the guest cannot read it, and its + /// generation can never satisfy `prove_generation`, so it is permanently + /// unreclaimable. GC still keeps it (fail-closed) but must SAY so. + #[cfg(unix)] #[test] - fn find_config_store_id_returns_not_found_for_empty_array() { - // Empty array IS a valid "store doesn't exist yet" signal, - // not schema drift — fastly CLI legitimately returns `[]` - // when no config-stores exist. - let drift = find_config_store_id("[]", "any"); + fn gc_warns_that_a_non_writer_split_root_is_not_runtime_readable() { + use crate::chunked_config::CHUNK_PAYLOAD_TARGET; + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + // Just over the entry limit => a full chunk plus a short remainder with + // room to absorb the shifted bytes. + let envelope = gen_envelope("shifted"); + let sha = sha256_hex(envelope.as_bytes()); + // Re-split 2 bytes early: still within every metadata bound the pointer + // validator checks, but NOT where this writer splits. + let cut = CHUNK_PAYLOAD_TARGET.saturating_sub(2); + let head = envelope.get(..cut).expect("ascii boundary"); + let tail = envelope.get(cut..).expect("ascii boundary"); + let key0 = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{sha}.0"); + let key1 = format!("{TEST_CONFIG_ID}{CHUNK_KEY_INFIX}{sha}.1"); + let pointer_json = serde_json::json!({ + "chunks": [ + {"key": key0, "len": head.len(), "sha256": sha256_hex(head.as_bytes())}, + {"key": key1, "len": tail.len(), "sha256": sha256_hex(tail.as_bytes())}, + ], + "data_sha256": "", + "edgezero_kind": "fastly_config_chunks", + "envelope_len": envelope.len(), + "envelope_sha256": sha, + "version": 1_u8, + }) + .to_string(); + + let stamp = stamp_secs_ago(604_800); + let listing = vec![ + (TEST_CONFIG_ID.to_owned(), stamp.clone(), pointer_json), + (key0.clone(), stamp.clone(), head.to_owned()), + (key1.clone(), stamp, tail.to_owned()), + ]; + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = run_gc(dir.path(), 86_400, false).expect("gc must not abort on such a root"); + let rendered = out.join("\n"); assert!( - matches!(drift, ConfigStoreLookup::NotFound), - "empty array must be NotFound, got {drift:?}" + rendered.contains("NOT runtime-readable"), + "GC must warn that this root is unreadable rather than call it healthy: {rendered}" ); + // Fail-closed: nothing is deleted, including its chunks. + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in [&key0, &key1] { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "must not delete a chunk of an unreadable root: `{key}`; log:\n{log}" + ); + } } - // ---------- push_config_entries (dry-run + error paths) ---------- - + /// A legitimate FOREIGN sibling (the documented `greeting = "hello"`) must + /// NOT block store-wide GC. The runtime returns such a value verbatim, so GC + /// protects it as a zero-reference root and still reclaims an unrelated dead + /// generation, rather than aborting the whole pass. + #[cfg(unix)] #[test] - fn push_dry_run_does_not_invoke_fastly() { + fn gc_reclaims_despite_a_foreign_sibling_value() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let entries = vec![ - ("greeting".to_owned(), "hello".to_owned()), - ("feature.new_checkout".to_owned(), "false".to_owned()), + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + + let mut listing = vec![ + listed_root(TEST_CONFIG_ID, &live, 172_800), + // A plain, non-envelope, non-pointer sibling entry. + ( + "greeting".to_owned(), + stamp_secs_ago(172_800), + "hello".to_owned(), + ), ]; - let out = FastlyCliAdapter - .push_config_entries( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, - &AdapterPushContext::new(), - true, - ) - .expect("dry-run succeeds"); - // First line names the resolve+publish flow; subsequent lines preview - // each key the push would create (so callers can eyeball the keyset - // before running for real). - assert_eq!(out.len(), 1 + entries.len(), "header + per-entry preview"); + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("a foreign sibling must not abort GC"); + for key in &dead_chunks { + assert!( + oplog_has(&oplog, &format!("delete {key}")), + "the dead generation must still be reclaimed: `{key}`" + ); + } assert!( - out[0].contains("would resolve fastly config-store `app_config`") - && out[0].contains("push entries"), - "dry-run header describes the would-be flow: {out:?}" + !oplog_has(&oplog, "delete greeting"), + "the foreign sibling must never be deleted" ); + } + + /// A DIRECT envelope from a NEWER writer at an ordinary key classifies as + /// `Foreign` (no `edgezero_kind`), so without the future-format guard GC would + /// wave it through as a zero-reference root and reclaim an otherwise-dead + /// generation -- yet the newer format may reference those chunks under a + /// scheme this build cannot read. GC must FAIL CLOSED and delete nothing. + #[cfg(unix)] + #[test] + fn gc_fails_closed_on_a_future_direct_envelope_at_an_ordinary_key() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let dead = gen_envelope("dead"); + + // Envelope-shaped, no discriminator, VERSION 2 -> a newer direct envelope. + let future = r#"{"data":{"x":1},"sha256":"0000000000000000000000000000000000000000000000000000000000000000","generated_at":"2026-01-01T00:00:00Z","version":2}"#; + let mut listing = vec![( + "app_config".to_owned(), + stamp_secs_ago(172_800), + future.to_owned(), + )]; + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let result = run_gc(dir.path(), 86_400, false); assert!( - out.iter().any(|line| line.contains("`greeting`")), - "dry-run lists `greeting`: {out:?}" + result.is_err(), + "a future direct envelope must abort GC (fail closed)" ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); assert!( - out.iter() - .any(|line| line.contains("`feature.new_checkout`")), - "dry-run lists `feature.new_checkout`: {out:?}" + !log.lines().any(|line| line.starts_with("delete ")), + "nothing may be deleted when GC fails closed; log:\n{log}" ); } + /// A valid v1 pointer whose chunks reassemble to a NEWER inner format. GC can + /// validate the outer pointer and reassemble the bytes, but the reassembled + /// value may reference generations this build cannot see, so trusting only the + /// outer chunks as the live set could delete live data. GC must fail closed -- + /// `BlobEnvelope` deserialize alone would silently ignore the newer format. + #[cfg(unix)] #[test] - fn push_with_no_entries_reports_no_op_without_invoking_fastly() { + fn gc_fails_closed_on_a_future_inner_generation() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let out = FastlyCliAdapter - .push_config_entries( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &[], - &AdapterPushContext::new(), - false, - ) - .expect("zero-entry push is fine"); - assert_eq!(out.len(), 1); + let oplog = dir.path().join("ops.log"); + + // A v1 envelope, chunked, then its inner `version` bumped to 2. The v1 + // pointer's content-address still matches the reassembled (v2) bytes. + let v1 = gen_envelope("live"); + let mut v2_value: serde_json::Value = serde_json::from_str(&v1).expect("parse"); + v2_value["version"] = serde_json::json!(2_u32); + let v2 = v2_value.to_string(); + + let mut listing = vec![listed_root(TEST_CONFIG_ID, &v2, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &v2, 172_800)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let result = run_gc(dir.path(), 86_400, false); assert!( - out[0].contains("no config entries"), - "status line names the no-op: {out:?}" + result + .as_ref() + .is_err_and(|err| err.contains("newer format")), + "a future inner generation must abort GC with a newer-format error: {result:?}" + ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + !log.lines().any(|line| line.starts_with("delete ")), + "nothing may be deleted when GC fails closed; log:\n{log}" ); } - // ---------- read_config_entry_local ---------- + /// A dry-run lists exactly what it would delete, and deletes nothing. + #[cfg(unix)] + #[test] + fn gc_dry_run_lists_but_deletes_nothing() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = run_gc(dir.path(), 86_400, true).expect("dry-run succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + !log.lines().any(|line| line.starts_with("delete ")), + "a dry-run must not delete; log:\n{log}" + ); + let rendered = out.join("\n"); + assert!( + rendered.contains("would delete"), + "lists intent: {rendered}" + ); + for key in &dead_chunks { + assert!(rendered.contains(key.as_str()), "names `{key}`: {rendered}"); + } + } + /// An unreadable `created_at` on a DELETE path fails CLOSED. + #[cfg(unix)] #[test] - fn read_local_returns_missing_store_when_fastly_toml_absent() { + fn gc_fails_closed_on_unreadable_timestamp() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - // No fastly.toml written — file missing. - let result = FastlyCliAdapter - .read_config_entry_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("missing file is not an error"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 3_600)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 3_600)); + // An orphan whose timestamp is garbage. + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + for key in dead_chunks { + listing.push((key, "not-a-timestamp".to_owned(), "X".to_owned())); + } + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); assert!( - matches!(result, ReadConfigEntry::MissingStore), - "absent fastly.toml => MissingStore" + err.contains("unreadable") && err.contains("nothing was deleted"), + "must refuse to reclaim: {err}" + ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + !log.lines().any(|line| line.starts_with("delete ")), + "nothing may be deleted when the state is unreadable; log:\n{log}" ); } + /// A root whose pointer cannot be classified fails CLOSED — we cannot know + /// what it references, so nothing may be deleted. + #[cfg(unix)] #[test] - fn read_local_returns_missing_store_when_no_local_server_contents() { + fn gc_fails_closed_on_unclassifiable_root() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - // fastly.toml exists but has no [local_server.config_stores.*] block. - fs::write(&path, "name = \"demo\"\n[setup.config_stores.app_config]\n").expect("write"); - let result = FastlyCliAdapter - .read_config_entry_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("missing local_server block is not an error"); + let oplog = dir.path().join("ops.log"); + + let dead = gen_envelope("dead"); + // Root value is pointer-kind but invalid. + let bad = r#"{"edgezero_kind":"fastly_config_chunks","version":2}"#.to_owned(); + let mut listing = vec![(TEST_CONFIG_ID.to_owned(), stamp_secs_ago(3_600), bad)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); assert!( - matches!(result, ReadConfigEntry::MissingStore), - "no local_server stanza => MissingStore" + err.contains("could not classify root") && err.contains("nothing was deleted"), + "must refuse to reclaim: {err}" + ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + !log.lines().any(|line| line.starts_with("delete ")), + "nothing may be deleted when a root is unclassifiable; log:\n{log}" ); } + /// A listing entry missing a required field fails CLOSED — a defaulted/empty + /// field could make a real root look like it references nothing, deleting + /// live chunks. + #[cfg(unix)] #[test] - fn read_local_returns_missing_key_when_key_absent_from_contents() { + fn gc_fails_closed_on_malformed_listing_entry() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - // Write a local_server block with a different key so the store exists - // but the requested key is absent. + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + let good = entry_list_json(&listing); + // Inject an entry with NO item_value (drop that field entirely). + let mut array: serde_json::Value = serde_json::from_str(&good).unwrap(); + array.as_array_mut().unwrap().push(serde_json::json!({ + "item_key": "some.__edgezero_chunks.deadbeef.0", + "created_at": stamp_secs_ago(1000), + })); + // Serve that hand-built listing. + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); fs::write( - &path, - format!( - "name = \"demo\"\n\ - [local_server.config_stores.{TEST_CONFIG_ID}]\n\ - format = \"inline-toml\"\n\ - [local_server.config_stores.{TEST_CONFIG_ID}.contents]\n\ - other_key = \"other_value\"\n" - ), + fake.path().join("entries.json"), + serde_json::to_string(&array).unwrap(), ) - .expect("write"); - let result = FastlyCliAdapter - .read_config_entry_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("missing key is not an error"); + .expect("overwrite entries"); + let _path = PathPrepend::new(fake.path()); + + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); assert!( - matches!(result, ReadConfigEntry::MissingKey), - "key absent from contents => MissingKey" + err.contains("missing a string") && err.contains("item_value"), + "must name the missing field: {err}" + ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + !log.lines().any(|line| line.starts_with("delete ")), + "nothing may be deleted on a malformed listing; log:\n{log}" ); } + /// A failed delete is a non-zero exit that names the failed key(s), so + /// automation can detect partial failure. + #[cfg(unix)] #[test] - fn read_local_returns_present_when_key_exists_in_contents() { - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; - + fn gc_delete_failure_is_non_zero_exit() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write initial toml"); + let oplog = dir.path().join("ops.log"); - // Use a valid BlobEnvelope value — the resolver requires BlobEnvelope - // or chunk-pointer JSON; raw strings are not accepted post-chunking. - let envelope_json = serde_json::to_string(&BlobEnvelope::new( - json!({"hello": "fastly"}), - "2026-06-22T00:00:00Z".into(), - )) - .expect("serialize"); - write_fastly_local_config_store( - &path, + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let dead_chunks = chunk_keys_of(TEST_CONFIG_ID, &dead); + let fail_key = dead_chunks.first().expect("a chunk").clone(); + + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + let fake = fake_fastly_gc( TEST_CONFIG_ID, - &[("greeting".to_owned(), envelope_json.clone())], - ) - .expect("setup write"); + &[], + &listing, + Some(&fail_key), + false, + &oplog, + ); + let _path = PathPrepend::new(fake.path()); - let result = FastlyCliAdapter - .read_config_entry_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ) - .expect("key present"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present variant"); - }; - assert_eq!(value, envelope_json, "value matches what was written"); + let err = run_gc(dir.path(), 86_400, false).expect_err("a failed delete must be non-zero"); + assert!( + err.contains("deletes FAILED") && err.contains(&fail_key), + "error names the failed key: {err}" + ); } + /// Every reclamation delete passes `--key` + `--auto-yes` and NEVER `--all`. + #[cfg(unix)] #[test] - fn read_local_roundtrips_with_push_local() { - // Write via push_config_entries_local, then read via - // read_config_entry_local — the two must agree on the value. - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; + fn gc_delete_uses_key_and_auto_yes_never_all() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let dead = gen_envelope("dead"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + listing.extend(listed_generation(TEST_CONFIG_ID, &dead, 604_800)); + + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + run_gc(dir.path(), 86_400, false).expect("gc succeeds"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + let argv_lines: Vec<&str> = log + .lines() + .filter(|line| line.starts_with("delete-argv ")) + .collect(); + assert!(!argv_lines.is_empty(), "a delete happened: {log}"); + for line in argv_lines { + assert!( + line.contains("--auto-yes"), + "delete passes --auto-yes: {line}" + ); + assert!(line.contains("--key="), "delete targets a --key: {line}"); + assert!( + !line.contains("--all"), + "delete must NEVER pass --all: {line}" + ); + } + } + /// A non-canonical chunk-like key (short/uppercase SHA, leading-zero index) + /// is NOT a delete candidate — the destructive validator is canonical-only. + #[cfg(unix)] + #[test] + fn gc_never_deletes_non_canonical_keys() { + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let path = dir.path().join("fastly.toml"); - fs::write(&path, "name = \"demo\"\n").expect("write"); + let oplog = dir.path().join("ops.log"); + + let live = gen_envelope("live"); + let mut listing = vec![listed_root(TEST_CONFIG_ID, &live, 172_800)]; + listing.extend(listed_generation(TEST_CONFIG_ID, &live, 172_800)); + // Foreign-shaped keys under the reserved infix but not canonical. + let noncanonical = [ + format!("{TEST_CONFIG_ID}.__edgezero_chunks.abc123.0"), // short sha + format!("{TEST_CONFIG_ID}.__edgezero_chunks.{}.00", "a".repeat(64)), // leading-zero idx + format!("{TEST_CONFIG_ID}.__edgezero_chunks.{}.0", "A".repeat(64)), // uppercase + ]; + for key in &noncanonical { + listing.push((key.clone(), stamp_secs_ago(604_800), "X".to_owned())); + } - // push_config_entries_local passes the value through the chunk-pointer - // helper which stores it verbatim when ≤ 8 000 chars. The reader then - // resolves it through the same resolver that requires BlobEnvelope JSON. - let envelope_json = serde_json::to_string(&BlobEnvelope::new( - json!({"hello": "roundtrip"}), - "2026-06-22T00:00:00Z".into(), - )) - .expect("serialize"); - let entries = vec![("greeting".to_owned(), envelope_json.clone())]; + let fake = fake_fastly_gc(TEST_CONFIG_ID, &[], &listing, None, false, &oplog); + let _path = PathPrepend::new(fake.path()); + + // A key that is NOT canonical is not one we wrote, so it is not a + // reclamation candidate. It sits in our reserved namespace, though, so + // it is also not an ordinary root: we cannot say what it is. Since the + // GC classifier fails closed on any root it cannot classify, the run + // aborts and names it -- which satisfies this test's invariant (a + // non-canonical key is never deleted) the strict way. + let err = run_gc(dir.path(), 86_400, false).expect_err("must fail closed"); + assert!( + err.contains("refusing to reclaim"), + "expected a fail-closed refusal, got: {err}" + ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + for key in &noncanonical { + assert!( + !oplog_has(&oplog, &format!("delete {key}")), + "a non-canonical key must never be deleted: `{key}`; log:\n{log}" + ); + } + assert!( + !log.contains("delete "), + "a fail-closed run deletes nothing at all; log:\n{log}" + ); + } + + // ---------- local chunk GC ---------- + + /// Config shrinks from chunked back under the 8 000-char limit: the + /// new value is a direct envelope, so GC prunes every prior chunk. + #[cfg(unix)] + #[test] + fn push_config_entries_local_prunes_prior_chunks_when_value_shrinks_to_direct() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + + let chunked = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); FastlyCliAdapter .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, + &[(TEST_CONFIG_ID.to_owned(), chunked)], &AdapterPushContext::new(), false, ) - .expect("push succeeds"); - let result = FastlyCliAdapter - .read_config_entry_local( + .expect("first push"); + + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", + &[(TEST_CONFIG_ID.to_owned(), direct.clone())], &AdapterPushContext::new(), + false, ) - .expect("read succeeds"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present after push+read roundtrip"); - }; - assert_eq!(value, envelope_json, "roundtrip value matches"); - } - - #[test] - fn read_local_requires_adapter_manifest_path() { - let dir = tempdir().expect("tempdir"); - let result = FastlyCliAdapter.read_config_entry_local( - dir.path(), - None, // adapter_manifest_path missing - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", - &AdapterPushContext::new(), - ); - match result { - Err(err) => assert!( - err.contains("[adapters.fastly.adapter].manifest"), - "error names the missing field: {err}" - ), - Ok(_) => panic!("expected Err when adapter_manifest_path is None"), - } - } + .expect("second push"); - // ---------- read_config_entry (fake fastly, remote shell-out) ---------- - - /// Build a tempdir containing a `fastly` shim script that: - /// - Responds to `config-store list --json` with a store-list JSON containing - /// `TEST_CONFIG_ID` mapped to `store-abc123`. - /// - Responds to `config-store-entry describe ...` with `stdout_body` on - /// stdout and `stderr_body` on stderr, exiting with `exit_code`. - /// - /// Payloads are written to separate sibling files so shell-active chars - /// in the content don't get re-interpreted by the script. - #[cfg(unix)] - fn fake_fastly_returning( - stdout_body: &str, - stderr_body: &str, - exit_code: i32, - ) -> tempfile::TempDir { - use std::os::unix::fs::PermissionsExt as _; - let dir = tempdir().expect("tempdir"); - let script_path = dir.path().join("fastly"); - let stdout_file = dir.path().join("stdout_payload.txt"); - let stderr_file = dir.path().join("stderr_payload.txt"); - let list_file = dir.path().join("list_payload.txt"); - // Store-list JSON: bare array with one entry matching TEST_CONFIG_ID. - let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); - fs::write(&stdout_file, stdout_body).expect("write stdout payload"); - fs::write(&stderr_file, stderr_body).expect("write stderr payload"); - fs::write(&list_file, list_json).expect("write list payload"); - let script = format!( - "#!/bin/sh\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\ncat '{}'\ncat '{}' >&2\nexit {exit_code}\n", - list_file.display(), - stdout_file.display(), - stderr_file.display(), - ); - fs::write(&script_path, script).expect("write fastly script"); - let mut perms = fs::metadata(&script_path).expect("meta").permissions(); - perms.set_mode(0o755); - fs::set_permissions(&script_path, perms).expect("chmod +x"); - dir - } - - /// Build a fake `fastly` that logs each argv token (one per line) to - /// `out_path`, handles the list call correctly, and exits 0 for both calls. - #[cfg(unix)] - fn fake_fastly_argv_log(out_path: &Path) -> tempfile::TempDir { - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; - use std::os::unix::fs::PermissionsExt as _; - let dir = tempdir().expect("tempdir"); - let script_path = dir.path().join("fastly"); - let list_file = dir.path().join("list_payload.txt"); - let entry_file = dir.path().join("entry_payload.txt"); - let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); - // item_value must be a valid BlobEnvelope JSON so the resolver accepts it. - let envelope_json = serde_json::to_string(&BlobEnvelope::new( - json!({"v": "logged"}), - "2026-06-22T00:00:00Z".into(), - )) - .expect("serialize"); - let entry_json = format!( - r#"{{"item_value":{},"store_id":"store-abc123"}}"#, - serde_json::to_string(&envelope_json).expect("escape") + let after = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = after.parse().expect("parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents"); + + assert_eq!( + contents + .get(TEST_CONFIG_ID) + .and_then(toml_edit::Item::as_str), + Some(direct.as_str()), + "root holds the direct envelope" ); - fs::write(&list_file, list_json).expect("write list payload"); - fs::write(&entry_file, &entry_json).expect("write entry payload"); - let script = format!( - "#!/bin/sh\nfor arg in \"$@\"; do printf '%s\\n' \"$arg\" >> '{}'; done\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\ncat '{}'\nexit 0\n", - out_path.display(), - list_file.display(), - entry_file.display(), + assert!( + !contents + .iter() + .any(|(key, _)| key.contains(CHUNK_KEY_INFIX)), + "prior chunks must be pruned: {after}" ); - fs::write(&script_path, script).expect("write script"); - let mut perms = fs::metadata(&script_path).expect("meta").permissions(); - perms.set_mode(0o755); - fs::set_permissions(&script_path, perms).expect("chmod +x"); - dir - } - - /// Process-wide mutex serialising PATH-mutating tests so parallel - /// test threads don't race on the environment variable. - #[cfg(unix)] - fn path_mutation_guard() -> &'static Mutex<()> { - use std::sync::OnceLock; - static GUARD: OnceLock> = OnceLock::new(); - GUARD.get_or_init(|| Mutex::new(())) } - #[cfg(unix)] + /// The local prune must NOT delete a prior chunk key whose VALUE is a + /// runtime-readable root (a valid direct envelope). A small envelope padded + /// with trailing whitespace chunks so that chunk 0 is itself a whole, + /// verifying envelope; deleting it would drop live config. #[test] - fn read_remote_returns_present_on_success() { + fn push_config_entries_local_keeps_a_chunk_key_holding_a_valid_envelope() { use edgezero_core::blob_envelope::BlobEnvelope; use serde_json::json; - let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - // Fake fastly: list succeeds with app_config → store-abc123; - // describe returns valid JSON with item_value that is a BlobEnvelope. - let envelope = serde_json::to_string(&BlobEnvelope::new( - json!({"hello": "fastly"}), - "2026-06-22T00:00:00Z".into(), - )) - .expect("serialize"); - let entry_json = format!( - r#"{{"item_value":{},"store_id":"store-abc123"}}"#, - serde_json::to_string(&envelope).expect("escape") - ); - let fake = fake_fastly_returning(&entry_json, "", 0); - let _path = PathPrepend::new(fake.path()); - let result = FastlyCliAdapter - .read_config_entry( + let fastly_toml = dir.path().join("fastly.toml"); + + // A padded envelope: chunk 0 is the whole envelope plus trailing spaces + // (still a valid, verifying envelope on its own). + let envelope = BlobEnvelope::new(json!({ "k": "v" }), "2026-06-22T00:00:00Z".into()); + let mut padded = serde_json::to_string(&envelope).unwrap(); + padded.push_str(&" ".repeat(8_200)); + let entries = prepare_fastly_config_entries(TEST_CONFIG_ID, &padded).expect("expand"); + let chunk0_key = entries[0].0.clone(); + // Confirm the fixture: chunk 0's value verifies as an envelope. + let parsed: BlobEnvelope = serde_json::from_str(&entries[0].1).expect("chunk0 parses"); + parsed.verify().expect("chunk0 verifies"); + + FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", + &[(TEST_CONFIG_ID.to_owned(), padded)], &AdapterPushContext::new(), + false, ) - .expect("fake fastly exit-0 must succeed"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present"); - }; - assert_eq!(value, envelope); - } + .expect("first push"); - #[cfg(unix)] - #[test] - fn read_remote_returns_missing_key_on_not_found_stderr() { - let _lock = path_mutation_guard().lock().expect("guard"); - let dir = tempdir().expect("tempdir"); - // describe exits non-zero with "not found" in stderr → MissingKey. - let fake = fake_fastly_returning("", "Error: item not found", 1); - let _path = PathPrepend::new(fake.path()); - let result = FastlyCliAdapter - .read_config_entry( + // Re-push a direct value: the prior generation's chunks become orphans. + let direct = make_test_envelope(100); + let expected_deletions = entries.len().saturating_sub(2); // chunks minus the protected chunk0 + + // DRY-RUN first: its count must MATCH what the real prune deletes, i.e. + // it must exclude the protected root-like chunk0. + let dry = FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", + &[(TEST_CONFIG_ID.to_owned(), direct.clone())], &AdapterPushContext::new(), + true, ) - .expect("not-found maps to MissingKey (not Err)"); + .expect("dry-run"); assert!( - matches!(result, ReadConfigEntry::MissingKey), - "not-found stderr => MissingKey" + dry.join("\n") + .contains(&format!("would delete {expected_deletions} orphan chunks")), + "dry-run must count only the prunable orphans (excluding the protected root): {dry:?}" + ); + + let warnings = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct)], + &AdapterPushContext::new(), + false, + ) + .expect("second push"); + + let after = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = after.parse().expect("parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents"); + + assert!( + contents.contains_key(&chunk0_key), + "a chunk key holding a valid envelope is a runtime-readable root and must be kept: \ + {after}" + ); + assert!( + warnings + .iter() + .any(|warning| warning.contains("runtime-readable root")), + "the operator must be warned that the key was kept: {warnings:?}" ); } - /// The Fastly impl distinguishes store-not-found from key-not-found via - /// `resolve_remote_config_store_id`: when the list call exits non-zero and - /// the error string contains "not found", `read_config_entry` returns - /// `MissingStore` without ever calling the describe subcommand. - #[cfg(unix)] + /// The local prune must NOT delete a prior chunk key whose value was written + /// by a NEWER format -- a v2 direct envelope (bumped version) stored under a + /// chunk-shaped key. Cloud GC fails closed on it; local prune must be + /// symmetric, or an older CLI destroys config a newer writer produced. #[test] - fn read_remote_returns_missing_store_on_appropriate_stderr() { - use std::os::unix::fs::PermissionsExt as _; - let _lock = path_mutation_guard().lock().expect("guard"); + fn push_config_entries_local_keeps_a_chunk_key_holding_a_future_envelope() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let dir = tempdir().expect("tempdir"); - // Script that exits non-zero for the list call so resolve fails with - // a "not found" error, causing read_config_entry to return MissingStore. - let fake_dir = tempdir().expect("tempdir"); - let stderr_file = fake_dir.path().join("stderr_payload.txt"); - fs::write(&stderr_file, "Error: config store not found for service").expect("write stderr"); - let script_path = fake_dir.path().join("fastly"); - let script = format!( - "#!/bin/sh\ncat '{stderr}' >&2\nexit 1\n", - stderr = stderr_file.display(), - ); - fs::write(&script_path, script).expect("write script"); - let mut perms = fs::metadata(&script_path).expect("meta").permissions(); - perms.set_mode(0o755); - fs::set_permissions(&script_path, perms).expect("chmod +x"); - let _path = PathPrepend::new(fake_dir.path()); - let result = FastlyCliAdapter - .read_config_entry( + let fastly_toml = dir.path().join("fastly.toml"); + let chunked = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(5_000)); + FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", + &[(TEST_CONFIG_ID.to_owned(), chunked)], + &AdapterPushContext::new(), + false, + ) + .expect("seed"); + + // A v2 direct envelope (version bumped) parked at a chunk key. + let mut v2_value: serde_json::Value = serde_json::from_str( + &serde_json::to_string(&BlobEnvelope::new( + json!({ "k": "v" }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .unwrap(), + ) + .unwrap(); + v2_value["version"] = json!(2_u32); + let v2 = v2_value.to_string(); + + let mut doc: toml_edit::DocumentMut = fs::read_to_string(&fastly_toml) + .expect("read") + .parse() + .expect("parse"); + let contents = doc["local_server"]["config_stores"][TEST_CONFIG_ID]["contents"] + .as_table_mut() + .expect("contents"); + let victim = contents + .iter() + .map(|(key, _)| key.to_owned()) + .find(|key| key.contains(CHUNK_KEY_INFIX)) + .expect("a chunk key"); + contents.insert(&victim, toml_edit::value(v2)); + fs::write(&fastly_toml, doc.to_string()).expect("write"); + + let direct = make_test_envelope(100); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct)], &AdapterPushContext::new(), + false, ) - .expect("list failure with not-found maps to MissingStore (not Err)"); + .expect("re-push"); + + let after = fs::read_to_string(&fastly_toml).expect("read"); + let after_doc: toml_edit::DocumentMut = after.parse().expect("parse"); assert!( - matches!(result, ReadConfigEntry::MissingStore), - "list not-found => MissingStore" + after_doc["local_server"]["config_stores"][TEST_CONFIG_ID]["contents"] + .as_table() + .expect("contents") + .contains_key(&victim), + "a v2 (future-format) envelope must be KEPT, not pruned: {after}" ); } - /// Verify that `read_config_entry` invokes - /// `fastly config-store-entry describe --store-id= --key= --json` - /// (after the resolve step that calls `fastly config-store list --json`). - #[cfg(unix)] + /// The local prune must NOT delete a prior chunk key whose value claims our + /// `edgezero_kind` namespace with an UNKNOWN/future kind. The cloud GC path + /// fails closed on such a value; local replacement must be symmetric, or it + /// would destroy a newer-format entry an older CLI cannot understand. #[test] - fn read_remote_invokes_correct_argv() { - let _lock = path_mutation_guard().lock().expect("guard"); + fn push_config_entries_local_keeps_a_chunk_key_holding_an_unknown_kind() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); - let argv_log = dir.path().join("argv.txt"); - let fake = fake_fastly_argv_log(&argv_log); - let _path = PathPrepend::new(fake.path()); - let result = FastlyCliAdapter - .read_config_entry( + let fastly_toml = dir.path().join("fastly.toml"); + + // Seed a real chunked generation, then overwrite ONE chunk value with a + // future-format value that claims our namespace but is not a v1 pointer. + let chunked = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(5_000)); + FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "greeting", + &[(TEST_CONFIG_ID.to_owned(), chunked)], &AdapterPushContext::new(), + false, ) - .expect("argv-log fake must succeed"); - assert!( - matches!(result, ReadConfigEntry::Present(_)), - "expected Present from argv-log fake" - ); - let captured = fs::read_to_string(&argv_log).expect("argv log"); - // The describe call must include these args (resolve call args - // are also captured but we only assert the describe shape here). - assert!( - captured.contains("config-store-entry"), - "must invoke config-store-entry; got:\n{captured}" - ); - assert!( - captured.contains("describe"), - "must pass describe subcommand; got:\n{captured}" - ); - assert!( - captured.contains("--store-id=store-abc123"), - "must pass resolved store id; got:\n{captured}" + .expect("seed"); + + let mut doc: toml_edit::DocumentMut = fs::read_to_string(&fastly_toml) + .expect("read") + .parse() + .expect("parse"); + let contents = doc["local_server"]["config_stores"][TEST_CONFIG_ID]["contents"] + .as_table_mut() + .expect("contents"); + let victim = contents + .iter() + .map(|(key, _)| key.to_owned()) + .find(|key| key.contains(CHUNK_KEY_INFIX)) + .expect("a chunk key"); + contents.insert( + &victim, + toml_edit::value(r#"{"edgezero_kind":"fastly_config_chunks_v2","new":true}"#), ); + fs::write(&fastly_toml, doc.to_string()).expect("write"); + + // Re-push a direct value: every prior chunk becomes an orphan. + let direct = make_test_envelope(100); + let warnings = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct)], + &AdapterPushContext::new(), + false, + ) + .expect("re-push"); + + let after = fs::read_to_string(&fastly_toml).expect("read"); + let after_doc: toml_edit::DocumentMut = after.parse().expect("parse"); + let present = after_doc["local_server"]["config_stores"][TEST_CONFIG_ID]["contents"] + .as_table() + .expect("contents") + .contains_key(&victim); assert!( - captured.contains("--key=greeting"), - "must pass --key=; got:\n{captured}" + present, + "an unknown/future-kind value must be KEPT (symmetric with cloud GC fail-closed): {after}" ); assert!( - captured.contains("--json"), - "must pass --json flag; got:\n{captured}" + warnings + .iter() + .any(|warning| warning.contains("kept") && warning.contains("edgezero_kind")), + "the operator must be warned the namespace-claiming key was kept: {warnings:?}" ); } - // ---------- chunked push integration tests ---------- - - /// Build a valid `BlobEnvelope` JSON string of approximately `target_len` bytes. - #[cfg(unix)] - fn make_test_envelope(target_len: usize) -> String { - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; - let pad = "x".repeat(target_len.saturating_add(64)); - let data = json!({ "pad": pad }); - let raw = - serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".into())).unwrap(); - if raw.len() >= target_len { - let overhead = raw.len().saturating_sub(pad.len()); - let adjusted = "x".repeat(target_len.saturating_sub(overhead)); - let data2 = json!({ "pad": adjusted }); - serde_json::to_string(&BlobEnvelope::new(data2, "2026-06-22T00:00:00Z".into())).unwrap() - } else { - raw - } - } + /// SYMMETRY with cloud GC: a local prune must NOT delete a truncated pointer + /// at a chunk-shaped key that HAS a canonical chunk nested beneath it — it is + /// a (broken) nested root, and removing it would orphan the nested chunks. + #[test] + fn push_config_entries_local_keeps_a_malformed_nested_root_holder() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); - /// Build a fake `fastly` script whose describe response depends on - /// the `--key=` argument: `key_responses` maps key names to JSON - /// item-value responses. Falls back to exit 1 "not found" for unknown keys. - #[cfg(unix)] - fn fake_fastly_with_key_dispatch( - _dir: &Path, - key_responses: &[(String, String)], - ) -> tempfile::TempDir { - use std::fmt::Write as _; - use std::os::unix::fs::PermissionsExt as _; - let fake_dir = tempdir().expect("tempdir"); - let list_file = fake_dir.path().join("list.json"); - let list_json = format!(r#"[{{"name":"{TEST_CONFIG_ID}","id":"store-abc123"}}]"#); - fs::write(&list_file, list_json).expect("write list"); - // Write each key response to a named file. - let mut dispatch_lines = String::new(); - for (key, response) in key_responses { - let resp_file = fake_dir.path().join(format!("resp_{key}.json")); - fs::write(&resp_file, response).expect("write resp"); - // Use exact-match: iterate argv and compare each token literally - // so that a root key like "app_config" does NOT match a chunk key - // like "app_config.__edgezero_chunks.abc.0". - writeln!( - dispatch_lines, - " for arg in \"$@\"; do if [ \"$arg\" = \"--key={key}\" ]; then cat '{}'; exit 0; fi; done", - resp_file.display() + // Seed a chunked generation, then turn ONE chunk key into a nested root: + // give it a truncated (unclassifiable) value AND a canonical chunk nested + // beneath it. + let chunked = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(5_000)); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), chunked)], + &AdapterPushContext::new(), + false, ) - .expect("write to String is infallible"); - } - // Fallback outputs "not found" so fetch_remote_config_store_entry - // maps it to Ok(None) rather than Err. - let script = format!( - "#!/bin/sh\nif [ \"$1\" = \"config-store\" ]; then\n cat '{}'\n exit 0\nfi\n{dispatch_lines}echo 'Error: item not found' >&2\nexit 1\n", - list_file.display() + .expect("seed"); + + let mut doc: toml_edit::DocumentMut = fs::read_to_string(&fastly_toml) + .expect("read") + .parse() + .expect("parse"); + let contents = doc["local_server"]["config_stores"][TEST_CONFIG_ID]["contents"] + .as_table_mut() + .expect("contents"); + let holder = contents + .iter() + .map(|(key, _)| key.to_owned()) + .find(|key| key.contains(CHUNK_KEY_INFIX)) + .expect("a chunk key"); + // Truncated pointer at the holder (unclassifiable, announces no kind). + contents.insert(&holder, toml_edit::value(r#"{"chunks":[{"key":"#)); + // A canonical chunk nested BENEATH the holder. + let nested_chunk = format!("{holder}{CHUNK_KEY_INFIX}{}.0", "b".repeat(64)); + contents.insert(&nested_chunk, toml_edit::value("nested-payload")); + fs::write(&fastly_toml, doc.to_string()).expect("write"); + + // Re-push a direct value: every prior chunk becomes an orphan, including + // the holder (which the OLD pointer referenced). + let direct = make_test_envelope(100); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct)], + &AdapterPushContext::new(), + false, + ) + .expect("re-push"); + + let after = fs::read_to_string(&fastly_toml).expect("read"); + let after_doc: toml_edit::DocumentMut = after.parse().expect("parse"); + let after_contents = after_doc["local_server"]["config_stores"][TEST_CONFIG_ID]["contents"] + .as_table() + .expect("contents"); + assert!( + after_contents.contains_key(&holder), + "a nested-root holder with chunks beneath it must be KEPT, not pruned: {after}" ); - let script_path = fake_dir.path().join("fastly"); - fs::write(&script_path, &script).expect("write script"); - let mut perms = fs::metadata(&script_path).expect("meta").permissions(); - perms.set_mode(0o755); - fs::set_permissions(&script_path, perms).expect("chmod"); - fake_dir } - #[cfg(unix)] + /// `preflight_config_write` rejects an infeasible push BEFORE any provider + /// I/O: a reserved key, an empty key, and a body whose DERIVED chunk keys + /// would exceed the store limit (caught by running expansion offline). #[test] - fn push_config_entries_writes_direct_entry_at_exactly_8000_chars() { + fn preflight_config_write_rejects_infeasible_pushes_offline() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; - let _lock = path_mutation_guard().lock().expect("guard"); - let dir = tempdir().expect("tempdir"); - let argv_log = dir.path().join("argv.txt"); - let fake = fake_fastly_argv_log(&argv_log); - let _path = PathPrepend::new(fake.path()); + let small = make_test_envelope(100); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); - assert_eq!(envelope.len(), FASTLY_CONFIG_ENTRY_LIMIT); + let reserved = format!("app_config{CHUNK_KEY_INFIX}deadbeef.0"); + assert!( + FastlyCliAdapter + .preflight_config_write(&reserved, &small) + .is_err_and(|err| err.contains("reserved infix")), + "a reserved-namespace key must be rejected" + ); - let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; - let out = FastlyCliAdapter - .push_config_entries( + assert!( + FastlyCliAdapter + .preflight_config_write("", &small) + .is_err_and(|err| err.contains("empty")), + "an empty key must be rejected" + ); + + // A ~200-char root with a CHUNKED body: derived chunk keys (root + ~85) + // exceed the 255-char limit. Caught offline by expansion. + let long_root = "r".repeat(200); + let big = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + assert!( + FastlyCliAdapter + .preflight_config_write(&long_root, &big) + .is_err(), + "an over-limit derived chunk key must be rejected before I/O" + ); + + // A normal push passes. + FastlyCliAdapter + .preflight_config_write("app_config", &small) + .expect("a normal push must pass preflight"); + } + + /// A logical key containing the reserved chunk infix is rejected + /// before any file I/O (it would collide with the chunk namespace). + #[cfg(unix)] + #[test] + fn push_config_entries_local_rejects_reserved_key() { + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + let bad_key = format!("app_config{CHUNK_KEY_INFIX}deadbeef.0"); + + let err = FastlyCliAdapter + .push_config_entries_local( dir.path(), - None, + Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, + &[(bad_key.clone(), "{}".to_owned())], &AdapterPushContext::new(), false, ) - .expect("push must succeed"); - // One physical entry written (direct). - let captured = fs::read_to_string(&argv_log).expect("argv log"); - assert!( - captured.contains(&format!("--key={TEST_CONFIG_ID}")), - "must write root key directly: {captured}" - ); + .expect_err("reserved key must be rejected"); + assert!(err.contains(&bad_key), "error names the key: {err}"); assert!( - out[0].contains("1 physical entries (1 logical)"), - "summary reports 1 physical entry: {out:?}" + !fastly_toml.exists(), + "rejection must happen before any write" ); } + /// A suspicious prior pointer (pointer-kind but invalid) makes GC + /// warn and delete nothing — pre-seeded chunk keys must survive. #[cfg(unix)] #[test] - fn push_config_entries_writes_chunks_and_root_pointer_for_8001_chars() { + fn push_config_entries_local_warns_on_suspicious_prior_pointer_and_keeps_chunks() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; - let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); - let argv_log = dir.path().join("argv.txt"); - let fake = fake_fastly_argv_log(&argv_log); - let _path = PathPrepend::new(fake.path()); - - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - assert!(envelope.len() > FASTLY_CONFIG_ENTRY_LIMIT); + let fastly_toml = dir.path().join("fastly.toml"); + // Seed the root with a pointer-kind-but-invalid value AND a real + // chunk-like key so "no deletes" is non-vacuous. + let seed = concat!( + "name = \"demo\"\n\n", + "[local_server.config_stores.app_config]\n", + "format = \"inline-toml\"\n\n", + "[local_server.config_stores.app_config.contents]\n", + "app_config = \"{\\\"edgezero_kind\\\":\\\"fastly_config_chunks\\\",\\\"version\\\":1}\"\n", + "\"app_config.__edgezero_chunks.deadbeef.0\" = \"seeded-chunk-payload\"\n", + ); + fs::write(&fastly_toml, seed).expect("seed"); - let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); let out = FastlyCliAdapter - .push_config_entries( + .push_config_entries_local( dir.path(), - None, + Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, + &[(TEST_CONFIG_ID.to_owned(), direct.clone())], &AdapterPushContext::new(), false, ) - .expect("push must succeed"); - let captured = fs::read_to_string(&argv_log).expect("argv log"); - // At least one chunk key must appear before the root key. + .expect("push must still succeed"); + + let combined = out.join("\n"); assert!( - captured.contains(".__edgezero_chunks."), - "chunk keys must be written: {captured}" + combined.contains("skipping chunk GC"), + "must warn about the suspicious prior pointer: {combined}" ); - // Root pointer must also be written. + + let after = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = after.parse().expect("parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents"); assert!( - captured.contains(&format!("--key={TEST_CONFIG_ID}")), - "root pointer must be written: {captured}" + contents + .get("app_config.__edgezero_chunks.deadbeef.0") + .is_some(), + "pre-seeded chunk key must survive a suspicious-pointer skip: {after}" ); - // Root key must be LAST in the log (chunk lines come before it). - let root_pos = captured.rfind(&format!("--key={TEST_CONFIG_ID}")).unwrap(); - let chunk_pos = captured.find(".__edgezero_chunks.").unwrap(); - assert!( - chunk_pos < root_pos, - "chunk writes must precede root pointer write: chunk_pos={chunk_pos} root_pos={root_pos}" + assert_eq!( + contents + .get(TEST_CONFIG_ID) + .and_then(toml_edit::Item::as_str), + Some(direct.as_str()), + "new value still written" ); - assert!(out[0].contains("logical"), "summary line present: {out:?}"); } + /// TOCTOU guard: if the locked reread finds the root now holds a NEWER format + /// (installed between the pre-push check and the lock), the writer must REFUSE + /// to overwrite it -- an older writer must never clobber a newer format. #[cfg(unix)] #[test] - fn push_config_entries_dry_run_reports_direct_vs_chunked() { + fn push_config_entries_local_refuses_to_overwrite_a_future_prior() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + // The root now holds a v2 direct envelope from a newer writer. + let seed = concat!( + "name = \"demo\"\n\n", + "[local_server.config_stores.app_config]\n", + "format = \"inline-toml\"\n\n", + "[local_server.config_stores.app_config.contents]\n", + "app_config = \"{\\\"data\\\":{},\\\"sha256\\\":\\\"x\\\",\\\"generated_at\\\":\\\"t\\\",\\\"version\\\":2}\"\n", + ); + fs::write(&fastly_toml, seed).expect("seed"); - let direct_envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); - let chunked_envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - - let entries = vec![ - ("cfg_direct".to_owned(), direct_envelope), - ("cfg_chunked".to_owned(), chunked_envelope), - ]; - let out = FastlyCliAdapter - .push_config_entries( + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let err = FastlyCliAdapter + .push_config_entries_local( dir.path(), - None, + Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, + &[(TEST_CONFIG_ID.to_owned(), direct)], &AdapterPushContext::new(), - true, // dry_run + false, ) - .expect("dry-run must not error"); - - // No shellout happens; output must describe intent. - let combined = out.join("\n"); + .expect_err("a future prior must abort the push under the lock"); assert!( - combined.contains("would push `cfg_direct` as direct entry"), - "must report direct: {combined}" + err.contains("newer format") && err.to_lowercase().contains("refusing"), + "must refuse to clobber a newer format: {err}" ); + // The v2 value must survive untouched. + let after = fs::read_to_string(&fastly_toml).expect("read"); assert!( - combined.contains("would push `cfg_chunked` as chunked"), - "must report chunked: {combined}" + after.contains("\\\"version\\\":2"), + "the newer-format value must be left intact: {after}" ); } - /// Spec 12.7: pushing two blobs under different root keys - /// (e.g. `app_config` + `app_config_staging`) must leave both - /// keys readable from the local fastly.toml so the runtime - /// `EDGEZERO__STORES__CONFIG__APP_CONFIG__KEY` override can - /// switch between them. Prior to the upsert fix the second - /// push wholesale-replaced the per-store contents table. + /// The dry-run count must EXCLUDE a prior chunk that is already absent from + /// the file: the real prune's `remove()` is a no-op there, so counting it + /// would over-report the number of deletions. #[cfg(unix)] #[test] - fn push_config_entries_local_preserves_sibling_keys() { + fn push_config_entries_local_dry_run_excludes_already_missing_prior_chunks() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); - let store = ResolvedStoreId::from_logical(TEST_CONFIG_ID); - let ctx = AdapterPushContext::new(); + // Seed a multi-chunk generation. + let chunked = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(5_000)); FastlyCliAdapter .push_config_entries_local( dir.path(), Some("fastly.toml"), None, - &store, - &[("app_config".to_owned(), "{\"envelope\":\"A\"}".to_owned())], - &ctx, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), chunked)], + &AdapterPushContext::new(), false, ) - .expect("first push"); - FastlyCliAdapter + .expect("seed"); + + // Manually delete ONE chunk entry: a prior chunk that is already gone. + let mut doc: toml_edit::DocumentMut = fs::read_to_string(&fastly_toml) + .expect("read") + .parse() + .expect("parse"); + let contents = doc["local_server"]["config_stores"][TEST_CONFIG_ID]["contents"] + .as_table_mut() + .expect("contents"); + let chunk_keys: Vec = contents + .iter() + .map(|(key, _)| key.to_owned()) + .filter(|key| key.contains(CHUNK_KEY_INFIX)) + .collect(); + assert!(chunk_keys.len() >= 2, "seed must have chunked"); + contents.remove(&chunk_keys[0]); + let present_after = chunk_keys.len().saturating_sub(1); + fs::write(&fastly_toml, doc.to_string()).expect("write"); + + // Dry-run a shrink-to-direct re-push: every remaining chunk is an orphan. + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let out = FastlyCliAdapter .push_config_entries_local( dir.path(), Some("fastly.toml"), None, - &store, - &[( - "app_config_staging".to_owned(), - "{\"envelope\":\"B\"}".to_owned(), - )], - &ctx, - false, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct)], + &AdapterPushContext::new(), + true, ) - .expect("second push (sibling key)"); - - let raw = fs::read_to_string(&fastly_toml).expect("read"); - let doc: toml_edit::DocumentMut = raw.parse().expect("parse"); - let contents = doc - .get("local_server") - .and_then(|ls| ls.get("config_stores")) - .and_then(|cs| cs.get(TEST_CONFIG_ID)) - .and_then(|st| st.get("contents")) - .and_then(toml_edit::Item::as_table) - .expect("contents after sibling push"); - let app_config = contents - .get("app_config") - .and_then(toml_edit::Item::as_str) - .expect("default key must survive sibling push"); + .expect("dry-run"); + + let reported = out + .join("\n") + .split("would delete ") + .nth(1) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|n| n.parse::().ok()) + .expect("a numeric orphan count"); assert_eq!( - app_config, "{\"envelope\":\"A\"}", - "default key value: {raw}" + reported, present_after, + "the already-absent chunk must not be counted (reported {reported}, present {present_after})" ); - let staging = contents - .get("app_config_staging") - .and_then(toml_edit::Item::as_str) - .expect("staging key must be present"); - assert_eq!(staging, "{\"envelope\":\"B\"}", "staging key value: {raw}"); } + /// Dry-run reports the orphan count and writes nothing. #[cfg(unix)] #[test] - fn push_config_entries_local_writes_literal_dotted_chunk_keys() { + fn push_config_entries_local_dry_run_reports_orphan_count() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); - fs::write(&fastly_toml, "name = \"demo\"\n").expect("write"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; + let envelope_a = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); FastlyCliAdapter .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, + &[(TEST_CONFIG_ID.to_owned(), envelope_a)], &AdapterPushContext::new(), false, ) - .expect("local push must succeed"); + .expect("seed push"); + let before = fs::read_to_string(&fastly_toml).expect("read"); - let after = fs::read_to_string(&fastly_toml).expect("read back"); - // Chunk keys contain '.' and must appear as quoted string keys, - // not as TOML nested tables (which would look like [table.sub]). + let envelope_b = { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let data = json!({ "alt": "y".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:02Z".to_owned())) + .expect("envelope B") + }; + let out = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), envelope_b)], + &AdapterPushContext::new(), + true, // dry_run + ) + .expect("dry-run"); + + let combined = out.join("\n"); assert!( - after.contains(".__edgezero_chunks."), - "chunk keys written to fastly.toml: {after}" + combined.contains("would delete") && combined.contains("orphan chunks"), + "dry-run must report orphan count: {combined}" ); - // Parse with toml_edit and confirm chunk keys are string-keyed entries. - let doc: toml_edit::DocumentMut = after.parse().expect("must parse"); - let contents = doc - .get("local_server") - .and_then(|ls| ls.get("config_stores")) - .and_then(|cs| cs.get(TEST_CONFIG_ID)) - .and_then(|st| st.get("contents")) - .expect("contents table must exist"); - // At least one chunk key must be present as a string value (not a table). - let has_chunk_string = contents.as_table().is_some_and(|tbl| { - tbl.iter() - .any(|(key, val)| key.contains(".__edgezero_chunks.") && val.as_value().is_some()) - }); + assert_eq!( + fs::read_to_string(&fastly_toml).expect("read"), + before, + "dry-run must not edit fastly.toml" + ); + } + + /// Two concurrent local pushes must not lose each other's edit. Each thread + /// adds a DISTINCT key; the cross-process lock serialises the whole + /// read-modify-write, so the second push reads what the first wrote and both + /// keys survive. Without the lock, both would read the same base and the + /// later rename would discard the earlier key -- the silent data loss. + #[cfg(unix)] + #[test] + fn concurrent_local_pushes_do_not_lose_edits() { + use std::sync::Arc; + use std::thread; + + let dir = tempdir().expect("tempdir"); + let path = Arc::new(dir.path().join("fastly.toml")); + fs::write(path.as_ref(), "name = \"demo\"\n").expect("seed"); + + // Many rounds to make the interleaving likely to hit the race window. + for round in 0_u32..25 { + let path_a = Arc::clone(&path); + let path_b = Arc::clone(&path); + let key_a = format!("alpha_{round}"); + let key_b = format!("beta_{round}"); + let (ka, kb) = (key_a.clone(), key_b.clone()); + let ta = thread::spawn(move || { + write_fastly_local_config_store( + &path_a, + TEST_CONFIG_ID, + &[(ka, "a".to_owned())], + &[], + ) + }); + let tb = thread::spawn(move || { + write_fastly_local_config_store( + &path_b, + TEST_CONFIG_ID, + &[(kb, "b".to_owned())], + &[], + ) + }); + ta.join().expect("thread a").expect("push a"); + tb.join().expect("thread b").expect("push b"); + + let after = fs::read_to_string(path.as_ref()).expect("read back"); + assert!( + after.contains(&format!("{key_a} = \"a\"")), + "round {round}: `{key_a}` was lost by a concurrent push:\n{after}" + ); + assert!( + after.contains(&format!("{key_b} = \"b\"")), + "round {round}: `{key_b}` was lost by a concurrent push:\n{after}" + ); + } + } + + /// A concurrent edit to `fastly.toml` between the push's read and its write + /// must NOT be clobbered: the rewrite refuses and reports, leaving the other + /// writer's file intact so no sibling change is silently lost. + #[cfg(unix)] + #[test] + fn local_rewrite_refuses_to_clobber_a_concurrent_edit() { + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + + // Simulate: we read one thing, another writer moved the file, we write. + let stale_view = "name = \"demo\"\n"; + fs::write(&fastly_toml, "name = \"demo\"\nother = \"sibling edit\"\n") + .expect("concurrent write"); + + let err = atomically_replace_file(&fastly_toml, stale_view, "name = \"clobbered\"\n") + .expect_err("a concurrent edit must not be overwritten"); assert!( - has_chunk_string, - "chunk keys must be literal string-valued entries, not nested tables: {after}" + err.contains("changed on disk"), + "must report the conflict: {err}" + ); + assert_eq!( + fs::read_to_string(&fastly_toml).expect("read"), + "name = \"demo\"\nother = \"sibling edit\"\n", + "the other writer's file must survive untouched" ); } + /// The happy path replaces contents and leaves no temp file behind. #[cfg(unix)] #[test] - fn push_config_entries_local_dry_run_reports_chunking_and_does_not_edit_fastly_toml() { - use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + fn local_rewrite_replaces_atomically_and_cleans_up() { let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); - let original = "name = \"demo\"\n"; - fs::write(&fastly_toml, original).expect("write"); + fs::write(&fastly_toml, "before\n").expect("seed"); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let entries = vec![(TEST_CONFIG_ID.to_owned(), envelope)]; - let out = FastlyCliAdapter - .push_config_entries_local( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &entries, - &AdapterPushContext::new(), - true, // dry_run - ) - .expect("local dry-run must not error"); + atomically_replace_file(&fastly_toml, "before\n", "after\n").expect("replace"); + assert_eq!( + fs::read_to_string(&fastly_toml).expect("read"), + "after\n", + "contents must be replaced" + ); + let leftovers: Vec<_> = fs::read_dir(dir.path()) + .expect("read_dir") + .filter_map(Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().contains(".tmp")) + .collect(); + assert!(leftovers.is_empty(), "no temp file may be left behind"); + } - // File must be untouched. - let after = fs::read_to_string(&fastly_toml).expect("read back"); - assert_eq!(after, original, "dry-run must not edit fastly.toml"); + /// The atomic replace must PRESERVE the target's permissions: a 0600 manifest + /// must not widen to the umask default when it is replaced. + #[cfg(unix)] + #[test] + fn atomic_replace_preserves_restrictive_permissions() { + use std::os::unix::fs::PermissionsExt as _; + let dir = tempdir().expect("tempdir"); + let manifest = dir.path().join("fastly.toml"); + fs::write(&manifest, "before\n").expect("seed"); + fs::set_permissions(&manifest, fs::Permissions::from_mode(0o600)).expect("chmod"); - // Output must describe chunking intent. - let combined = out.join("\n"); - assert!( - combined.contains("would set") && combined.contains("chunked"), - "must report chunked intent: {combined}" + atomically_replace_file(&manifest, "before\n", "after\n").expect("replace"); + + let mode = fs::metadata(&manifest).expect("meta").permissions().mode() & 0o777; + assert_eq!( + mode, 0o600, + "restrictive permissions must survive the replace" ); + assert_eq!(fs::read_to_string(&manifest).expect("read"), "after\n"); } - // ---------- chunked read integration tests ---------- - + /// A symlinked manifest must be updated THROUGH the link: the real file's + /// contents change and the symlink itself is preserved (not replaced with a + /// regular file). The lock and the replace both resolve to the real target. #[cfg(unix)] #[test] - fn read_config_entry_resolves_direct_value_unchanged() { - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; - let _lock = path_mutation_guard().lock().expect("guard"); + fn local_rewrite_follows_a_symlinked_manifest() { + use std::os::unix::fs::symlink; let dir = tempdir().expect("tempdir"); + let real = dir.path().join("real-fastly.toml"); + let link = dir.path().join("fastly.toml"); + fs::write(&real, "name = \"demo\"\n").expect("seed real"); + symlink(&real, &link).expect("symlink"); - let envelope = BlobEnvelope::new(json!({"hello": "world"}), "2026-06-22T00:00:00Z".into()); - let json_str = serde_json::to_string(&envelope).unwrap(); - let item_json = format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(&json_str).unwrap() - ); - let fake = fake_fastly_returning(&item_json, "", 0); - let _path = PathPrepend::new(fake.path()); + write_fastly_local_config_store( + &link, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "hi".to_owned())], + &[], + ) + .expect("push through symlink"); - let result = FastlyCliAdapter - .read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "cfg", - &AdapterPushContext::new(), - ) - .expect("read must succeed"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present"); - }; - assert_eq!(value, json_str, "direct envelope passes through unchanged"); + assert!( + fs::symlink_metadata(&link) + .expect("lstat") + .file_type() + .is_symlink(), + "the manifest symlink must be preserved, not replaced with a file" + ); + assert!( + fs::read_to_string(&real) + .expect("read real") + .contains("greeting = \"hi\""), + "the real target behind the symlink must be updated" + ); } + /// A DANGLING manifest symlink (points at a not-yet-created file) must be + /// FOLLOWED: the write creates the intended target and preserves the symlink, + /// rather than replacing the link with a regular file and leaving the target + /// absent. #[cfg(unix)] #[test] - fn read_config_entry_reconstructs_chunked_envelope() { - use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; - let _lock = path_mutation_guard().lock().expect("guard"); + fn local_rewrite_follows_a_dangling_symlinked_manifest() { + use std::os::unix::fs::symlink; let dir = tempdir().expect("tempdir"); + let target = dir.path().join("real-fastly.toml"); // does NOT exist yet + let link = dir.path().join("fastly.toml"); + symlink(&target, &link).expect("dangling symlink"); + assert!(!target.exists(), "target must start absent"); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); - let (_, pointer_json) = physical.last().unwrap(); - // Build a key→response map for every physical entry. - let mut key_responses: Vec<(String, String)> = Vec::new(); - for (pk, pv) in &physical { - let resp = format!(r#"{{"item_value":{}}}"#, serde_json::to_string(pv).unwrap()); - key_responses.push((pk.clone(), resp)); - } - // The root key should return the pointer. - let ptr_resp = format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(pointer_json).unwrap() + write_fastly_local_config_store( + &link, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "hi".to_owned())], + &[], + ) + .expect("push through dangling symlink"); + + assert!( + fs::symlink_metadata(&link) + .expect("lstat") + .file_type() + .is_symlink(), + "the symlink must be preserved, not replaced with a regular file" ); - key_responses.push((TEST_CONFIG_ID.to_owned(), ptr_resp)); + assert!( + fs::read_to_string(&target) + .expect("target must now exist") + .contains("greeting = \"hi\""), + "the intended (formerly-missing) target must be created and written" + ); + } - let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); - let _path = PathPrepend::new(fake.path()); + /// A MULTI-HOP dangling symlink chain (fastly.toml -> middle.toml -> + /// missing.toml) must be followed to the FINAL target: the last file is + /// created and every intermediate link is preserved. A direct write to the + /// final target also resolves to the same lock. + #[cfg(unix)] + #[test] + fn local_rewrite_follows_a_multi_hop_dangling_symlink_chain() { + use std::os::unix::fs::symlink; + let dir = tempdir().expect("tempdir"); + let final_target = dir.path().join("missing.toml"); // absent + let middle = dir.path().join("middle.toml"); + let link = dir.path().join("fastly.toml"); + symlink(&final_target, &middle).expect("middle -> missing"); + symlink(&middle, &link).expect("fastly -> middle"); + assert!(!final_target.exists(), "final target must start absent"); - let result = FastlyCliAdapter - .read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - TEST_CONFIG_ID, - &AdapterPushContext::new(), - ) - .expect("chunked read must succeed"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present"); - }; + write_fastly_local_config_store( + &link, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "hi".to_owned())], + &[], + ) + .expect("push through the symlink chain"); + + for intermediate in [&link, &middle] { + assert!( + fs::symlink_metadata(intermediate) + .expect("lstat") + .file_type() + .is_symlink(), + "every intermediate link must be preserved: {}", + intermediate.display() + ); + } + assert!( + fs::read_to_string(&final_target) + .expect("final target must now exist") + .contains("greeting = \"hi\""), + "the final target at the end of the chain must be created and written" + ); + // Symmetry: a direct write to the final target resolves to the same real + // file the chain does, so both share one lock. assert_eq!( - value, envelope, - "reconstructed envelope must equal original" + canonical_manifest_target(&link).expect("chain resolves"), + canonical_manifest_target(&final_target).expect("direct resolves"), + "the chain and a direct path must resolve to the same lock target" ); } + /// A HARD-LINKED manifest cannot be replaced safely (rename breaks the link; + /// path-based locks miss the other names), so the writer FAILS CLOSED with a + /// fix rather than silently diverging. #[cfg(unix)] #[test] - fn read_config_entry_errors_on_missing_chunk() { - use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; - let _lock = path_mutation_guard().lock().expect("guard"); + fn local_rewrite_refuses_a_hard_linked_manifest() { let dir = tempdir().expect("tempdir"); + let manifest = dir.path().join("fastly.toml"); + let other = dir.path().join("other-name.toml"); + fs::write(&manifest, "name = \"demo\"\n").expect("seed"); + fs::hard_link(&manifest, &other).expect("hard link"); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); - let (_, pointer_json) = physical.last().unwrap(); - // Only provide the root pointer; omit chunk responses so chunk fetch returns not-found. - let ptr_resp = format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(pointer_json).unwrap() - ); - let key_responses = vec![(TEST_CONFIG_ID.to_owned(), ptr_resp)]; - let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); - let _path = PathPrepend::new(fake.path()); - - let result = FastlyCliAdapter.read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + let err = write_fastly_local_config_store( + &manifest, TEST_CONFIG_ID, - &AdapterPushContext::new(), - ); - let Err(err) = result else { - panic!("missing chunk must error") - }; + &[("greeting".to_owned(), "hi".to_owned())], + &[], + ) + .expect_err("a hard-linked manifest must be refused"); assert!( - err.contains("missing chunk"), - "error must mention missing chunk: {err}" + err.contains("hard link"), + "must explain the hard-link refusal: {err}" ); + // Nothing was written -- the original content is intact. + assert_eq!( + fs::read_to_string(&manifest).expect("read"), + "name = \"demo\"\n", + "a refused write must not modify the manifest" + ); + } + + /// A provision write and a local push serialise on the SAME manifest lock, so + /// neither loses the other's edit even though they are different writers. + #[cfg(unix)] + #[test] + fn provision_and_push_serialise_on_the_manifest_lock() { + use std::sync::Arc; + use std::thread; + + let dir = tempdir().expect("tempdir"); + let manifest = Arc::new(dir.path().join("fastly.toml")); + fs::write(manifest.as_ref(), "name = \"demo\"\n").expect("seed"); + + for _round in 0_u32..25 { + let p_provision = Arc::clone(&manifest); + let p_push = Arc::clone(&manifest); + let provision = + thread::spawn(move || append_fastly_setup(&p_provision, "config", "app_config")); + let push = thread::spawn(move || { + write_fastly_local_config_store( + &p_push, + TEST_CONFIG_ID, + &[("greeting".to_owned(), "hi".to_owned())], + &[], + ) + }); + provision + .join() + .expect("provision thread") + .expect("provision"); + push.join().expect("push thread").expect("push"); + + let after = fs::read_to_string(manifest.as_ref()).expect("read"); + assert!( + after.contains("[setup.config_stores.app_config]"), + "provision's setup block must survive:\n{after}" + ); + assert!( + after.contains("greeting = \"hi\""), + "push's config edit must survive:\n{after}" + ); + } } + /// PARITY: the dry-run's reported orphan count equals the number of chunk + /// keys the real (non-dry-run) push actually deletes, on ONE fixture. A + /// divergence would make the dry-run a misleading preview of the delete. #[cfg(unix)] #[test] - fn read_config_entry_errors_on_corrupt_chunk_hash() { + fn push_config_entries_local_dry_run_count_matches_real_deletions() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; - let _lock = path_mutation_guard().lock().expect("guard"); + + fn count_chunk_keys(toml_src: &str) -> usize { + let doc: toml_edit::DocumentMut = toml_src.parse().expect("parse"); + doc.get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .map_or(0, |table| { + table + .iter() + .filter(|(key, _)| key.contains(CHUNK_KEY_INFIX)) + .count() + }) + } + fn parse_would_delete_count(text: &str) -> Option { + let marker = "would delete "; + let idx = text.find(marker)?; + text.get(idx.saturating_add(marker.len())..)? + .split_whitespace() + .next()? + .parse::() + .ok() + } + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); - let (_, pointer_json) = physical.last().unwrap(); - let mut key_responses: Vec<(String, String)> = Vec::new(); - // Corrupt first chunk's content. - let (first_chunk_key, first_chunk_val) = &physical[0]; - let corrupted: String = first_chunk_val.chars().map(|_| 'Z').collect(); - let corrupt_resp = format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(&corrupted).unwrap() + // Seed a multi-chunk generation, then measure how many chunk keys exist. + let chunked = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(5_000)); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), chunked)], + &AdapterPushContext::new(), + false, + ) + .expect("seed push"); + let seeded = fs::read_to_string(&fastly_toml).expect("read"); + let prior_chunk_count = count_chunk_keys(&seeded); + assert!(prior_chunk_count >= 2, "seed must have chunked: {seeded}"); + + // Dry-run a shrink-to-direct re-push: capture the reported orphan count. + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let out = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct.clone())], + &AdapterPushContext::new(), + true, // dry_run + ) + .expect("dry-run"); + let reported = parse_would_delete_count(&out.join("\n")) + .expect("dry-run must report a numeric orphan count"); + assert_eq!( + fs::read_to_string(&fastly_toml).expect("read"), + seeded, + "dry-run must not edit fastly.toml" ); - key_responses.push((first_chunk_key.clone(), corrupt_resp)); - // Remaining chunks as normal. - for (pk, pv) in physical - .iter() - .take(physical.len().saturating_sub(1)) - .skip(1) - { - key_responses.push(( - pk.clone(), - format!(r#"{{"item_value":{}}}"#, serde_json::to_string(pv).unwrap()), - )); - } - key_responses.push(( - TEST_CONFIG_ID.to_owned(), - format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(pointer_json).unwrap() - ), - )); - let fake = fake_fastly_with_key_dispatch(dir.path(), &key_responses); - let _path = PathPrepend::new(fake.path()); - let result = FastlyCliAdapter.read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - TEST_CONFIG_ID, - &AdapterPushContext::new(), + // Real re-push: count the chunk keys actually removed. + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct)], + &AdapterPushContext::new(), + false, + ) + .expect("real push"); + let after = fs::read_to_string(&fastly_toml).expect("read"); + let actually_deleted = prior_chunk_count.saturating_sub(count_chunk_keys(&after)); + + assert_eq!( + reported, actually_deleted, + "dry-run count {reported} must equal real deletions {actually_deleted}" ); - let Err(err) = result else { - panic!("corrupt chunk must error") - }; - assert!( - err.contains("SHA mismatch") || err.contains("mismatch"), - "error must mention hash mismatch: {err}" + assert_eq!( + reported, prior_chunk_count, + "a shrink-to-direct re-push orphans every prior chunk" ); } + /// Real (non-dry-run) push over a MALFORMED prior pointer WARNS and deletes + /// nothing: its chunk list is untrustworthy, so no key is removed and the + /// root is simply overwritten with the new value. This is the real-push + /// counterpart to the dry-run "unknown" degradation on the same prior state. #[cfg(unix)] #[test] - fn read_config_entry_errors_on_malformed_pointer() { - let _lock = path_mutation_guard().lock().expect("guard"); - let dir = tempdir().expect("tempdir"); - // Root value is JSON but neither a BlobEnvelope nor a valid pointer. - let bad_json = r#"{"some_field":"not a pointer or envelope"}"#; - let item_json = format!( - r#"{{"item_value":{}}}"#, - serde_json::to_string(bad_json).unwrap() + fn push_config_entries_local_real_push_over_malformed_prior_warns_and_deletes_nothing() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + // A pointer-kind prior value missing its required fields — malformed, so + // `prior_chunk_keys` returns Err (warn, delete nothing). + let seed = concat!( + "name = \"demo\"\n\n", + "[local_server.config_stores.app_config]\n", + "format = \"inline-toml\"\n\n", + "[local_server.config_stores.app_config.contents]\n", + "app_config = \"{\\\"edgezero_kind\\\":\\\"fastly_config_chunks\\\",\\\"version\\\":1}\"\n", ); - let fake = fake_fastly_returning(&item_json, "", 0); - let _path = PathPrepend::new(fake.path()); + fs::write(&fastly_toml, seed).expect("seed"); + + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let out = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), direct.clone())], + &AdapterPushContext::new(), + false, + ) + .expect("real push must not fail on a malformed prior"); - let result = FastlyCliAdapter.read_config_entry( - dir.path(), - Some("fastly.toml"), - None, - &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "cfg", - &AdapterPushContext::new(), - ); - let Err(err) = result else { - panic!("malformed pointer must error") - }; assert!( - err.contains("neither a valid BlobEnvelope") || err.contains("chunk pointer"), - "error must describe parse failure: {err}" + out.iter().any(|line| line.contains("skipping chunk GC")), + "must warn about the malformed prior pointer: {out:?}" ); - } - // ---------- local read integration tests ---------- + let after = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = after.parse().expect("parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents"); + assert_eq!( + contents + .get(TEST_CONFIG_ID) + .and_then(toml_edit::Item::as_str), + Some(direct.as_str()), + "root is overwritten with the new direct envelope: {after}" + ); + } + /// Dry-run of an identical re-push reports zero orphans (new keys + /// equal prior keys — regression for expanding `new_keys`). + #[cfg(unix)] #[test] - fn read_config_entry_local_resolves_direct_value() { - use edgezero_core::blob_envelope::BlobEnvelope; - use serde_json::json; + fn push_config_entries_local_dry_run_identical_repush_counts_zero() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); - let envelope = BlobEnvelope::new(json!({"x": 1_i32}), "2026-06-22T00:00:00Z".into()); - let json_str = serde_json::to_string(&envelope).unwrap(); - // Write directly as a single entry (not via push_config_entries_local so we - // control the exact TOML content). - write_fastly_local_config_store( - &fastly_toml, - TEST_CONFIG_ID, - &[("cfg".to_owned(), json_str.clone())], - ) - .expect("write"); + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(TEST_CONFIG_ID.to_owned(), envelope.clone())], + &AdapterPushContext::new(), + false, + ) + .expect("seed push"); - let result = FastlyCliAdapter - .read_config_entry_local( + let out = FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - "cfg", + &[(TEST_CONFIG_ID.to_owned(), envelope)], &AdapterPushContext::new(), + true, // dry_run, same bytes ) - .expect("local read must succeed"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present"); - }; - assert_eq!(value, json_str, "direct envelope passes through unchanged"); + .expect("dry-run"); + + assert!( + out.join("\n").contains("would delete 0 orphan chunks"), + "identical re-push must count 0 orphans: {out:?}" + ); } + /// Dry-run over a suspicious prior pointer reports an unknown count + /// and does not fail. + #[cfg(unix)] #[test] - fn read_config_entry_local_reconstructs_chunked_envelope() { + fn push_config_entries_local_dry_run_suspicious_prior_pointer_unknown() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); + let seed = concat!( + "name = \"demo\"\n\n", + "[local_server.config_stores.app_config]\n", + "format = \"inline-toml\"\n\n", + "[local_server.config_stores.app_config.contents]\n", + "app_config = \"{\\\"edgezero_kind\\\":\\\"fastly_config_chunks\\\",\\\"version\\\":1}\"\n", + ); + fs::write(&fastly_toml, seed).expect("seed"); - let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - let physical = prepare_fastly_config_entries(TEST_CONFIG_ID, &envelope).unwrap(); - // Write all physical entries (chunks + pointer) to the local store. - write_fastly_local_config_store(&fastly_toml, TEST_CONFIG_ID, &physical).expect("write"); - - let result = FastlyCliAdapter - .read_config_entry_local( + let direct = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let out = FastlyCliAdapter + .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - TEST_CONFIG_ID, + &[(TEST_CONFIG_ID.to_owned(), direct)], &AdapterPushContext::new(), + true, // dry_run ) - .expect("local chunked read must succeed"); - let ReadConfigEntry::Present(value) = result else { - panic!("expected Present"); - }; - assert_eq!( - value, envelope, - "reconstructed envelope must equal original" + .expect("dry-run must not fail on suspicious pointer"); + + assert!( + out.join("\n").contains("unknown: suspicious prior pointer"), + "dry-run must degrade to unknown: {out:?}" ); } - /// Spec 12.3 + 9.3: a second oversized push must converge the - /// runtime on the NEW envelope — chunk keys are content-addressed - /// by the full-envelope SHA, so push B writes a new chunk-set and - /// installs a new root pointer. - /// - /// The local fastly.toml writer upserts per-key (so a sibling - /// `--key app_config_staging` push leaves `app_config` intact per - /// spec 12.7). Within the SAME root key, old chunks for envelope - /// A remain in the contents table after envelope B's push — they're - /// unreferenced (the root pointer at `app_config` now names B's - /// chunks), matching the remote Fastly behaviour where the - /// per-entry `update --upsert` shell-out has no atomic-delete - /// pairing. The runtime-correctness property holds either way: a - /// read after push B follows the active pointer and reconstructs - /// envelope B, not A. + /// A present-but-malformed `contents` (non-table) is prior state the + /// real writer would reject — the dry-run count must degrade to + /// `unknown: could not read prior state`, not silently report 0. #[cfg(unix)] #[test] - #[expect( - clippy::too_many_lines, - reason = "linear test scenario: push A, inspect, push B, inspect, read; splitting would obscure the chunk-set comparison" - )] - fn second_oversized_push_converges_runtime_on_new_envelope() { + fn push_config_entries_local_dry_run_non_table_contents_unknown() { use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; let dir = tempdir().expect("tempdir"); let fastly_toml = dir.path().join("fastly.toml"); - fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + let seed = concat!( + "name = \"demo\"\n\n", + "[local_server.config_stores.app_config]\n", + "format = \"inline-toml\"\n", + "contents = \"bad\"\n", + ); + fs::write(&fastly_toml, seed).expect("seed"); - // First push: envelope A. Records the chunk-key set so we can - // confirm they survive the second push (no garbage collection - // in v1 — spec 9.3 + Q6). - let envelope_a = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); - FastlyCliAdapter + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let out = FastlyCliAdapter .push_config_entries_local( dir.path(), Some("fastly.toml"), None, &ResolvedStoreId::from_logical(TEST_CONFIG_ID), - &[(TEST_CONFIG_ID.to_owned(), envelope_a.clone())], + &[(TEST_CONFIG_ID.to_owned(), envelope)], &AdapterPushContext::new(), - false, + true, // dry_run ) - .expect("first push must succeed"); + .expect("dry-run must not fail on malformed contents"); - let after_a = fs::read_to_string(&fastly_toml).expect("read"); - let doc_a: toml_edit::DocumentMut = after_a.parse().expect("parse"); - let contents_a = doc_a - .get("local_server") - .and_then(|ls| ls.get("config_stores")) - .and_then(|cs| cs.get(TEST_CONFIG_ID)) - .and_then(|st| st.get("contents")) - .and_then(toml_edit::Item::as_table) - .expect("contents table after push A"); - let chunks_a: Vec = contents_a - .iter() - .map(|(key, _)| key.to_owned()) - .filter(|key| key.contains(".__edgezero_chunks.")) - .collect(); assert!( - !chunks_a.is_empty(), - "push A must have produced chunk entries: {after_a}" + out.join("\n") + .contains("unknown: could not read prior state"), + "non-table contents must degrade to unknown: {out:?}" ); + } - // Second push: a DIFFERENT oversized envelope B. The - // content-addressed chunk keys must shift to B's sha; old - // A-chunks may remain in the table (v1 doesn't GC). Build - // envelope B with a distinct payload key so its SHA differs - // from A's even at the same total length. - let envelope_b = { + /// A duplicate root key in one batch is rejected before any I/O. + /// Otherwise the earlier tuple's GC plan would reclaim the chunks the + /// LAST tuple just installed, leaving the final pointer dangling. + /// Regression: prior B, batch `[(root, A), (root, B)]` — the root must + /// still resolve afterwards. + #[cfg(unix)] + #[test] + fn push_config_entries_local_rejects_duplicate_root_keys() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + + let make = |tag: &str| { use edgezero_core::blob_envelope::BlobEnvelope; use serde_json::json; - let data = json!({ "alt": "x".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); - serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:01Z".to_owned())) - .expect("envelope B serialises") + let data = json!({ tag: "x".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".to_owned())) + .expect("envelope") }; - assert_ne!(envelope_a, envelope_b, "test fixtures must differ"); + let envelope_a = make("aaa"); + let envelope_b = make("bbb"); + + // Prior generation B is live. FastlyCliAdapter .push_config_entries_local( dir.path(), @@ -3341,50 +9071,35 @@ build = \"cargo build --release\" &AdapterPushContext::new(), false, ) - .expect("second push must succeed"); + .expect("seed push"); + let before = fs::read_to_string(&fastly_toml).expect("read"); - let after_b = fs::read_to_string(&fastly_toml).expect("read"); - let doc_b: toml_edit::DocumentMut = after_b.parse().expect("parse"); - let contents_b = doc_b - .get("local_server") - .and_then(|ls| ls.get("config_stores")) - .and_then(|cs| cs.get(TEST_CONFIG_ID)) - .and_then(|st| st.get("contents")) - .and_then(toml_edit::Item::as_table) - .expect("contents table after push B"); - let chunks_b: Vec = contents_b - .iter() - .map(|(key, _)| key.to_owned()) - .filter(|key| key.contains(".__edgezero_chunks.")) - .collect(); + // Duplicate-root batch must be rejected outright. + let err = FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[ + (TEST_CONFIG_ID.to_owned(), envelope_a), + (TEST_CONFIG_ID.to_owned(), envelope_b.clone()), + ], + &AdapterPushContext::new(), + false, + ) + .expect_err("duplicate root keys must be rejected"); assert!( - !chunks_b.is_empty(), - "push B must have produced chunk entries: {after_b}" + err.contains("more than once"), + "error explains the duplicate: {err}" ); - - // Chunk keys are content-addressed by envelope SHA, so the B - // push installs a fresh chunk-set whose keys are all distinct - // from A's. Under the upsert semantic the A-chunks remain in - // the contents table (no GC in v1); B's chunks are simply added. - let new_b_chunks: Vec<&String> = chunks_b - .iter() - .filter(|key| !chunks_a.contains(*key)) - .collect(); - assert!( - !new_b_chunks.is_empty(), - "push B must have added at least one new content-addressed chunk: A-set={chunks_a:?} B-set={chunks_b:?}" + assert_eq!( + fs::read_to_string(&fastly_toml).expect("read"), + before, + "rejection must happen before any write" ); - // Old A-chunks remain in the table (orphan-but-present — - // matches the remote Fastly write-only-upsert semantic). - for chunk_key in &chunks_a { - assert!( - chunks_b.contains(chunk_key), - "old A-chunk `{chunk_key}` must remain in the local table after push B (v1 has no GC); B-set={chunks_b:?}" - ); - } - // Runtime-correctness property: a fresh read after push B - // reconstructs envelope B (NOT envelope A). + // The live root still resolves to B (nothing was reclaimed). let read = FastlyCliAdapter .read_config_entry_local( dir.path(), @@ -3394,17 +9109,134 @@ build = \"cargo build --release\" TEST_CONFIG_ID, &AdapterPushContext::new(), ) - .expect("local read after push B"); + .expect("root must still resolve"); let ReadConfigEntry::Present(value) = read else { - panic!("expected Present after push B"); + panic!("expected Present"); }; - assert_eq!( - value, envelope_b, - "read after second push must reconstruct envelope B, not A" - ); - assert_ne!( - value, envelope_a, - "old envelope A's chunks must be inert -- read must NOT return A" - ); + assert_eq!(value, envelope_b, "root still reconstructs envelope B"); + } + + /// GC of a chunked root must not touch a chunked SIBLING's chunks — + /// the prefix `app_config.__edgezero_chunks.` must not match + /// `app_config_staging.__edgezero_chunks.` (shared string prefix). + #[cfg(unix)] + #[test] + fn push_config_entries_local_gc_preserves_sibling_chunks() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let dir = tempdir().expect("tempdir"); + let fastly_toml = dir.path().join("fastly.toml"); + fs::write(&fastly_toml, "name = \"demo\"\n").expect("seed"); + + let make = |tag: &str| { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + let data = json!({ tag: "x".repeat(FASTLY_CONFIG_ENTRY_LIMIT) }); + serde_json::to_string(&BlobEnvelope::new(data, "2026-06-22T00:00:00Z".to_owned())) + .expect("envelope") + }; + let push = |key: &str, body: String| { + FastlyCliAdapter + .push_config_entries_local( + dir.path(), + Some("fastly.toml"), + None, + &ResolvedStoreId::from_logical(TEST_CONFIG_ID), + &[(key.to_owned(), body)], + &AdapterPushContext::new(), + false, + ) + .expect("push"); + }; + + // app_config gen X, then a chunked sibling, then app_config gen Z. + push("app_config", make("x1")); + push("app_config_staging", make("staging")); + let staging_chunks = chunk_keys_of("app_config_staging", &make("staging")); + push("app_config", make("z2")); // GCs app_config's gen-X chunks + + let after = fs::read_to_string(&fastly_toml).expect("read"); + let doc: toml_edit::DocumentMut = after.parse().expect("parse"); + let contents = doc + .get("local_server") + .and_then(|ls| ls.get("config_stores")) + .and_then(|cs| cs.get(TEST_CONFIG_ID)) + .and_then(|st| st.get("contents")) + .and_then(toml_edit::Item::as_table) + .expect("contents"); + for key in &staging_chunks { + assert!( + contents.get(key).is_some(), + "sibling chunk `{key}` must survive app_config GC: {after}" + ); + } + } + + // ---- chunk GC helpers ---- + + #[test] + fn reject_reserved_root_keys_accepts_clean_keys() { + let entries = vec![ + ("app_config".to_owned(), "{}".to_owned()), + ("app_config_staging".to_owned(), "{}".to_owned()), + ]; + reject_reserved_root_keys(&entries).expect("clean keys accepted"); + } + + #[test] + fn reject_reserved_root_keys_rejects_infix_key() { + let bad = format!("app_config{CHUNK_KEY_INFIX}deadbeef.0"); + let entries = vec![(bad.clone(), "{}".to_owned())]; + let err = reject_reserved_root_keys(&entries).expect_err("reserved infix must reject"); + assert!(err.contains(&bad), "error names the key: {err}"); + assert!(err.contains("reserved"), "error explains why: {err}"); + } + + #[test] + fn orphan_chunk_keys_subtracts_new_keys() { + let mut new_keys = HashSet::new(); + new_keys.insert("keep".to_owned()); + let plan = FastlyConfigGcPlan { + new_keys, + prior_keys: Ok(vec![ + "gone1".to_owned(), + "keep".to_owned(), + "gone2".to_owned(), + ]), + }; + let orphans = orphan_chunk_keys(&plan).expect("ok"); + assert_eq!(orphans, vec!["gone1".to_owned(), "gone2".to_owned()]); + } + + #[test] + fn orphan_chunk_keys_propagates_prior_err() { + let plan = FastlyConfigGcPlan { + new_keys: HashSet::new(), + prior_keys: Err("suspicious".to_owned()), + }; + orphan_chunk_keys(&plan).unwrap_err(); + } + + #[test] + fn expand_root_direct_value_has_single_entry() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT); + let (expanded, new_keys, new_root_value) = expand_root(TEST_CONFIG_ID, &envelope).unwrap(); + assert_eq!(expanded.len(), 1); + assert_eq!(new_root_value, envelope); + assert!(new_keys.contains(TEST_CONFIG_ID)); + assert_eq!(new_keys.len(), 1); + } + + #[test] + fn expand_root_chunked_value_carries_pointer_as_root_value() { + use crate::chunked_config::FASTLY_CONFIG_ENTRY_LIMIT; + let envelope = make_test_envelope(FASTLY_CONFIG_ENTRY_LIMIT.saturating_add(1)); + let (expanded, new_keys, new_root_value) = expand_root(TEST_CONFIG_ID, &envelope).unwrap(); + assert!(expanded.len() >= 2, "chunks + pointer"); + let (last_key, last_value) = expanded.last().unwrap(); + assert_eq!(last_key, TEST_CONFIG_ID); + assert_eq!(&new_root_value, last_value); + assert!(new_keys.contains(TEST_CONFIG_ID)); + assert_eq!(new_keys.len(), expanded.len()); } } diff --git a/crates/edgezero-adapter-fastly/src/config_store.rs b/crates/edgezero-adapter-fastly/src/config_store.rs index 6896369f..bad34170 100644 --- a/crates/edgezero-adapter-fastly/src/config_store.rs +++ b/crates/edgezero-adapter-fastly/src/config_store.rs @@ -1,9 +1,10 @@ //! Fastly adapter config store: wraps `fastly::ConfigStore`. +use std::cell::Cell; #[cfg(test)] use std::collections::HashMap; -use crate::chunked_config::resolve_fastly_config_value; +use crate::chunked_config::resolve_fastly_config_value_typed; use async_trait::async_trait; use edgezero_core::config_store::{ConfigStore, ConfigStoreError}; use fastly::ConfigStore as FastlyConfigStoreInner; @@ -28,18 +29,6 @@ impl FastlyConfigStore { } } - /// Synchronous key lookup used by the chunk-pointer resolver callback. - /// Returns `Ok(Some(value))`, `Ok(None)` (missing), or `Err(message)`. - fn get_sync(&self, key: &str) -> Result, String> { - match &self.inner { - FastlyConfigStoreBackend::Fastly(inner) => inner - .try_get(key) - .map_err(|err| format!("config store lookup failed for `{key}`: {err}")), - #[cfg(test)] - FastlyConfigStoreBackend::InMemory(data) => Ok(data.get(key).cloned()), - } - } - /// Open a Fastly Config Store by resource link name. /// /// Returns an error if the configured store cannot be opened. @@ -68,30 +57,116 @@ impl ConfigStore for FastlyConfigStore { let Some(value) = root_value else { return Ok(None); }; - // Resolve chunk pointers transparently. Direct BlobEnvelope values - // pass through unchanged; pointer values fan out to chunk entries - // in the same store. Missing / malformed / hash-mismatched chunks - // are corrupt platform state — spec 9.3 (line 6272) calls this an - // internal config-store error with re-push remediation, NOT a - // transient unavailable. Mapping to `internal` surfaces as HTTP - // 500 and pushes operators toward ` config push` instead - // of waiting for a 503 to clear. - let resolved = resolve_fastly_config_value(key, value, |chunk_key| { - self.get_sync(chunk_key) - }) - .map_err(|err| { - log::warn!( - "Fastly config-store chunk resolution failed for `{key}`: {err}. \ - Re-run ` config push` to repair the store." - ); - ConfigStoreError::internal(anyhow::anyhow!( - "config store entry is corrupt or incomplete; re-run config push to repair: {err}" - )) - })?; - Ok(Some(resolved)) + // Resolve chunk pointers transparently. Direct BlobEnvelope values and + // any other raw value pass through; pointer values fan out to chunk + // entries in the same store. + // + // A chunk fetch can fail in two classes that must NOT collapse to one + // status: + // - Corrupt state (a hash mismatch, a bad/oversized derived key) → + // Internal (HTTP 500) with re-push remediation, per spec 9.3. + // Re-pushing rewrites the generation and fixes it. + // - Transient (invalid store handle, lookup exhaustion, an unclassified + // error, a value that outgrew the read buffer, OR a referenced chunk + // not yet visible at this POP) → Unavailable (HTTP 503, retryable). + // + // A MISSING referenced chunk is deliberately transient. Config Store is + // eventually consistent ACROSS keys, so right after a push the flipped + // root pointer can be visible at a POP before all of its + // content-addressed chunks have propagated there. That gap is not + // corruption — a retry moments later resolves it — so it must read as 503, + // not a re-push-me 500. (Genuine, lasting corruption then shows as a + // persistent 503 the operator repairs by re-pushing; that is strictly + // safer than a spurious 500 during the normal propagation window.) + // `transient` records whether any chunk fetch hit that class so the outer + // result can pick the right status. + // + // A DIRECT value is returned VERBATIM (the resolver only touches our chunk + // pointers): the store layer must not parse or judge arbitrary values -- + // that is the typed app-config extractor's job, which gives the + // upgrade/redeploy remediation for a newer DIRECT envelope. Only a value + // the resolver actually processes (an unknown `edgezero_kind`, or a + // future pointer/inner-envelope version -- typed `FutureFormat`) is handled + // here, because those ARE our reserved namespace. + let transient = Cell::new(false); + let outcome = resolve_fastly_config_value_typed(key, value, |chunk_key| { + let got = match &self.inner { + FastlyConfigStoreBackend::Fastly(inner) => { + inner.try_get(chunk_key).map_err(|err| { + if is_transient_lookup(&err) { + transient.set(true); + } + // The pointer-controlled chunk key is not echoed (the resolver + // adds a safe position locator); the SDK `err` carries no value. + format!("config store lookup failed: {err}") + })? + } + #[cfg(test)] + FastlyConfigStoreBackend::InMemory(data) => data.get(chunk_key).cloned(), + }; + if got.is_none() { + // Referenced chunk absent at this POP: treat as propagation lag + // (transient) rather than corruption. + transient.set(true); + } + Ok(got) + }); + match outcome { + Ok(resolved) => Ok(Some(resolved)), + Err(err) if err.is_future_format() => { + // A NEWER format in OUR reserved namespace (an unknown + // `edgezero_kind`, or a future pointer/inner-envelope version): + // re-pushing the same config will not help; the deployed build + // must be UPGRADED. + log::warn!( + "Fastly config-store value for `{key}` uses a NEWER format than this build \ + understands: {}. Re-pushing the same config will not help -- redeploy this \ + service with an updated EdgeZero build.", + err.into_message() + ); + Err(ConfigStoreError::internal(anyhow::anyhow!( + "config store value uses a newer format than this build understands; redeploy \ + this service with an updated build (re-pushing will not help)" + ))) + } + Err(err) => { + let message = err.into_message(); + if transient.get() { + log::warn!( + "Fastly config-store chunk lookup for `{key}` was transiently \ + unavailable: {message}" + ); + Err(ConfigStoreError::unavailable( + "config store temporarily unavailable", + )) + } else { + log::warn!( + "Fastly config-store chunk resolution failed for `{key}`: {message}. \ + Re-run ` config push` to repair the store." + ); + Err(ConfigStoreError::internal(anyhow::anyhow!( + "config store entry is corrupt or incomplete; re-run config push to \ + repair: {message}" + ))) + } + } + } } } +/// Is a CHUNK lookup failure environmental (retry) rather than corrupt config +/// (`config push` to repair)? Only a bad KEY names corrupt state a re-push +/// rewrites; everything else — an invalid store handle, lookup exhaustion, an +/// unclassified/future failure, or a value that outgrew the read buffer — is +/// transient, because re-pushing cannot fix a request-scoped condition. +fn is_transient_lookup(err: &LookupError) -> bool { + // `ValueTooLong` is TRANSIENT, not corruption: the SDK already retried with + // the reported buffer size, so a `ValueTooLong` reaching us means the value + // GREW between host calls (a concurrent write) — a race a retry resolves, not + // a re-push-me corruption. + !matches!(err, LookupError::KeyInvalid | LookupError::KeyTooLong) +} + fn map_lookup_error(err: &LookupError) -> ConfigStoreError { // `LookupError` is from the `fastly` crate; using a wildcard arm guards // against new variants being added in upstream point releases without @@ -134,6 +209,53 @@ mod tests { assert!(matches!(err, ConfigStoreError::InvalidKey { .. })); } + /// A CHUNK lookup that fails transiently (lookup exhaustion, an invalid + /// store handle, an unclassified error, or a value that outgrew the read + /// buffer) must keep Unavailable semantics — re-pushing cannot repair a + /// request-scoped condition. Only a bad KEY is corrupt state a re-push fixes. + #[test] + fn transient_chunk_lookups_are_not_treated_as_corruption() { + assert!(is_transient_lookup(&LookupError::TooManyLookups)); + assert!(is_transient_lookup(&LookupError::ConfigStoreInvalid)); + assert!(is_transient_lookup(&LookupError::Other)); + // ValueTooLong is TRANSIENT: the SDK already retried at the reported size, + // so it reaching us means the value grew between host calls (a race). + assert!(is_transient_lookup(&LookupError::ValueTooLong)); + assert!(!is_transient_lookup(&LookupError::KeyInvalid)); + assert!(!is_transient_lookup(&LookupError::KeyTooLong)); + } + + /// A referenced chunk that is ABSENT maps to Unavailable (HTTP 503), not + /// Internal. Config Store is eventually consistent across keys, so a flipped + /// root pointer can reach a POP before all its chunks propagate there; that + /// window is retryable, not a re-push-me corruption. + #[test] + fn a_missing_chunk_maps_to_unavailable_not_internal() { + use crate::chunked_config::prepare_fastly_config_entries; + use futures::executor::block_on; + + // A real chunked value, but seed ONLY the root pointer -- the chunks it + // references are "not yet propagated" to this POP. + let envelope = { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + serde_json::to_string(&BlobEnvelope::new( + json!({ "pad": "x".repeat(9_000) }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .expect("envelope") + }; + let entries = prepare_fastly_config_entries("app_config", &envelope).expect("expand"); + let (root_key, pointer_json) = entries.last().expect("pointer").clone(); + let store = FastlyConfigStore::from_entries([(root_key.clone(), pointer_json)]); + + let err = block_on(store.get(&root_key)).expect_err("a missing chunk must error"); + assert!( + matches!(err, ConfigStoreError::Unavailable { .. }), + "a not-yet-propagated chunk must be retryable (Unavailable), not Internal: {err:?}" + ); + } + /// Spec 9.3 (line 6272): missing chunks, hash mismatches, pointer /// parse failures, and full-envelope mismatches are CORRUPT PLATFORM /// STATE — the runtime returns an internal config-store error with @@ -142,14 +264,13 @@ mod tests { #[test] fn corrupt_chunk_pointer_maps_to_internal_not_unavailable() { use futures::executor::block_on; - // A root value that is neither a valid BlobEnvelope nor a valid - // FastlyChunkPointer triggers the resolver's "malformed pointer" - // path. We do NOT add the chunks the (would-be) pointer expects, - // so any chunk fetch would also fail — but the resolver bails - // earlier on the parse failure. + // A root value that ANNOUNCES our chunk-pointer kind but is malformed. + // It must be a pointer-kind value: an unrelated raw value is a + // legitimate Config Store entry and passes through untouched, so it + // would not exercise the corruption path at all. let store = FastlyConfigStore::from_entries([( "app_config".to_owned(), - "not-a-valid-envelope-or-pointer".to_owned(), + r#"{"edgezero_kind":"fastly_config_chunks"}"#.to_owned(), )]); let err = block_on(store.get("app_config")) .expect_err("corrupt root must map to a ConfigStoreError"); @@ -165,4 +286,54 @@ mod tests { "error message must point operators at the remediation: {err}" ); } + + /// A value written by a NEWER format (an unknown `edgezero_kind`, or a bumped + /// version) must map to Internal with an UPGRADE remediation, NOT the + /// re-push-to-repair message: re-pushing the same config cannot help a guest + /// that is older than the config. + #[test] + fn future_format_value_asks_to_redeploy_not_repush() { + use futures::executor::block_on; + let store = FastlyConfigStore::from_entries([( + "app_config".to_owned(), + r#"{"edgezero_kind":"fastly_config_chunks","version":2,"chunks":[]}"#.to_owned(), + )]); + let err = block_on(store.get("app_config")).expect_err("a future format must error"); + assert!( + matches!(err, ConfigStoreError::Internal { .. }), + "a future format is Internal, not transient: {err:?}" + ); + let message = err.to_string().to_lowercase(); + assert!( + message.contains("redeploy") || message.contains("newer format"), + "must ask the operator to redeploy an updated build: {err}" + ); + assert!( + !message.contains("re-run config push") + && !message.contains("re-push to repair") + && !message.contains("push to repair"), + "must NOT instruct the operator to re-push to repair a future format: {err}" + ); + } + + /// The store layer returns arbitrary DIRECT values VERBATIM (the shared + /// `ConfigStore` contract): it must NOT parse or judge them -- a direct + /// envelope from a newer writer carries no `edgezero_kind`, so the resolver + /// passes it through as an `Ok`. Judging its version is the typed app-config + /// extractor's job, which gives the redeploy remediation. Reverts an earlier + /// store-layer inspection that broke the "return verbatim" contract. + #[test] + fn direct_future_envelope_is_returned_verbatim() { + use futures::executor::block_on; + // A v2 direct envelope: envelope-shaped, no `edgezero_kind`, version 2. + let raw = r#"{"data":{"x":1},"sha256":"0000000000000000000000000000000000000000000000000000000000000000","generated_at":"2026-01-01T00:00:00Z","version":2}"#; + let store = FastlyConfigStore::from_entries([("app_config".to_owned(), raw.to_owned())]); + let got = block_on(store.get("app_config")) + .expect("the store must return a direct value verbatim, not judge it"); + assert_eq!( + got.as_deref(), + Some(raw), + "a direct value (even a future envelope) must be returned VERBATIM by the store layer" + ); + } } diff --git a/crates/edgezero-adapter-fastly/src/lib.rs b/crates/edgezero-adapter-fastly/src/lib.rs index e3c40e4a..4df9ef7d 100644 --- a/crates/edgezero-adapter-fastly/src/lib.rs +++ b/crates/edgezero-adapter-fastly/src/lib.rs @@ -1,6 +1,10 @@ //! Utilities for bridging Fastly Compute@Edge requests into the //! `edgezero-core` service abstractions. +// Only compiled where it is actually used (the CLI push/GC path and the Fastly +// runtime resolver). Gating it keeps a `--no-default-features` build dead-code +// clean instead of dragging in helpers no feature references. +#[cfg(any(feature = "cli", feature = "fastly", test))] pub(crate) mod chunked_config; #[cfg(feature = "cli")] pub mod cli; diff --git a/crates/edgezero-adapter/src/registry.rs b/crates/edgezero-adapter/src/registry.rs index 0f8850d0..76d09e0b 100644 --- a/crates/edgezero-adapter/src/registry.rs +++ b/crates/edgezero-adapter/src/registry.rs @@ -204,6 +204,34 @@ impl<'entry> TypedSecretEntry<'entry> { /// Outcome of a single-key read. See spec 9.0. #[non_exhaustive] pub enum ReadConfigEntry { + /// The entry EXISTS but its stored value is PROVABLY unusable, AND that was + /// established without an incomplete read — a hash mismatch, a malformed/foreign + /// pointer that resolves to a non-envelope, or a referenced chunk CONFIRMED + /// absent by an authoritative complete listing (persistent loss the blob spec + /// repairs by re-pushing). Distinct from a read/IO failure (which still + /// errors): a `config push` can safely overwrite it. This is the runtime's + /// "re-run config push to repair" contract made real — the push treats it like + /// an absent remote (proceeds to write) rather than aborting, so a broken + /// generation is recoverable in-band. The `&'static str` carries a redacted, + /// human-readable reason. + /// + /// The critical qualifier is CONFIRMED. An absence inferred from a single + /// not-found/404 response is NOT `Corrupt`: a proxy/endpoint or auth failure + /// returns the same shape, so trusting it could clobber a healthy generation + /// on an incomplete read. An adapter reports a missing chunk as `Corrupt` ONLY + /// when a complete, unpaginated listing of the store omits it; otherwise it + /// fails closed (a hard read error the operator retries). A NEWER format (an + /// unknown `edgezero_kind`, or a bumped envelope/pointer version) is likewise + /// NOT `Corrupt`: it is refused with an upgrade error and never overwritten. + /// + /// Adapters MAY, but need not, produce this: an adapter that returns a + /// corrupt value as `Present` still gets the same repair behaviour, because + /// the generic push layer classifies a `Present` body that does not verify as + /// an envelope and overwrites it (while a NEWER envelope version is refused + /// with an upgrade error, never overwritten). So the repair contract holds + /// for every adapter, and `Corrupt` is an optional precision an adapter adds + /// when it can cheaply tell corruption from a plain read. + Corrupt(&'static str), /// The store exists but the key is absent (operator hasn't pushed yet, /// or pushed under a different key). MissingKey, @@ -243,6 +271,51 @@ pub trait Adapter: Sync + Send { /// Returns an error string if the requested adapter action fails. fn execute(&self, action: AdapterAction, args: &[String]) -> Result<(), String>; + /// Reclaim chunk entries that no LIVE config pointer references. + /// + /// Deliberately NOT part of `config push`. On an eventually-consistent + /// store, a chunk may only be deleted once the pointer that referenced it + /// has stopped being served everywhere — and the platform may record no + /// pointer-supersession time (Fastly does not: `updated_at` is not bumped + /// by an upsert) and offer no compare-and-swap with which to record one + /// safely. Only the OPERATOR knows their deploy history, so `older_than` + /// carries that assertion. + /// + /// **`older_than` is a STORE-WIDE assertion, and it is stronger than "old + /// creations are no longer served".** `config gc` sweeps every root in the + /// selected store, so the caller asserts: **no root in this store changed + /// within the window, AND no writer is targeting the store** while `gc` runs. + /// A recently-changed sibling root — especially one that shrank to a direct + /// value, leaving no live chunk to age by — makes a wide window unsafe. Direct + /// trait callers MUST honour this stronger contract, not the weaker + /// paraphrase. A `dry_run` lists exactly what would be deleted, for review. + /// + /// # Errors + /// + /// Returns `Err` if the adapter has no `config gc` impl, if the platform + /// state cannot be read, or if it cannot be classified with confidence — + /// reclamation FAILS CLOSED: when in doubt, nothing is deleted. + #[inline] + #[expect( + clippy::too_many_arguments, + reason = "mirrors `push_config_entries`: manifest root, adapter manifest path, component selector, resolved store, push-time overlay, the operator's age assertion, and dry-run — each distinct; an aggregate struct is a worse ergonomic trade for implementers." + )] + fn gc_config_entries( + &self, + _manifest_root: &Path, + _adapter_manifest_path: Option<&str>, + _component_selector: Option<&str>, + _store: &ResolvedStoreId, + _push_ctx: &AdapterPushContext<'_>, + _older_than_secs: u64, + _dry_run: bool, + ) -> Result, String> { + Err(format!( + "adapter `{}` does not implement `config gc`", + self.name() + )) + } + /// Store kinds whose logical-id namespaces the adapter merges into /// a single backend at runtime — declaring the SAME logical id /// under two merged kinds causes silent write collisions because @@ -263,6 +336,26 @@ pub trait Adapter: Sync + Send { /// Name used to reference the adapter (case-insensitive). fn name(&self) -> &'static str; + /// Reject a config `(key, body)` that this adapter cannot store, BEFORE any + /// provider I/O. Called by `config push` ahead of the remote read, so an + /// infeasible push fails offline instead of after a `list`/`describe` + /// round-trip. + /// + /// `body` is the serialised blob-envelope JSON that would be written, so + /// body-dependent feasibility (e.g. Fastly's derived chunk-key length and + /// pointer-size limits, which depend on whether the value chunks) can be + /// checked here rather than during the later write expansion. + /// + /// Default: accept. The Fastly adapter overrides this to reject a reserved + /// or over-limit key and to run the full chunk expansion offline. + /// + /// # Errors + /// Returns a human-readable error string if the push is infeasible. + #[inline] + fn preflight_config_write(&self, _key: &str, _body: &str) -> Result<(), String> { + Ok(()) + } + /// Provision the platform resources backing each store id the /// user declared. Returns a list of human-readable /// status lines the CLI logs verbatim — one line per resource diff --git a/crates/edgezero-cli/src/args.rs b/crates/edgezero-cli/src/args.rs index 65f033d6..b12c458d 100644 --- a/crates/edgezero-cli/src/args.rs +++ b/crates/edgezero-cli/src/args.rs @@ -25,8 +25,14 @@ pub enum Command { Auth(AuthArgs), /// Build the project for a target edge. Build(BuildArgs), - /// Inspect or mutate the typed `.toml` app config. - #[command(subcommand, after_help = crate::args::STUB_POINTER_AFTER_HELP)] + /// Inspect or mutate the typed `.toml` app config, or reclaim + /// orphaned config-store entries (`gc`). + /// + /// NOTE: no group-level typed-CLI after-help here. `push`/`diff`/`validate` + /// each carry their own, because they need a typed `C`; `gc` does NOT -- + /// it inspects the store's physical entries and runs from this bundled + /// binary. A group-level notice would tell operators `gc` is unavailable. + #[command(subcommand)] Config(ConfigCmd), /// Run the bundled `app-demo` example locally (contributor-only). #[cfg(feature = "demo-example")] @@ -56,6 +62,23 @@ pub enum ConfigCmd { /// (Bundled `edgezero` stub — see after-help for the typed CLI.) #[command(after_help = STUB_POINTER_AFTER_HELP)] Diff(ConfigCmdStubArgs), + /// Reclaim chunk entries in the adapter's config store that no live + /// config pointer references. + /// + /// Deliberately NOT part of `config push`. On an eventually-consistent + /// store a chunk may only be deleted once the pointer that referenced it + /// has stopped being served everywhere — and the platform may record no + /// such timestamp (Fastly does not). Only YOU know your deploy history, + /// so `--older-than` is your assertion — see its help: it covers EVERY + /// root in the selected store, not only the config you have in mind. + /// + /// UNTYPED: unlike `push`/`diff`/`validate`, `gc` inspects the store's + /// physical entries rather than your `AppConfig`, so the bundled + /// `edgezero` binary can run it. + /// + /// SAFE BY DEFAULT: without `--yes` this only reports what it would + /// delete. Nothing is removed until you pass `--yes`. + Gc(ConfigGcArgs), /// Push the typed `.toml` as a single blob envelope to the /// adapter's config store. The blob carries every field verbatim /// (per spec 3.3 Model A — `#[secret]` fields store the key NAME, @@ -79,6 +102,68 @@ pub struct ConfigCmdStubArgs { pub trailing: Vec, } +/// Arguments for `config gc`. +/// +/// Unlike `push` / `diff`, `gc` needs no typed app-config: it reclaims +/// unreferenced chunk entries by inspecting the store, so it runs in-band in +/// the bundled `edgezero` binary. +#[derive(clap::Args, Debug)] +#[non_exhaustive] +pub struct ConfigGcArgs { + /// Adapter whose config store to reclaim (e.g. `fastly`). + #[arg(long)] + pub adapter: String, + /// Path to `edgezero.toml`. + #[arg(long, default_value = "edgezero.toml")] + pub manifest: PathBuf, + /// Ignore `EDGEZERO__STORES__CONFIG____NAME` when resolving which + /// PHYSICAL store to sweep, so the logical id `` is used as the physical + /// store name instead. + /// + /// This is NOT the app-config overlay the other `config` subcommands mean by + /// `--no-env` — `gc` never loads your typed app config. On a DESTRUCTIVE + /// command it selects a DIFFERENT TARGET: if that variable is what maps your + /// logical id onto the real store, `--no-env` points `gc` at a different + /// store (or one that does not exist). Check the store id `gc` reports before + /// passing `--yes`. + #[arg(long)] + pub no_env: bool, + /// Only reclaim entries older than this. This is YOUR SAFETY ASSERTION, + /// and it is about the WHOLE PHYSICAL STORE, not just one config: `gc` + /// sweeps every root in the selected store, so you are asserting "NO root + /// in this store changed within this window, and no writer is targeting + /// it" — so nothing superseded more recently (which POPs may still serve) + /// is deleted. A sibling root you re-pushed minutes ago is enough to make + /// a wide window unsafe, especially if it changed to a value small enough + /// to store directly (that leaves no chunk for `gc` to date it by). + /// Accepts `s`/`m`/`h`/`d` suffixes (e.g. `7d`, `24h`, `90m`); a bare + /// number means seconds, and 0 is rejected for `--yes`. REQUIRED for + /// `--yes` (a destructive run must not guess it); a dry-run without it + /// previews every orphan and its age. + #[arg(long)] + pub older_than: Option, + /// Override the config-store id (defaults to the manifest's). + #[arg(long)] + pub store: Option, + /// Actually delete. Without this, `gc` only reports what it WOULD delete. + #[arg(long)] + pub yes: bool, +} + +impl Default for ConfigGcArgs { + #[inline] + fn default() -> Self { + Self { + adapter: String::new(), + manifest: PathBuf::from("edgezero.toml"), + no_env: false, + older_than: None, + store: None, + yes: false, + } + } +} + /// Arguments for the `auth` command. /// /// Intentionally has no `Default` impl: unlike the other `*Args` @@ -417,6 +502,44 @@ fn default_manifest_path() -> PathBuf { PathBuf::from("edgezero.toml") } +/// Parse a human duration (`7d`, `24h`, `90m`, `30s`, or bare seconds) into +/// seconds. +/// +/// # Errors +/// +/// Returns `Err` when the value is empty, non-numeric, or carries an unknown +/// suffix — a destructive command must never guess at its safety threshold. +#[must_use = "the parsed threshold gates a destructive command"] +#[inline] +pub fn parse_duration_secs(raw: &str) -> Result { + let trimmed = raw.trim(); + let unknown = || { + format!( + "could not parse `--older-than {raw}`; expected e.g. `7d`, `24h`, `90m`, `30s`, or a number of seconds" + ) + }; + let (digits, multiplier) = match trimmed.strip_suffix('s') { + Some(rest) => (rest, 1_u64), + None => match trimmed.strip_suffix('m') { + Some(rest) => (rest, 60_u64), + None => match trimmed.strip_suffix('h') { + Some(rest) => (rest, 3_600_u64), + None => match trimmed.strip_suffix('d') { + Some(rest) => (rest, 86_400_u64), + None => (trimmed, 1_u64), + }, + }, + }, + }; + if digits.is_empty() { + return Err(unknown()); + } + let value: u64 = digits.parse().map_err(|_err| unknown())?; + value + .checked_mul(multiplier) + .ok_or_else(|| format!("`--older-than {raw}` overflows")) +} + #[cfg(test)] mod tests { use super::*; @@ -844,4 +967,23 @@ mod tests { "`[TRAILING]` placeholder leaked into push help: {help}" ); } + #[test] + fn parse_duration_secs_accepts_suffixes_and_bare_seconds() { + assert_eq!(parse_duration_secs("30s").unwrap(), 30); + assert_eq!(parse_duration_secs("90m").unwrap(), 5_400); + assert_eq!(parse_duration_secs("24h").unwrap(), 86_400); + assert_eq!(parse_duration_secs("7d").unwrap(), 604_800); + assert_eq!(parse_duration_secs("3600").unwrap(), 3_600); + assert_eq!(parse_duration_secs(" 7d ").unwrap(), 604_800); + } + + /// A destructive command must never guess at its safety threshold. + #[test] + fn parse_duration_secs_rejects_garbage() { + parse_duration_secs("").unwrap_err(); + parse_duration_secs("soon").unwrap_err(); + parse_duration_secs("7w").unwrap_err(); + parse_duration_secs("-1d").unwrap_err(); + parse_duration_secs("d").unwrap_err(); + } } diff --git a/crates/edgezero-cli/src/config.rs b/crates/edgezero-cli/src/config.rs index ef4ac6a4..2af56228 100644 --- a/crates/edgezero-cli/src/config.rs +++ b/crates/edgezero-cli/src/config.rs @@ -18,7 +18,10 @@ //! env-overlay unless `--no-env` is passed, so the validation sees //! the values the runtime would. -use crate::args::{ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, DiffFormat}; +use crate::args::{ + ConfigDiffArgs, ConfigGcArgs, ConfigPushArgs, ConfigValidateArgs, DiffFormat, + parse_duration_secs, +}; use crate::diff::{collect_changes, render_json, render_structured}; use crate::ensure_adapter_defined; use edgezero_adapter::registry::{ @@ -28,7 +31,7 @@ use edgezero_core::app_config::{ self, AppConfigError, AppConfigLoadOptions, AppConfigMeta, SecretField, SecretKind, SecretPathSegment, }; -use edgezero_core::blob_envelope::BlobEnvelope; +use edgezero_core::blob_envelope::{BlobEnvelope, BlobEnvelopeError, ENVELOPE_VERSION_V1}; use edgezero_core::env_config::EnvConfig; use edgezero_core::manifest::{Manifest, ManifestLoader, StoreDeclaration}; use serde::Serialize; @@ -36,11 +39,28 @@ use serde::de::DeserializeOwned; use similar::TextDiff; use std::collections::BTreeMap; use std::io::{Error as IoError, IsTerminal as _, Write, stdin}; +use std::iter; use std::path::{Path, PathBuf}; use toml::Value; use toml::value::Table; use validator::Validate; +/// Hard-error message shown when a push would overwrite a NEWER envelope format. +const FUTURE_FORMAT_PUSH_ERROR: &str = "the remote value uses an envelope version this CLI does not recognise (a newer format); \ + UPGRADE the CLI to push to this store rather than overwrite a newer format."; + +/// A `Present` remote body, classified for the push/diff flow. +enum PresentBody { + /// The value exists but is corrupt (malformed, or a SHA mismatch). Treated + /// like `Corrupt`: the push OVERWRITES it (the in-band repair). This makes the + /// repair contract hold for EVERY adapter, not just Fastly -- axum, Cloudflare + /// and Spin return a raw stored value as `Present`, and the generic layer here + /// (not each adapter) turns an unverifiable one into a repairable overwrite. + Corrupt, + /// A valid, integrity-verified envelope to diff against. + Valid(Box), +} + /// Pre-loaded state for either push flow. Shares /// [`ValidationContext`] with the validate flows (manifest, raw /// config, env-overlay-aware app-config path) and adds the resolved @@ -125,6 +145,10 @@ pub struct DiffExit { /// Internal outcome of a `config diff` run. Drives `apply_exit_code`. /// Variants are alphabetical per `clippy::arbitrary_source_item_ordering`. enum DiffOutcome { + /// The remote entry exists but is CORRUPT — no valid comparison was + /// possible. Reported distinctly (never as "no changes" or "all added") so a + /// scripted `config diff` cannot read it as a successful clean comparison. + CorruptRemote, /// Diff is present (remote != local, or remote absent). DiffPresent, /// SHA matched — no changes. @@ -272,6 +296,126 @@ pub fn run_config_push(_args: &ConfigPushArgs) -> Result<(), String> { ) } +/// `config gc` — reclaim chunk entries no live config pointer references. +/// +/// Runs in-band in the bundled binary: unlike `push` / `diff` it needs no typed +/// app-config, only the store's own contents. +/// +/// SAFE BY DEFAULT: without `--yes` this is a dry-run. The adapter fails closed +/// — if it cannot classify the store's state with confidence it deletes nothing. +/// +/// # Errors +/// +/// Returns `Err` if the manifest/adapter cannot be resolved, `--older-than` is +/// unparseable, or the adapter refuses to reclaim (unreadable/unclassifiable +/// state). +#[inline] +pub fn run_config_gc(args: &ConfigGcArgs) -> Result<(), String> { + // A destructive run must not invent the operator's safety assertion. + let older_than_secs = match (args.yes, args.older_than.as_deref()) { + (true, None) => { + return Err( + "`config gc --yes` requires an explicit `--older-than `: a destructive run \ + must not guess it. It asserts that NO root in the selected store changed within \ + that window and that no writer is targeting the store, so nothing POPs may still \ + be serving is deleted -- `gc` sweeps the whole physical store, not just the \ + config you have in mind. Run without `--yes` first to preview every orphan and \ + its age." + .to_owned(), + ); + } + (yes, Some(raw)) => { + let secs = parse_duration_secs(raw)?; + // `--older-than 0 --yes` asserts nothing: it makes EVERY orphan + // eligible, including one superseded a second ago whose pointer + // POPs are still serving. Dry-run may preview at zero; a delete + // may not run at it. + if yes && secs == 0 { + return Err(format!( + "`config gc --yes --older-than {raw}` resolves to 0 seconds, which asserts \ + nothing: it would make every orphan eligible, including chunks a pointer POPs \ + are still serving. Pass a window at least as long as Fastly's propagation \ + time and no longer than the time since ANY root in this store last changed." + )); + } + secs + } + // Dry-run with no threshold: preview EVERY orphan (age >= 0) with ages, + // so the operator can choose `--older-than` from real data. + (false, None) => 0, + }; + + // Manifest-only resolution. Unlike `push`/`diff`, `gc` reclaims by + // inspecting the STORE, so it must NOT require the typed app-config file to + // exist — resolve the adapter + store straight from `edgezero.toml`. + let manifest_loader = ManifestLoader::from_path(&args.manifest) + .map_err(|err| format!("failed to load {}: {err}", args.manifest.display()))?; + let manifest = manifest_loader.manifest(); + ensure_adapter_defined(&args.adapter, Some(&manifest_loader))?; + let adapter = adapter_registry::get_adapter(&args.adapter).ok_or_else(|| { + format!( + "adapter `{}` is declared in {} but not registered in this build (rebuild the CLI with its feature enabled)", + args.adapter, + args.manifest.display() + ) + })?; + let (_canonical, adapter_cfg) = manifest.adapter_entry(adapter.name()).ok_or_else(|| { + format!( + "adapter `{}` has no `[adapters.{}]` block", + args.adapter, args.adapter + ) + })?; + + let logical = resolve_config_store_id(args.store.as_deref(), manifest)?; + // `--no-env` means: do not overlay `EDGEZERO__*`. An empty config yields the + // logical id as the platform name (`store_name`'s fallback). + let env_config = if args.no_env { + EnvConfig::from_vars(iter::empty::<(String, String)>()) + } else { + EnvConfig::from_env() + }; + let platform = env_config.store_name("config", &logical); + let store = ResolvedStoreId::new(logical, platform); + + let manifest_root = args + .manifest + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let adapter_manifest_path = adapter_cfg.adapter.manifest.clone(); + let component_selector = adapter_cfg.adapter.component.clone(); + let mut push_ctx = adapter_registry::AdapterPushContext::new(); + if let Some(deploy_cmd) = adapter_cfg.commands.deploy.as_deref() { + push_ctx = push_ctx.with_manifest_adapter_deploy_cmd(deploy_cmd); + } + + // A run without --yes is a dry-run: report, delete nothing. + let dry_run = !args.yes; + let lines = adapter.gc_config_entries( + manifest_root, + adapter_manifest_path.as_deref(), + component_selector.as_deref(), + &store, + &push_ctx, + older_than_secs, + dry_run, + )?; + for line in lines { + log::info!("[edgezero] {line}"); + } + if dry_run { + match args.older_than.as_deref() { + Some(dur) => log::info!( + "[edgezero] dry-run (no --yes): nothing was deleted. Re-run with `--yes --older-than {dur}` to apply (a destructive run requires the explicit window). `--older-than {dur}` asserts that NO root in this store changed within that window and that no writer is targeting it -- this sweeps the whole physical store, not just one config." + ), + None => log::info!( + "[edgezero] dry-run (no --yes): previewing ALL orphans and their ages. Choose an `--older-than ` that is (a) at least Fastly's propagation window, so POPs have stopped serving the superseded pointer, and (b) no longer than the time since ANY root in this store last changed -- gc sweeps every root here, so a sibling you re-pushed recently also constrains the window. Then re-run with `--yes --older-than ` (a bare `--yes` is rejected)." + ), + } + } + Ok(()) +} + /// Typed flow — push the user's `C` struct. Runs strict pre-flight /// validation, then builds a `BlobEnvelope`, reads back the current /// remote for skip-on-equal + inline diff, prompts for consent, and @@ -323,6 +467,12 @@ where serde_json::from_str(&body).map_err(|err| format!("local envelope parse failed: {err}"))?; let local_sha = local_envelope.sha256.clone(); + // Reject an infeasible push (bad/over-limit key, over-limit derived chunk + // keys, oversized pointer) BEFORE any provider I/O — otherwise it would + // trigger a list/describe round-trip only to be rejected by the write path + // afterwards. `body` is the envelope JSON the write would store. + ctx.adapter.preflight_config_write(&key, &body)?; + // First read + diff. let remote = read_remote(ctx.adapter, args.local, &paths, &ctx.store, &key)?; let approved_remote_sha = @@ -379,7 +529,10 @@ fn diff_info(msg: &str) { /// per Q10's table. fn apply_exit_code(exit_code_flag: bool, outcome: DiffOutcome) -> DiffExit { let code = match (exit_code_flag, outcome) { - (_, DiffOutcome::Unsupported) => 2_i32, + // "Could not compare" (unsupported read-back, or a corrupt remote) is + // exit 2 REGARDLESS of --exit-code, so it is never mistaken for a clean + // comparison by a script. + (_, DiffOutcome::Unsupported | DiffOutcome::CorruptRemote) => 2_i32, (true, DiffOutcome::DiffPresent | DiffOutcome::RemoteAbsent) => 1_i32, // false + any, or true + NoChanges → no signal. _ => 0_i32, @@ -485,26 +638,32 @@ where // Branch per variant, render, determine outcome. let outcome: DiffOutcome = match &remote { - ReadConfigEntry::Present(body) => { - let remote_envelope: BlobEnvelope = serde_json::from_str(body) - .map_err(|err| format!("remote envelope parse failed: {err}"))?; - remote_envelope - .verify() - .map_err(|err| format!("remote envelope verification failed: {err}"))?; - if remote_envelope.sha256 == local_sha { - diff_info(&format!("# no changes (sha256 matches: {local_sha})")); - DiffOutcome::NoChanges - } else { - dispatch_diff_format( - &remote_envelope.data, - &local_envelope.data, - &remote_envelope.sha256, - &local_sha, - &args.format, - ); - DiffOutcome::DiffPresent + ReadConfigEntry::Present(body) => match classify_present_body(body)? { + // A future envelope version returns Err above (hard error). + PresentBody::Valid(remote_envelope) => { + if remote_envelope.sha256 == local_sha { + diff_info(&format!("# no changes (sha256 matches: {local_sha})")); + DiffOutcome::NoChanges + } else { + dispatch_diff_format( + &remote_envelope.data, + &local_envelope.data, + &remote_envelope.sha256, + &local_sha, + &args.format, + ); + DiffOutcome::DiffPresent + } } - } + PresentBody::Corrupt => { + diff_info(&format!( + "# existing value at key `{key}` is unusable; NO diff was computed. \ + ` config push --adapter {} --yes` will REPLACE it.", + args.adapter, + )); + DiffOutcome::CorruptRemote + } + }, ReadConfigEntry::MissingKey => { let leaf_count = collect_changes( &serde_json::Value::Object(serde_json::Map::default()), @@ -543,6 +702,14 @@ where ); DiffOutcome::RemoteAbsent } + ReadConfigEntry::Corrupt(reason) => { + diff_info(&format!( + "# existing value at key `{key}` is unusable ({reason}); NO diff was computed. \ + ` config push --adapter {} --yes` will REPLACE it.", + args.adapter, + )); + DiffOutcome::CorruptRemote + } ReadConfigEntry::Unsupported(reason) => { diff_info(&format!( "config diff for {} is unsupported ({reason}). Re-run with --local for the \ @@ -590,7 +757,20 @@ fn dispatch_diff_format( /// Consent gate: 8.3 Spin Cloud four-branch UX when `remote` is /// `Unsupported`; 8.2 default flow otherwise. fn handle_consent(args: &ConfigPushArgs, remote: &ReadConfigEntry) -> Result<(), String> { + // A CORRUPT remote (the value exists but does not resolve) is NOT handled + // here — it falls to the `else` NORMAL consent path below, so a push + // overwrites the broken generation with just `--yes`. That is the in-band + // repair the runtime and spec promise. + // + // The `Unsupported` branch below is for a read that could not be COMPUTED at + // all: a CLOUD read that cannot reach the backend (Spin Cloud), where a + // dry-run is genuinely impossible; or a LOCAL read of malformed on-disk state + // (unreadable/invalid TOML, a non-table parent) — a recoverable single-file + // overwrite that takes the normal consent path (`if args.local`). if let ReadConfigEntry::Unsupported(reason) = remote { + if args.local { + return require_consent(args, remote); + } if args.dry_run { return Err(format!( "config push --dry-run --adapter spin against Spin Cloud is unsupported \ @@ -684,12 +864,15 @@ fn recheck_before_write( approved_remote_sha: Option<&str>, ) -> Result { let remote_now = read_remote(adapter, args.local, paths, store, key)?; - if let ReadConfigEntry::Present(body_now) = remote_now { - let remote_now_env: BlobEnvelope = serde_json::from_str(&body_now) - .map_err(|err| format!("post-consent remote envelope parse failed: {err}"))?; - remote_now_env - .verify() - .map_err(|err| format!("post-consent remote envelope verification failed: {err}"))?; + // A Present body that no longer verifies (a concurrent write landed a corrupt + // value on any adapter) is handled like the `Corrupt` variant below: overwrite + // to repair. A FUTURE envelope version returns Err (hard error). + let present_now = if let ReadConfigEntry::Present(body_now) = &remote_now { + Some(classify_present_body(body_now)?) + } else { + None + }; + if let Some(PresentBody::Valid(remote_now_env)) = &present_now { if remote_now_env.sha256 == local_sha { push_info(&format!( "# concurrent push reached the same state (sha256 matches: {local_sha}); skipping write" @@ -708,6 +891,15 @@ fn recheck_before_write( short_ref(&remote_now_env.sha256), )); } + } else if matches!(remote_now, ReadConfigEntry::Corrupt(_)) + || matches!(present_now, Some(PresentBody::Corrupt)) + { + // The remote is (or became) CORRUPT: report it precisely instead of + // modelling it as a removal, then overwrite. A Present→Corrupt transition + // is a concurrent write that landed a broken value, not a deletion. + push_info(&format!( + "# remote value at key `{key}` is unusable at write time; overwriting to repair it" + )); } else if matches!(first_read, ReadConfigEntry::Present(_)) { // First read Present, second read MissingKey/MissingStore: // the remote was removed while we waited for consent. Warn @@ -719,17 +911,67 @@ fn recheck_before_write( )); } } else { - // First read MissingKey/MissingStore, second also Missing — - // nothing changed; fall through to write silently. + // First read MissingKey/MissingStore/Corrupt, second also non-Present — + // nothing to compare; fall through to write. } Ok(RecheckOutcome::Write) } +/// Classify a `Present` body. `Err` for a FUTURE format (a bumped envelope +/// version) — the push must refuse rather than overwrite a newer format. +fn classify_present_body(body: &str) -> Result { + // Detect a NEWER envelope version BEFORE deserializing as the exact v1 schema. + // A v2 envelope may add/rename/retype fields and fail to deserialize as the v1 + // `BlobEnvelope`, which would otherwise fall through to `Corrupt` and let a + // downgrade push overwrite a newer format. Version detection must come first. + if body_is_future_envelope(body) { + return Err(FUTURE_FORMAT_PUSH_ERROR.to_owned()); + } + match serde_json::from_str::(body) { + Ok(envelope) => match envelope.verify() { + Ok(()) => Ok(PresentBody::Valid(Box::new(envelope))), + // A bumped envelope version is a newer format: do NOT overwrite it. + Err(BlobEnvelopeError::UnknownVersion(_)) => Err(FUTURE_FORMAT_PUSH_ERROR.to_owned()), + // A SHA mismatch is corruption a push repairs. + Err(_) => Ok(PresentBody::Corrupt), + }, + // Not a parseable envelope at all: corruption a push repairs. + Err(_) => Ok(PresentBody::Corrupt), + } +} + +/// Is `body` a NEWER format this CLI must not overwrite? +/// +/// A cheap, schema-agnostic pre-check that parses only as a generic JSON object. +/// True when EITHER: +/// - it carries an `edgezero_kind` field. A v1 `BlobEnvelope` never has one; its +/// presence means a chunk pointer or a newer discriminated format. serde would +/// otherwise ignore the unknown field and deserialize a v1-shaped envelope with +/// `edgezero_kind: "new_format"` as valid, letting a push overwrite it; or +/// - it carries a `version` field that is present but NOT 1 -- a newer envelope +/// even when it no longer deserializes as the exact v1 [`BlobEnvelope`]. +/// +/// A missing/absent version with no `edgezero_kind` is NOT future (that stays +/// repairable corruption). +fn body_is_future_envelope(body: &str) -> bool { + let Ok(serde_json::Value::Object(obj)) = serde_json::from_str::(body) else { + return false; + }; + if obj.contains_key("edgezero_kind") { + return true; + } + obj.get("version") + .and_then(serde_json::Value::as_u64) + .is_some_and(|version| version != u64::from(ENVELOPE_VERSION_V1)) +} + /// Render the first-read diff and return the outcome. /// /// - `Present` with matching SHA → `NoChange`. /// - `Present` with differing SHA → renders diff (when `!no_diff`) and /// returns `ProceedFromPresent` with the remote SHA. +/// - `Present` but unverifiable (corrupt) → proceeds to overwrite (repair); +/// a FUTURE envelope version is a hard error instead. /// - `MissingKey` / `MissingStore` → renders "all leaves added" diff /// (when `!no_diff`) and returns `ProceedFromMissingOrUnsupported`. /// - `Unsupported` → returns `ProceedFromMissingOrUnsupported` without @@ -746,27 +988,33 @@ fn render_first_read_diff( no_diff: bool, ) -> Result { match remote { - ReadConfigEntry::Present(body_str) => { - let remote_envelope: BlobEnvelope = serde_json::from_str(body_str) - .map_err(|err| format!("remote envelope parse failed: {err}"))?; - remote_envelope - .verify() - .map_err(|err| format!("remote envelope verification failed: {err}"))?; - if remote_envelope.sha256 == local_sha { - return Ok(FirstReadOutcome::NoChange); + ReadConfigEntry::Present(body_str) => match classify_present_body(body_str)? { + PresentBody::Valid(remote_envelope) => { + if remote_envelope.sha256 == local_sha { + return Ok(FirstReadOutcome::NoChange); + } + if !no_diff { + print_unified_diff_inline( + &remote_envelope.data, + &local_envelope.data, + &remote_envelope.sha256, + local_sha, + ); + } + Ok(FirstReadOutcome::ProceedFromPresent { + approved_remote_sha: remote_envelope.sha256.clone(), + }) } - if !no_diff { - print_unified_diff_inline( - &remote_envelope.data, - &local_envelope.data, - &remote_envelope.sha256, - local_sha, - ); + PresentBody::Corrupt => { + // Corrupt on ANY adapter (an unverifiable Present value): no real + // prior to diff. Report it and overwrite -- the in-band repair. + push_info(&format!( + "# existing value at key `{key}` is unusable; NO diff was computed. It will be \ + REPLACED by this push." + )); + Ok(FirstReadOutcome::ProceedFromMissingOrUnsupported) } - Ok(FirstReadOutcome::ProceedFromPresent { - approved_remote_sha: remote_envelope.sha256.clone(), - }) - } + }, ReadConfigEntry::MissingKey if !no_diff => { push_info(&format!("# no remote at key `{key}`; all leaves added")); print_unified_diff_inline( @@ -791,6 +1039,17 @@ fn render_first_read_diff( ); Ok(FirstReadOutcome::ProceedFromMissingOrUnsupported) } + ReadConfigEntry::Corrupt(reason) => { + // The existing remote value is unreadable, so there is NO real prior + // to diff against. Do NOT fabricate a local-vs-empty diff (that reads + // as "the remote was empty"); just report the corruption and proceed + // to overwrite (the in-band repair). + push_info(&format!( + "# existing value at key `{key}` is unusable ({reason}); NO diff was computed. It \ + will be REPLACED by this push." + )); + Ok(FirstReadOutcome::ProceedFromMissingOrUnsupported) + } // Unsupported, MissingKey/MissingStore with no_diff, and future // #[non_exhaustive] variants — fall through. _ => Ok(FirstReadOutcome::ProceedFromMissingOrUnsupported), @@ -1640,6 +1899,259 @@ mod tests { use std::sync::Mutex; use tempfile::TempDir; + // ---------- config gc argument gating ---------- + + /// A destructive `config gc --yes` MUST NOT invent the safety assertion: it + /// requires an explicit `--older-than`. The check runs before any manifest + /// or store access, so it is testable in isolation. + #[test] + fn config_gc_yes_requires_explicit_older_than() { + let args = ConfigGcArgs { + adapter: "fastly".to_owned(), + yes: true, + older_than: None, + ..ConfigGcArgs::default() + }; + let err = run_config_gc(&args).expect_err("--yes without --older-than must be rejected"); + assert!( + err.contains("requires an explicit"), + "must explain the requirement: {err}" + ); + } + + /// `--older-than 0 --yes` parses, but asserts NOTHING -- + /// it makes every orphan eligible, including one superseded a second ago whose + /// pointer POPs still serve. A destructive run must reject it (a dry-run may + /// still preview at zero — see `config_gc_dry_run_allows_missing_older_than`). + #[test] + fn config_gc_yes_rejects_zero_older_than() { + for raw in ["0", "0s", "0d"] { + let args = ConfigGcArgs { + adapter: "fastly".to_owned(), + yes: true, + older_than: Some(raw.to_owned()), + ..ConfigGcArgs::default() + }; + let err = run_config_gc(&args) + .expect_err("a zero window on a destructive run must be rejected"); + assert!( + err.contains("resolves to 0 seconds"), + "`--older-than {raw}` must be rejected as a no-op assertion, got: {err}" + ); + } + } + + /// Build a fake `fastly` that maps store name→id via `config-store list` + /// (`store_list_json`), serves `entry_list_json` for `config-store-entry + /// list`, and logs every `config-store-entry` invocation's argv to `oplog` + /// (so a test can assert which store-id the resolved name flowed to, and any + /// delete arguments). + #[cfg(unix)] + fn fake_fastly_gc_list( + store_list_json: &str, + entry_list_json: &str, + oplog: &Path, + ) -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt as _; + let tmp = tempfile::TempDir::new().expect("tempdir"); + let list_file = tmp.path().join("list.json"); + fs::write(&list_file, entry_list_json).expect("write entry list json"); + let script = format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"config-store\" ] && [ \"$2\" = \"list\" ]; then echo '{stores}'; exit 0; fi\n\ + case \"$2\" in\n list) echo \"list $*\" >> '{log}'; cat '{list}'; exit 0;;\n delete) echo \"delete $*\" >> '{log}'; exit 0;;\nesac\nexit 0\n", + stores = store_list_json, + list = list_file.display(), + log = oplog.display(), + ); + let script_path = tmp.path().join("fastly"); + fs::write(&script_path, &script).expect("write script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod"); + tmp + } + + const GC_MANIFEST: &str = r#" +[app] +name = "demo-app" + +[adapters.fastly.adapter] +crate = "crates/demo-app-adapter-fastly" +manifest = "fastly.toml" + +[adapters.fastly.commands] +build = "echo" +deploy = "echo" +serve = "echo" + +[stores.config] +ids = ["app_config"] +"#; + + /// COMMAND-LEVEL GC: a dry-run drives the whole wrapper — manifest load, store + /// resolution, adapter-registry dispatch, listing, classification, and + /// reporting — end to end. A store with only a live root and a foreign sibling + /// has nothing to reclaim, so the command SUCCEEDS and deletes nothing. + #[cfg(unix)] + #[test] + fn config_gc_command_dispatches_and_reports_nothing_to_reclaim() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let _lock = manifest_guard().lock().expect("manifest guard"); + let _path_lock = path_mutation_guard().lock().expect("path guard"); + let (dir, manifest, _) = setup_project(GC_MANIFEST, FIXTURE_APP_CONFIG); + + let live = serde_json::to_string(&BlobEnvelope::new( + json!({ "greeting": "hi" }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .expect("envelope"); + let list = json!([ + { "item_key": "app_config", "item_value": live, "created_at": "2026-07-01T00:00:00Z" }, + { "item_key": "greeting", "item_value": "hello", "created_at": "2026-07-01T00:00:00Z" }, + ]) + .to_string(); + let oplog = dir.path().join("fastly-ops.log"); + let fake = fake_fastly_gc_list(r#"[{"name":"app_config","id":"store-1"}]"#, &list, &oplog); + let _prepend = PathPrepend::new(fake.path()); + + let args = ConfigGcArgs { + adapter: "fastly".to_owned(), + manifest, + store: None, + no_env: true, + yes: false, + older_than: None, + ..ConfigGcArgs::default() + }; + run_config_gc(&args).expect("a dry-run over a clean store must succeed end to end"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + !log.contains("delete "), + "a dry-run (and a nothing-to-reclaim store) must delete nothing: {log}" + ); + } + + /// COMMAND-LEVEL fail-closed regression: a store whose root claims an + /// UNKNOWN/future `edgezero_kind` must abort the whole command (nothing + /// deleted), driven through the real dispatch path, not just the internal + /// classifier. + #[cfg(unix)] + #[test] + fn config_gc_command_fails_closed_on_unknown_kind_root() { + use serde_json::json; + + let _lock = manifest_guard().lock().expect("manifest guard"); + let _path_lock = path_mutation_guard().lock().expect("path guard"); + let (dir, manifest, _) = setup_project(GC_MANIFEST, FIXTURE_APP_CONFIG); + + let list = json!([ + { + "item_key": "app_config", + "item_value": r#"{"edgezero_kind":"fastly_config_chunks_v2","data":{}}"#, + "created_at": "2026-07-01T00:00:00Z" + }, + ]) + .to_string(); + let oplog = dir.path().join("fastly-ops.log"); + let fake = fake_fastly_gc_list(r#"[{"name":"app_config","id":"store-1"}]"#, &list, &oplog); + let _prepend = PathPrepend::new(fake.path()); + + let args = ConfigGcArgs { + adapter: "fastly".to_owned(), + manifest, + store: None, + no_env: true, + yes: false, + older_than: None, + ..ConfigGcArgs::default() + }; + let err = run_config_gc(&args).expect_err("an unknown-kind root must fail closed"); + assert!( + err.contains("refusing to reclaim") || err.contains("does not recognise"), + "must fail closed on the unknown kind: {err}" + ); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + !log.contains("delete "), + "a fail-closed run deletes nothing: {log}" + ); + } + + /// COMMAND-LEVEL store routing: the ENV-derived platform name + /// (`EDGEZERO__STORES__CONFIG__APP_CONFIG__NAME`) must drive store selection. + /// The gc resolves that name to a Fastly store id via `config-store list`, + /// then lists entries under that id. The fake only knows the env name, so a + /// wrong-store regression (ignoring the overlay) would fail resolution — and + /// we assert the resolved id flowed through to the entry listing. + #[cfg(unix)] + #[test] + fn config_gc_command_selects_the_env_derived_store() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let _lock = manifest_guard().lock().expect("manifest guard"); + let _path_lock = path_mutation_guard().lock().expect("path guard"); + let _env = EnvOverride::set( + "EDGEZERO__STORES__CONFIG__APP_CONFIG__NAME", + "shared_config", + ); + let (dir, manifest, _) = setup_project(GC_MANIFEST, FIXTURE_APP_CONFIG); + + let live = serde_json::to_string(&BlobEnvelope::new( + json!({ "greeting": "hi" }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .expect("envelope"); + let entries = json!([ + { "item_key": "app_config", "item_value": live, "created_at": "2026-07-01T00:00:00Z" }, + ]) + .to_string(); + // The store list is keyed ONLY on the ENV name -> a specific id. If the + // overlay were ignored (logical "app_config" looked up), resolution fails. + let stores = r#"[{"name":"shared_config","id":"store-42"}]"#; + let oplog = dir.path().join("fastly-ops.log"); + let fake = fake_fastly_gc_list(stores, &entries, &oplog); + let _prepend = PathPrepend::new(fake.path()); + + let args = ConfigGcArgs { + adapter: "fastly".to_owned(), + manifest, + store: None, + no_env: false, // apply the EDGEZERO__* overlay + yes: false, + older_than: None, + ..ConfigGcArgs::default() + }; + run_config_gc(&args).expect("the env-derived store must resolve and list"); + let log = fs::read_to_string(&oplog).unwrap_or_default(); + assert!( + log.contains("--store-id=store-42"), + "the entry listing must run against the ENV-derived store id: {log}" + ); + } + + /// A dry-run (no `--yes`) is allowed without `--older-than`; it fails later, + /// only because the fixture manifest/adapter is absent — NOT on the + /// threshold gate. (Proves the gate does not fire for a dry-run.) + #[test] + fn config_gc_dry_run_allows_missing_older_than() { + let args = ConfigGcArgs { + adapter: "fastly".to_owned(), + manifest: PathBuf::from("does-not-exist.toml"), + yes: false, + older_than: None, + ..ConfigGcArgs::default() + }; + let err = run_config_gc(&args).expect_err("no manifest here"); + assert!( + !err.contains("requires an explicit"), + "the threshold gate must not fire for a dry-run: {err}" + ); + } + // ---------- shared fixtures ---------- const FIXTURE_APP_CONFIG: &str = r#" @@ -2739,6 +3251,383 @@ deep = true // ---------- typed push ---------- + /// A corrupt local prior value must not block `config push --local`: the + /// diff read degrades to `Unsupported`, and the shared handler must route a + /// LOCAL `Unsupported` through the normal consent path (reaching the writer) + /// rather than the Spin-Cloud dry-run rejection. Exercises the real + /// orchestration end-to-end, not the adapter methods in isolation. + #[test] + fn local_dry_run_over_corrupt_prior_reaches_the_writer() { + const FASTLY_ONLY_MANIFEST: &str = r#" +[app] +name = "demo-app" + +[adapters.fastly.adapter] +crate = "crates/demo-app-adapter-fastly" +manifest = "fastly.toml" + +[adapters.fastly.commands] +build = "echo" +deploy = "echo" +serve = "echo" + +[stores.config] +ids = ["app_config"] + +[stores.secrets] +ids = ["default"] +"#; + let _lock = manifest_guard().lock().expect("manifest guard"); + + // Every kind of unreadable prior STATE must let a local dry-run reach the + // writer's degrading count rather than hitting the Spin-Cloud rejection: + // a corrupt-but-parsing pointer, malformed TOML, and a non-string root. + let corrupt_pointer = format!( + "app_config = {}", + toml_string_literal(&format!( + "{{\"edgezero_kind\":\"fastly_config_chunks\",\"version\":1,\"chunks\":[{{\"key\":\"app_config.__edgezero_chunks.{sha}.0\",\"len\":10,\"sha256\":\"x\"}}],\"data_sha256\":\"\",\"envelope_len\":10,\"envelope_sha256\":\"{sha}\"}}", + sha = "a".repeat(64), + )), + ); + let cases = [ + ( + "corrupt pointer", + format!("[local_server.config_stores.app_config.contents]\n{corrupt_pointer}\n"), + ), + ( + "malformed TOML", + "[local_server.config_stores.app_config.contents]\nthis is not valid toml = = =" + .to_owned(), + ), + ( + "non-string root", + "[local_server.config_stores.app_config.contents]\napp_config = 42\n".to_owned(), + ), + ]; + + for (label, fastly_toml) in cases { + let (dir, manifest, _) = setup_project(FASTLY_ONLY_MANIFEST, FIXTURE_APP_CONFIG); + fs::write(dir.path().join("fastly.toml"), &fastly_toml).expect("write fastly.toml"); + + let mut args = push_args(&manifest, "fastly"); + args.local = true; + args.dry_run = true; + args.app_config = Some(dir.path().join("demo-app.toml")); + + run_config_push_typed::(&args).unwrap_or_else(|err| { + panic!("a local dry-run over {label} must reach the writer, not be rejected: {err}") + }); + } + } + + /// The dry-run degradation does NOT weaken the real push: a real + /// `config push --local` over malformed TOML still fails fatally at the + /// writer (which cannot parse the file to write into it). + #[test] + fn local_real_push_over_malformed_toml_still_fails() { + const FASTLY_ONLY_MANIFEST: &str = r#" +[app] +name = "demo-app" + +[adapters.fastly.adapter] +crate = "crates/demo-app-adapter-fastly" +manifest = "fastly.toml" + +[adapters.fastly.commands] +build = "echo" +deploy = "echo" +serve = "echo" + +[stores.config] +ids = ["app_config"] + +[stores.secrets] +ids = ["default"] +"#; + let _lock = manifest_guard().lock().expect("manifest guard"); + let (dir, manifest, _) = setup_project(FASTLY_ONLY_MANIFEST, FIXTURE_APP_CONFIG); + fs::write(dir.path().join("fastly.toml"), "this is not = = valid toml") + .expect("write fastly.toml"); + + let mut args = push_args(&manifest, "fastly"); + args.local = true; + args.yes = true; // real write, non-interactive + args.app_config = Some(dir.path().join("demo-app.toml")); + + run_config_push_typed::(&args) + .expect_err("a real push over malformed TOML must fail at the writer"); + } + + /// The body-aware preflight runs BEFORE any remote I/O: an infeasible cloud + /// push (here, a reserved `--key`) fails with the preflight error, not a + /// `fastly`-not-found / auth error from the remote read. If preflight ran + /// after `read_remote`, the error would be about the missing/failed shell-out. + #[test] + fn cloud_push_preflight_rejects_reserved_key_before_remote_io() { + const FASTLY_ONLY_MANIFEST: &str = r#" +[app] +name = "demo-app" + +[adapters.fastly.adapter] +crate = "crates/demo-app-adapter-fastly" +manifest = "fastly.toml" + +[adapters.fastly.commands] +build = "echo" +deploy = "echo" +serve = "echo" + +[stores.config] +ids = ["app_config"] + +[stores.secrets] +ids = ["default"] +"#; + let _lock = manifest_guard().lock().expect("manifest guard"); + let _path_lock = path_mutation_guard().lock().expect("path guard"); + let (dir, manifest, _) = setup_project(FASTLY_ONLY_MANIFEST, FIXTURE_APP_CONFIG); + // A fake `fastly` on PATH records any invocation, so an ordering + // regression shows up as a logged call rather than a real shell-out. + let oplog = dir.path().join("fastly-ops.log"); + let fake = fake_fastly_logging(&oplog); + let _prepend = PathPrepend::new(fake.path()); + + let mut args = push_args(&manifest, "fastly"); + // A reserved-namespace --key: infeasible, and preflight-detectable. + args.key = Some("app_config.__edgezero_chunks.deadbeef.0".to_owned()); + args.yes = true; + args.app_config = Some(dir.path().join("demo-app.toml")); + + let err = run_config_push_typed::(&args) + .expect_err("a reserved --key must be rejected"); + assert!( + err.contains("reserved infix"), + "must fail at preflight (before any remote read), not on a shell-out: {err}" + ); + assert!( + !oplog.exists(), + "preflight must reject BEFORE any `fastly` invocation; log: {:?}", + fs::read_to_string(&oplog).unwrap_or_default() + ); + } + + /// Stronger ordering proof: the generic push runs the FULL body-aware + /// preflight offline before ANY remote I/O. Unlike a reserved key (rejectable + /// by key SHAPE alone), a DERIVED-key overflow is only detectable by actually + /// expanding the envelope into chunks — a ~200-char root key is itself valid, + /// but once the >8 000-byte body chunks, the derived `.__edgezero_chunks. + /// <64-char-sha>.` key exceeds the 255-char store limit. If preflight + /// ran AFTER `read_remote`, the error would be a `fastly`-not-found shell-out + /// (no CLI on PATH in tests), never the key-limit message — so this pins that + /// the generic path performs no list/describe/update/delete before the offline + /// feasibility check has passed. + #[test] + fn cloud_push_preflight_rejects_derived_key_overflow_before_remote_io() { + const FASTLY_ONLY_MANIFEST: &str = r#" +[app] +name = "demo-app" + +[adapters.fastly.adapter] +crate = "crates/demo-app-adapter-fastly" +manifest = "fastly.toml" + +[adapters.fastly.commands] +build = "echo" +deploy = "echo" +serve = "echo" + +[stores.config] +ids = ["app_config"] + +[stores.secrets] +ids = ["default"] +"#; + let _lock = manifest_guard().lock().expect("manifest guard"); + let _path_lock = path_mutation_guard().lock().expect("path guard"); + let (dir, manifest, _) = setup_project(FASTLY_ONLY_MANIFEST, FIXTURE_APP_CONFIG); + // A fake `fastly` on PATH records any invocation, so an ordering + // regression shows up as a logged call rather than a real shell-out + // against the developer's authenticated CLI. + let oplog = dir.path().join("fastly-ops.log"); + let fake = fake_fastly_logging(&oplog); + let _prepend = PathPrepend::new(fake.path()); + // A body large enough to force chunking (envelope > 8 000 bytes). The + // huge `greeting` passes `#[validate(length(min = 1))]`. + let big_app_config = format!( + "api_token = \"demo_api_token\"\ngreeting = \"{}\"\nvault = \"default\"\n\n[service]\ntimeout_ms = 1500\n", + "x".repeat(9_000) + ); + fs::write(dir.path().join("demo-app.toml"), big_app_config).expect("write big app config"); + + let mut args = push_args(&manifest, "fastly"); + // A VALID root key (<= 255 chars, no reserved infix) whose DERIVED chunk + // key (+~85 chars) overflows the store's 255-char limit once chunked. + args.key = Some("r".repeat(200)); + args.yes = true; + args.app_config = Some(dir.path().join("demo-app.toml")); + + let err = run_config_push_typed::(&args) + .expect_err("a derived-key overflow must be rejected"); + assert!( + err.contains("character limit"), + "must fail at the offline body-aware preflight (before any remote read), not on a \ + shell-out: {err}" + ); + // The decisive assertion: NO list/describe/update/delete was attempted. + assert!( + !oplog.exists(), + "the body-aware preflight must reject BEFORE any `fastly` invocation; log: {:?}", + fs::read_to_string(&oplog).unwrap_or_default() + ); + } + + /// A CORRUPT remote (the value exists but does not resolve) must let the push + /// PROCEED to overwrite — the in-band repair contract. `render_first_read_diff` + /// returns `ProceedFromMissingOrUnsupported`, and the consent gate takes the + /// normal `--yes` path (not the Spin-Cloud Unsupported branch). + #[test] + fn corrupt_remote_proceeds_to_overwrite_not_abort() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let local = BlobEnvelope::new(json!({ "k": "v" }), "2026-01-01T00:00:00Z".to_owned()); + let outcome = render_first_read_diff( + &ReadConfigEntry::Corrupt("corrupt or incomplete chunk state"), + "app_config", + &local, + &local.sha256, + true, + ) + .expect("a corrupt remote must not error the diff read"); + assert!( + matches!(outcome, FirstReadOutcome::ProceedFromMissingOrUnsupported), + "a corrupt remote must proceed to overwrite, not abort" + ); + + // Consent: with --yes the push proceeds (repair), and it does NOT take the + // Spin-Cloud Unsupported branch (which would mis-message a fastly repair). + let mut args = push_args(Path::new("unused.toml"), "fastly"); + args.yes = true; + handle_consent(&args, &ReadConfigEntry::Corrupt("corrupt")).expect("--yes must proceed"); + } + + /// A read-only `config diff` against a CORRUPT remote must NOT read as a + /// clean/absent comparison: it exits 2 ("could not compare") regardless of + /// `--exit-code`, distinct from `NoChanges` (0) and `RemoteAbsent`/`DiffPresent`. + #[test] + fn corrupt_remote_diff_exits_could_not_compare() { + assert_eq!( + apply_exit_code(false, DiffOutcome::CorruptRemote).code, + 2_i32 + ); + assert_eq!( + apply_exit_code(true, DiffOutcome::CorruptRemote).code, + 2_i32 + ); + // For contrast: a clean match is 0, and an absent remote with --exit-code + // signals a change (1) -- neither is what a corrupt remote reports. + assert_eq!(apply_exit_code(false, DiffOutcome::NoChanges).code, 0_i32); + assert_eq!(apply_exit_code(true, DiffOutcome::RemoteAbsent).code, 1_i32); + } + + /// The repair contract is GENERIC, not Fastly-only: any adapter that returns a + /// malformed value as `Present` still gets the overwrite-to-repair behavior + /// (the generic layer classifies it), while a FUTURE envelope version is a + /// hard error the push refuses. + #[test] + fn present_body_classification_drives_generic_repair() { + use edgezero_core::blob_envelope::BlobEnvelope; + use serde_json::json; + + let valid = serde_json::to_string(&BlobEnvelope::new( + json!({ "k": "v" }), + "2026-01-01T00:00:00Z".to_owned(), + )) + .unwrap(); + assert!(matches!( + classify_present_body(&valid), + Ok(PresentBody::Valid(_)) + )); + + // A malformed value returned as Present (axum/cloudflare/spin path) is + // repairable Corrupt -> the push overwrites it, on ANY adapter. + assert!(matches!( + classify_present_body("not an envelope"), + Ok(PresentBody::Corrupt) + )); + let mut bad_sha: serde_json::Value = serde_json::from_str(&valid).unwrap(); + bad_sha["sha256"] = json!("0".repeat(64)); + assert!(matches!( + classify_present_body(&bad_sha.to_string()), + Ok(PresentBody::Corrupt) + )); + + // A future envelope VERSION is a hard error -- never overwritten. + let mut v2: serde_json::Value = serde_json::from_str(&valid).unwrap(); + v2["version"] = json!(2_u32); + assert!( + classify_present_body(&v2.to_string()).is_err_and(|err| err.contains("UPGRADE")), + "a future envelope version must refuse the push" + ); + + // A future envelope whose SCHEMA also changed (so it no longer + // deserializes as the exact v1 `BlobEnvelope`) is STILL refused: version + // detection runs BEFORE v1 deserialization, so it never falls through to + // repairable Corrupt and gets overwritten by a downgrade push. + let future_schema = json!({ + "version": 2_u32, + "data": { "k": "v" }, + "generated_at": "2026-01-01T00:00:00Z", + "sha256": 12_u32, // a NUMBER: v1 expects a string, so v1 deserialize fails + }); + assert!( + classify_present_body(&future_schema.to_string()) + .is_err_and(|err| err.contains("UPGRADE")), + "a future envelope that fails v1 deserialize must still refuse the push" + ); + + // An UNKNOWN `edgezero_kind` on a v1-SHAPED envelope must be refused, not + // overwritten: serde ignores the extra field and would otherwise accept it + // as a valid v1 envelope. Non-Fastly adapters return raw Present values, so + // this generic guard is the only thing standing between them and a + // downgrade overwrite of a newer discriminated format. + let mut kinded: serde_json::Value = serde_json::from_str(&valid).unwrap(); + kinded["edgezero_kind"] = json!("new_format"); + assert!( + classify_present_body(&kinded.to_string()).is_err_and(|err| err.contains("UPGRADE")), + "a v1-shaped envelope carrying an unknown edgezero_kind must refuse the push" + ); + + // And the first-read-diff flow proceeds to overwrite on a corrupt Present. + let local = BlobEnvelope::new(json!({ "k": "v2" }), "2026-01-01T00:00:00Z".to_owned()); + let outcome = render_first_read_diff( + &ReadConfigEntry::Present("garbage".to_owned()), + "app_config", + &local, + &local.sha256, + true, + ) + .expect("a corrupt Present must not abort the push"); + assert!(matches!( + outcome, + FirstReadOutcome::ProceedFromMissingOrUnsupported + )); + } + + /// Serialise a string as a TOML basic-string literal (test helper). + fn toml_string_literal(value: &str) -> String { + let mut out = String::from('"'); + for ch in value.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + other => out.push(other), + } + } + out.push('"'); + out + } + #[test] fn typed_push_writes_blob_envelope_to_local_config_file() { use edgezero_core::blob_envelope::{BlobEnvelope, ENVELOPE_VERSION_V1}; @@ -3328,6 +4217,26 @@ ids = ["default"] GUARD.get_or_init(|| Mutex::new(())) } + /// Build a tempdir containing a `fastly` script that APPENDS every + /// invocation to `oplog` and fails. Injected via PATH so an ordering + /// regression is caught as a recorded invocation instead of silently + /// shelling out to the developer's real, authenticated Fastly CLI. + #[cfg(unix)] + fn fake_fastly_logging(oplog: &Path) -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt as _; + let tmp = tempfile::TempDir::new().expect("tempdir"); + let script_path = tmp.path().join("fastly"); + let script = format!( + "#!/bin/sh\necho \"$@\" >> '{log}'\necho 'fake fastly: refusing' >&2\nexit 1\n", + log = oplog.display(), + ); + fs::write(&script_path, &script).expect("write fastly script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod +x"); + tmp + } + /// Build a tempdir containing a `spin` script that emits fixed /// stdout/stderr and exits with the given code. Payloads are written /// to sidecar files so shell-active chars are never re-interpreted. diff --git a/crates/edgezero-cli/src/generator.rs b/crates/edgezero-cli/src/generator.rs index 720d8243..eda4a01d 100644 --- a/crates/edgezero-cli/src/generator.rs +++ b/crates/edgezero-cli/src/generator.rs @@ -1345,6 +1345,7 @@ mod tests { for import in [ "AuthArgs", "BuildArgs", + "ConfigGcArgs", "ConfigPushArgs", "ConfigValidateArgs", "DeployArgs", @@ -1382,12 +1383,22 @@ mod tests { ); } + // `config gc` reclaims leaked chunk entries. It is UNTYPED on purpose: + // it inspects the store's physical entries, not the app-config struct, + // so it takes `ConfigGcArgs` and dispatches to the untyped runner. A + // generated CLI that omits it leaves operators no way to reclaim. + assert!( + main.contains("Gc(ConfigGcArgs)"), + "-cli ConfigCmd must include the untyped `Gc(ConfigGcArgs)` variant: {main}" + ); + // Typed dispatch — the whole reason a downstream CLI // exists. Raw push/validate would defeat the point. for call in [ "run_config_push_typed::", "run_config_validate_typed::", "edgezero_cli::run_auth", + "edgezero_cli::run_config_gc", "edgezero_cli::run_provision", ] { assert!( diff --git a/crates/edgezero-cli/src/lib.rs b/crates/edgezero-cli/src/lib.rs index 68eac9d8..512d42f9 100644 --- a/crates/edgezero-cli/src/lib.rs +++ b/crates/edgezero-cli/src/lib.rs @@ -48,8 +48,8 @@ pub mod args; pub use auth::run_auth; #[cfg(feature = "cli")] pub use config::{ - DiffExit, run_config_diff_typed, run_config_push, run_config_push_typed, run_config_validate, - run_config_validate_typed, + DiffExit, run_config_diff_typed, run_config_gc, run_config_push, run_config_push_typed, + run_config_validate, run_config_validate_typed, }; #[cfg(feature = "cli")] pub use provision::run_provision; diff --git a/crates/edgezero-cli/src/main.rs b/crates/edgezero-cli/src/main.rs index f94fe817..81d263af 100644 --- a/crates/edgezero-cli/src/main.rs +++ b/crates/edgezero-cli/src/main.rs @@ -25,6 +25,9 @@ fn main() { }; process::exit(2); } + // `config gc` needs no typed app-config -- it reclaims unreferenced + // chunk entries by inspecting the store -- so it runs in-band here. + Command::Config(ConfigCmd::Gc(cmd_args)) => edgezero_cli::run_config_gc(&cmd_args), Command::Config(ConfigCmd::Validate(cmd_args)) => { edgezero_cli::run_config_validate(&cmd_args) } diff --git a/crates/edgezero-cli/src/templates/cli/src/main.rs.hbs b/crates/edgezero-cli/src/templates/cli/src/main.rs.hbs index 3586a187..99721b1d 100644 --- a/crates/edgezero-cli/src/templates/cli/src/main.rs.hbs +++ b/crates/edgezero-cli/src/templates/cli/src/main.rs.hbs @@ -13,7 +13,8 @@ use clap::{Parser, Subcommand}; use edgezero_cli::DiffExit; use edgezero_cli::args::{ - AuthArgs, BuildArgs, ConfigDiffArgs, ConfigPushArgs, ConfigValidateArgs, DeployArgs, NewArgs, + AuthArgs, BuildArgs, ConfigDiffArgs, ConfigGcArgs, ConfigPushArgs, ConfigValidateArgs, + DeployArgs, NewArgs, ProvisionArgs, ServeArgs, }; use {{proj_core_mod}}::config::{{NameUpperCamel}}Config; @@ -67,6 +68,12 @@ enum {{NameUpperCamel}}ConfigCmd { /// Validate `edgezero.toml` and `{{name}}.toml` against the /// typed `{{NameUpperCamel}}Config` contract. Validate(ConfigValidateArgs), + /// Reclaim orphaned chunk entries the config store leaked from prior + /// oversized pushes. Store-derived and untyped (no `{{NameUpperCamel}}Config`). + /// A dry-run by default; deletes only with `--yes` + an explicit + /// `--older-than` (YOUR assertion that nothing superseded within that + /// window is still being served, and no push is running). + Gc(ConfigGcArgs), } fn main() { @@ -93,6 +100,9 @@ fn main() { Cmd::Config({{NameUpperCamel}}ConfigCmd::Validate(args)) => { edgezero_cli::run_config_validate_typed::<{{NameUpperCamel}}Config>(&args) } + // `gc` inspects the STORE, not the typed config, so it is not + // parameterised over `{{NameUpperCamel}}Config`. + Cmd::Config({{NameUpperCamel}}ConfigCmd::Gc(args)) => edgezero_cli::run_config_gc(&args), Cmd::Deploy(args) => edgezero_cli::run_deploy(&args), Cmd::New(args) => edgezero_cli::run_new(&args), Cmd::Provision(args) => edgezero_cli::run_provision(&args), diff --git a/crates/edgezero-cli/src/templates/root/gitignore.hbs b/crates/edgezero-cli/src/templates/root/gitignore.hbs index 938f60cc..90b13520 100644 --- a/crates/edgezero-cli/src/templates/root/gitignore.hbs +++ b/crates/edgezero-cli/src/templates/root/gitignore.hbs @@ -14,6 +14,11 @@ target/ .spin/ .edgezero/ +# Fastly local config-push lock (a persistent advisory-lock sidecar next to +# the manifest; serialises concurrent `config push --local` / `provision`). +# The glob also covers a manifest named something other than `fastly.toml`. +.*.edgezero-lock + # env .env diff --git a/crates/edgezero-core/src/blob_envelope.rs b/crates/edgezero-core/src/blob_envelope.rs index 8ca8b955..f0c319d1 100644 --- a/crates/edgezero-core/src/blob_envelope.rs +++ b/crates/edgezero-core/src/blob_envelope.rs @@ -5,6 +5,8 @@ //! { "data": {...}, "sha256": "", "version": 1, "generated_at": "" } //! ``` +use core::fmt; + use crate::canonical_form::canonical_data_sha256; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -21,14 +23,34 @@ pub struct BlobEnvelope { pub version: u32, } -#[derive(Debug, Error)] +#[derive(Error)] pub enum BlobEnvelopeError { - #[error("sha mismatch: stored {stored}, computed {computed}")] + // The stored/computed hashes are NOT interpolated into the Display message. + // `stored` is a blob-controlled string (a malformed envelope can set it to + // anything, including a secret), and this error is formatted into diagnostics + // that reach HTTP responses and logs. Redacting at the source covers every + // caller — the extractor, the chunk resolver, the CLI push/diff paths, and + // the introspection endpoint — instead of each having to remember to. The + // fields stay on the struct for programmatic inspection. + #[error("stored SHA-256 does not match the computed hash (hashes redacted)")] ShaMismatch { stored: String, computed: String }, #[error("unknown envelope version {0}; expected {expected}", expected = ENVELOPE_VERSION_V1)] UnknownVersion(u32), } +// Debug is hand-written (NOT derived): a derived `Debug` would print the +// `stored`/`computed` hashes, so `{err:?}` / `?err` (anyhow) would leak what the +// Display message deliberately redacts. Debug therefore mirrors Display. +impl fmt::Debug for BlobEnvelopeError { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ShaMismatch { .. } => f.write_str("ShaMismatch { hashes redacted }"), + Self::UnknownVersion(version) => write!(f, "UnknownVersion({version})"), + } + } +} + impl BlobEnvelope { /// Consume the envelope and return only the inner `data` value. #[must_use] @@ -89,6 +111,29 @@ mod tests { envelope.verify().unwrap(); } + #[test] + fn sha_mismatch_display_does_not_leak_the_stored_hash() { + // The stored SHA is blob-controlled and this Display is formatted into + // diagnostics that reach HTTP responses and logs by many callers. + // Redacting here is the single point that covers all of them. + const SENTINEL: &str = "SUPER_SECRET_STORED_HASH"; + let mut envelope = BlobEnvelope::new(json!({ "greeting": "hi" }), "2026-06-17".into()); + envelope.sha256 = SENTINEL.to_owned(); + let err = envelope + .verify() + .expect_err("a tampered sha must fail verify"); + assert!( + !err.to_string().contains(SENTINEL), + "the stored hash must never appear in the error Display: {err}" + ); + // Debug must redact too: `{err:?}` / `?err` (anyhow) would otherwise + // print the stored hash even though Display redacts it. + assert!( + !format!("{err:?}").contains(SENTINEL), + "the stored hash must never appear in the error Debug: {err:?}" + ); + } + #[test] fn rejects_unknown_version() { let mut envelope = BlobEnvelope::new(json!({ "x": 1_i32 }), "2026-06-17T00:00:00Z".into()); diff --git a/crates/edgezero-core/src/error.rs b/crates/edgezero-core/src/error.rs index c5a4d7f7..3928f9ab 100644 --- a/crates/edgezero-core/src/error.rs +++ b/crates/edgezero-core/src/error.rs @@ -70,9 +70,29 @@ impl EdgeError { #[must_use] #[inline] pub fn config_out_of_date_from_serde(serde_err: &SerdePathError) -> Self { + // The serde message embeds the offending stored VALUE (e.g. `invalid + // type: string "hunter2", expected u32`), and this message is serialised + // into the HTTP error body. The config blob may hold secrets, so the + // VALUE must not escape — report only the category. + // + // The `field_path` STRING segments are redacted (structure kept): a map + // key is indistinguishable from a struct field here and may be a secret, + // and the redaction invariant forbids a stored string on any path. The + // exact path is available from a local `config validate`. See + // `redact_serde_path`. + use serde_json::error::Category; + let category = match serde_err.inner().classify() { + Category::Data => "wrong type or invalid value", + Category::Syntax => "malformed JSON", + Category::Eof => "unexpected end of input", + Category::Io => "i/o error while reading", + }; Self::ConfigOutOfDate { - message: serde_err.inner().to_string(), - field_path: serde_err.path().to_string(), + message: format!( + "typed app-config is out of date ({category}; value redacted) — \ + run ` config push` for this deploy" + ), + field_path: redact_serde_path(serde_err.path()), } } @@ -257,6 +277,49 @@ impl IntoResponse for EdgeError { } } +/// Render a `serde_path_to_error` path with its STRING segments redacted while +/// preserving structure (dots and sequence indices). +/// +/// A struct field and a map KEY are the same `Segment::Map` in this API +/// (verified empirically), so they cannot be told apart. A map key is stored +/// config DATA and may be a secret, and this path is serialised into the HTTP +/// error body — which the redaction invariant forbids for any stored string. So +/// every string segment becomes ``; sequence indices (positions, not +/// values) are kept. The operator recovers the EXACT path by running +/// `config validate` locally, which deserializes the same blob without an HTTP +/// boundary. See the `field_path` note in spec 2026-06-16. +fn redact_serde_path(path: &serde_path_to_error::Path) -> String { + use serde_path_to_error::Segment; + let mut out = String::new(); + for segment in path { + match segment { + Segment::Seq { index } => { + out.push('['); + out.push_str(&index.to_string()); + out.push(']'); + } + Segment::Map { .. } | Segment::Enum { .. } => { + if !out.is_empty() { + out.push('.'); + } + out.push_str(""); + } + Segment::Unknown => { + if !out.is_empty() { + out.push('.'); + } + out.push('?'); + } + } + } + // No segments => a root-level error; keep serde's "." sentinel (a marker, + // not data). + if out.is_empty() { + return path.to_string(); + } + out +} + fn json_or_text(payload: &T) -> Body { Body::json(payload).unwrap_or_else(|_| Body::text("internal error")) } @@ -327,14 +390,68 @@ mod tests { let result: Result = serde_path_to_error::deserialize(de); let serde_err = result.expect_err("expected deserialization error"); - let expected_path = serde_err.path().to_string(); let err = EdgeError::config_out_of_date_from_serde(&serde_err); assert_eq!(err.status(), StatusCode::SERVICE_UNAVAILABLE); assert!(!err.message().is_empty()); match err { EdgeError::ConfigOutOfDate { field_path, .. } => { - assert_eq!(field_path, expected_path); + // String segments redacted, structure preserved. + assert_eq!(field_path, "."); + } + EdgeError::BadRequest { .. } + | EdgeError::Internal { .. } + | EdgeError::MethodNotAllowed { .. } + | EdgeError::NotFound { .. } + | EdgeError::NotImplemented { .. } + | EdgeError::ServiceUnavailable { .. } + | EdgeError::Validation { .. } => panic!("expected ConfigOutOfDate"), + } + } + + #[test] + fn config_out_of_date_from_serde_redacts_map_key_from_path_and_message() { + use serde::Deserialize; + use std::collections::BTreeMap; + + #[derive(Debug, Deserialize)] + struct Outer { + #[expect(dead_code, reason = "only used to drive deserialization")] + items: BTreeMap, + } + #[derive(Debug, Deserialize)] + struct Inner { + #[expect(dead_code, reason = "only used to drive deserialization")] + port: u32, + } + + // A MAP KEY holding a secret. The type error is under that key, so the + // serde path is `items.SECRET_MAP_KEY.port`. It must reach NEITHER the + // message NOR the field_path: a map key is indistinguishable from a struct + // field in the serde path, so every string segment is redacted (structure + // kept). + const SENTINEL: &str = "SUPER_SECRET_MAP_KEY"; + let json = format!(r#"{{"items": {{"{SENTINEL}": {{"port": "nope"}}}}}}"#); + let de = &mut serde_json::Deserializer::from_str(&json); + let result: Result = serde_path_to_error::deserialize(de); + let serde_err = result.expect_err("expected deserialization error"); + + let err = EdgeError::config_out_of_date_from_serde(&serde_err); + match err { + EdgeError::ConfigOutOfDate { + field_path, + message, + } => { + assert!( + !message.contains(SENTINEL), + "a stored map key must never reach the message: {message}" + ); + assert!( + !field_path.contains(SENTINEL), + "a stored map key must never reach the field_path: {field_path}" + ); + // Structure is preserved (items..port -> redacted, dotted). + assert_eq!(field_path, ".."); } EdgeError::BadRequest { .. } | EdgeError::Internal { .. } diff --git a/crates/edgezero-core/src/extractor.rs b/crates/edgezero-core/src/extractor.rs index 42dda715..bc8e6c17 100644 --- a/crates/edgezero-core/src/extractor.rs +++ b/crates/edgezero-core/src/extractor.rs @@ -9,7 +9,7 @@ use serde::de::DeserializeOwned; use validator::Validate; use crate::app_config::{AppConfigMeta, SecretField, SecretKind, SecretPathSegment}; -use crate::blob_envelope::BlobEnvelope; +use crate::blob_envelope::{BlobEnvelope, BlobEnvelopeError}; use crate::config_store::ConfigStoreHandle; use crate::context::RequestContext; use crate::error::EdgeError; @@ -811,6 +811,35 @@ where } } +/// A redacted reason when `raw` is a NEWER format this build must not apply, or +/// `None` when it is not (that stays ordinary corruption). +/// +/// A cheap, schema-agnostic pre-check that parses only as a generic JSON object, +/// so it catches a newer format even when the value no longer deserializes as the +/// exact v1 [`BlobEnvelope`]. It is future when EITHER: +/// - it carries an `edgezero_kind` field. A v1 `BlobEnvelope` never has one; +/// serde silently ignores the unknown field, so without this a v1-shaped +/// envelope tagged `edgezero_kind: "new_format"` would deserialize and apply on +/// non-Fastly runtimes (the generic push already refuses that exact value); or +/// - it carries a `version` field present but NOT 1. +/// +/// The reason names only the integer version or the field name — never a value, +/// so it is safe to surface in the (HTTP-visible) remediation message. +fn future_format_reason(raw: &str) -> Option { + use crate::blob_envelope::ENVELOPE_VERSION_V1; + let serde_json::Value::Object(obj) = serde_json::from_str::(raw).ok()? + else { + return None; + }; + if obj.contains_key("edgezero_kind") { + return Some("an unrecognised `edgezero_kind` discriminator".to_owned()); + } + obj.get("version") + .and_then(serde_json::Value::as_u64) + .filter(|version| *version != u64::from(ENVELOPE_VERSION_V1)) + .map(|version| format!("envelope version {version}")) +} + /// Shared body: fetch + envelope + sha + secret walk + deserialise + validate. /// /// The `FromRequest` impl and the `named`/`from_store` inherent methods all @@ -855,10 +884,39 @@ where String::new(), ) })?; - let envelope: BlobEnvelope = serde_json::from_str(&raw) - .map_err(|err| EdgeError::internal(anyhow::anyhow!("envelope parse failed: {err}")))?; - envelope.verify().map_err(|err| { - EdgeError::internal(anyhow::anyhow!("envelope verification failed: {err}")) + // Neither the parse error nor the verify error is echoed into the + // client-facing message: a serde error embeds the offending input, and a + // `BlobEnvelope` integrity failure names the stored hashes — both are + // config-store values that may hold secrets, and this message reaches the + // HTTP body. Report a category only. + // + // A value written by a NEWER format (a bumped envelope version OR an unknown + // `edgezero_kind` discriminator) needs a DIFFERENT remediation than + // corruption: re-pushing the same config cannot help a build older than its + // config; the deployed build must be UPGRADED. This is the typed app-config + // layer's job (the store returns arbitrary direct values verbatim), so it is + // detected here, BEFORE the exact-v1 deserialize (a newer format may not even + // parse as v1, and serde ignores an unknown discriminator). The reason names + // only the version/field, never a value, so surfacing it is safe. + if let Some(reason) = future_format_reason(&raw) { + return Err(EdgeError::internal(anyhow::anyhow!( + "typed app-config blob uses {reason}, which this build does not understand; redeploy \ + this service with an updated build (re-pushing will not help)" + ))); + } + let envelope: BlobEnvelope = serde_json::from_str(&raw).map_err(|_err| { + EdgeError::internal(anyhow::anyhow!( + "typed app-config blob is not a valid envelope (details redacted)" + )) + })?; + envelope.verify().map_err(|err| match err { + BlobEnvelopeError::UnknownVersion(version) => EdgeError::internal(anyhow::anyhow!( + "typed app-config blob uses envelope version {version}, which this build does not \ + understand; redeploy this service with an updated build (re-pushing will not help)" + )), + BlobEnvelopeError::ShaMismatch { .. } => EdgeError::internal(anyhow::anyhow!( + "typed app-config blob failed its integrity check (details redacted)" + )), })?; let mut data = envelope.into_data(); // Secret walk per spec 3.3.3. @@ -1039,10 +1097,13 @@ async fn resolve_leaf( })? .to_owned(); let bound = ctx.secret_store(&store_id_str).ok_or_else(|| { + // `store_id_str` is the blob's store_ref VALUE — config data that + // may be sensitive — and this message reaches the HTTP body. Name + // the field, not the stored id. EdgeError::config_out_of_date( format!( - "blob declared store_ref `{store_id_str}` but \ - [stores.secrets] has no such id" + "secret field `{leaf_path}` names a store_ref that is not declared in \ + [stores.secrets] (id redacted)" ), leaf_path.clone(), ) @@ -1062,23 +1123,33 @@ async fn resolve_leaf( fn map_secret_error( err: SecretError, field_name: &str, - store_id: &str, - key_name: &str, + // The stored key name, the store id, and the provider's message/source are + // deliberately UNUSED in the messages below: they are blob- or + // provider-controlled strings that may reveal the secret or infrastructure, + // and these messages reach the HTTP body / logs. Every branch names only the + // offending FIELD (schema, safe). Kept in the signature so the redaction is + // visible at the one place these values could have been formatted. + _store_id: &str, + _key_name: &str, ) -> EdgeError { match err { - SecretError::NotFound { name } => EdgeError::config_out_of_date( - format!("secret `{name}` in store `{store_id}` not found"), + SecretError::NotFound { .. } => EdgeError::config_out_of_date( + format!( + "the secret referenced by `{field_name}` was not found in its store (identifier redacted)" + ), field_name.to_owned(), ), - SecretError::Validation(msg) => EdgeError::config_out_of_date( - format!("secret `{key_name}` in store `{store_id}` rejected: {msg}"), + SecretError::Validation(_msg) => EdgeError::config_out_of_date( + format!( + "the secret referenced by `{field_name}` was rejected by its store (details redacted)" + ), field_name.to_owned(), ), - SecretError::Unavailable => { - EdgeError::service_unavailable(format!("secret store `{store_id}` unreachable")) - } - SecretError::Internal(source) => EdgeError::internal(anyhow::anyhow!( - "secret `{key_name}` in store `{store_id}` produced unexpected store error: {source}" + SecretError::Unavailable => EdgeError::service_unavailable(format!( + "the secret store for `{field_name}` is unreachable" + )), + SecretError::Internal(_source) => EdgeError::internal(anyhow::anyhow!( + "secret resolution for `{field_name}` failed (details redacted)" )), } } @@ -2308,16 +2379,20 @@ mod tests { #[test] fn app_config_extractor_returns_internal_on_sha_mismatch() { + const SENTINEL: &str = "SUPER_SECRET_STORED_HASH"; struct TamperedStore; #[async_trait(?Send)] impl ConfigStore for TamperedStore { async fn get(&self, _key: &str) -> Result, ConfigStoreError> { - // Build a valid envelope then corrupt its sha. + // A valid envelope whose stored sha is a SENTINEL secret. The + // stored hash is attacker-influenced (it comes from the config + // store), and this error becomes the HTTP 500 body — so the + // message must NOT echo it. let mut env = BlobEnvelope::new( serde_json::json!({ "greeting": "hi", "timeout_ms": 100_u32 }), "2026-01-01T00:00:00Z".into(), ); - env.sha256 = "ff".repeat(32); + env.sha256 = SENTINEL.to_owned(); Ok(Some(serde_json::to_string(&env).unwrap())) } } @@ -2330,12 +2405,131 @@ mod tests { matches!(err, EdgeError::Internal { .. }), "SHA mismatch must surface as Internal: {err:?}" ); + // The client-facing message (the HTTP body) must not carry the stored + // hash, only a redacted category. assert!( - err.message().contains("envelope verification failed"), - "message names the problem: {err:?}" + !err.message().contains(SENTINEL), + "the stored hash must never reach the client-facing message: {err:?}" + ); + assert!( + err.message().contains("integrity check"), + "message must still name the category: {err:?}" + ); + } + + #[test] + fn app_config_extractor_asks_to_redeploy_on_a_future_envelope_version() { + // The store returns a NEWER envelope version VERBATIM (its contract). The + // typed app-config layer -- NOT the store -- must give the upgrade/redeploy + // remediation, distinct from the corruption "re-push" path, because a + // build older than its config cannot be fixed by re-pushing the config. + struct FutureVersionStore; + #[async_trait(?Send)] + impl ConfigStore for FutureVersionStore { + async fn get(&self, _key: &str) -> Result, ConfigStoreError> { + // A v1-shaped envelope with the version bumped to 2. + let env = BlobEnvelope::new( + serde_json::json!({ "greeting": "hi", "timeout_ms": 100_u32 }), + "2026-01-01T00:00:00Z".into(), + ); + let mut value: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&env).unwrap()).unwrap(); + value["version"] = serde_json::json!(2_u32); + Ok(Some(value.to_string())) + } + } + + let ctx = ctx_with_config_store(FutureVersionStore, "key"); + let err = block_on(AppConfig::::from_request(&ctx)) + .expect_err("a future envelope version must error"); + assert!( + matches!(err, EdgeError::Internal { .. }), + "a future version is Internal, not transient: {err:?}" + ); + let message = err.message().to_lowercase(); + assert!( + message.contains("redeploy") && message.contains("version 2"), + "must ask to redeploy an updated build and name the version: {err:?}" + ); + assert!( + !message.contains("re-run config push") && !message.contains("push to repair"), + "must NOT tell the operator to re-push a future format: {err:?}" + ); + } + + #[test] + fn app_config_extractor_asks_to_redeploy_on_an_unknown_edgezero_kind() { + // A v1-SHAPED envelope carrying an `edgezero_kind` discriminator. serde + // ignores the unknown field, so without the reason check the extractor + // would deserialize and APPLY it. It must instead ask to redeploy, matching + // the generic push's refusal of the same value. + struct KindTaggedStore; + #[async_trait(?Send)] + impl ConfigStore for KindTaggedStore { + async fn get(&self, _key: &str) -> Result, ConfigStoreError> { + let env = BlobEnvelope::new( + serde_json::json!({ "greeting": "hi", "timeout_ms": 100_u32 }), + "2026-01-01T00:00:00Z".into(), + ); + let mut value: serde_json::Value = + serde_json::from_str(&serde_json::to_string(&env).unwrap()).unwrap(); + value["edgezero_kind"] = serde_json::json!("new_format"); + Ok(Some(value.to_string())) + } + } + + let ctx = ctx_with_config_store(KindTaggedStore, "key"); + let err = block_on(AppConfig::::from_request(&ctx)) + .expect_err("an unknown edgezero_kind must error"); + let message = err.message().to_lowercase(); + assert!( + matches!(err, EdgeError::Internal { .. }) + && message.contains("redeploy") + && message.contains("edgezero_kind"), + "an unknown discriminator must ask to redeploy: {err:?}" ); } + #[test] + fn app_config_extractor_does_not_leak_data_value_in_deserialize_error() { + const SENTINEL: &str = "SUPER_SECRET_FIELD_VALUE"; + struct TypeErrorStore; + #[async_trait(?Send)] + impl ConfigStore for TypeErrorStore { + async fn get(&self, _key: &str) -> Result, ConfigStoreError> { + // A VALID envelope (verify passes) whose data has the wrong type + // for `timeout_ms` — a string sentinel where u32 is expected. + // The deserialize error names that value; it must not reach the + // client-facing message. + let env = BlobEnvelope::new( + serde_json::json!({ "greeting": "hi", "timeout_ms": SENTINEL }), + "2026-01-01T00:00:00Z".into(), + ); + Ok(Some(serde_json::to_string(&env).unwrap())) + } + } + + let ctx = ctx_with_config_store(TypeErrorStore, "key"); + let err = block_on(AppConfig::::from_request(&ctx)) + .expect_err("a type mismatch in the typed config must error"); + assert!( + matches!(err, EdgeError::ConfigOutOfDate { .. }), + "a typed deserialize failure must be ConfigOutOfDate: {err:?}" + ); + assert!( + !err.message().contains(SENTINEL), + "the stored field value must never reach the client-facing message: {err:?}" + ); + // The field path segments are redacted (a map key is indistinguishable + // from a struct field and may be a secret); structure is kept. + if let EdgeError::ConfigOutOfDate { field_path, .. } = &err { + assert!( + !field_path.contains("timeout_ms") && field_path.contains(""), + "the field path must be redacted: {field_path}" + ); + } + } + #[test] fn app_config_extractor_returns_internal_on_bad_envelope_json() { struct GarbageStore; @@ -2354,9 +2548,15 @@ mod tests { "Envelope parse failure must be Internal: {err:?}" ); assert!( - err.message().contains("envelope parse failed"), + err.message().contains("not a valid envelope"), "message names the problem: {err:?}" ); + // The offending input is not echoed (a serde error would embed it, and a + // stored value may hold secrets). + assert!( + !err.message().contains("not-json-at-all"), + "the offending stored value must not reach the client message: {err:?}" + ); } #[test] @@ -2383,11 +2583,11 @@ mod tests { matches!(err, EdgeError::ConfigOutOfDate { .. }), "deserialise failure must be ConfigOutOfDate: {err:?}" ); - // serde_path_to_error should give us the field path. + // The field path segment is redacted (indistinguishable from a map key). if let EdgeError::ConfigOutOfDate { field_path, .. } = &err { assert_eq!( - field_path, "timeout_ms", - "serde_path_to_error must supply the field name: {err:?}" + field_path, "", + "the field path must be redacted: {err:?}" ); } } @@ -2479,6 +2679,32 @@ mod tests { } } + #[test] + fn app_config_secret_walk_error_does_not_leak_the_stored_key_name() { + use crate::config_store::{ConfigStore, ConfigStoreError}; + struct BlobStore(String); + #[async_trait(?Send)] + impl ConfigStore for BlobStore { + async fn get(&self, _key: &str) -> Result, ConfigStoreError> { + Ok(Some(self.0.clone())) + } + } + + // The blob's secret field holds the secret's KEY NAME — config data that + // may be sensitive. When resolution fails, that name (and any store id / + // provider message) must not reach the error, which becomes the HTTP body. + const SENTINEL: &str = "SUPER_SECRET_KEY_NAME"; + let data = serde_json::json!({ "greeting": "hi", "api_token": SENTINEL }); + let blob = make_envelope(data); + let ctx = ctx_with_config_and_secrets(BlobStore(blob), "key", NoopSecretStore, "vault"); + let err = block_on(AppConfig::::from_request(&ctx)) + .expect_err("missing secret must error"); + assert!( + !err.message().contains(SENTINEL), + "the stored secret key name must never reach the error message: {err:?}" + ); + } + // Build a RequestContext whose default secret store maps `default/{key}` -> // `value`, for exercising `secret_walk` directly. fn ctx_with_default_secret_store(key: &str, value: &str) -> RequestContext { diff --git a/crates/edgezero-core/src/introspection.rs b/crates/edgezero-core/src/introspection.rs index 9404f063..d74ba664 100644 --- a/crates/edgezero-core/src/introspection.rs +++ b/crates/edgezero-core/src/introspection.rs @@ -99,11 +99,17 @@ pub async fn config(ctx: RequestContext) -> Result { .await .map_err(EdgeError::from)? .ok_or_else(|| EdgeError::not_found("no config blob in default store"))?; - let envelope: BlobEnvelope = serde_json::from_str(&raw) - .map_err(|err| EdgeError::internal(anyhow::anyhow!("envelope parse failed: {err}")))?; - envelope.verify().map_err(|err| { - EdgeError::internal(anyhow::anyhow!("envelope verification failed: {err}")) + // Neither error echoes the stored value: a serde parse error embeds the + // input, and the envelope integrity error is redacted at its source. `raw` + // is the stored config blob and may hold secrets; this reaches the HTTP body. + let envelope: BlobEnvelope = serde_json::from_str(&raw).map_err(|_err| { + EdgeError::internal(anyhow::anyhow!( + "config blob is not a valid envelope (details redacted)" + )) })?; + envelope + .verify() + .map_err(|err| EdgeError::internal(anyhow::anyhow!("config blob {err}")))?; let body = Body::json(&envelope.into_data()).map_err(EdgeError::internal)?; json_response(StatusCode::OK, body) } diff --git a/docs/guide/blob-app-config-migration.md b/docs/guide/blob-app-config-migration.md index dfbba996..255f3fd5 100644 --- a/docs/guide/blob-app-config-migration.md +++ b/docs/guide/blob-app-config-migration.md @@ -117,8 +117,12 @@ The push uses `fastly config-store-entry update --upsert --stdin` to write the envelope as the value of one Config Store entry. **Oversized envelopes** are handled automatically. Fastly's per-entry -limit is 8,000 characters. If your envelope fits, it's stored -directly. Otherwise the adapter: +limit is 8,000 characters. The adapter applies a conservative +**UTF-8 byte** check (bytes are always ≥ characters, so an envelope +that fits by bytes always fits by characters): if the envelope JSON is +at most 8,000 bytes it's stored directly. This byte threshold is the +v1 read/write contract — it is stable across upgrades, so a value +stored by an earlier release always resolves. Otherwise the adapter: 1. Splits the envelope JSON into UTF-8-safe chunks (target 7,000 bytes each). @@ -306,16 +310,39 @@ Listing the orphans before deletion: # Cloudflare wrangler kv key list --namespace-id= --remote | jq -r '.[].name' -# Fastly -fastly config-store-entry list --store-id= --json | jq -r '.[].key' +# Fastly (the JSON field is `item_key`, not `key`) +fastly config-store-entry list --store-id= --json | jq -r '.[].item_key' # Spin sqlite3 .spin/sqlite_key_value.db "SELECT key FROM spin_key_value WHERE store=''" ``` -A future `config gc --adapter ` will automate this; v1 is -manual on the rationale that orphan cleanup is best done with operator -oversight. +> **`config gc` does NOT do this per-leaf cleanup.** It reclaims a different kind +> of orphan — the **chunk** entries an _oversized_ blob leaves behind when it is +> re-pushed (each generation is content-addressed, so a change orphans the whole +> previous chunk set). It never touches your old per-leaf keys. Run the per-leaf +> deletes above for the migration cutover; run `config gc` (below) as an ongoing +> reclamation for chunked stores. + +### Reclaiming orphaned chunk entries (Fastly, oversized configs) + +If your app-config blob exceeds Fastly's 8 000-character entry limit it is stored +as content-addressed chunks, and every re-push orphans the previous generation. +`config gc` reclaims them safely — it derives the live set from the store and +deletes only unreferenced chunk entries you have asserted are old enough: + +```sh +# Preview every orphan and its age (dry-run by default): +-cli config gc --adapter fastly + +# Delete, asserting that NO root in this store changed in the last 7 days AND no +# writer is targeting the store (config gc sweeps every root, store-wide): +-cli config gc --adapter fastly --older-than 7d --yes +``` + +`--older-than` is REQUIRED for `--yes` and is YOUR safety assertion; do not run +`config gc` while a `config push` to the same store is in flight (Fastly has no +compare-and-swap). Cloudflare/Spin orphan cleanup remains manual. ### Fastly chunk-pointer hygiene @@ -325,11 +352,11 @@ under 8,000 characters, the old chunks remain in the store unreferenced ```sh fastly config-store-entry list --store-id= --json \ - | jq -r '.[].key | select(contains(".__edgezero_chunks."))' + | jq -r '.[].item_key | select(contains(".__edgezero_chunks."))' ``` -They're safe to delete via the same per-key delete command above. -A future `config gc` will sweep them automatically. +`config gc --adapter fastly` (shown above) reclaims these safely — it will only +delete chunk keys no live pointer references. ## Troubleshooting @@ -356,13 +383,22 @@ project's typed downstream CLI. The bundled binary intentionally cannot push (it has no typed `C` in scope). Run `-cli config push` instead — `edgezero new` generates this CLI for you. -### Local fastly.toml writer wipes old chunks on re-push - -The local writer wholesale-replaces the per-store contents block on -every push so stale entries don't accumulate during dev work. This is -intentional (and STRONGER than the remote behaviour, where orphan -chunks remain inert). The runtime correctness property holds either -way: a read after push B reconstructs envelope B, not A. +### Local fastly.toml writer prunes old chunks on re-push + +The local writer **upserts the keys it is pushing and prunes the chunks the +prior generation of those roots referenced** — except any prior chunk key whose +VALUE is itself a runtime-readable root (a valid envelope or pointer), which is +kept with a warning rather than deleted. It does not wholesale-replace the +per-store contents block, so keys it isn't pushing (including sibling roots and +their chunks) are preserved. Pruning is safe +here in a way it is not in the cloud: `fastly.toml` is a single file that +Viceroy reads at startup, so there is no propagation window and no POP +that could still be serving the prior pointer. + +This is STRONGER than the remote behaviour, where a `config push` +reclaims nothing and orphan chunks remain inert until you run +`config gc`. The runtime correctness property holds either way: a read +after push B reconstructs envelope B, not A. ## Reference diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index f25c7404..56d5819e 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -236,14 +236,18 @@ edgezero config push --adapter [--manifest ] [--app-config ] - `--runtime-config ` — adapter runtime configuration file. Currently only consumed by Spin, which reads `[key_value_store.