From 54af3b11346cc9439f2ce2e77576e2c1ed1795cd Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 06:46:17 -0700 Subject: [PATCH 01/36] Refactor: share catalog preflight and split head replacement (#125) Extract the pure candidate and snapshot admission from publish_catalog_generation, and split head execution into staging and replacement, so a gated publication route can run the same protocol and insert a check before HEAD replacement. Version-1 behavior is unchanged. --- src/adapters/catalog_publication.rs | 50 +++++++++++++------ src/adapters/catalog_publication_execution.rs | 18 ++++++- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/adapters/catalog_publication.rs b/src/adapters/catalog_publication.rs index 4dcfb714..5e6d1b43 100644 --- a/src/adapters/catalog_publication.rs +++ b/src/adapters/catalog_publication.rs @@ -5,8 +5,8 @@ use super::catalog_transition; use super::{ AdmittedCatalog, AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationReadiness, - CatalogPublicationReceipt, CatalogPublicationStorage, ChecksummedPublicationHead, - SegmentPublication, catalog_publication_execution, + CatalogPublicationReceipt, CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, + ChecksummedPublicationHead, SegmentPublication, catalog_publication_execution, }; /// Publishes one fully admitted canonical catalog generation. @@ -29,20 +29,10 @@ pub fn publish_catalog_generation( catalog: &CanonicalCatalog, segments: &[AdmittedSegment<'_>], ) -> Result { - validate_staged_segment(&segment, segments)?; let checksummed = catalog.checksummed(); - let admitted = checksummed.admit(segments).map_err(|source| { - CatalogPublicationError::CatalogAdmission { - source: Box::new(source), - } - })?; - validate_transition(expectation, &admitted)?; + let admitted = admit_candidate(expectation, &segment, checksummed, segments)?; let head = CanonicalPublicationHead::for_catalog(checksummed); - let checked_head = ChecksummedPublicationHead::decode(head.encoded()) - .map_err(|source| CatalogPublicationError::HeadVerification { source })?; - let snapshot = checked_head - .admit(admitted) - .map_err(|source| CatalogPublicationError::SnapshotAdmission { source })?; + let snapshot = admit_snapshot(&head, admitted)?; let readiness = catalog_publication_execution::execute_current(storage, expectation, &snapshot, &segment)?; if readiness == CatalogPublicationReadiness::AlreadyPublished { @@ -62,6 +52,38 @@ pub fn publish_catalog_generation( )) } +/// Admits the complete candidate catalog and its successor relationship. +/// +/// This pure preflight performs no I/O. Every catalog publication route runs +/// it before its first storage call. +pub(super) fn admit_candidate<'catalog, 'records>( + expectation: CatalogPublicationExpectation, + segment: &SegmentPublication<'_, '_>, + checksummed: ChecksummedCatalog<'catalog>, + segments: &'records [AdmittedSegment<'records>], +) -> Result, CatalogPublicationError> { + validate_staged_segment(segment, segments)?; + let admitted = checksummed.admit(segments).map_err(|source| { + CatalogPublicationError::CatalogAdmission { + source: Box::new(source), + } + })?; + validate_transition(expectation, &admitted)?; + Ok(admitted) +} + +/// Binds generated next-head bytes to the admitted candidate catalog. +pub(super) fn admit_snapshot<'head, 'catalog, 'records>( + head: &'head CanonicalPublicationHead, + admitted: AdmittedCatalog<'catalog, 'records>, +) -> Result, CatalogPublicationError> { + let checked_head = ChecksummedPublicationHead::decode(head.encoded()) + .map_err(|source| CatalogPublicationError::HeadVerification { source })?; + checked_head + .admit(admitted) + .map_err(|source| CatalogPublicationError::SnapshotAdmission { source }) +} + fn validate_transition( expectation: CatalogPublicationExpectation, candidate: &AdmittedCatalog<'_, '_>, diff --git a/src/adapters/catalog_publication_execution.rs b/src/adapters/catalog_publication_execution.rs index 1b5e6993..3793ad3a 100644 --- a/src/adapters/catalog_publication_execution.rs +++ b/src/adapters/catalog_publication_execution.rs @@ -100,6 +100,16 @@ pub(super) fn execute_head( storage: &mut impl CatalogPublicationStorage, head: &CanonicalPublicationHead, snapshot: &CatalogSnapshot<'_, '_, '_>, +) -> Result<(), CatalogPublicationError> { + execute_head_stage(storage, head, snapshot)?; + execute_head_replacement(storage) +} + +/// Stages, synchronizes, and verifies `head.next` without replacing `HEAD`. +pub(super) fn execute_head_stage( + storage: &mut impl CatalogPublicationStorage, + head: &CanonicalPublicationHead, + snapshot: &CatalogSnapshot<'_, '_, '_>, ) -> Result<(), CatalogPublicationError> { phase( CatalogPublicationPhase::CreateHeadStage, @@ -114,7 +124,13 @@ pub(super) fn execute_head( phase( CatalogPublicationPhase::VerifyHeadView, storage.verify_head_view(head, snapshot), - )?; + ) +} + +/// Atomically replaces `HEAD` with the verified stage and synchronizes the root. +pub(super) fn execute_head_replacement( + storage: &mut impl CatalogPublicationStorage, +) -> Result<(), CatalogPublicationError> { phase(CatalogPublicationPhase::ReplaceHead, storage.replace_head())?; phase( CatalogPublicationPhase::SynchronizeRoot, From a607a1760c9b70dcd345db232937023455a4c6ab Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 06:46:23 -0700 Subject: [PATCH 02/36] Feat: gate catalog publication on every retained closure (#125) Add publish_retained_catalog_generation over the RetainedCatalogPublicationStorage port. Under writer authority it admits the current retention head, its manifest, and every manifest root, verifies each root closure against the candidate before the first catalog storage call, and rereads the retention head after head.next verification, refusing HEAD replacement if it changed. The receipt binds the preserved manifest generation and digest. Catalog stages are reachable only with a gate-issued RetainedClosureAdmission. Record the decision, crash states, and recovery obligation in the v2 format rationale; update closure, retention-publication, README, ledger, and changelog. --- CHANGELOG.md | 2 + docs/formats/segment-store-v2/README.md | 2 +- docs/formats/segment-store-v2/closure.md | 2 +- docs/formats/segment-store-v2/rationale.md | 69 +++++ docs/formats/segment-store-v2/requirements.md | 1 + .../segment-store-v2/retention-publication.md | 10 +- src/adapters/exports.rs | 5 + src/adapters/mod.rs | 1 + src/adapters/retained_catalog_publication.rs | 122 ++++++++ .../preserved_retention_manifest.rs | 42 +++ .../retained_catalog_publication_error.rs | 142 ++++++++++ ...ained_catalog_publication_error_display.rs | 51 ++++ .../retained_catalog_publication_receipt.rs | 55 ++++ .../retained_catalog_publication_storage.rs | 66 +++++ .../retained_closure_admission.rs | 52 ++++ .../retained_evidence_admission.rs | 154 ++++++++++ .../retention_evidence_phase.rs | 24 ++ src/lib.rs | 5 + tests/retained_catalog_publication.rs | 210 ++++++++++++++ .../catalog_fixture.rs | 146 ++++++++++ .../closure_model_laws.rs | 145 ++++++++++ .../closure_refusal_laws.rs | 118 ++++++++ .../evidence_refusal_laws.rs | 263 ++++++++++++++++++ .../model_storage.rs | 124 +++++++++ .../retention_fixture.rs | 152 ++++++++++ .../revalidation_laws.rs | 120 ++++++++ 26 files changed, 2080 insertions(+), 3 deletions(-) create mode 100644 src/adapters/retained_catalog_publication.rs create mode 100644 src/adapters/retained_catalog_publication/preserved_retention_manifest.rs create mode 100644 src/adapters/retained_catalog_publication/retained_catalog_publication_error.rs create mode 100644 src/adapters/retained_catalog_publication/retained_catalog_publication_error_display.rs create mode 100644 src/adapters/retained_catalog_publication/retained_catalog_publication_receipt.rs create mode 100644 src/adapters/retained_catalog_publication/retained_catalog_publication_storage.rs create mode 100644 src/adapters/retained_catalog_publication/retained_closure_admission.rs create mode 100644 src/adapters/retained_catalog_publication/retained_evidence_admission.rs create mode 100644 src/adapters/retained_catalog_publication/retention_evidence_phase.rs create mode 100644 tests/retained_catalog_publication.rs create mode 100644 tests/retained_catalog_publication/catalog_fixture.rs create mode 100644 tests/retained_catalog_publication/closure_model_laws.rs create mode 100644 tests/retained_catalog_publication/closure_refusal_laws.rs create mode 100644 tests/retained_catalog_publication/evidence_refusal_laws.rs create mode 100644 tests/retained_catalog_publication/model_storage.rs create mode 100644 tests/retained_catalog_publication/retention_fixture.rs create mode 100644 tests/retained_catalog_publication/revalidation_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 08e02f4b..0ed92fe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ after its public API and format compatibility policies are established. ## [Unreleased] +- Version-2 catalog publication can refuse candidates that lose retained content: `publish_retained_catalog_generation` verifies every current retained closure against the candidate before any catalog mutation and revalidates the retention head before `HEAD` replacement (#125). + - Update the repository-only serde_json parser to 1.0.154 in both locked workspaces and refresh its dependency admission; the resolved transitive graph, features, license and MSRV are unchanged (#186). - Update the in-process BLAKE3 dependency to 1.8.7 in both locked workspaces, remove the unused arrayref license exception and renew its admission while retaining the external b3sum oracle and version-one identity contract (#94). diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 4ef7b927..12f17db3 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -105,7 +105,7 @@ Writer authority coordinates cooperating writers in a managed namespace; it does `FilesystemRetentionSnapshot` binds a catalog, retention head and manifest under a shared reader fence and verifies selected roots on demand. `DurableStore` composes that view with authenticated whole-blob and exact-range reads, delivered in [PR #164](https://github.com/flyingrobots/keep/pull/164); the [read contract](../../invariants/authenticated-reconstruction/README.md) records proof and allocation limits. -General version-two catalog publication still needs the candidate-catalog retained-closure admission gate tracked in [#125](https://github.com/flyingrobots/keep/issues/125); verifying a new retention root against the current catalog is a different operation. +The candidate-catalog retained-closure gate is implemented over its storage port ([#125](https://github.com/flyingrobots/keep/issues/125)). No filesystem version-two catalog publisher uses it yet ([#198](https://github.com/flyingrobots/keep/issues/198)). Verifying a new retention root against the current catalog is a different operation. [Durable verification reports](../../invariants/verification/README.md) are delivered in [PR #165](https://github.com/flyingrobots/keep/pull/165), with explicit subject/depth evidence and typed non-success outcomes. diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/segment-store-v2/closure.md index 5e4c1912..b967e8b6 100644 --- a/docs/formats/segment-store-v2/closure.md +++ b/docs/formats/segment-store-v2/closure.md @@ -1,6 +1,6 @@ # Closure Verification -- Status: Normative version-2 protocol; the storage-independent verifier and live member re-verification under filesystem retention authority are implemented. General candidate-catalog retained-closure admission remains in [#125](https://github.com/flyingrobots/keep/issues/125). +- Status: Normative version-2 protocol; the storage-independent verifier, live member re-verification under filesystem retention authority, and the candidate-catalog retained-closure gate over its storage port ([#125](https://github.com/flyingrobots/keep/issues/125)) are implemented. No filesystem version-2 catalog publisher uses the gate yet ([#198](https://github.com/flyingrobots/keep/issues/198)). - Format coordinate: `keep.segment-store/v2` - Requirement: [`KEEP-RETENTION-005`](requirements.md#retention-transitions) - Decision record: diff --git a/docs/formats/segment-store-v2/rationale.md b/docs/formats/segment-store-v2/rationale.md index 4f8388df..9994562f 100644 --- a/docs/formats/segment-store-v2/rationale.md +++ b/docs/formats/segment-store-v2/rationale.md @@ -123,3 +123,72 @@ silently substitute a different identity coordinate. An exact receipt proves the migration records agree; it does not prove the current reserved namespace still admits. Recovery therefore invokes the read-only `StoreMigrationRecoveryStorage::verify_complete` capability after planning selects `Complete` and before returning success. The filesystem adapter reuses version-two namespace admission, retaining the original cause through `Observation` and `NamespacePreflight`. Existing corruption/planning refusals retain priority because this check follows planning. No namespace effect, synchronization or retention recovery is initiated. Completed stores may contain published retention roots, manifests and heads. Reusing the partial-migration empty-directory preflight would reject valid post-migration state, so completion uses the existing version-two admission policy instead. Migration completion verifies its root/reserved directory contract; retention owns the interpretation of retention artifacts and stages. This adds a required method to the public recovery storage port; external implementations must implement equivalent effect-free admission. On-disk formats and identities are unchanged. + +## Gate catalog publication on every retained closure + +ADR-0009 requires version-2 catalog publication to verify every current +retained closure against the candidate catalog before it replaces `HEAD`. +`publish_retained_catalog_generation` implements that rule over the +`RetainedCatalogPublicationStorage` port (#125). It is the entry point a +version-2 catalog writer plugs into; this change adds no filesystem version-2 +catalog publisher. + +Under writer authority, the gate runs the pure catalog preflight first. It +then reads the retention head, the manifest that head names, and every root +the manifest names. Each record is decoded and cross-checked against the +record that selected it: the head names the manifest's digest, generation, +predecessor and length, and each manifest entry names its root's namespace, +generation and digest. The gate then calls `verify_retention_closure` for each +root against the candidate snapshot, using the root's own profile and limits. +Only then does the first catalog storage call run. A refused candidate +therefore leaves no stage, pool entry, or head change. After `head.next` +verification, the gate rereads the retention head and requires it to equal the +admitted head before it calls `replace_head`. + +Storage returns exact bytes and the gate does all the decoding. Storage that +decoded its own records would be vouching for evidence it supplies. The +catalog stages are reachable only through `catalog_stages`, which takes a +`RetainedClosureAdmission`. Only the gate can create that value. A version-2 +publisher whose stage type has no other constructor therefore cannot be passed +to the ungated `publish_catalog_generation`, which stays unchanged for +version 1. + +The gate holds one manifest of at most 4,096 entries and one root at a time. +Each closure is limited by its root's admitted limits. Roots are verified in +manifest order, and the first refusal is returned. + +Revalidation compares only the head, because manifest and root pool entries +are immutable and named by digest. An unchanged canonical head therefore +selects exactly the admitted evidence. The candidate catalog is held in +memory, and `verify_head_view` reverifies its pool copy. + +Crash and refusal states: + +- Refusal during evidence admission or closure verification happens before any + catalog call, so there is nothing to recover. +- A head change refused at revalidation leaves the linked catalog, the linked + segment if one was staged, and a verified `head.next`. These are the same + states a crash at that point leaves in the version-1 protocol. They are + recovery state, never repaired automatically. +- Process death at any point leaves exactly the version-1 crash states. The + gate adds only reads. + +Recovery implication: today's next-head finalization admits only version-1 +writer authority, so it cannot finalize a version-2 `head.next`. When the +single-format writer moves stage recovery onto version 2 (#198), finalizing a +staged catalog head must run this same retained-closure admission against the +staged catalog. Changing the admission type that recovery accepts is not +enough. + +Rejected alternatives: + +- Extending `CatalogPublicationStorage` with retention reads. Any implementation + could then be driven through the ungated entry point. +- Having storage return admitted evidence. The gate would trust the storage + it is checking. +- Revalidating before `head.next` creation. That leaves a wider unchecked + window before the atomic replacement. +- Rereading every root at commit. Immutable digest-named records make the head + comparison sufficient. +- Relying on ingestion's preserve-all-records behaviour. That is a property of + one writer, not of the publication boundary. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 09057b01..d2a1a9d2 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -19,6 +19,7 @@ case is not evidence. | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | `ReaderFence` holds a shared kernel lock on a verified zero-length `reader.lock`; `collect_retention_view` accepts a view only when both head coordinates agree before and after loading and refuses an exhausted attempt limit (`retention_view_collector_tests`); `FilesystemRetentionSnapshot` binds the catalog snapshot, retention head, and manifest under the fence and verifies each selected root on demand while the fence is held, refusing a substituted root and a replaced fence, and two readers share the fence while an exclusive lock waits (`filesystem_retention_snapshot_tests`); Linux public snapshot process laws prove live-reader exclusion, SIGKILL fence release with persistent inode preservation, and kernel-queued new-reader exclusion until collection releases the fence (`tests/reader_fence_process.rs`; [evidence](../../testing-evidence/reader-fence-process.md)) | Implemented | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | all three-operation histories over initial publications of two namespaces, successor, release, restore, byte-identical retry and stale initial (343 histories, each in a fresh migrated store) compare fenced namespace generations, anchor sets and liveness against an operation-derived model after every step; exact typed refusals remain checked. The count describes exploration, not correctness. See [release/restore evidence](../../testing-evidence/retention-release-restore-model.md). Core architecture checks remain separate static evidence. | Implemented; release/restore model coverage added for #128 | +| `KEEP-RETENTION-011` | Catalog publication over retention state admits the current retention head, manifest, and every manifest root under writer authority; verifies every retained closure against the candidate before any catalog storage call; and refuses `HEAD` replacement if the retention head changed | port-level laws in `tests/retained_catalog_publication.rs`: dropped and substituted members refuse with the exact missing identity and no catalog stage; every manifest root is verified wherever it sorts; absent, corrupt, substituted, and contradictory head, manifest, and root evidence refuse with typed expected and observed state; a head changed, removed, or published before commit refuses after `head.next` verification without `replace_head`; an exhaustive 24-case candidate-by-root-set model and a repeat-run determinism check | Implemented at the storage port; no filesystem version-2 catalog publisher or next-head finalization uses it yet (#198) | diff --git a/docs/formats/segment-store-v2/retention-publication.md b/docs/formats/segment-store-v2/retention-publication.md index e696fa2c..3654ee26 100644 --- a/docs/formats/segment-store-v2/retention-publication.md +++ b/docs/formats/segment-store-v2/retention-publication.md @@ -22,7 +22,15 @@ exact already-committed retry revalidates authority and performs no mutation. Version-2 catalog publication must hold the same writer authority and prove every current retained closure against its candidate catalog before replacing the catalog `HEAD`. -That general candidate-catalog gate remains unimplemented and is tracked in [#125](https://github.com/flyingrobots/keep/issues/125). The implemented retention publication and complete-stage retention recovery paths reverify a root's closure against the current catalog; this does not establish that an arbitrary successor catalog preserves every retained root. +`publish_retained_catalog_generation` implements that gate over the `RetainedCatalogPublicationStorage` port ([#125](https://github.com/flyingrobots/keep/issues/125)). Under writer authority, before its first catalog storage call, it: + +1. admits the current retention head, the manifest that head names, and every root that manifest names; +2. cross-checks each record against the record that selected it; and +3. verifies each root's closure against the candidate catalog with that root's profile and limits. + +After `head.next` verification, it rereads the retention head and refuses `HEAD` replacement if the head changed. Its receipt binds the preserved manifest generation and digest. The [format rationale](rationale.md#gate-catalog-publication-on-every-retained-closure) records the crash states and the recovery obligation. + +No filesystem version-2 catalog publisher or version-2 next-head finalization uses the gate yet; that writer is tracked in [#198](https://github.com/flyingrobots/keep/issues/198). Retention publication and complete-stage retention recovery separately reverify a new root's closure against the current catalog. ## Generation transition diff --git a/src/adapters/exports.rs b/src/adapters/exports.rs index 8a900830..3edc85e7 100644 --- a/src/adapters/exports.rs +++ b/src/adapters/exports.rs @@ -71,6 +71,11 @@ pub use super::publication_head_decode_error::PublicationHeadDecodeError; pub use super::recovery::*; #[cfg(feature = "repository-tasks")] pub use super::repository_initialization_storage::RepositoryInitializationStorage; +pub use super::retained_catalog_publication::{ + PreservedRetentionManifest, RetainedCatalogPublicationError, RetainedCatalogPublicationReceipt, + RetainedCatalogPublicationStorage, RetainedClosureAdmission, RetentionEvidencePhase, + publish_retained_catalog_generation, +}; pub use super::retention::*; pub use super::sealed_segment::SealedSegment; pub use super::segment_header::SegmentHeader; diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 31a2b951..b82e01da 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -173,6 +173,7 @@ mod publication_head_encoder; mod recovery; #[cfg(feature = "repository-tasks")] mod repository_initialization_storage; +mod retained_catalog_publication; mod retention; mod sealed_segment; mod segment_digest_builder; diff --git a/src/adapters/retained_catalog_publication.rs b/src/adapters/retained_catalog_publication.rs new file mode 100644 index 00000000..0a81cff8 --- /dev/null +++ b/src/adapters/retained_catalog_publication.rs @@ -0,0 +1,122 @@ +//! This module owns catalog publication that refuses to lose retained content. +//! +//! A store with retention state publishes a candidate catalog only through +//! [`publish_retained_catalog_generation`]. The entry point admits the current +//! global retention head, its manifest, and every root that manifest names, +//! then verifies each root's closure against the candidate before the first +//! catalog mutation. It confirms the retention head is unchanged again +//! immediately before `HEAD` replacement. The decision and its crash states are +//! recorded in `docs/formats/segment-store-v2/rationale.md`. + +mod preserved_retention_manifest; +mod retained_catalog_publication_error; +mod retained_catalog_publication_error_display; +mod retained_catalog_publication_receipt; +mod retained_catalog_publication_storage; +mod retained_closure_admission; +mod retained_evidence_admission; +mod retention_evidence_phase; + +pub use preserved_retention_manifest::PreservedRetentionManifest; +pub use retained_catalog_publication_error::RetainedCatalogPublicationError; +pub use retained_catalog_publication_receipt::RetainedCatalogPublicationReceipt; +pub use retained_catalog_publication_storage::RetainedCatalogPublicationStorage; +pub use retained_closure_admission::RetainedClosureAdmission; +pub use retention_evidence_phase::RetentionEvidencePhase; + +use super::catalog_publication_execution as execution; +use super::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, + CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogSnapshot, SegmentPublication, + catalog_publication, +}; + +/// Publishes one catalog generation only if it preserves every retained closure. +/// +/// The caller's storage must hold exclusive writer authority for the complete +/// call. The order is: +/// +/// 1. admit the candidate catalog, its successor relationship, and its +/// generated next head without I/O, exactly as +/// [`publish_catalog_generation`](crate::publish_catalog_generation) does; +/// 2. read and admit the current retention head, the manifest it names, and +/// every root that manifest names, in manifest order; +/// 3. verify each root's closure against the candidate with the root's own +/// registered profile and admitted limits; +/// 4. run the catalog publication protocol through `head.next` verification; +/// 5. reread the retention head and require it to equal the admitted head; and +/// 6. replace `HEAD` and synchronize the store root. +/// +/// Steps 2 and 3 complete before the first catalog storage call, so a refused +/// candidate leaves no stage, pool entry, or head change. Memory is bounded by +/// one manifest and one root at a time; closure work is bounded by each root's +/// limits and the manifest's 4,096-entry ceiling. The first refusal in +/// manifest order is returned. +/// +/// # Errors +/// +/// Returns [`RetainedCatalogPublicationError`] with the exact catalog preflight +/// or storage failure, retention read failure, retention record refusal, +/// manifest or root binding disagreement, closure refusal and the manifest +/// entry it belongs to, or a retention head that changed before commit. A +/// failure returns no receipt. +pub fn publish_retained_catalog_generation( + storage: &mut S, + expectation: CatalogPublicationExpectation, + segment: SegmentPublication<'_, '_>, + catalog: &CanonicalCatalog, + segments: &[AdmittedSegment<'_>], +) -> Result { + let checksummed = catalog.checksummed(); + let admitted = + catalog_publication::admit_candidate(expectation, &segment, checksummed, segments)?; + let head = CanonicalPublicationHead::for_catalog(checksummed); + let snapshot = catalog_publication::admit_snapshot(&head, admitted)?; + let admission = retained_evidence_admission::admit(storage, &snapshot)?; + let preserved = admission.preserved_manifest(); + let current_stages = storage.catalog_stages(&admission); + let readiness = execution::execute_current(current_stages, expectation, &snapshot, &segment)?; + if readiness == CatalogPublicationReadiness::AlreadyPublished { + let catalog_receipt = CatalogPublicationReceipt::already_published( + snapshot.generation(), + snapshot.catalog_digest(), + ); + return Ok(RetainedCatalogPublicationReceipt::new( + catalog_receipt, + preserved, + )); + } + let candidate = StagedCandidate { + catalog, + head: &head, + snapshot: &snapshot, + }; + stage_candidate(storage.catalog_stages(&admission), segment, &candidate)?; + retained_evidence_admission::revalidate(storage, &admission)?; + execution::execute_head_replacement(storage.catalog_stages(&admission))?; + let catalog_receipt = + CatalogPublicationReceipt::published(snapshot.generation(), snapshot.catalog_digest()); + Ok(RetainedCatalogPublicationReceipt::new( + catalog_receipt, + preserved, + )) +} + +struct StagedCandidate<'call, 'head, 'catalog, 'records> { + catalog: &'call CanonicalCatalog, + head: &'call CanonicalPublicationHead, + snapshot: &'call CatalogSnapshot<'head, 'catalog, 'records>, +} + +fn stage_candidate( + stages: &mut impl super::CatalogPublicationStorage, + segment: SegmentPublication<'_, '_>, + candidate: &StagedCandidate<'_, '_, '_, '_>, +) -> Result<(), RetainedCatalogPublicationError> { + if let Some(admitted) = segment.into_admitted() { + execution::execute_segment(stages, admitted)?; + } + execution::execute_catalog(stages, candidate.catalog, candidate.catalog.checksummed())?; + execution::execute_head_stage(stages, candidate.head, candidate.snapshot)?; + Ok(()) +} diff --git a/src/adapters/retained_catalog_publication/preserved_retention_manifest.rs b/src/adapters/retained_catalog_publication/preserved_retention_manifest.rs new file mode 100644 index 00000000..29504bb1 --- /dev/null +++ b/src/adapters/retained_catalog_publication/preserved_retention_manifest.rs @@ -0,0 +1,42 @@ +//! This module owns the retention coordinates a catalog publication preserved. + +use crate::{LivenessGeneration, RetentionManifestDigest}; + +/// Exact retention manifest whose every root closure a candidate preserved. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PreservedRetentionManifest { + generation: LivenessGeneration, + digest: RetentionManifestDigest, + root_count: u32, +} + +impl PreservedRetentionManifest { + /// Returns the preserved global liveness generation. + pub const fn generation(self) -> LivenessGeneration { + self.generation + } + + /// Returns the preserved canonical manifest digest. + pub const fn digest(self) -> RetentionManifestDigest { + self.digest + } + + /// Returns how many manifest roots were verified against the candidate. + #[must_use] + pub const fn root_count(self) -> u32 { + self.root_count + } + + pub(super) const fn new( + generation: LivenessGeneration, + digest: RetentionManifestDigest, + root_count: u32, + ) -> Self { + Self { + generation, + digest, + root_count, + } + } +} diff --git a/src/adapters/retained_catalog_publication/retained_catalog_publication_error.rs b/src/adapters/retained_catalog_publication/retained_catalog_publication_error.rs new file mode 100644 index 00000000..c86925fe --- /dev/null +++ b/src/adapters/retained_catalog_publication/retained_catalog_publication_error.rs @@ -0,0 +1,142 @@ +//! This module owns refusals of retained-closure catalog publication. + +use std::error::Error; +use std::io; + +use super::super::CatalogPublicationError; +use super::RetentionEvidencePhase; +use crate::{ + LivenessGeneration, RetentionClosureVerificationError, RetentionHead, RetentionHeadDecodeError, + RetentionManifestDecodeError, RetentionManifestDigest, RetentionManifestEntry, + RetentionManifestLength, RetentionRootDecodeError, +}; + +/// Failure before a candidate catalog that must preserve retention becomes current. +/// +/// Retention evidence and closure refusals occur before the first catalog +/// storage call. [`Self::RetentionChanged`] and failures in +/// [`RetentionEvidencePhase::RevalidateHead`] occur after `head.next` +/// verification and before `HEAD` replacement. [`Self::Catalog`] preserves +/// the catalog protocol's own preflight refusal or storage phase. +#[derive(Debug)] +pub enum RetainedCatalogPublicationError { + /// Candidate preflight or one catalog protocol transition failed. + Catalog { + /// Preserved catalog publication refusal. + source: CatalogPublicationError, + }, + /// A retention head or manifest read failed under writer authority. + RetentionStorage { + /// Evidence read attempted. + phase: RetentionEvidencePhase, + /// Preserved storage source. + source: io::Error, + }, + /// The root one manifest entry selects could not be read. + RootStorage { + /// Manifest entry whose root was requested. + entry: RetentionManifestEntry, + /// Preserved storage source. + source: io::Error, + }, + /// Retention head bytes failed canonical admission. + HeadDecode { + /// Evidence read whose bytes were refused. + phase: RetentionEvidencePhase, + /// Preserved head decoder refusal. + source: RetentionHeadDecodeError, + }, + /// The selected manifest's length differs from the length the head names. + ManifestLength { + /// Length the admitted head names. + expected: RetentionManifestLength, + /// Byte length storage returned. + observed: usize, + }, + /// The selected manifest failed canonical admission. + ManifestDecode { + /// Preserved manifest decoder refusal. + source: Box, + }, + /// The selected manifest's digest differs from the digest the head names. + ManifestDigest { + /// Digest the admitted head names. + expected: RetentionManifestDigest, + /// Digest of the admitted manifest bytes. + observed: RetentionManifestDigest, + }, + /// The selected manifest's generation differs from the head's generation. + ManifestGeneration { + /// Generation the admitted head names. + expected: LivenessGeneration, + /// Generation the admitted manifest records. + observed: LivenessGeneration, + }, + /// The selected manifest's predecessor differs from the head's predecessor. + ManifestPredecessor { + /// Predecessor the admitted head names. + expected: Option, + /// Predecessor the admitted manifest records. + observed: Option, + }, + /// One selected root failed canonical admission. + RootDecode { + /// Manifest entry whose root was refused. + entry: RetentionManifestEntry, + /// Preserved root decoder refusal. + source: Box, + }, + /// An admitted root is not the root its manifest entry selects. + /// + /// The observed coordinate is boxed so every refusal stays small. + RootBinding { + /// Manifest entry that selected the root. + expected: RetentionManifestEntry, + /// Namespace, generation, and digest of the admitted root. + observed: Box, + }, + /// The candidate catalog cannot reconstruct one retained closure. + Closure { + /// Manifest entry whose root closure was refused. + entry: RetentionManifestEntry, + /// Preserved closure refusal against the candidate catalog. + source: Box, + }, + /// The retention head changed between admission and `HEAD` replacement. + /// + /// The observed head is boxed so every refusal stays small. `None` on + /// either side means no retention head was published. + RetentionChanged { + /// Head admitted before the first catalog storage call. + expected: Option, + /// Head observed after `head.next` verification. + observed: Box>, + }, +} + +impl From for RetainedCatalogPublicationError { + fn from(source: CatalogPublicationError) -> Self { + Self::Catalog { source } + } +} + +impl Error for RetainedCatalogPublicationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Catalog { source } => Some(source), + Self::RetentionStorage { source, .. } | Self::RootStorage { source, .. } => { + Some(source) + } + Self::HeadDecode { source, .. } => Some(source), + Self::ManifestDecode { source } => Some(source.as_ref()), + Self::RootDecode { source, .. } => Some(source.as_ref()), + Self::Closure { source, .. } => Some(source.as_ref()), + Self::ManifestLength { .. } + | Self::ManifestDigest { .. } + | Self::ManifestGeneration { .. } + | Self::ManifestPredecessor { .. } + | Self::RootBinding { .. } + | Self::RetentionChanged { .. } => None, + } + } +} diff --git a/src/adapters/retained_catalog_publication/retained_catalog_publication_error_display.rs b/src/adapters/retained_catalog_publication/retained_catalog_publication_error_display.rs new file mode 100644 index 00000000..3c5431e0 --- /dev/null +++ b/src/adapters/retained_catalog_publication/retained_catalog_publication_error_display.rs @@ -0,0 +1,51 @@ +//! This module owns diagnostic text for retained-closure publication refusals. + +use std::fmt; + +use super::RetainedCatalogPublicationError; + +impl fmt::Display for RetainedCatalogPublicationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Catalog { .. } => formatter.write_str("retained catalog publication refused"), + Self::RetentionStorage { phase, .. } | Self::HeadDecode { phase, .. } => { + write!(formatter, "{phase} failed") + } + Self::RootStorage { entry, .. } => write!( + formatter, + "retained root read failed at root generation {}", + entry.root_generation().get() + ), + Self::ManifestLength { expected, observed } => write!( + formatter, + "retention manifest length must be {}, observed {observed}", + expected.get() + ), + Self::ManifestDecode { .. } => { + formatter.write_str("retention manifest admission failed") + } + Self::ManifestDigest { .. } => { + formatter.write_str("retention manifest digest differs from the head") + } + Self::ManifestGeneration { expected, observed } => write!( + formatter, + "retention manifest generation must be {}, observed {}", + expected.get(), + observed.get() + ), + Self::ManifestPredecessor { .. } => { + formatter.write_str("retention manifest predecessor differs from the head") + } + Self::RootDecode { .. } => formatter.write_str("retained root admission failed"), + Self::RootBinding { .. } => { + formatter.write_str("retained root differs from its manifest entry") + } + Self::Closure { .. } => { + formatter.write_str("candidate catalog does not preserve a retained closure") + } + Self::RetentionChanged { .. } => { + formatter.write_str("retention head changed before catalog head replacement") + } + } + } +} diff --git a/src/adapters/retained_catalog_publication/retained_catalog_publication_receipt.rs b/src/adapters/retained_catalog_publication/retained_catalog_publication_receipt.rs new file mode 100644 index 00000000..c1f9fe2a --- /dev/null +++ b/src/adapters/retained_catalog_publication/retained_catalog_publication_receipt.rs @@ -0,0 +1,55 @@ +//! This module owns the receipt for one retained-closure catalog publication. + +use super::super::{CatalogPublicationOutcome, CatalogPublicationReceipt}; +use super::PreservedRetentionManifest; +use crate::{CatalogDigest, CatalogGeneration}; + +/// Proof that a candidate became or remained current while preserving retention. +/// +/// The receipt binds the catalog coordinates with the exact retention manifest +/// whose every root closure the candidate was verified to preserve, and whose +/// head was unchanged immediately before `HEAD` replacement. +#[must_use] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct RetainedCatalogPublicationReceipt { + catalog: CatalogPublicationReceipt, + preserved: Option, +} + +impl RetainedCatalogPublicationReceipt { + /// Returns the underlying catalog publication receipt. + pub const fn catalog(self) -> CatalogPublicationReceipt { + self.catalog + } + + /// Returns the exact published catalog generation. + pub const fn generation(self) -> CatalogGeneration { + self.catalog.generation() + } + + /// Returns the exact published physical catalog digest. + pub const fn catalog_digest(self) -> CatalogDigest { + self.catalog.catalog_digest() + } + + /// Returns whether this call published or reverified the generation. + pub const fn outcome(self) -> CatalogPublicationOutcome { + self.catalog.outcome() + } + + /// Returns the preserved retention manifest. + /// + /// `None` means no retention head was published when the gate admitted + /// evidence, so there were no retained closures to preserve. + #[must_use] + pub const fn preserved_manifest(self) -> Option { + self.preserved + } + + pub(super) const fn new( + catalog: CatalogPublicationReceipt, + preserved: Option, + ) -> Self { + Self { catalog, preserved } + } +} diff --git a/src/adapters/retained_catalog_publication/retained_catalog_publication_storage.rs b/src/adapters/retained_catalog_publication/retained_catalog_publication_storage.rs new file mode 100644 index 00000000..a0b3cb2e --- /dev/null +++ b/src/adapters/retained_catalog_publication/retained_catalog_publication_storage.rs @@ -0,0 +1,66 @@ +//! This module owns the blocking port for retained-closure catalog publication. + +use std::io; + +use super::super::CatalogPublicationStorage; +use super::RetainedClosureAdmission; +use crate::{RetentionHead, RetentionManifestEntry}; + +/// Writer-authorized storage for catalog publication over retention state. +/// +/// An implementation holds exclusive store writer authority and one pinned +/// store root for the complete call, the same authority retention publication +/// and recovery require. Retention reads return exact record bytes without +/// interpreting them; the publication gate decodes, cross-checks, and admits +/// every byte itself, so storage cannot vouch for evidence it supplies. +/// +/// The catalog protocol transitions are reachable only through +/// [`catalog_stages`](Self::catalog_stages), which requires a +/// [`RetainedClosureAdmission`]. Only the gate can create that proof. An +/// implementation whose stage type is not otherwise constructible therefore +/// cannot be driven through the ungated +/// [`publish_catalog_generation`](crate::publish_catalog_generation). +pub trait RetainedCatalogPublicationStorage { + /// Catalog protocol transitions guarded by retained-closure admission. + type Stages: CatalogPublicationStorage; + + /// Reads the exact current retention head bytes. + /// + /// Returns `None` only when no retention head is published and no + /// retention artifact contradicts that absence: no root or manifest pool + /// entry and no fixed retention stage. Any fixed retention stage means + /// retention recovery is required and must refuse. A present head is read + /// without following links and bounded by its fixed 144-byte canonical + /// length, plus at most one byte to detect trailing data. + /// + /// # Errors + /// + /// Returns the exact open, kind, read, contradiction, or + /// recovery-required failure. + fn read_retention_head(&mut self) -> io::Result>>; + + /// Reads the exact immutable manifest that `head` selects. + /// + /// The read is bounded by `head.manifest_length()`. An absent manifest is + /// an error, never an empty manifest. + /// + /// # Errors + /// + /// Returns the exact open, kind, absence, or read failure. + fn read_retention_manifest(&mut self, head: &RetentionHead) -> io::Result>; + + /// Reads the exact immutable root that one manifest entry selects. + /// + /// The read is bounded by the version-2 root grammar's maximum encoded + /// length. An absent root is an error, never an empty anchor set. + /// + /// # Errors + /// + /// Returns the exact namespace, open, kind, absence, or read failure. + fn read_retained_root(&mut self, entry: RetentionManifestEntry) -> io::Result>; + + /// Returns the catalog protocol transitions after retained-closure admission. + /// + /// The method must not perform I/O; it only grants access to the stages. + fn catalog_stages(&mut self, admission: &RetainedClosureAdmission) -> &mut Self::Stages; +} diff --git a/src/adapters/retained_catalog_publication/retained_closure_admission.rs b/src/adapters/retained_catalog_publication/retained_closure_admission.rs new file mode 100644 index 00000000..a1999557 --- /dev/null +++ b/src/adapters/retained_catalog_publication/retained_closure_admission.rs @@ -0,0 +1,52 @@ +//! This module owns proof that every current retained closure was admitted. + +use super::PreservedRetentionManifest; +use crate::RetentionHead; + +/// Proof that one call admitted current retention evidence for its candidate. +/// +/// Only [`publish_retained_catalog_generation`](super::publish_retained_catalog_generation) +/// creates this value, after it has admitted the current retention head, the +/// manifest it names, and every root that manifest names, and has verified each +/// root's closure against the candidate catalog. It is neither `Clone` nor +/// constructible outside the gate, so holding a reference proves the gate ran +/// for the current call. +#[must_use] +#[derive(Debug, Eq, PartialEq)] +pub struct RetainedClosureAdmission { + head: Option, + root_count: u32, +} + +impl RetainedClosureAdmission { + /// Returns the preserved manifest, absent when no retention head exists. + #[must_use] + pub const fn preserved_manifest(&self) -> Option { + match self.head { + Some(head) => Some(PreservedRetentionManifest::new( + head.generation(), + head.manifest_digest(), + self.root_count, + )), + None => None, + } + } + + pub(super) const fn unretained() -> Self { + Self { + head: None, + root_count: 0, + } + } + + pub(super) const fn retained(head: RetentionHead, root_count: u32) -> Self { + Self { + head: Some(head), + root_count, + } + } + + pub(super) const fn head(&self) -> Option { + self.head + } +} diff --git a/src/adapters/retained_catalog_publication/retained_evidence_admission.rs b/src/adapters/retained_catalog_publication/retained_evidence_admission.rs new file mode 100644 index 00000000..17ebc990 --- /dev/null +++ b/src/adapters/retained_catalog_publication/retained_evidence_admission.rs @@ -0,0 +1,154 @@ +//! This module owns admission of current retention evidence for one candidate. +//! +//! Storage supplies exact bytes; this module decodes and cross-checks them so +//! evidence admission never depends on the storage being checked. Closure +//! work reuses the storage-independent [`verify_retention_closure`]. + +use super::{ + RetainedCatalogPublicationError as Error, RetainedCatalogPublicationStorage, + RetainedClosureAdmission, RetentionEvidencePhase as Phase, +}; +use crate::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CatalogSnapshot, ChecksummedRetentionHead, + RetentionHead, RetentionManifestEntry, verify_retention_closure, +}; + +/// Admits current retention evidence and verifies every closure against `candidate`. +/// +/// Roots are read, admitted, and verified one at a time in manifest order, so +/// at most one root's bytes are held. The first refusal is returned. +pub(super) fn admit( + storage: &mut impl RetainedCatalogPublicationStorage, + candidate: &CatalogSnapshot<'_, '_, '_>, +) -> Result { + let Some(head) = read_head(storage, Phase::AdmitHead)? else { + return Ok(RetainedClosureAdmission::unretained()); + }; + let encoded = + storage + .read_retention_manifest(&head) + .map_err(|source| Error::RetentionStorage { + phase: Phase::ReadManifest, + source, + })?; + let manifest = admit_manifest(&head, &encoded)?; + for entry in manifest.manifest().entries().iter().copied() { + verify_root(storage, entry, candidate)?; + } + Ok(RetainedClosureAdmission::retained( + head, + manifest.manifest().entry_count(), + )) +} + +/// Requires the current retention head to equal the admitted head. +/// +/// Head decoding is canonical, so semantic equality is byte equality. Manifest +/// and root pool entries are immutable and named by digest, so an unchanged +/// head selects exactly the admitted manifest and roots. +pub(super) fn revalidate( + storage: &mut impl RetainedCatalogPublicationStorage, + admission: &RetainedClosureAdmission, +) -> Result<(), Error> { + let expected = admission.head(); + let observed = read_head(storage, Phase::RevalidateHead)?; + if observed == expected { + Ok(()) + } else { + Err(Error::RetentionChanged { + expected, + observed: Box::new(observed), + }) + } +} + +fn read_head( + storage: &mut impl RetainedCatalogPublicationStorage, + phase: Phase, +) -> Result, Error> { + let Some(encoded) = storage + .read_retention_head() + .map_err(|source| Error::RetentionStorage { phase, source })? + else { + return Ok(None); + }; + let checksummed = ChecksummedRetentionHead::decode(&encoded) + .map_err(|source| Error::HeadDecode { phase, source })?; + Ok(Some(*checksummed.head())) +} + +fn admit_manifest<'encoded>( + head: &RetentionHead, + encoded: &'encoded [u8], +) -> Result, Error> { + let expected = head.manifest_length(); + if u64::try_from(encoded.len()).ok() != Some(expected.get()) { + return Err(Error::ManifestLength { + expected, + observed: encoded.len(), + }); + } + let admitted = + AdmittedRetentionManifest::decode(encoded).map_err(|source| Error::ManifestDecode { + source: Box::new(source), + })?; + require_manifest_binding(head, &admitted)?; + Ok(admitted) +} + +fn require_manifest_binding( + head: &RetentionHead, + admitted: &AdmittedRetentionManifest<'_>, +) -> Result<(), Error> { + let manifest = admitted.manifest(); + if admitted.digest() != head.manifest_digest() { + return Err(Error::ManifestDigest { + expected: head.manifest_digest(), + observed: admitted.digest(), + }); + } + if manifest.generation() != head.generation() { + return Err(Error::ManifestGeneration { + expected: head.generation(), + observed: manifest.generation(), + }); + } + if manifest.predecessor() != head.predecessor() { + return Err(Error::ManifestPredecessor { + expected: head.predecessor(), + observed: manifest.predecessor(), + }); + } + Ok(()) +} + +fn verify_root( + storage: &mut impl RetainedCatalogPublicationStorage, + entry: RetentionManifestEntry, + candidate: &CatalogSnapshot<'_, '_, '_>, +) -> Result<(), Error> { + let encoded = storage + .read_retained_root(entry) + .map_err(|source| Error::RootStorage { entry, source })?; + let admitted = AdmittedRetentionRoot::decode(&encoded).map_err(|source| Error::RootDecode { + entry, + source: Box::new(source), + })?; + let root = admitted.root(); + let observed = RetentionManifestEntry::new( + root.namespace().digest(), + root.generation(), + admitted.digest(), + ); + if observed != entry { + return Err(Error::RootBinding { + expected: entry, + observed: Box::new(observed), + }); + } + let _verified = verify_retention_closure(root, candidate).map_err(|source| Error::Closure { + entry, + source: Box::new(source), + })?; + Ok(()) +} diff --git a/src/adapters/retained_catalog_publication/retention_evidence_phase.rs b/src/adapters/retained_catalog_publication/retention_evidence_phase.rs new file mode 100644 index 00000000..7e64f3e6 --- /dev/null +++ b/src/adapters/retained_catalog_publication/retention_evidence_phase.rs @@ -0,0 +1,24 @@ +//! This module owns the retention evidence reads of catalog publication. + +use std::fmt; + +/// Retention evidence read attempted by retained-closure catalog publication. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetentionEvidencePhase { + /// Read the current retention head before any catalog storage call. + AdmitHead, + /// Read the manifest the admitted retention head selects. + ReadManifest, + /// Reread the retention head after `head.next` verification. + RevalidateHead, +} + +impl fmt::Display for RetentionEvidencePhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::AdmitHead => "retention-head admission", + Self::ReadManifest => "retention-manifest read", + Self::RevalidateHead => "retention-head revalidation", + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index 74ef5b55..888bc2dc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -196,6 +196,11 @@ pub use adapters::{ ObservedSegmentStage, RepositoryInitializationStorage, SegmentStageDurabilityEvent, SegmentStageObserver, }; +pub use adapters::{ + PreservedRetentionManifest, RetainedCatalogPublicationError, RetainedCatalogPublicationReceipt, + RetainedCatalogPublicationStorage, RetainedClosureAdmission, RetentionEvidencePhase, + publish_retained_catalog_generation, +}; pub use adapters::{VerificationError, VerificationSource, verify_catalog_bytes, verify_segment}; pub use blob::{ BlobHashError, BlobHasher, BlobId, BlobLength, BlobReadError, ByteLength, ByteOffset, diff --git a/tests/retained_catalog_publication.rs b/tests/retained_catalog_publication.rs new file mode 100644 index 00000000..fddb369d --- /dev/null +++ b/tests/retained_catalog_publication.rs @@ -0,0 +1,210 @@ +//! Retained-closure catalog publication laws. + +#[path = "retained_catalog_publication/catalog_fixture.rs"] +pub mod catalog_fixture; +#[path = "retained_catalog_publication/closure_model_laws.rs"] +mod closure_model_laws; +#[path = "retained_catalog_publication/closure_refusal_laws.rs"] +mod closure_refusal_laws; +#[path = "retained_catalog_publication/evidence_refusal_laws.rs"] +mod evidence_refusal_laws; +#[path = "retention_closure/memory_stage.rs"] +mod memory_stage; +#[path = "retained_catalog_publication/model_storage.rs"] +pub mod model_storage; +#[path = "catalog_publication/recording_storage.rs"] +pub mod recording_storage; +#[path = "retained_catalog_publication/retention_fixture.rs"] +pub mod retention_fixture; +#[path = "retained_catalog_publication/revalidation_laws.rs"] +mod revalidation_laws; +mod support; + +use std::error::Error; + +use catalog_fixture::{Candidate, CandidateCall, with_candidate}; +use keep::{ + CatalogGeneration, CatalogPublicationError, CatalogPublicationOutcome, CatalogPublicationPhase, + RetainedCatalogPublicationError, RetainedCatalogPublicationReceipt, SegmentPublication, + publish_retained_catalog_generation, +}; +use model_storage::{EvidenceRead, ModelStorage}; +use recording_storage::{EXPECTED_WITH_SEGMENT, RecordingStorage}; +use retention_fixture::RetentionRecords; +use support::require_error; + +/// Catalog-only protocol order: every phase except the five segment phases. +fn catalog_only_phases() -> Vec { + EXPECTED_WITH_SEGMENT + .iter() + .copied() + .filter(|phase| !is_segment_phase(*phase)) + .collect() +} + +const fn is_segment_phase(phase: CatalogPublicationPhase) -> bool { + matches!( + phase, + CatalogPublicationPhase::LinkSegment + | CatalogPublicationPhase::VerifySegmentPool + | CatalogPublicationPhase::SynchronizeSegments + | CatalogPublicationPhase::RemoveSegmentStage + | CatalogPublicationPhase::SynchronizeStagingAfterSegment + ) +} + +/// Number of catalog-only stages that precede `HEAD` replacement. +fn stages_before_replacement() -> Result> { + catalog_only_phases() + .iter() + .position(|phase| *phase == CatalogPublicationPhase::ReplaceHead) + .ok_or_else(|| "catalog order omits head replacement".into()) +} + +/// Publishes one catalog-only candidate through `storage`. +fn publish( + storage: &mut ModelStorage, + call: &CandidateCall<'_, '_>, +) -> Result { + publish_retained_catalog_generation( + storage, + call.expectation, + SegmentPublication::none(), + call.catalog, + call.segments, + ) +} + +#[test] +fn preserving_successor_publishes_and_binds_the_preserved_manifest() -> Result<(), Box> { + let records = RetentionRecords::golden()?; + let head = records.current_head()?; + let entries = records.entries()?; + let mut storage = ModelStorage::new(records); + + let (receipt, digest) = with_candidate(Candidate::Preserving, |call| { + ( + publish(&mut storage, &call), + call.catalog.checksummed().digest(), + ) + })?; + let receipt = receipt?; + let preserved = receipt + .preserved_manifest() + .ok_or("receipt omits retention")?; + let entry = *entries.first().ok_or("golden manifest has no entry")?; + + assert_eq!(receipt.outcome(), CatalogPublicationOutcome::Published); + assert_eq!(receipt.generation(), CatalogGeneration::new(2)?); + assert_eq!(receipt.catalog_digest(), digest); + assert_eq!(preserved.generation(), head.generation()); + assert_eq!(preserved.digest(), head.manifest_digest()); + assert_eq!(preserved.root_count(), 1); + assert_eq!(storage.stages(), catalog_only_phases()); + assert_eq!( + storage.reads(), + [ + (EvidenceRead::Head, 0), + (EvidenceRead::Manifest, 0), + (EvidenceRead::Root(entry), 0), + (EvidenceRead::Head, stages_before_replacement()?), + ] + ); + Ok(()) +} + +#[test] +fn unretained_store_publishes_with_no_preserved_manifest() -> Result<(), Box> { + let mut storage = ModelStorage::new(RetentionRecords::unretained()); + + let receipt = with_candidate(Candidate::Empty, |call| publish(&mut storage, &call))??; + + assert_eq!(receipt.outcome(), CatalogPublicationOutcome::Published); + assert_eq!(receipt.preserved_manifest(), None); + assert_eq!( + storage.reads(), + [ + (EvidenceRead::Head, 0), + (EvidenceRead::Head, stages_before_replacement()?), + ] + ); + Ok(()) +} + +#[test] +fn already_published_candidate_proves_retention_before_reverification() -> Result<(), Box> +{ + let records = RetentionRecords::golden()?; + let mut storage = ModelStorage::with_stages(records, RecordingStorage::already_published()); + + let receipt = with_candidate(Candidate::Preserving, |call| publish(&mut storage, &call))??; + + assert_eq!( + receipt.outcome(), + CatalogPublicationOutcome::AlreadyPublished + ); + assert_eq!( + receipt + .preserved_manifest() + .map(keep::PreservedRetentionManifest::root_count), + Some(1) + ); + assert_eq!( + storage.stages(), + [ + CatalogPublicationPhase::VerifyCurrent, + CatalogPublicationPhase::SynchronizeRoot + ] + ); + assert!(storage.reads().iter().all(|(_, stages)| *stages == 0)); + Ok(()) +} + +#[test] +fn catalog_preflight_refuses_before_any_retention_read() -> Result<(), Box> { + let mut storage = ModelStorage::new(RetentionRecords::golden()?); + + let error = with_candidate(Candidate::Preserving, |call| { + publish_retained_catalog_generation( + &mut storage, + keep::CatalogPublicationExpectation::uninitialized(), + SegmentPublication::none(), + call.catalog, + call.segments, + ) + })?; + let error = require_error(error, "non-initial candidate published over no catalog")?; + + assert!(matches!( + error, + RetainedCatalogPublicationError::Catalog { + source: CatalogPublicationError::InitialGeneration { observed } + } if observed.get() == 2 + )); + assert!(storage.reads().is_empty()); + assert!(storage.stages().is_empty()); + Ok(()) +} + +#[test] +fn every_catalog_fault_after_admission_keeps_its_exact_phase() -> Result<(), Box> { + let replacement = stages_before_replacement()?; + for (index, failing) in catalog_only_phases().into_iter().enumerate() { + let stages = RecordingStorage::failing_at(failing); + let mut storage = ModelStorage::with_stages(RetentionRecords::golden()?, stages); + + let result = with_candidate(Candidate::Preserving, |call| publish(&mut storage, &call))?; + let error = require_error(result, "faulting publication returned a receipt")?; + let revalidated = storage.reads().len() == 4; + + assert!(matches!( + error, + RetainedCatalogPublicationError::Catalog { + source: CatalogPublicationError::Storage { phase, .. } + } if phase == failing + )); + assert_eq!(storage.stages().last(), Some(&failing)); + assert_eq!(revalidated, index >= replacement, "{failing:?}"); + } + Ok(()) +} diff --git a/tests/retained_catalog_publication/catalog_fixture.rs b/tests/retained_catalog_publication/catalog_fixture.rs new file mode 100644 index 00000000..9814d54f --- /dev/null +++ b/tests/retained_catalog_publication/catalog_fixture.rs @@ -0,0 +1,146 @@ +//! Current and candidate catalogs for retained-closure publication laws. + +use std::error::Error; + +use keep::{ + AdmittedLayout, AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, + CanonicalPublicationHead, CatalogGeneration, CatalogPublicationExpectation, + ChecksummedPublicationHead, LayoutDecodePolicy, LayoutEntryLimit, SegmentReadPolicy, + SegmentRecordIdentity, SegmentRecordLimit, +}; + +use crate::memory_stage::segment_bytes; +use crate::retention_fixture::fixture; +use crate::support::layout_record_bytes; + +const BUNDLE_SEGMENT_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-bundle-segment.hex"); +const CHUNK_SEGMENT_HEX: &str = + include_str!("../../conformance/segment-store/v1/one-zero-segment.hex"); + +/// Generation-two candidate shape relative to the retained one-zero blob. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Candidate { + /// Keeps the retained layout and chunk records. + Preserving, + /// Keeps the retained chunk but drops its layout. + DroppedLayout, + /// Keeps the retained layout but substitutes another chunk for its chunk. + SubstitutedChunk, + /// Drops every record. + Empty, +} + +impl Candidate { + /// Every candidate shape, in a fixed order. + pub const ALL: [Self; 4] = [ + Self::Preserving, + Self::DroppedLayout, + Self::SubstitutedChunk, + Self::Empty, + ]; +} + +/// Inputs to one candidate publication call. +pub struct CandidateCall<'call, 'records> { + /// Exact current catalog the candidate succeeds. + pub expectation: CatalogPublicationExpectation, + /// Canonical generation-two candidate. + pub catalog: &'call CanonicalCatalog, + /// Admitted candidate segment set. + pub segments: &'call [AdmittedSegment<'records>], +} + +/// Runs `operation` with one candidate succeeding the retained bundle catalog. +/// +/// # Errors +/// +/// Returns the exact fixture, segment, catalog, or snapshot refusal. +pub fn with_candidate( + candidate: Candidate, + operation: impl FnOnce(CandidateCall<'_, '_>) -> T, +) -> Result> { + let current_bytes = fixture(BUNDLE_SEGMENT_HEX)?; + let current_segments = [AdmittedSegment::decode(¤t_bytes, maximum_policy())?]; + let current = + CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, ¤t_segments)?; + let expectation = expectation_for(¤t, ¤t_segments)?; + let candidate_bytes = candidate_segment(candidate)?; + let segments = candidate_bytes + .as_deref() + .map(|bytes| AdmittedSegment::decode(bytes, maximum_policy())) + .transpose()? + .into_iter() + .collect::>(); + let catalog = CanonicalCatalog::from_segments( + CatalogGeneration::new(2)?, + Some(current.checksummed().digest()), + &segments, + )?; + Ok(operation(CandidateCall { + expectation, + catalog: &catalog, + segments: &segments, + })) +} + +/// Returns the identity a candidate drops from the retained closure. +/// +/// # Errors +/// +/// Returns the exact layout fixture refusal or `Preserving`, which drops none. +pub fn dropped_identity(candidate: Candidate) -> Result> { + let layout = one_zero_layout()?; + match candidate { + Candidate::Preserving => Err("a preserving candidate drops no identity".into()), + Candidate::DroppedLayout | Candidate::Empty => { + Ok(SegmentRecordIdentity::Layout(layout.encode_record()?.id())) + } + Candidate::SubstitutedChunk => { + let entry = layout + .entries() + .first() + .copied() + .ok_or("one-zero layout omits its chunk")?; + Ok(SegmentRecordIdentity::Chunk(entry.chunk_id())) + } + } +} + +/// Returns the candidate's only segment, absent for a catalog with no records. +fn candidate_segment(candidate: Candidate) -> Result>, Box> { + match candidate { + Candidate::Preserving => fixture(BUNDLE_SEGMENT_HEX).map(Some), + Candidate::DroppedLayout => fixture(CHUNK_SEGMENT_HEX).map(Some), + Candidate::Empty => Ok(None), + Candidate::SubstitutedChunk => { + let canonical = one_zero_layout()?.encode_record()?; + let records = [ + AdmittedSegmentRecord::for_chunk(b"substituted chunk")?, + AdmittedSegmentRecord::for_layout(&canonical)?, + ]; + Ok(Some(segment_bytes(&records)?)) + } + } +} + +fn expectation_for( + current: &CanonicalCatalog, + segments: &[AdmittedSegment<'_>], +) -> Result> { + let head = CanonicalPublicationHead::for_catalog(current.checksummed()); + let admitted = current.checksummed().admit(segments)?; + let snapshot = ChecksummedPublicationHead::decode(head.encoded())?.admit(admitted)?; + Ok(CatalogPublicationExpectation::successor_of(&snapshot)) +} + +fn one_zero_layout() -> Result> { + Ok(AdmittedLayout::decode_record( + &layout_record_bytes("one-zero")?, + LayoutDecodePolicy::new(LayoutEntryLimit::MAXIMUM), + )?) +} + +const fn maximum_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/tests/retained_catalog_publication/closure_model_laws.rs b/tests/retained_catalog_publication/closure_model_laws.rs new file mode 100644 index 00000000..497a316f --- /dev/null +++ b/tests/retained_catalog_publication/closure_model_laws.rs @@ -0,0 +1,145 @@ +//! Exhaustive comparison of the gate with a boring retained-closure model. + +use std::error::Error; + +use keep::{ + AdmittedRetentionRoot, RetainedCatalogPublicationError, RetentionClosureVerificationError, + RetentionManifestEntry, SegmentRecordIdentity, +}; + +use crate::catalog_fixture::{Candidate, dropped_identity, with_candidate}; +use crate::model_storage::{EvidenceRead, ModelStorage}; +use crate::publish; +use crate::retention_fixture::{RetentionRecords, RootShape}; + +const RETAIN: RootShape = RootShape::Retaining; +const RELEASE: RootShape = RootShape::Released; + +/// Typed result of one publication call. +#[derive(Debug, Eq, PartialEq)] +enum Outcome { + /// Published, preserving this many manifest roots, or no retention head. + Published(Option), + /// Refused at this manifest entry for this missing closure member. + Refused(RetentionManifestEntry, SegmentRecordIdentity), +} + +/// Outcome, catalog stage count, and every evidence read of one call. +#[derive(Debug, Eq, PartialEq)] +struct Observation { + outcome: Outcome, + stages: usize, + roots_read: Vec, +} + +fn configurations() -> Result, Box> { + Ok(vec![ + RetentionRecords::unretained(), + RetentionRecords::golden()?, + RetentionRecords::built(&[(b"a", RELEASE)])?, + RetentionRecords::built(&[(b"a", RETAIN), (b"b", RELEASE)])?, + RetentionRecords::built(&[(b"a", RETAIN), (b"b", RETAIN)])?, + RetentionRecords::built(&[(b"a", RELEASE), (b"b", RELEASE), (b"c", RETAIN)])?, + ]) +} + +#[test] +fn gate_agrees_with_the_model_for_every_candidate_and_root_set() -> Result<(), Box> { + let mut cases = 0_u32; + for records in configurations()? { + for candidate in Candidate::ALL { + let first = run(&records, candidate)?; + let second = run(&records, candidate)?; + assert_eq!(first, second, "verification must be deterministic"); + assert_eq!(first, model(&records, candidate)?, "{candidate:?}"); + cases = cases.checked_add(1).ok_or("case count overflowed")?; + } + } + assert_eq!(cases, 24); + Ok(()) +} + +fn run(records: &RetentionRecords, candidate: Candidate) -> Result> { + let mut storage = ModelStorage::new(records.clone()); + let result = with_candidate(candidate, |call| publish(&mut storage, &call))?; + let outcome = match result { + Ok(receipt) => Outcome::Published( + receipt + .preserved_manifest() + .map(keep::PreservedRetentionManifest::root_count), + ), + Err(error) => refused(&error)?, + }; + let roots_read = storage + .reads() + .iter() + .filter_map(|(read, _)| match read { + EvidenceRead::Root(entry) => Some(*entry), + EvidenceRead::Head | EvidenceRead::Manifest => None, + }) + .collect(); + Ok(Observation { + outcome, + stages: storage.stages().len(), + roots_read, + }) +} + +fn refused(error: &RetainedCatalogPublicationError) -> Result> { + let RetainedCatalogPublicationError::Closure { entry, source } = error else { + return Err(format!("unexpected refusal: {error:?}").into()); + }; + let RetentionClosureVerificationError::MissingMember { identity } = source.as_ref() else { + return Err(format!("unexpected closure refusal: {source:?}").into()); + }; + Ok(Outcome::Refused(*entry, *identity)) +} + +/// The model: the first retaining root in manifest order refuses any candidate +/// that does not keep the one-zero blob; otherwise every root is verified. +fn model(records: &RetentionRecords, candidate: Candidate) -> Result> { + let entries = match records.head { + Some(_) => records.entries()?, + None => Vec::new(), + }; + let refusal = + first_retaining(records, &entries)?.filter(|_| candidate != Candidate::Preserving); + let Some(position) = refusal else { + let count = records + .head + .as_ref() + .map(|_| u32::try_from(entries.len())) + .transpose()?; + return Ok(Observation { + outcome: Outcome::Published(count), + stages: crate::catalog_only_phases().len(), + roots_read: entries, + }); + }; + let entry = *entries + .get(position) + .ok_or("model position is out of range")?; + let read = position.checked_add(1).ok_or("model position overflowed")?; + Ok(Observation { + outcome: Outcome::Refused(entry, dropped_identity(candidate)?), + stages: 0, + roots_read: entries.into_iter().take(read).collect(), + }) +} + +fn first_retaining( + records: &RetentionRecords, + entries: &[RetentionManifestEntry], +) -> Result, Box> { + for (position, entry) in entries.iter().enumerate() { + let (_, bytes) = records + .roots + .iter() + .find(|(digest, _)| *digest == entry.root_digest()) + .ok_or("records omit a manifest root")?; + if AdmittedRetentionRoot::decode(bytes)?.root().anchor_count() > 0 { + return Ok(Some(position)); + } + } + Ok(None) +} diff --git a/tests/retained_catalog_publication/closure_refusal_laws.rs b/tests/retained_catalog_publication/closure_refusal_laws.rs new file mode 100644 index 00000000..bd2dd7ca --- /dev/null +++ b/tests/retained_catalog_publication/closure_refusal_laws.rs @@ -0,0 +1,118 @@ +//! Candidates that lose retained content refuse before any catalog mutation. + +use std::error::Error; + +use keep::{RetainedCatalogPublicationError, RetentionClosureVerificationError}; + +use crate::catalog_fixture::{Candidate, dropped_identity, with_candidate}; +use crate::model_storage::{EvidenceRead, ModelStorage}; +use crate::publish; +use crate::retention_fixture::{RetentionRecords, RootShape}; +use crate::support::require_error; + +#[test] +fn candidate_dropping_a_retained_layout_refuses_before_any_stage() -> Result<(), Box> { + require_loss_refused(Candidate::DroppedLayout) +} + +#[test] +fn candidate_substituting_a_retained_chunk_refuses_before_any_stage() -> Result<(), Box> +{ + require_loss_refused(Candidate::SubstitutedChunk) +} + +#[test] +fn every_manifest_root_is_verified_wherever_it_sorts() -> Result<(), Box> { + let mut positions = Vec::new(); + for names in [[b"alpha".as_slice(), b"omega"], [b"omega", b"alpha"]] { + let [retaining, released] = names; + let records = RetentionRecords::built(&[ + (retaining, RootShape::Retaining), + (released, RootShape::Released), + ])?; + let entries = records.entries()?; + let retained = retained_entry(&records, &entries)?; + positions.push(entries.iter().position(|entry| *entry == retained)); + let mut storage = ModelStorage::new(records.clone()); + + let result = with_candidate(Candidate::DroppedLayout, |call| { + publish(&mut storage, &call) + })?; + let error = require_error(result, "candidate dropped a retained root's layout")?; + + assert!(matches!( + error, + RetainedCatalogPublicationError::Closure { entry, .. } if entry == retained + )); + assert!(storage.stages().is_empty()); + let mut storage = ModelStorage::new(records); + let receipt = with_candidate(Candidate::Preserving, |call| publish(&mut storage, &call))??; + assert_eq!( + receipt + .preserved_manifest() + .map(keep::PreservedRetentionManifest::root_count), + Some(2) + ); + } + positions.sort_unstable(); + assert_eq!( + positions, + [Some(0), Some(1)], + "both manifest orders are exercised" + ); + Ok(()) +} + +fn require_loss_refused(candidate: Candidate) -> Result<(), Box> { + let records = RetentionRecords::golden()?; + let golden_entry = *records + .entries()? + .first() + .ok_or("golden manifest is empty")?; + let expected = dropped_identity(candidate)?; + let mut storage = ModelStorage::new(records); + + let result = with_candidate(candidate, |call| publish(&mut storage, &call))?; + let error = require_error(result, "candidate that loses retained content published")?; + + let RetainedCatalogPublicationError::Closure { entry, source } = &error else { + return Err(format!("unexpected refusal: {error:?}").into()); + }; + assert_eq!(*entry, golden_entry); + assert!(matches!( + source.as_ref(), + RetentionClosureVerificationError::MissingMember { identity } if *identity == expected + )); + assert!(std::error::Error::source(&error).is_some()); + assert!(storage.stages().is_empty(), "no catalog stage may run"); + assert_eq!( + storage.reads(), + [ + (EvidenceRead::Head, 0), + (EvidenceRead::Manifest, 0), + (EvidenceRead::Root(golden_entry), 0), + ] + ); + Ok(()) +} + +fn retained_entry( + records: &RetentionRecords, + entries: &[keep::RetentionManifestEntry], +) -> Result> { + for entry in entries { + let (_, bytes) = records + .roots + .iter() + .find(|(digest, _)| *digest == entry.root_digest()) + .ok_or("built records omit a root")?; + if keep::AdmittedRetentionRoot::decode(bytes)? + .root() + .anchor_count() + > 0 + { + return Ok(*entry); + } + } + Err("built records retain nothing".into()) +} diff --git a/tests/retained_catalog_publication/evidence_refusal_laws.rs b/tests/retained_catalog_publication/evidence_refusal_laws.rs new file mode 100644 index 00000000..6344e924 --- /dev/null +++ b/tests/retained_catalog_publication/evidence_refusal_laws.rs @@ -0,0 +1,263 @@ +//! Absent, corrupt, or contradictory retention evidence refuses before any stage. + +use std::error::Error; +use std::io; + +use keep::{ + LivenessGeneration, RetainedCatalogPublicationError as Refusal, RetentionEvidencePhase, + RetentionHead, RetentionHeadDecodeError, RetentionManifestDecodeError, RetentionManifestLength, + RetentionRootDecodeError, +}; + +use crate::catalog_fixture::{Candidate, with_candidate}; +use crate::model_storage::ModelStorage; +use crate::publish; +use crate::retention_fixture::{RetentionRecords, RootShape, encode_head}; +use crate::support::require_error; + +/// Publishes the preserving candidate and requires a refusal with no stage. +fn refuse(records: RetentionRecords) -> Result> { + refuse_with(ModelStorage::new(records)) +} + +fn refuse_with(mut storage: ModelStorage) -> Result> { + let result = with_candidate(Candidate::Preserving, |call| publish(&mut storage, &call))?; + let error = require_error(result, "unadmitted retention evidence published")?; + assert!(storage.stages().is_empty(), "no catalog stage may run"); + Ok(error) +} + +fn flip_last(bytes: &mut [u8]) -> Result<(), Box> { + let last = bytes.last_mut().ok_or("record is empty")?; + *last ^= 1; + Ok(()) +} + +#[test] +fn absent_manifest_refuses_with_its_exact_read() -> Result<(), Box> { + let mut records = RetentionRecords::golden()?; + records.manifest = None; + + let error = refuse(records)?; + + assert!(matches!( + error, + Refusal::RetentionStorage { phase: RetentionEvidencePhase::ReadManifest, ref source } + if source.kind() == io::ErrorKind::NotFound + )); + Ok(()) +} + +#[test] +fn absent_root_refuses_with_its_manifest_entry() -> Result<(), Box> { + let mut records = RetentionRecords::golden()?; + let expected = *records + .entries()? + .first() + .ok_or("golden manifest is empty")?; + records.roots.clear(); + + let error = refuse(records)?; + + assert!(matches!( + error, + Refusal::RootStorage { entry, ref source } + if entry == expected && source.kind() == io::ErrorKind::NotFound + )); + Ok(()) +} + +#[test] +fn failed_head_read_refuses_at_admission() -> Result<(), Box> { + let storage = ModelStorage::new(RetentionRecords::golden()?).failing_head_read(0); + + let error = refuse_with(storage)?; + + assert!(matches!( + error, + Refusal::RetentionStorage { + phase: RetentionEvidencePhase::AdmitHead, + .. + } + )); + Ok(()) +} + +#[test] +fn corrupt_head_refuses_with_its_checksum() -> Result<(), Box> { + let mut records = RetentionRecords::golden()?; + flip_last(records.head.as_mut().ok_or("golden head is absent")?)?; + + let error = refuse(records)?; + + assert!(matches!( + error, + Refusal::HeadDecode { + phase: RetentionEvidencePhase::AdmitHead, + source: RetentionHeadDecodeError::ChecksumMismatch { .. } + } + )); + Ok(()) +} + +#[test] +fn corrupt_manifest_refuses_with_its_checksum() -> Result<(), Box> { + let mut records = RetentionRecords::golden()?; + let (_, manifest) = records + .manifest + .as_mut() + .ok_or("golden manifest is absent")?; + flip_last(manifest)?; + + let error = refuse(records)?; + + assert!(matches!( + error, + Refusal::ManifestDecode { ref source } + if matches!(**source, RetentionManifestDecodeError::ChecksumMismatch { .. }) + )); + Ok(()) +} + +#[test] +fn corrupt_root_refuses_with_its_entry_and_checksum() -> Result<(), Box> { + let mut records = RetentionRecords::golden()?; + let expected = *records + .entries()? + .first() + .ok_or("golden manifest is empty")?; + let (_, root) = records.roots.first_mut().ok_or("golden root is absent")?; + flip_last(root)?; + + let error = refuse(records)?; + + assert!(matches!( + error, + Refusal::RootDecode { entry, ref source } + if entry == expected + && matches!(**source, RetentionRootDecodeError::ChecksumMismatch { .. }) + )); + Ok(()) +} + +#[test] +fn substituted_root_refuses_with_expected_and_observed_entries() -> Result<(), Box> { + let mut records = RetentionRecords::golden()?; + let expected = *records + .entries()? + .first() + .ok_or("golden manifest is empty")?; + let other = RetentionRecords::built(&[(b"substitute", RootShape::Retaining)])?; + let observed = *other.entries()?.first().ok_or("built manifest is empty")?; + let (_, substitute) = other.roots.first().ok_or("built root is absent")?; + let (_, root) = records.roots.first_mut().ok_or("golden root is absent")?; + root.clone_from(substitute); + + let error = refuse(records)?; + + assert!(matches!( + error, + Refusal::RootBinding { expected: e, observed: ref o } if e == expected && **o == observed + )); + Ok(()) +} + +#[test] +fn substituted_manifest_refuses_with_both_digests() -> Result<(), Box> { + let mut records = RetentionRecords::golden()?; + let expected = records.current_head()?.manifest_digest(); + let other = RetentionRecords::built(&[(b"contract", RootShape::Released)])?; + let (observed, substitute) = other.manifest.ok_or("built manifest is absent")?; + let (_, manifest) = records + .manifest + .as_mut() + .ok_or("golden manifest is absent")?; + *manifest = substitute; + + let error = refuse(records)?; + + assert!(matches!( + error, + Refusal::ManifestDigest { expected: e, observed: o } if e == expected && o == observed + )); + Ok(()) +} + +#[test] +fn head_naming_another_manifest_length_refuses_before_decoding() -> Result<(), Box> { + let mut records = RetentionRecords::golden()?; + let current = records.current_head()?; + let actual = current.manifest_length().get(); + let forged = RetentionManifestLength::new(actual.checked_add(72).ok_or("length overflow")?)?; + let head = RetentionHead::new( + current.generation(), + forged, + current.manifest_digest(), + None, + )?; + records.head = Some(encode_head(&head)); + + let error = refuse(records)?; + + assert!(matches!( + error, + Refusal::ManifestLength { expected, observed } + if expected == forged && u64::try_from(observed).ok() == Some(actual) + )); + Ok(()) +} + +#[test] +fn head_naming_another_generation_refuses_with_both_generations() -> Result<(), Box> { + let mut records = RetentionRecords::golden()?; + let current = records.current_head()?; + let forged = LivenessGeneration::new(2)?; + let head = RetentionHead::new( + forged, + current.manifest_length(), + current.manifest_digest(), + Some(current.manifest_digest()), + )?; + records.head = Some(encode_head(&head)); + + let error = refuse(records)?; + + assert!(matches!( + error, + Refusal::ManifestGeneration { expected, observed } + if expected == forged && observed == current.generation() + )); + Ok(()) +} + +#[test] +fn head_naming_another_predecessor_refuses_with_both_predecessors() -> Result<(), Box> { + let golden = RetentionRecords::golden()?; + let golden_digest = golden.current_head()?.manifest_digest(); + let generation = LivenessGeneration::new(2)?; + let manifest = + keep::RetentionManifest::new(generation, Some(golden_digest), golden.entries()?)?; + let canonical = keep::CanonicalRetentionManifest::from_manifest(&manifest)?; + let length = RetentionManifestLength::new(u64::try_from(canonical.encoded().len())?)?; + let forged = RetentionRecords::built(&[(b"forged", RootShape::Released)])?.current_head()?; + let head = RetentionHead::new( + generation, + length, + canonical.digest(), + Some(forged.manifest_digest()), + )?; + let records = RetentionRecords { + head: Some(encode_head(&head)), + manifest: Some((canonical.digest(), canonical.encoded().to_vec())), + roots: golden.roots, + }; + + let error = refuse(records)?; + + assert!(matches!( + error, + Refusal::ManifestPredecessor { expected, observed } + if expected == Some(forged.manifest_digest()) && observed == Some(golden_digest) + )); + Ok(()) +} diff --git a/tests/retained_catalog_publication/model_storage.rs b/tests/retained_catalog_publication/model_storage.rs new file mode 100644 index 00000000..7abf4d2d --- /dev/null +++ b/tests/retained_catalog_publication/model_storage.rs @@ -0,0 +1,124 @@ +//! In-memory writer-authorized storage that records every port call. + +use std::io; + +use keep::{ + CatalogPublicationPhase, RetainedCatalogPublicationStorage, RetainedClosureAdmission, + RetentionHead, RetentionManifestEntry, +}; + +use crate::recording_storage::RecordingStorage; +use crate::retention_fixture::RetentionRecords; + +/// One retention evidence read the gate requested. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum EvidenceRead { + /// `read_retention_head`. + Head, + /// `read_retention_manifest`. + Manifest, + /// `read_retained_root` for one exact entry. + Root(RetentionManifestEntry), +} + +/// Storage model over exact retention records and recorded catalog stages. +pub struct ModelStorage { + stages: RecordingStorage, + records: RetentionRecords, + later_heads: Vec>>, + failing_head_read: Option, + reads: Vec<(EvidenceRead, usize)>, +} + +impl ModelStorage { + /// Serves `records` and admits every catalog stage. + pub const fn new(records: RetentionRecords) -> Self { + Self::with_stages(records, RecordingStorage::succeeding()) + } + + /// Serves `records` through the supplied catalog stage recorder. + pub const fn with_stages(records: RetentionRecords, stages: RecordingStorage) -> Self { + Self { + stages, + records, + later_heads: Vec::new(), + failing_head_read: None, + reads: Vec::new(), + } + } + + /// Serves `head` on every head read after the first. + #[must_use] + pub fn then_head(mut self, head: Option>) -> Self { + self.later_heads.push(head); + self + } + + /// Fails the zero-based `ordinal` head read. + #[must_use] + pub const fn failing_head_read(mut self, ordinal: usize) -> Self { + self.failing_head_read = Some(ordinal); + self + } + + /// Returns every catalog stage attempted. + pub fn stages(&self) -> &[CatalogPublicationPhase] { + self.stages.observed() + } + + /// Returns every evidence read with the catalog stage count before it. + pub fn reads(&self) -> &[(EvidenceRead, usize)] { + &self.reads + } + + fn record(&mut self, read: EvidenceRead) { + self.reads.push((read, self.stages.observed().len())); + } + + fn head_reads(&self) -> usize { + self.reads + .iter() + .filter(|(read, _)| *read == EvidenceRead::Head) + .count() + } +} + +impl RetainedCatalogPublicationStorage for ModelStorage { + type Stages = RecordingStorage; + + fn read_retention_head(&mut self) -> io::Result>> { + let ordinal = self.head_reads(); + self.record(EvidenceRead::Head); + if self.failing_head_read == Some(ordinal) { + return Err(io::Error::other("injected retention-head read failure")); + } + let later = ordinal.checked_sub(1).and_then(|index| { + self.later_heads + .get(index) + .or_else(|| self.later_heads.last()) + }); + Ok(later.map_or_else(|| self.records.head.clone(), Clone::clone)) + } + + fn read_retention_manifest(&mut self, head: &RetentionHead) -> io::Result> { + self.record(EvidenceRead::Manifest); + match &self.records.manifest { + Some((digest, bytes)) if *digest == head.manifest_digest() => Ok(bytes.clone()), + _ => Err(io::Error::from(io::ErrorKind::NotFound)), + } + } + + fn read_retained_root(&mut self, entry: RetentionManifestEntry) -> io::Result> { + self.record(EvidenceRead::Root(entry)); + self.records + .roots + .iter() + .find(|(digest, _)| *digest == entry.root_digest()) + .map(|(_, bytes)| bytes.clone()) + .ok_or_else(|| io::Error::from(io::ErrorKind::NotFound)) + } + + fn catalog_stages(&mut self, _admission: &RetainedClosureAdmission) -> &mut RecordingStorage { + &mut self.stages + } +} diff --git a/tests/retained_catalog_publication/retention_fixture.rs b/tests/retained_catalog_publication/retention_fixture.rs new file mode 100644 index 00000000..444fc8c2 --- /dev/null +++ b/tests/retained_catalog_publication/retention_fixture.rs @@ -0,0 +1,152 @@ +//! Exact retention records for retained-closure publication laws. + +use std::error::Error; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, + CanonicalRetentionManifest, CanonicalRetentionRoot, ChecksummedRetentionHead, + LivenessGeneration, RetentionHead, RetentionManifest, RetentionManifestDigest, + RetentionManifestEntry, RetentionManifestLength, RetentionNamespace, RetentionPolicy, + RetentionRoot, RetentionRootDigest, RootGeneration, +}; + +use crate::support::decode_hex; + +const ROOT_HEX: &str = include_str!("../../conformance/segment-store/v2/one-anchor-root.hex"); +const MANIFEST_HEX: &str = include_str!("../../conformance/segment-store/v2/one-root-manifest.hex"); +const HEAD_HEX: &str = include_str!("../../conformance/segment-store/v2/one-root-head.hex"); + +/// Whether a built root retains the one-zero blob or nothing. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RootShape { + /// Anchors the frozen one-zero blob and layout. + Retaining, + /// Publishes an empty anchor set. + Released, +} + +/// Exact retention records served by a storage model. +#[derive(Clone, Debug)] +pub struct RetentionRecords { + /// Current `retention/HEAD` bytes, absent when unpublished. + pub head: Option>, + /// Manifest pool entry keyed by the digest storage resolves it with. + pub manifest: Option<(RetentionManifestDigest, Vec)>, + /// Root pool entries keyed by the digest storage resolves them with. + pub roots: Vec<(RetentionRootDigest, Vec)>, +} + +impl RetentionRecords { + /// Records of a store that never published retention. + pub const fn unretained() -> Self { + Self { + head: None, + manifest: None, + roots: Vec::new(), + } + } + + /// Frozen one-root generation-one records. + /// + /// # Errors + /// + /// Returns the exact fixture or decoder refusal. + pub fn golden() -> Result> { + let root = fixture(ROOT_HEX)?; + let manifest = fixture(MANIFEST_HEX)?; + let root_digest = AdmittedRetentionRoot::decode(&root)?.digest(); + let manifest_digest = AdmittedRetentionManifest::decode(&manifest)?.digest(); + Ok(Self { + head: Some(fixture(HEAD_HEX)?), + manifest: Some((manifest_digest, manifest)), + roots: vec![(root_digest, root)], + }) + } + + /// Builds generation-one records naming one root per namespace. + /// + /// # Errors + /// + /// Returns the exact namespace, root, manifest, or head refusal. + pub fn built(roots: &[(&[u8], RootShape)]) -> Result> { + let mut entries = Vec::new(); + let mut encoded_roots = Vec::new(); + for (namespace, shape) in roots { + let canonical = CanonicalRetentionRoot::from_root(&root(namespace, *shape)?)?; + let namespace_digest = RetentionNamespace::try_from(*namespace)?.digest(); + entries.push(RetentionManifestEntry::new( + namespace_digest, + RootGeneration::INITIAL, + canonical.digest(), + )); + encoded_roots.push((canonical.digest(), canonical.encoded().to_vec())); + } + let manifest = RetentionManifest::new(LivenessGeneration::INITIAL, None, entries)?; + let canonical = CanonicalRetentionManifest::from_manifest(&manifest)?; + let length = RetentionManifestLength::new(u64::try_from(canonical.encoded().len())?)?; + let head = RetentionHead::new( + LivenessGeneration::INITIAL, + length, + canonical.digest(), + None, + )?; + Ok(Self { + head: Some(encode_head(&head)), + manifest: Some((canonical.digest(), canonical.encoded().to_vec())), + roots: encoded_roots, + }) + } + + /// Returns the admitted current head. + /// + /// # Errors + /// + /// Returns a missing-head or decoder refusal. + pub fn current_head(&self) -> Result> { + let encoded = self.head.as_ref().ok_or("records publish no head")?; + Ok(*ChecksummedRetentionHead::decode(encoded)?.head()) + } + + /// Returns manifest entries in canonical order. + /// + /// # Errors + /// + /// Returns a missing-manifest or decoder refusal. + pub fn entries(&self) -> Result, Box> { + let (_digest, encoded) = self.manifest.as_ref().ok_or("records hold no manifest")?; + Ok(AdmittedRetentionManifest::decode(encoded)? + .manifest() + .entries() + .to_vec()) + } +} + +/// Encodes one semantic head with a correct checksum. +pub fn encode_head(head: &RetentionHead) -> Vec { + CanonicalRetentionHead::from_head(head).encoded().to_vec() +} + +/// Decodes one LF-terminated lowercase hexadecimal fixture. +/// +/// # Errors +/// +/// Returns a framing or hexadecimal transport refusal. +pub fn fixture(hex: &str) -> Result, Box> { + decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) +} + +fn root(namespace: &[u8], shape: RootShape) -> Result> { + let template_bytes = fixture(ROOT_HEX)?; + let template = AdmittedRetentionRoot::decode(&template_bytes)?; + let anchors = match shape { + RootShape::Retaining => template.root().anchors().to_vec(), + RootShape::Released => Vec::new(), + }; + Ok(RetentionRoot::new( + RetentionNamespace::try_from(namespace)?, + RootGeneration::INITIAL, + RetentionPolicy::new(template.root().profile(), template.root().limits()), + None, + anchors, + )?) +} diff --git a/tests/retained_catalog_publication/revalidation_laws.rs b/tests/retained_catalog_publication/revalidation_laws.rs new file mode 100644 index 00000000..6e1af304 --- /dev/null +++ b/tests/retained_catalog_publication/revalidation_laws.rs @@ -0,0 +1,120 @@ +//! Retention evidence that changes before `HEAD` replacement refuses the commit. + +use std::error::Error; + +use keep::{ + CatalogPublicationPhase, RetainedCatalogPublicationError as Refusal, RetentionEvidencePhase, + RetentionHeadDecodeError, +}; + +use crate::catalog_fixture::{Candidate, with_candidate}; +use crate::model_storage::{EvidenceRead, ModelStorage}; +use crate::retention_fixture::{RetentionRecords, RootShape}; +use crate::support::require_error; +use crate::{publish, stages_before_replacement}; + +/// Publishes the preserving candidate and requires a refusal before replacement. +fn refuse_at_commit(mut storage: ModelStorage) -> Result> { + let result = with_candidate(Candidate::Preserving, |call| publish(&mut storage, &call))?; + let error = require_error(result, "changed retention evidence was committed")?; + let replacement = stages_before_replacement()?; + assert_eq!(storage.stages().len(), replacement); + assert_eq!( + storage.stages().last(), + Some(&CatalogPublicationPhase::VerifyHeadView) + ); + assert!( + !storage + .stages() + .contains(&CatalogPublicationPhase::ReplaceHead) + ); + assert_eq!( + storage.reads().last(), + Some(&(EvidenceRead::Head, replacement)) + ); + Ok(error) +} + +#[test] +fn head_replaced_between_admission_and_commit_refuses() -> Result<(), Box> { + let records = RetentionRecords::golden()?; + let expected = records.current_head()?; + let successor = RetentionRecords::built(&[(b"later", RootShape::Released)])?; + let observed = successor.current_head()?; + let storage = ModelStorage::new(records).then_head(successor.head); + + let error = refuse_at_commit(storage)?; + + assert!(matches!( + error, + Refusal::RetentionChanged { expected: e, observed: ref o } + if e == Some(expected) && **o == Some(observed) + )); + Ok(()) +} + +#[test] +fn head_removed_between_admission_and_commit_refuses() -> Result<(), Box> { + let records = RetentionRecords::golden()?; + let expected = records.current_head()?; + let storage = ModelStorage::new(records).then_head(None); + + let error = refuse_at_commit(storage)?; + + assert!(matches!( + error, + Refusal::RetentionChanged { expected: e, ref observed } if e == Some(expected) && observed.is_none() + )); + Ok(()) +} + +#[test] +fn head_published_between_admission_and_commit_refuses() -> Result<(), Box> { + let appeared = RetentionRecords::golden()?; + let observed = appeared.current_head()?; + let storage = ModelStorage::new(RetentionRecords::unretained()).then_head(appeared.head); + + let error = refuse_at_commit(storage)?; + + assert!(matches!( + error, + Refusal::RetentionChanged { expected: None, observed: ref o } if **o == Some(observed) + )); + Ok(()) +} + +#[test] +fn corrupt_head_at_commit_refuses_with_the_revalidation_phase() -> Result<(), Box> { + let records = RetentionRecords::golden()?; + let mut corrupt = records.head.clone().ok_or("golden head is absent")?; + let last = corrupt.last_mut().ok_or("golden head is empty")?; + *last ^= 1; + let storage = ModelStorage::new(records).then_head(Some(corrupt)); + + let error = refuse_at_commit(storage)?; + + assert!(matches!( + error, + Refusal::HeadDecode { + phase: RetentionEvidencePhase::RevalidateHead, + source: RetentionHeadDecodeError::ChecksumMismatch { .. } + } + )); + Ok(()) +} + +#[test] +fn failed_head_read_at_commit_refuses_with_the_revalidation_phase() -> Result<(), Box> { + let storage = ModelStorage::new(RetentionRecords::golden()?).failing_head_read(1); + + let error = refuse_at_commit(storage)?; + + assert!(matches!( + error, + Refusal::RetentionStorage { + phase: RetentionEvidencePhase::RevalidateHead, + .. + } + )); + Ok(()) +} From 95227de8fd1553f8a56404084c1a2c6a833efdca Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 07:14:02 -0700 Subject: [PATCH 03/36] Refactor: share filesystem catalog stages across writer authorities (#198) Move the pinned staging, segment-pool and catalog-pool directories, the retained writable stages and the stage-authority token into one FilesystemCatalogStages value. It is the single CatalogPublicationStorage implementation; FilesystemCatalogPublisher owns it beside its writer lock and forwards every transition unchanged. Segment stage creation and selection move with it. No behavior change: this lets a version-two writer authority run the same stage code. --- src/adapters/filesystem_catalog_catalog.rs | 63 +++++----- src/adapters/filesystem_catalog_current.rs | 46 +++---- src/adapters/filesystem_catalog_head.rs | 55 ++++----- src/adapters/filesystem_catalog_publisher.rs | 50 ++------ .../filesystem_catalog_publisher_storage.rs | 113 ++++++++++++++++++ .../filesystem_catalog_publisher_tests.rs | 24 ++-- src/adapters/filesystem_catalog_segment.rs | 32 ++--- src/adapters/filesystem_catalog_stages.rs | 86 +++++++++++++ src/adapters/filesystem_catalog_storage.rs | 13 +- src/adapters/filesystem_segment_stage.rs | 9 +- src/adapters/mod.rs | 2 + 11 files changed, 341 insertions(+), 152 deletions(-) create mode 100644 src/adapters/filesystem_catalog_publisher_storage.rs create mode 100644 src/adapters/filesystem_catalog_stages.rs diff --git a/src/adapters/filesystem_catalog_catalog.rs b/src/adapters/filesystem_catalog_catalog.rs index 4539e908..3c419fce 100644 --- a/src/adapters/filesystem_catalog_catalog.rs +++ b/src/adapters/filesystem_catalog_catalog.rs @@ -2,100 +2,101 @@ use std::io::{self, Write}; +use super::filesystem_catalog_stages::FilesystemCatalogStages; use super::{ CanonicalCatalog, CatalogRestartArtifact, ChecksummedCatalog, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, filesystem_catalog_artifact, - filesystem_catalog_publisher, physical_pool_name, + FilesystemCatalogPublicationError, filesystem_catalog_artifact, filesystem_catalog_publisher, + physical_pool_name, }; -pub(super) fn create_stage(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { - if publisher.catalog_stage.is_some() { +pub(super) fn create_stage(stages: &mut FilesystemCatalogStages) -> io::Result<()> { + if stages.catalog_stage.is_some() { return Err(stage_state()); } - publisher.catalog_stage = Some(filesystem_catalog_artifact::create_exclusive( - &publisher.staging, + stages.catalog_stage = Some(filesystem_catalog_artifact::create_exclusive( + &stages.staging, filesystem_catalog_publisher::CURRENT_CATALOG, )?); Ok(()) } pub(super) fn write( - publisher: &mut FilesystemCatalogPublisher, + stages: &mut FilesystemCatalogStages, catalog: &CanonicalCatalog, ) -> io::Result<()> { - write_bytes(publisher, catalog.encoded()) + write_bytes(stages, catalog.encoded()) } #[cfg(feature = "repository-tasks")] pub(super) fn write_prefix( - publisher: &mut FilesystemCatalogPublisher, + stages: &mut FilesystemCatalogStages, catalog: &CanonicalCatalog, prefix: usize, ) -> io::Result<()> { let bytes = catalog.encoded().get(..prefix).ok_or_else(|| { io::Error::new(io::ErrorKind::InvalidInput, "catalog prefix exceeds bytes") })?; - write_bytes(publisher, bytes) + write_bytes(stages, bytes) } -fn write_bytes(publisher: &mut FilesystemCatalogPublisher, bytes: &[u8]) -> io::Result<()> { - stage_mut(publisher)?.write_all(bytes) +fn write_bytes(stages: &mut FilesystemCatalogStages, bytes: &[u8]) -> io::Result<()> { + stage_mut(stages)?.write_all(bytes) } -pub(super) fn flush(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { - stage_mut(publisher)?.flush() +pub(super) fn flush(stages: &mut FilesystemCatalogStages) -> io::Result<()> { + stage_mut(stages)?.flush() } -pub(super) fn synchronize(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { - stage_mut(publisher)?.sync_all() +pub(super) fn synchronize(stages: &mut FilesystemCatalogStages) -> io::Result<()> { + stage_mut(stages)?.sync_all() } pub(super) fn link( - publisher: &mut FilesystemCatalogPublisher, + stages: &mut FilesystemCatalogStages, catalog: ChecksummedCatalog<'_>, ) -> io::Result<()> { - let stage = publisher.catalog_stage.take().ok_or_else(stage_state)?; + let stage = stages.catalog_stage.take().ok_or_else(stage_state)?; drop(stage); filesystem_catalog_artifact::verify_catalog( - &publisher.staging, + &stages.staging, filesystem_catalog_publisher::CURRENT_CATALOG, catalog, )?; super::filesystem_exact_record::link_without_replacement( - &publisher.staging, + &stages.staging, filesystem_catalog_publisher::CURRENT_CATALOG, - &publisher.catalogs, + &stages.catalogs, &physical_pool_name::catalog(catalog.generation(), catalog.digest()), ) } pub(super) fn verify_pool( - publisher: &FilesystemCatalogPublisher, + stages: &FilesystemCatalogStages, catalog: ChecksummedCatalog<'_>, ) -> io::Result<()> { filesystem_catalog_artifact::verify_catalog( - &publisher.catalogs, + &stages.catalogs, &physical_pool_name::catalog(catalog.generation(), catalog.digest()), catalog, ) } -pub(super) fn synchronize_pool(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { - filesystem_catalog_artifact::synchronize_directory(&publisher.catalogs) +pub(super) fn synchronize_pool(stages: &FilesystemCatalogStages) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(&stages.catalogs) } -pub(super) fn remove_stage(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { - publisher +pub(super) fn remove_stage(stages: &FilesystemCatalogStages) -> io::Result<()> { + stages .staging .remove_file(filesystem_catalog_publisher::CURRENT_CATALOG) } -pub(super) fn synchronize_staging(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { - filesystem_catalog_artifact::synchronize_directory(&publisher.staging) +pub(super) fn synchronize_staging(stages: &FilesystemCatalogStages) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(&stages.staging) } -fn stage_mut(publisher: &mut FilesystemCatalogPublisher) -> io::Result<&mut cap_std::fs::File> { - publisher.catalog_stage.as_mut().ok_or_else(stage_state) +fn stage_mut(stages: &mut FilesystemCatalogStages) -> io::Result<&mut cap_std::fs::File> { + stages.catalog_stage.as_mut().ok_or_else(stage_state) } fn stage_state() -> io::Error { diff --git a/src/adapters/filesystem_catalog_current.rs b/src/adapters/filesystem_catalog_current.rs index cf56a399..b36501db 100644 --- a/src/adapters/filesystem_catalog_current.rs +++ b/src/adapters/filesystem_catalog_current.rs @@ -2,29 +2,29 @@ use std::io; +use super::filesystem_catalog_stages::FilesystemCatalogStages; use super::{ CatalogPublicationExpectation, CatalogPublicationReadiness, CatalogRestartError, - CatalogRestartPhase, CatalogSnapshot, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, SegmentPublication, catalog_restart_loader, - filesystem_catalog_artifact, filesystem_catalog_publisher, + CatalogRestartPhase, CatalogSnapshot, FilesystemCatalogPublicationError, SegmentPublication, + catalog_restart_loader, filesystem_catalog_artifact, filesystem_catalog_publisher, }; pub(super) fn verify( - publisher: &FilesystemCatalogPublisher, + stages: &FilesystemCatalogStages, expected: CatalogPublicationExpectation, candidate: &CatalogSnapshot<'_, '_, '_>, segment: &SegmentPublication<'_, '_>, ) -> io::Result { - require_segment_authority(publisher, segment)?; - require_no_next_head(publisher)?; - require_no_catalog_stage(publisher)?; + require_segment_authority(stages, segment)?; + require_no_next_head(stages)?; + require_no_catalog_stage(stages)?; if segment.admitted().is_none() { - require_no_segment_stage(publisher)?; + require_no_segment_stage(stages)?; } let readiness = match catalog_restart_loader::load_from_directory( - &publisher.root, + &stages.root, filesystem_catalog_publisher::HEAD, - publisher.policy, + stages.policy, ) { Ok(observed) => { let observed_generation = Some(observed.generation()); @@ -49,22 +49,22 @@ pub(super) fn verify( } } Err(source) if head_is_absent(&source) && expected.current_generation().is_none() => { - require_empty_durable_pools(publisher)?; + require_empty_durable_pools(stages)?; Ok(CatalogPublicationReadiness::Ready) } Err(source) => Err(filesystem_catalog_artifact::invalid_data(source)), }?; if readiness == CatalogPublicationReadiness::AlreadyPublished { - require_no_segment_stage(publisher)?; + require_no_segment_stage(stages)?; } Ok(readiness) } fn require_segment_authority( - publisher: &FilesystemCatalogPublisher, + stages: &FilesystemCatalogStages, segment: &SegmentPublication<'_, '_>, ) -> io::Result<()> { - if segment.is_bound_to(&publisher.authority) { + if segment.is_bound_to(&stages.authority) { Ok(()) } else { Err(filesystem_catalog_artifact::invalid_data( @@ -73,8 +73,8 @@ fn require_segment_authority( } } -fn require_no_next_head(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { - match publisher +fn require_no_next_head(stages: &FilesystemCatalogStages) -> io::Result<()> { + match stages .root .symlink_metadata(filesystem_catalog_publisher::NEXT_HEAD) { @@ -86,29 +86,29 @@ fn require_no_next_head(publisher: &FilesystemCatalogPublisher) -> io::Result<() } } -fn require_no_catalog_stage(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { +fn require_no_catalog_stage(stages: &FilesystemCatalogStages) -> io::Result<()> { require_absent( - &publisher.staging, + &stages.staging, filesystem_catalog_publisher::CURRENT_CATALOG, FilesystemCatalogPublicationError::CatalogRecoveryRequired, ) } -fn require_no_segment_stage(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { +fn require_no_segment_stage(stages: &FilesystemCatalogStages) -> io::Result<()> { require_absent( - &publisher.staging, + &stages.staging, filesystem_catalog_publisher::CURRENT_SEGMENT, FilesystemCatalogPublicationError::SegmentRecoveryRequired, ) } -fn require_empty_durable_pools(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { +fn require_empty_durable_pools(stages: &FilesystemCatalogStages) -> io::Result<()> { require_empty_pool( - &publisher.segments, + &stages.segments, FilesystemCatalogPublicationError::SegmentPoolRecoveryRequired, )?; require_empty_pool( - &publisher.catalogs, + &stages.catalogs, FilesystemCatalogPublicationError::CatalogPoolRecoveryRequired, ) } diff --git a/src/adapters/filesystem_catalog_head.rs b/src/adapters/filesystem_catalog_head.rs index 27df44f0..5d3f3831 100644 --- a/src/adapters/filesystem_catalog_head.rs +++ b/src/adapters/filesystem_catalog_head.rs @@ -2,33 +2,34 @@ use std::io::{self, Write}; +use super::filesystem_catalog_stages::FilesystemCatalogStages; use super::{ CanonicalPublicationHead, CatalogRestartArtifact, CatalogSnapshot, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, catalog_restart_loader, - filesystem_catalog_artifact, filesystem_catalog_publisher, + FilesystemCatalogPublicationError, catalog_restart_loader, filesystem_catalog_artifact, + filesystem_catalog_publisher, }; -pub(super) fn create_stage(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { - if publisher.head_stage.is_some() { +pub(super) fn create_stage(stages: &mut FilesystemCatalogStages) -> io::Result<()> { + if stages.head_stage.is_some() { return Err(stage_state()); } - publisher.head_stage = Some(filesystem_catalog_artifact::create_exclusive( - &publisher.root, + stages.head_stage = Some(filesystem_catalog_artifact::create_exclusive( + &stages.root, filesystem_catalog_publisher::NEXT_HEAD, )?); Ok(()) } pub(super) fn write( - publisher: &mut FilesystemCatalogPublisher, + stages: &mut FilesystemCatalogStages, head: &CanonicalPublicationHead, ) -> io::Result<()> { - write_bytes(publisher, head.encoded()) + write_bytes(stages, head.encoded()) } #[cfg(feature = "repository-tasks")] pub(super) fn write_prefix( - publisher: &mut FilesystemCatalogPublisher, + stages: &mut FilesystemCatalogStages, head: &CanonicalPublicationHead, prefix: usize, ) -> io::Result<()> { @@ -36,32 +37,32 @@ pub(super) fn write_prefix( .encoded() .get(..prefix) .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "head prefix exceeds bytes"))?; - write_bytes(publisher, bytes) + write_bytes(stages, bytes) } -fn write_bytes(publisher: &mut FilesystemCatalogPublisher, bytes: &[u8]) -> io::Result<()> { - stage_mut(publisher)?.write_all(bytes) +fn write_bytes(stages: &mut FilesystemCatalogStages, bytes: &[u8]) -> io::Result<()> { + stage_mut(stages)?.write_all(bytes) } -pub(super) fn flush(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { - stage_mut(publisher)?.flush() +pub(super) fn flush(stages: &mut FilesystemCatalogStages) -> io::Result<()> { + stage_mut(stages)?.flush() } -pub(super) fn synchronize(publisher: &mut FilesystemCatalogPublisher) -> io::Result<()> { - stage_mut(publisher)?.sync_all() +pub(super) fn synchronize(stages: &mut FilesystemCatalogStages) -> io::Result<()> { + stage_mut(stages)?.sync_all() } pub(super) fn verify_view( - publisher: &mut FilesystemCatalogPublisher, + stages: &mut FilesystemCatalogStages, head: &CanonicalPublicationHead, expected: &CatalogSnapshot<'_, '_, '_>, ) -> io::Result<()> { - let stage = publisher.head_stage.take().ok_or_else(stage_state)?; + let stage = stages.head_stage.take().ok_or_else(stage_state)?; drop(stage); let observed = catalog_restart_loader::load_from_directory( - &publisher.root, + &stages.root, filesystem_catalog_publisher::NEXT_HEAD, - publisher.policy, + stages.policy, ) .map_err(filesystem_catalog_artifact::invalid_data)?; if observed.head_bytes() != head.encoded() { @@ -87,20 +88,20 @@ pub(super) fn verify_view( } } -pub(super) fn replace(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { - publisher.root.rename( +pub(super) fn replace(stages: &FilesystemCatalogStages) -> io::Result<()> { + stages.root.rename( filesystem_catalog_publisher::NEXT_HEAD, - &publisher.root, + &stages.root, filesystem_catalog_publisher::HEAD, ) } -pub(super) fn synchronize_root(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { - filesystem_catalog_artifact::synchronize_directory(&publisher.root) +pub(super) fn synchronize_root(stages: &FilesystemCatalogStages) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(&stages.root) } -fn stage_mut(publisher: &mut FilesystemCatalogPublisher) -> io::Result<&mut cap_std::fs::File> { - publisher.head_stage.as_mut().ok_or_else(stage_state) +fn stage_mut(stages: &mut FilesystemCatalogStages) -> io::Result<&mut cap_std::fs::File> { + stages.head_stage.as_mut().ok_or_else(stage_state) } fn stage_state() -> io::Error { diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index 6034f662..ad554871 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -2,13 +2,11 @@ use std::io; -use cap_std::fs::{Dir, File}; - -use super::filesystem_publisher_authority::FilesystemPublisherAuthority; +use super::filesystem_catalog_stages::FilesystemCatalogStages; use super::{ - AdmittedSegment, CatalogRestartPolicy, ClosedSegment, FilesystemPlatformAdmission, - FilesystemSegmentStage, FilesystemWriterLock, SealedSegment, SegmentPublication, - SegmentPublicationError, SegmentStageCreateError, sync_capable_directory, + AdmittedSegment, CatalogRestartPolicy, FilesystemPlatformAdmission, FilesystemSegmentStage, + FilesystemWriterLock, SealedSegment, SegmentPublication, SegmentPublicationError, + SegmentStageCreateError, }; #[cfg(feature = "repository-tasks")] use super::{CanonicalCatalog, CanonicalPublicationHead, filesystem_catalog_catalog}; @@ -29,14 +27,7 @@ pub(super) const NEXT_HEAD: &str = "head.next"; /// initializing a new store or reopening a completely published store. #[must_use] pub struct FilesystemCatalogPublisher { - pub(super) root: Dir, - pub(super) staging: Dir, - pub(super) segments: Dir, - pub(super) catalogs: Dir, - pub(super) policy: CatalogRestartPolicy, - pub(super) authority: FilesystemPublisherAuthority, - pub(super) catalog_stage: Option, - pub(super) head_stage: Option, + pub(super) stages: FilesystemCatalogStages, // Fields drop in declaration order. Writer authority must outlive every // directory capability and retained writable stage. pub(super) _lock: FilesystemWriterLock, @@ -58,19 +49,9 @@ impl FilesystemCatalogPublisher { ) -> io::Result { let lock = admission.into_lock(); let pinned_root = lock.clone_directory()?; - let root = sync_capable_directory::open(&pinned_root, ".")?; - let staging = sync_capable_directory::open(&root, "staging")?; - let segments = sync_capable_directory::open(&root, "segments")?; - let catalogs = sync_capable_directory::open(&root, "catalogs")?; + let stages = FilesystemCatalogStages::open(&pinned_root, policy)?; Ok(Self { - root, - staging, - segments, - catalogs, - policy, - authority: FilesystemPublisherAuthority::new(), - catalog_stage: None, - head_stage: None, + stages, _lock: lock, }) } @@ -125,7 +106,7 @@ impl FilesystemCatalogPublisher { pub fn create_segment_stage( &self, ) -> Result, SegmentStageCreateError> { - FilesystemSegmentStage::create(self) + self.stages.create_segment_stage() } /// Closes and selects one synchronized stage created by this publisher. @@ -144,16 +125,7 @@ impl FilesystemCatalogPublisher { sealed: SealedSegment>, admitted: &'selection AdmittedSegment<'records>, ) -> Result, SegmentPublicationError> { - let (stage, record_count, segment_length, digest) = sealed.into_parts(); - let authority = stage.close(); - if !self.authority.matches(&authority) { - return Err(SegmentPublicationError::PublisherAuthority); - } - SegmentPublication::one_bound( - ClosedSegment::admitted(record_count, segment_length, digest), - admitted, - authority, - ) + self.stages.select_segment(sealed, admitted) } /// Writes a strict prefix through the production catalog-stage adapter. @@ -168,7 +140,7 @@ impl FilesystemCatalogPublisher { catalog: &CanonicalCatalog, prefix: usize, ) -> io::Result<()> { - filesystem_catalog_catalog::write_prefix(self, catalog, prefix) + filesystem_catalog_catalog::write_prefix(&mut self.stages, catalog, prefix) } /// Writes a strict prefix through the production head-stage adapter. @@ -183,6 +155,6 @@ impl FilesystemCatalogPublisher { head: &CanonicalPublicationHead, prefix: usize, ) -> io::Result<()> { - super::filesystem_catalog_head::write_prefix(self, head, prefix) + super::filesystem_catalog_head::write_prefix(&mut self.stages, head, prefix) } } diff --git a/src/adapters/filesystem_catalog_publisher_storage.rs b/src/adapters/filesystem_catalog_publisher_storage.rs new file mode 100644 index 00000000..6d09a5a7 --- /dev/null +++ b/src/adapters/filesystem_catalog_publisher_storage.rs @@ -0,0 +1,113 @@ +//! This module binds version-one publication authority to the shared stages. +//! +//! Every transition forwards unchanged to +//! [`FilesystemCatalogStages`](super::filesystem_catalog_stages::FilesystemCatalogStages), +//! so the version-one port and the version-two retained-closure port execute +//! one implementation. + +use std::io; + +use super::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, + CatalogPublicationReadiness, CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, + FilesystemCatalogPublisher, SegmentPublication, +}; + +impl CatalogPublicationStorage for FilesystemCatalogPublisher { + fn verify_current( + &mut self, + expected: CatalogPublicationExpectation, + candidate: &CatalogSnapshot<'_, '_, '_>, + segment: &SegmentPublication<'_, '_>, + ) -> io::Result { + self.stages.verify_current(expected, candidate, segment) + } + + fn link_segment(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { + self.stages.link_segment(segment) + } + + fn verify_segment_pool(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { + self.stages.verify_segment_pool(segment) + } + + fn synchronize_segments(&mut self) -> io::Result<()> { + self.stages.synchronize_segments() + } + + fn remove_segment_stage(&mut self) -> io::Result<()> { + self.stages.remove_segment_stage() + } + + fn synchronize_staging_after_segment(&mut self) -> io::Result<()> { + self.stages.synchronize_staging_after_segment() + } + + fn create_catalog_stage(&mut self) -> io::Result<()> { + self.stages.create_catalog_stage() + } + + fn write_catalog(&mut self, catalog: &CanonicalCatalog) -> io::Result<()> { + self.stages.write_catalog(catalog) + } + + fn flush_catalog(&mut self) -> io::Result<()> { + self.stages.flush_catalog() + } + + fn synchronize_catalog(&mut self) -> io::Result<()> { + self.stages.synchronize_catalog() + } + + fn link_catalog(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { + self.stages.link_catalog(catalog) + } + + fn verify_catalog_pool(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { + self.stages.verify_catalog_pool(catalog) + } + + fn synchronize_catalogs(&mut self) -> io::Result<()> { + self.stages.synchronize_catalogs() + } + + fn remove_catalog_stage(&mut self) -> io::Result<()> { + self.stages.remove_catalog_stage() + } + + fn synchronize_staging_after_catalog(&mut self) -> io::Result<()> { + self.stages.synchronize_staging_after_catalog() + } + + fn create_head_stage(&mut self) -> io::Result<()> { + self.stages.create_head_stage() + } + + fn write_head(&mut self, head: &CanonicalPublicationHead) -> io::Result<()> { + self.stages.write_head(head) + } + + fn flush_head(&mut self) -> io::Result<()> { + self.stages.flush_head() + } + + fn synchronize_head(&mut self) -> io::Result<()> { + self.stages.synchronize_head() + } + + fn verify_head_view( + &mut self, + head: &CanonicalPublicationHead, + snapshot: &CatalogSnapshot<'_, '_, '_>, + ) -> io::Result<()> { + self.stages.verify_head_view(head, snapshot) + } + + fn replace_head(&mut self) -> io::Result<()> { + self.stages.replace_head() + } + + fn synchronize_root(&mut self) -> io::Result<()> { + self.stages.synchronize_root() + } +} diff --git a/src/adapters/filesystem_catalog_publisher_tests.rs b/src/adapters/filesystem_catalog_publisher_tests.rs index 8ff44ae7..87d70297 100644 --- a/src/adapters/filesystem_catalog_publisher_tests.rs +++ b/src/adapters/filesystem_catalog_publisher_tests.rs @@ -182,17 +182,25 @@ fn fixture(hex: &str) -> Result, Box> { fn publisher_drop_closes_writable_stages_before_releasing_writer_authority() -> Result<(), Box> { let source = include_str!("filesystem_catalog_publisher.rs"); - let catalog_stage = source - .find("pub(super) catalog_stage:") - .ok_or("publisher must retain the catalog stage")?; - let head_stage = source - .find("pub(super) head_stage:") - .ok_or("publisher must retain the head stage")?; + let stages = source + .find("pub(super) stages: FilesystemCatalogStages,") + .ok_or("publisher must retain the shared catalog stages")?; let writer_lock = source .find("pub(super) _lock:") .ok_or("publisher must retain writer authority")?; + let shared = include_str!("filesystem_catalog_stages.rs"); + let owner = shared + .find("pub(super) struct FilesystemCatalogStages") + .ok_or("shared stages must be one owned type")?; + let catalog_stage = shared + .find("pub(super) catalog_stage:") + .ok_or("shared stages must retain the catalog stage")?; + let head_stage = shared + .find("pub(super) head_stage:") + .ok_or("shared stages must retain the head stage")?; - assert!(catalog_stage < writer_lock); - assert!(head_stage < writer_lock); + assert!(stages < writer_lock); + assert!(owner < catalog_stage && owner < head_stage); + assert!(!shared.contains("FilesystemWriterLock")); Ok(()) } diff --git a/src/adapters/filesystem_catalog_segment.rs b/src/adapters/filesystem_catalog_segment.rs index 5af91ecc..3a498d88 100644 --- a/src/adapters/filesystem_catalog_segment.rs +++ b/src/adapters/filesystem_catalog_segment.rs @@ -2,51 +2,51 @@ use std::io; +use super::filesystem_catalog_stages::FilesystemCatalogStages; use super::{ - AdmittedSegment, FilesystemCatalogPublisher, filesystem_catalog_artifact, - filesystem_catalog_publisher, physical_pool_name, + AdmittedSegment, filesystem_catalog_artifact, filesystem_catalog_publisher, physical_pool_name, }; pub(super) fn link( - publisher: &FilesystemCatalogPublisher, + stages: &FilesystemCatalogStages, segment: &AdmittedSegment<'_>, ) -> io::Result<()> { filesystem_catalog_artifact::verify_segment( - &publisher.staging, + &stages.staging, filesystem_catalog_publisher::CURRENT_SEGMENT, segment, - publisher.policy.segment_read(), + stages.policy.segment_read(), )?; super::filesystem_exact_record::link_without_replacement( - &publisher.staging, + &stages.staging, filesystem_catalog_publisher::CURRENT_SEGMENT, - &publisher.segments, + &stages.segments, &physical_pool_name::segment(segment.digest()), ) } pub(super) fn verify_pool( - publisher: &FilesystemCatalogPublisher, + stages: &FilesystemCatalogStages, segment: &AdmittedSegment<'_>, ) -> io::Result<()> { filesystem_catalog_artifact::verify_segment( - &publisher.segments, + &stages.segments, &physical_pool_name::segment(segment.digest()), segment, - publisher.policy.segment_read(), + stages.policy.segment_read(), ) } -pub(super) fn synchronize_pool(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { - filesystem_catalog_artifact::synchronize_directory(&publisher.segments) +pub(super) fn synchronize_pool(stages: &FilesystemCatalogStages) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(&stages.segments) } -pub(super) fn remove_stage(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { - publisher +pub(super) fn remove_stage(stages: &FilesystemCatalogStages) -> io::Result<()> { + stages .staging .remove_file(filesystem_catalog_publisher::CURRENT_SEGMENT) } -pub(super) fn synchronize_staging(publisher: &FilesystemCatalogPublisher) -> io::Result<()> { - filesystem_catalog_artifact::synchronize_directory(&publisher.staging) +pub(super) fn synchronize_staging(stages: &FilesystemCatalogStages) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(&stages.staging) } diff --git a/src/adapters/filesystem_catalog_stages.rs b/src/adapters/filesystem_catalog_stages.rs new file mode 100644 index 00000000..1f6d2c70 --- /dev/null +++ b/src/adapters/filesystem_catalog_stages.rs @@ -0,0 +1,86 @@ +//! This module owns the catalog protocol stages shared by every writer authority. +//! +//! One implementation of the segment, catalog, and head transitions runs under +//! version-one publication authority and under version-two retained-closure +//! authority. Each authority owns its writer lock and decides which protocol +//! entry point may drive these stages; the stages themselves never acquire, +//! release, or widen authority. + +use std::io; + +use cap_std::fs::{Dir, File}; + +use super::filesystem_publisher_authority::FilesystemPublisherAuthority; +use super::{ + AdmittedSegment, CatalogRestartPolicy, ClosedSegment, FilesystemSegmentStage, SealedSegment, + SegmentPublication, SegmentPublicationError, SegmentStageCreateError, sync_capable_directory, +}; + +/// Pinned catalog-protocol directories and the stages one writer retains. +/// +/// The value pins the store root, `staging`, `segments`, and `catalogs` +/// capabilities and holds at most one writable catalog stage and one writable +/// `head.next`. Dropping it closes those handles but never publishes, removes, +/// truncates, or repairs protocol state. It does not hold writer authority; +/// the owning publisher does, and must drop this value first. +pub(super) struct FilesystemCatalogStages { + pub(super) root: Dir, + pub(super) staging: Dir, + pub(super) segments: Dir, + pub(super) catalogs: Dir, + pub(super) policy: CatalogRestartPolicy, + pub(super) authority: FilesystemPublisherAuthority, + pub(super) catalog_stage: Option, + pub(super) head_stage: Option, +} + +impl FilesystemCatalogStages { + /// Pins the protocol directories under `pinned_root` without following links. + /// + /// # Errors + /// + /// Returns the exact no-follow directory-open or inspection failure. A + /// protocol entry that is not a directory returns + /// [`io::ErrorKind::NotADirectory`]. + pub(super) fn open(pinned_root: &Dir, policy: CatalogRestartPolicy) -> io::Result { + let root = sync_capable_directory::open(pinned_root, ".")?; + let staging = sync_capable_directory::open(&root, "staging")?; + let segments = sync_capable_directory::open(&root, "segments")?; + let catalogs = sync_capable_directory::open(&root, "catalogs")?; + Ok(Self { + root, + staging, + segments, + catalogs, + policy, + authority: FilesystemPublisherAuthority::new(), + catalog_stage: None, + head_stage: None, + }) + } + + /// Exclusively creates `staging/current.seg` bound to these stages. + pub(super) fn create_segment_stage( + &self, + ) -> Result, SegmentStageCreateError> { + FilesystemSegmentStage::create(self) + } + + /// Closes one sealed stage and binds it to `admitted` under this authority. + pub(super) fn select_segment<'selection, 'records>( + &self, + sealed: SealedSegment>, + admitted: &'selection AdmittedSegment<'records>, + ) -> Result, SegmentPublicationError> { + let (stage, record_count, segment_length, digest) = sealed.into_parts(); + let authority = stage.close(); + if !self.authority.matches(&authority) { + return Err(SegmentPublicationError::PublisherAuthority); + } + SegmentPublication::one_bound( + ClosedSegment::admitted(record_count, segment_length, digest), + admitted, + authority, + ) + } +} diff --git a/src/adapters/filesystem_catalog_storage.rs b/src/adapters/filesystem_catalog_storage.rs index c6a778b5..4454f656 100644 --- a/src/adapters/filesystem_catalog_storage.rs +++ b/src/adapters/filesystem_catalog_storage.rs @@ -1,15 +1,20 @@ -//! This module binds filesystem catalog transitions to the publication port. +//! This module binds the shared filesystem catalog stages to the publication port. +//! +//! This is the one stage implementation. Each writer authority forwards its own +//! port to it, so version-one and version-two publication execute identical +//! transitions. use std::io; +use super::filesystem_catalog_stages::FilesystemCatalogStages; use super::{ AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, CatalogPublicationReadiness, CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, - FilesystemCatalogPublisher, SegmentPublication, filesystem_catalog_catalog, - filesystem_catalog_current, filesystem_catalog_head, filesystem_catalog_segment, + SegmentPublication, filesystem_catalog_catalog, filesystem_catalog_current, + filesystem_catalog_head, filesystem_catalog_segment, }; -impl CatalogPublicationStorage for FilesystemCatalogPublisher { +impl CatalogPublicationStorage for FilesystemCatalogStages { fn verify_current( &mut self, expected: CatalogPublicationExpectation, diff --git a/src/adapters/filesystem_segment_stage.rs b/src/adapters/filesystem_segment_stage.rs index 1d540576..afeea1f0 100644 --- a/src/adapters/filesystem_segment_stage.rs +++ b/src/adapters/filesystem_segment_stage.rs @@ -6,8 +6,9 @@ use cap_std::fs::File; use super::filesystem_catalog_artifact; use super::filesystem_catalog_publisher::CURRENT_SEGMENT; +use super::filesystem_catalog_stages::FilesystemCatalogStages; use super::filesystem_publisher_authority::FilesystemPublisherAuthority; -use super::{FilesystemCatalogPublisher, SegmentStage, SegmentStageCreateError}; +use super::{SegmentStage, SegmentStageCreateError}; /// An exclusively created empty `current.seg` staging file under writer authority. /// @@ -16,7 +17,7 @@ use super::{FilesystemCatalogPublisher, SegmentStage, SegmentStageCreateError}; /// the value closes the file but deliberately leaves its bytes and name for /// explicit recovery. /// -/// The lifetime keeps the locked publisher borrowed until the writable file is +/// The lifetime keeps the locked publisher's stages borrowed until the writable file is /// closed. This type does not synchronize the staging-directory entry, publish /// the file, or establish that the surrounding filesystem satisfies Keep's /// complete platform contract. It retains this publisher's private authority @@ -24,12 +25,12 @@ use super::{FilesystemCatalogPublisher, SegmentStage, SegmentStageCreateError}; pub struct FilesystemSegmentStage<'publisher> { file: File, authority: FilesystemPublisherAuthority, - _publisher: &'publisher FilesystemCatalogPublisher, + _publisher: &'publisher FilesystemCatalogStages, } impl<'publisher> FilesystemSegmentStage<'publisher> { pub(super) fn create( - publisher: &'publisher FilesystemCatalogPublisher, + publisher: &'publisher FilesystemCatalogStages, ) -> Result { let file = filesystem_catalog_artifact::create_exclusive(&publisher.staging, CURRENT_SEGMENT) diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index b82e01da..96291b09 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -83,10 +83,12 @@ mod filesystem_catalog_current; mod filesystem_catalog_head; mod filesystem_catalog_publication_error; mod filesystem_catalog_publisher; +mod filesystem_catalog_publisher_storage; #[cfg(test)] mod filesystem_catalog_publisher_tests; mod filesystem_catalog_segment; mod filesystem_catalog_snapshot; +mod filesystem_catalog_stages; mod filesystem_catalog_storage; mod filesystem_exact_record; mod filesystem_initialization_namespace; From 111e49403dc242668510a64df33d32f02d722416 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 07:14:02 -0700 Subject: [PATCH 04/36] Refactor: read retained-closure evidence through one internal capability (#198) The gate's evidence admission now reads through a crate-internal RetainedEvidenceSource. Forward publication adapts its RetainedCatalogPublicationStorage to it without exposing catalog stages, so recovery finalization can run the same admission later. No behavior change. --- src/adapters/retained_catalog_publication.rs | 8 +++- .../retained_evidence_admission.rs | 13 +++--- .../retained_evidence_source.rs | 46 +++++++++++++++++++ 3 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 src/adapters/retained_catalog_publication/retained_evidence_source.rs diff --git a/src/adapters/retained_catalog_publication.rs b/src/adapters/retained_catalog_publication.rs index 0a81cff8..d8ce5a68 100644 --- a/src/adapters/retained_catalog_publication.rs +++ b/src/adapters/retained_catalog_publication.rs @@ -15,6 +15,7 @@ mod retained_catalog_publication_receipt; mod retained_catalog_publication_storage; mod retained_closure_admission; mod retained_evidence_admission; +mod retained_evidence_source; mod retention_evidence_phase; pub use preserved_retention_manifest::PreservedRetentionManifest; @@ -24,6 +25,8 @@ pub use retained_catalog_publication_storage::RetainedCatalogPublicationStorage; pub use retained_closure_admission::RetainedClosureAdmission; pub use retention_evidence_phase::RetentionEvidencePhase; +use retained_evidence_source::PublicationEvidence; + use super::catalog_publication_execution as execution; use super::{ AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, @@ -72,7 +75,8 @@ pub fn publish_retained_catalog_generation catalog_publication::admit_candidate(expectation, &segment, checksummed, segments)?; let head = CanonicalPublicationHead::for_catalog(checksummed); let snapshot = catalog_publication::admit_snapshot(&head, admitted)?; - let admission = retained_evidence_admission::admit(storage, &snapshot)?; + let admission = + retained_evidence_admission::admit(&mut PublicationEvidence(storage), &snapshot)?; let preserved = admission.preserved_manifest(); let current_stages = storage.catalog_stages(&admission); let readiness = execution::execute_current(current_stages, expectation, &snapshot, &segment)?; @@ -92,7 +96,7 @@ pub fn publish_retained_catalog_generation snapshot: &snapshot, }; stage_candidate(storage.catalog_stages(&admission), segment, &candidate)?; - retained_evidence_admission::revalidate(storage, &admission)?; + retained_evidence_admission::revalidate(&mut PublicationEvidence(storage), &admission)?; execution::execute_head_replacement(storage.catalog_stages(&admission))?; let catalog_receipt = CatalogPublicationReceipt::published(snapshot.generation(), snapshot.catalog_digest()); diff --git a/src/adapters/retained_catalog_publication/retained_evidence_admission.rs b/src/adapters/retained_catalog_publication/retained_evidence_admission.rs index 17ebc990..76c355b4 100644 --- a/src/adapters/retained_catalog_publication/retained_evidence_admission.rs +++ b/src/adapters/retained_catalog_publication/retained_evidence_admission.rs @@ -4,9 +4,10 @@ //! evidence admission never depends on the storage being checked. Closure //! work reuses the storage-independent [`verify_retention_closure`]. +use super::retained_evidence_source::RetainedEvidenceSource; use super::{ - RetainedCatalogPublicationError as Error, RetainedCatalogPublicationStorage, - RetainedClosureAdmission, RetentionEvidencePhase as Phase, + RetainedCatalogPublicationError as Error, RetainedClosureAdmission, + RetentionEvidencePhase as Phase, }; use crate::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CatalogSnapshot, ChecksummedRetentionHead, @@ -18,7 +19,7 @@ use crate::{ /// Roots are read, admitted, and verified one at a time in manifest order, so /// at most one root's bytes are held. The first refusal is returned. pub(super) fn admit( - storage: &mut impl RetainedCatalogPublicationStorage, + storage: &mut impl RetainedEvidenceSource, candidate: &CatalogSnapshot<'_, '_, '_>, ) -> Result { let Some(head) = read_head(storage, Phase::AdmitHead)? else { @@ -47,7 +48,7 @@ pub(super) fn admit( /// and root pool entries are immutable and named by digest, so an unchanged /// head selects exactly the admitted manifest and roots. pub(super) fn revalidate( - storage: &mut impl RetainedCatalogPublicationStorage, + storage: &mut impl RetainedEvidenceSource, admission: &RetainedClosureAdmission, ) -> Result<(), Error> { let expected = admission.head(); @@ -63,7 +64,7 @@ pub(super) fn revalidate( } fn read_head( - storage: &mut impl RetainedCatalogPublicationStorage, + storage: &mut impl RetainedEvidenceSource, phase: Phase, ) -> Result, Error> { let Some(encoded) = storage @@ -123,7 +124,7 @@ fn require_manifest_binding( } fn verify_root( - storage: &mut impl RetainedCatalogPublicationStorage, + storage: &mut impl RetainedEvidenceSource, entry: RetentionManifestEntry, candidate: &CatalogSnapshot<'_, '_, '_>, ) -> Result<(), Error> { diff --git a/src/adapters/retained_catalog_publication/retained_evidence_source.rs b/src/adapters/retained_catalog_publication/retained_evidence_source.rs new file mode 100644 index 00000000..db9532d6 --- /dev/null +++ b/src/adapters/retained_catalog_publication/retained_evidence_source.rs @@ -0,0 +1,46 @@ +//! This module owns the crate-internal capability to read retention evidence. +//! +//! The retained-closure gate needs exact retention bytes on two routes: forward +//! catalog publication, through [`RetainedCatalogPublicationStorage`], and +//! recovery finalization of a staged catalog head, which owns no catalog +//! stages. Both routes supply evidence through this one capability so the same +//! admission code decodes, binds, and verifies it. + +use std::io; + +use super::RetainedCatalogPublicationStorage; +use crate::{RetentionHead, RetentionManifestEntry}; + +/// Exact retention record bytes read under writer authority. +/// +/// The contract of each method is the contract of the identically named +/// [`RetainedCatalogPublicationStorage`] method: exact bytes, no-follow and +/// bounded reads, and `None` for the head only when nothing contradicts that +/// absence. +pub(in crate::adapters) trait RetainedEvidenceSource { + /// Reads the exact current retention head bytes, if one is published. + fn read_retention_head(&mut self) -> io::Result>>; + + /// Reads the exact immutable manifest that `head` selects. + fn read_retention_manifest(&mut self, head: &RetentionHead) -> io::Result>; + + /// Reads the exact immutable root that one manifest entry selects. + fn read_retained_root(&mut self, entry: RetentionManifestEntry) -> io::Result>; +} + +/// The evidence reads of one publication storage, without its catalog stages. +pub(super) struct PublicationEvidence<'storage, S>(pub(super) &'storage mut S); + +impl RetainedEvidenceSource for PublicationEvidence<'_, S> { + fn read_retention_head(&mut self) -> io::Result>> { + self.0.read_retention_head() + } + + fn read_retention_manifest(&mut self, head: &RetentionHead) -> io::Result> { + self.0.read_retention_manifest(head) + } + + fn read_retained_root(&mut self, entry: RetentionManifestEntry) -> io::Result> { + self.0.read_retained_root(entry) + } +} From e3930f058d7f17cb3bdb8f81e87eca7226cd71b7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 07:36:53 -0700 Subject: [PATCH 05/36] Feat: version-two catalog writer and catalog-stage recovery (#198) FilesystemVersionTwoCatalogPublisher implements RetainedCatalogPublicationStorage under FilesystemVersionTwoAdmission. Its Stages type, FilesystemVersionTwoCatalogStages, wraps the shared FilesystemCatalogStages and has no public constructor, so it is reachable only through the retained-closure gate. Retention evidence is read no-follow and bounded; an absent head is reported only when no fixed retention stage and no root or manifest pool entry contradicts it. Publication takes no reader fence; readers now admit a retained catalog head.next, which version 2 preserves and readers never select. FilesystemVersionTwoRecoveryAdmission admits a migrated root that may retain head.next. The inventory reader, discarder, completer, segment resumer and next-head finalizer gain version-two constructors over the same stage code, classify_version_two_recovery_names classifies every version-two root entry as known, and version-two next-head finalization runs the forward gate's retained-closure admission immediately before every candidate verification that can precede HEAD replacement. Version-one entry points keep refusing version-two roots and version-two entry points refuse version-one roots. Supported envelope: one staged segment per publication. --- src/adapters/exports.rs | 7 + .../filesystem_initialization_namespace.rs | 41 ++- .../filesystem_recovery_inventory_reader.rs | 77 ++++- ...filesystem_recovery_next_head_candidate.rs | 31 +- ...filesystem_recovery_next_head_finalizer.rs | 62 +++- .../filesystem_recovery_segment_resumer.rs | 21 +- .../filesystem_recovery_stage_completer.rs | 22 +- .../filesystem_recovery_stage_discarder.rs | 34 ++- .../filesystem_version_two_admission.rs | 56 +++- ...ilesystem_version_two_catalog_publisher.rs | 131 +++++++++ ...stem_version_two_catalog_recovery_tests.rs | 118 ++++++++ .../filesystem_version_two_catalog_stages.rs | 235 +++++++++++++++ .../filesystem_version_two_catalog_storage.rs | 29 ++ ...lesystem_version_two_recovery_admission.rs | 80 +++++ src/adapters/mod.rs | 6 + src/adapters/recovery.rs | 4 +- src/adapters/recovery/recovery_entry_role.rs | 14 + .../recovery/recovery_name_classification.rs | 153 ++++++++-- ...ry_next_head_finalization_storage_error.rs | 15 + .../recovery/recovery_required_entry.rs | 29 +- src/adapters/retained_catalog_publication.rs | 20 ++ src/adapters/retention.rs | 2 + .../retention/filesystem_retained_evidence.rs | 122 ++++++++ .../filesystem_retention_snapshot.rs | 4 +- .../retention/filesystem_retention_storage.rs | 2 +- src/lib.rs | 67 +++-- tests/version_two_catalog_writer.rs | 10 + .../finalization_laws.rs | 112 +++++++ .../forward_laws.rs | 64 ++++ .../recovery_fixture.rs | 128 ++++++++ .../refusal_laws.rs | 96 ++++++ .../stage_recovery_laws.rs | 52 ++++ tests/version_two_catalog_writer/suite.rs | 28 ++ .../version_laws.rs | 141 +++++++++ .../writer_fixture.rs | 274 ++++++++++++++++++ 35 files changed, 2186 insertions(+), 101 deletions(-) create mode 100644 src/adapters/filesystem_version_two_catalog_publisher.rs create mode 100644 src/adapters/filesystem_version_two_catalog_recovery_tests.rs create mode 100644 src/adapters/filesystem_version_two_catalog_stages.rs create mode 100644 src/adapters/filesystem_version_two_catalog_storage.rs create mode 100644 src/adapters/filesystem_version_two_recovery_admission.rs create mode 100644 src/adapters/retention/filesystem_retained_evidence.rs create mode 100644 tests/version_two_catalog_writer.rs create mode 100644 tests/version_two_catalog_writer/finalization_laws.rs create mode 100644 tests/version_two_catalog_writer/forward_laws.rs create mode 100644 tests/version_two_catalog_writer/recovery_fixture.rs create mode 100644 tests/version_two_catalog_writer/refusal_laws.rs create mode 100644 tests/version_two_catalog_writer/stage_recovery_laws.rs create mode 100644 tests/version_two_catalog_writer/suite.rs create mode 100644 tests/version_two_catalog_writer/version_laws.rs create mode 100644 tests/version_two_catalog_writer/writer_fixture.rs diff --git a/src/adapters/exports.rs b/src/adapters/exports.rs index 3edc85e7..9fb376f2 100644 --- a/src/adapters/exports.rs +++ b/src/adapters/exports.rs @@ -56,7 +56,14 @@ pub use super::filesystem_recovery_stage_error::{ }; pub use super::filesystem_segment_stage::FilesystemSegmentStage; pub use super::filesystem_version_two_admission::FilesystemVersionTwoAdmission; +pub use super::filesystem_version_two_catalog_publisher::FilesystemVersionTwoCatalogPublisher; +pub use super::filesystem_version_two_catalog_stages::FilesystemVersionTwoCatalogStages; +#[cfg(feature = "repository-tasks")] +pub use super::filesystem_version_two_catalog_stages::{ + CatalogPublicationBoundary, CatalogPublicationObserver, +}; pub use super::filesystem_version_two_record_refusal::VersionTwoRecordRefusal; +pub use super::filesystem_version_two_recovery_admission::FilesystemVersionTwoRecoveryAdmission; pub use super::filesystem_writer_lock::FilesystemWriterLock; pub use super::layout_decode_error::LayoutDecodeError; pub use super::layout_decode_policy::LayoutDecodePolicy; diff --git a/src/adapters/filesystem_initialization_namespace.rs b/src/adapters/filesystem_initialization_namespace.rs index e3a23352..39564d9b 100644 --- a/src/adapters/filesystem_initialization_namespace.rs +++ b/src/adapters/filesystem_initialization_namespace.rs @@ -58,6 +58,22 @@ const VERSION_TWO_NAMES: [&str; 12] = [ GC_NAME, RECOVERY_NAME, ]; +const NEXT_HEAD_NAME: &str = "head.next"; +const VERSION_TWO_STAGED_NAMES: [&str; 13] = [ + LOCK_NAME, + STAGING_NAME, + SEGMENTS_NAME, + CATALOGS_NAME, + HEAD_NAME, + NEXT_HEAD_NAME, + READER_LOCK_NAME, + MARKER_NAME, + INTENT_NAME, + RECEIPT_NAME, + RETENTION_NAME, + GC_NAME, + RECOVERY_NAME, +]; pub(super) fn admit(directory: &Dir) -> io::Result<()> { admit_optional_file(directory, LOCK_NAME)?; @@ -101,6 +117,27 @@ pub(super) fn admit_recoverable(directory: &Dir) -> io::Result<()> { /// marker, both migration records, and all three protocol directories must be /// present. Any other entry is unrecoverable ambiguity. pub(super) fn admit_version_two(directory: &Dir) -> io::Result<()> { + admit_version_two_required(directory)?; + admit_membership(directory, &VERSION_TWO_NAMES)?; + admit_version_two_protocol_directories(directory) +} + +/// Admits a completely migrated version-2 root that may retain a catalog `head.next`. +/// +/// `head.next` is the version-1 catalog-protocol head stage, which version 2 +/// preserves unchanged. A reader selects only `HEAD`, so a retained or +/// in-flight `head.next` does not change its view, and catalog-stage recovery +/// must be able to open the store that holds one. Writer admission keeps the +/// exact law of [`admit_version_two`] and refuses it. Every other entry and +/// every protocol directory is admitted exactly as [`admit_version_two`] does. +pub(super) fn admit_version_two_with_catalog_stage(directory: &Dir) -> io::Result<()> { + admit_version_two_required(directory)?; + admit_optional_file(directory, NEXT_HEAD_NAME)?; + admit_membership(directory, &VERSION_TWO_STAGED_NAMES)?; + admit_version_two_protocol_directories(directory) +} + +fn admit_version_two_required(directory: &Dir) -> io::Result<()> { admit_required_file(directory, LOCK_NAME)?; admit_required_directory(directory, STAGING_NAME)?; admit_required_directory(directory, SEGMENTS_NAME)?; @@ -112,9 +149,7 @@ pub(super) fn admit_version_two(directory: &Dir) -> io::Result<()> { admit_required_file(directory, RECEIPT_NAME)?; admit_required_directory(directory, RETENTION_NAME)?; admit_required_directory(directory, GC_NAME)?; - admit_required_directory(directory, RECOVERY_NAME)?; - admit_membership(directory, &VERSION_TWO_NAMES)?; - admit_version_two_protocol_directories(directory) + admit_required_directory(directory, RECOVERY_NAME) } /// Admits the nested version-2 protocol directories the migration writer left. diff --git a/src/adapters/filesystem_recovery_inventory_reader.rs b/src/adapters/filesystem_recovery_inventory_reader.rs index 392522fb..4acc2b10 100644 --- a/src/adapters/filesystem_recovery_inventory_reader.rs +++ b/src/adapters/filesystem_recovery_inventory_reader.rs @@ -3,10 +3,11 @@ use std::io; use std::path::Path; -#[cfg(test)] +#[cfg(any(test, feature = "repository-tasks"))] use cap_std::ambient_authority; use cap_std::fs::Dir; +use super::filesystem_version_two_admission::admit_root; use super::{ FilesystemRecoveryStageError, RecoveryEntryName, RecoveryInventory, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryOperation, RecoveryInventoryStorage, @@ -70,14 +71,66 @@ impl FilesystemRecoveryInventoryReader { Self::from_root(root) } + /// Opens one migrated version-two store for read-only stage inventory. + /// + /// Admission is the version-two recovery law: the production platform for + /// every version-two protocol directory, the exact namespace with an + /// optional retained catalog `head.next`, the jointly admitted migration + /// records, and the restart-stable root identity. No writer lock is taken + /// and no protocol state is mutated. Classify the resulting inventory with + /// [`classify_version_two_recovery_names`](crate::classify_version_two_recovery_names). + /// + /// # Errors + /// + /// Returns [`RecoveryInventoryError::Io`] for the store root when platform + /// or version-two admission refuses; the source carries the exact + /// [`FilesystemPlatformAdmissionError`](crate::FilesystemPlatformAdmissionError). + /// A version-one root refuses because its marker and records are absent. + pub fn open_version_two(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open_version_two(store_root).map_err(root_error)?; + admit_root( + &root, + filesystem_initialization_namespace::admit_version_two_with_catalog_stage, + ) + .map_err(|source| root_error(io::Error::new(io::ErrorKind::InvalidData, source)))?; + Self::pin(root) + } + + /// Opens a migrated root without platform admission for repository tasks. + /// + /// Every namespace, record, and identity law of [`Self::open_version_two`] + /// still applies. + /// + /// # Errors + /// + /// As [`Self::open_version_two`] after platform admission. + #[cfg(any(test, feature = "repository-tasks"))] + #[doc(hidden)] + pub fn open_version_two_unchecked_for_repository_tasks( + store_root: &Path, + ) -> Result { + let root = Dir::open_ambient_dir(store_root, ambient_authority()).map_err(root_error)?; + admit_root( + &root, + filesystem_initialization_namespace::admit_version_two_with_catalog_stage, + ) + .map_err(|source| root_error(io::Error::new(io::ErrorKind::InvalidData, source)))?; + Self::pin(root) + } + pub(super) fn from_root(root: Dir) -> Result { - filesystem_initialization_namespace::admit_recoverable(&root).map_err(|source| { - RecoveryInventoryError::io( - RecoveryNamespace::Root, - RecoveryInventoryOperation::OpenNamespace, - source, - ) - })?; + filesystem_initialization_namespace::admit_recoverable(&root).map_err(root_error)?; + Self::pin(root) + } + + /// Pins an already admitted, writer-locked version-two root. + pub(super) fn from_version_two_root(root: Dir) -> Result { + filesystem_initialization_namespace::admit_version_two_with_catalog_stage(&root) + .map_err(root_error)?; + Self::pin(root) + } + + fn pin(root: Dir) -> Result { let staging = PinnedRecoveryDirectory::open(&root, RecoveryNamespace::Staging, STAGING_NAME)?; let segments = @@ -221,3 +274,11 @@ impl RecoveryInventoryStorage for FilesystemRecoveryInventoryReader { ) } } + +const fn root_error(source: io::Error) -> RecoveryInventoryError { + RecoveryInventoryError::io( + RecoveryNamespace::Root, + RecoveryInventoryOperation::OpenNamespace, + source, + ) +} diff --git a/src/adapters/filesystem_recovery_next_head_candidate.rs b/src/adapters/filesystem_recovery_next_head_candidate.rs index 14641c40..f75dffda 100644 --- a/src/adapters/filesystem_recovery_next_head_candidate.rs +++ b/src/adapters/filesystem_recovery_next_head_candidate.rs @@ -2,8 +2,10 @@ use std::io; +use super::filesystem_recovery_next_head_finalizer::FinalizationRetention; +use super::retained_catalog_publication::admit_retained_closures; use super::{ - FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageError, + CatalogSnapshot, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageError, RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, RecoveryStage, RecoveryStageEvidence, RecoveryStageNamespacePhase, catalog_restart_loader, filesystem_recovery_stage, @@ -37,6 +39,7 @@ pub(super) fn verify( }, ); } + require_retained_closures(finalizer, &snapshot)?; verify_evidence(finalizer, request) } @@ -110,3 +113,29 @@ fn stage(source: FilesystemRecoveryStageError) -> RecoveryNextHeadFinalizationSt source: Box::new(source), } } + +/// Runs the forward catalog gate's retained-closure admission on the candidate. +/// +/// Version-two finalization changes `HEAD` exactly as forward publication +/// does, so every root named by the current retention head must reconstruct +/// its complete closure from the candidate catalog. The check reads retention +/// evidence under the finalizer's writer authority and mutates nothing; a +/// refusal leaves `head.next` and `HEAD` untouched. +fn require_retained_closures( + finalizer: &FilesystemRecoveryNextHeadFinalizer, + candidate: &CatalogSnapshot<'_, '_, '_>, +) -> Result<(), RecoveryNextHeadFinalizationStorageError> { + match &finalizer.retention { + FinalizationRetention::VersionOne => Ok(()), + FinalizationRetention::VersionTwo(evidence) => { + let mut source = evidence; + admit_retained_closures(&mut source, candidate) + .map(|_admission| ()) + .map_err( + |source| RecoveryNextHeadFinalizationStorageError::RetainedClosure { + source: Box::new(source), + }, + ) + } + } +} diff --git a/src/adapters/filesystem_recovery_next_head_finalizer.rs b/src/adapters/filesystem_recovery_next_head_finalizer.rs index 1f1daeb0..cf66241f 100644 --- a/src/adapters/filesystem_recovery_next_head_finalizer.rs +++ b/src/adapters/filesystem_recovery_next_head_finalizer.rs @@ -4,9 +4,11 @@ use std::path::Path; use cap_std::fs::Dir; +use super::retention::FilesystemRetainedEvidence; use super::{ CatalogRestartPolicy, FilesystemRecoveryNextHeadFinalizationOpenError, - FilesystemRecoveryStageDiscarder, RecoveryStageParent, + FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, + FilesystemVersionTwoRecoveryAdmission, RecoveryStageParent, }; /// Writer-authorized pinned filesystem adapter for exact next-head finalization. @@ -15,10 +17,26 @@ use super::{ /// root and `writer.lock`, then pins all three protocol child directories /// without following links. The synchronous adapter may block on bounded /// filesystem I/O and retains writer authority until dropped. +/// +/// A finalizer opened over a version-two store also retains the admitted +/// retention capabilities and, immediately before every candidate +/// verification that can precede `HEAD` replacement, verifies every current +/// retained closure against the staged candidate with the same admission the +/// forward catalog gate runs. A candidate that would lose retained content +/// refuses and leaves `head.next` in place. #[must_use] pub struct FilesystemRecoveryNextHeadFinalizer { pub(super) discarder: FilesystemRecoveryStageDiscarder, pub(super) policy: CatalogRestartPolicy, + pub(super) retention: FinalizationRetention, +} + +/// Retention evidence a finalizer must preserve, by store format. +pub(super) enum FinalizationRetention { + /// A version-one store has no retention state to preserve. + VersionOne, + /// A version-two store's retained closures gate every replacement. + VersionTwo(FilesystemRetainedEvidence), } impl FilesystemRecoveryNextHeadFinalizer { @@ -36,20 +54,58 @@ impl FilesystemRecoveryNextHeadFinalizer { policy: CatalogRestartPolicy, ) -> Result { FilesystemRecoveryStageDiscarder::open(store_root) - .map(|discarder| Self { discarder, policy }) + .map(|discarder| Self::version_one(discarder, policy)) .map_err(Into::into) } + /// Pins a migrated version-two store for gated head finalization. + /// + /// The call performs no protocol mutation and reads no record. Execution + /// verifies every retained closure named by the current retention head + /// against the candidate before `HEAD` can change. + /// + /// # Errors + /// + /// Returns [`FilesystemRecoveryNextHeadFinalizationOpenError`] on root-clone + /// or namespace pinning failure. + pub fn open_version_two( + admission: FilesystemVersionTwoRecoveryAdmission, + policy: CatalogRestartPolicy, + ) -> Result { + let (lock, retention, roots, manifests) = admission.into_parts(); + let root = lock + .clone_directory() + .map_err(|source| FilesystemRecoveryStageDiscardOpenError::CloneRoot { source })?; + let discarder = FilesystemRecoveryStageDiscarder::from_version_two_lock(lock)?; + let evidence = FilesystemRetainedEvidence::new(root, retention, roots, manifests); + Ok(Self { + discarder, + policy, + retention: FinalizationRetention::VersionTwo(evidence), + }) + } + #[cfg(test)] pub(super) fn open_unchecked_for_tests( store_root: &Path, policy: CatalogRestartPolicy, ) -> Result { FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(store_root) - .map(|discarder| Self { discarder, policy }) + .map(|discarder| Self::version_one(discarder, policy)) .map_err(Into::into) } + const fn version_one( + discarder: FilesystemRecoveryStageDiscarder, + policy: CatalogRestartPolicy, + ) -> Self { + Self { + discarder, + policy, + retention: FinalizationRetention::VersionOne, + } + } + pub(super) const fn root(&self) -> &Dir { self.discarder .inventory diff --git a/src/adapters/filesystem_recovery_segment_resumer.rs b/src/adapters/filesystem_recovery_segment_resumer.rs index 9789c1d3..435e2788 100644 --- a/src/adapters/filesystem_recovery_segment_resumer.rs +++ b/src/adapters/filesystem_recovery_segment_resumer.rs @@ -2,7 +2,10 @@ use std::path::Path; -use super::{FilesystemRecoverySegmentResumeOpenError, FilesystemRecoveryStageDiscarder}; +use super::{ + FilesystemRecoverySegmentResumeOpenError, FilesystemRecoveryStageDiscarder, + FilesystemVersionTwoRecoveryAdmission, +}; /// Writer-authorized pinned filesystem adapter for reusable segment recovery. /// @@ -32,6 +35,22 @@ impl FilesystemRecoverySegmentResumer { .map_err(Into::into) } + /// Pins a migrated version-two store for explicit segment continuation. + /// + /// The call performs no protocol mutation. + /// + /// # Errors + /// + /// Returns [`FilesystemRecoverySegmentResumeOpenError`] on root-clone or + /// namespace pinning failure. + pub fn open_version_two( + admission: FilesystemVersionTwoRecoveryAdmission, + ) -> Result { + FilesystemRecoveryStageDiscarder::open_version_two(admission) + .map(Self::from_discarder) + .map_err(Into::into) + } + const fn from_discarder(discarder: FilesystemRecoveryStageDiscarder) -> Self { Self { discarder, diff --git a/src/adapters/filesystem_recovery_stage_completer.rs b/src/adapters/filesystem_recovery_stage_completer.rs index 90debae4..6d7e9ab6 100644 --- a/src/adapters/filesystem_recovery_stage_completer.rs +++ b/src/adapters/filesystem_recovery_stage_completer.rs @@ -2,7 +2,10 @@ use std::path::Path; -use super::{FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscarder}; +use super::{ + FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscarder, + FilesystemVersionTwoRecoveryAdmission, +}; /// Writer-authorized pinned filesystem adapter for exact stage completion. /// @@ -30,6 +33,23 @@ impl FilesystemRecoveryStageCompleter { .map_err(Into::into) } + /// Pins a migrated version-two store for explicit stage completion. + /// + /// The call performs no protocol mutation. Completion links a stage into + /// an immutable pool and never changes `HEAD`. + /// + /// # Errors + /// + /// Returns [`FilesystemRecoveryStageCompletionOpenError`] on root-clone or + /// namespace pinning failure. + pub fn open_version_two( + admission: FilesystemVersionTwoRecoveryAdmission, + ) -> Result { + FilesystemRecoveryStageDiscarder::open_version_two(admission) + .map(|discarder| Self { discarder }) + .map_err(Into::into) + } + #[cfg(test)] pub(super) fn open_unchecked_for_tests( store_root: &Path, diff --git a/src/adapters/filesystem_recovery_stage_discarder.rs b/src/adapters/filesystem_recovery_stage_discarder.rs index e29cb928..c3db169d 100644 --- a/src/adapters/filesystem_recovery_stage_discarder.rs +++ b/src/adapters/filesystem_recovery_stage_discarder.rs @@ -8,7 +8,7 @@ use cap_std::fs::Dir; use super::{ FilesystemRecoveryInventoryReader, FilesystemRecoveryStageDiscardOpenError, - FilesystemWriterLock, filesystem_platform_profile, + FilesystemVersionTwoRecoveryAdmission, FilesystemWriterLock, filesystem_platform_profile, }; #[cfg(any(test, feature = "repository-tasks"))] use super::{RecoveryInventoryError, RecoveryInventoryOperation, RecoveryNamespace}; @@ -75,6 +75,38 @@ impl FilesystemRecoveryStageDiscarder { Self::from_root(root) } + /// Pins a migrated version-two store for explicit stage discard. + /// + /// The admission already holds writer authority over a root that passed + /// version-two recovery admission; this call pins the catalog-protocol + /// directories under it and performs no protocol mutation. The retained + /// retention capabilities are released: discard never changes `HEAD`. + /// + /// # Errors + /// + /// Returns [`FilesystemRecoveryStageDiscardOpenError`] on root-clone or + /// namespace pinning failure. + pub fn open_version_two( + admission: FilesystemVersionTwoRecoveryAdmission, + ) -> Result { + let (lock, _retention, _roots, _manifests) = admission.into_parts(); + Self::from_version_two_lock(lock) + } + + pub(super) fn from_version_two_lock( + authority: FilesystemWriterLock, + ) -> Result { + let inventory_root = authority + .clone_directory() + .map_err(|source| FilesystemRecoveryStageDiscardOpenError::CloneRoot { source })?; + let inventory = FilesystemRecoveryInventoryReader::from_version_two_root(inventory_root) + .map_err(|source| FilesystemRecoveryStageDiscardOpenError::Namespace { source })?; + Ok(Self { + inventory, + _authority: authority, + }) + } + fn from_root(root: Dir) -> Result { let authority = FilesystemWriterLock::try_acquire_in(root) .map_err(|source| FilesystemRecoveryStageDiscardOpenError::WriterLock { source })?; diff --git a/src/adapters/filesystem_version_two_admission.rs b/src/adapters/filesystem_version_two_admission.rs index 3ef32471..2d2d2f58 100644 --- a/src/adapters/filesystem_version_two_admission.rs +++ b/src/adapters/filesystem_version_two_admission.rs @@ -79,21 +79,8 @@ impl FilesystemVersionTwoAdmission { } fn admit(root: Dir) -> Result { - let lock = FilesystemWriterLock::try_acquire_in(root) - .map_err(|source| FilesystemPlatformAdmissionError::WriterLock { source })?; - let directory = lock - .clone_directory() - .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; - filesystem_initialization_namespace::admit_version_two(&directory) - .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; - let observed = filesystem_platform_profile::root_identity(&directory) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; - let bound = filesystem_version_two_records::admit(&directory) - .map_err(|source| FilesystemPlatformAdmissionError::MigrationRecord { source })?; - require_root_identity(bound, observed)?; - let retention = pin(&directory, "retention")?; - let roots = pin(&retention, "roots")?; - let manifests = pin(&retention, "manifests")?; + let (lock, retention, roots, manifests) = + admit_locked(root, filesystem_initialization_namespace::admit_version_two)?; Ok(Self { lock, retention, @@ -103,6 +90,45 @@ impl FilesystemVersionTwoAdmission { } } +/// Writer authority and the three pinned retention capabilities of one admitted root. +pub(super) type AdmittedVersionTwoParts = (FilesystemWriterLock, Dir, Dir, Dir); + +/// Acquires writer authority, then admits the root under `namespace`. +/// +/// Forward writers pass the exact version-two namespace law; catalog-stage +/// recovery passes the law that also admits a retained `head.next`. Every +/// other check is identical: the marker, intent, and receipt records, the +/// restart-stable root identity, and the pinned retention directories. +pub(super) fn admit_locked( + root: Dir, + namespace: fn(&Dir) -> std::io::Result<()>, +) -> Result { + let lock = FilesystemWriterLock::try_acquire_in(root) + .map_err(|source| FilesystemPlatformAdmissionError::WriterLock { source })?; + let directory = lock + .clone_directory() + .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; + admit_root(&directory, namespace)?; + let retention = pin(&directory, "retention")?; + let roots = pin(&retention, "roots")?; + let manifests = pin(&retention, "manifests")?; + Ok((lock, retention, roots, manifests)) +} + +/// Admits the namespace, migration records, and root identity without mutation. +pub(super) fn admit_root( + directory: &Dir, + namespace: fn(&Dir) -> std::io::Result<()>, +) -> Result<(), FilesystemPlatformAdmissionError> { + namespace(directory) + .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; + let observed = filesystem_platform_profile::root_identity(directory) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + let bound = filesystem_version_two_records::admit(directory) + .map_err(|source| FilesystemPlatformAdmissionError::MigrationRecord { source })?; + require_root_identity(bound, observed) +} + /// Pins one admitted protocol directory without following links. fn pin(parent: &Dir, name: &str) -> Result { parent diff --git a/src/adapters/filesystem_version_two_catalog_publisher.rs b/src/adapters/filesystem_version_two_catalog_publisher.rs new file mode 100644 index 00000000..68178b5c --- /dev/null +++ b/src/adapters/filesystem_version_two_catalog_publisher.rs @@ -0,0 +1,131 @@ +//! This module owns writer-locked version-two catalog publication state. + +use std::io; + +use super::filesystem_catalog_stages::FilesystemCatalogStages; +#[cfg(feature = "repository-tasks")] +use super::filesystem_version_two_catalog_stages::CatalogPublicationObserver; +use super::retention::FilesystemRetainedEvidence; +use super::{ + AdmittedSegment, CatalogRestartPolicy, FilesystemSegmentStage, FilesystemVersionTwoAdmission, + FilesystemVersionTwoCatalogStages, FilesystemWriterLock, SealedSegment, SegmentPublication, + SegmentPublicationError, SegmentStageCreateError, +}; + +/// Exclusive version-two authority to ingest one segment and publish a catalog. +/// +/// The publisher owns the writer lock, the pinned store root, the catalog +/// protocol directories, and the `retention`, `retention/roots`, and +/// `retention/manifests` capabilities that version-two admission pinned. It +/// publishes only through +/// [`publish_retained_catalog_generation`](crate::publish_retained_catalog_generation): +/// it implements [`RetainedCatalogPublicationStorage`](crate::RetainedCatalogPublicationStorage) +/// and not the ungated port, so the type system refuses +/// [`publish_catalog_generation`](crate::publish_catalog_generation): +/// +/// ```compile_fail +/// fn publish( +/// mut publisher: keep::FilesystemVersionTwoCatalogPublisher, +/// catalog: &keep::CanonicalCatalog, +/// ) { +/// let _ = keep::publish_catalog_generation( +/// &mut publisher, +/// keep::CatalogPublicationExpectation::uninitialized(), +/// keep::SegmentPublication::none(), +/// catalog, +/// &[], +/// ); +/// } +/// ``` +/// +/// Publication takes no reader fence; readers keep their views beside it. +/// Dropping the publisher closes open stages and every directory capability +/// before releasing the writer lock, but never publishes, removes, truncates, +/// or repairs protocol state. +#[must_use] +pub struct FilesystemVersionTwoCatalogPublisher { + pub(super) evidence: FilesystemRetainedEvidence, + pub(super) stages: FilesystemVersionTwoCatalogStages, + // Fields drop in declaration order. Writer authority must outlive every + // directory capability and retained writable stage. + _lock: FilesystemWriterLock, +} + +impl FilesystemVersionTwoCatalogPublisher { + /// Pins the catalog and retention directories under version-two writer authority. + /// + /// Only [`FilesystemVersionTwoAdmission`] is accepted, so a version-one + /// root or version-one writer authority cannot reach this publisher: + /// + /// ```compile_fail + /// fn open(admission: keep::FilesystemPlatformAdmission, policy: keep::CatalogRestartPolicy) { + /// let _ = keep::FilesystemVersionTwoCatalogPublisher::open(admission, policy); + /// } + /// ``` + /// + /// The synchronous call performs no protocol mutation and reads no record. + /// + /// # Errors + /// + /// Returns the exact root-clone, no-follow directory-open, or + /// directory-inspection failure. A protocol entry that is not a directory + /// returns [`io::ErrorKind::NotADirectory`]. A failure releases the writer + /// lock. + pub fn open( + admission: FilesystemVersionTwoAdmission, + policy: CatalogRestartPolicy, + ) -> io::Result { + let (lock, retention, roots, manifests) = admission.into_parts(); + let pinned_root = lock.clone_directory()?; + let stages = FilesystemCatalogStages::open(&pinned_root, policy)?; + Ok(Self { + evidence: FilesystemRetainedEvidence::new(pinned_root, retention, roots, manifests), + stages: FilesystemVersionTwoCatalogStages::new(stages), + _lock: lock, + }) + } + + /// Exclusively creates `staging/current.seg` under this writer authority. + /// + /// The stage borrows this publisher until it is dropped or sealed and + /// selected. Creation performs blocking, capability-relative I/O. + /// + /// # Errors + /// + /// Returns [`SegmentStageCreateError`] without opening or truncating an + /// existing filesystem entry. + pub fn create_segment_stage( + &self, + ) -> Result, SegmentStageCreateError> { + self.stages.shared().create_segment_stage() + } + + /// Closes and selects one synchronized stage created by this publisher. + /// + /// # Errors + /// + /// Returns [`SegmentPublicationError::PublisherAuthority`] when `sealed` + /// was created by another publisher, including a version-one publisher. + /// Other variants preserve exact closed-stage to admitted-segment + /// disagreements. + pub fn select_segment<'selection, 'records>( + &self, + sealed: SealedSegment>, + admitted: &'selection AdmittedSegment<'records>, + ) -> Result, SegmentPublicationError> { + self.stages.shared().select_segment(sealed, admitted) + } + + /// Installs an observer around every catalog transition for repository tasks. + /// + /// The crash matrix blocks inside the observer for process death; an + /// observer error fails the transition at that boundary. + #[cfg(feature = "repository-tasks")] + #[doc(hidden)] + pub fn observe_catalog_phases_for_repository_tasks( + &mut self, + observer: CatalogPublicationObserver, + ) { + self.stages.observe(observer); + } +} diff --git a/src/adapters/filesystem_version_two_catalog_recovery_tests.rs b/src/adapters/filesystem_version_two_catalog_recovery_tests.rs new file mode 100644 index 00000000..e1f0d6e7 --- /dev/null +++ b/src/adapters/filesystem_version_two_catalog_recovery_tests.rs @@ -0,0 +1,118 @@ +//! Interrupted version-two publication and its stage recovery. +//! +//! Each law fails the real version-two writer at one storage-port boundary, +//! then recovers with the public version-two recovery tools and requires the +//! store to read both blobs exactly with every retained root preserved. + +#[path = "../../tests/golden_file_worldline/durable_fixture.rs"] +#[allow( + dead_code, + reason = "interruption laws use complete stores; golden laws cover partial records" +)] +mod durable_fixture; +#[path = "../../tests/version_two_catalog_writer/recovery_fixture.rs"] +#[allow( + dead_code, + reason = "interruption laws finalize and complete; the public suite also inspects roles" +)] +mod recovery_fixture; +#[path = "../../tests/version_two_catalog_writer/writer_fixture.rs"] +#[allow( + dead_code, + reason = "interruption laws ingest and retain; the public suite also stages by hand" +)] +mod writer_fixture; + +use std::io; +use std::path::Path; + +use super::filesystem_test_sandbox as durable_sandbox; + +use crate::{CatalogPublicationError, CatalogPublicationPhase, RetainedCatalogPublicationError}; + +use super::filesystem_version_two_catalog_stages::CatalogPublicationBoundary; +use recovery_fixture::{complete, discard, finalize, stage_roles}; +use writer_fixture::{ + Predecessor, TestResult, assert_reads, first_blob, open_publisher, pool_segments, publish, + republish, retain, second_blob, +}; + +const INJECTED: &str = "injected version-two publication interruption"; + +/// Publishes the second blob, failing after `phase` completes. +fn interrupted_after(root: &Path, phase: CatalogPublicationPhase) -> TestResult<()> { + let mut publisher = open_publisher(root)?; + publisher + .stages + .observe(Box::new(move |observed, boundary| { + if observed == phase && boundary == CatalogPublicationBoundary::After { + Err(io::Error::other(INJECTED)) + } else { + Ok(()) + } + })); + let outcome = publish(&mut publisher, root, &second_blob(), Predecessor::Kept)?; + match outcome { + Err(RetainedCatalogPublicationError::Catalog { + source: CatalogPublicationError::Storage { phase: failed, .. }, + }) if failed == phase => Ok(()), + other => Err(format!("expected interruption at {phase}: {other:?}").into()), + } +} + +/// Requires both blobs to read back once the second blob is retained. +fn assert_both_read(root: &Path) -> TestResult<()> { + let first = first_blob(); + assert_reads(root, &first)?; + retain(root, b"second", &second_blob())?; + assert_reads(root, &first)?; + assert_reads(root, &second_blob()) +} + +#[test] +fn a_head_stage_interrupted_after_synchronization_is_finalized_through_the_gate() -> TestResult<()> +{ + let sandbox = durable_fixture::build("v2-interrupt-head", &[&first_blob()])?; + let root = sandbox.path(); + interrupted_after(root, CatalogPublicationPhase::SynchronizeHead)?; + assert_eq!( + stage_roles(root)?, + [crate::RecoveryEntryRole::NextHeadStage] + ); + assert_reads(root, &first_blob())?; + + let candidate = pool_segments(root)?; + let _receipt = finalize(root, &candidate)??; + assert!(stage_roles(root)?.is_empty()); + assert_both_read(root) +} + +#[test] +fn a_linked_segment_stage_is_completed_and_the_catalog_republished() -> TestResult<()> { + let sandbox = durable_fixture::build("v2-interrupt-segment", &[&first_blob()])?; + let root = sandbox.path(); + interrupted_after(root, CatalogPublicationPhase::LinkSegment)?; + assert_eq!(stage_roles(root)?, [crate::RecoveryEntryRole::SegmentStage]); + + let _receipt = complete(root, crate::RecoveryStage::Segment)?; + assert!(stage_roles(root)?.is_empty()); + let mut publisher = open_publisher(root)?; + let _published = republish(&mut publisher, root)??; + drop(publisher); + assert_both_read(root) +} + +#[test] +fn an_empty_catalog_stage_is_discarded_and_the_catalog_republished() -> TestResult<()> { + let sandbox = durable_fixture::build("v2-interrupt-catalog", &[&first_blob()])?; + let root = sandbox.path(); + interrupted_after(root, CatalogPublicationPhase::CreateCatalogStage)?; + assert_eq!(stage_roles(root)?, [crate::RecoveryEntryRole::CatalogStage]); + + let _receipt = discard(root, crate::RecoveryStage::Catalog)?; + assert!(stage_roles(root)?.is_empty()); + let mut publisher = open_publisher(root)?; + let _published = republish(&mut publisher, root)??; + drop(publisher); + assert_both_read(root) +} diff --git a/src/adapters/filesystem_version_two_catalog_stages.rs b/src/adapters/filesystem_version_two_catalog_stages.rs new file mode 100644 index 00000000..c03ac03a --- /dev/null +++ b/src/adapters/filesystem_version_two_catalog_stages.rs @@ -0,0 +1,235 @@ +//! This module owns the catalog stages reachable only after retained-closure admission. +//! +//! Version-two catalog publication executes the shared +//! [`FilesystemCatalogStages`] implementation through this type. It has no +//! public constructor and is reachable only through +//! [`RetainedCatalogPublicationStorage::catalog_stages`](crate::RetainedCatalogPublicationStorage::catalog_stages), +//! which requires a [`RetainedClosureAdmission`](crate::RetainedClosureAdmission) +//! that only the gate creates. The ungated +//! [`publish_catalog_generation`](crate::publish_catalog_generation) can +//! therefore never drive version-two stages. + +use std::io; + +use super::filesystem_catalog_stages::FilesystemCatalogStages; +use super::{ + AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, + CatalogPublicationPhase as Phase, CatalogPublicationReadiness, CatalogPublicationStorage, + CatalogSnapshot, ChecksummedCatalog, SegmentPublication, +}; + +/// Where a repository-task observer runs relative to one catalog transition. +#[cfg(any(test, feature = "repository-tasks"))] +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CatalogPublicationBoundary { + /// Before the transition begins. + Before, + /// After the transition returned successfully. + After, +} + +/// Repository-task observer of version-two catalog transitions. +/// +/// Returning an error fails the transition at that boundary exactly as a +/// storage failure would; the crash matrix blocks inside it for process death. +#[cfg(any(test, feature = "repository-tasks"))] +#[doc(hidden)] +pub type CatalogPublicationObserver = + Box io::Result<()>>; + +/// Version-two catalog protocol transitions guarded by retained-closure admission. +/// +/// Every transition is the shared version-one stage implementation, executed +/// under the version-two publisher's writer authority. Fields are private and +/// no constructor is public. +#[must_use] +pub struct FilesystemVersionTwoCatalogStages { + stages: FilesystemCatalogStages, + #[cfg(any(test, feature = "repository-tasks"))] + observer: Option, +} + +impl FilesystemVersionTwoCatalogStages { + pub(super) const fn new(stages: FilesystemCatalogStages) -> Self { + Self { + stages, + #[cfg(any(test, feature = "repository-tasks"))] + observer: None, + } + } + + pub(super) const fn shared(&self) -> &FilesystemCatalogStages { + &self.stages + } + + #[cfg(any(test, feature = "repository-tasks"))] + pub(super) fn observe(&mut self, observer: CatalogPublicationObserver) { + self.observer = Some(observer); + } + + /// Runs one shared transition, notifying a repository-task observer around it. + #[cfg(any(test, feature = "repository-tasks"))] + fn transition( + &mut self, + phase: Phase, + operation: impl FnOnce(&mut FilesystemCatalogStages) -> io::Result, + ) -> io::Result { + self.notify(phase, CatalogPublicationBoundary::Before)?; + let value = operation(&mut self.stages)?; + self.notify(phase, CatalogPublicationBoundary::After)?; + Ok(value) + } + + /// Runs one shared transition; production builds observe no boundary. + #[cfg(not(any(test, feature = "repository-tasks")))] + fn transition( + &mut self, + _phase: Phase, + operation: impl FnOnce(&mut FilesystemCatalogStages) -> io::Result, + ) -> io::Result { + operation(&mut self.stages) + } + + #[cfg(any(test, feature = "repository-tasks"))] + fn notify(&mut self, phase: Phase, boundary: CatalogPublicationBoundary) -> io::Result<()> { + self.observer + .as_mut() + .map_or(Ok(()), |observer| observer(phase, boundary)) + } +} + +impl CatalogPublicationStorage for FilesystemVersionTwoCatalogStages { + fn verify_current( + &mut self, + expected: CatalogPublicationExpectation, + candidate: &CatalogSnapshot<'_, '_, '_>, + segment: &SegmentPublication<'_, '_>, + ) -> io::Result { + self.transition(Phase::VerifyCurrent, |stages| { + stages.verify_current(expected, candidate, segment) + }) + } + + fn link_segment(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { + self.transition(Phase::LinkSegment, |stages| stages.link_segment(segment)) + } + + fn verify_segment_pool(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { + self.transition(Phase::VerifySegmentPool, |stages| { + stages.verify_segment_pool(segment) + }) + } + + fn synchronize_segments(&mut self) -> io::Result<()> { + self.transition(Phase::SynchronizeSegments, |stages| { + stages.synchronize_segments() + }) + } + + fn remove_segment_stage(&mut self) -> io::Result<()> { + self.transition(Phase::RemoveSegmentStage, |stages| { + stages.remove_segment_stage() + }) + } + + fn synchronize_staging_after_segment(&mut self) -> io::Result<()> { + self.transition(Phase::SynchronizeStagingAfterSegment, |stages| { + stages.synchronize_staging_after_segment() + }) + } + + fn create_catalog_stage(&mut self) -> io::Result<()> { + self.transition(Phase::CreateCatalogStage, |stages| { + stages.create_catalog_stage() + }) + } + + fn write_catalog(&mut self, catalog: &CanonicalCatalog) -> io::Result<()> { + self.transition(Phase::WriteCatalog, |stages| stages.write_catalog(catalog)) + } + + fn flush_catalog(&mut self) -> io::Result<()> { + self.transition( + Phase::FlushCatalog, + CatalogPublicationStorage::flush_catalog, + ) + } + + fn synchronize_catalog(&mut self) -> io::Result<()> { + self.transition(Phase::SynchronizeCatalog, |stages| { + stages.synchronize_catalog() + }) + } + + fn link_catalog(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { + self.transition(Phase::LinkCatalog, |stages| stages.link_catalog(catalog)) + } + + fn verify_catalog_pool(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { + self.transition(Phase::VerifyCatalogPool, |stages| { + stages.verify_catalog_pool(catalog) + }) + } + + fn synchronize_catalogs(&mut self) -> io::Result<()> { + self.transition(Phase::SynchronizeCatalogs, |stages| { + stages.synchronize_catalogs() + }) + } + + fn remove_catalog_stage(&mut self) -> io::Result<()> { + self.transition(Phase::RemoveCatalogStage, |stages| { + stages.remove_catalog_stage() + }) + } + + fn synchronize_staging_after_catalog(&mut self) -> io::Result<()> { + self.transition(Phase::SynchronizeStagingAfterCatalog, |stages| { + stages.synchronize_staging_after_catalog() + }) + } + + fn create_head_stage(&mut self) -> io::Result<()> { + self.transition( + Phase::CreateHeadStage, + CatalogPublicationStorage::create_head_stage, + ) + } + + fn write_head(&mut self, head: &CanonicalPublicationHead) -> io::Result<()> { + self.transition(Phase::WriteHead, |stages| stages.write_head(head)) + } + + fn flush_head(&mut self) -> io::Result<()> { + self.transition(Phase::FlushHead, CatalogPublicationStorage::flush_head) + } + + fn synchronize_head(&mut self) -> io::Result<()> { + self.transition( + Phase::SynchronizeHead, + CatalogPublicationStorage::synchronize_head, + ) + } + + fn verify_head_view( + &mut self, + head: &CanonicalPublicationHead, + snapshot: &CatalogSnapshot<'_, '_, '_>, + ) -> io::Result<()> { + self.transition(Phase::VerifyHeadView, |stages| { + stages.verify_head_view(head, snapshot) + }) + } + + fn replace_head(&mut self) -> io::Result<()> { + self.transition(Phase::ReplaceHead, CatalogPublicationStorage::replace_head) + } + + fn synchronize_root(&mut self) -> io::Result<()> { + self.transition( + Phase::SynchronizeRoot, + CatalogPublicationStorage::synchronize_root, + ) + } +} diff --git a/src/adapters/filesystem_version_two_catalog_storage.rs b/src/adapters/filesystem_version_two_catalog_storage.rs new file mode 100644 index 00000000..e7f58241 --- /dev/null +++ b/src/adapters/filesystem_version_two_catalog_storage.rs @@ -0,0 +1,29 @@ +//! This module binds version-two writer authority to the retained-closure port. + +use std::io; + +use super::{ + FilesystemVersionTwoCatalogPublisher, FilesystemVersionTwoCatalogStages, + RetainedCatalogPublicationStorage, RetainedClosureAdmission, +}; +use crate::{RetentionHead, RetentionManifestEntry}; + +impl RetainedCatalogPublicationStorage for FilesystemVersionTwoCatalogPublisher { + type Stages = FilesystemVersionTwoCatalogStages; + + fn read_retention_head(&mut self) -> io::Result>> { + self.evidence.read_head() + } + + fn read_retention_manifest(&mut self, head: &RetentionHead) -> io::Result> { + self.evidence.read_manifest(head) + } + + fn read_retained_root(&mut self, entry: RetentionManifestEntry) -> io::Result> { + self.evidence.read_root(entry) + } + + fn catalog_stages(&mut self, _admission: &RetainedClosureAdmission) -> &mut Self::Stages { + &mut self.stages + } +} diff --git a/src/adapters/filesystem_version_two_recovery_admission.rs b/src/adapters/filesystem_version_two_recovery_admission.rs new file mode 100644 index 00000000..62561f31 --- /dev/null +++ b/src/adapters/filesystem_version_two_recovery_admission.rs @@ -0,0 +1,80 @@ +//! This module owns proof that a version-two root admits catalog-stage recovery. + +use std::path::Path; + +#[cfg(any(test, feature = "repository-tasks"))] +use cap_std::ambient_authority; +use cap_std::fs::Dir; + +use super::filesystem_version_two_admission::{AdmittedVersionTwoParts, admit_locked}; +use super::{ + FilesystemPlatformAdmissionError, filesystem_initialization_namespace, + filesystem_platform_profile, +}; + +/// Exclusive writer authority over a migrated version-two root for stage recovery. +/// +/// Admission is the version-two writer admission with one difference: the +/// root may retain the catalog `head.next` that an interrupted publication +/// left. Every other law is the same: the production platform, the writer +/// lock, the exact namespace with its protocol directories, the jointly +/// admitted marker, intent, and receipt records, and the restart-stable root +/// identity. The admitted `retention`, `roots`, and `manifests` capabilities +/// are retained so recovery finalization verifies retained closures against +/// the directories that passed admission. +/// +/// Only the version-two recovery constructors consume this value; version-one +/// recovery keeps refusing a version-two root, and this admission refuses a +/// version-one root because the marker and migration records are required. +#[must_use] +pub struct FilesystemVersionTwoRecoveryAdmission { + parts: AdmittedVersionTwoParts, +} + +impl FilesystemVersionTwoRecoveryAdmission { + /// Acquires writer authority over one migrated store for stage recovery. + /// + /// The call mutates no protocol state and may block on filesystem I/O. + /// + /// # Errors + /// + /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, + /// writer-lock, namespace, or migration-record boundary and preserved + /// source. + pub fn reopen(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open_version_two(store_root) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + Self::admit(root) + } + + /// Reopens a migrated root without platform admission for repository tasks. + /// + /// Every namespace, record, and identity law still applies. Production + /// callers use [`Self::reopen`]. + /// + /// # Errors + /// + /// Returns [`FilesystemPlatformAdmissionError`] exactly as [`Self::reopen`] + /// does for every boundary after platform admission. + #[cfg(any(test, feature = "repository-tasks"))] + pub fn reopen_unchecked_for_repository_tasks( + store_root: &Path, + ) -> Result { + let root = Dir::open_ambient_dir(store_root, ambient_authority()) + .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + Self::admit(root) + } + + /// Releases the writer lock and the three pinned retention capabilities. + pub(super) fn into_parts(self) -> AdmittedVersionTwoParts { + self.parts + } + + fn admit(root: Dir) -> Result { + admit_locked( + root, + filesystem_initialization_namespace::admit_version_two_with_catalog_stage, + ) + .map(|parts| Self { parts }) + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 96291b09..c013ed62 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -144,8 +144,14 @@ mod filesystem_store_initializer_tests; #[path = "../../tests/segment_filesystem_stage/sandbox.rs"] mod filesystem_test_sandbox; mod filesystem_version_two_admission; +mod filesystem_version_two_catalog_publisher; +#[cfg(all(test, target_os = "linux"))] +mod filesystem_version_two_catalog_recovery_tests; +mod filesystem_version_two_catalog_stages; +mod filesystem_version_two_catalog_storage; mod filesystem_version_two_record_refusal; mod filesystem_version_two_records; +mod filesystem_version_two_recovery_admission; mod filesystem_writer_lock; mod framed_blake3; mod layout_decode_error; diff --git a/src/adapters/recovery.rs b/src/adapters/recovery.rs index ab1d3263..5bde5474 100644 --- a/src/adapters/recovery.rs +++ b/src/adapters/recovery.rs @@ -107,7 +107,9 @@ pub use recovery_inventory_error::RecoveryInventoryError; pub use recovery_inventory_limit::{RecoveryInventoryLimit, RecoveryInventoryLimitError}; pub use recovery_inventory_operation::RecoveryInventoryOperation; pub use recovery_inventory_storage::RecoveryInventoryStorage; -pub use recovery_name_classification::classify_recovery_names; +pub use recovery_name_classification::{ + classify_recovery_names, classify_version_two_recovery_names, +}; pub use recovery_name_classification_error::RecoveryNameClassificationError; pub use recovery_name_manifest::{RecoveryNameManifest, RecoveryNamedEntry}; pub use recovery_namespace::RecoveryNamespace; diff --git a/src/adapters/recovery/recovery_entry_role.rs b/src/adapters/recovery/recovery_entry_role.rs index a9aef90f..c54450b9 100644 --- a/src/adapters/recovery/recovery_entry_role.rs +++ b/src/adapters/recovery/recovery_entry_role.rs @@ -34,6 +34,20 @@ pub enum RecoveryEntryRole { /// Physical catalog digest parsed from the name. digest: CatalogDigest, }, + /// Version-two persistent reader fence `reader.lock`. + ReaderFence, + /// Version-two format marker `FORMAT`. + FormatMarker, + /// Version-two migration intent record. + MigrationIntent, + /// Version-two migration receipt record. + MigrationReceipt, + /// Version-two `retention` protocol directory, admitted by retention recovery. + RetentionDirectory, + /// Version-two reserved `gc` protocol directory. + GcDirectory, + /// Version-two reserved `recovery` protocol directory. + RecoveryDirectory, } impl RecoveryEntryRole { diff --git a/src/adapters/recovery/recovery_name_classification.rs b/src/adapters/recovery/recovery_name_classification.rs index 81a8204b..fe7bde87 100644 --- a/src/adapters/recovery/recovery_name_classification.rs +++ b/src/adapters/recovery/recovery_name_classification.rs @@ -6,11 +6,12 @@ use super::{ recovery_pool_name, }; -/// Classifies one complete inventory without opening artifact bytes. +/// Classifies one complete version-one inventory without opening artifact bytes. /// /// The operation preserves inventory order and allocates at most one manifest /// entry per inventory entry. It rejects unknown names, missing initialized /// root entries, noncanonical pool coordinates, and simultaneous fixed stages. +/// Every version-two root entry is unknown here. /// /// # Errors /// @@ -18,13 +19,47 @@ use super::{ /// namespace, grammar, required-entry, or stage-conflict failure. pub fn classify_recovery_names( inventory: RecoveryInventory, +) -> Result { + classify_inventory(inventory, StoreLayout::VersionOne) +} + +/// Classifies one complete migrated version-two inventory without opening bytes. +/// +/// The version-one laws apply unchanged, and the version-two root entries +/// `reader.lock`, `FORMAT`, `migration.intent`, `migration.receipt`, +/// `retention`, `gc`, and `recovery` are classified as known roles, never as +/// unexpected or ignored. Each of them and the published `HEAD` is required. +/// Migration stages are unknown: a completely migrated root never holds one. +/// The contents of `retention`, `gc`, and `recovery` are not inventoried here; +/// version-two admission and retention recovery own them. +/// +/// # Errors +/// +/// Returns [`RecoveryNameClassificationError`] at the first deterministic +/// namespace, grammar, required-entry, or stage-conflict failure. +pub fn classify_version_two_recovery_names( + inventory: RecoveryInventory, +) -> Result { + classify_inventory(inventory, StoreLayout::VersionTwo) +} + +/// The root layout one classification admits. +#[derive(Clone, Copy)] +enum StoreLayout { + VersionOne, + VersionTwo, +} + +fn classify_inventory( + inventory: RecoveryInventory, + layout: StoreLayout, ) -> Result { let mut required = RequiredEntries::default(); let mut first_stage = None; let mut named = Vec::with_capacity(inventory.entries().len()); for entry in inventory.into_entries() { let (namespace, name) = entry.into_parts(); - let role = match classify(namespace, &name) { + let role = match classify(namespace, &name, layout) { Ok(role) => role, Err(NameFailure::Unexpected) => { return Err(RecoveryNameClassificationError::Unexpected { namespace, name }); @@ -49,7 +84,7 @@ pub fn classify_recovery_names( } named.push(RecoveryNamedEntry::new(namespace, name, role)); } - if let Some(required) = required.first_missing() { + if let Some(required) = required.first_missing(layout) { return Err(RecoveryNameClassificationError::Missing { required }); } Ok(RecoveryNameManifest::new(named)) @@ -58,9 +93,10 @@ pub fn classify_recovery_names( fn classify( namespace: RecoveryNamespace, name: &RecoveryEntryName, + layout: StoreLayout, ) -> Result { match namespace { - RecoveryNamespace::Root => classify_root(name), + RecoveryNamespace::Root => classify_root(name, layout), RecoveryNamespace::Staging => classify_staging(name), RecoveryNamespace::Segments => recovery_pool_name::segment(name) .map(|digest| RecoveryEntryRole::ImmutableSegment { digest }) @@ -71,7 +107,10 @@ fn classify( } } -fn classify_root(name: &RecoveryEntryName) -> Result { +fn classify_root( + name: &RecoveryEntryName, + layout: StoreLayout, +) -> Result { match name.as_bytes() { b"writer.lock" => Ok(RecoveryEntryRole::WriterLock), b"staging" => Ok(RecoveryEntryRole::StagingDirectory), @@ -79,6 +118,22 @@ fn classify_root(name: &RecoveryEntryName) -> Result Ok(RecoveryEntryRole::CatalogPoolDirectory), b"HEAD" => Ok(RecoveryEntryRole::CurrentHead), b"head.next" => Ok(RecoveryEntryRole::NextHeadStage), + other => match layout { + StoreLayout::VersionOne => Err(NameFailure::Unexpected), + StoreLayout::VersionTwo => classify_version_two_root(other), + }, + } +} + +fn classify_version_two_root(name: &[u8]) -> Result { + match name { + b"reader.lock" => Ok(RecoveryEntryRole::ReaderFence), + b"FORMAT" => Ok(RecoveryEntryRole::FormatMarker), + b"migration.intent" => Ok(RecoveryEntryRole::MigrationIntent), + b"migration.receipt" => Ok(RecoveryEntryRole::MigrationReceipt), + b"retention" => Ok(RecoveryEntryRole::RetentionDirectory), + b"gc" => Ok(RecoveryEntryRole::GcDirectory), + b"recovery" => Ok(RecoveryEntryRole::RecoveryDirectory), _ => Err(NameFailure::Unexpected), } } @@ -96,41 +151,75 @@ enum NameFailure { Pool(super::RecoveryPoolNameError), } +/// Required root entries in report order; version one requires the prefix. +const REQUIRED: [RecoveryRequiredEntry; 12] = [ + RecoveryRequiredEntry::WriterLock, + RecoveryRequiredEntry::StagingDirectory, + RecoveryRequiredEntry::SegmentPoolDirectory, + RecoveryRequiredEntry::CatalogPoolDirectory, + RecoveryRequiredEntry::CurrentHead, + RecoveryRequiredEntry::ReaderFence, + RecoveryRequiredEntry::FormatMarker, + RecoveryRequiredEntry::MigrationIntent, + RecoveryRequiredEntry::MigrationReceipt, + RecoveryRequiredEntry::RetentionDirectory, + RecoveryRequiredEntry::GcDirectory, + RecoveryRequiredEntry::RecoveryDirectory, +]; +const VERSION_ONE_REQUIRED: usize = 4; + #[derive(Default)] -struct RequiredEntries(u8); +struct RequiredEntries { + observed: [bool; REQUIRED.len()], +} impl RequiredEntries { - const WRITER_LOCK: u8 = 1 << 0; - const STAGING: u8 = 1 << 1; - const SEGMENTS: u8 = 1 << 2; - const CATALOGS: u8 = 1 << 3; - - const fn observe(&mut self, role: RecoveryEntryRole) { - match role { - RecoveryEntryRole::WriterLock => self.0 |= Self::WRITER_LOCK, - RecoveryEntryRole::StagingDirectory => self.0 |= Self::STAGING, - RecoveryEntryRole::SegmentPoolDirectory => self.0 |= Self::SEGMENTS, - RecoveryEntryRole::CatalogPoolDirectory => self.0 |= Self::CATALOGS, - RecoveryEntryRole::CurrentHead - | RecoveryEntryRole::NextHeadStage - | RecoveryEntryRole::SegmentStage - | RecoveryEntryRole::CatalogStage - | RecoveryEntryRole::ImmutableSegment { .. } - | RecoveryEntryRole::ImmutableCatalog { .. } => {} + fn observe(&mut self, role: RecoveryEntryRole) { + let Some(required) = required_entry(role) else { + return; + }; + for (seen, entry) in self.observed.iter_mut().zip(REQUIRED) { + if entry == required { + *seen = true; + } } } - const fn first_missing(&self) -> Option { - if self.0 & Self::WRITER_LOCK == 0 { - Some(RecoveryRequiredEntry::WriterLock) - } else if self.0 & Self::STAGING == 0 { - Some(RecoveryRequiredEntry::StagingDirectory) - } else if self.0 & Self::SEGMENTS == 0 { + fn first_missing(&self, layout: StoreLayout) -> Option { + let count = match layout { + StoreLayout::VersionOne => VERSION_ONE_REQUIRED, + StoreLayout::VersionTwo => REQUIRED.len(), + }; + REQUIRED + .into_iter() + .zip(self.observed) + .take(count) + .find_map(|(entry, seen)| (!seen).then_some(entry)) + } +} + +const fn required_entry(role: RecoveryEntryRole) -> Option { + match role { + RecoveryEntryRole::WriterLock => Some(RecoveryRequiredEntry::WriterLock), + RecoveryEntryRole::StagingDirectory => Some(RecoveryRequiredEntry::StagingDirectory), + RecoveryEntryRole::SegmentPoolDirectory => { Some(RecoveryRequiredEntry::SegmentPoolDirectory) - } else if self.0 & Self::CATALOGS == 0 { + } + RecoveryEntryRole::CatalogPoolDirectory => { Some(RecoveryRequiredEntry::CatalogPoolDirectory) - } else { - None } + RecoveryEntryRole::CurrentHead => Some(RecoveryRequiredEntry::CurrentHead), + RecoveryEntryRole::ReaderFence => Some(RecoveryRequiredEntry::ReaderFence), + RecoveryEntryRole::FormatMarker => Some(RecoveryRequiredEntry::FormatMarker), + RecoveryEntryRole::MigrationIntent => Some(RecoveryRequiredEntry::MigrationIntent), + RecoveryEntryRole::MigrationReceipt => Some(RecoveryRequiredEntry::MigrationReceipt), + RecoveryEntryRole::RetentionDirectory => Some(RecoveryRequiredEntry::RetentionDirectory), + RecoveryEntryRole::GcDirectory => Some(RecoveryRequiredEntry::GcDirectory), + RecoveryEntryRole::RecoveryDirectory => Some(RecoveryRequiredEntry::RecoveryDirectory), + RecoveryEntryRole::NextHeadStage + | RecoveryEntryRole::SegmentStage + | RecoveryEntryRole::CatalogStage + | RecoveryEntryRole::ImmutableSegment { .. } + | RecoveryEntryRole::ImmutableCatalog { .. } => None, } } diff --git a/src/adapters/recovery/recovery_next_head_finalization_storage_error.rs b/src/adapters/recovery/recovery_next_head_finalization_storage_error.rs index 7f83d78c..9b3e2b54 100644 --- a/src/adapters/recovery/recovery_next_head_finalization_storage_error.rs +++ b/src/adapters/recovery/recovery_next_head_finalization_storage_error.rs @@ -8,6 +8,7 @@ use super::{ CatalogPublicationExpectation, CatalogRestartError, FilesystemRecoveryStageError, RecoveryNextHeadFinalizationTarget, RecoveryStageEvidence, }; +use crate::RetainedCatalogPublicationError; /// Why storage could not continue one exact next-head finalization request. #[derive(Debug)] @@ -58,6 +59,15 @@ pub enum RecoveryNextHeadFinalizationStorageError { /// Exact bounded restart-loading failure. source: Box, }, + /// The candidate would lose content a current retention root retains. + /// + /// Only version-two finalization reports this. The source is the forward + /// catalog gate's own evidence, binding, or closure refusal; its `Catalog` + /// and `RetentionChanged` variants never arise here. + RetainedClosure { + /// Exact retained-closure admission refusal. + source: Box, + }, /// The storage boundary failed while observing or mutating. Storage { /// Exact underlying storage failure. @@ -112,6 +122,10 @@ impl fmt::Display for RecoveryNextHeadFinalizationStorageError { Self::CurrentView { source } => { write!(formatter, "durable current view is invalid: {source}") } + Self::RetainedClosure { source } => write!( + formatter, + "recovery candidate would not preserve retained content: {source}" + ), Self::Storage { source } => { write!( formatter, @@ -128,6 +142,7 @@ impl Error for RecoveryNextHeadFinalizationStorageError { Self::Storage { source } => Some(source), Self::Stage { source } => Some(source.as_ref()), Self::CandidateView { source } | Self::CurrentView { source } => Some(source.as_ref()), + Self::RetainedClosure { source } => Some(source.as_ref()), Self::EvidenceMismatch { .. } | Self::CurrentMismatch { .. } | Self::CandidateMismatch { .. } diff --git a/src/adapters/recovery/recovery_required_entry.rs b/src/adapters/recovery/recovery_required_entry.rs index 1639c985..dcb45ee0 100644 --- a/src/adapters/recovery/recovery_required_entry.rs +++ b/src/adapters/recovery/recovery_required_entry.rs @@ -2,7 +2,10 @@ use std::fmt; -/// One fixed entry required in every initialized store root. +/// One fixed entry required in an initialized store root. +/// +/// The first four are required in every root. The remaining entries are +/// required only when a migrated version-two root is classified. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RecoveryRequiredEntry { /// Persistent `writer.lock`. @@ -13,6 +16,22 @@ pub enum RecoveryRequiredEntry { SegmentPoolDirectory, /// `catalogs` directory. CatalogPoolDirectory, + /// Published catalog `HEAD`, required in a version-two root. + CurrentHead, + /// Persistent `reader.lock`. + ReaderFence, + /// `FORMAT` marker. + FormatMarker, + /// `migration.intent` record. + MigrationIntent, + /// `migration.receipt` record. + MigrationReceipt, + /// `retention` directory. + RetentionDirectory, + /// `gc` directory. + GcDirectory, + /// `recovery` directory. + RecoveryDirectory, } impl fmt::Display for RecoveryRequiredEntry { @@ -22,6 +41,14 @@ impl fmt::Display for RecoveryRequiredEntry { Self::StagingDirectory => "staging", Self::SegmentPoolDirectory => "segments", Self::CatalogPoolDirectory => "catalogs", + Self::CurrentHead => "HEAD", + Self::ReaderFence => "reader.lock", + Self::FormatMarker => "FORMAT", + Self::MigrationIntent => "migration.intent", + Self::MigrationReceipt => "migration.receipt", + Self::RetentionDirectory => "retention", + Self::GcDirectory => "gc", + Self::RecoveryDirectory => "recovery", }) } } diff --git a/src/adapters/retained_catalog_publication.rs b/src/adapters/retained_catalog_publication.rs index d8ce5a68..171b9f95 100644 --- a/src/adapters/retained_catalog_publication.rs +++ b/src/adapters/retained_catalog_publication.rs @@ -25,6 +25,8 @@ pub use retained_catalog_publication_storage::RetainedCatalogPublicationStorage; pub use retained_closure_admission::RetainedClosureAdmission; pub use retention_evidence_phase::RetentionEvidencePhase; +pub(in crate::adapters) use retained_evidence_source::RetainedEvidenceSource; + use retained_evidence_source::PublicationEvidence; use super::catalog_publication_execution as execution; @@ -124,3 +126,21 @@ fn stage_candidate( execution::execute_head_stage(stages, candidate.head, candidate.snapshot)?; Ok(()) } + +/// Verifies every current retained closure against one candidate snapshot. +/// +/// This is the gate's own admission, exposed to routes that change `HEAD` +/// without running the forward catalog protocol: recovery finalization of a +/// staged `head.next` runs it immediately before replacement. It performs only +/// the reads described by [`RetainedEvidenceSource`] and no mutation. +/// +/// # Errors +/// +/// Returns the forward gate's evidence, binding, or closure refusal. The +/// `Catalog` and `RetentionChanged` variants never arise from this check. +pub(in crate::adapters) fn admit_retained_closures( + evidence: &mut impl RetainedEvidenceSource, + candidate: &CatalogSnapshot<'_, '_, '_>, +) -> Result { + retained_evidence_admission::admit(evidence, candidate) +} diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 7dcc6450..344f3074 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -19,6 +19,7 @@ mod durable_read_law_tests; mod durable_view_law_tests; #[cfg(test)] mod filesystem_recovery_admission_tests; +mod filesystem_retained_evidence; #[cfg(test)] mod filesystem_retention_anchor_order_prefix_tests; mod filesystem_retention_attempt; @@ -242,6 +243,7 @@ pub use canonical_root::CanonicalRetentionRoot; pub use checksummed_head::ChecksummedRetentionHead; pub use closure_error::RetentionClosureVerificationError; pub use closure_verifier::verify_retention_closure; +pub(in crate::adapters) use filesystem_retained_evidence::FilesystemRetainedEvidence; pub use filesystem_retention_authority::FilesystemRetentionPublicationAuthority; pub use filesystem_retention_authority_error::{ FilesystemRetentionAuthorityError, RetentionAuthorityDirectory, diff --git a/src/adapters/retention/filesystem_retained_evidence.rs b/src/adapters/retention/filesystem_retained_evidence.rs new file mode 100644 index 00000000..65449832 --- /dev/null +++ b/src/adapters/retention/filesystem_retained_evidence.rs @@ -0,0 +1,122 @@ +//! This module owns bounded no-follow reads of retained-closure evidence. +//! +//! Version-two catalog publication and catalog-head recovery finalization read +//! the current retention head, its manifest, and every root it selects through +//! this one reader before `HEAD` may change. The reader returns exact bytes; +//! the retained-closure gate decodes and binds every record itself. + +use std::io; + +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; + +use super::RetentionCurrentStateRefusal as Refusal; +use super::filesystem_retention_current::read_exact_optional; +use super::filesystem_retention_pool_name as pool_name; +use super::filesystem_retention_stage::retention_error; +use super::filesystem_retention_storage::{require_no_retained_stage, require_pinned_directories}; +use super::{filesystem_retention_namespace, head_decoder, root_header_decoder}; +use crate::adapters::filesystem_exact_record; +use crate::adapters::retained_catalog_publication::RetainedEvidenceSource; +use crate::{RetentionHead, RetentionManifestEntry}; + +/// Pinned `retention`, `retention/roots`, and `retention/manifests` capabilities. +/// +/// The owner holds writer authority and the store-root capability these +/// directories were admitted under. Every read follows no links, reads no more +/// than the record's fixed or grammar-bounded length plus one trailing-byte +/// probe, and mutates nothing. +pub(in crate::adapters) struct FilesystemRetainedEvidence { + root: Dir, + retention: Dir, + roots: Dir, + manifests: Dir, +} + +impl FilesystemRetainedEvidence { + /// Binds the admitted retention capabilities to the pinned store root. + pub(in crate::adapters) const fn new( + root: Dir, + retention: Dir, + roots: Dir, + manifests: Dir, + ) -> Self { + Self { + root, + retention, + roots, + manifests, + } + } + + /// Reads the head after proving its absence is uncontradicted. + /// + /// The protocol names must still resolve to the pinned directories, no + /// fixed retention stage may exist (one means retention recovery is + /// required), and the retention namespace and both pools must admit. Only + /// then is an absent `retention/HEAD` reported as `None`, and only while + /// neither pool holds an artifact. + pub(in crate::adapters) fn read_head(&self) -> io::Result>> { + require_pinned_directories(&self.root, &self.retention, &self.roots, &self.manifests)?; + require_no_retained_stage(&self.retention)?; + let census = + filesystem_retention_namespace::admit(&self.retention, &self.roots, &self.manifests)?; + let head = read_exact_optional( + &self.retention, + pool_name::HEAD, + head_decoder::ENCODED_LENGTH, + )?; + match head { + Some(bytes) => Ok(Some(bytes.into_vec())), + None if census.is_empty() => Ok(None), + None => Err(Refusal::HeadAbsentWithArtifacts.into_io()), + } + } + + pub(in crate::adapters) fn read_manifest(&self, head: &RetentionHead) -> io::Result> { + let length = usize::try_from(head.manifest_length().get()) + .map_err(|_source| Refusal::RecordLengthOverflow.into_io())?; + let name = pool_name::manifest(head.generation(), head.manifest_digest()); + read_required(&self.manifests, &name, length) + } + + pub(in crate::adapters) fn read_root( + &self, + entry: RetentionManifestEntry, + ) -> io::Result> { + let namespace = self + .roots + .open_dir_nofollow(pool_name::namespace(entry.namespace()))?; + let name = pool_name::root(entry.root_generation(), entry.root_digest()); + let metadata = namespace.symlink_metadata(&name)?; + if !metadata.is_file() { + return Err(Refusal::RecordKindOrLength.into_io()); + } + let length = usize::try_from(metadata.len()) + .map_err(|_source| Refusal::RecordLengthOverflow.into_io())?; + if length > root_header_decoder::MAXIMUM_ENCODED_LENGTH { + return Err(Refusal::RecordKindOrLength.into_io()); + } + read_required(&namespace, &name, length) + } +} + +impl RetainedEvidenceSource for &FilesystemRetainedEvidence { + fn read_retention_head(&mut self) -> io::Result>> { + self.read_head() + } + + fn read_retention_manifest(&mut self, head: &RetentionHead) -> io::Result> { + self.read_manifest(head) + } + + fn read_retained_root(&mut self, entry: RetentionManifestEntry) -> io::Result> { + self.read_root(entry) + } +} + +/// Reads one required immutable record; absence is the filesystem's `NotFound`. +fn read_required(directory: &Dir, name: &str, length: usize) -> io::Result> { + filesystem_exact_record::read_exact_regular(directory, name, length) + .map_err(|error| io::Error::from(retention_error(error))) +} diff --git a/src/adapters/retention/filesystem_retention_snapshot.rs b/src/adapters/retention/filesystem_retention_snapshot.rs index 0cd2e9c1..13e9e797 100644 --- a/src/adapters/retention/filesystem_retention_snapshot.rs +++ b/src/adapters/retention/filesystem_retention_snapshot.rs @@ -120,6 +120,8 @@ impl FilesystemRetentionSnapshot { /// Linux ext4 profile across every present version-two protocol directory. /// The same opened root capability is retained through namespace, migration /// identity, fence, and coordinate admission; the ambient path is not reopened. + /// A retained or in-flight catalog `head.next` is admitted and ignored: the + /// view selects only `HEAD`, so catalog publication runs beside readers. /// /// # Errors /// @@ -147,7 +149,7 @@ impl FilesystemRetentionSnapshot { ) -> Result { let root = filesystem_platform_profile::open_version_two(store_root) .map_err(|source| Error::Admission { source })?; - filesystem_initialization_namespace::admit_version_two(&root) + filesystem_initialization_namespace::admit_version_two_with_catalog_stage(&root) .map_err(|source| Error::Admission { source })?; let bound = filesystem_version_two_records::admit(&root) .map_err(|source| Error::Admission { source })?; diff --git a/src/adapters/retention/filesystem_retention_storage.rs b/src/adapters/retention/filesystem_retention_storage.rs index a9f756c1..f13fa4ea 100644 --- a/src/adapters/retention/filesystem_retention_storage.rs +++ b/src/adapters/retention/filesystem_retention_storage.rs @@ -285,7 +285,7 @@ pub(super) fn require_pinned_directories( Ok(()) } -fn require_no_retained_stage(retention: &Dir) -> io::Result<()> { +pub(super) fn require_no_retained_stage(retention: &Dir) -> io::Result<()> { for stage in [ pool_name::ROOT_STAGE, pool_name::MANIFEST_STAGE, diff --git a/src/lib.rs b/src/lib.rs index 888bc2dc..e32dc0c8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -103,27 +103,29 @@ pub use adapters::{ FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemStoreMigrationAuthority, - FilesystemStoreMigrationInventoryReader, FilesystemVersionTwoAdmission, FilesystemWriterLock, - ImmutablePoolInventoryDigest, InitialGcStateDigest, InitialRetentionStateDigest, - LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, - LayoutIdTextParseError, MigrationInventoryNamespace, MigrationInventoryPool, - MigrationSynchronizationMask, OpenedReusableSegment, PublicationHeadDecodeError, - RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, - RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, - RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, - RecoveryInventoryStorage, RecoveryNameClassificationError, RecoveryNameManifest, - RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, - RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, - RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, - RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, - RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, - RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, - RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, - RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, - RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, - RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, - RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, - RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + FilesystemStoreMigrationInventoryReader, FilesystemVersionTwoAdmission, + FilesystemVersionTwoCatalogPublisher, FilesystemVersionTwoCatalogStages, + FilesystemVersionTwoRecoveryAdmission, FilesystemWriterLock, ImmutablePoolInventoryDigest, + InitialGcStateDigest, InitialRetentionStateDigest, LayoutDecodeError, LayoutDecodePolicy, + LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, + MigrationInventoryNamespace, MigrationInventoryPool, MigrationSynchronizationMask, + OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, + RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, + RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, + RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, + RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, + RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, + RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, + RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, + RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, + RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, + RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, + RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, + RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, + RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, + RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, + RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, + RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, @@ -149,11 +151,12 @@ pub use adapters::{ VersionTwoRecordRefusal, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, - execute_recovery_next_head_finalization, execute_recovery_segment_resume, - execute_recovery_stage_completion, execute_recovery_stage_discard, execute_store_migration, - fingerprint_recovery_stage, initialize_store, plan_recovery_next_head_finalization, - plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, - publish_catalog_generation, read_recovery_inventory, + classify_version_two_recovery_names, execute_recovery_next_head_finalization, + execute_recovery_segment_resume, execute_recovery_stage_completion, + execute_recovery_stage_discard, execute_store_migration, fingerprint_recovery_stage, + initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, + plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, + read_recovery_inventory, }; pub use adapters::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionHead, @@ -183,6 +186,12 @@ pub use adapters::{ plan_retention_transition, preflight_retention_transition, prepare_retention_publication, verify_retention_closure, }; +#[cfg(feature = "repository-tasks")] +#[doc(hidden)] +pub use adapters::{ + CatalogPublicationBoundary, CatalogPublicationObserver, ObservedSegmentStage, + RepositoryInitializationStorage, SegmentStageDurabilityEvent, SegmentStageObserver, +}; pub use adapters::{ FilesystemMigrationRecoveryRefusal, FilesystemMigrationResidueKind, MIGRATION_NAMESPACE_PREFIX, StoreMigrationEffect, StoreMigrationFixedStage, StoreMigrationNamespacePrefix, @@ -190,12 +199,6 @@ pub use adapters::{ StoreMigrationRecoveryReceipt, StoreMigrationRecoveryStorage, StoreMigrationResidue, StoreMigrationStageDecodeError, plan_store_migration_recovery, recover_store_migration, }; -#[cfg(feature = "repository-tasks")] -#[doc(hidden)] -pub use adapters::{ - ObservedSegmentStage, RepositoryInitializationStorage, SegmentStageDurabilityEvent, - SegmentStageObserver, -}; pub use adapters::{ PreservedRetentionManifest, RetainedCatalogPublicationError, RetainedCatalogPublicationReceipt, RetainedCatalogPublicationStorage, RetainedClosureAdmission, RetentionEvidencePhase, diff --git a/tests/version_two_catalog_writer.rs b/tests/version_two_catalog_writer.rs new file mode 100644 index 00000000..dbe14956 --- /dev/null +++ b/tests/version_two_catalog_writer.rs @@ -0,0 +1,10 @@ +//! Version-two catalog writer and stage-recovery laws on migrated stores. +//! +//! Every law builds a real version-one store, migrates it, publishes a +//! retention root, and then drives the version-two writer and recovery tools +//! through their public production entry points. + +#![cfg(target_os = "linux")] + +#[path = "version_two_catalog_writer/suite.rs"] +mod suite; diff --git a/tests/version_two_catalog_writer/finalization_laws.rs b/tests/version_two_catalog_writer/finalization_laws.rs new file mode 100644 index 00000000..d67d0ac4 --- /dev/null +++ b/tests/version_two_catalog_writer/finalization_laws.rs @@ -0,0 +1,112 @@ +//! This module owns gated recovery finalization of a staged catalog head. +//! +//! Each law stages exactly the residue an interrupted publication leaves after +//! `head.next` synchronization: the new segment and candidate catalog in their +//! immutable pools and a complete `head.next`. Recovery then decides. + +use std::fs; +use std::path::Path; + +use keep::{ + CanonicalCatalog, CanonicalPublicationHead, CatalogGeneration, ChecksummedPublicationHead, + FilesystemCatalogSnapshot, RecoveryEntryRole, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationStorageError, + RetainedCatalogPublicationError, +}; + +use super::durable_fixture; +use super::recovery_fixture::{catalog_name, finalize, stage_roles}; +use super::writer_fixture::{ + Predecessor, TestResult, admit_all, assert_reads, first_blob, open_publisher, pool_segments, + second_blob, witness, +}; + +/// Pools a new segment for `content` and stages `head.next` over a candidate. +/// +/// Returns the exact segment bytes the staged candidate references. +fn stage_next_head( + root: &Path, + content: &[u8], + predecessor: Predecessor, +) -> TestResult>> { + let previous = pool_segments(root)?; + let publisher = open_publisher(root)?; + let staged = super::writer_fixture::stage_only(&publisher, root, content)?; + drop(publisher); + let segment = admit_all(std::slice::from_ref(&staged))?; + let name = super::writer_fixture::segment_name(segment.first().ok_or("segment absent")?)?; + fs::rename( + root.join("staging/current.seg"), + root.join("segments").join(name), + )?; + let mut candidate = match predecessor { + Predecessor::Kept => previous, + Predecessor::Dropped => Vec::new(), + }; + candidate.push(staged); + let segments = admit_all(&candidate)?; + let current = FilesystemCatalogSnapshot::load(root, durable_fixture::policy()?)?; + let catalog = CanonicalCatalog::from_segments( + CatalogGeneration::new(2)?, + Some(current.catalog_digest()), + &segments, + )?; + let head = CanonicalPublicationHead::for_catalog(catalog.checksummed()); + let checked = ChecksummedPublicationHead::decode(head.encoded())?; + fs::write( + root.join("catalogs").join(catalog_name(&checked)?), + catalog.encoded(), + )?; + fs::write(root.join("head.next"), head.encoded())?; + Ok(candidate) +} + +#[test] +fn recovery_finalizes_a_staged_head_that_preserves_retained_content() -> TestResult<()> { + let first = first_blob(); + let sandbox = durable_fixture::build("v2-finalize-preserving", &[&first])?; + let root = sandbox.path(); + let candidate = stage_next_head(root, &second_blob(), Predecessor::Kept)?; + assert_eq!(stage_roles(root)?, [RecoveryEntryRole::NextHeadStage]); + // A retained head stage does not stop readers: they select only HEAD. + assert_reads(root, &first)?; + + let receipt = finalize(root, &candidate)??; + assert_eq!( + receipt.outcome(), + RecoveryNextHeadFinalizationOutcome::Finalized + ); + assert_eq!(receipt.target().generation().get(), 2); + assert!(stage_roles(root)?.is_empty()); + assert_reads(root, &first) +} + +#[test] +fn recovery_refuses_a_staged_head_that_would_lose_retained_content() -> TestResult<()> { + let first = first_blob(); + let sandbox = durable_fixture::build("v2-finalize-dropping", &[&first])?; + let root = sandbox.path(); + let candidate = stage_next_head(root, &second_blob(), Predecessor::Dropped)?; + let before = witness(root)?; + + let outcome = finalize(root, &candidate)?; + let Err(RecoveryNextHeadFinalizationError::Verify { source, .. }) = outcome else { + return Err( + format!("a head that loses retained content was finalized: {outcome:?}").into(), + ); + }; + let RecoveryNextHeadFinalizationStorageError::RetainedClosure { source } = *source else { + return Err(format!("finalization refused for another reason: {source:?}").into()); + }; + assert!(matches!( + *source, + RetainedCatalogPublicationError::Closure { .. } + )); + assert_eq!( + witness(root)?, + before, + "refusal must leave its evidence intact" + ); + assert_eq!(stage_roles(root)?, [RecoveryEntryRole::NextHeadStage]); + assert_reads(root, &first) +} diff --git a/tests/version_two_catalog_writer/forward_laws.rs b/tests/version_two_catalog_writer/forward_laws.rs new file mode 100644 index 00000000..0dc36853 --- /dev/null +++ b/tests/version_two_catalog_writer/forward_laws.rs @@ -0,0 +1,64 @@ +//! This module owns forward version-two ingestion laws. + +use keep::{CatalogPublicationOutcome, DurableStore, ReaderAttemptLimit}; + +use super::durable_fixture; +use super::writer_fixture::{ + Predecessor, TestResult, assert_reads, first_blob, open_publisher, publish, retain, + second_blob, witness, +}; + +#[test] +fn a_successor_that_preserves_retained_content_publishes_and_reads_back() -> TestResult<()> { + let first = first_blob(); + let second = second_blob(); + let sandbox = durable_fixture::build("v2-writer-forward", &[&first])?; + let root = sandbox.path(); + let mut publisher = open_publisher(root)?; + let receipt = publish(&mut publisher, root, &second, Predecessor::Kept)??; + drop(publisher); + + assert_eq!(receipt.generation().get(), 2); + assert_eq!(receipt.outcome(), CatalogPublicationOutcome::Published); + let preserved = receipt + .preserved_manifest() + .ok_or("receipt must bind the preserved retention manifest")?; + assert_eq!(preserved.generation().get(), 1); + assert_eq!(preserved.root_count(), 1); + let staging = witness(&root.join("staging"))?; + assert!( + staging.is_empty(), + "publication left stage residue: {staging:?}" + ); + + // Every retained root still verifies against the successor catalog. + assert_reads(root, &first)?; + retain(root, b"second", &second)?; + assert_reads(root, &first)?; + assert_reads(root, &second) +} + +#[test] +fn publication_runs_beside_a_pinned_reader_view() -> TestResult<()> { + let first = first_blob(); + let second = second_blob(); + let sandbox = durable_fixture::build("v2-writer-reader", &[&first])?; + let root = sandbox.path(); + let store = DurableStore::open( + root, + durable_fixture::policy()?, + ReaderAttemptLimit::DEFAULT, + )?; + let view = store.snapshot()?; + + let mut publisher = open_publisher(root)?; + let _receipt = publish(&mut publisher, root, &second, Predecessor::Kept)??; + drop(publisher); + + let target = durable_fixture::identify(&first)?.target; + let mut output = Vec::new(); + let _receipt = view.reconstruct(target, &mut output)?; + assert_eq!(output, first, "the pinned view must keep its generation"); + drop(view); + assert_reads(root, &first) +} diff --git a/tests/version_two_catalog_writer/recovery_fixture.rs b/tests/version_two_catalog_writer/recovery_fixture.rs new file mode 100644 index 00000000..5ef56d9d --- /dev/null +++ b/tests/version_two_catalog_writer/recovery_fixture.rs @@ -0,0 +1,128 @@ +//! This module owns version-two stage recovery through the public recovery tools. +//! +//! Every helper reopens the store through version-two recovery admission, +//! fingerprints the fixed stage under the pinned inventory, admits exactly the +//! fingerprinted bytes, and executes one planned transition. + +use std::fmt::Write as _; +use std::fs; +use std::path::Path; + +use keep::{ + CatalogPublicationExpectation, ChecksummedCatalog, ChecksummedPublicationHead, + FilesystemCatalogSnapshot, FilesystemRecoveryInventoryReader, + FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageCompleter, + FilesystemRecoveryStageDiscarder, FilesystemVersionTwoRecoveryAdmission, RecoveryEntryRole, + RecoveryInventoryLimit, RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationReceipt, + RecoveryStage, RecoveryStageCompletionReceipt, RecoveryStageDiscardReceipt, + admit_recovery_stage_bytes, assess_recovery_stage, classify_version_two_recovery_names, + execute_recovery_next_head_finalization, execute_recovery_stage_completion, + execute_recovery_stage_discard, plan_recovery_next_head_finalization, + plan_recovery_stage_completion, plan_recovery_stage_discard, +}; + +use super::durable_fixture; +use super::writer_fixture::{TestResult, admit_all, segment_policy}; + +/// Classifies a fresh version-two inventory and returns every stage role. +pub(super) fn stage_roles(root: &Path) -> TestResult> { + let mut reader = FilesystemRecoveryInventoryReader::open_version_two(root)?; + let inventory = reader.read(RecoveryInventoryLimit::new(4096)?)?; + let manifest = classify_version_two_recovery_names(inventory)?; + Ok(manifest + .entries() + .iter() + .map(keep::RecoveryNamedEntry::role) + .filter(|role| { + matches!( + role, + RecoveryEntryRole::SegmentStage + | RecoveryEntryRole::CatalogStage + | RecoveryEntryRole::NextHeadStage + ) + }) + .collect()) +} + +/// Links one complete segment or catalog stage into its immutable pool. +pub(super) fn complete( + root: &Path, + stage: RecoveryStage, +) -> TestResult { + let reader = FilesystemRecoveryInventoryReader::open_version_two(root)?; + let evidence = reader.fingerprint_stage(stage)?; + let bytes = fs::read(stage_path(root, stage))?; + let admitted = admit_recovery_stage_bytes(stage, evidence, &bytes)?; + let assessment = assess_recovery_stage(&admitted, segment_policy())?; + let request = plan_recovery_stage_completion(&assessment)?; + let admission = FilesystemVersionTwoRecoveryAdmission::reopen(root)?; + let mut completer = FilesystemRecoveryStageCompleter::open_version_two(admission)?; + Ok(execute_recovery_stage_completion(&mut completer, request)?) +} + +/// Removes one exactly fingerprinted incomplete stage. +pub(super) fn discard( + root: &Path, + stage: RecoveryStage, +) -> TestResult { + let reader = FilesystemRecoveryInventoryReader::open_version_two(root)?; + let evidence = reader.fingerprint_stage(stage)?; + let bytes = fs::read(stage_path(root, stage))?; + let admitted = admit_recovery_stage_bytes(stage, evidence, &bytes)?; + let assessment = assess_recovery_stage(&admitted, segment_policy())?; + let request = plan_recovery_stage_discard(&assessment)?; + let admission = FilesystemVersionTwoRecoveryAdmission::reopen(root)?; + let mut discarder = FilesystemRecoveryStageDiscarder::open_version_two(admission)?; + Ok(execute_recovery_stage_discard(&mut discarder, request)?) +} + +/// Plans and executes gated finalization of the complete `head.next`. +/// +/// `candidate_segments` are the exact segments the staged catalog references. +/// The outer result carries fixture failures; the inner result is the +/// finalization outcome. +pub(super) fn finalize( + root: &Path, + candidate_segments: &[Vec], +) -> TestResult> { + let reader = FilesystemRecoveryInventoryReader::open_version_two(root)?; + let evidence = reader.fingerprint_stage(RecoveryStage::NextHead)?; + drop(reader); + let head_bytes = fs::read(root.join("head.next"))?; + let head = ChecksummedPublicationHead::decode(&head_bytes)?; + let catalog_bytes = fs::read(root.join("catalogs").join(catalog_name(&head)?))?; + let segments = admit_all(candidate_segments)?; + let candidate = head.admit(ChecksummedCatalog::decode(&catalog_bytes)?.admit(&segments)?)?; + let admitted = admit_recovery_stage_bytes(RecoveryStage::NextHead, evidence, &head_bytes)?; + let assessment = assess_recovery_stage(&admitted, segment_policy())?; + let current = FilesystemCatalogSnapshot::load(root, durable_fixture::policy()?)?; + let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); + let request = plan_recovery_next_head_finalization(&assessment, &candidate, expectation)?; + let admission = FilesystemVersionTwoRecoveryAdmission::reopen(root)?; + let mut finalizer = FilesystemRecoveryNextHeadFinalizer::open_version_two( + admission, + durable_fixture::policy()?, + )?; + Ok(execute_recovery_next_head_finalization( + &mut finalizer, + request, + )) +} + +/// The canonical catalog-pool name a publication head selects. +pub(super) fn catalog_name(head: &ChecksummedPublicationHead<'_>) -> TestResult { + let mut name = format!("{:016x}-", head.generation().get()); + for byte in head.catalog_digest().as_bytes() { + write!(name, "{byte:02x}")?; + } + name.push_str(".cat"); + Ok(name) +} + +fn stage_path(root: &Path, stage: RecoveryStage) -> std::path::PathBuf { + match stage { + RecoveryStage::Segment => root.join("staging/current.seg"), + RecoveryStage::Catalog => root.join("staging/current.cat"), + RecoveryStage::NextHead => root.join("head.next"), + } +} diff --git a/tests/version_two_catalog_writer/refusal_laws.rs b/tests/version_two_catalog_writer/refusal_laws.rs new file mode 100644 index 00000000..e2f6ea9a --- /dev/null +++ b/tests/version_two_catalog_writer/refusal_laws.rs @@ -0,0 +1,96 @@ +//! This module owns version-two writer refusals that must precede every mutation. + +use std::fs; +use std::io; +use std::path::Path; + +use keep::{RetainedCatalogPublicationError, RetentionCurrentStateRefusal, RetentionEvidencePhase}; + +use super::durable_fixture; +use super::writer_fixture::{ + Predecessor, StoreWitness, TestResult, assert_reads, first_blob, open_publisher, publish, + second_blob, witness, +}; + +/// Requires that only the caller's sealed segment stage was added. +fn assert_only_caller_stage_added(before: &StoreWitness, root: &Path) -> TestResult<()> { + let mut after = witness(root)?; + let position = after + .iter() + .position(|(path, _)| path == "staging/current.seg") + .ok_or("the caller's sealed segment stage is absent")?; + after.remove(position); + assert_eq!(before, &after, "refusal mutated the store"); + Ok(()) +} + +fn current_state_refusal(source: &io::Error) -> Option<&RetentionCurrentStateRefusal> { + source + .get_ref() + .and_then(|inner| inner.downcast_ref::()) +} + +#[test] +fn a_candidate_that_drops_retained_content_refuses_before_any_mutation() -> TestResult<()> { + let first = first_blob(); + let sandbox = durable_fixture::build("v2-writer-dropped", &[&first])?; + let root = sandbox.path(); + let before = witness(root)?; + let mut publisher = open_publisher(root)?; + let outcome = publish(&mut publisher, root, &second_blob(), Predecessor::Dropped)?; + drop(publisher); + + let Err(RetainedCatalogPublicationError::Closure { entry, .. }) = outcome else { + return Err(format!("dropping retained content was not refused: {outcome:?}").into()); + }; + assert_eq!(entry.root_generation().get(), 1); + assert_only_caller_stage_added(&before, root)?; + assert_reads(root, &first) +} + +#[test] +fn absent_retention_head_contradicted_by_pool_entries_refuses() -> TestResult<()> { + let first = first_blob(); + let sandbox = durable_fixture::build("v2-writer-headless", &[&first])?; + let root = sandbox.path(); + fs::remove_file(root.join("retention/HEAD"))?; + let before = witness(root)?; + let mut publisher = open_publisher(root)?; + let outcome = publish(&mut publisher, root, &second_blob(), Predecessor::Dropped)?; + drop(publisher); + + let Err(RetainedCatalogPublicationError::RetentionStorage { phase, source }) = outcome else { + return Err(format!("contradicted head absence was admitted: {outcome:?}").into()); + }; + assert_eq!(phase, RetentionEvidencePhase::AdmitHead); + assert!(matches!( + current_state_refusal(&source), + Some(RetentionCurrentStateRefusal::HeadAbsentWithArtifacts) + )); + assert_only_caller_stage_added(&before, root) +} + +#[test] +fn a_fixed_retention_stage_requires_retention_recovery_first() -> TestResult<()> { + let first = first_blob(); + let sandbox = durable_fixture::build("v2-writer-retention-stage", &[&first])?; + let root = sandbox.path(); + fs::write( + root.join("retention/root.next"), + b"interrupted retention stage", + )?; + let before = witness(root)?; + let mut publisher = open_publisher(root)?; + let outcome = publish(&mut publisher, root, &second_blob(), Predecessor::Kept)?; + drop(publisher); + + let Err(RetainedCatalogPublicationError::RetentionStorage { phase, source }) = outcome else { + return Err(format!("a retained retention stage was ignored: {outcome:?}").into()); + }; + assert_eq!(phase, RetentionEvidencePhase::AdmitHead); + assert!(matches!( + current_state_refusal(&source), + Some(RetentionCurrentStateRefusal::RetainedStage) + )); + assert_only_caller_stage_added(&before, root) +} diff --git a/tests/version_two_catalog_writer/stage_recovery_laws.rs b/tests/version_two_catalog_writer/stage_recovery_laws.rs new file mode 100644 index 00000000..d40528f2 --- /dev/null +++ b/tests/version_two_catalog_writer/stage_recovery_laws.rs @@ -0,0 +1,52 @@ +//! This module owns version-two segment and catalog stage recovery laws. + +use std::fs; + +use keep::{RecoveryEntryRole, RecoveryStage, RecoveryStageDiscardOutcome}; + +use super::durable_fixture; +use super::recovery_fixture::{complete, discard, stage_roles}; +use super::writer_fixture::{ + Predecessor, TestResult, assert_reads, first_blob, open_publisher, publish, republish, retain, + second_blob, +}; + +#[test] +fn a_refused_candidate_leaves_a_segment_stage_recovery_completes() -> TestResult<()> { + let first = first_blob(); + let second = second_blob(); + let sandbox = durable_fixture::build("v2-recover-refused-stage", &[&first])?; + let root = sandbox.path(); + let mut publisher = open_publisher(root)?; + let refused = publish(&mut publisher, root, &second, Predecessor::Dropped)?; + drop(publisher); + assert!(refused.is_err(), "a dropping candidate published"); + assert_eq!(stage_roles(root)?, [RecoveryEntryRole::SegmentStage]); + + let _receipt = complete(root, RecoveryStage::Segment)?; + assert!(stage_roles(root)?.is_empty()); + let mut publisher = open_publisher(root)?; + let _published = republish(&mut publisher, root)??; + drop(publisher); + retain(root, b"second", &second)?; + assert_reads(root, &first)?; + assert_reads(root, &second) +} + +#[test] +fn an_incomplete_catalog_stage_is_discarded_by_version_two_recovery() -> TestResult<()> { + let first = first_blob(); + let sandbox = durable_fixture::build("v2-recover-partial-catalog", &[&first])?; + let root = sandbox.path(); + fs::write(root.join("staging/current.cat"), b"KEEP")?; + assert_eq!(stage_roles(root)?, [RecoveryEntryRole::CatalogStage]); + assert_reads(root, &first)?; + + let receipt = discard(root, RecoveryStage::Catalog)?; + assert_eq!(receipt.outcome(), RecoveryStageDiscardOutcome::Removed); + assert!(stage_roles(root)?.is_empty()); + let mut publisher = open_publisher(root)?; + let _published = republish(&mut publisher, root)??; + drop(publisher); + assert_reads(root, &first) +} diff --git a/tests/version_two_catalog_writer/suite.rs b/tests/version_two_catalog_writer/suite.rs new file mode 100644 index 00000000..1b8e65c0 --- /dev/null +++ b/tests/version_two_catalog_writer/suite.rs @@ -0,0 +1,28 @@ +//! This module owns the version-two writer suite's fixture imports. + +#[path = "../golden_file_worldline/durable_fixture.rs"] +#[allow( + dead_code, + reason = "writer laws use complete stores; golden laws cover partial-record construction" +)] +mod durable_fixture; +#[path = "../segment_filesystem_stage/sandbox.rs"] +#[allow( + dead_code, + reason = "writer laws use Drop cleanup; other filesystem laws check explicit removal" +)] +mod durable_sandbox; +#[path = "finalization_laws.rs"] +mod finalization_laws; +#[path = "forward_laws.rs"] +mod forward_laws; +#[path = "recovery_fixture.rs"] +mod recovery_fixture; +#[path = "refusal_laws.rs"] +mod refusal_laws; +#[path = "stage_recovery_laws.rs"] +mod stage_recovery_laws; +#[path = "version_laws.rs"] +mod version_laws; +#[path = "writer_fixture.rs"] +mod writer_fixture; diff --git a/tests/version_two_catalog_writer/version_laws.rs b/tests/version_two_catalog_writer/version_laws.rs new file mode 100644 index 00000000..6e9135d4 --- /dev/null +++ b/tests/version_two_catalog_writer/version_laws.rs @@ -0,0 +1,141 @@ +//! This module owns typed refusal of cross-version writer and recovery entry. + +use std::io; + +use keep::{ + AdmittedSegment, AdmittedSegmentRecord, FilesystemCatalogPublisher, + FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, + FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageDiscardOpenError, + FilesystemRecoveryStageDiscarder, FilesystemVersionTwoAdmission, + FilesystemVersionTwoRecoveryAdmission, RecoveryEntryRole, RecoveryInventoryError, + RecoveryInventoryLimit, RecoveryNameClassificationError, RecoveryNamespace, + RecoveryRequiredEntry, SegmentPublicationError, SegmentRecordLimit, StagedSegment, + classify_recovery_names, classify_version_two_recovery_names, +}; + +use super::durable_fixture; +use super::durable_sandbox::TestDirectory; +use super::writer_fixture::{TestResult, first_blob, open_publisher, segment_policy}; + +fn limit() -> TestResult { + Ok(RecoveryInventoryLimit::new(4096)?) +} + +#[test] +fn version_one_entry_points_refuse_a_version_two_root() -> TestResult<()> { + let sandbox = durable_fixture::build("v2-versions-refuse-v1", &[&first_blob()])?; + let root = sandbox.path(); + let policy = durable_fixture::policy()?; + + assert!(matches!( + FilesystemPlatformAdmission::reopen(root), + Err(FilesystemPlatformAdmissionError::Namespace { .. }) + )); + assert!(matches!( + FilesystemRecoveryInventoryReader::open(root), + Err(RecoveryInventoryError::Io { + namespace: RecoveryNamespace::Root, + .. + }) + )); + assert!(matches!( + FilesystemRecoveryStageDiscarder::open(root), + Err(FilesystemRecoveryStageDiscardOpenError::Namespace { .. }) + )); + assert!(matches!( + FilesystemRecoveryNextHeadFinalizer::open(root, policy), + Err(FilesystemRecoveryNextHeadFinalizationOpenError::Namespace { .. }) + )); + + let inventory = FilesystemRecoveryInventoryReader::open_version_two(root)?.read(limit()?)?; + assert!(matches!( + classify_recovery_names(inventory.clone()), + Err(RecoveryNameClassificationError::Unexpected { + namespace: RecoveryNamespace::Root, + .. + }) + )); + let manifest = classify_version_two_recovery_names(inventory)?; + for role in [ + RecoveryEntryRole::ReaderFence, + RecoveryEntryRole::FormatMarker, + RecoveryEntryRole::MigrationIntent, + RecoveryEntryRole::MigrationReceipt, + RecoveryEntryRole::RetentionDirectory, + RecoveryEntryRole::GcDirectory, + RecoveryEntryRole::RecoveryDirectory, + RecoveryEntryRole::CurrentHead, + ] { + assert!( + manifest.entries().iter().any(|entry| entry.role() == role), + "version-two root entry {role:?} was not classified as known" + ); + } + Ok(()) +} + +#[test] +fn version_two_entry_points_refuse_a_version_one_root() -> TestResult<()> { + let sandbox = TestDirectory::create("v2-versions-refuse-v2")?; + let root = sandbox.path(); + drop(FilesystemPlatformAdmission::initialize(root)?); + let missing = |error: &FilesystemPlatformAdmissionError| { + matches!( + error, + FilesystemPlatformAdmissionError::Namespace { source } + if source.kind() == io::ErrorKind::NotFound + ) + }; + + let Err(writer) = FilesystemVersionTwoAdmission::reopen(root) else { + return Err("version-two writer admission admitted a version-one root".into()); + }; + assert!(missing(&writer), "unexpected refusal: {writer:?}"); + let Err(recovery) = FilesystemVersionTwoRecoveryAdmission::reopen(root) else { + return Err("version-two recovery admission admitted a version-one root".into()); + }; + assert!(missing(&recovery), "unexpected refusal: {recovery:?}"); + let Err(RecoveryInventoryError::Io { + namespace: RecoveryNamespace::Root, + source, + .. + }) = FilesystemRecoveryInventoryReader::open_version_two(root) + else { + return Err("version-two inventory admitted a version-one root".into()); + }; + let admission = source + .get_ref() + .and_then(|inner| inner.downcast_ref::()) + .ok_or("inventory refusal lost its admission source")?; + assert!(missing(admission), "unexpected refusal: {admission:?}"); + + let inventory = FilesystemRecoveryInventoryReader::open(root)?.read(limit()?)?; + assert!(matches!( + classify_version_two_recovery_names(inventory), + Err(RecoveryNameClassificationError::Missing { + required: RecoveryRequiredEntry::CurrentHead + }) + )); + Ok(()) +} + +#[test] +fn a_version_one_segment_stage_cannot_be_selected_by_the_version_two_writer() -> TestResult<()> { + let version_one = TestDirectory::create("v2-versions-stage-v1")?; + let admission = FilesystemPlatformAdmission::initialize(version_one.path())?; + let legacy = FilesystemCatalogPublisher::open(admission, durable_fixture::policy()?)?; + let sealed = StagedSegment::begin(legacy.create_segment_stage()?, SegmentRecordLimit::MAXIMUM)? + .append(AdmittedSegmentRecord::for_chunk(b"version-one stage")?)? + .seal()?; + let bytes = std::fs::read(version_one.path().join("staging/current.seg"))?; + let segment = AdmittedSegment::decode(&bytes, segment_policy())?; + + let version_two = durable_fixture::build("v2-versions-stage-v2", &[&first_blob()])?; + let publisher = open_publisher(version_two.path())?; + assert!(matches!( + publisher.select_segment(sealed, &segment), + Err(SegmentPublicationError::PublisherAuthority) + )); + Ok(()) +} diff --git a/tests/version_two_catalog_writer/writer_fixture.rs b/tests/version_two_catalog_writer/writer_fixture.rs new file mode 100644 index 00000000..adb1402c --- /dev/null +++ b/tests/version_two_catalog_writer/writer_fixture.rs @@ -0,0 +1,274 @@ +//! This module owns one blob's ingestion through the version-two catalog writer. +//! +//! The library's interruption laws include this file by path, so it names the +//! crate as `keep` and reaches the shared store fixture through `super`. + +use std::collections::BTreeSet; +use std::error::Error; +use std::fmt::Write as _; +use std::fs; +use std::path::Path; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, AdmittedSegment, AdmittedSegmentRecord, + CanonicalCatalog, CanonicalRetentionRoot, CatalogGeneration, CatalogPublicationExpectation, + DurableStore, FilesystemCatalogSnapshot, FilesystemRetentionPublicationAuthority, + FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, + ReaderAttemptLimit, RegisteredRetentionProfile, RetainedCatalogPublicationError, + RetainedCatalogPublicationReceipt, RetentionAnchor, RetentionClosureLimits, + RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionRoot, + RootGeneration, SegmentPublication, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, + execute_retention_publication, preflight_retention_transition, prepare_retention_publication, + publish_retained_catalog_generation, +}; + +use super::durable_fixture::{self, Identified}; + +pub(super) type TestResult = Result>; + +/// The retained content every migrated fixture store starts with. +pub(super) fn first_blob() -> Vec { + b"keep version-two writer: first retained blob. ".repeat(3_000) +} + +/// The content each law ingests through the version-two writer. +pub(super) fn second_blob() -> Vec { + b"KEEP V2 WRITER -- the second blob, ingested after migration | ".repeat(2_500) +} + +/// Whether a candidate catalog keeps every predecessor segment. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum Predecessor { + /// The candidate references every current segment and the new one. + Kept, + /// The candidate references only the new segment. + Dropped, +} + +pub(super) const fn segment_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +/// Opens the version-two writer over a migrated store. +pub(super) fn open_publisher(root: &Path) -> TestResult { + let admission = FilesystemVersionTwoAdmission::reopen(root)?; + Ok(FilesystemVersionTwoCatalogPublisher::open( + admission, + durable_fixture::policy()?, + )?) +} + +/// Reads every immutable segment in deterministic name order. +pub(super) fn pool_segments(root: &Path) -> TestResult>> { + let mut names = fs::read_dir(root.join("segments"))? + .map(|entry| entry.map(|entry| entry.file_name())) + .collect::, _>>()?; + names.sort(); + names + .iter() + .map(|name| Ok(fs::read(root.join("segments").join(name))?)) + .collect() +} + +/// Stages `content` as one new segment and publishes the next catalog generation. +/// +/// The candidate references the current segments when `predecessor` is +/// [`Predecessor::Kept`]. The outer result carries fixture failures; the inner +/// result is the gate's exact outcome. +pub(super) fn publish( + publisher: &mut FilesystemVersionTwoCatalogPublisher, + root: &Path, + content: &[u8], + predecessor: Predecessor, +) -> TestResult> { + let identified = durable_fixture::identify(content)?; + let current = FilesystemCatalogSnapshot::load(root, durable_fixture::policy()?)?; + let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); + let mut encoded = match predecessor { + Predecessor::Kept => pool_segments(root)?, + Predecessor::Dropped => Vec::new(), + }; + let sealed = stage_records(publisher, content, &identified)?; + encoded.push(fs::read(root.join("staging/current.seg"))?); + let segments = admit_all(&encoded)?; + let staged = segments.last().ok_or("staged segment absent")?; + let selection = publisher.select_segment(sealed, staged)?; + let generation = CatalogGeneration::new( + current + .generation() + .get() + .checked_add(1) + .ok_or("generation overflow")?, + )?; + let catalog = + CanonicalCatalog::from_segments(generation, Some(current.catalog_digest()), &segments)?; + Ok(publish_retained_catalog_generation( + publisher, + expectation, + selection, + &catalog, + &segments, + )) +} + +/// Publishes the next catalog generation over already pooled segments. +pub(super) fn republish( + publisher: &mut FilesystemVersionTwoCatalogPublisher, + root: &Path, +) -> TestResult> { + let current = FilesystemCatalogSnapshot::load(root, durable_fixture::policy()?)?; + let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); + let encoded = pool_segments(root)?; + let segments = admit_all(&encoded)?; + let generation = CatalogGeneration::new( + current + .generation() + .get() + .checked_add(1) + .ok_or("generation overflow")?, + )?; + let catalog = + CanonicalCatalog::from_segments(generation, Some(current.catalog_digest()), &segments)?; + Ok(publish_retained_catalog_generation( + publisher, + expectation, + SegmentPublication::none(), + &catalog, + &segments, + )) +} + +pub(super) fn admit_all(encoded: &[Vec]) -> TestResult>> { + encoded + .iter() + .map(|bytes| Ok(AdmittedSegment::decode(bytes, segment_policy())?)) + .collect() +} + +fn stage_records<'publisher>( + publisher: &'publisher FilesystemVersionTwoCatalogPublisher, + content: &[u8], + identified: &Identified, +) -> TestResult>> { + let mut stage = StagedSegment::begin( + publisher.create_segment_stage()?, + SegmentRecordLimit::MAXIMUM, + )?; + let mut chunks = BTreeSet::new(); + for span in &identified.spans { + if chunks.insert(span.id()) { + let start = usize::try_from(span.offset().get())?; + let end = usize::try_from(span.end().get())?; + let bytes = content.get(start..end).ok_or("chunk outside source")?; + stage = stage.append(AdmittedSegmentRecord::for_chunk(bytes)?)?; + } + } + stage = stage.append(AdmittedSegmentRecord::for_layout(&identified.record)?)?; + Ok(stage.seal()?) +} + +/// Publishes a new generation-one retention root that anchors `content`. +pub(super) fn retain(root: &Path, namespace: &[u8], content: &[u8]) -> TestResult<()> { + let identified = durable_fixture::identify(content)?; + let retained = RetentionRoot::new( + RetentionNamespace::try_from(namespace)?, + RootGeneration::INITIAL, + RetentionPolicy::new( + RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + RetentionClosureLimits::new(4096, 8, 16_777_216, 67_108_864)?, + ), + None, + vec![RetentionAnchor::new( + identified.target, + identified.record.id(), + )], + )?; + let canonical = CanonicalRetentionRoot::from_root(&retained)?; + let candidate = AdmittedRetentionRoot::decode(canonical.encoded())?; + let current = FilesystemCatalogSnapshot::load(root, durable_fixture::policy()?)?; + let snapshot = current.snapshot()?; + let preflight = preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + &snapshot, + )?; + let admission = FilesystemVersionTwoAdmission::reopen(root)?; + let mut authority = FilesystemRetentionPublicationAuthority::open(admission)?; + let observed = authority.observe_current()?; + let manifest = observed + .as_ref() + .map(|state| AdmittedRetentionManifest::decode(state.manifest_bytes())) + .transpose()?; + let prepared = prepare_retention_publication(preflight, manifest.as_ref())?; + let _receipt = execute_retention_publication(&mut authority, &prepared)?; + Ok(()) +} + +/// Requires a fresh fenced view to reconstruct exactly `content`. +pub(super) fn assert_reads(root: &Path, content: &[u8]) -> TestResult<()> { + let target = durable_fixture::identify(content)?.target; + let store = DurableStore::open( + root, + durable_fixture::policy()?, + ReaderAttemptLimit::DEFAULT, + )?; + let mut output = Vec::new(); + let _receipt = store.reconstruct(target, &mut output)?; + if output == content { + Ok(()) + } else { + Err("reconstructed bytes differ from the ingested content".into()) + } +} + +/// Every store path, with the bytes of each regular file, in path order. +pub(super) type StoreWitness = Vec<(String, Option>)>; + +/// Every path and regular-file byte string under `root`, in name order. +pub(super) fn witness(root: &Path) -> TestResult { + let mut entries = Vec::new(); + let mut pending = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + for entry in fs::read_dir(&directory)? { + let path = entry?.path(); + let relative = path + .strip_prefix(root)? + .to_str() + .ok_or("non-UTF-8 store path")? + .to_owned(); + if path.is_dir() { + pending.push(path); + entries.push((relative, None)); + } else { + entries.push((relative, Some(fs::read(&path)?))); + } + } + } + entries.sort(); + Ok(entries) +} + +/// Stages and seals `content` as `staging/current.seg` without publishing it. +/// +/// Returns the exact sealed bytes. The closed stage remains for the caller. +pub(super) fn stage_only( + publisher: &FilesystemVersionTwoCatalogPublisher, + root: &Path, + content: &[u8], +) -> TestResult> { + let identified = durable_fixture::identify(content)?; + let sealed = stage_records(publisher, content, &identified)?; + drop(sealed); + Ok(fs::read(root.join("staging/current.seg"))?) +} + +/// The canonical segment-pool name of one admitted segment. +pub(super) fn segment_name(segment: &AdmittedSegment<'_>) -> TestResult { + let mut name = String::new(); + for byte in segment.digest().as_bytes() { + write!(name, "{byte:02x}")?; + } + name.push_str(".seg"); + Ok(name) +} From 6a34e3c865aa362f1130048e569c591fafa19b61 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 07:43:31 -0700 Subject: [PATCH 06/36] Test: process-death rows for the version-two writer and its recovery (#198) Add the version-two-catalog crash sequence, KEEP-CRASH-074 through 077: segment link, catalog-stage creation, head-stage synchronization and HEAD replacement of the production version-two writer, each at before, during and after. The child installs an observer on the writer's own stage type and blocks there for process death. Restart requires the exact stage residue, recovers it with the public version-two recovery tools (segment completion, catalog discard, gated head finalization), republishes where the candidate never reached HEAD, retains the ingested blob and reads both blobs through DurableStore. --- README.md | 1 + .../production_protocol.rs | 5 + .../production_protocol/retention.rs | 2 +- .../production_protocol/version_two_blob.rs | 90 ++++++ .../version_two_catalog.rs | 164 +++++++++++ xtask/src/durability_crash_matrix/restart.rs | 5 + .../restart/expectation.rs | 4 +- .../restart/version_two_catalog.rs | 266 ++++++++++++++++++ .../restart/version_two_recovery.rs | 174 ++++++++++++ xtask/src/durability_crash_point.rs | 25 +- xtask/src/durability_crash_point_identity.rs | 4 + .../tests/durability_crash_point_contract.rs | 22 ++ .../durability_crash_production_contract.rs | 2 + 13 files changed, 760 insertions(+), 4 deletions(-) create mode 100644 xtask/src/durability_crash_matrix/production_protocol/version_two_blob.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs create mode 100644 xtask/src/durability_crash_matrix/restart/version_two_catalog.rs create mode 100644 xtask/src/durability_crash_matrix/restart/version_two_recovery.rs diff --git a/README.md b/README.md index 7f75e8fe..ffd34551 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ Select a crash campaign with `--sequence NAME` using these exact CLI names: | `initialization` | Writer-locked initialization | | `retention` | Retention root, manifest, and head publication | | `migration` | Version-one to version-two migration | +| `version-two-catalog` | Version-two catalog writer and stage recovery | ```bash cargo xtask durability-crash-matrix --sequence retention diff --git a/xtask/src/durability_crash_matrix/production_protocol.rs b/xtask/src/durability_crash_matrix/production_protocol.rs index 0b47f3bc..a7ed62ea 100644 --- a/xtask/src/durability_crash_matrix/production_protocol.rs +++ b/xtask/src/durability_crash_matrix/production_protocol.rs @@ -13,6 +13,8 @@ mod recovery_storage; pub(super) mod retention; mod retention_storage; mod segment_stage; +pub(super) mod version_two_blob; +pub(super) mod version_two_catalog; use std::error::Error; use std::fs; @@ -51,6 +53,9 @@ pub(super) fn run( DurabilityCrashSequence::Migration => { migration::run(&store_root, &mut control)?; } + DurabilityCrashSequence::VersionTwoCatalog => { + version_two_catalog::run(&store_root, control)?; + } } Err(DurabilityCrashMatrixError::PointSequenceMismatch { point: case.point(), diff --git a/xtask/src/durability_crash_matrix/production_protocol/retention.rs b/xtask/src/durability_crash_matrix/production_protocol/retention.rs index 486f54ba..fa3eb9ee 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/retention.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/retention.rs @@ -34,7 +34,7 @@ pub(super) fn run( /// Initializes, populates, and migrates the bundle store, then reopens it as /// version two and returns retention authority over it. -fn migrated_authority( +pub(super) fn migrated_authority( store_root: &Path, ) -> Result { let lock = initialization::initialized_lock(store_root)?; diff --git a/xtask/src/durability_crash_matrix/production_protocol/version_two_blob.rs b/xtask/src/durability_crash_matrix/production_protocol/version_two_blob.rs new file mode 100644 index 00000000..6dd9048e --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/version_two_blob.rs @@ -0,0 +1,90 @@ +//! This module owns the one blob the version-two crash campaign ingests. +//! +//! The child stages exactly these records, and restart reconstructs exactly +//! these bytes, so both sides derive the blob from one definition. + +use keep::{ + AdmittedLayout, AdmittedSegmentRecord, BlobId, CanonicalLayoutRecord, ChunkSpan, FastCdc, + LayoutEntryLimit, RegisteredStorageProfile, RetentionAnchor, +}; + +use super::super::DurabilityCrashMatrixError; +use super::verification; + +/// The exact bytes the version-two writer ingests before process death. +pub(in crate::durability_crash_matrix) const CONTENT: &[u8] = + b"keep version-two crash campaign: one blob ingested after migration"; + +/// The canonical layout and chunk spans of [`CONTENT`]. +pub(in crate::durability_crash_matrix) struct VersionTwoBlob { + spans: Vec, + record: CanonicalLayoutRecord, + target: BlobId, +} + +impl VersionTwoBlob { + pub(in crate::durability_crash_matrix) fn identify() -> Result + { + let target = BlobId::hash_bytes(CONTENT) + .map_err(|source| verification("identify version-two crash blob", source))?; + let mut detector = FastCdc::new(); + let mut spans = Vec::new(); + detector + .feed(CONTENT, |span| spans.push(span)) + .map_err(|source| verification("chunk version-two crash blob", source))?; + if let Some(span) = detector + .finish() + .map_err(|source| verification("finish version-two crash blob", source))? + { + spans.push(span); + } + let layout = AdmittedLayout::from_spans( + target, + RegisteredStorageProfile::FAST_CDC_64K_V1, + spans.clone(), + LayoutEntryLimit::MAXIMUM, + ) + .map_err(|source| verification("admit version-two crash layout", source))?; + let record = layout + .encode_record() + .map_err(|source| verification("encode version-two crash layout", source))?; + Ok(Self { + spans, + record, + target, + }) + } + + /// The chunk records followed by the layout record, in staging order. + pub(in crate::durability_crash_matrix) fn records( + &self, + ) -> Result>, DurabilityCrashMatrixError> { + let mut records = Vec::with_capacity(self.spans.len()); + for span in &self.spans { + let start = usize::try_from(span.offset().get()) + .map_err(|source| verification("bound version-two crash chunk", source))?; + let end = usize::try_from(span.end().get()) + .map_err(|source| verification("bound version-two crash chunk", source))?; + let bytes = CONTENT + .get(start..end) + .ok_or(DurabilityCrashMatrixError::FixtureRange)?; + records.push( + AdmittedSegmentRecord::for_chunk(bytes) + .map_err(|source| verification("admit version-two crash chunk", source))?, + ); + } + records.push( + AdmittedSegmentRecord::for_layout(&self.record) + .map_err(|source| verification("admit version-two crash layout", source))?, + ); + Ok(records) + } + + pub(in crate::durability_crash_matrix) const fn target(&self) -> BlobId { + self.target + } + + pub(in crate::durability_crash_matrix) const fn anchor(&self) -> RetentionAnchor { + RetentionAnchor::new(self.target, self.record.id()) + } +} diff --git a/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs b/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs new file mode 100644 index 00000000..23afc6f3 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs @@ -0,0 +1,164 @@ +//! This module owns execution of the production version-two catalog writer. +//! +//! The child migrates the bundle store, publishes retention generation one, +//! then ingests one new blob through the version-two writer. An observer +//! installed on the writer's own stage type blocks at the selected boundary +//! for process death; no transition is reimplemented here. + +use std::fs; +use std::path::Path; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationBoundary, + CatalogPublicationExpectation, CatalogPublicationObserver, CatalogPublicationPhase, + CatalogSnapshot, FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, + SegmentRecordLimit, StagedSegment, execute_retention_publication, + publish_retained_catalog_generation, +}; +use xtask::DurabilityCrashPoint; + +use super::control::{CrashControl, DuringTiming}; +use super::fixture::GoldenFixture; +use super::version_two_blob::VersionTwoBlob; +use super::{DurabilityCrashMatrixError, initialization, retention, verification}; + +/// Ingests the version-two crash blob, dying at the selected coordinate. +pub(super) fn run( + store_root: &Path, + control: CrashControl, +) -> Result<(), DurabilityCrashMatrixError> { + retained_bundle_store(store_root)?; + let admission = + FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("reopen crash store for version-two writer", source))?; + let mut publisher = FilesystemVersionTwoCatalogPublisher::open(admission, policy()?) + .map_err(|source| verification("open version-two catalog publisher", source))?; + publisher.observe_catalog_phases_for_repository_tasks(observer(control)); + let blob = VersionTwoBlob::identify()?; + let mut stage = StagedSegment::begin( + publisher + .create_segment_stage() + .map_err(|source| verification("create version-two segment stage", source))?, + SegmentRecordLimit::MAXIMUM, + ) + .map_err(|source| verification("write version-two segment header", source))?; + for record in blob.records()? { + stage = stage + .append(record) + .map_err(|source| verification("append version-two segment record", source))?; + } + let sealed = stage + .seal() + .map_err(|source| verification("seal version-two segment", source))?; + let bundle = GoldenFixture::bundle_segment()?; + let staged = fs::read(store_root.join("staging/current.seg")) + .map_err(|source| DurabilityCrashMatrixError::io("read version-two segment", source))?; + let segments = admit_segments(&[bundle.bytes(), &staged])?; + let new_segment = segments + .last() + .ok_or(DurabilityCrashMatrixError::FixtureRange)?; + let selection = publisher + .select_segment(sealed, new_segment) + .map_err(|source| verification("select version-two segment stage", source))?; + let head = GoldenFixture::bundle_head()?; + let catalog_fixture = GoldenFixture::bundle_catalog()?; + let predecessor = segments + .get(..1) + .ok_or(DurabilityCrashMatrixError::FixtureRange)?; + let current = bundle_snapshot(head.bytes(), catalog_fixture.bytes(), predecessor)?; + let expectation = CatalogPublicationExpectation::successor_of(¤t); + let generation = CatalogGeneration::new(2) + .map_err(|source| verification("construct version-two catalog generation", source))?; + let catalog = + CanonicalCatalog::from_segments(generation, Some(current.catalog_digest()), &segments) + .map_err(|source| verification("encode version-two catalog", source))?; + publish_retained_catalog_generation(&mut publisher, expectation, selection, &catalog, &segments) + .map(|_receipt| ()) + .map_err(|source| verification("execute version-two catalog publication", source)) +} + +/// Migrates the bundle store and publishes its golden retention root. +pub(in crate::durability_crash_matrix) fn retained_bundle_store( + store_root: &Path, +) -> Result<(), DurabilityCrashMatrixError> { + let mut authority = retention::migrated_authority(store_root)?; + let root = GoldenFixture::retention_root()?; + let preparation = retention::preparation(root.bytes())?; + execute_retention_publication(&mut authority, &preparation) + .map(|_receipt| ()) + .map_err(|source| verification("publish crash retention generation one", source)) +} + +pub(in crate::durability_crash_matrix) fn policy() +-> Result { + let limit = keep::CatalogRestartByteLimit::new(1_048_576) + .map_err(|source| verification("bound version-two catalog read", source))?; + Ok(keep::CatalogRestartPolicy::new( + initialization::segment_policy(), + limit, + )) +} + +pub(in crate::durability_crash_matrix) fn admit_segments<'bytes>( + encoded: &[&'bytes [u8]], +) -> Result>, DurabilityCrashMatrixError> { + encoded + .iter() + .map(|bytes| { + AdmittedSegment::decode(bytes, initialization::segment_policy()) + .map_err(|source| verification("admit version-two crash segment", source)) + }) + .collect() +} + +fn bundle_snapshot<'bytes>( + head: &'bytes [u8], + catalog: &'bytes [u8], + segments: &'bytes [AdmittedSegment<'bytes>], +) -> Result, DurabilityCrashMatrixError> { + let catalog = keep::ChecksummedCatalog::decode(catalog) + .map_err(|source| verification("decode bundle catalog", source))? + .admit(segments) + .map_err(|source| verification("admit bundle catalog", source))?; + keep::ChecksummedPublicationHead::decode(head) + .map_err(|source| verification("decode bundle head", source))? + .admit(catalog) + .map_err(|source| verification("admit bundle snapshot", source)) +} + +/// Maps the writer's own transitions onto the four version-two crash points. +fn observer(mut control: CrashControl) -> CatalogPublicationObserver { + Box::new(move |phase, boundary| { + let Some((point, timing)) = crash_point(phase) else { + return Ok(()); + }; + match boundary { + CatalogPublicationBoundary::Before => control.before(point, timing), + CatalogPublicationBoundary::After => control.after(point, timing), + } + }) +} + +const fn crash_point( + phase: CatalogPublicationPhase, +) -> Option<(DurabilityCrashPoint, DuringTiming)> { + match phase { + CatalogPublicationPhase::LinkSegment => Some(( + DurabilityCrashPoint::VersionTwoLinkSegment, + DuringTiming::After, + )), + CatalogPublicationPhase::CreateCatalogStage => Some(( + DurabilityCrashPoint::VersionTwoCreateCatalogStage, + DuringTiming::After, + )), + CatalogPublicationPhase::SynchronizeHead => Some(( + DurabilityCrashPoint::VersionTwoSynchronizeHead, + DuringTiming::Before, + )), + CatalogPublicationPhase::ReplaceHead => Some(( + DurabilityCrashPoint::VersionTwoReplaceHead, + DuringTiming::After, + )), + _ => None, + } +} diff --git a/xtask/src/durability_crash_matrix/restart.rs b/xtask/src/durability_crash_matrix/restart.rs index 2c2a6834..eda5a0db 100644 --- a/xtask/src/durability_crash_matrix/restart.rs +++ b/xtask/src/durability_crash_matrix/restart.rs @@ -7,6 +7,8 @@ mod retention; mod retention_incomplete; mod retention_snapshot; mod semantic; +mod version_two_catalog; +mod version_two_recovery; use std::collections::BTreeSet; use std::fs; @@ -28,6 +30,9 @@ pub(super) fn verify( if case.point().sequence() == DurabilityCrashSequence::Migration { return migration::verify(store_root, case); } + if case.point().sequence() == DurabilityCrashSequence::VersionTwoCatalog { + return version_two_catalog::verify(store_root, case); + } let expected = ExpectedStoreState::for_case(case)?; let observed_paths = inventory(store_root)?; if observed_paths != expected.paths() { diff --git a/xtask/src/durability_crash_matrix/restart/expectation.rs b/xtask/src/durability_crash_matrix/restart/expectation.rs index e54c7961..32a52923 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation.rs @@ -64,7 +64,9 @@ impl ExpectedStoreState { DurabilityCrashSequence::Head => sequence::head(case), DurabilityCrashSequence::RecoveryDiscard => sequence::recovery(case), DurabilityCrashSequence::Initialization => sequence::initialization(case), - DurabilityCrashSequence::Retention | DurabilityCrashSequence::Migration => { + DurabilityCrashSequence::Retention + | DurabilityCrashSequence::Migration + | DurabilityCrashSequence::VersionTwoCatalog => { Err(DurabilityCrashMatrixError::PointSequenceMismatch { point: case.point(), }) diff --git a/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs b/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs new file mode 100644 index 00000000..c2836546 --- /dev/null +++ b/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs @@ -0,0 +1,266 @@ +//! This module owns post-process-death verification of the version-two writer. +//! +//! After the child dies at its coordinate, restart requires the exact stage +//! residue that coordinate leaves, recovers it with the public version-two +//! recovery tools, republishes when the candidate never reached `HEAD`, and +//! then reads both the migrated retained blob and the ingested blob through +//! the production durable reader. + +use std::fs; +use std::io; +use std::path::Path; + +use keep::{ + AdmittedRetentionManifest, AdmittedRetentionRoot, BlobId, CanonicalCatalog, + CanonicalRetentionRoot, CatalogGeneration, CatalogPublicationExpectation, + CatalogPublicationOutcome, DurableStore, FilesystemCatalogSnapshot, + FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, ReaderAttemptLimit, + RecoveryEntryRole, RegisteredRetentionProfile, RetentionClosureLimits, + RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionRoot, + RootGeneration, SegmentPublication, execute_retention_publication, + preflight_retention_transition, prepare_retention_publication, + publish_retained_catalog_generation, +}; +use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::super::DurabilityCrashMatrixError; +use super::super::production_protocol::fixture::GoldenFixture; +use super::super::production_protocol::retention::reopened_authority; +use super::super::production_protocol::verification; +use super::super::production_protocol::version_two_blob::{CONTENT, VersionTwoBlob}; +use super::super::production_protocol::version_two_catalog::{admit_segments, policy}; +use super::version_two_recovery as recovery; + +/// The catalog-protocol residue one coordinate leaves. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Residue { + /// A sealed `current.seg`, linked or not. + SegmentStage, + /// An empty `current.cat`. + CatalogStage, + /// A complete `head.next`. + HeadStage, + /// No stage; `HEAD` still names generation one. + Unpublished, + /// No stage; `HEAD` names the candidate. + Published, +} + +pub(super) fn verify( + store_root: &Path, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + let residue = residue(case)?; + let observed = recovery::stage_roles(store_root)?; + let expected: &[RecoveryEntryRole] = match residue { + Residue::SegmentStage => &[RecoveryEntryRole::SegmentStage], + Residue::CatalogStage => &[RecoveryEntryRole::CatalogStage], + Residue::HeadStage => &[RecoveryEntryRole::NextHeadStage], + Residue::Unpublished | Residue::Published => &[], + }; + if observed != expected { + return Err(mismatch(format!( + "{} {:?}: expected stages {expected:?}, observed {observed:?}", + case.point().identifier(), + case.position() + ))); + } + recover(store_root, residue)?; + require_generation_two(store_root)?; + retain_blob(store_root)?; + read_back(store_root) +} + +fn residue(case: DurabilityCrashCase) -> Result { + let before = case.position() == DurabilityCrashPosition::Before; + match case.point() { + DurabilityCrashPoint::VersionTwoLinkSegment => Ok(Residue::SegmentStage), + DurabilityCrashPoint::VersionTwoCreateCatalogStage if before => Ok(Residue::Unpublished), + DurabilityCrashPoint::VersionTwoCreateCatalogStage => Ok(Residue::CatalogStage), + DurabilityCrashPoint::VersionTwoSynchronizeHead => Ok(Residue::HeadStage), + DurabilityCrashPoint::VersionTwoReplaceHead if before => Ok(Residue::HeadStage), + DurabilityCrashPoint::VersionTwoReplaceHead => Ok(Residue::Published), + point => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), + } +} + +fn recover(store_root: &Path, residue: Residue) -> Result<(), DurabilityCrashMatrixError> { + match residue { + Residue::SegmentStage => { + recovery::complete_segment(store_root)?; + republish(store_root) + } + Residue::CatalogStage => { + recovery::discard_catalog(store_root)?; + republish(store_root) + } + Residue::Unpublished => republish(store_root), + Residue::HeadStage => { + let pool = pool_segments(store_root)?; + let candidate: Vec<&[u8]> = pool.iter().map(Vec::as_slice).collect(); + recovery::finalize_head(store_root, &candidate) + } + Residue::Published => Ok(()), + } +} + +/// Publishes generation two over the pooled segments through the gate. +fn republish(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let pool = pool_segments(store_root)?; + let encoded: Vec<&[u8]> = pool.iter().map(Vec::as_slice).collect(); + let segments = admit_segments(&encoded)?; + let current = FilesystemCatalogSnapshot::load(store_root, policy()?) + .map_err(|source| verification("load generation-one catalog", source))?; + let expectation = CatalogPublicationExpectation::successor_of( + ¤t + .snapshot() + .map_err(|source| verification("admit generation-one catalog", source))?, + ); + let generation = CatalogGeneration::new(2) + .map_err(|source| verification("construct republished generation", source))?; + let catalog = + CanonicalCatalog::from_segments(generation, Some(current.catalog_digest()), &segments) + .map_err(|source| verification("encode republished catalog", source))?; + let admission = + FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("reopen recovered store for publication", source))?; + let mut publisher = FilesystemVersionTwoCatalogPublisher::open(admission, policy()?) + .map_err(|source| verification("open recovered version-two publisher", source))?; + let receipt = publish_retained_catalog_generation( + &mut publisher, + expectation, + SegmentPublication::none(), + &catalog, + &segments, + ) + .map_err(|source| verification("republish recovered catalog", source))?; + if receipt.outcome() == CatalogPublicationOutcome::Published { + Ok(()) + } else { + Err(mismatch(format!( + "republication reported {:?}", + receipt.outcome() + ))) + } +} + +fn require_generation_two(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let current = FilesystemCatalogSnapshot::load(store_root, policy()?) + .map_err(|source| verification("load recovered catalog", source))?; + if current.generation().get() == 2 { + Ok(()) + } else { + Err(mismatch(format!( + "recovered HEAD names generation {}", + current.generation().get() + ))) + } +} + +/// Retains the ingested blob in its own namespace, gated by the current catalog. +fn retain_blob(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let blob = VersionTwoBlob::identify()?; + let root = RetentionRoot::new( + RetentionNamespace::try_from(&b"version-two-crash"[..]) + .map_err(|source| verification("name crash retention namespace", source))?, + RootGeneration::INITIAL, + RetentionPolicy::new( + RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + RetentionClosureLimits::new(4096, 8, 16_777_216, 67_108_864) + .map_err(|source| verification("bound crash retention closure", source))?, + ), + None, + vec![blob.anchor()], + ) + .map_err(|source| verification("construct crash retention root", source))?; + let canonical = CanonicalRetentionRoot::from_root(&root) + .map_err(|source| verification("encode crash retention root", source))?; + let candidate = AdmittedRetentionRoot::decode(canonical.encoded()) + .map_err(|source| verification("admit crash retention root", source))?; + let current = FilesystemCatalogSnapshot::load(store_root, policy()?) + .map_err(|source| verification("load recovered catalog for retention", source))?; + let snapshot = current + .snapshot() + .map_err(|source| verification("admit recovered catalog for retention", source))?; + let preflight = preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + &snapshot, + ) + .map_err(|source| verification("preflight crash blob retention", source))?; + let mut authority = reopened_authority(store_root)?; + let observed = authority + .observe_current() + .map_err(|source| verification("observe recovered retention head", source))?; + let manifest = observed + .as_ref() + .map(|state| AdmittedRetentionManifest::decode(state.manifest_bytes())) + .transpose() + .map_err(|source| verification("admit recovered retention manifest", source))?; + let prepared = prepare_retention_publication(preflight, manifest.as_ref()) + .map_err(|source| verification("prepare crash blob retention", source))?; + execute_retention_publication(&mut authority, &prepared) + .map(|_receipt| ()) + .map_err(|source| verification("publish crash blob retention", source)) +} + +/// Reads the migrated retained blob and the ingested blob through the durable reader. +fn read_back(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let store = DurableStore::open(store_root, policy()?, ReaderAttemptLimit::DEFAULT) + .map_err(|source| verification("open recovered durable store", source))?; + let golden = GoldenFixture::retention_root()?; + let retained = AdmittedRetentionRoot::decode(golden.bytes()) + .map_err(|source| verification("decode golden retention root", source))?; + for anchor in retained.root().anchors() { + let bytes = reconstruct(&store, anchor.blob_id())?; + let observed = BlobId::hash_bytes(&bytes) + .map_err(|source| verification("identify recovered retained blob", source))?; + if observed != anchor.blob_id() { + return Err(mismatch("recovered retained blob changed identity".into())); + } + } + let ingested = reconstruct(&store, VersionTwoBlob::identify()?.target())?; + if ingested == CONTENT { + Ok(()) + } else { + Err(mismatch( + "recovered ingested blob differs from its content".into(), + )) + } +} + +fn reconstruct( + store: &DurableStore, + target: BlobId, +) -> Result, DurabilityCrashMatrixError> { + let mut output = Vec::new(); + store + .reconstruct(target, &mut output) + .map(|_receipt| output) + .map_err(|source| verification("reconstruct recovered blob", source)) +} + +fn pool_segments(store_root: &Path) -> Result>, DurabilityCrashMatrixError> { + let directory = store_root.join("segments"); + let mut names = fs::read_dir(&directory) + .map_err(|source| DurabilityCrashMatrixError::io("list recovered segment pool", source))? + .map(|entry| entry.map(|entry| entry.file_name())) + .collect::, _>>() + .map_err(|source| DurabilityCrashMatrixError::io("read recovered segment pool", source))?; + names.sort(); + names + .iter() + .map(|name| { + fs::read(directory.join(name)) + .map_err(|source| DurabilityCrashMatrixError::io("read recovered segment", source)) + }) + .collect() +} + +fn mismatch(message: String) -> DurabilityCrashMatrixError { + verification( + "verify version-two crash recovery", + io::Error::other(message), + ) +} diff --git a/xtask/src/durability_crash_matrix/restart/version_two_recovery.rs b/xtask/src/durability_crash_matrix/restart/version_two_recovery.rs new file mode 100644 index 00000000..2eae37d4 --- /dev/null +++ b/xtask/src/durability_crash_matrix/restart/version_two_recovery.rs @@ -0,0 +1,174 @@ +//! This module owns version-two stage recovery after one writer process death. +//! +//! Each step reopens the store through the production version-two recovery +//! admission (without the platform probe), fingerprints the fixed stage under +//! the pinned inventory, admits exactly those bytes, and executes one planned +//! transition with the public recovery executors. + +use std::fmt::Write as _; +use std::fs; +use std::path::{Path, PathBuf}; + +use keep::{ + CatalogPublicationExpectation, ChecksummedCatalog, ChecksummedPublicationHead, + FilesystemCatalogSnapshot, FilesystemRecoveryInventoryReader, + FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageCompleter, + FilesystemRecoveryStageDiscarder, FilesystemVersionTwoRecoveryAdmission, RecoveryEntryRole, + RecoveryInventoryLimit, RecoveryStage, admit_recovery_stage_bytes, assess_recovery_stage, + classify_version_two_recovery_names, execute_recovery_next_head_finalization, + execute_recovery_stage_completion, execute_recovery_stage_discard, + plan_recovery_next_head_finalization, plan_recovery_stage_completion, + plan_recovery_stage_discard, +}; + +use super::super::DurabilityCrashMatrixError; +use super::super::production_protocol::initialization::segment_policy; +use super::super::production_protocol::verification; +use super::super::production_protocol::version_two_catalog::{admit_segments, policy}; + +/// Every fixed catalog-protocol stage a version-two inventory classifies. +pub(super) fn stage_roles( + root: &Path, +) -> Result, DurabilityCrashMatrixError> { + let mut reader = + FilesystemRecoveryInventoryReader::open_version_two_unchecked_for_repository_tasks(root) + .map_err(|source| verification("open version-two recovery inventory", source))?; + let limit = RecoveryInventoryLimit::new(4096) + .map_err(|source| verification("bound version-two recovery inventory", source))?; + let inventory = reader + .read(limit) + .map_err(|source| verification("read version-two recovery inventory", source))?; + let manifest = classify_version_two_recovery_names(inventory) + .map_err(|source| verification("classify version-two recovery names", source))?; + Ok(manifest + .entries() + .iter() + .map(keep::RecoveryNamedEntry::role) + .filter(|role| { + matches!( + role, + RecoveryEntryRole::SegmentStage + | RecoveryEntryRole::CatalogStage + | RecoveryEntryRole::NextHeadStage + ) + }) + .collect()) +} + +/// Links the complete fixed segment stage into the segment pool. +pub(super) fn complete_segment(root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let bytes = stage_bytes(root, RecoveryStage::Segment)?; + let evidence = evidence(root, RecoveryStage::Segment)?; + let admitted = admit_recovery_stage_bytes(RecoveryStage::Segment, evidence, &bytes) + .map_err(|source| verification("admit recovered segment stage", source))?; + let assessment = assess_recovery_stage(&admitted, segment_policy()) + .map_err(|source| verification("assess recovered segment stage", source))?; + let request = plan_recovery_stage_completion(&assessment) + .map_err(|source| verification("plan recovered segment completion", source))?; + let mut completer = FilesystemRecoveryStageCompleter::open_version_two(admission(root)?) + .map_err(|source| verification("open version-two stage completer", source))?; + execute_recovery_stage_completion(&mut completer, request) + .map(|_receipt| ()) + .map_err(|source| verification("complete recovered segment stage", source)) +} + +/// Removes the incomplete fixed catalog stage. +pub(super) fn discard_catalog(root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let bytes = stage_bytes(root, RecoveryStage::Catalog)?; + let evidence = evidence(root, RecoveryStage::Catalog)?; + let admitted = admit_recovery_stage_bytes(RecoveryStage::Catalog, evidence, &bytes) + .map_err(|source| verification("admit recovered catalog stage", source))?; + let assessment = assess_recovery_stage(&admitted, segment_policy()) + .map_err(|source| verification("assess recovered catalog stage", source))?; + let request = plan_recovery_stage_discard(&assessment) + .map_err(|source| verification("plan recovered catalog discard", source))?; + let mut discarder = FilesystemRecoveryStageDiscarder::open_version_two(admission(root)?) + .map_err(|source| verification("open version-two stage discarder", source))?; + execute_recovery_stage_discard(&mut discarder, request) + .map(|_receipt| ()) + .map_err(|source| verification("discard recovered catalog stage", source)) +} + +/// Finalizes the complete `head.next` through the retained-closure gate. +pub(super) fn finalize_head( + root: &Path, + candidate_segments: &[&[u8]], +) -> Result<(), DurabilityCrashMatrixError> { + let head_bytes = stage_bytes(root, RecoveryStage::NextHead)?; + let evidence = evidence(root, RecoveryStage::NextHead)?; + let head = ChecksummedPublicationHead::decode(&head_bytes) + .map_err(|source| verification("decode recovered next head", source))?; + let catalog_bytes = + fs::read(root.join("catalogs").join(catalog_name(&head)?)).map_err(|source| { + DurabilityCrashMatrixError::io("read recovered candidate catalog", source) + })?; + let segments = admit_segments(candidate_segments)?; + let catalog = ChecksummedCatalog::decode(&catalog_bytes) + .map_err(|source| verification("decode recovered candidate catalog", source))? + .admit(&segments) + .map_err(|source| verification("admit recovered candidate catalog", source))?; + let candidate = head + .admit(catalog) + .map_err(|source| verification("admit recovered candidate snapshot", source))?; + let admitted = admit_recovery_stage_bytes(RecoveryStage::NextHead, evidence, &head_bytes) + .map_err(|source| verification("admit recovered next-head stage", source))?; + let assessment = assess_recovery_stage(&admitted, segment_policy()) + .map_err(|source| verification("assess recovered next-head stage", source))?; + let current = FilesystemCatalogSnapshot::load(root, policy()?) + .map_err(|source| verification("load recovered current catalog", source))?; + let expectation = CatalogPublicationExpectation::successor_of( + ¤t + .snapshot() + .map_err(|source| verification("admit recovered current catalog", source))?, + ); + let request = plan_recovery_next_head_finalization(&assessment, &candidate, expectation) + .map_err(|source| verification("plan recovered next-head finalization", source))?; + let mut finalizer = + FilesystemRecoveryNextHeadFinalizer::open_version_two(admission(root)?, policy()?) + .map_err(|source| verification("open version-two head finalizer", source))?; + execute_recovery_next_head_finalization(&mut finalizer, request) + .map(|_receipt| ()) + .map_err(|source| verification("finalize recovered next head", source)) +} + +fn admission( + root: &Path, +) -> Result { + FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(root) + .map_err(|source| verification("admit version-two stage recovery", source)) +} + +fn evidence( + root: &Path, + stage: RecoveryStage, +) -> Result { + FilesystemRecoveryInventoryReader::open_version_two_unchecked_for_repository_tasks(root) + .map_err(|source| verification("open version-two stage inventory", source))? + .fingerprint_stage(stage) + .map_err(|source| verification("fingerprint version-two recovery stage", source)) +} + +fn stage_bytes(root: &Path, stage: RecoveryStage) -> Result, DurabilityCrashMatrixError> { + fs::read(stage_path(root, stage)) + .map_err(|source| DurabilityCrashMatrixError::io("read version-two recovery stage", source)) +} + +fn stage_path(root: &Path, stage: RecoveryStage) -> PathBuf { + match stage { + RecoveryStage::Segment => root.join("staging/current.seg"), + RecoveryStage::Catalog => root.join("staging/current.cat"), + RecoveryStage::NextHead => root.join("head.next"), + } +} + +fn catalog_name( + head: &ChecksummedPublicationHead<'_>, +) -> Result { + let mut name = format!("{:016x}-", head.generation().get()); + for byte in head.catalog_digest().as_bytes() { + write!(name, "{byte:02x}") + .map_err(|source| verification("name recovered candidate catalog", source))?; + } + name.push_str(".cat"); + Ok(name) +} diff --git a/xtask/src/durability_crash_point.rs b/xtask/src/durability_crash_point.rs index 0cd6ad04..bbcff405 100644 --- a/xtask/src/durability_crash_point.rs +++ b/xtask/src/durability_crash_point.rs @@ -17,11 +17,14 @@ pub enum DurabilityCrashSequence { Retention, /// One-way version-1 to version-2 store migration. Migration, + /// Version-two catalog publication and its stage recovery, `KEEP-CRASH-074` + /// through `077`. + VersionTwoCatalog, } impl DurabilityCrashSequence { /// Every sequence in stable protocol order. - pub const ALL: [Self; 7] = [ + pub const ALL: [Self; 8] = [ Self::Segment, Self::Catalog, Self::Head, @@ -29,6 +32,7 @@ impl DurabilityCrashSequence { Self::Initialization, Self::Retention, Self::Migration, + Self::VersionTwoCatalog, ]; /// Returns the stable identifier used by the crash-matrix command line. @@ -42,6 +46,7 @@ impl DurabilityCrashSequence { Self::Initialization => "initialization", Self::Retention => "retention", Self::Migration => "migration", + Self::VersionTwoCatalog => "version-two-catalog", } } @@ -204,11 +209,19 @@ pub enum DurabilityCrashPoint { MigrationRemoveReceiptStage, /// Synchronize the store root after receipt-stage cleanup. MigrationSynchronizeRootAfterReceiptCleanup, + /// Link the version-two writer's sealed segment into the immutable pool. + VersionTwoLinkSegment, + /// Exclusively create the version-two writer's `staging/current.cat`. + VersionTwoCreateCatalogStage, + /// Synchronize the version-two writer's complete `head.next`. + VersionTwoSynchronizeHead, + /// Replace `HEAD` after the version-two retained-closure revalidation. + VersionTwoReplaceHead, } impl DurabilityCrashPoint { /// Every crash boundary in stable protocol order. - pub const ALL: [Self; 73] = [ + pub const ALL: [Self; 77] = [ Self::CreateSegmentStage, Self::WriteSegmentHeader, Self::AppendSegmentRecord, @@ -282,6 +295,10 @@ impl DurabilityCrashPoint { Self::MigrationSynchronizeRootAfterReceipt, Self::MigrationRemoveReceiptStage, Self::MigrationSynchronizeRootAfterReceiptCleanup, + Self::VersionTwoLinkSegment, + Self::VersionTwoCreateCatalogStage, + Self::VersionTwoSynchronizeHead, + Self::VersionTwoReplaceHead, ]; /// The migration boundaries in `StoreMigrationPhase::ALL` order. @@ -401,6 +418,10 @@ impl DurabilityCrashPoint { | Self::MigrationSynchronizeRootAfterReceiptCleanup => { DurabilityCrashSequence::Migration } + Self::VersionTwoLinkSegment + | Self::VersionTwoCreateCatalogStage + | Self::VersionTwoSynchronizeHead + | Self::VersionTwoReplaceHead => DurabilityCrashSequence::VersionTwoCatalog, } } diff --git a/xtask/src/durability_crash_point_identity.rs b/xtask/src/durability_crash_point_identity.rs index 168af5a7..0316962e 100644 --- a/xtask/src/durability_crash_point_identity.rs +++ b/xtask/src/durability_crash_point_identity.rs @@ -80,6 +80,10 @@ impl DurabilityCrashPoint { Self::MigrationSynchronizeRootAfterReceipt => "KEEP-CRASH-071", Self::MigrationRemoveReceiptStage => "KEEP-CRASH-072", Self::MigrationSynchronizeRootAfterReceiptCleanup => "KEEP-CRASH-073", + Self::VersionTwoLinkSegment => "KEEP-CRASH-074", + Self::VersionTwoCreateCatalogStage => "KEEP-CRASH-075", + Self::VersionTwoSynchronizeHead => "KEEP-CRASH-076", + Self::VersionTwoReplaceHead => "KEEP-CRASH-077", } } } diff --git a/xtask/tests/durability_crash_point_contract.rs b/xtask/tests/durability_crash_point_contract.rs index 57b4c26c..f7b417a0 100644 --- a/xtask/tests/durability_crash_point_contract.rs +++ b/xtask/tests/durability_crash_point_contract.rs @@ -6,6 +6,7 @@ use xtask::{DurabilityCrashPoint, DurabilityCrashSequence}; use DurabilityCrashSequence::{ Catalog, Head, Initialization, Migration, RecoveryDiscard, Retention, Segment, + VersionTwoCatalog, }; const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ @@ -350,6 +351,26 @@ const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ "KEEP-CRASH-073", Migration, ), + ( + DurabilityCrashPoint::VersionTwoLinkSegment, + "KEEP-CRASH-074", + VersionTwoCatalog, + ), + ( + DurabilityCrashPoint::VersionTwoCreateCatalogStage, + "KEEP-CRASH-075", + VersionTwoCatalog, + ), + ( + DurabilityCrashPoint::VersionTwoSynchronizeHead, + "KEEP-CRASH-076", + VersionTwoCatalog, + ), + ( + DurabilityCrashPoint::VersionTwoReplaceHead, + "KEEP-CRASH-077", + VersionTwoCatalog, + ), ]; #[test] @@ -427,6 +448,7 @@ fn crash_matrix_cli_admits_its_stable_sequence_names() { ("initialization", Initialization), ("retention", Retention), ("migration", Migration), + ("version-two-catalog", VersionTwoCatalog), ] { assert_eq!( DurabilityCrashSequence::from_identifier(name), diff --git a/xtask/tests/durability_crash_production_contract.rs b/xtask/tests/durability_crash_production_contract.rs index eba7adcc..3d1c2277 100644 --- a/xtask/tests/durability_crash_production_contract.rs +++ b/xtask/tests/durability_crash_production_contract.rs @@ -16,6 +16,7 @@ fn crash_children_execute_every_claimed_production_protocol() -> Result<(), Box< "durability_crash_matrix/production_protocol/publication.rs", "durability_crash_matrix/production_protocol/recovery.rs", "durability_crash_matrix/production_protocol/migration.rs", + "durability_crash_matrix/production_protocol/version_two_catalog.rs", ] .into_iter() .map(|path| fs::read_to_string(source_root.join(path))) @@ -29,6 +30,7 @@ fn crash_children_execute_every_claimed_production_protocol() -> Result<(), Box< "initialize_store(", "execute_recovery_stage_discard(", "execute_store_migration(", + "publish_retained_catalog_generation(", ] { assert!( protocol.contains(required), From 9499fdd0b2185a5bbc467d5280409fb4c78140ef Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 07:47:25 -0700 Subject: [PATCH 07/36] Test: resume a reusable segment prefix under version-two authority (#198) Interrupt staging after one record, resume through FilesystemRecoverySegmentResumer::open_version_two, seal, complete the stage into the pool, republish through the gate and read both blobs. --- .../stage_recovery_laws.rs | 55 ++++++++++++++++++- .../writer_fixture.rs | 18 +++++- 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/tests/version_two_catalog_writer/stage_recovery_laws.rs b/tests/version_two_catalog_writer/stage_recovery_laws.rs index d40528f2..59614b11 100644 --- a/tests/version_two_catalog_writer/stage_recovery_laws.rs +++ b/tests/version_two_catalog_writer/stage_recovery_laws.rs @@ -2,13 +2,18 @@ use std::fs; -use keep::{RecoveryEntryRole, RecoveryStage, RecoveryStageDiscardOutcome}; +use keep::{ + FilesystemRecoveryInventoryReader, FilesystemRecoverySegmentResumer, + FilesystemVersionTwoRecoveryAdmission, RecoveryEntryRole, RecoveryStage, + RecoveryStageDiscardOutcome, SegmentRecordLimit, StagedSegment, admit_recovery_stage_bytes, + assess_recovery_stage, execute_recovery_segment_resume, plan_recovery_segment_resume, +}; use super::durable_fixture; use super::recovery_fixture::{complete, discard, stage_roles}; use super::writer_fixture::{ - Predecessor, TestResult, assert_reads, first_blob, open_publisher, publish, republish, retain, - second_blob, + Predecessor, TestResult, assert_reads, blob_records, first_blob, open_publisher, publish, + republish, retain, second_blob, segment_policy, }; #[test] @@ -50,3 +55,47 @@ fn an_incomplete_catalog_stage_is_discarded_by_version_two_recovery() -> TestRes drop(publisher); assert_reads(root, &first) } + +#[test] +fn a_reusable_segment_prefix_resumes_under_version_two_authority() -> TestResult<()> { + let first = first_blob(); + let second = second_blob(); + let sandbox = durable_fixture::build("v2-recover-resume", &[&first])?; + let root = sandbox.path(); + let identified = durable_fixture::identify(&second)?; + let mut records = blob_records(&second, &identified)?.into_iter(); + let publisher = open_publisher(root)?; + let prefix = StagedSegment::begin( + publisher.create_segment_stage()?, + SegmentRecordLimit::MAXIMUM, + )? + .append(records.next().ok_or("blob has no record")?)?; + drop(prefix); + drop(publisher); + assert_eq!(stage_roles(root)?, [RecoveryEntryRole::SegmentStage]); + + let request = resume_plan(root)?; + let admission = FilesystemVersionTwoRecoveryAdmission::reopen(root)?; + let resumer = FilesystemRecoverySegmentResumer::open_version_two(admission)?; + let mut continued = execute_recovery_segment_resume(resumer, request)?; + for record in records { + continued = continued.append(record)?; + } + drop(continued.seal()?); + let _receipt = complete(root, RecoveryStage::Segment)?; + let mut publisher = open_publisher(root)?; + let _published = republish(&mut publisher, root)??; + drop(publisher); + retain(root, b"second", &second)?; + assert_reads(root, &first)?; + assert_reads(root, &second) +} + +fn resume_plan(root: &std::path::Path) -> TestResult { + let reader = FilesystemRecoveryInventoryReader::open_version_two(root)?; + let evidence = reader.fingerprint_stage(RecoveryStage::Segment)?; + let bytes = fs::read(root.join("staging/current.seg"))?; + let admitted = admit_recovery_stage_bytes(RecoveryStage::Segment, evidence, &bytes)?; + let assessment = assess_recovery_stage(&admitted, segment_policy())?; + Ok(plan_recovery_segment_resume(&assessment, segment_policy())?) +} diff --git a/tests/version_two_catalog_writer/writer_fixture.rs b/tests/version_two_catalog_writer/writer_fixture.rs index adb1402c..ab4dcef4 100644 --- a/tests/version_two_catalog_writer/writer_fixture.rs +++ b/tests/version_two_catalog_writer/writer_fixture.rs @@ -154,17 +154,29 @@ fn stage_records<'publisher>( publisher.create_segment_stage()?, SegmentRecordLimit::MAXIMUM, )?; + for record in blob_records(content, identified)? { + stage = stage.append(record)?; + } + Ok(stage.seal()?) +} + +/// Every distinct chunk record of `content` followed by its layout record. +pub(super) fn blob_records<'content>( + content: &'content [u8], + identified: &'content Identified, +) -> TestResult>> { let mut chunks = BTreeSet::new(); + let mut records = Vec::new(); for span in &identified.spans { if chunks.insert(span.id()) { let start = usize::try_from(span.offset().get())?; let end = usize::try_from(span.end().get())?; let bytes = content.get(start..end).ok_or("chunk outside source")?; - stage = stage.append(AdmittedSegmentRecord::for_chunk(bytes)?)?; + records.push(AdmittedSegmentRecord::for_chunk(bytes)?); } } - stage = stage.append(AdmittedSegmentRecord::for_layout(&identified.record)?)?; - Ok(stage.seal()?) + records.push(AdmittedSegmentRecord::for_layout(&identified.record)?); + Ok(records) } /// Publishes a new generation-one retention root that anchors `content`. From 8300146dc2ac935b9650a2837a8edfe4267eabbf Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 07:47:39 -0700 Subject: [PATCH 08/36] Docs: specify version-two catalog publication and stage recovery (#198) Add the catalog-publication page: supported envelope (one staged segment per publication, no rollover), authority, the ten-step forward protocol, every crash state with its recovery, version-two recovery admission and classification, gated next-head finalization, typed cross-version refusal, and the KEEP-CRASH-074..077 rows with 078..080 planned. Route the version-two README, namespace, retention-publication and rationale pages to it, record the shared-stage decision, add KEEP-RETENTION-012..015 and update 011. --- CHANGELOG.md | 2 + docs/formats/segment-store-v2/README.md | 7 +- .../segment-store-v2/catalog-publication.md | 183 ++++++++++++++++++ docs/formats/segment-store-v2/rationale.md | 49 ++++- docs/formats/segment-store-v2/recovery.md | 7 +- docs/formats/segment-store-v2/requirements.md | 6 +- .../segment-store-v2/retention-publication.md | 2 +- 7 files changed, 245 insertions(+), 11 deletions(-) create mode 100644 docs/formats/segment-store-v2/catalog-publication.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed92fe9..eb251281 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ after its public API and format compatibility policies are established. ## [Unreleased] +- Version-2 stores can ingest: `FilesystemVersionTwoCatalogPublisher` stages one segment and publishes a catalog successor through the retained-closure gate, version-2 catalog-stage recovery classifies version-2 roots and finalizes a staged head only after the same gate admits it, and `KEEP-CRASH-074` through `077` cover writer process death (#198). + - Version-2 catalog publication can refuse candidates that lose retained content: `publish_retained_catalog_generation` verifies every current retained closure against the candidate before any catalog mutation and revalidates the retention head before `HEAD` replacement (#125). - Update the repository-only serde_json parser to 1.0.154 in both locked workspaces and refresh its dependency admission; the resolved transitive graph, features, license and MSRV are unchanged (#186). diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 12f17db3..f56bbd45 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -42,6 +42,9 @@ The following pages form one protocol: manifest, and retention-head rules. - [Retention publication](retention-publication.md) owns closure admission and the generation transition. +- [Catalog publication and stage recovery](catalog-publication.md) owns the + version-2 segment and catalog writer, its crash states, and version-2 + catalog-stage recovery, including gated next-head finalization. - [Retention publication recovery](retention-recovery.md) owns interrupted stages, restart publication, and retention process-death boundaries. - [Closure verification](closure.md) owns deterministic traversal, exact resource accounting, authenticated reconstruction, and closure evidence. @@ -105,11 +108,11 @@ Writer authority coordinates cooperating writers in a managed namespace; it does `FilesystemRetentionSnapshot` binds a catalog, retention head and manifest under a shared reader fence and verifies selected roots on demand. `DurableStore` composes that view with authenticated whole-blob and exact-range reads, delivered in [PR #164](https://github.com/flyingrobots/keep/pull/164); the [read contract](../../invariants/authenticated-reconstruction/README.md) records proof and allocation limits. -The candidate-catalog retained-closure gate is implemented over its storage port ([#125](https://github.com/flyingrobots/keep/issues/125)). No filesystem version-two catalog publisher uses it yet ([#198](https://github.com/flyingrobots/keep/issues/198)). Verifying a new retention root against the current catalog is a different operation. +The candidate-catalog retained-closure gate is implemented over its storage port ([#125](https://github.com/flyingrobots/keep/issues/125)). `FilesystemVersionTwoCatalogPublisher` ingests one segment and publishes a catalog successor through it, and version-two next-head finalization runs the same admission before replacing `HEAD` ([#198](https://github.com/flyingrobots/keep/issues/198)); see [catalog publication and stage recovery](catalog-publication.md). Verifying a new retention root against the current catalog is a different operation. [Durable verification reports](../../invariants/verification/README.md) are delivered in [PR #165](https://github.com/flyingrobots/keep/pull/165), with explicit subject/depth evidence and typed non-success outcomes. -Production durable ingestion ([#82](https://github.com/flyingrobots/keep/issues/82)), garbage collection and compaction ([#21](https://github.com/flyingrobots/keep/issues/21)) remain absent from main; those prepared portions of unmerged PR #107 are not delivered APIs here. +Production durable ingestion ([#82](https://github.com/flyingrobots/keep/issues/82)) builds on the version-two segment and catalog writer but is not delivered by it. Durable ingestion, garbage collection and compaction ([#21](https://github.com/flyingrobots/keep/issues/21)) remain absent from main; those prepared portions of unmerged PR #107 are not delivered APIs here. The `KEEP-CRASH-036..052` initial-publication process-death sequence includes independent recovered-reader checks; its [evidence receipt](../../testing-evidence/retention-crash-reader-oracle.md) records the assertions, calibration, and scope. diff --git a/docs/formats/segment-store-v2/catalog-publication.md b/docs/formats/segment-store-v2/catalog-publication.md new file mode 100644 index 00000000..5f06480a --- /dev/null +++ b/docs/formats/segment-store-v2/catalog-publication.md @@ -0,0 +1,183 @@ +# Catalog Publication and Stage Recovery + +This page owns segment ingestion and catalog publication on a completely +migrated `keep.segment-store/v2` store, and recovery of the catalog-protocol +stages that publication leaves. It adds no byte format: the segment, catalog, +and publication-head grammars stay the version-1 grammars, and the retention +records stay those of [Retention records](retention.md). Version-2 stores are +still created only by migration. + +## Supported envelope + +- One publication stages at most one new segment, through the writer's own + `staging/current.seg`, or none. A candidate that needs more than one new + segment is outside this envelope. Segment rollover ([#121](https://github.com/flyingrobots/keep/issues/121)) + is not claimed. +- The caller supplies the complete candidate catalog and every segment it + references. The version-1 catalog, segment, and head bounds apply unchanged. +- Retention evidence is bounded by one manifest of at most 4,096 entries and + one root at a time. Each closure is bounded by its root's admitted limits. +- The store must be a completely migrated version-2 root. Direct version-2 + initialization is still undefined. + +## Authority + +`FilesystemVersionTwoAdmission::reopen` admits the root and acquires the writer +lock. `FilesystemVersionTwoCatalogPublisher::open` consumes that admission. It +pins the store root, `staging`, `segments`, `catalogs`, `retention`, +`retention/roots`, and `retention/manifests`. Version-1 writer authority +cannot open it. + +The publisher implements only `RetainedCatalogPublicationStorage`, so it +publishes only through `publish_retained_catalog_generation`. Its stage type, +`FilesystemVersionTwoCatalogStages`, has private fields and no public +constructor. It is reachable only through `catalog_stages`, which requires the +gate's `RetainedClosureAdmission`. The ungated `publish_catalog_generation` +therefore cannot drive version-2 stages. + +The stage transitions are the version-1 implementation. One +`FilesystemCatalogStages` value runs them under either writer authority, so the +two formats cannot diverge in their segment, catalog, or head steps. + +Publication never takes the reader fence. Readers hold `reader.lock` shared +and select only `HEAD`. A retained or in-flight catalog `head.next` is admitted +and ignored by readers, so a pinned view keeps its generation while a +publication runs beside it. Only GC takes the fence exclusively. + +## Forward protocol + +Under one writer lock, one call to `publish_retained_catalog_generation`: + +1. admits the candidate catalog, its successor relationship, and its + generated head without I/O; +2. requires `retention`, `retention/roots`, and `retention/manifests` to still + be the pinned directories, refuses any fixed retention stage + (`root.next`, `manifest.next`, `head.next` under `retention`), and admits the + retention namespace and both pools; +3. reads `retention/HEAD` no-follow and bounded to 144 bytes plus one + trailing-byte probe. An absent head is reported only when both pools are + empty; otherwise the call refuses with `HeadAbsentWithArtifacts`; +4. reads the selected manifest, bounded by the head's declared length, and + every root the manifest names, bounded by the root grammar's maximum + length. The gate decodes and binds each record and verifies each root's + closure against the candidate; +5. verifies the current catalog state: no `head.next`, no `current.cat`, the + expected `HEAD`, and a `current.seg` only when this call selected it; +6. for a selected segment: links `current.seg` into `segments` without + replacement, reverifies the pool entry, synchronizes `segments`, removes + `current.seg`, and synchronizes `staging`; +7. exclusively creates `staging/current.cat`, writes, flushes, and + synchronizes the canonical catalog, links it into `catalogs` without + replacement, reverifies it, synchronizes `catalogs`, removes the stage, and + synchronizes `staging`; +8. exclusively creates `head.next`, writes, flushes, and synchronizes the + canonical head, then reopens and verifies its complete transitive view; +9. rereads `retention/HEAD` and requires it to equal the admitted head; and +10. atomically renames `head.next` over `HEAD` and synchronizes the store root. + +Steps 1 to 4 complete before the first catalog storage call. A refusal there +leaves no stage, pool entry, or head change. The caller's sealed +`current.seg`, created before the call, stays for explicit recovery. + +## Crash states and recovery + +Every state below is ordinary recovery evidence. Nothing is repaired +automatically, and recovery never removes or rewrites an immutable pool entry. + +| Interrupted at | Durable residue | Recovery | +| --- | --- | --- | +| Before step 6, including any gate refusal | the caller's `current.seg`, complete or incomplete | complete a complete stage into `segments`; discard an incomplete one; republish | +| Step 6, after the link | `current.seg` and an identical pool entry | completion verifies the existing entry and removes the stage; republish | +| After step 6, before `current.cat` exists | no stage; the new segment is pooled | republish with no segment selection | +| Step 7, `current.cat` incomplete | an incomplete `current.cat` | discard; republish | +| Step 7, `current.cat` complete | a complete `current.cat`, possibly also pooled | completion links or verifies it and removes the stage; republish | +| Step 8, `head.next` incomplete | an incomplete `head.next` | discard; republish | +| Step 8 or 9, `head.next` complete | a complete `head.next` naming a pooled catalog | gated next-head finalization | +| Step 10, after the rename | `HEAD` names the candidate | none; a byte-identical retry returns already published | + +A retained retention stage refuses publication at step 2. Retention recovery +must resolve it first. + +### Recovery admission + +`FilesystemVersionTwoRecoveryAdmission::reopen` applies the writer admission +with one difference: the root may also hold `head.next`. It requires the +platform profile, the writer lock, the exact version-2 namespace and protocol +directories, the jointly admitted marker, intent, and receipt, and the +restart-stable root identity. The recovery tools take it through their +`open_version_two` constructors: + +- `FilesystemRecoveryStageDiscarder`; +- `FilesystemRecoveryStageCompleter`; +- `FilesystemRecoverySegmentResumer`; and +- `FilesystemRecoveryNextHeadFinalizer`. + +The read-only `FilesystemRecoveryInventoryReader::open_version_two` applies +the same namespace, record, and identity laws without the writer lock. + +`classify_version_two_recovery_names` classifies `reader.lock`, `FORMAT`, +`migration.intent`, `migration.receipt`, `retention`, `gc`, and `recovery` as +known roles. It requires each of them and `HEAD`. Migration stages are unknown +entries. Version-2 admission owns the contents of the three protocol +directories, and retention recovery owns `retention`. + +Stage discard, completion, and segment resume execute the version-1 +executors unchanged. None of them changes `HEAD`. + +### Gated next-head finalization + +A version-2 finalizer retains the admitted retention directories. Before every +candidate verification that can precede `HEAD` replacement, including the one +immediately before the rename, it runs the forward gate's retained-closure +admission against the staged candidate catalog. A candidate that would lose +retained content refuses with +`RecoveryNextHeadFinalizationStorageError::RetainedClosure`. The error carries +the gate's exact refusal, and `head.next`, the pools, and `HEAD` stay +unchanged. A retained retention stage, or a retention head absent while the +pools hold artifacts, refuses in the same way. + +## Cross-version entry + +- Version-1 publisher admission, version-1 inventory, and every version-1 + recovery tool keep refusing a version-2 root through + `admit_recoverable` or the exact published namespace. +- Version-2 writer admission, version-2 recovery admission, and the version-2 + inventory refuse a version-1 root because its marker, records, and reader + fence are absent. +- `classify_recovery_names` refuses every version-2 root entry as unexpected. + `classify_version_two_recovery_names` refuses a version-1 inventory as + missing a required entry. +- A segment stage sealed under one publisher's authority, version-1 or + version-2, refuses selection by any other publisher. + +## Process-death matrix + +`cargo xtask durability-crash-matrix --sequence version-two-catalog` runs the +production version-2 writer in a child process. The child blocks at the +selected boundary through an observer on the writer's own stage type. Restart +then requires the exact stage residue, recovers it with the public version-2 +recovery tools, retains the ingested blob, and reads both blobs through +`DurableStore`. + + + +| ID | Boundary | Positions | Expected residue and recovery | Status | +| --- | --- | --- | --- | --- | +| `KEEP-CRASH-074` | segment link | before, during, after | sealed `current.seg`; completion, then republication | Implemented | +| `KEEP-CRASH-075` | catalog-stage creation | before, during, after | none and republication before; empty `current.cat`, discard, and republication otherwise | Implemented | +| `KEEP-CRASH-076` | `head.next` synchronization | before, during, after | complete `head.next`; gated finalization | Implemented | +| `KEEP-CRASH-077` | `HEAD` replacement | before, during, after | complete `head.next` and gated finalization before; generation two published otherwise | Implemented | +| `KEEP-CRASH-078` | version-2 recovery stage removal | before, during, after | the version-1 discard executor's law under version-2 admission | Planned | +| `KEEP-CRASH-079` | version-2 recovery stage completion link | before, during, after | the version-1 completion executor's law under version-2 admission | Planned | +| `KEEP-CRASH-080` | version-2 gated finalization replacement | before, during, after | finalized or retryable `head.next`; the gate reruns on retry | Planned | + + + +The implemented rows are 12 cases. CI runs them with the complete matrix in +debug and release builds. They establish recovery after process death; they do +not simulate power loss. In-process laws add an injected storage-port failure +after segment link, catalog-stage creation, and `head.next` synchronization +(`filesystem_version_two_catalog_recovery_tests`), plus hand-staged complete +`head.next` candidates that preserve or drop retained content +(`tests/version_two_catalog_writer.rs`). Other boundaries rely on the shared +version-1 stage implementation and its `KEEP-CRASH-001` to `030` evidence. diff --git a/docs/formats/segment-store-v2/rationale.md b/docs/formats/segment-store-v2/rationale.md index 9994562f..3032743c 100644 --- a/docs/formats/segment-store-v2/rationale.md +++ b/docs/formats/segment-store-v2/rationale.md @@ -173,12 +173,10 @@ Crash and refusal states: - Process death at any point leaves exactly the version-1 crash states. The gate adds only reads. -Recovery implication: today's next-head finalization admits only version-1 -writer authority, so it cannot finalize a version-2 `head.next`. When the -single-format writer moves stage recovery onto version 2 (#198), finalizing a -staged catalog head must run this same retained-closure admission against the -staged catalog. Changing the admission type that recovery accepts is not -enough. +Recovery implication: finalizing a staged catalog head must run this same +retained-closure admission against the staged catalog; changing the admission +type that recovery accepts is not enough. Version-2 next-head finalization +(#198) does exactly that, as recorded below. Rejected alternatives: @@ -192,3 +190,42 @@ Rejected alternatives: comparison sufficient. - Relying on ingestion's preserve-all-records behaviour. That is a property of one writer, not of the publication boundary. + +## Run one stage implementation under both writer authorities + +The version-2 writer (#198) needs the same segment, catalog, and head +transitions as version 1. A copy would let the two formats drift in exactly +the steps that decide durability. `FilesystemCatalogStages` therefore holds the +pinned catalog directories, the writable stages, and the stage-authority +token, and it is the only `CatalogPublicationStorage` implementation of those +transitions. `FilesystemCatalogPublisher` owns one beside its version-1 writer +lock. `FilesystemVersionTwoCatalogStages` wraps one under the version-2 writer +lock and has no public constructor. Each forwards transitions unchanged. + +The version-2 stage type is reachable only through `catalog_stages`, so only +the retained-closure gate can drive it. Repository tasks install an observer +on that type for crash injection rather than wrapping the port, because a +wrapper cannot obtain the stages without the gate's admission. + +Readers admit a retained catalog `head.next`. It is a version-1 file that +version 2 preserves, and a reader selects only `HEAD`, so refusing it would +stop readers for the whole window in which publication stages a head. Writer +admission still refuses it, so a writer never starts over unrecovered state. + +Catalog-stage recovery on version 2 reuses the version-1 executors under a +separate `FilesystemVersionTwoRecoveryAdmission`. That admission differs from +writer admission only in admitting `head.next`. Next-head finalization is the +one recovery route that changes `HEAD`, so a version-2 finalizer runs the +gate's own `admit_retained_closures` before every candidate verification that +can precede replacement, including the one immediately before the rename. +Refusal leaves `head.next` and the pools unchanged. + +Rejected alternatives: + +- Copying the stage code into a version-2 publisher. The formats would drift. +- Implementing the ungated port for the version-2 publisher behind a runtime + check. The type system would no longer prevent ungated publication. +- Letting `FilesystemVersionTwoAdmission` admit `head.next`. Writers and + retention publication would then start over unrecovered catalog state. +- Gating only the planner. Planning is pure and runs before the writer lock. + The retention head can change between planning and replacement. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index d339caae..f2404793 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -29,7 +29,10 @@ recovery/dispositions/.receipt ``` `retention/HEAD`, every fixed `.next` stage, `gc/intent`, and `gc/receipt` are -optional according to the exact state tables below. Immutable-pool coordinates +optional according to the exact state tables below. The version-1 catalog +`head.next` stays a version-1 file: writer admission refuses it as recovery +state, while readers and version-2 catalog-stage recovery admit it, as +[catalog publication and stage recovery](catalog-publication.md) specifies. Immutable-pool coordinates are data-dependent but canonically named. Every other root or protocol-directory entry is an unknown entry and unrecoverable ambiguity. Operations are capability-relative and never follow links. @@ -65,6 +68,8 @@ corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of retained writer authority. The fresh writer executes once; separate migration recovery plans and executes the lawful remaining suffix after process death. Version-1 reopen and version-1 recovery both refuse a migrated root. +Version-2 catalog-stage recovery admits it through +`FilesystemVersionTwoRecoveryAdmission`. ## Reader fence diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index d2a1a9d2..9625ad25 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -19,7 +19,11 @@ case is not evidence. | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | `ReaderFence` holds a shared kernel lock on a verified zero-length `reader.lock`; `collect_retention_view` accepts a view only when both head coordinates agree before and after loading and refuses an exhausted attempt limit (`retention_view_collector_tests`); `FilesystemRetentionSnapshot` binds the catalog snapshot, retention head, and manifest under the fence and verifies each selected root on demand while the fence is held, refusing a substituted root and a replaced fence, and two readers share the fence while an exclusive lock waits (`filesystem_retention_snapshot_tests`); Linux public snapshot process laws prove live-reader exclusion, SIGKILL fence release with persistent inode preservation, and kernel-queued new-reader exclusion until collection releases the fence (`tests/reader_fence_process.rs`; [evidence](../../testing-evidence/reader-fence-process.md)) | Implemented | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | all three-operation histories over initial publications of two namespaces, successor, release, restore, byte-identical retry and stale initial (343 histories, each in a fresh migrated store) compare fenced namespace generations, anchor sets and liveness against an operation-derived model after every step; exact typed refusals remain checked. The count describes exploration, not correctness. See [release/restore evidence](../../testing-evidence/retention-release-restore-model.md). Core architecture checks remain separate static evidence. | Implemented; release/restore model coverage added for #128 | -| `KEEP-RETENTION-011` | Catalog publication over retention state admits the current retention head, manifest, and every manifest root under writer authority; verifies every retained closure against the candidate before any catalog storage call; and refuses `HEAD` replacement if the retention head changed | port-level laws in `tests/retained_catalog_publication.rs`: dropped and substituted members refuse with the exact missing identity and no catalog stage; every manifest root is verified wherever it sorts; absent, corrupt, substituted, and contradictory head, manifest, and root evidence refuse with typed expected and observed state; a head changed, removed, or published before commit refuses after `head.next` verification without `replace_head`; an exhaustive 24-case candidate-by-root-set model and a repeat-run determinism check | Implemented at the storage port; no filesystem version-2 catalog publisher or next-head finalization uses it yet (#198) | +| `KEEP-RETENTION-011` | Catalog publication over retention state admits the current retention head, manifest, and every manifest root under writer authority; verifies every retained closure against the candidate before any catalog storage call; and refuses `HEAD` replacement if the retention head changed | port-level laws in `tests/retained_catalog_publication.rs`: dropped and substituted members refuse with the exact missing identity and no catalog stage; every manifest root is verified wherever it sorts; absent, corrupt, substituted, and contradictory head, manifest, and root evidence refuse with typed expected and observed state; a head changed, removed, or published before commit refuses after `head.next` verification without `replace_head`; an exhaustive 24-case candidate-by-root-set model and a repeat-run determinism check | Implemented at the storage port; the filesystem version-2 publisher and version-2 next-head finalization use it (#198, `KEEP-RETENTION-012` and `013`) | +| `KEEP-RETENTION-012` | Under version-2 writer authority, one staged segment and a catalog successor publish only through the retained-closure gate, with no-follow bounded retention reads, head absence admitted only when uncontradicted, fixed retention stages refused, and no reader fence taken | `FilesystemVersionTwoCatalogPublisher`; `tests/version_two_catalog_writer.rs`: a preserving successor publishes, reopens, and reads both blobs while the retention root still verifies; a pinned reader view keeps its generation across publication; a candidate that drops retained content, a contradicted head absence, and a fixed retention stage each refuse with the exact typed error and an unchanged store witness; runtime RED for the contradiction and stage checks | Implemented for one staged segment per publication; segment rollover is #121 | +| `KEEP-RETENTION-013` | Version-2 segment and catalog stage recovery admits version-2 root entries as known, reuses the version-1 executors, and finalizes a staged `head.next` only after the retained-closure gate admits it | `FilesystemVersionTwoRecoveryAdmission`, `classify_version_two_recovery_names`, and the `open_version_two` constructors; `tests/version_two_catalog_writer.rs` covers classification, completion, discard, reusable-prefix segment resume, and finalization that preserves or would lose retained content (runtime RED with the gate result ignored); `filesystem_version_two_catalog_recovery_tests` recovers injected interruptions after segment link, catalog-stage creation, and `head.next` synchronization | Implemented | +| `KEEP-RETENTION-014` | Version-1 writer and recovery entry points refuse version-2 roots, and version-2 entry points refuse version-1 roots, with typed errors | `tests/version_two_catalog_writer.rs` version laws, including cross-publisher segment selection | Implemented | +| `KEEP-RETENTION-015` | Process death in the version-2 writer and its recovery reaches documented recoverable states | `cargo xtask durability-crash-matrix --sequence version-two-catalog` runs `KEEP-CRASH-074..=077`, 12 cases, through recovery and durable reads; `KEEP-CRASH-078..=080` (process death inside version-2 recovery) are planned | In progress: writer rows implemented, recovery rows planned | diff --git a/docs/formats/segment-store-v2/retention-publication.md b/docs/formats/segment-store-v2/retention-publication.md index 3654ee26..a3750d8c 100644 --- a/docs/formats/segment-store-v2/retention-publication.md +++ b/docs/formats/segment-store-v2/retention-publication.md @@ -30,7 +30,7 @@ Version-2 catalog publication must hold the same writer authority and prove ever After `head.next` verification, it rereads the retention head and refuses `HEAD` replacement if the head changed. Its receipt binds the preserved manifest generation and digest. The [format rationale](rationale.md#gate-catalog-publication-on-every-retained-closure) records the crash states and the recovery obligation. -No filesystem version-2 catalog publisher or version-2 next-head finalization uses the gate yet; that writer is tracked in [#198](https://github.com/flyingrobots/keep/issues/198). Retention publication and complete-stage retention recovery separately reverify a new root's closure against the current catalog. +`FilesystemVersionTwoCatalogPublisher` publishes through the gate, and version-2 next-head finalization runs the same retained-closure admission against a staged candidate ([#198](https://github.com/flyingrobots/keep/issues/198)). [Catalog publication and stage recovery](catalog-publication.md) owns that writer and its recovery. Retention publication and complete-stage retention recovery separately reverify a new root's closure against the current catalog. ## Generation transition From c1e67caa397f34adc2fde7c052d631343a58fcbc Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 07:48:28 -0700 Subject: [PATCH 09/36] Fix: export the catalog observer types to featureless test builds (#198) The library's own interruption laws use the observer without the repository-tasks feature, so the hidden types must be reachable there too; clippy --no-default-features --all-targets otherwise denies them as unreachable pub items. --- src/adapters/exports.rs | 2 +- src/lib.rs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/adapters/exports.rs b/src/adapters/exports.rs index 9fb376f2..ab8aaebd 100644 --- a/src/adapters/exports.rs +++ b/src/adapters/exports.rs @@ -58,7 +58,7 @@ pub use super::filesystem_segment_stage::FilesystemSegmentStage; pub use super::filesystem_version_two_admission::FilesystemVersionTwoAdmission; pub use super::filesystem_version_two_catalog_publisher::FilesystemVersionTwoCatalogPublisher; pub use super::filesystem_version_two_catalog_stages::FilesystemVersionTwoCatalogStages; -#[cfg(feature = "repository-tasks")] +#[cfg(any(test, feature = "repository-tasks"))] pub use super::filesystem_version_two_catalog_stages::{ CatalogPublicationBoundary, CatalogPublicationObserver, }; diff --git a/src/lib.rs b/src/lib.rs index e32dc0c8..9c53fc1a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -189,9 +189,12 @@ pub use adapters::{ #[cfg(feature = "repository-tasks")] #[doc(hidden)] pub use adapters::{ - CatalogPublicationBoundary, CatalogPublicationObserver, ObservedSegmentStage, - RepositoryInitializationStorage, SegmentStageDurabilityEvent, SegmentStageObserver, + ObservedSegmentStage, RepositoryInitializationStorage, SegmentStageDurabilityEvent, + SegmentStageObserver, }; +#[cfg(any(test, feature = "repository-tasks"))] +#[doc(hidden)] +pub use adapters::{CatalogPublicationBoundary, CatalogPublicationObserver}; pub use adapters::{ FilesystemMigrationRecoveryRefusal, FilesystemMigrationResidueKind, MIGRATION_NAMESPACE_PREFIX, StoreMigrationEffect, StoreMigrationFixedStage, StoreMigrationNamespacePrefix, From cde36ef33c01972cb7233031dd2bf12bea67c32b Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 07:54:09 -0700 Subject: [PATCH 10/36] Style: order the hidden repository-task exports as rustfmt requires (#198) --- src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 9c53fc1a..03c07e71 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -186,12 +186,6 @@ pub use adapters::{ plan_retention_transition, preflight_retention_transition, prepare_retention_publication, verify_retention_closure, }; -#[cfg(feature = "repository-tasks")] -#[doc(hidden)] -pub use adapters::{ - ObservedSegmentStage, RepositoryInitializationStorage, SegmentStageDurabilityEvent, - SegmentStageObserver, -}; #[cfg(any(test, feature = "repository-tasks"))] #[doc(hidden)] pub use adapters::{CatalogPublicationBoundary, CatalogPublicationObserver}; @@ -202,6 +196,12 @@ pub use adapters::{ StoreMigrationRecoveryReceipt, StoreMigrationRecoveryStorage, StoreMigrationResidue, StoreMigrationStageDecodeError, plan_store_migration_recovery, recover_store_migration, }; +#[cfg(feature = "repository-tasks")] +#[doc(hidden)] +pub use adapters::{ + ObservedSegmentStage, RepositoryInitializationStorage, SegmentStageDurabilityEvent, + SegmentStageObserver, +}; pub use adapters::{ PreservedRetentionManifest, RetainedCatalogPublicationError, RetainedCatalogPublicationReceipt, RetainedCatalogPublicationStorage, RetainedClosureAdmission, RetentionEvidencePhase, From 22082e56b531337954274527fc6d0ec4c4fe8e7b Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 08:16:56 -0700 Subject: [PATCH 11/36] Test: build populated stores through one shared writer-backed builder (#198) Move populated-store construction out of the golden worldline and version-two writer fixtures into tests/support/populated_store.rs and tests/support/successor_catalog.rs. The builder creates a store through the public writer (version-one initialization, the first catalog through FilesystemCatalogPublisher, migration), publishes retention roots, and publishes successor catalogs through FilesystemVersionTwoCatalogPublisher and the retained-closure gate. Admission is a typed route, so unit tests outside the platform profile reuse the same steps. durable_fixture and writer_fixture keep their names and signatures and delegate, so every law's construction and assertions are unchanged. When direct initialization replaces migration, only populated_store::create changes. --- ...stem_version_two_catalog_recovery_tests.rs | 7 + src/adapters/mod.rs | 4 + tests/durable_locator/suite.rs | 6 + .../golden_file_worldline/durable_fixture.rs | 205 ++----------- tests/golden_file_worldline/suite.rs | 7 + tests/support/populated_store.rs | 290 ++++++++++++++++++ tests/support/successor_catalog.rs | 160 ++++++++++ .../stage_recovery_laws.rs | 5 +- tests/version_two_catalog_writer/suite.rs | 8 + .../writer_fixture.rs | 216 ++----------- 10 files changed, 535 insertions(+), 373 deletions(-) create mode 100644 tests/support/populated_store.rs create mode 100644 tests/support/successor_catalog.rs diff --git a/src/adapters/filesystem_version_two_catalog_recovery_tests.rs b/src/adapters/filesystem_version_two_catalog_recovery_tests.rs index e1f0d6e7..c6125964 100644 --- a/src/adapters/filesystem_version_two_catalog_recovery_tests.rs +++ b/src/adapters/filesystem_version_two_catalog_recovery_tests.rs @@ -16,6 +16,12 @@ mod durable_fixture; reason = "interruption laws finalize and complete; the public suite also inspects roles" )] mod recovery_fixture; +#[path = "../../tests/support/successor_catalog.rs"] +#[allow( + dead_code, + reason = "interruption laws publish and republish; the public suite also stages by hand" +)] +mod successor_catalog; #[path = "../../tests/version_two_catalog_writer/writer_fixture.rs"] #[allow( dead_code, @@ -27,6 +33,7 @@ use std::io; use std::path::Path; use super::filesystem_test_sandbox as durable_sandbox; +use super::populated_store; use crate::{CatalogPublicationError, CatalogPublicationPhase, RetainedCatalogPublicationError}; diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index c013ed62..aa2f2f88 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -174,6 +174,10 @@ mod lower_hex; mod observed_segment_stage; mod opened_reusable_segment; mod physical_pool_name; +#[cfg(test)] +#[path = "../../tests/support/populated_store.rs"] +#[allow(dead_code, reason = "each consumer uses a builder subset")] +mod populated_store; mod publication_head_decode_error; mod publication_head_decode_error_display; mod publication_head_decoder; diff --git a/tests/durable_locator/suite.rs b/tests/durable_locator/suite.rs index e1a1ac2d..cfab97cd 100644 --- a/tests/durable_locator/suite.rs +++ b/tests/durable_locator/suite.rs @@ -14,3 +14,9 @@ mod durable_locator_laws; reason = "locator laws use Drop cleanup; other filesystem laws check explicit removal" )] mod durable_sandbox; +#[path = "../support/populated_store.rs"] +#[allow( + dead_code, + reason = "each consumer uses a subset of the shared builder; none reads every receipt" +)] +mod populated_store; diff --git a/tests/golden_file_worldline/durable_fixture.rs b/tests/golden_file_worldline/durable_fixture.rs index 265250db..72e36fbb 100644 --- a/tests/golden_file_worldline/durable_fixture.rs +++ b/tests/golden_file_worldline/durable_fixture.rs @@ -1,203 +1,58 @@ -//! This module owns real version-one publication followed by migration and retention. +//! This module owns the Golden File Worldline's sandboxed populated stores. +//! +//! Construction belongs to the shared `tests/support` builder; this module +//! owns only the sandbox and the worldline retention namespace. -use std::collections::BTreeSet; use std::error::Error; -use std::fs; - -use keep::{ - AdmittedLayout, AdmittedRetentionRoot, AdmittedSegment, AdmittedSegmentRecord, BlobId, - CanonicalCatalog, CanonicalLayoutRecord, CanonicalRetentionRoot, CatalogGeneration, - CatalogPublicationExpectation, CatalogRestartByteLimit, CatalogRestartPolicy, ChunkSpan, - FastCdc, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, - FilesystemRetentionPublicationAuthority, FilesystemStoreMigrationAuthority, - FilesystemVersionTwoAdmission, LayoutEntryLimit, RegisteredRetentionProfile, - RegisteredStorageProfile, RetentionAnchor, RetentionClosureLimits, - RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionRoot, - RootGeneration, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, - execute_retention_publication, execute_store_migration, preflight_retention_transition, - prepare_retention_publication, publish_catalog_generation, -}; use super::durable_sandbox::TestDirectory; +use super::populated_store::{self, FirstCatalogRecords, StoreAdmission}; +pub(super) use super::populated_store::{identify, policy}; type TestResult = Result>; -#[derive(Clone, Copy, Eq, PartialEq)] -enum Records { - Complete, - LayoutsOnly, - OnlyChunk(keep::ChunkId), -} - -pub(super) struct Identified { - pub(super) spans: Vec, - pub(super) record: CanonicalLayoutRecord, - pub(super) target: BlobId, -} - -pub(super) fn identify(bytes: &[u8]) -> TestResult { - let target = BlobId::hash_bytes(bytes)?; - let mut detector = FastCdc::new(); - let mut spans = Vec::new(); - detector.feed(bytes, |span| spans.push(span))?; - if let Some(span) = detector.finish()? { - spans.push(span); - } - let layout = AdmittedLayout::from_spans( - target, - RegisteredStorageProfile::FAST_CDC_64K_V1, - spans.clone(), - LayoutEntryLimit::MAXIMUM, - )?; - Ok(Identified { - spans, - record: layout.encode_record()?, - target, - }) -} - -pub(super) fn policy() -> TestResult { - Ok(CatalogRestartPolicy::new( - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), - CatalogRestartByteLimit::new(16_777_216)?, - )) -} - +/// A store holding `contents` with one "worldline" root that retains them all. pub(super) fn build(name: &str, contents: &[&[u8]]) -> TestResult { - let sandbox = TestDirectory::create(name)?; - let identified = contents - .iter() - .map(|bytes| identify(bytes)) - .collect::>>()?; - publish_catalog(&sandbox, contents, &identified, Records::Complete)?; - migrate(&sandbox)?; - publish_retention(&sandbox, &identified)?; + let sandbox = created(name, contents, FirstCatalogRecords::Complete)?; + let _receipt = populated_store::retain( + sandbox.path(), + StoreAdmission::PLATFORM, + b"worldline", + contents, + )?; Ok(sandbox) } +/// An unretained store whose catalog holds `content`'s layout but no chunk. pub(super) fn build_missing_chunk(name: &str, content: &[u8]) -> TestResult { - let sandbox = TestDirectory::create(name)?; - let identified = [identify(content)?]; - publish_catalog(&sandbox, &[content], &identified, Records::LayoutsOnly)?; - migrate(&sandbox)?; - Ok(sandbox) + created(name, &[content], FirstCatalogRecords::LayoutsOnly) } +/// An unretained store whose catalog holds `content`'s layout and only chunk `index`. pub(super) fn build_selected_chunk( name: &str, content: &[u8], index: usize, ) -> TestResult { - let sandbox = TestDirectory::create(name)?; - let identified = [identify(content)?]; - let selected = identified - .first() - .ok_or("identity absent")? + let selected = identify(content)? .spans .get(index) .ok_or("selected chunk absent")? .id(); - publish_catalog( - &sandbox, - &[content], - &identified, - Records::OnlyChunk(selected), - )?; - migrate(&sandbox)?; - Ok(sandbox) + created(name, &[content], FirstCatalogRecords::OnlyChunk(selected)) } -fn migrate(sandbox: &TestDirectory) -> TestResult<()> { - let admission = FilesystemPlatformAdmission::reopen(sandbox.path())?; - let mut migration = FilesystemStoreMigrationAuthority::open( - admission, - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), - )?; - let intent = migration.observe_intent()?; - let _receipt = execute_store_migration(&mut migration, &intent)?; - drop(migration); - Ok(()) -} - -fn publish_catalog( - sandbox: &TestDirectory, +fn created( + name: &str, contents: &[&[u8]], - identified: &[Identified], - records: Records, -) -> TestResult<()> { - let admission = FilesystemPlatformAdmission::initialize(sandbox.path())?; - let mut publisher = FilesystemCatalogPublisher::open(admission, policy()?)?; - let mut stage = StagedSegment::begin( - publisher.create_segment_stage()?, - SegmentRecordLimit::MAXIMUM, - )?; - let mut chunks = BTreeSet::new(); - let mut layouts = BTreeSet::new(); - for (bytes, identity) in contents.iter().zip(identified) { - for span in &identity.spans { - if matches!(records, Records::Complete) || records == Records::OnlyChunk(span.id()) { - if !chunks.insert(span.id()) { - continue; - } - let start = usize::try_from(span.offset().get())?; - let end = usize::try_from(span.end().get())?; - stage = stage.append(AdmittedSegmentRecord::for_chunk( - bytes.get(start..end).ok_or("chunk outside source")?, - )?)?; - } - } - if layouts.insert(identity.record.id()) { - stage = stage.append(AdmittedSegmentRecord::for_layout(&identity.record)?)?; - } - } - let sealed = stage.seal()?; - let bytes = fs::read(sandbox.path().join("staging/current.seg"))?; - let segments = [AdmittedSegment::decode( - &bytes, - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), - )?]; - let selected = publisher.select_segment(sealed, segments.first().ok_or("segment absent")?)?; - let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let _receipt = publish_catalog_generation( - &mut publisher, - CatalogPublicationExpectation::uninitialized(), - selected, - &catalog, - &segments, - )?; - Ok(()) -} - -fn publish_retention(sandbox: &TestDirectory, identified: &[Identified]) -> TestResult<()> { - let anchors = identified - .iter() - .map(|entry| RetentionAnchor::new(entry.target, entry.record.id())) - .collect::>() - .into_iter() - .collect(); - let root = RetentionRoot::new( - RetentionNamespace::try_from(&b"worldline"[..])?, - RootGeneration::INITIAL, - RetentionPolicy::new( - RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, - RetentionClosureLimits::new(4096, 8, 16_777_216, 67_108_864)?, - ), - None, - anchors, - )?; - let canonical = CanonicalRetentionRoot::from_root(&root)?; - let candidate = AdmittedRetentionRoot::decode(canonical.encoded())?; - let current = FilesystemCatalogSnapshot::load(sandbox.path(), policy()?)?; - let snapshot = current.snapshot()?; - let preflight = preflight_retention_transition( - RetentionGenerationExpectation::Absent, - None, - candidate, - &snapshot, + selection: FirstCatalogRecords, +) -> TestResult { + let sandbox = TestDirectory::create(name)?; + let _created = populated_store::create( + sandbox.path(), + StoreAdmission::PLATFORM, + contents, + selection, )?; - let prepared = prepare_retention_publication(preflight, None)?; - let admission = FilesystemVersionTwoAdmission::reopen(sandbox.path())?; - let mut authority = FilesystemRetentionPublicationAuthority::open(admission)?; - let _receipt = execute_retention_publication(&mut authority, &prepared)?; - Ok(()) + Ok(sandbox) } diff --git a/tests/golden_file_worldline/suite.rs b/tests/golden_file_worldline/suite.rs index 7519b499..8841eea2 100644 --- a/tests/golden_file_worldline/suite.rs +++ b/tests/golden_file_worldline/suite.rs @@ -48,6 +48,13 @@ mod durable_refusal_laws; #[cfg(target_os = "linux")] #[path = "../segment_filesystem_stage/sandbox.rs"] mod durable_sandbox; +#[cfg(target_os = "linux")] +#[path = "../support/populated_store.rs"] +#[allow( + dead_code, + reason = "each consumer uses a subset of the shared builder; none reads every receipt" +)] +mod populated_store; use std::error::Error; use std::io::ErrorKind; diff --git a/tests/support/populated_store.rs b/tests/support/populated_store.rs new file mode 100644 index 00000000..5098513a --- /dev/null +++ b/tests/support/populated_store.rs @@ -0,0 +1,290 @@ +//! This module owns populated-store construction for test code. +//! +//! Every populated version-two test store is created by [`create`] and then +//! extended by [`retain`] and by the version-two writer in `successor_catalog.rs`. +//! Only public production entry points run, so test code is not a second +//! implementation of the store format. Today [`create`] initializes version +//! one, publishes the first catalog through `FilesystemCatalogPublisher`, and +//! migrates. When direct initialization replaces migration (#198), only that +//! creation step changes. +//! +//! Consumers include this file by path beside their laws, so it names the +//! crate as `keep` and uses `super`-relative visibility. + +use std::collections::BTreeSet; +use std::error::Error; +use std::fs; +use std::path::Path; + +use keep::{ + AdmittedLayout, AdmittedRetentionManifest, AdmittedRetentionRoot, AdmittedSegment, + AdmittedSegmentRecord, BlobId, CanonicalCatalog, CanonicalLayoutRecord, CanonicalRetentionRoot, + CanonicalStoreMigrationReceipt, CatalogGeneration, CatalogPublicationExpectation, + CatalogPublicationReceipt, CatalogRestartByteLimit, CatalogRestartPolicy, ChunkId, ChunkSpan, + FastCdc, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, + FilesystemRetentionPublicationAuthority, FilesystemStoreMigrationAuthority, + FilesystemVersionTwoAdmission, LayoutEntryLimit, RegisteredRetentionProfile, + RegisteredStorageProfile, RetentionAnchor, RetentionClosureLimits, + RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, + RetentionPublicationReceipt, RetentionRoot, RootGeneration, SegmentReadPolicy, + SegmentRecordLimit, StagedSegment, execute_retention_publication, execute_store_migration, + preflight_retention_transition, prepare_retention_publication, publish_catalog_generation, +}; + +pub(super) type TestResult = Result>; + +/// How the builder acquires writer authority over a store root. +/// +/// [`Self::PLATFORM`] is the production platform profile. Private unit tests +/// that run outside that profile supply their own unchecked route; every step +/// after admission is identical. +#[derive(Clone, Copy)] +pub(super) struct StoreAdmission { + /// Initializes an empty root. + pub(super) initialize: fn(&Path) -> TestResult, + /// Reacquires authority over the published pre-migration root. + pub(super) reopen_version_one: fn(&Path) -> TestResult, + /// Reacquires authority over the created version-two root. + pub(super) reopen_version_two: fn(&Path) -> TestResult, +} + +impl StoreAdmission { + /// The production platform profile. + pub(super) const PLATFORM: Self = Self { + initialize: |root| Ok(FilesystemPlatformAdmission::initialize(root)?), + reopen_version_one: |root| Ok(FilesystemPlatformAdmission::reopen(root)?), + reopen_version_two: |root| Ok(FilesystemVersionTwoAdmission::reopen(root)?), + }; +} + +/// Which records of the given contents the first catalog references. +/// +/// Anything but [`Self::Complete`] deliberately builds a store whose catalog +/// is valid but whose content is incomplete. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum FirstCatalogRecords { + /// Every distinct chunk, then each layout. + Complete, + /// Only the layout records. + LayoutsOnly, + /// One chunk record and every layout. + OnlyChunk(ChunkId), +} + +impl FirstCatalogRecords { + fn keeps_chunk(self, chunk: ChunkId) -> bool { + match self { + Self::Complete => true, + Self::LayoutsOnly => false, + Self::OnlyChunk(selected) => selected == chunk, + } + } +} + +/// The receipts of one completed creation step. +#[must_use] +pub(super) struct CreatedStore { + /// The first catalog publication. + pub(super) catalog: CatalogPublicationReceipt, + /// The step that made the root version two. + pub(super) migration: CanonicalStoreMigrationReceipt, +} + +/// The canonical identity of one blob under the registered storage profile. +pub(super) struct Identified { + pub(super) spans: Vec, + pub(super) record: CanonicalLayoutRecord, + pub(super) target: BlobId, +} + +/// Chunks `bytes` and encodes its canonical layout record. +pub(super) fn identify(bytes: &[u8]) -> TestResult { + let target = BlobId::hash_bytes(bytes)?; + let mut detector = FastCdc::new(); + let mut spans = Vec::new(); + detector.feed(bytes, |span| spans.push(span))?; + if let Some(span) = detector.finish()? { + spans.push(span); + } + let layout = AdmittedLayout::from_spans( + target, + RegisteredStorageProfile::FAST_CDC_64K_V1, + spans.clone(), + LayoutEntryLimit::MAXIMUM, + )?; + Ok(Identified { + spans, + record: layout.encode_record()?, + target, + }) +} + +pub(super) fn policy() -> TestResult { + Ok(CatalogRestartPolicy::new( + segment_policy(), + CatalogRestartByteLimit::new(16_777_216)?, + )) +} + +pub(super) const fn segment_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} + +/// Creates a version-two store at the empty directory `root`. +/// +/// The first catalog holds one segment with the `selection` of `contents`' +/// records. Writer authority is released before this returns. +pub(super) fn create( + root: &Path, + admission: StoreAdmission, + contents: &[&[u8]], + selection: FirstCatalogRecords, +) -> TestResult { + let identified = contents + .iter() + .map(|bytes| identify(bytes)) + .collect::>>()?; + let blobs = contents.iter().copied().zip(&identified); + let records = select_records(blobs, selection)?; + let catalog = publish_first_catalog(root, (admission.initialize)(root)?, records)?; + let migration = migrate(root, admission)?; + Ok(CreatedStore { catalog, migration }) +} + +/// Makes the published root version two and releases writer authority. +fn migrate(root: &Path, admission: StoreAdmission) -> TestResult { + let mut authority = FilesystemStoreMigrationAuthority::open( + (admission.reopen_version_one)(root)?, + segment_policy(), + )?; + let intent = authority.observe_intent()?; + let receipt = execute_store_migration(&mut authority, &intent)?; + drop(authority); + Ok(receipt) +} + +/// Every selected chunk record, deduplicated across blobs, then each new layout. +pub(super) fn select_records<'content>( + blobs: impl IntoIterator, + selection: FirstCatalogRecords, +) -> TestResult>> { + let mut chunks = BTreeSet::new(); + let mut layouts = BTreeSet::new(); + let mut records = Vec::new(); + for (bytes, identity) in blobs { + for span in &identity.spans { + if selection.keeps_chunk(span.id()) && chunks.insert(span.id()) { + let start = usize::try_from(span.offset().get())?; + let end = usize::try_from(span.end().get())?; + let chunk = bytes.get(start..end).ok_or("chunk outside source")?; + records.push(AdmittedSegmentRecord::for_chunk(chunk)?); + } + } + if layouts.insert(identity.record.id()) { + records.push(AdmittedSegmentRecord::for_layout(&identity.record)?); + } + } + Ok(records) +} + +fn publish_first_catalog( + root: &Path, + admission: FilesystemPlatformAdmission, + records: Vec>, +) -> TestResult { + let mut publisher = FilesystemCatalogPublisher::open(admission, policy()?)?; + let mut stage = StagedSegment::begin( + publisher.create_segment_stage()?, + SegmentRecordLimit::MAXIMUM, + )?; + for record in records { + stage = stage.append(record)?; + } + let sealed = stage.seal()?; + let bytes = fs::read(root.join("staging/current.seg"))?; + let segments = [AdmittedSegment::decode(&bytes, segment_policy())?]; + let segment = segments.first().ok_or("segment absent")?; + let selected = publisher.select_segment(sealed, segment)?; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + Ok(publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + selected, + &catalog, + &segments, + )?) +} + +/// Publishes a generation-one root in `namespace` that anchors every blob. +pub(super) fn retain( + root: &Path, + admission: StoreAdmission, + namespace: &[u8], + contents: &[&[u8]], +) -> TestResult { + let anchors = contents + .iter() + .map(|bytes| { + let identity = identify(bytes)?; + Ok(RetentionAnchor::new(identity.target, identity.record.id())) + }) + .collect::>>()?; + let retained = RetentionRoot::new( + RetentionNamespace::try_from(namespace)?, + RootGeneration::INITIAL, + RetentionPolicy::new( + RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, + RetentionClosureLimits::new(4096, 8, 16_777_216, 67_108_864)?, + ), + None, + anchors.into_iter().collect(), + )?; + let canonical = CanonicalRetentionRoot::from_root(&retained)?; + let candidate = AdmittedRetentionRoot::decode(canonical.encoded())?; + let current = FilesystemCatalogSnapshot::load(root, policy()?)?; + let snapshot = current.snapshot()?; + let preflight = preflight_retention_transition( + RetentionGenerationExpectation::Absent, + None, + candidate, + &snapshot, + )?; + let mut authority = + FilesystemRetentionPublicationAuthority::open((admission.reopen_version_two)(root)?)?; + let observed = authority.observe_current()?; + let manifest = observed + .as_ref() + .map(|state| AdmittedRetentionManifest::decode(state.manifest_bytes())) + .transpose()?; + let prepared = prepare_retention_publication(preflight, manifest.as_ref())?; + Ok(execute_retention_publication(&mut authority, &prepared)?) +} + +/// Requires the first catalog's pools and head to hold exactly these bytes. +/// +/// Laws that prepare against frozen corpus bytes call this, so a writer whose +/// output drifts refuses at construction instead of failing an unrelated law. +pub(super) fn require_first_catalog( + root: &Path, + segment: &[u8], + catalog: &[u8], + head: &[u8], +) -> TestResult<()> { + require_sole_entry(root, "segments", segment)?; + require_sole_entry(root, "catalogs", catalog)?; + if fs::read(root.join("HEAD"))? == head { + Ok(()) + } else { + Err("writer-built HEAD differs from the expected bytes".into()) + } +} + +fn require_sole_entry(root: &Path, pool: &str, expected: &[u8]) -> TestResult<()> { + let entries = fs::read_dir(root.join(pool))? + .map(|entry| entry.map(|entry| entry.path())) + .collect::, _>>()?; + match entries.as_slice() { + [only] if fs::read(only)? == expected => Ok(()), + _ => Err(format!("writer-built {pool} pool is not exactly the expected entry").into()), + } +} diff --git a/tests/support/successor_catalog.rs b/tests/support/successor_catalog.rs new file mode 100644 index 00000000..a6e3def1 --- /dev/null +++ b/tests/support/successor_catalog.rs @@ -0,0 +1,160 @@ +//! This module owns test publication of catalogs after a store's first one. +//! +//! Every successor goes through `FilesystemVersionTwoCatalogPublisher` and the +//! retained-closure gate, on a store created by `populated_store.rs`. Callers +//! pass the publisher so they can observe or interrupt its transitions. +//! +//! Consumers include this file by path beside `populated_store`. + +use std::fs; +use std::path::Path; + +use keep::{ + AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, + CatalogPublicationExpectation, FilesystemCatalogSnapshot, FilesystemSegmentStage, + FilesystemVersionTwoCatalogPublisher, RetainedCatalogPublicationError, + RetainedCatalogPublicationReceipt, SealedSegment, SegmentPublication, SegmentRecordLimit, + StagedSegment, publish_retained_catalog_generation, +}; + +use super::populated_store::{ + FirstCatalogRecords, Identified, StoreAdmission, TestResult, identify, policy, segment_policy, + select_records, +}; + +/// The gate's exact outcome, inside the fixture's own failure channel. +pub(super) type GateOutcome = + Result; + +/// Whether a candidate catalog keeps every predecessor segment. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum Predecessor { + /// The candidate references every current segment and the new one. + Kept, + /// The candidate references only the new segment. + Dropped, +} + +/// Opens the version-two writer over a created store. +pub(super) fn open_publisher( + root: &Path, + admission: StoreAdmission, +) -> TestResult { + Ok(FilesystemVersionTwoCatalogPublisher::open( + (admission.reopen_version_two)(root)?, + policy()?, + )?) +} + +/// Reads every immutable segment in deterministic name order. +pub(super) fn pool_segments(root: &Path) -> TestResult>> { + let mut names = fs::read_dir(root.join("segments"))? + .map(|entry| entry.map(|entry| entry.file_name())) + .collect::, _>>()?; + names.sort(); + names + .iter() + .map(|name| Ok(fs::read(root.join("segments").join(name))?)) + .collect() +} + +pub(super) fn admit_all(encoded: &[Vec]) -> TestResult>> { + encoded + .iter() + .map(|bytes| Ok(AdmittedSegment::decode(bytes, segment_policy())?)) + .collect() +} + +/// Every distinct chunk record of `content` followed by its layout record. +pub(super) fn blob_records<'content>( + content: &'content [u8], + identified: &'content Identified, +) -> TestResult>> { + select_records([(content, identified)], FirstCatalogRecords::Complete) +} + +/// Stages and seals every record of `content` without selecting the stage. +pub(super) fn stage_blob<'publisher>( + publisher: &'publisher FilesystemVersionTwoCatalogPublisher, + content: &[u8], + identified: &Identified, +) -> TestResult>> { + let mut stage = StagedSegment::begin( + publisher.create_segment_stage()?, + SegmentRecordLimit::MAXIMUM, + )?; + for record in blob_records(content, identified)? { + stage = stage.append(record)?; + } + Ok(stage.seal()?) +} + +/// Stages `content` as one new segment and publishes the next catalog generation. +/// +/// The candidate references the current segments when `predecessor` is +/// [`Predecessor::Kept`]. +pub(super) fn publish( + publisher: &mut FilesystemVersionTwoCatalogPublisher, + root: &Path, + content: &[u8], + predecessor: Predecessor, +) -> TestResult { + let identified = identify(content)?; + let current = FilesystemCatalogSnapshot::load(root, policy()?)?; + let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); + let mut encoded = match predecessor { + Predecessor::Kept => pool_segments(root)?, + Predecessor::Dropped => Vec::new(), + }; + let sealed = stage_blob(publisher, content, &identified)?; + encoded.push(fs::read(root.join("staging/current.seg"))?); + let segments = admit_all(&encoded)?; + let staged = segments.last().ok_or("staged segment absent")?; + let selection = publisher.select_segment(sealed, staged)?; + let catalog = successor(¤t, &segments)?; + Ok(publish_retained_catalog_generation( + publisher, + expectation, + selection, + &catalog, + &segments, + )) +} + +/// Publishes the next catalog generation over already pooled segments. +pub(super) fn republish( + publisher: &mut FilesystemVersionTwoCatalogPublisher, + root: &Path, +) -> TestResult { + let current = FilesystemCatalogSnapshot::load(root, policy()?)?; + let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); + let encoded = pool_segments(root)?; + let segments = admit_all(&encoded)?; + let catalog = successor(¤t, &segments)?; + Ok(publish_retained_catalog_generation( + publisher, + expectation, + SegmentPublication::none(), + &catalog, + &segments, + )) +} + +/// The catalog one generation after `current` over exactly `segments`. +fn successor( + current: &FilesystemCatalogSnapshot, + segments: &[AdmittedSegment<'_>], +) -> TestResult { + let generation = CatalogGeneration::new( + current + .generation() + .get() + .checked_add(1) + .ok_or("generation overflow")?, + )?; + Ok(CanonicalCatalog::from_segments( + generation, + Some(current.catalog_digest()), + segments, + )?) +} diff --git a/tests/version_two_catalog_writer/stage_recovery_laws.rs b/tests/version_two_catalog_writer/stage_recovery_laws.rs index 59614b11..d855865b 100644 --- a/tests/version_two_catalog_writer/stage_recovery_laws.rs +++ b/tests/version_two_catalog_writer/stage_recovery_laws.rs @@ -11,9 +11,10 @@ use keep::{ use super::durable_fixture; use super::recovery_fixture::{complete, discard, stage_roles}; +use super::successor_catalog::blob_records; use super::writer_fixture::{ - Predecessor, TestResult, assert_reads, blob_records, first_blob, open_publisher, publish, - republish, retain, second_blob, segment_policy, + Predecessor, TestResult, assert_reads, first_blob, open_publisher, publish, republish, retain, + second_blob, segment_policy, }; #[test] diff --git a/tests/version_two_catalog_writer/suite.rs b/tests/version_two_catalog_writer/suite.rs index 1b8e65c0..94559109 100644 --- a/tests/version_two_catalog_writer/suite.rs +++ b/tests/version_two_catalog_writer/suite.rs @@ -16,12 +16,20 @@ mod durable_sandbox; mod finalization_laws; #[path = "forward_laws.rs"] mod forward_laws; +#[path = "../support/populated_store.rs"] +#[allow( + dead_code, + reason = "each consumer uses a subset of the shared builder; none reads every receipt" +)] +mod populated_store; #[path = "recovery_fixture.rs"] mod recovery_fixture; #[path = "refusal_laws.rs"] mod refusal_laws; #[path = "stage_recovery_laws.rs"] mod stage_recovery_laws; +#[path = "../support/successor_catalog.rs"] +mod successor_catalog; #[path = "version_laws.rs"] mod version_laws; #[path = "writer_fixture.rs"] diff --git a/tests/version_two_catalog_writer/writer_fixture.rs b/tests/version_two_catalog_writer/writer_fixture.rs index ab4dcef4..62673f2b 100644 --- a/tests/version_two_catalog_writer/writer_fixture.rs +++ b/tests/version_two_catalog_writer/writer_fixture.rs @@ -1,32 +1,26 @@ -//! This module owns one blob's ingestion through the version-two catalog writer. +//! This module owns the version-two writer suite's blobs and store witnesses. //! -//! The library's interruption laws include this file by path, so it names the -//! crate as `keep` and reaches the shared store fixture through `super`. +//! Store construction and successor publication belong to the shared +//! `tests/support` builder; this module fixes the suite's blobs and the +//! platform admission route. The library's interruption laws include this file +//! by path, so it names the crate as `keep` and reaches the builder through +//! `super`. -use std::collections::BTreeSet; -use std::error::Error; use std::fmt::Write as _; use std::fs; use std::path::Path; use keep::{ - AdmittedRetentionManifest, AdmittedRetentionRoot, AdmittedSegment, AdmittedSegmentRecord, - CanonicalCatalog, CanonicalRetentionRoot, CatalogGeneration, CatalogPublicationExpectation, - DurableStore, FilesystemCatalogSnapshot, FilesystemRetentionPublicationAuthority, - FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, - ReaderAttemptLimit, RegisteredRetentionProfile, RetainedCatalogPublicationError, - RetainedCatalogPublicationReceipt, RetentionAnchor, RetentionClosureLimits, - RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionRoot, - RootGeneration, SegmentPublication, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, - execute_retention_publication, preflight_retention_transition, prepare_retention_publication, - publish_retained_catalog_generation, + AdmittedSegment, DurableStore, FilesystemVersionTwoCatalogPublisher, ReaderAttemptLimit, }; -use super::durable_fixture::{self, Identified}; - -pub(super) type TestResult = Result>; +use super::populated_store::{self, StoreAdmission}; +pub(super) use super::populated_store::{TestResult, segment_policy}; +pub(super) use super::successor_catalog::{ + Predecessor, admit_all, pool_segments, publish, republish, +}; -/// The retained content every migrated fixture store starts with. +/// The retained content every fixture store starts with. pub(super) fn first_blob() -> Vec { b"keep version-two writer: first retained blob. ".repeat(3_000) } @@ -36,193 +30,23 @@ pub(super) fn second_blob() -> Vec { b"KEEP V2 WRITER -- the second blob, ingested after migration | ".repeat(2_500) } -/// Whether a candidate catalog keeps every predecessor segment. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum Predecessor { - /// The candidate references every current segment and the new one. - Kept, - /// The candidate references only the new segment. - Dropped, -} - -pub(super) const fn segment_policy() -> SegmentReadPolicy { - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) -} - -/// Opens the version-two writer over a migrated store. +/// Opens the version-two writer under the production platform profile. pub(super) fn open_publisher(root: &Path) -> TestResult { - let admission = FilesystemVersionTwoAdmission::reopen(root)?; - Ok(FilesystemVersionTwoCatalogPublisher::open( - admission, - durable_fixture::policy()?, - )?) -} - -/// Reads every immutable segment in deterministic name order. -pub(super) fn pool_segments(root: &Path) -> TestResult>> { - let mut names = fs::read_dir(root.join("segments"))? - .map(|entry| entry.map(|entry| entry.file_name())) - .collect::, _>>()?; - names.sort(); - names - .iter() - .map(|name| Ok(fs::read(root.join("segments").join(name))?)) - .collect() -} - -/// Stages `content` as one new segment and publishes the next catalog generation. -/// -/// The candidate references the current segments when `predecessor` is -/// [`Predecessor::Kept`]. The outer result carries fixture failures; the inner -/// result is the gate's exact outcome. -pub(super) fn publish( - publisher: &mut FilesystemVersionTwoCatalogPublisher, - root: &Path, - content: &[u8], - predecessor: Predecessor, -) -> TestResult> { - let identified = durable_fixture::identify(content)?; - let current = FilesystemCatalogSnapshot::load(root, durable_fixture::policy()?)?; - let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); - let mut encoded = match predecessor { - Predecessor::Kept => pool_segments(root)?, - Predecessor::Dropped => Vec::new(), - }; - let sealed = stage_records(publisher, content, &identified)?; - encoded.push(fs::read(root.join("staging/current.seg"))?); - let segments = admit_all(&encoded)?; - let staged = segments.last().ok_or("staged segment absent")?; - let selection = publisher.select_segment(sealed, staged)?; - let generation = CatalogGeneration::new( - current - .generation() - .get() - .checked_add(1) - .ok_or("generation overflow")?, - )?; - let catalog = - CanonicalCatalog::from_segments(generation, Some(current.catalog_digest()), &segments)?; - Ok(publish_retained_catalog_generation( - publisher, - expectation, - selection, - &catalog, - &segments, - )) -} - -/// Publishes the next catalog generation over already pooled segments. -pub(super) fn republish( - publisher: &mut FilesystemVersionTwoCatalogPublisher, - root: &Path, -) -> TestResult> { - let current = FilesystemCatalogSnapshot::load(root, durable_fixture::policy()?)?; - let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); - let encoded = pool_segments(root)?; - let segments = admit_all(&encoded)?; - let generation = CatalogGeneration::new( - current - .generation() - .get() - .checked_add(1) - .ok_or("generation overflow")?, - )?; - let catalog = - CanonicalCatalog::from_segments(generation, Some(current.catalog_digest()), &segments)?; - Ok(publish_retained_catalog_generation( - publisher, - expectation, - SegmentPublication::none(), - &catalog, - &segments, - )) -} - -pub(super) fn admit_all(encoded: &[Vec]) -> TestResult>> { - encoded - .iter() - .map(|bytes| Ok(AdmittedSegment::decode(bytes, segment_policy())?)) - .collect() -} - -fn stage_records<'publisher>( - publisher: &'publisher FilesystemVersionTwoCatalogPublisher, - content: &[u8], - identified: &Identified, -) -> TestResult>> { - let mut stage = StagedSegment::begin( - publisher.create_segment_stage()?, - SegmentRecordLimit::MAXIMUM, - )?; - for record in blob_records(content, identified)? { - stage = stage.append(record)?; - } - Ok(stage.seal()?) -} - -/// Every distinct chunk record of `content` followed by its layout record. -pub(super) fn blob_records<'content>( - content: &'content [u8], - identified: &'content Identified, -) -> TestResult>> { - let mut chunks = BTreeSet::new(); - let mut records = Vec::new(); - for span in &identified.spans { - if chunks.insert(span.id()) { - let start = usize::try_from(span.offset().get())?; - let end = usize::try_from(span.end().get())?; - let bytes = content.get(start..end).ok_or("chunk outside source")?; - records.push(AdmittedSegmentRecord::for_chunk(bytes)?); - } - } - records.push(AdmittedSegmentRecord::for_layout(&identified.record)?); - Ok(records) + super::successor_catalog::open_publisher(root, StoreAdmission::PLATFORM) } /// Publishes a new generation-one retention root that anchors `content`. pub(super) fn retain(root: &Path, namespace: &[u8], content: &[u8]) -> TestResult<()> { - let identified = durable_fixture::identify(content)?; - let retained = RetentionRoot::new( - RetentionNamespace::try_from(namespace)?, - RootGeneration::INITIAL, - RetentionPolicy::new( - RegisteredRetentionProfile::SINGLE_CANONICAL_WITNESS_V1, - RetentionClosureLimits::new(4096, 8, 16_777_216, 67_108_864)?, - ), - None, - vec![RetentionAnchor::new( - identified.target, - identified.record.id(), - )], - )?; - let canonical = CanonicalRetentionRoot::from_root(&retained)?; - let candidate = AdmittedRetentionRoot::decode(canonical.encoded())?; - let current = FilesystemCatalogSnapshot::load(root, durable_fixture::policy()?)?; - let snapshot = current.snapshot()?; - let preflight = preflight_retention_transition( - RetentionGenerationExpectation::Absent, - None, - candidate, - &snapshot, - )?; - let admission = FilesystemVersionTwoAdmission::reopen(root)?; - let mut authority = FilesystemRetentionPublicationAuthority::open(admission)?; - let observed = authority.observe_current()?; - let manifest = observed - .as_ref() - .map(|state| AdmittedRetentionManifest::decode(state.manifest_bytes())) - .transpose()?; - let prepared = prepare_retention_publication(preflight, manifest.as_ref())?; - let _receipt = execute_retention_publication(&mut authority, &prepared)?; + let _receipt = populated_store::retain(root, StoreAdmission::PLATFORM, namespace, &[content])?; Ok(()) } /// Requires a fresh fenced view to reconstruct exactly `content`. pub(super) fn assert_reads(root: &Path, content: &[u8]) -> TestResult<()> { - let target = durable_fixture::identify(content)?.target; + let target = populated_store::identify(content)?.target; let store = DurableStore::open( root, - durable_fixture::policy()?, + populated_store::policy()?, ReaderAttemptLimit::DEFAULT, )?; let mut output = Vec::new(); @@ -269,8 +93,8 @@ pub(super) fn stage_only( root: &Path, content: &[u8], ) -> TestResult> { - let identified = durable_fixture::identify(content)?; - let sealed = stage_records(publisher, content, &identified)?; + let identified = populated_store::identify(content)?; + let sealed = super::successor_catalog::stage_blob(publisher, content, &identified)?; drop(sealed); Ok(fs::read(root.join("staging/current.seg"))?) } From 0340591388dc24f56a263fde7444f5d8e0e886db Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 08:17:07 -0700 Subject: [PATCH 12/36] Test: build the retention fixture's bundle store through the builder (#198) The src retention fixture and the tmpfs reader-profile fixture wrote the frozen one-zero bundle segment, catalog, and HEAD by hand into an initialized version-one root and then migrated it. Both now ingest the one-zero blob through the shared builder under an unchecked test admission route. The retention laws prepare against the frozen bundle snapshot and root, so populate_bundle requires each pool to hold exactly the frozen entry and HEAD to equal the frozen head; a drifting writer refuses at construction. Staging the layout record before the chunk makes it refuse with "writer-built segments pool is not exactly the expected entry". --- .../filesystem_retention_test_fixture.rs | 59 ++++++++++++------- .../retention/reader_platform_law_tests.rs | 31 ++-------- 2 files changed, 41 insertions(+), 49 deletions(-) diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs index 76aa77ac..6fe05d01 100644 --- a/src/adapters/retention/filesystem_retention_test_fixture.rs +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -16,16 +16,16 @@ use super::{ use super::{RetentionPublicationPhase as Phase, RetentionPublicationStorage}; use crate::LayoutEntryLimit; use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::populated_store::{self, FirstCatalogRecords, StoreAdmission}; use crate::adapters::test_support::decode_hex; use crate::adapters::{ AdmittedCatalog, AdmittedSegment, CatalogSnapshot, ChecksummedCatalog, - ChecksummedPublicationHead, FilesystemPlatformAdmission, FilesystemStoreMigrationAuthority, - FilesystemVersionTwoAdmission, SegmentReadPolicy, SegmentRecordLimit, + ChecksummedPublicationHead, FilesystemPlatformAdmission, FilesystemVersionTwoAdmission, + SegmentReadPolicy, SegmentRecordLimit, }; use crate::{ RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionRoot, - RootGeneration, execute_store_migration, preflight_retention_transition, - prepare_retention_publication, + RootGeneration, preflight_retention_transition, prepare_retention_publication, }; /// Frozen canonical generation-one root. @@ -243,29 +243,44 @@ fn hex(bytes: &[u8; 32]) -> String { }) } -/// Builds one completely migrated version-2 store with writer authority released. +/// The shared builder's route on hosts outside the admitted platform profile. +const UNCHECKED: StoreAdmission = StoreAdmission { + initialize: |root| { + Ok(FilesystemPlatformAdmission::initialize_unchecked_for_tests( + root, + )?) + }, + reopen_version_one: |root| { + Ok(FilesystemPlatformAdmission::reopen_unchecked_for_tests( + root, + )?) + }, + reopen_version_two: |root| { + Ok(FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(root)?) + }, +}; + +/// Builds one version-2 store holding the frozen bundle, with writer authority released. pub(super) fn migrated_store(name: &str) -> Result> { let sandbox = TestDirectory::create(name)?; - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - write_version_one(&sandbox)?; - let mut authority = FilesystemStoreMigrationAuthority::open(admission, maximum_policy())?; - let intent = authority.observe_intent()?; - let _receipt = execute_store_migration(&mut authority, &intent)?; - drop(authority); + populate_bundle(sandbox.path())?; Ok(sandbox) } -fn write_version_one(sandbox: &TestDirectory) -> Result<(), Box> { - fs::write( - sandbox.path().join("segments").join(SEGMENT_NAME), - fixture(SEGMENT_HEX)?, - )?; - fs::write( - sandbox.path().join("catalogs").join(CATALOG_NAME), - fixture(CATALOG_HEX)?, - )?; - fs::write(sandbox.path().join("HEAD"), fixture(CATALOG_HEAD_HEX)?)?; - Ok(()) +/// Creates the one-zero blob's store at the empty directory `root`. +/// +/// The shared builder ingests the blob through the public writer. These laws +/// prepare against the frozen bundle snapshot and root, so the pools must hold +/// exactly the frozen bundle bytes; any other bytes refuse here. +pub(super) fn populate_bundle(root: &Path) -> Result<(), Box> { + let _created = + populated_store::create(root, UNCHECKED, &[&[0]], FirstCatalogRecords::Complete)?; + populated_store::require_first_catalog( + root, + &fixture(SEGMENT_HEX)?, + &fixture(CATALOG_HEX)?, + &fixture(CATALOG_HEAD_HEX)?, + ) } const fn maximum_policy() -> SegmentReadPolicy { diff --git a/src/adapters/retention/reader_platform_law_tests.rs b/src/adapters/retention/reader_platform_law_tests.rs index 3f31c1c7..b09afb26 100644 --- a/src/adapters/retention/reader_platform_law_tests.rs +++ b/src/adapters/retention/reader_platform_law_tests.rs @@ -10,14 +10,11 @@ use std::fs; use std::io; use std::path::PathBuf; -use super::filesystem_retention_test_fixture::{ - CATALOG_NAME, SEGMENT_NAME, fixture, migrated_store, -}; +use super::filesystem_retention_test_fixture::{migrated_store, populate_bundle}; use crate::{ CatalogRestartByteLimit, CatalogRestartPolicy, DurableSnapshot, DurableStoreError, - FilesystemPlatformAdmission, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, - FilesystemStoreMigrationAuthority, FilesystemWriterLock, ReaderAttemptLimit, SegmentReadPolicy, - execute_store_migration, + FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, FilesystemWriterLock, + ReaderAttemptLimit, SegmentReadPolicy, }; #[test] @@ -97,27 +94,7 @@ impl TmpfsStore { "negative profile requires Linux tmpfs" ); let store = Self::reserve(name)?; - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(&store.0)?; - for (name, encoded) in [ - ( - format!("segments/{SEGMENT_NAME}"), - include_str!("../../../conformance/segment-store/v1/one-zero-bundle-segment.hex"), - ), - ( - format!("catalogs/{CATALOG_NAME}"), - include_str!("../../../conformance/segment-store/v1/one-zero-bundle-catalog.hex"), - ), - ( - "HEAD".into(), - include_str!("../../../conformance/segment-store/v1/one-zero-bundle-head.hex"), - ), - ] { - fs::write(store.0.join(name), fixture(encoded)?)?; - } - let mut authority = - FilesystemStoreMigrationAuthority::open(admission, SegmentReadPolicy::MAXIMUM)?; - let intent = authority.observe_intent()?; - let _receipt = execute_store_migration(&mut authority, &intent)?; + populate_bundle(&store.0)?; Ok(store) } } From cdfdf4d8fe47ab237acaf6099ae27f8f5d4f3229 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 08:17:07 -0700 Subject: [PATCH 13/36] Test: build crash-matrix bundle stores through the public writer (#198) The retention and version-two catalog crash rows wrote the frozen bundle bytes by hand into an initialized root and migrated it. A new production_protocol/populated_store.rs creates the bundle store through FilesystemCatalogPublisher and migration, then requires the pools to equal the frozen bundle before any crash row runs. The crash rows, their coordinates, and restart verifiers are unchanged. xtask cannot include integration-test support, so this module is the matrix's counterpart of tests/support/populated_store.rs over the same public entry points. The version-two crash blob becomes CrashBlob, which also identifies the one-zero bundle blob; the unused initialized-lock helper is removed. --- .../production_protocol.rs | 3 +- .../{version_two_blob.rs => crash_blob.rs} | 58 +++++--- .../production_protocol/initialization.rs | 13 -- .../production_protocol/populated_store.rs | 125 ++++++++++++++++++ .../production_protocol/retention.rs | 40 +----- .../version_two_catalog.rs | 4 +- .../restart/version_two_catalog.rs | 8 +- 7 files changed, 176 insertions(+), 75 deletions(-) rename xtask/src/durability_crash_matrix/production_protocol/{version_two_blob.rs => crash_blob.rs} (51%) create mode 100644 xtask/src/durability_crash_matrix/production_protocol/populated_store.rs diff --git a/xtask/src/durability_crash_matrix/production_protocol.rs b/xtask/src/durability_crash_matrix/production_protocol.rs index a7ed62ea..d80f24ec 100644 --- a/xtask/src/durability_crash_matrix/production_protocol.rs +++ b/xtask/src/durability_crash_matrix/production_protocol.rs @@ -1,11 +1,13 @@ //! This module owns real production protocol execution for crash children. mod control; +pub(super) mod crash_blob; pub(super) mod fixture; pub(super) mod initialization; mod initialization_storage; mod migration; mod migration_storage; +mod populated_store; mod publication; mod publication_storage; mod recovery; @@ -13,7 +15,6 @@ mod recovery_storage; pub(super) mod retention; mod retention_storage; mod segment_stage; -pub(super) mod version_two_blob; pub(super) mod version_two_catalog; use std::error::Error; diff --git a/xtask/src/durability_crash_matrix/production_protocol/version_two_blob.rs b/xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs similarity index 51% rename from xtask/src/durability_crash_matrix/production_protocol/version_two_blob.rs rename to xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs index 6dd9048e..601d5d23 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/version_two_blob.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs @@ -1,7 +1,7 @@ -//! This module owns the one blob the version-two crash campaign ingests. +//! This module owns the blobs crash children ingest through the public writer. //! -//! The child stages exactly these records, and restart reconstructs exactly -//! these bytes, so both sides derive the blob from one definition. +//! A child stages exactly these records, and restart reconstructs exactly +//! these bytes, so both sides derive each blob from one definition. use keep::{ AdmittedLayout, AdmittedSegmentRecord, BlobId, CanonicalLayoutRecord, ChunkSpan, FastCdc, @@ -12,29 +12,43 @@ use super::super::DurabilityCrashMatrixError; use super::verification; /// The exact bytes the version-two writer ingests before process death. -pub(in crate::durability_crash_matrix) const CONTENT: &[u8] = +pub(in crate::durability_crash_matrix) const VERSION_TWO_CONTENT: &[u8] = b"keep version-two crash campaign: one blob ingested after migration"; -/// The canonical layout and chunk spans of [`CONTENT`]. -pub(in crate::durability_crash_matrix) struct VersionTwoBlob { +/// The synthetic source byte of the frozen one-zero bundle corpus. +const BUNDLE_CONTENT: &[u8] = &[0]; + +/// One blob with its canonical layout and chunk spans. +pub(in crate::durability_crash_matrix) struct CrashBlob { + content: &'static [u8], spans: Vec, record: CanonicalLayoutRecord, target: BlobId, } -impl VersionTwoBlob { - pub(in crate::durability_crash_matrix) fn identify() -> Result - { - let target = BlobId::hash_bytes(CONTENT) - .map_err(|source| verification("identify version-two crash blob", source))?; +impl CrashBlob { + /// The blob the version-two writer ingests before process death. + pub(in crate::durability_crash_matrix) fn version_two() + -> Result { + Self::identify(VERSION_TWO_CONTENT) + } + + /// The blob whose first catalog is the frozen bundle corpus. + pub(in crate::durability_crash_matrix) fn bundle() -> Result { + Self::identify(BUNDLE_CONTENT) + } + + fn identify(content: &'static [u8]) -> Result { + let target = BlobId::hash_bytes(content) + .map_err(|source| verification("identify crash blob", source))?; let mut detector = FastCdc::new(); let mut spans = Vec::new(); detector - .feed(CONTENT, |span| spans.push(span)) - .map_err(|source| verification("chunk version-two crash blob", source))?; + .feed(content, |span| spans.push(span)) + .map_err(|source| verification("chunk crash blob", source))?; if let Some(span) = detector .finish() - .map_err(|source| verification("finish version-two crash blob", source))? + .map_err(|source| verification("finish crash blob", source))? { spans.push(span); } @@ -44,11 +58,12 @@ impl VersionTwoBlob { spans.clone(), LayoutEntryLimit::MAXIMUM, ) - .map_err(|source| verification("admit version-two crash layout", source))?; + .map_err(|source| verification("admit crash layout", source))?; let record = layout .encode_record() - .map_err(|source| verification("encode version-two crash layout", source))?; + .map_err(|source| verification("encode crash layout", source))?; Ok(Self { + content, spans, record, target, @@ -62,20 +77,21 @@ impl VersionTwoBlob { let mut records = Vec::with_capacity(self.spans.len()); for span in &self.spans { let start = usize::try_from(span.offset().get()) - .map_err(|source| verification("bound version-two crash chunk", source))?; + .map_err(|source| verification("bound crash chunk", source))?; let end = usize::try_from(span.end().get()) - .map_err(|source| verification("bound version-two crash chunk", source))?; - let bytes = CONTENT + .map_err(|source| verification("bound crash chunk", source))?; + let bytes = self + .content .get(start..end) .ok_or(DurabilityCrashMatrixError::FixtureRange)?; records.push( AdmittedSegmentRecord::for_chunk(bytes) - .map_err(|source| verification("admit version-two crash chunk", source))?, + .map_err(|source| verification("admit crash chunk", source))?, ); } records.push( AdmittedSegmentRecord::for_layout(&self.record) - .map_err(|source| verification("admit version-two crash layout", source))?, + .map_err(|source| verification("admit crash layout", source))?, ); Ok(records) } diff --git a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs index 298ef994..6ee9cea9 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs @@ -26,19 +26,6 @@ pub(super) fn run( .map_err(|source| verification("execute production store initialization", source)) } -/// Initializes a fresh store and returns its retained writer lock. -pub(super) fn initialized_lock( - store_root: &Path, -) -> Result { - let mut storage = RepositoryInitializationStorage::admit_unchecked(store_root) - .map_err(|source| DurabilityCrashMatrixError::io("open initialization storage", source))?; - let _receipt = initialize_store(&mut storage) - .map_err(|source| verification("initialize production crash store", source))?; - storage - .into_writer_lock() - .map_err(|source| DurabilityCrashMatrixError::io("retain initialized writer lock", source)) -} - pub(super) fn publisher( store_root: &Path, ) -> Result { diff --git a/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs b/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs new file mode 100644 index 00000000..045b2832 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs @@ -0,0 +1,125 @@ +//! This module owns crash-child store construction through the public writer. +//! +//! It is the crash matrix's counterpart of `tests/support/populated_store.rs`: +//! the same public entry points, under the matrix's typed error, because a +//! repository task cannot include integration-test support. Today creation is +//! version-one initialization, the first catalog through +//! `FilesystemCatalogPublisher`, and migration. When direct initialization +//! replaces migration (#198), only [`create_bundle_store`] changes. + +use std::fs; +use std::path::Path; + +use keep::{ + AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationExpectation, + FilesystemCatalogPublisher, FilesystemPlatformAdmission, FilesystemSegmentStage, + FilesystemStoreMigrationAuthority, SealedSegment, SegmentRecordLimit, StagedSegment, + execute_store_migration, publish_catalog_generation, +}; + +use super::crash_blob::CrashBlob; +use super::fixture::{BUNDLE_CATALOG_NAME, BUNDLE_SEGMENT_NAME, GoldenFixture}; +use super::{DurabilityCrashMatrixError, initialization, verification}; + +/// Creates the one-zero bundle store at the empty directory `store_root`. +/// +/// Writer authority is released before this returns. Crash rows prepare +/// against the frozen bundle snapshot and retention root, so the pools must +/// hold exactly the frozen bundle bytes; any other bytes refuse here. +pub(super) fn create_bundle_store(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + publish_bundle_catalog(store_root)?; + migrate(store_root)?; + require_frozen_bundle(store_root) +} + +fn publish_bundle_catalog(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let mut publisher = initialization::publisher(store_root)?; + let blob = CrashBlob::bundle()?; + let sealed = stage_bundle(&publisher, &blob)?; + let bytes = fs::read(store_root.join("staging/current.seg")) + .map_err(|source| DurabilityCrashMatrixError::io("read bundle segment", source))?; + let segments = [ + AdmittedSegment::decode(&bytes, initialization::segment_policy()) + .map_err(|source| verification("admit bundle segment", source))?, + ]; + let selection = publisher + .select_segment(sealed, &segments[0]) + .map_err(|source| verification("select bundle segment stage", source))?; + let generation = CatalogGeneration::new(1) + .map_err(|source| verification("construct bundle catalog generation", source))?; + let catalog = CanonicalCatalog::from_segments(generation, None, &segments) + .map_err(|source| verification("encode bundle catalog", source))?; + publish_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + selection, + &catalog, + &segments, + ) + .map(|_receipt| ()) + .map_err(|source| verification("publish bundle catalog", source)) +} + +fn stage_bundle<'publisher>( + publisher: &'publisher FilesystemCatalogPublisher, + blob: &CrashBlob, +) -> Result>, DurabilityCrashMatrixError> { + let stage = publisher + .create_segment_stage() + .map_err(|source| verification("create bundle segment stage", source))?; + let mut stage = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM) + .map_err(|source| verification("write bundle segment header", source))?; + for record in blob.records()? { + stage = stage + .append(record) + .map_err(|source| verification("append bundle segment record", source))?; + } + stage + .seal() + .map_err(|source| verification("seal bundle segment", source)) +} + +fn migrate(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let admission = FilesystemPlatformAdmission::reopen(store_root) + .map_err(|source| verification("reopen bundle store for migration", source))?; + let mut migration = + FilesystemStoreMigrationAuthority::open(admission, initialization::segment_policy()) + .map_err(|source| verification("open crash migration authority", source))?; + let intent = migration + .observe_intent() + .map_err(|source| verification("observe crash migration intent", source))?; + execute_store_migration(&mut migration, &intent) + .map(|_receipt| ()) + .map_err(|source| verification("execute crash store migration", source)) +} + +fn require_frozen_bundle(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + for (artifact, relative, expected) in [ + ( + "bundle segment", + format!("segments/{BUNDLE_SEGMENT_NAME}"), + GoldenFixture::bundle_segment()?, + ), + ( + "bundle catalog", + format!("catalogs/{BUNDLE_CATALOG_NAME}"), + GoldenFixture::bundle_catalog()?, + ), + ( + "bundle head", + "HEAD".to_owned(), + GoldenFixture::bundle_head()?, + ), + ] { + let observed = fs::read(store_root.join(relative)) + .map_err(|source| DurabilityCrashMatrixError::io("read writer-built bundle", source))?; + if observed != expected.bytes() { + return Err(DurabilityCrashMatrixError::artifact_bytes( + artifact, + expected.bytes(), + &observed, + )); + } + } + Ok(()) +} diff --git a/xtask/src/durability_crash_matrix/production_protocol/retention.rs b/xtask/src/durability_crash_matrix/production_protocol/retention.rs index fa3eb9ee..1cd5663a 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/retention.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/retention.rs @@ -1,19 +1,18 @@ //! This module owns execution of the production retention publication protocol. -use std::fs; use std::path::Path; use keep::{ AdmittedCatalog, AdmittedRetentionRoot, AdmittedSegment, ChecksummedCatalog, ChecksummedPublicationHead, FilesystemRetentionPublicationAuthority, - FilesystemStoreMigrationAuthority, FilesystemVersionTwoAdmission, - RetentionGenerationExpectation, RetentionPublicationPreparation, execute_retention_publication, - execute_store_migration, preflight_retention_transition, prepare_retention_publication, + FilesystemVersionTwoAdmission, RetentionGenerationExpectation, RetentionPublicationPreparation, + execute_retention_publication, preflight_retention_transition, prepare_retention_publication, }; use super::control::CrashControl; -use super::fixture::{BUNDLE_CATALOG_NAME, BUNDLE_SEGMENT_NAME, GoldenFixture}; +use super::fixture::GoldenFixture; use super::initialization; +use super::populated_store; use super::retention_storage::CrashRetentionStorage; use super::{DurabilityCrashMatrixError, verification}; @@ -32,24 +31,12 @@ pub(super) fn run( .map_err(|source| verification("execute production retention publication", source)) } -/// Initializes, populates, and migrates the bundle store, then reopens it as +/// Creates the bundle store through the public writer, then reopens it as /// version two and returns retention authority over it. pub(super) fn migrated_authority( store_root: &Path, ) -> Result { - let lock = initialization::initialized_lock(store_root)?; - write_bundle(store_root)?; - let mut migration = FilesystemStoreMigrationAuthority::open_unchecked_for_repository_tasks( - lock, - initialization::segment_policy(), - ) - .map_err(|source| verification("open crash migration authority", source))?; - let intent = migration - .observe_intent() - .map_err(|source| verification("observe crash migration intent", source))?; - let _receipt = execute_store_migration(&mut migration, &intent) - .map_err(|source| verification("execute crash store migration", source))?; - drop(migration); + populated_store::create_bundle_store(store_root)?; reopened_authority(store_root) } @@ -64,21 +51,6 @@ pub(in crate::durability_crash_matrix) fn reopened_authority( .map_err(|source| verification("open crash retention authority", source)) } -fn write_bundle(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let segment = GoldenFixture::bundle_segment()?; - let catalog = GoldenFixture::bundle_catalog()?; - let head = GoldenFixture::bundle_head()?; - for (relative, bytes) in [ - (format!("segments/{BUNDLE_SEGMENT_NAME}"), segment.bytes()), - (format!("catalogs/{BUNDLE_CATALOG_NAME}"), catalog.bytes()), - ("HEAD".to_owned(), head.bytes()), - ] { - fs::write(store_root.join(&relative), bytes) - .map_err(|source| DurabilityCrashMatrixError::io("write bundle corpus", source))?; - } - Ok(()) -} - /// Prepares the frozen generation-one root as an initial publication against /// the bundle catalog snapshot. pub(in crate::durability_crash_matrix) fn preparation( diff --git a/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs b/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs index 23afc6f3..cb87fa55 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs @@ -18,8 +18,8 @@ use keep::{ use xtask::DurabilityCrashPoint; use super::control::{CrashControl, DuringTiming}; +use super::crash_blob::CrashBlob; use super::fixture::GoldenFixture; -use super::version_two_blob::VersionTwoBlob; use super::{DurabilityCrashMatrixError, initialization, retention, verification}; /// Ingests the version-two crash blob, dying at the selected coordinate. @@ -34,7 +34,7 @@ pub(super) fn run( let mut publisher = FilesystemVersionTwoCatalogPublisher::open(admission, policy()?) .map_err(|source| verification("open version-two catalog publisher", source))?; publisher.observe_catalog_phases_for_repository_tasks(observer(control)); - let blob = VersionTwoBlob::identify()?; + let blob = CrashBlob::version_two()?; let mut stage = StagedSegment::begin( publisher .create_segment_stage() diff --git a/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs b/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs index c2836546..74961c9f 100644 --- a/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs +++ b/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs @@ -24,10 +24,10 @@ use keep::{ use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; use super::super::DurabilityCrashMatrixError; +use super::super::production_protocol::crash_blob::{CrashBlob, VERSION_TWO_CONTENT}; use super::super::production_protocol::fixture::GoldenFixture; use super::super::production_protocol::retention::reopened_authority; use super::super::production_protocol::verification; -use super::super::production_protocol::version_two_blob::{CONTENT, VersionTwoBlob}; use super::super::production_protocol::version_two_catalog::{admit_segments, policy}; use super::version_two_recovery as recovery; @@ -159,7 +159,7 @@ fn require_generation_two(store_root: &Path) -> Result<(), DurabilityCrashMatrix /// Retains the ingested blob in its own namespace, gated by the current catalog. fn retain_blob(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let blob = VersionTwoBlob::identify()?; + let blob = CrashBlob::version_two()?; let root = RetentionRoot::new( RetentionNamespace::try_from(&b"version-two-crash"[..]) .map_err(|source| verification("name crash retention namespace", source))?, @@ -220,8 +220,8 @@ fn read_back(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { return Err(mismatch("recovered retained blob changed identity".into())); } } - let ingested = reconstruct(&store, VersionTwoBlob::identify()?.target())?; - if ingested == CONTENT { + let ingested = reconstruct(&store, CrashBlob::version_two()?.target())?; + if ingested == VERSION_TWO_CONTENT { Ok(()) } else { Err(mismatch( From 25c98f89c78453357e7aaea3fdb9a6e2064721cb Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 09:42:52 -0700 Subject: [PATCH 14/36] Feat: replace migration with direct initialization of the one store format (#198) ADR-0010 makes keep.segment-store/v2 the only format and requires initialization to replace migration in the same change that deletes it. FilesystemVersionTwoAdmission::initialize creates a store under exclusive writer authority as a recoverable state machine: it classifies the root before creating writer.lock and again under the lock, makes FORMAT.next durable before any directory exists, creates the reader fence and the nine protocol directories with a parent synchronization each, links FORMAT without replacement, and commits with the root synchronization after the link. Every admission refuses FORMAT.next, and the stage is removed only after the commit synchronization, so a store never admits before its commit point. Interrupted states resume idempotently; every other nonempty root, and a committed one, refuses with typed evidence before mutation. An initialized store has no catalog HEAD: writers admit it, readers refuse it as Unpublished, version-2 recovery classification no longer requires HEAD, and the version-2 writer publishes the first catalog through the retained-closure gate. The test builders and the reader-fence fixture now initialize and publish through that writer. Migration is deleted: src/adapters/store_migration, its public API, tests, fuzz target, crash rows KEEP-CRASH-053..073, golden records and source tables, and format pages. The persisted root device-and-inode binding goes with it; live lock, fence, and directory checks remain. The FORMAT codec moves to src/adapters/store_format. Removing the migration rows from definition.tsv changes the definition digest to f1a0ce10...361d; the marker golden is regenerated from the handwritten oracle. The initialization crash matrix runs 41 cases at KEEP-CRASH-081..091 and recovers, publishes, retains, and reconstructs after each. Retired identifiers are mapped in docs/formats/segment-store-v2/retired-identifiers.md. Retires #108. --- CHANGELOG.md | 2 + README.md | 37 +- conformance/segment-store/v2/ORIGIN.md | 51 ++- conformance/segment-store/v2/README.md | 64 ++- conformance/segment-store/v2/artifacts.tsv | 4 +- conformance/segment-store/v2/definition.tsv | 20 - .../segment-store/v2/format-marker.hex | 2 +- conformance/segment-store/v2/inventory.tsv | 4 - .../segment-store/v2/migration-intent.hex | 1 - .../segment-store/v2/migration-receipt.hex | 1 - .../segment-store/v2/migration-source.tsv | 3 - conformance/segment-store/v2/transitions.tsv | 32 +- docs/adr/retention-reader-root-identity.md | 2 +- docs/dependencies/blake3-1.8.7.md | 2 +- docs/formats/README.md | 2 +- docs/formats/segment-store-v1/recovery.md | 2 +- docs/formats/segment-store-v2/README.md | 88 ++-- .../segment-store-v2/catalog-publication.md | 24 +- .../segment-store-v2/initialization.md | 190 ++++++++ .../segment-store-v2/migration-crash.md | 122 ----- .../segment-store-v2/migration-inventory.md | 66 --- .../segment-store-v2/migration-recovery.md | 102 ----- docs/formats/segment-store-v2/rationale.md | 79 ++-- docs/formats/segment-store-v2/recovery.md | 236 ++++------ docs/formats/segment-store-v2/requirements.md | 48 +- .../segment-store-v2/retention-publication.md | 2 +- .../segment-store-v2/retention-recovery.md | 2 +- .../segment-store-v2/retired-identifiers.md | 103 +++++ .../authenticated-reconstruction/README.md | 2 +- .../current-durable-surfaces.md | 2 +- .../migration-clone-failure.md | 21 - .../migration-compatibility-fuzz.md | 50 --- .../migration-restart-matrix.md | 69 --- .../retention-reader-root-identity.md | 2 + fuzz/Cargo.toml | 12 +- fuzz/README.md | 2 +- fuzz/fuzz_targets/format_marker.rs | 17 + fuzz/fuzz_targets/migration_format.rs | 58 --- .../fuzz_targets/migration_format/recovery.rs | 115 ----- src/adapters/admitted_segment.rs | 4 - src/adapters/durable/rationale.md | 2 +- src/adapters/durable/store.rs | 4 +- src/adapters/exports.rs | 9 +- src/adapters/filesystem_catalog_publisher.rs | 2 +- src/adapters/filesystem_exact_record.rs | 30 -- src/adapters/filesystem_format_marker.rs | 44 ++ .../filesystem_initialization_namespace.rs | 113 +++-- src/adapters/filesystem_namespace_refusal.rs | 9 + src/adapters/filesystem_platform_admission.rs | 34 +- .../filesystem_platform_admission_error.rs | 26 +- src/adapters/filesystem_platform_profile.rs | 124 +---- ...ilesystem_platform_profile_policy_tests.rs | 27 -- .../filesystem_platform_profile_tests.rs | 10 +- .../filesystem_recovery_inventory_reader.rs | 8 +- ...filesystem_recovery_next_head_finalizer.rs | 2 +- .../filesystem_recovery_segment_resumer.rs | 2 +- .../filesystem_recovery_stage_completer.rs | 2 +- .../filesystem_recovery_stage_discarder.rs | 2 +- src/adapters/filesystem_root_identity.rs | 34 -- src/adapters/filesystem_store_initializer.rs | 19 +- .../filesystem_version_two_admission.rs | 197 +++++--- .../filesystem_version_two_record_refusal.rs | 28 +- .../filesystem_version_two_records.rs | 87 ---- ...lesystem_version_two_recovery_admission.rs | 17 +- src/adapters/mod.rs | 6 +- src/adapters/recovery/recovery_entry_role.rs | 4 - .../recovery/recovery_name_classification.rs | 24 +- .../recovery/recovery_required_entry.rs | 12 +- src/adapters/retention.rs | 5 - .../filesystem_recovery_admission_tests.rs | 16 +- .../filesystem_retention_fifo_tests.rs | 4 +- ...em_retention_migration_completion_tests.rs | 48 -- .../filesystem_retention_snapshot.rs | 36 +- ...tem_retention_snapshot_coordinate_tests.rs | 4 +- .../filesystem_retention_snapshot_error.rs | 7 + ...lesystem_retention_snapshot_error_tests.rs | 10 +- ...ystem_retention_snapshot_identity_tests.rs | 53 --- ...system_retention_snapshot_pinning_tests.rs | 4 +- .../filesystem_retention_snapshot_tests.rs | 14 +- .../filesystem_retention_storage_tests.rs | 10 +- .../filesystem_retention_test_fixture.rs | 30 +- .../filesystem_retention_verification.rs | 31 +- .../filesystem_verification_law_tests.rs | 2 +- .../filesystem_version_two_admission_tests.rs | 132 +++--- src/adapters/retention/reader_fence_tests.rs | 4 +- .../retention/reader_platform_law_tests.rs | 8 +- .../retention/retention_model_tests.rs | 2 +- .../verification_namespace_law_tests.rs | 8 +- .../verification_store_admission_tests.rs | 95 +--- src/adapters/store_format.rs | 21 + .../admitted_format_marker.rs | 0 .../canonical_format_marker.rs | 0 .../format_definition_digest.rs | 6 +- .../format_marker_decode_error.rs | 0 .../format_marker_decode_error_display.rs | 0 .../format_marker_decoder.rs | 0 .../format_marker_digest.rs | 0 .../format_marker_encoder.rs | 0 src/adapters/store_migration.rs | 191 -------- .../admitted_migration_intent.rs | 122 ----- .../admitted_migration_receipt.rs | 97 ---- .../canonical_migration_intent.rs | 147 ------ .../canonical_migration_receipt.rs | 35 -- .../empty_disposition_set_digest.rs | 18 - .../filesystem_inventory_catalog_errors.rs | 107 ----- .../filesystem_inventory_catalogs.rs | 179 -------- ...system_inventory_catalogs_refusal_tests.rs | 184 -------- ...esystem_inventory_catalogs_test_fixture.rs | 109 ----- .../filesystem_inventory_catalogs_tests.rs | 35 -- .../filesystem_inventory_directory.rs | 74 --- .../filesystem_inventory_error.rs | 155 ------- .../filesystem_inventory_error_display.rs | 147 ------ .../filesystem_inventory_file.rs | 128 ------ .../filesystem_inventory_file_tests.rs | 67 --- .../filesystem_inventory_names.rs | 61 --- .../filesystem_inventory_names_tests.rs | 41 -- .../filesystem_inventory_reader.rs | 196 -------- .../filesystem_inventory_reader_tests.rs | 80 ---- .../filesystem_inventory_segments.rs | 184 -------- ...system_inventory_segments_refusal_tests.rs | 156 ------- ...esystem_inventory_segments_test_fixture.rs | 79 ---- .../filesystem_inventory_segments_tests.rs | 36 -- .../filesystem_migration_authority.rs | 236 ---------- .../filesystem_migration_authority_error.rs | 134 ------ ...ystem_migration_authority_error_display.rs | 121 ----- .../filesystem_migration_authority_tests.rs | 79 ---- ...lesystem_migration_authority_validation.rs | 80 ---- ...ilesystem_migration_compatibility_tests.rs | 57 --- ...system_migration_current_recovery_tests.rs | 50 --- .../filesystem_migration_fixed_artifact.rs | 256 ----------- .../filesystem_migration_namespace.rs | 307 ------------- ...ilesystem_migration_namespace_directory.rs | 178 -------- ...lesystem_migration_pair_admission_tests.rs | 66 --- .../filesystem_migration_reader_fence.rs | 62 --- ...system_migration_receipt_evidence_tests.rs | 79 ---- .../filesystem_migration_recovery.rs | 189 -------- .../filesystem_migration_recovery_refusal.rs | 110 ----- .../filesystem_migration_recovery_tests.rs | 159 ------- ...tem_migration_recovery_truncation_tests.rs | 123 ----- .../filesystem_migration_remount_tests.rs | 131 ------ .../filesystem_migration_repository_tasks.rs | 106 ----- .../filesystem_migration_residue.rs | 88 ---- ...filesystem_migration_residue_kind_tests.rs | 48 -- ...ystem_migration_restart_namespace_tests.rs | 190 -------- ...ilesystem_migration_restart_order_tests.rs | 148 ------ ...filesystem_migration_restart_pair_tests.rs | 62 --- ...filesystem_migration_restart_pool_tests.rs | 192 -------- ...lesystem_migration_restart_record_tests.rs | 163 ------- ...filesystem_migration_restart_root_tests.rs | 46 -- ...lesystem_migration_restart_test_fixture.rs | 89 ---- .../filesystem_migration_storage.rs | 242 ---------- .../filesystem_migration_storage_tests.rs | 228 ---------- .../filesystem_migration_test_fixture.rs | 40 -- .../immutable_pool_inventory_digest.rs | 20 - .../initial_gc_state_digest.rs | 18 - .../initial_retention_state_digest.rs | 18 - .../migration_catalog_admission.rs | 106 ----- .../migration_catalog_coordinates.rs | 52 --- .../store_migration/migration_catalog_plan.rs | 51 --- .../migration_catalog_records.rs | 93 ---- .../store_migration/migration_error.rs | 45 -- .../store_migration/migration_execution.rs | 136 ------ .../migration_intent_decode_error.rs | 84 ---- .../migration_intent_decode_error_display.rs | 64 --- .../migration_intent_decoder.rs | 156 ------- .../migration_intent_digest.rs | 18 - .../migration_intent_encoder.rs | 86 ---- .../migration_intent_format.rs | 61 --- .../migration_inventory_entry.rs | 62 --- .../migration_inventory_entry_count.rs | 35 -- .../migration_inventory_entry_count_error.rs | 29 -- .../migration_inventory_error.rs | 65 --- .../migration_inventory_hasher.rs | 91 ---- .../migration_namespace_prefix.rs | 47 -- .../store_migration/migration_phase.rs | 114 ----- .../migration_receipt_decode_error.rs | 100 ----- .../migration_receipt_decode_error_display.rs | 63 --- .../migration_receipt_decoder.rs | 145 ------ .../migration_receipt_encoder.rs | 52 --- .../migration_receipt_format.rs | 15 - .../migration_receipt_initial_state.rs | 76 ---- .../store_migration/migration_record_bytes.rs | 69 --- .../migration_recovery_ambiguity.rs | 86 ---- .../migration_recovery_ambiguity_display.rs | 69 --- .../migration_recovery_execution.rs | 244 ---------- .../migration_recovery_plan.rs | 40 -- .../migration_recovery_planner.rs | 272 ----------- .../migration_recovery_residue.rs | 98 ---- .../migration_recovery_storage.rs | 57 --- .../store_migration/migration_resumption.rs | 162 ------- .../migration_stage_decode_error.rs | 48 -- .../store_migration/migration_storage.rs | 173 ------- .../migration_synchronization_mask.rs | 20 - src/adapters/store_migration/rationale.md | 34 -- .../store_migration/store_identifier.rs | 18 - .../store_migration/store_root_identity.rs | 81 ---- src/adapters/version_two_initialization.rs | 56 +++ .../initialization_entries.rs | 138 ++++++ .../initialization_error.rs | 84 ++++ .../initialization_execution.rs | 142 ++++++ .../initialization_namespace.rs | 40 ++ .../initialization_observation.rs | 253 +++++++++++ .../initialization_phase.rs | 153 +++++++ .../initialization_prefix.rs | 33 ++ .../initialization_prefix_tests.rs | 183 ++++++++ .../initialization_reader_fence.rs | 39 ++ .../initialization_receipt.rs | 38 ++ .../initialization_refusal.rs | 91 ++++ .../initialization_resume_tests.rs | 200 +++++++++ .../initialization_test_fixture.rs | 126 ++++++ src/lib.rs | 125 +++--- tests/adapters_layout_contract.rs | 22 +- .../durable_refusal_laws.rs | 2 +- tests/migration_descriptor_exhaustion.rs | 85 ---- tests/reader_fence_process.rs | 10 +- tests/reader_fence_process/fixture.rs | 76 ++-- tests/reader_fence_process/reader.rs | 2 +- tests/store_format_marker.rs | 4 +- tests/store_migration_compatibility.rs | 100 ----- tests/store_migration_execution.rs | 97 ---- tests/store_migration_intent.rs | 200 --------- tests/store_migration_intent/fixture.rs | 33 -- tests/store_migration_intent_encoding.rs | 129 ------ tests/store_migration_inventory.rs | 135 ------ tests/store_migration_phase.rs | 101 ----- tests/store_migration_receipt.rs | 96 ---- tests/store_migration_receipt/binding_laws.rs | 104 ----- tests/store_migration_receipt/fixture.rs | 38 -- tests/store_migration_receipt/harness.rs | 107 ----- tests/store_migration_receipt_encoding.rs | 46 -- tests/store_migration_recovery.rs | 424 ------------------ tests/store_migration_recovery_order.rs | 121 ----- tests/store_migration_storage.rs | 87 ---- .../recording_storage.rs | 149 ------ tests/support/populated_store.rs | 93 ++-- tests/version_two_admission_contract.rs | 13 +- tests/version_two_catalog_writer.rs | 6 +- .../version_laws.rs | 11 +- .../writer_fixture.rs | 2 +- xtask/src/durability_crash_case.rs | 6 +- xtask/src/durability_crash_matrix/error.rs | 10 +- .../durability_crash_matrix/error/display.rs | 8 +- .../production_protocol.rs | 9 +- .../production_protocol/crash_blob.rs | 2 +- .../production_protocol/migration.rs | 39 -- .../migration_stage_prefix_laws.rs | 45 -- .../production_protocol/migration_storage.rs | 315 ------------- .../production_protocol/populated_store.rs | 53 +-- .../production_protocol/retention.rs | 12 +- .../version_two_catalog.rs | 6 +- .../version_two_initialization.rs | 178 ++++++++ xtask/src/durability_crash_matrix/restart.rs | 7 +- .../restart/expectation.rs | 4 +- .../restart/migration.rs | 122 ----- .../restart/migration_expectation.rs | 246 ---------- .../restart/retention.rs | 2 +- .../restart/version_two_catalog.rs | 4 +- .../restart/version_two_initialization.rs | 303 +++++++++++++ xtask/src/durability_crash_point.rs | 185 +++----- xtask/src/durability_crash_point_identity.rs | 32 +- xtask/src/fuzz_campaign/target/tests.rs | 2 +- xtask/src/fuzz_seed_corpus.rs | 5 +- .../fuzz_seed_corpus/format_marker_seeds.rs | 32 ++ .../migration_recovery_seeds.rs | 65 --- xtask/src/fuzz_seed_corpus/migration_seeds.rs | 135 ------ .../fuzz_seed_corpus/tests/materialization.rs | 4 +- xtask/tests/durability_crash_case_contract.rs | 24 +- xtask/tests/durability_crash_documentation.rs | 20 +- .../tests/durability_crash_point_contract.rs | 196 ++++---- .../durability_crash_production_contract.rs | 4 +- ...retention_store_v2_conformance_contract.rs | 4 - .../tests/retention_store_v2_format_oracle.rs | 32 -- .../artifacts.rs | 102 +---- .../artifacts/format_marker.rs | 28 ++ .../artifacts/migration.rs | 96 ---- .../encoding.rs | 10 - .../fixture_assertion.rs | 58 +-- .../retention_store_v2_protocol_contract.rs | 15 +- .../initialization_contract_laws.rs | 72 +++ .../migration_contract_laws.rs | 84 ---- .../parser_fuzz_laws.rs | 14 +- .../transition_laws.rs | 104 ++--- ...ment_store_implementation_documentation.rs | 2 +- 283 files changed, 3810 insertions(+), 16013 deletions(-) delete mode 100644 conformance/segment-store/v2/inventory.tsv delete mode 100644 conformance/segment-store/v2/migration-intent.hex delete mode 100644 conformance/segment-store/v2/migration-receipt.hex delete mode 100644 conformance/segment-store/v2/migration-source.tsv create mode 100644 docs/formats/segment-store-v2/initialization.md delete mode 100644 docs/formats/segment-store-v2/migration-crash.md delete mode 100644 docs/formats/segment-store-v2/migration-inventory.md delete mode 100644 docs/formats/segment-store-v2/migration-recovery.md create mode 100644 docs/formats/segment-store-v2/retired-identifiers.md delete mode 100644 docs/testing-evidence/migration-clone-failure.md delete mode 100644 docs/testing-evidence/migration-compatibility-fuzz.md delete mode 100644 docs/testing-evidence/migration-restart-matrix.md create mode 100644 fuzz/fuzz_targets/format_marker.rs delete mode 100644 fuzz/fuzz_targets/migration_format.rs delete mode 100644 fuzz/fuzz_targets/migration_format/recovery.rs create mode 100644 src/adapters/filesystem_format_marker.rs delete mode 100644 src/adapters/filesystem_platform_profile_policy_tests.rs delete mode 100644 src/adapters/filesystem_root_identity.rs delete mode 100644 src/adapters/filesystem_version_two_records.rs delete mode 100644 src/adapters/retention/filesystem_retention_migration_completion_tests.rs delete mode 100644 src/adapters/retention/filesystem_retention_snapshot_identity_tests.rs create mode 100644 src/adapters/store_format.rs rename src/adapters/{store_migration => store_format}/admitted_format_marker.rs (100%) rename src/adapters/{store_migration => store_format}/canonical_format_marker.rs (100%) rename src/adapters/{store_migration => store_format}/format_definition_digest.rs (74%) rename src/adapters/{store_migration => store_format}/format_marker_decode_error.rs (100%) rename src/adapters/{store_migration => store_format}/format_marker_decode_error_display.rs (100%) rename src/adapters/{store_migration => store_format}/format_marker_decoder.rs (100%) rename src/adapters/{store_migration => store_format}/format_marker_digest.rs (100%) rename src/adapters/{store_migration => store_format}/format_marker_encoder.rs (100%) delete mode 100644 src/adapters/store_migration.rs delete mode 100644 src/adapters/store_migration/admitted_migration_intent.rs delete mode 100644 src/adapters/store_migration/admitted_migration_receipt.rs delete mode 100644 src/adapters/store_migration/canonical_migration_intent.rs delete mode 100644 src/adapters/store_migration/canonical_migration_receipt.rs delete mode 100644 src/adapters/store_migration/empty_disposition_set_digest.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_catalog_errors.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_directory.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_error.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_error_display.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_file.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_file_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_names.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_names_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_reader.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_reader_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_segments.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs delete mode 100644 src/adapters/store_migration/filesystem_inventory_segments_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_authority.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_authority_error.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_authority_error_display.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_authority_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_authority_validation.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_compatibility_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_current_recovery_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_fixed_artifact.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_namespace.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_namespace_directory.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_pair_admission_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_reader_fence.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_receipt_evidence_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_recovery.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_recovery_refusal.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_recovery_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_recovery_truncation_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_remount_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_repository_tasks.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_residue.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_residue_kind_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_restart_namespace_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_restart_order_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_restart_pair_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_restart_pool_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_restart_record_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_restart_root_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_restart_test_fixture.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_storage.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_storage_tests.rs delete mode 100644 src/adapters/store_migration/filesystem_migration_test_fixture.rs delete mode 100644 src/adapters/store_migration/immutable_pool_inventory_digest.rs delete mode 100644 src/adapters/store_migration/initial_gc_state_digest.rs delete mode 100644 src/adapters/store_migration/initial_retention_state_digest.rs delete mode 100644 src/adapters/store_migration/migration_catalog_admission.rs delete mode 100644 src/adapters/store_migration/migration_catalog_coordinates.rs delete mode 100644 src/adapters/store_migration/migration_catalog_plan.rs delete mode 100644 src/adapters/store_migration/migration_catalog_records.rs delete mode 100644 src/adapters/store_migration/migration_error.rs delete mode 100644 src/adapters/store_migration/migration_execution.rs delete mode 100644 src/adapters/store_migration/migration_intent_decode_error.rs delete mode 100644 src/adapters/store_migration/migration_intent_decode_error_display.rs delete mode 100644 src/adapters/store_migration/migration_intent_decoder.rs delete mode 100644 src/adapters/store_migration/migration_intent_digest.rs delete mode 100644 src/adapters/store_migration/migration_intent_encoder.rs delete mode 100644 src/adapters/store_migration/migration_intent_format.rs delete mode 100644 src/adapters/store_migration/migration_inventory_entry.rs delete mode 100644 src/adapters/store_migration/migration_inventory_entry_count.rs delete mode 100644 src/adapters/store_migration/migration_inventory_entry_count_error.rs delete mode 100644 src/adapters/store_migration/migration_inventory_error.rs delete mode 100644 src/adapters/store_migration/migration_inventory_hasher.rs delete mode 100644 src/adapters/store_migration/migration_namespace_prefix.rs delete mode 100644 src/adapters/store_migration/migration_phase.rs delete mode 100644 src/adapters/store_migration/migration_receipt_decode_error.rs delete mode 100644 src/adapters/store_migration/migration_receipt_decode_error_display.rs delete mode 100644 src/adapters/store_migration/migration_receipt_decoder.rs delete mode 100644 src/adapters/store_migration/migration_receipt_encoder.rs delete mode 100644 src/adapters/store_migration/migration_receipt_format.rs delete mode 100644 src/adapters/store_migration/migration_receipt_initial_state.rs delete mode 100644 src/adapters/store_migration/migration_record_bytes.rs delete mode 100644 src/adapters/store_migration/migration_recovery_ambiguity.rs delete mode 100644 src/adapters/store_migration/migration_recovery_ambiguity_display.rs delete mode 100644 src/adapters/store_migration/migration_recovery_execution.rs delete mode 100644 src/adapters/store_migration/migration_recovery_plan.rs delete mode 100644 src/adapters/store_migration/migration_recovery_planner.rs delete mode 100644 src/adapters/store_migration/migration_recovery_residue.rs delete mode 100644 src/adapters/store_migration/migration_recovery_storage.rs delete mode 100644 src/adapters/store_migration/migration_resumption.rs delete mode 100644 src/adapters/store_migration/migration_stage_decode_error.rs delete mode 100644 src/adapters/store_migration/migration_storage.rs delete mode 100644 src/adapters/store_migration/migration_synchronization_mask.rs delete mode 100644 src/adapters/store_migration/rationale.md delete mode 100644 src/adapters/store_migration/store_identifier.rs delete mode 100644 src/adapters/store_migration/store_root_identity.rs create mode 100644 src/adapters/version_two_initialization.rs create mode 100644 src/adapters/version_two_initialization/initialization_entries.rs create mode 100644 src/adapters/version_two_initialization/initialization_error.rs create mode 100644 src/adapters/version_two_initialization/initialization_execution.rs create mode 100644 src/adapters/version_two_initialization/initialization_namespace.rs create mode 100644 src/adapters/version_two_initialization/initialization_observation.rs create mode 100644 src/adapters/version_two_initialization/initialization_phase.rs create mode 100644 src/adapters/version_two_initialization/initialization_prefix.rs create mode 100644 src/adapters/version_two_initialization/initialization_prefix_tests.rs create mode 100644 src/adapters/version_two_initialization/initialization_reader_fence.rs create mode 100644 src/adapters/version_two_initialization/initialization_receipt.rs create mode 100644 src/adapters/version_two_initialization/initialization_refusal.rs create mode 100644 src/adapters/version_two_initialization/initialization_resume_tests.rs create mode 100644 src/adapters/version_two_initialization/initialization_test_fixture.rs delete mode 100644 tests/migration_descriptor_exhaustion.rs delete mode 100644 tests/store_migration_compatibility.rs delete mode 100644 tests/store_migration_execution.rs delete mode 100644 tests/store_migration_intent.rs delete mode 100644 tests/store_migration_intent/fixture.rs delete mode 100644 tests/store_migration_intent_encoding.rs delete mode 100644 tests/store_migration_inventory.rs delete mode 100644 tests/store_migration_phase.rs delete mode 100644 tests/store_migration_receipt.rs delete mode 100644 tests/store_migration_receipt/binding_laws.rs delete mode 100644 tests/store_migration_receipt/fixture.rs delete mode 100644 tests/store_migration_receipt/harness.rs delete mode 100644 tests/store_migration_receipt_encoding.rs delete mode 100644 tests/store_migration_recovery.rs delete mode 100644 tests/store_migration_recovery_order.rs delete mode 100644 tests/store_migration_storage.rs delete mode 100644 tests/store_migration_storage/recording_storage.rs delete mode 100644 xtask/src/durability_crash_matrix/production_protocol/migration.rs delete mode 100644 xtask/src/durability_crash_matrix/production_protocol/migration_stage_prefix_laws.rs delete mode 100644 xtask/src/durability_crash_matrix/production_protocol/migration_storage.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/version_two_initialization.rs delete mode 100644 xtask/src/durability_crash_matrix/restart/migration.rs delete mode 100644 xtask/src/durability_crash_matrix/restart/migration_expectation.rs create mode 100644 xtask/src/durability_crash_matrix/restart/version_two_initialization.rs create mode 100644 xtask/src/fuzz_seed_corpus/format_marker_seeds.rs delete mode 100644 xtask/src/fuzz_seed_corpus/migration_recovery_seeds.rs delete mode 100644 xtask/src/fuzz_seed_corpus/migration_seeds.rs create mode 100644 xtask/tests/retention_store_v2_format_oracle/artifacts/format_marker.rs delete mode 100644 xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs create mode 100644 xtask/tests/retention_store_v2_protocol_contract/initialization_contract_laws.rs delete mode 100644 xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index eb251281..1ef27b17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ after its public API and format compatibility policies are established. ## [Unreleased] +- Keep has one storage format: `FilesystemVersionTwoAdmission::initialize` creates a store directly through a recoverable protocol with one durable commit point and a 41-case crash matrix (`KEEP-CRASH-081`–`091`), migration and its public API, records, golden fixtures, and crash rows `KEEP-CRASH-053`–`073` are deleted, the persisted root device-and-inode binding is removed so quiescent copies and backups open, and the format-definition digest changes to `f1a0ce10…361d` (#198). + - Version-2 stores can ingest: `FilesystemVersionTwoCatalogPublisher` stages one segment and publishes a catalog successor through the retained-closure gate, version-2 catalog-stage recovery classifies version-2 roots and finalizes a staged head only after the same gate admits it, and `KEEP-CRASH-074` through `077` cover writer process death (#198). - Version-2 catalog publication can refuse candidates that lose retained content: `publish_retained_catalog_generation` verifies every current retained closure against the candidate before any catalog mutation and revalidates the retention head before `HEAD` replacement (#125). diff --git a/README.md b/README.md index ffd34551..e2194c82 100644 --- a/README.md +++ b/README.md @@ -55,25 +55,24 @@ Keep is required to refuse all three, before mutating anything. at 156 before/during/after coordinates (`KEEP-CRASH-001`–`052`) and verifies the store lands in exactly one documented lawful state each time, for version-1 publication and for version-2 retention publication. -- **Version-2 retention and migration, forward path.** Explicit retention - roots, deterministic closure verification, a one-way 21-phase migration, - and a 17-phase retention publication — all with production filesystem - writers, all preserving every version-1 byte. Reopening a migrated store - jointly admits its marker, intent, and receipt, binds the root's device and - inode identity to the intent, and pins the directories it admitted. Mount - identity remains a check within the live migration process. Publication +- **One store format, created directly.** `FilesystemVersionTwoAdmission::initialize` + creates a version-2 store through a recoverable state machine with one + durable commit point; an interrupted call resumes, and any other nonempty + directory refuses before it is touched. A 41-case process-death matrix + (`KEEP-CRASH-081`–`091`) covers every boundary. Nothing persisted binds a + store to its directory: a quiescent copy or restored backup opens wherever + it is placed. There is no migration; see the + [retirement ledger](docs/formats/segment-store-v2/retired-identifiers.md). +- **Version-2 retention, forward path.** Explicit retention roots, + deterministic closure verification, the gated segment and catalog writer, + and a 17-phase retention publication, all with production filesystem + writers. Reopening a store admits its namespace and marker and pins the + directories it admitted. Publication binds this store's own catalog head and the catalog it selects, and refuses retained stages, superseded candidates, substituted files, replaced protocol directories, and every namespace or capacity violation before it writes anything. Each refusal is a typed value, not a string. -- **Migration restart recovery.** Recovery verifies current authority, - classifies the observed prefix, and resumes an exact persisted migration - intent through the remaining phases. An incomplete pre-effect intent stage - is discarded and rebuilt from freshly verified current intent. Its crash - matrix kills real writer processes - at 68 before/during/after coordinates (`KEEP-CRASH-053`–`073`), preserving - every version-1 byte. The [restart matrix](docs/testing-evidence/migration-restart-matrix.md) records additional hostile-prefix evidence and its limits. Version-2 complete-stage retention recovery, fenced retention snapshots and model-based transitions are implemented on main through [PR #99](https://github.com/flyingrobots/keep/pull/99). @@ -95,9 +94,7 @@ Incomplete retention stages are preserved and block publication pending explicit Complete orphans remain recovery-protected until explicit disposition lands with garbage collection (#21). -Migration restart recovery is implemented and does not grant retention authority. - -A version-1 store stays admitted until its owner migrates it. +Version-1 admission still exists and refuses every version-2 store; no conversion exists, and a later unit of [#198](https://github.com/flyingrobots/keep/issues/198) deletes version 1. | Gap | Tracked | | --- | --- | @@ -197,7 +194,7 @@ assert_eq!(output, b"exact bytes, or nothing"); # Ok::<(), Box>(()) ``` -On Linux with the admitted ext4 profile, read an existing migrated version-two store whose retention roots anchor the requested blob. The following example is also compiled in the `DurableStore` API documentation. Its 16 MiB limit applies to aggregate catalog-selected segment bytes; catalog bytes and decoded metadata allocate separately. +On Linux with the admitted ext4 profile, read an initialized version-two store whose retention roots anchor the requested blob. The following example is also compiled in the `DurableStore` API documentation. Its 16 MiB limit applies to aggregate catalog-selected segment bytes; catalog bytes and decoded metadata allocate separately. ```rust #[cfg(target_os = "linux")] @@ -242,8 +239,8 @@ Select a crash campaign with `--sequence NAME` using these exact CLI names: | `recovery-discard` | Explicit recovery evidence discard | | `initialization` | Writer-locked initialization | | `retention` | Retention root, manifest, and head publication | -| `migration` | Version-one to version-two migration | | `version-two-catalog` | Version-two catalog writer and stage recovery | +| `version-two-initialization` | Direct initialization and its recovery | ```bash cargo xtask durability-crash-matrix --sequence retention @@ -282,7 +279,7 @@ has one job; this one is the front door. | Understand what is proved and what is not | [`docs/invariants/`](docs/invariants/) | | Read the byte-level formats | [`docs/formats/`](docs/formats/) | | See the architecture and port boundaries | [`docs/architecture/`](docs/architecture/) | -| Follow the crash and recovery rules | [`segment-store-v1/recovery.md`](docs/formats/segment-store-v1/recovery.md) · [`segment-store-v2/recovery.md`](docs/formats/segment-store-v2/recovery.md) · [`segment-store-v2/migration-recovery.md`](docs/formats/segment-store-v2/migration-recovery.md) | +| Follow the crash and recovery rules | [`segment-store-v1/recovery.md`](docs/formats/segment-store-v1/recovery.md) · [`segment-store-v2/recovery.md`](docs/formats/segment-store-v2/recovery.md) · [`segment-store-v2/initialization.md`](docs/formats/segment-store-v2/initialization.md) | | See how a retention generation is published | [`segment-store-v2/retention-publication.md`](docs/formats/segment-store-v2/retention-publication.md) | | Check reproducible performance evidence | [`docs/benchmarks/`](docs/benchmarks/) | | Run the language-neutral corpora | [`conformance/`](conformance/) | diff --git a/conformance/segment-store/v2/ORIGIN.md b/conformance/segment-store/v2/ORIGIN.md index 84f00f7c..92490cc6 100644 --- a/conformance/segment-store/v2/ORIGIN.md +++ b/conformance/segment-store/v2/ORIGIN.md @@ -6,10 +6,11 @@ The corpus was constructed on 2026-07-29 with: - `cargo 1.96.0 (30a34c682 2026-05-25)`; and - `b3sum 1.8.5`. -`transitions.tsv` was added on 2026-09-30 by transcribing the 21 boundaries of -`StoreMigrationPhase::ALL` and the recovery table in -`docs/formats/segment-store-v2/migration-recovery.md`. No independent oracle -constructs this ledger. Its committed-byte guard is `transition_laws.rs` +`transitions.tsv` was first added on 2026-09-30 for the 21 migration +boundaries. On 2026-10-27 its migration rows were removed and its 11 rows were +transcribed from the initialization crash points `KEEP-CRASH-081` to `-091` +and the state table in `docs/formats/segment-store-v2/initialization.md`. No +independent oracle constructs this ledger. Its committed-byte guard is `transition_laws.rs` under `xtask/tests/retention_store_v2_protocol_contract/`. The crash matrix checks runtime behavior at the same boundaries without reading the TSV. @@ -17,18 +18,14 @@ checks runtime behavior at the same boundaries without reading the TSV. The oracle imports exact bytes only from these previously accepted fixtures: -- `conformance/segment-store/v1/one-zero-segment.hex`; -- `conformance/segment-store/v1/one-zero-catalog.hex`; -- `conformance/segment-store/v1/one-zero-head.hex`; - the one-zero `BlobId` canonical text and `LayoutId` binary identity from `conformance/layout/v1/layouts.tsv`. -It parses the version-1 head coordinate, catalog predecessor, and segment and -catalog semantic digests directly from fixed offsets. The oracle constructs the +The oracle constructs the 59-byte `BlobId` from the accepted binary grammar and verifies its length and digest against the layout table; the table directly supplies the 60-byte `LayoutId`. It does not call a production encoder, decoder, retention type, -migration adapter, serializer, or filesystem implementation. +initializer, serializer, or filesystem implementation. ## Definition verification @@ -47,7 +44,10 @@ Exact output: db1c1c1a50613ef11f7c0ee0882e37b6d24e2db2ca57783d01197ba51b61ce59 ``` -The format-definition digest was checked independently with: +The format-definition digest of the original definition was checked +independently with `b3sum`. For the current definition, this command is +expected to print the digest below, which the handwritten oracle computed; see +[Regeneration on 2026-10-27](#regeneration-on-2026-10-27): ```bash { @@ -56,12 +56,35 @@ The format-definition digest was checked independently with: } | b3sum --no-names ``` -Exact output: +Expected output: ```text -32381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427 +f1a0ce10c0d363d145f8a693ea22651e47cf356caa14f1ef4f8958d8b29f361d ``` +## Regeneration on 2026-10-27 + +Removing the migration rows from `definition.tsv` (ADR-0010, +[#198](https://github.com/flyingrobots/keep/issues/198)) changed the +definition digest from +`32381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427` to the +value above, so `format-marker.hex` and its `artifacts.tsv` row were +regenerated. The removed rows are the `domain.migration-*`, +`domain.store-identifier`, `domain.initial-retention-state`, +`domain.initial-gc-state`, `domain.empty-disposition-set`, and `migration.*` +keys; each was read only by the migration records. + +The new bytes were taken from the committed handwritten oracle, +`xtask/tests/retention_store_v2_format_oracle.rs`, which hashes the edited +`definition.tsv` and constructs the marker from fixed offsets without any +production codec: its failure output on the unchanged fixtures named the exact +expected table row and marker bytes, which were written verbatim, and the +oracle then passed. `b3sum` was not available on the regeneration host, so the +shell check above was not rerun there; a reviewer should rerun it and expect +the digest shown. The retention fixtures do not depend on the definition and +are unchanged. The migration intent, receipt, inventory, and source table +were deleted rather than regenerated. + ## Materialization boundary A temporary ignored Rust test wrote the initially reviewed TSV and hexadecimal @@ -73,5 +96,5 @@ boundary described above. Changing any fixture requires a deliberate specification change, an updated definition or profile digest when affected, fresh independent construction, -and review of every dependent migration and retention coordinate. A fixture is +and review of every dependent retention coordinate. A fixture is never regenerated to make a production implementation pass. diff --git a/conformance/segment-store/v2/README.md b/conformance/segment-store/v2/README.md index c995c8b2..836f0964 100644 --- a/conformance/segment-store/v2/README.md +++ b/conformance/segment-store/v2/README.md @@ -3,7 +3,7 @@ This corpus freezes independent canonical inputs and golden bytes for `keep.segment-store/v2`. It proves the written format has one executable byte interpretation. It does not prove that a production encoder, decoder, -migration, retention transition, or garbage collector exists. +initializer, retention transition, or garbage collector exists. ## Corpus files @@ -11,13 +11,9 @@ migration, retention transition, or garbage collector exists. | --- | --- | | `definition.tsv` | Sorted format-definition key/value bytes | | `retention-profile.tsv` | Registered realization-profile definition | -| `inventory.tsv` | Canonical one-segment, one-catalog migration inventory | -| `migration-source.tsv` | Exact version-1 and derived migration coordinates | | `artifacts.tsv` | Golden artifact lengths, digests, checksums, and filenames | -| `transitions.tsv` | One stable crash identifier per migration boundary, `KEEP-CRASH-053` to `-073` | +| `transitions.tsv` | One stable crash identifier per initialization boundary, `KEEP-CRASH-081` to `-091` | | `format-marker.hex` | Canonical 96-byte `FORMAT` record | -| `migration-intent.hex` | Canonical 256-byte migration intent | -| `migration-receipt.hex` | Canonical 256-byte migration receipt | | `one-anchor-root.hex` | Generation-1 root with one nontext namespace | | `one-root-manifest.hex` | Generation-1 one-namespace manifest | | `one-root-head.hex` | Generation-1 retention head | @@ -26,10 +22,14 @@ migration, retention transition, or garbage collector exists. Every text file uses UTF-8 or ASCII, LF line endings, and one final newline. Every hex fixture is one lowercase hexadecimal line with one final newline. In `artifacts.tsv`, `bound_digest_hex` is the marker content digest for -`format-marker`, the intent digest for `migration-intent`, the referenced -intent digest for `migration-receipt`, the canonical record digest for -`retention-root` and `retention-manifest`, and the referenced manifest digest -for `retention-head`. +`format-marker`, the canonical record digest for `retention-root` and +`retention-manifest`, and the referenced manifest digest for +`retention-head`. + +The migration records, their inventory and source tables, and the +`KEEP-CRASH-053` to `-073` transition rows were removed with migration; the +[retirement ledger](../../../docs/formats/segment-store-v2/retired-identifiers.md) +records their last evidence. ## Frozen identities @@ -39,18 +39,10 @@ It hashes the exact `retention-profile.tsv` bytes under the registered profile domain. The format-definition digest is -`32381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427`. +`f1a0ce10c0d363d145f8a693ea22651e47cf356caa14f1ef4f8958d8b29f361d`. It hashes the exact `definition.tsv` bytes under the registered format domain. The definition binds the profile digest, every named domain, magic, version, -field order, record width, format limit, and migration synchronization mask. - -The migration fixture preserves the version-1 one-zero segment and generation-1 -catalog. Its canonical two-entry inventory digest is -`40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9`. -The derived logical store identifier is -`0cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd79`. -Fixture-only root device, mount, and file coordinates are `1`, `2`, and `3`; -they bind in-place recovery but do not enter the logical store identifier. +field order, record width, and format limit. The retention fixture uses namespace bytes `00 2f ff`, proving the namespace is opaque and not a path or Unicode string. Its one anchor combines the canonical @@ -58,21 +50,21 @@ one-zero `BlobId` and `LayoutId` values from the existing layout corpus. ## Transition protocol -`transitions.tsv` mirrors the version-1 table: one row per migration +`transitions.tsv` mirrors the version-1 table: one row per initialization durability operation with its pre-state, interrupted-state classification, -post-state, and recovery posture. `KEEP-CRASH-053`, `-062`, and `-068` are -the only rows whose interruption may leave an incomplete pre-effect stage and -therefore the only rows that plan a discard; `-072` and `-073` admit the -complete migration. - -The `cargo xtask durability-crash-matrix --sequence migration` harness -executes 68 canonical process-death cases at the same 21 boundaries: each at -three positions plus one `during` case per admitted directory-prefix length -for `KEEP-CRASH-060`. It kills an isolated writer process group, compares the +post-state, and recovery posture. `KEEP-CRASH-082` and `-091` are the only +rows whose interruption may leave an incomplete pre-commit stage and therefore +the only rows that plan a discard; `-089` and `-090` admit the committed +store. + +The `cargo xtask durability-crash-matrix --sequence version-two-initialization` +harness executes 41 canonical initialization process-death cases at the same +11 boundaries: each at three positions plus one `during` case per directory +for `KEEP-CRASH-086`. It kills an isolated writer process group, compares the restarted root against an independent expected-state model, requires the -production planner to report the predicted recovery plan, and requires the -recovered store to be one complete migration with every version-1 byte -intact. Host power loss remains outside its claim. +resumed initializer to report the predicted state, and requires the recovered +store to publish, retain, and reconstruct its first blob. Host power loss +remains outside its claim. ## Verification @@ -95,7 +87,5 @@ shape, operation order, and recovery-posture claims. The format oracle does not construct or compare `transitions.tsv`; the crash matrix checks runtime behavior without reading its bytes. -Passing this corpus is necessary but insufficient for migration recovery -(#108), restart corruption (#111), or compatibility and fuzz coverage (#112). -Production code still needs parser, corruption, property, model, crash, recovery, -concurrency, fuzz, and public API evidence. +Production code still needs parser, corruption, property, model, crash, +recovery, concurrency, fuzz, and public API evidence. diff --git a/conformance/segment-store/v2/artifacts.tsv b/conformance/segment-store/v2/artifacts.tsv index dace87c5..dadd99de 100644 --- a/conformance/segment-store/v2/artifacts.tsv +++ b/conformance/segment-store/v2/artifacts.tsv @@ -1,8 +1,6 @@ keep.segment-store-v2.artifacts/v1 case kind byte_length generation entry_count bound_digest_hex final_checksum_hex fixture -format-marker format-marker 96 - - 4b063c329085abdebe86b256d531b112c7ea33cb2f545caa40a7a869ff3337ce 06384cbaf2b69e0a12eeb2bf62df4c49e193d56f2bde940b3c5637320458abc1 format-marker.hex -migration-intent migration-intent 256 1 2 a15a00000219df20979da36419046eae9a0ba998645fbfe308ea4335a8326b44 7bec10cc8c1eef5ab0e8e8b6a33240bba291252d4263147df134062eb70d3f1f migration-intent.hex -migration-receipt migration-receipt 256 1 2 a15a00000219df20979da36419046eae9a0ba998645fbfe308ea4335a8326b44 3a6a5f29bfafeffb9401de5ba814c09c345adbad69e8ba0531e3eb1ebb0b681d migration-receipt.hex +format-marker format-marker 96 - - ec115cb44700d1f12ad882f1ea07d0921c65c21c4ad7dc80e4448ad459d20ce6 ea63c9e69346312f60dfb896de4a507adf2bd58df2bb6ba5a269e5c044967e90 format-marker.hex one-anchor-root retention-root 378 1 1 ca4c11f265c3bed07073bdc3b6aef003e964ac8cb36fcfcc92f20fa6f0b60085 28c52ff0f8d6533234be083f425e921d699639e204e2c66dec0cae2ff0a2dc34 one-anchor-root.hex one-root-manifest retention-manifest 296 1 1 f46b96a2bf3379320cf59e8af15b9d108de06025c415307b28953714bd7a80eb 10597643c3fc9485c7ecd3bb511d6726e726fd92f0f769a204b899c5fdc77d2c one-root-manifest.hex one-root-head retention-head 144 1 1 f46b96a2bf3379320cf59e8af15b9d108de06025c415307b28953714bd7a80eb ac049edb33af7e957c6ff11ead7e1bcf9c40fa9793cc84979215ffbba5f630b7 one-root-head.hex diff --git a/conformance/segment-store/v2/definition.tsv b/conformance/segment-store/v2/definition.tsv index cea06274..e6afdffe 100644 --- a/conformance/segment-store/v2/definition.tsv +++ b/conformance/segment-store/v2/definition.tsv @@ -1,6 +1,5 @@ keep.segment-store.definition/v2 key value -domain.empty-disposition-set keep.empty-disposition-set/v2\0 domain.format-definition keep.segment-store-definition/v2\0 domain.format-marker keep.store-format-marker/v2\0 domain.format-marker-checksum keep.segment-store-marker-checksum/v2\0 @@ -8,12 +7,6 @@ domain.gc-candidate-set keep.gc-candidate-set/v2\0 domain.gc-intent keep.gc-retirement-intent/v2\0 domain.gc-intent-checksum keep.gc-retirement-intent-checksum/v2\0 domain.gc-receipt-checksum keep.gc-retirement-receipt-checksum/v2\0 -domain.initial-gc-state keep.initial-gc-state/v2\0 -domain.initial-retention-state keep.initial-retention-state/v2\0 -domain.migration-intent keep.store-migration-intent/v2\0 -domain.migration-intent-checksum keep.store-migration-intent-checksum/v2\0 -domain.migration-inventory keep.store-v1-pool-inventory/v2\0 -domain.migration-receipt-checksum keep.store-migration-receipt-checksum/v2\0 domain.recovery-disposition-checksum keep.recovery-disposition-receipt-checksum/v2\0 domain.retention-anchor-set keep.retention-anchor-set/v2\0 domain.retention-head-checksum keep.retention-head-checksum/v2\0 @@ -24,7 +17,6 @@ domain.retention-namespace keep.retention-namespace/v1\0 domain.retention-profile keep.retention-realization-profile/v1\0 domain.retention-root keep.retention-root/v2\0 domain.retention-root-checksum keep.retention-root-checksum/v2\0 -domain.store-identifier keep.store-identifier/v2\0 format.coordinate keep.segment-store/v2 format.marker.fields magic:16,version:u16,record_length:u16,flags:u32,definition_digest:32,maximum_namespace_count:u32,reserved:u32,checksum:32 format.marker.length 96 @@ -41,18 +33,6 @@ gc.receipt.fields magic:16,version:u16,record_length:u16,flags:u32,generation:u6 gc.receipt.length 320 gc.receipt.magic KEEP:GC:RECEIPT2 gc.receipt.version 2 -migration.intent.fields magic:16,version:u16,record_length:u16,flags:u32,catalog_generation:u64,catalog_length:u64,catalog_digest:32,predecessor_digest:32,inventory_digest:32,root_device:u64,root_mount:u64,root_file:u64,definition_digest:32,store_id:32,checksum:32 -migration.intent.length 256 -migration.intent.magic KEEP:MIG:INT2\0\0\0 -migration.intent.version 2 -migration.inventory.entry-fields kind:u8,reserved:7,catalog_generation:u64,artifact_length:u64,artifact_digest:32 -migration.inventory.entry-width 56 -migration.inventory.maximum-entries 2097152 -migration.receipt.fields magic:16,version:u16,record_length:u16,flags:u32,intent_digest:32,store_id:32,format_marker_digest:32,initial_retention_digest:32,initial_gc_digest:32,disposition_set_digest:32,synchronization_mask:u64,checksum:32 -migration.receipt.length 256 -migration.receipt.magic KEEP:MIG:REC2\0\0\0 -migration.receipt.synchronization-mask 0x00000000000003ff -migration.receipt.version 2 recovery.disposition.fields magic:16,version:u16,record_length:u16,flags:u32,artifact_kind:u16,decision:u16,classification:u16,reserved:u16,artifact_length:u64,artifact_identity_digest:32,artifact_content_digest:32,publication_generation:u64,publication_checksum:32,catalog_generation:u64,catalog_digest:32,liveness_generation:u64,manifest_digest:32,reader_device:u64,reader_mount:u64,reader_file:u64,decision_evidence_digest:32,reserved:8,checksum:32 recovery.disposition.length 320 recovery.disposition.magic KEEP:REC:DISP2\0\0 diff --git a/conformance/segment-store/v2/format-marker.hex b/conformance/segment-store/v2/format-marker.hex index 30640a92..0445623f 100644 --- a/conformance/segment-store/v2/format-marker.hex +++ b/conformance/segment-store/v2/format-marker.hex @@ -1 +1 @@ -4b4545503a53544f52453a5632000000000200600000000032381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427000010000000000006384cbaf2b69e0a12eeb2bf62df4c49e193d56f2bde940b3c5637320458abc1 +4b4545503a53544f52453a56320000000002006000000000f1a0ce10c0d363d145f8a693ea22651e47cf356caa14f1ef4f8958d8b29f361d0000100000000000ea63c9e69346312f60dfb896de4a507adf2bd58df2bb6ba5a269e5c044967e90 diff --git a/conformance/segment-store/v2/inventory.tsv b/conformance/segment-store/v2/inventory.tsv deleted file mode 100644 index cd0443ae..00000000 --- a/conformance/segment-store/v2/inventory.tsv +++ /dev/null @@ -1,4 +0,0 @@ -keep.segment-store-v2.inventory/v1 -kind generation byte_length artifact_digest_hex source_fixture -segment 0 337 b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc one-zero-segment.hex -catalog 1 352 04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320 one-zero-catalog.hex diff --git a/conformance/segment-store/v2/migration-intent.hex b/conformance/segment-store/v2/migration-intent.hex deleted file mode 100644 index 5ce426b9..00000000 --- a/conformance/segment-store/v2/migration-intent.hex +++ /dev/null @@ -1 +0,0 @@ -4b4545503a4d49473a494e543200000000020100000000000000000000000001000000000000016004b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320000000000000000000000000000000000000000000000000000000000000000040bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f900000000000000010000000000000002000000000000000332381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b8734270cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd797bec10cc8c1eef5ab0e8e8b6a33240bba291252d4263147df134062eb70d3f1f diff --git a/conformance/segment-store/v2/migration-receipt.hex b/conformance/segment-store/v2/migration-receipt.hex deleted file mode 100644 index 66b524ea..00000000 --- a/conformance/segment-store/v2/migration-receipt.hex +++ /dev/null @@ -1 +0,0 @@ -4b4545503a4d49473a524543320000000002010000000000a15a00000219df20979da36419046eae9a0ba998645fbfe308ea4335a8326b440cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd794b063c329085abdebe86b256d531b112c7ea33cb2f545caa40a7a869ff3337ced52f1f022edb1de7b840c5bf8fb55de7932ca69370ae85e2bee4179143792bc3ba0ea200a5b06741564c43a79a91945bef0b0fac51c960ea4f8207094f3e1e31a80259fcd1237203ea6c6cc5065514abdeb01da603c3194b096a045cf694c95a00000000000003ff3a6a5f29bfafeffb9401de5ba814c09c345adbad69e8ba0531e3eb1ebb0b681d diff --git a/conformance/segment-store/v2/migration-source.tsv b/conformance/segment-store/v2/migration-source.tsv deleted file mode 100644 index 5d0204bf..00000000 --- a/conformance/segment-store/v2/migration-source.tsv +++ /dev/null @@ -1,3 +0,0 @@ -keep.segment-store-v2.migration-source/v1 -case catalog_generation catalog_length catalog_digest_hex predecessor_digest_hex inventory_digest_hex definition_digest_hex store_id_hex root_device root_mount root_file -one-zero 1 352 04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320 0000000000000000000000000000000000000000000000000000000000000000 40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9 32381f1ac332d1277a7e1faf8f11576993cb55b7e85d2a110b74dc9c3b873427 0cd9d3dfbec9b349fe42d21475271b0e8de23c043440d6427a1c37898ad1dd79 1 2 3 diff --git a/conformance/segment-store/v2/transitions.tsv b/conformance/segment-store/v2/transitions.tsv index cce218d3..3cd06375 100644 --- a/conformance/segment-store/v2/transitions.tsv +++ b/conformance/segment-store/v2/transitions.tsv @@ -1,23 +1,13 @@ keep.segment-store.transitions/v2 crash_id phase operation pre_state interrupted_class post_state recovery_posture -KEEP-CRASH-053 migration write-intent-stage admitted-version-one-store absent-or-incomplete-intent-stage complete-intent-stage discard-incomplete-stage-or-resume-stage-sync -KEEP-CRASH-054 migration sync-intent-stage complete-intent-stage complete-intent-stage durable-intent-stage resume-stage-sync -KEEP-CRASH-055 migration link-intent durable-intent-stage durable-intent-stage-or-linked-intent linked-intent verify-no-clobber-link-and-resume-root-sync -KEEP-CRASH-056 migration sync-root-after-intent linked-intent linked-intent durable-intent resume-root-sync -KEEP-CRASH-057 migration remove-intent-stage durable-intent durable-intent-with-or-without-stage durable-intent-alone resume-cleanup-sync -KEEP-CRASH-058 migration sync-root-after-intent-cleanup durable-intent-alone durable-intent-alone intent-cleanup-synchronized resume-cleanup-sync -KEEP-CRASH-059 migration admit-reader-fence intent-cleanup-synchronized intent-with-or-without-reader-fence reader-fence-admitted resume-namespace-prefix -KEEP-CRASH-060 migration admit-namespace-prefix reader-fence-admitted directory-prefix-length-zero-to-six namespace-prefix-admitted resume-namespace-prefix-or-root-sync -KEEP-CRASH-061 migration sync-root-after-namespace namespace-prefix-admitted namespace-prefix-admitted durable-namespace-prefix resume-root-sync -KEEP-CRASH-062 migration write-marker-stage durable-namespace-prefix absent-or-incomplete-marker-stage complete-marker-stage discard-incomplete-stage-or-resume-stage-sync -KEEP-CRASH-063 migration sync-marker-stage complete-marker-stage complete-marker-stage durable-marker-stage resume-stage-sync -KEEP-CRASH-064 migration link-marker durable-marker-stage durable-marker-stage-or-linked-marker linked-marker verify-no-clobber-link-and-resume-root-sync -KEEP-CRASH-065 migration sync-root-after-marker linked-marker linked-marker durable-marker resume-root-sync -KEEP-CRASH-066 migration remove-marker-stage durable-marker durable-marker-with-or-without-stage durable-marker-alone resume-cleanup-sync -KEEP-CRASH-067 migration sync-root-after-marker-cleanup durable-marker-alone durable-marker-alone marker-cleanup-synchronized resume-cleanup-sync -KEEP-CRASH-068 migration write-receipt-stage marker-cleanup-synchronized absent-or-incomplete-receipt-stage complete-receipt-stage discard-incomplete-stage-or-resume-stage-sync -KEEP-CRASH-069 migration sync-receipt-stage complete-receipt-stage complete-receipt-stage durable-receipt-stage resume-stage-sync -KEEP-CRASH-070 migration link-receipt durable-receipt-stage durable-receipt-stage-or-linked-receipt linked-receipt verify-no-clobber-link-and-resume-root-sync -KEEP-CRASH-071 migration sync-root-after-receipt linked-receipt linked-receipt durable-receipt resume-root-sync -KEEP-CRASH-072 migration remove-receipt-stage durable-receipt durable-receipt-with-or-without-stage complete-migration admit-complete-migration -KEEP-CRASH-073 migration sync-root-after-receipt-cleanup complete-migration complete-migration durable-complete-migration admit-complete-migration +KEEP-CRASH-081 initialization open-and-lock-writer-file empty-root empty-or-locked-root locked-root resume-marker-stage-write +KEEP-CRASH-082 initialization write-marker-stage locked-root absent-or-incomplete-marker-stage complete-marker-stage discard-incomplete-stage-or-resume-stage-sync +KEEP-CRASH-083 initialization sync-marker-stage complete-marker-stage complete-marker-stage durable-marker-stage resume-stage-sync +KEEP-CRASH-084 initialization sync-root-after-marker-stage durable-marker-stage durable-marker-stage durable-marker-stage-name resume-stage-sync +KEEP-CRASH-085 initialization admit-reader-fence durable-marker-stage-name marker-stage-with-or-without-reader-fence reader-fence-admitted resume-stage-sync +KEEP-CRASH-086 initialization admit-directory reader-fence-admitted directory-prefix-length-zero-to-nine complete-namespace resume-stage-sync +KEEP-CRASH-087 initialization link-marker complete-namespace marker-stage-or-linked-marker linked-marker verify-no-clobber-link-and-resume-commit-sync +KEEP-CRASH-088 initialization sync-root-after-marker linked-marker linked-marker committed-marker resume-commit-sync +KEEP-CRASH-089 initialization remove-marker-stage committed-marker linked-or-committed-marker committed-store resume-commit-sync-or-admit-committed-store +KEEP-CRASH-090 initialization sync-root-after-cleanup committed-store committed-store durable-committed-store admit-committed-store +KEEP-CRASH-091 initialization discard-incomplete-marker-stage incomplete-marker-stage incomplete-or-discarded-marker-stage locked-root discard-incomplete-stage-or-resume-stage-write diff --git a/docs/adr/retention-reader-root-identity.md b/docs/adr/retention-reader-root-identity.md index d3512c51..fc0a6ba0 100644 --- a/docs/adr/retention-reader-root-identity.md +++ b/docs/adr/retention-reader-root-identity.md @@ -1,6 +1,6 @@ # Retention reader root identity -Status: Accepted +Status: Superseded by [ADR-0010](https://github.com/flyingrobots/keep/pull/199) (#198). The persisted binding this record relied on lived in `migration.intent`, which was deleted with migration; readers now admit the committed namespace and the `FORMAT` marker, and a store is portable. The live checks that remain are listed under [Portable stores](../formats/segment-store-v2/recovery.md#portable-stores). The text below is kept as the record of the retired decision. Joint admission of `FORMAT`, `migration.intent`, and `migration.receipt` establishes that those records agree with each other. It does not establish that the directory receiving the read request is the directory the migration intent names. Reader admission must compare the opened root's physical identity before accepting the version-two read capability. diff --git a/docs/dependencies/blake3-1.8.7.md b/docs/dependencies/blake3-1.8.7.md index a83bf7af..79338ae1 100644 --- a/docs/dependencies/blake3-1.8.7.md +++ b/docs/dependencies/blake3-1.8.7.md @@ -10,7 +10,7 @@ ## Admitted use -Keep admits the `blake3` crate at locked version 1.8.7 for the 32-byte `BlobId` and `ChunkId` version-1 digests and the existing domain-separated segment, catalog, layout, retention and migration hashes. No dependency-owned type appears in Keep's public API. +Keep admits the `blake3` crate at locked version 1.8.7 for the 32-byte `BlobId` and `ChunkId` version-1 digests and the existing domain-separated segment, catalog, layout, retention and format-marker hashes. No dependency-owned type appears in Keep's public API. The private `xtask` crate also uses the same pinned implementation to recompute Golden File Worldline digest witnesses directly from corpus source bytes and to name deterministic fuzz seeds. The oracle does not import Keep production types or hashing wrappers. This separation independently checks preimage framing, length encoding, canonical text and binary encodings, mutation semantics, and committed witness bytes. For every identity and content mutation, the repository checker also streams the canonical preimage through external `b3sum`; a mismatch with the in-process result is a refusal. Golden File Worldline and protocol conformance share one deadline-bounded process adapter, but retain separate preimage construction. The [Golden File Worldline reference model](../conformance/golden-file-worldline.md#reference-model) records the process-adapter contract. The checked-in vectors and runtime cross-check therefore cover the algorithm boundary without claiming that the Rust path independently implements the BLAKE3 compression function. diff --git a/docs/formats/README.md b/docs/formats/README.md index 17cdb32c..3ccbaaf3 100644 --- a/docs/formats/README.md +++ b/docs/formats/README.md @@ -9,7 +9,7 @@ admitted merely because one Rust type can serialize and deserialize it. | --- | --- | --- | --- | | [Flat Chunk Layout v1](flat-chunk-layout-v1/README.md) | `keep.flat-chunks/v1` | Implemented through verified reconstruction in issues #10 and #13 | [Golden corpus](../../conformance/layout/v1/README.md) | | [Durable Segment Store v1](segment-store-v1/README.md) | `keep.segment-store/v1` | Implemented through initialization, publication, restart, and recovery in issues #14–#17 | [Golden corpus](../../conformance/segment-store/v1/README.md) | -| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | One-way migration, version-two reopen, and forward retention publication implemented; retention recovery, reader fencing, and collection planned in issue #19 | [Golden corpus](../../conformance/segment-store/v2/README.md) | +| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | The one store format: direct initialization, reopen, the gated catalog writer and its recovery, and retention publication and recovery implemented; collection planned in issue #21 | [Golden corpus](../../conformance/segment-store/v2/README.md) | The registry records protocol specifications, including formats whose implementation is still planned. Each format page states its exact proof diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index f81d0d89..91f3ce5c 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -128,7 +128,7 @@ cargo xtask durability-crash-matrix ``` The 105 version-one cases are a subset of the complete command, which also -executes version-two migration cases. The version-one subset executes the +executes version-two cases. The version-one subset executes the three ordered positions for each stable `KEEP-CRASH-001`–`KEEP-CRASH-035` point. Each version-one case owns a fresh filesystem store and an isolated child process group. The child retains diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index f56bbd45..fef0360c 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -1,14 +1,15 @@ # Durable Segment Store Version 2 -`keep.segment-store/v2` is the accepted successor to -`keep.segment-store/v1`. It preserves every admitted version-1 segment, -catalog, and publication-head byte while adding explicit retention state, -reader fences, migration evidence, and reserved GC and recovery-disposition -namespaces. +`keep.segment-store/v2` is the successor to `keep.segment-store/v1` and the +one store format Keep creates. It keeps the version-1 segment, catalog, and +publication-head grammars byte for byte and adds the `FORMAT` marker, the +reader fence, explicit retention state, and reserved GC and +recovery-disposition namespaces. -ADR-0009 owns the cross-cutting retention and liveness decision. These pages own its durable representation. - -The one-way migration, version-two reopen, forward retention publication, partial-prefix migration recovery, complete-stage retention recovery, and fenced retention snapshots are implemented on main through [PR #99](https://github.com/flyingrobots/keep/pull/99). +ADR-0009 owns the cross-cutting retention and liveness decision. +[ADR-0010](https://github.com/flyingrobots/keep/pull/199) makes this the only +storage format and replaces migration with direct initialization. These pages +own the durable representation. The [requirements ledger](requirements.md) records evidence and remaining obligations; the laws below are normative requirements, not a claim that every corresponding runtime surface exists. @@ -17,10 +18,13 @@ The [requirements ledger](requirements.md) records evidence and remaining obliga Version 2 retains every version-1 physical law and adds these: - the format version is explicit and cannot be inferred from path existence; -- a complete version-2 store is entered only by the specified version-1 - migration; direct version-2 initialization is undefined; -- migration is one-way, writer-authorized, durable, and recoverable from every - documented prefix; +- a store is created only by direct initialization, a recoverable state + machine under writer authority with one durable commit point, and it does + not admit before that point; +- an initialized store with no published catalog is a defined, admitted + state; +- nothing persisted binds a store to its directory: a quiescent copy is the + same store, and live authority comes from the opening process; - retention authority exists only through one verified retention head and its complete immutable manifest; - each manifest binds every admitted namespace to one exact root generation @@ -38,6 +42,12 @@ Version 2 retains every version-1 physical law and adds these: The following pages form one protocol: +- [Direct initialization](initialization.md) owns how a store comes into + existence: the forward protocol, its commit point, every interrupted state + and its one result, and the initialization process-death boundaries. +- [Format, namespace, and portability](recovery.md) owns the exact root + namespace, the `FORMAT` marker, the reader fence, the GC and + recovery-disposition reservation, and the laws that keep a store portable. - [Retention records](retention.md) owns canonical namespace, root-generation, manifest, and retention-head rules. - [Retention publication](retention-publication.md) owns closure admission and @@ -52,23 +62,16 @@ The following pages form one protocol: ingress proof and its exact refusal evidence. - [GC and disposition records](gc.md) owns the canonical planned intent, completion, and recovery-disposition byte grammars. -- [Migration and recovery](recovery.md) owns the exact root namespace, - version marker, reader fence, migration records, GC reservation, - recovery-disposition reservation, and restart behavior. -- [Migration protocol and recovery](migration-recovery.md) owns the ordered - one-way migration protocol and partial-migration recovery. -- [Migration crash points](migration-crash.md) owns fixed-stage publication and - the exact process-death boundaries for migration. -- [Migration inventory](migration-inventory.md) owns the bounded canonical - digest over preserved version-1 immutable pools. - [Requirements and evidence](requirements.md) owns stable requirement and crash identifiers, evidence status, compatibility, and nonclaims. +- [Retired identifiers](retired-identifiers.md) maps every retired + requirement and crash identifier to its last evidence. - [Format rationale](rationale.md) records format-local choices and rejected alternatives. The [version-2 golden corpus](../../../conformance/segment-store/v2/README.md) -freezes independent definition, profile, inventory, and record bytes. It is -format evidence, not production-writer evidence. +freezes independent definition, profile, marker, and retention record bytes. +It is format evidence, not production-writer evidence. The version-1 [segment](../segment-store-v1/segment.md), [catalog](../segment-store-v1/catalog.md), and @@ -78,23 +81,23 @@ re-encode them. ## Status -The format contract is frozen by ADR-0009 and this specification. +The format contract is frozen by ADR-0009, ADR-0010, and this specification. Implemented with executable evidence: public core types for namespaces, generations, realization profiles, closure policies, anchors, and roots; -canonical root, manifest, and head codecs matching their golden records; -storage-independent transition planning, bounded closure verification against -one pinned catalog, preflight, preparation, and the 17-phase publication port; -fresh writer-locked filesystem migration through all 21 phases, refusing a -version-one store that still holds a retained stage; -`FilesystemVersionTwoAdmission::reopen`, which jointly admits the marker, -intent, and receipt, binds the root's restart-stable device and inode identity -to the intent, and pins the retention directories it admitted; and -`FilesystemRetentionPublicationAuthority`, which publishes initial and -successor generations against the observed head, binds this store's catalog -head and the catalog it selects, and refuses superseded candidates, retained -stages, replaced protocol directories, and every namespace or capacity -violation before mutation, each as a typed `RetentionCurrentStateRefusal`. +canonical root, manifest, head, and marker codecs matching their golden +records; storage-independent transition planning, bounded closure +verification against one pinned catalog, preflight, preparation, and the +17-phase publication port; `FilesystemVersionTwoAdmission::initialize`, which +creates or resumes a store through the initialization protocol and its 41-case +process-death matrix; `FilesystemVersionTwoAdmission::reopen`, which admits +the committed namespace and the marker and pins the retention directories it +admitted; and `FilesystemRetentionPublicationAuthority`, which publishes +initial and successor generations against the observed head, binds this +store's catalog head and the catalog it selects, and refuses superseded +candidates, retained stages, replaced protocol directories, and every +namespace or capacity violation before mutation, each as a typed +`RetentionCurrentStateRefusal`. Retention publication recovery, fenced retention snapshots and model-based transition evidence are implemented on main. @@ -108,7 +111,7 @@ Writer authority coordinates cooperating writers in a managed namespace; it does `FilesystemRetentionSnapshot` binds a catalog, retention head and manifest under a shared reader fence and verifies selected roots on demand. `DurableStore` composes that view with authenticated whole-blob and exact-range reads, delivered in [PR #164](https://github.com/flyingrobots/keep/pull/164); the [read contract](../../invariants/authenticated-reconstruction/README.md) records proof and allocation limits. -The candidate-catalog retained-closure gate is implemented over its storage port ([#125](https://github.com/flyingrobots/keep/issues/125)). `FilesystemVersionTwoCatalogPublisher` ingests one segment and publishes a catalog successor through it, and version-two next-head finalization runs the same admission before replacing `HEAD` ([#198](https://github.com/flyingrobots/keep/issues/198)); see [catalog publication and stage recovery](catalog-publication.md). Verifying a new retention root against the current catalog is a different operation. +The candidate-catalog retained-closure gate is implemented over its storage port ([#125](https://github.com/flyingrobots/keep/issues/125)). `FilesystemVersionTwoCatalogPublisher` ingests one segment and publishes a catalog successor through it, including the first catalog of an initialized store, and version-two next-head finalization runs the same admission before replacing `HEAD` ([#198](https://github.com/flyingrobots/keep/issues/198)); see [catalog publication and stage recovery](catalog-publication.md). Verifying a new retention root against the current catalog is a different operation. [Durable verification reports](../../invariants/verification/README.md) are delivered in [PR #165](https://github.com/flyingrobots/keep/pull/165), with explicit subject/depth evidence and typed non-success outcomes. @@ -116,11 +119,4 @@ Production durable ingestion ([#82](https://github.com/flyingrobots/keep/issues/ The `KEEP-CRASH-036..052` initial-publication process-death sequence includes independent recovered-reader checks; its [evidence receipt](../../testing-evidence/retention-crash-reader-oracle.md) records the assertions, calibration, and scope. -Partial-prefix migration recovery and the 68-case `KEEP-CRASH-053..073` -process-death matrix are implemented. - -The [migration restart matrix](../../testing-evidence/migration-restart-matrix.md) and [compatibility and fuzz evidence](../../testing-evidence/migration-compatibility-fuzz.md) record the additional merged #111/#112 evidence and its remaining limits. - -Reopen compares restart-stable root coordinates, device and inode, against the intent; see [root identity across restart](recovery.md#root-identity-across-restart). - -A version-1 store remains admitted until its owner migrates it. The [requirements ledger](requirements.md) records implementation and evidence gaps; planned cases are not proof. +Version-1 admission still exists and refuses every version-2 store; version-2 admission refuses every version-1 store. No conversion between them exists. Version-1 admission is removed by a later unit of [#198](https://github.com/flyingrobots/keep/issues/198). diff --git a/docs/formats/segment-store-v2/catalog-publication.md b/docs/formats/segment-store-v2/catalog-publication.md index 5f06480a..ec3d841b 100644 --- a/docs/formats/segment-store-v2/catalog-publication.md +++ b/docs/formats/segment-store-v2/catalog-publication.md @@ -1,11 +1,11 @@ # Catalog Publication and Stage Recovery -This page owns segment ingestion and catalog publication on a completely -migrated `keep.segment-store/v2` store, and recovery of the catalog-protocol +This page owns segment ingestion and catalog publication on a committed +`keep.segment-store/v2` store, and recovery of the catalog-protocol stages that publication leaves. It adds no byte format: the segment, catalog, and publication-head grammars stay the version-1 grammars, and the retention records stay those of [Retention records](retention.md). Version-2 stores are -still created only by migration. +created by [direct initialization](initialization.md). ## Supported envelope @@ -17,8 +17,10 @@ still created only by migration. references. The version-1 catalog, segment, and head bounds apply unchanged. - Retention evidence is bounded by one manifest of at most 4,096 entries and one root at a time. Each closure is bounded by its root's admitted limits. -- The store must be a completely migrated version-2 root. Direct version-2 - initialization is still undefined. +- The store must be a committed version-2 root. A freshly initialized store + has no `HEAD`; its first publication uses the uninitialized expectation, + has no predecessor, and is admitted only when `segments`, `catalogs`, and + both retention pools are empty. ## Authority @@ -103,8 +105,7 @@ must resolve it first. `FilesystemVersionTwoRecoveryAdmission::reopen` applies the writer admission with one difference: the root may also hold `head.next`. It requires the platform profile, the writer lock, the exact version-2 namespace and protocol -directories, the jointly admitted marker, intent, and receipt, and the -restart-stable root identity. The recovery tools take it through their +directories, and the canonical `FORMAT` marker. The recovery tools take it through their `open_version_two` constructors: - `FilesystemRecoveryStageDiscarder`; @@ -113,12 +114,13 @@ restart-stable root identity. The recovery tools take it through their - `FilesystemRecoveryNextHeadFinalizer`. The read-only `FilesystemRecoveryInventoryReader::open_version_two` applies -the same namespace, record, and identity laws without the writer lock. +the same namespace and marker laws without the writer lock. `classify_version_two_recovery_names` classifies `reader.lock`, `FORMAT`, -`migration.intent`, `migration.receipt`, `retention`, `gc`, and `recovery` as -known roles. It requires each of them and `HEAD`. Migration stages are unknown -entries. Version-2 admission owns the contents of the three protocol +`retention`, `gc`, and `recovery` as known roles and requires each of them. It +does not require `HEAD`: the first publication of an initialized store can be +interrupted before `HEAD` exists. `FORMAT.next` is an unknown entry. +Version-2 admission owns the contents of the three protocol directories, and retention recovery owns `retention`. Stage discard, completion, and segment resume execute the version-1 diff --git a/docs/formats/segment-store-v2/initialization.md b/docs/formats/segment-store-v2/initialization.md new file mode 100644 index 00000000..42790f76 --- /dev/null +++ b/docs/formats/segment-store-v2/initialization.md @@ -0,0 +1,190 @@ +# Direct Initialization + +This page owns how a `keep.segment-store/v2` store comes into existence. It +adds no byte format: `FORMAT` keeps the grammar in +[Format marker](recovery.md#format-marker), and every other record keeps the +grammar of its own page. [ADR-0010](https://github.com/flyingrobots/keep/pull/199) +records the decision that initialization replaces migration as the only entry +into the store format. + +## Committed namespace + +An initialized store root holds exactly: + +```text +writer.lock +reader.lock +FORMAT +staging/ +segments/ +catalogs/ +retention/roots/ +retention/manifests/ +gc/ +recovery/dispositions/ +``` + +Catalog `HEAD` is absent until the first catalog is published. Each directory +is empty, and `writer.lock` and `reader.lock` are zero-length regular files. +Nothing in the store records the root directory's device, inode, mount, or +path, so a quiescent copy of a committed store is the same store; see +[Portable stores](recovery.md#portable-stores). + +A committed store with no catalog `HEAD` is a defined state, not an inferred +one. Writer reopen admits it. The first catalog publication proves the +absence instead of assuming it: it refuses unless `segments` and `catalogs` +are empty, and the retained-closure gate refuses unless `retention/roots` and +`retention/manifests` are empty, as +[catalog publication](catalog-publication.md#forward-protocol) specifies. A +reader refuses it with `FilesystemRetentionSnapshotError::Unpublished` after +admission and the reader fence. + +## Forward protocol + +`FilesystemVersionTwoAdmission::initialize` runs under exclusive writer +authority. It performs these phases in order: + +| Phase | Operation | Durable effect | +| --- | --- | --- | +| `AdmitPlatform` | open the root no-follow and admit the platform profile | none | +| `ObservePrefix` | classify the root without mutation | none | +| `OpenAndLockWriterFile` | create or open `writer.lock`, lock it, synchronize it and the root | `writer.lock` | +| `DiscardIncompleteMarkerStage` | only on resume: remove an incomplete `FORMAT.next`, synchronize the root | stage removed | +| `WriteMarkerStage` | exclusively create `FORMAT.next` and write the 96 canonical bytes | stage bytes | +| `SynchronizeMarkerStage` | synchronize `FORMAT.next` and reverify its bytes and identity | durable stage | +| `SynchronizeRootAfterMarkerStage` | synchronize the root | durable stage name | +| `AdmitReaderFence` | create zero-length `reader.lock`, synchronize the root | `reader.lock` | +| `AdmitDirectory` | create each directory below in order, synchronize its parent | one directory | +| `LinkMarker` | hard-link `FORMAT.next` to `FORMAT` without replacement | `FORMAT` | +| `SynchronizeRootAfterMarker` | synchronize the root | **commit point** | +| `RemoveMarkerStage` | remove `FORMAT.next` | stage removed | +| `SynchronizeRootAfterCleanup` | synchronize the root | durable cleanup | + +The directory order is `staging`, `segments`, `catalogs`, `retention`, +`retention/roots`, `retention/manifests`, `gc`, `recovery`, and +`recovery/dispositions`. A directory that already exists is admitted only if it +is a real directory that holds nothing but the canonical children already +created before it. Every creation is followed by a synchronization of its +parent, so each name is durable before the next one is created. + +Initialization never writes `FORMAT` in place. `FORMAT.next` is written +before any directory exists and made durable first. +It is the evidence that this root is a Keep initialization in progress. A root +of directories without it is never adopted. + +### Commit point + +The commit point is the root synchronization that follows `LinkMarker`. +Before it, the store does not admit. Neither the existence of `FORMAT` nor the +presence of the protocol directories proves completion: `FORMAT` is linked +while `FORMAT.next` still names the same inode, and every admission refuses a +root that holds `FORMAT.next`. `FORMAT.next` is removed only after the commit +synchronization returned, so a root holding `FORMAT` without `FORMAT.next` can +be observed only after the commit is durable. Initialization returns writer +authority only after `SynchronizeRootAfterCleanup`. + +## States and results + +Initialization classifies the root before it creates `writer.lock`, and again +after it holds the lock. Only the locked classification authorizes effects; +the first exists so that a refused directory is never mutated, not even by +creating `writer.lock`. Every state has exactly one result for each entry +point: + +| State | Root holds | `initialize` | writer and reader admission | +| --- | --- | --- | --- | +| empty | nothing | create from the start | refuse: no `writer.lock` | +| locked | `writer.lock` | resume at `WriteMarkerStage` | refuse `MissingFormatMarker` | +| incomplete marker stage | + a strict canonical prefix of `FORMAT.next` | discard it, then write it | refuse `InitializationIncomplete` | +| marker staged | + canonical `FORMAT.next` | resume at `SynchronizeMarkerStage` | refuse `InitializationIncomplete` | +| reader fence | + `reader.lock` | resume at `SynchronizeMarkerStage` | refuse `InitializationIncomplete` | +| directory prefix *k* | + the first *k* directories | resume at `SynchronizeMarkerStage` | refuse `InitializationIncomplete` | +| marker linked | + `FORMAT` naming the inode of `FORMAT.next` | resume at `SynchronizeRootAfterMarker` | refuse `InitializationIncomplete` | +| committed | `FORMAT` without `FORMAT.next` | refuse `AlreadyInitialized` | admit | +| anything else | | refuse, no mutation | refuse | + +Every state that holds the canonical `FORMAT.next` and no `FORMAT` resumes at +`SynchronizeMarkerStage` and replays the later phases. Each replayed phase is +idempotent: the stage is reopened and verified by its exact bytes and inode, +an existing `reader.lock` or directory is admitted by kind, length, and +membership and its parent is synchronized again, and the link never replaces +an entry. Replay therefore never depends on knowing which synchronization +completed before process death. A synchronization failure returns +`VersionTwoInitializationError::Io` at that phase and leaves the state on the +table, so a retry resumes and repeats the synchronization. + +Refusals carry their evidence as `VersionTwoInitializationRefusal`: + +- an entry outside the initialization namespace, a directory or `reader.lock` + without `FORMAT.next`, a directory out of order, or an entry inside a + protocol directory other than its canonical children is `NotAPrefix`, + naming the entry by its path; +- a canonical entry of the wrong kind, such as a link where a directory + belongs, is `WrongKind`, with expected and observed kinds; +- a `FORMAT.next` whose bytes are not a prefix of the canonical marker, or + that is longer than 96 bytes, is `MarkerStageMismatch`, with expected and + observed lengths; +- a `FORMAT` beside `FORMAT.next` that is not the same inode with the same + canonical bytes is `MarkerLinkMismatch`; +- a `writer.lock` or `reader.lock` that is not zero-length is + `NonEmptyLockFile`, with its observed length; and +- `FORMAT` without `FORMAT.next` is `AlreadyInitialized`. + +The marker-linked state admits a populated committed namespace, not only an +empty one. Process death cannot produce one, because admission refuses +`FORMAT.next`. A filesystem that loses the stage removal at power failure +could, after later writes made other entries durable. Removing a second name +for the committed marker's inode loses nothing, so initialization completes the +cleanup instead of refusing a committed store forever. + +### Retention absence + +The initialization prefixes require `retention/roots` and +`retention/manifests` to be empty and `retention` to hold nothing else. Missing +retention state therefore never means "nothing is retained" when an artifact +contradicts it: a stray root, manifest, head, or stage refuses as +`NotAPrefix`. After commit, retention publication and the +retained-closure gate own that law. + +### Concurrent attempts + +Writer authority is the root directory lock and then the `writer.lock` lock, +both nonblocking. Of two concurrent attempts, one acquires them. The other +refuses with `WriterLock { source: Busy }`, or, if it acquires them after the +first committed, with `AlreadyInitialized`. Both classifications run under the +lock, so the second attempt never resumes the first one's phases. + +### Version 1 + +A root holding only an empty `writer.lock` is also a version-1 initialization +prefix; it holds nothing of value and either protocol may continue it. Every +later state holds `FORMAT.next` or `FORMAT`, which version-1 admission and +version-1 recovery refuse. Version-1 admission is removed by a later unit of +[#198](https://github.com/flyingrobots/keep/issues/198). + +## Process-death matrix + +`cargo xtask durability-crash-matrix --sequence version-two-initialization` +runs the production initializer in a subprocess and kills it before, during, +and after each boundary. The parent then checks the exact inventory and bytes, +the admission refusal or result, and runs `initialize` again as recovery. It +then requires writer reopen, publishes a first catalog through the version-2 +writer, and reads the blob back. + +| ID | Boundary | During | +| --- | --- | --- | +| `KEEP-CRASH-081` | `OpenAndLockWriterFile` | after the lock | +| `KEEP-CRASH-082` | `WriteMarkerStage` | a 48-byte strict prefix | +| `KEEP-CRASH-083` | `SynchronizeMarkerStage` | before | +| `KEEP-CRASH-084` | `SynchronizeRootAfterMarkerStage` | before | +| `KEEP-CRASH-085` | `AdmitReaderFence` | after | +| `KEEP-CRASH-086` | `AdmitDirectory` | after each of the nine directories | +| `KEEP-CRASH-087` | `LinkMarker` | after | +| `KEEP-CRASH-088` | `SynchronizeRootAfterMarker` (commit point) | before | +| `KEEP-CRASH-089` | `RemoveMarkerStage` | after | +| `KEEP-CRASH-090` | `SynchronizeRootAfterCleanup` | before | +| `KEEP-CRASH-091` | `DiscardIncompleteMarkerStage`, inside recovery | after | + +The matrix has 41 cases. It is process-death evidence. It does not claim +power-loss behavior, which depends on the filesystem honoring each +synchronization. diff --git a/docs/formats/segment-store-v2/migration-crash.md b/docs/formats/segment-store-v2/migration-crash.md deleted file mode 100644 index 0afd372c..00000000 --- a/docs/formats/segment-store-v2/migration-crash.md +++ /dev/null @@ -1,122 +0,0 @@ -# Migration Crash Points - -This page owns fixed-record publication and process-death boundaries for the -one-way `keep.segment-store/v1` to `keep.segment-store/v2` migration. - -## Fixed-stage law - -Migration never writes canonical fixed names in place: - -| Stage | Canonical target | -| --- | --- | -| `migration.intent.next` | `migration.intent` | -| `FORMAT.next` | `FORMAT` | -| `migration.receipt.next` | `migration.receipt` | - -For each pair, migration: - -1. creates the stage exclusively as a pinned regular file; -2. writes bounded complete bytes, synchronizes, reopens, and verifies them; -3. links the stage to the canonical target without replacement; -4. synchronizes the store root; -5. removes the retained stage; and -6. synchronizes the store root again. - -The verified stage is linked without replacement. The canonical target is -immutable. Recovery never truncates, replaces, or repairs it. An exact stage -with an absent target resumes at stage synchronization. Exact stage and target -bytes resume at the required synchronization or cleanup. Different bytes, a -substituted inode, a link, or a wrong file kind refuse. - -A pre-effect incomplete stage may be removed only when its canonical target and -every later-ordered migration effect are absent and every earlier effect admits -exactly. Migration recovery revalidates the present stage's regular kind and -strictly incomplete length immediately before removal, removes it, synchronizes -the store root, and returns a typed discard report. It does not retain an -incomplete-stage handle. Any later effect makes incomplete or corrupt stage -bytes unrecoverable ambiguity. - -The fixed stage is not authority. `migration.intent` becomes migration -authority only after its canonical link and store-root synchronization. -`migration.receipt` becomes completion evidence at the equivalent boundary. - -## Receipt synchronization mask - -`migration.receipt` records the exact pre-receipt mask -`0x00000000000003ff`. Bits are: - -| Bit | Completed evidence | -| ---: | --- | -| 0 | canonical migration intent and store root synchronized | -| 1 | `reader.lock` verified, synchronized, and root-synchronized | -| 2 | `retention` created and parent synchronized | -| 3 | `retention/roots` created and parent synchronized | -| 4 | `retention/manifests` created and parent synchronized | -| 5 | `gc` created and parent synchronized | -| 6 | `recovery` created and parent synchronized | -| 7 | `recovery/dispositions` created and parent synchronized | -| 8 | canonical format marker and store root synchronized | -| 9 | complete version-2 view reopened and verified | - -Bits 10 through 63 are zero and refuse when set. The mask records only -evidence completed before receipt construction; receipt-stage publication and -its final root synchronizations are established by admission of the canonical -receipt, not claimed by its own bytes. - -## Namespace prefix - -After durable intent publication, migration creates persistent `reader.lock` -and the exact nested directory prefix in the order specified by -[migration recovery](recovery.md). Each existing name must be the exact pinned -file or directory expected at that position. Each new nested name is followed -by synchronization of its parent. The final store-root synchronization admits -the complete prefix. A wrong kind, link, out-of-order name, or unknown entry -refuses. - -## Process-death matrix - -| Identifier | Boundary | -| --- | --- | -| `KEEP-CRASH-053` | migration-intent stage write | -| `KEEP-CRASH-054` | migration-intent stage synchronization | -| `KEEP-CRASH-055` | migration-intent canonical link | -| `KEEP-CRASH-056` | store-root synchronization after intent link | -| `KEEP-CRASH-057` | migration-intent stage removal | -| `KEEP-CRASH-058` | store-root synchronization after intent cleanup | -| `KEEP-CRASH-059` | persistent reader-fence creation | -| `KEEP-CRASH-060` | canonical nested directory-prefix creation | -| `KEEP-CRASH-061` | store-root synchronization after namespace creation | -| `KEEP-CRASH-062` | format-marker stage write | -| `KEEP-CRASH-063` | format-marker stage synchronization | -| `KEEP-CRASH-064` | format-marker canonical link | -| `KEEP-CRASH-065` | store-root synchronization after marker link | -| `KEEP-CRASH-066` | format-marker stage removal | -| `KEEP-CRASH-067` | store-root synchronization after marker cleanup | -| `KEEP-CRASH-068` | migration-receipt stage write | -| `KEEP-CRASH-069` | migration-receipt stage synchronization | -| `KEEP-CRASH-070` | migration-receipt canonical link | -| `KEEP-CRASH-071` | store-root synchronization after receipt link | -| `KEEP-CRASH-072` | migration-receipt stage removal | -| `KEEP-CRASH-073` | final store-root synchronization | - -Every identifier requires before, during, and after process-death evidence. -`KEEP-CRASH-060` additionally requires one case for every admitted directory -prefix length. Restart must classify exact stages, canonical targets, namespace -prefix, marker, receipt, and cleanup state without depending on a clock, -filesystem iteration order, or file existence alone. - -Namespace occurrence coordinates are zero-based: `during` accepts zero -through five, while `before` and `after` accept only zero. Invalid coordinates -refuse at crash-case admission before a child starts. Segment record -occurrences remain dependent on the write's record count. - -`StoreMigrationPhase::ALL` freezes the 21 boundaries above in exact order. -The production recovery planner and filesystem adapter execute the lawful -remaining suffix. `cargo xtask durability-crash-matrix --sequence migration` -runs 68 process-death cases: before/during/after each boundary, with all six -directory-prefix occurrences at `KEEP-CRASH-060`. The existing CI matrix -runs these cases in debug and release builds. - -These cases establish recovery after process death; they do not simulate power -loss. Broader restart corruption and compatibility/fuzz coverage remain tracked -by #111 and #112. diff --git a/docs/formats/segment-store-v2/migration-inventory.md b/docs/formats/segment-store-v2/migration-inventory.md deleted file mode 100644 index 2eed8dcf..00000000 --- a/docs/formats/segment-store-v2/migration-inventory.md +++ /dev/null @@ -1,66 +0,0 @@ -# Migration Inventory - -This page owns the bounded canonical digest over version-1 immutable segment -and catalog pools used by `migration.intent`. - -## Entry grammar - -One migration inventory entry is exactly 56 bytes: - -| Offset | Width | Field | Canonical value | -| ---: | ---: | --- | --- | -| 0 | 1 | artifact kind | `1` segment or `2` catalog | -| 1 | 7 | reserved | zero | -| 8 | 8 | catalog generation | zero for segment; positive for catalog | -| 16 | 8 | artifact length | exact positive length | -| 24 | 32 | artifact digest | exact admitted segment or catalog digest | - -Entries are sorted by their complete 56-byte canonical encoding and are -duplicate-free. The maximum is 2,097,152 entries across both pools. Count and -length arithmetic is checked before bytes are retained or allocated. - -The inventory digest is: - -```text -BLAKE3-256("keep.store-v1-pool-inventory/v2\0" || - entry-count-u32 || - canonical-entry-bytes) -``` - -The digest is streamed; the complete encoded inventory is never required in -memory. - -## Admission - -Migration inventories the exact pinned version-1 `segments` and `catalogs` -directories under writer authority. Every regular entry must have the one -canonical physical name derived from its verified semantic digest and, for a -catalog, generation. Each artifact is reopened without following links and -completely admitted before its semantic coordinate enters the digest. - -An unknown name, alternate case or width, alias, duplicate semantic coordinate, -wrong kind, link, changed directory, changed artifact, corrupt bytes, count -overflow, or entry above the fixed maximum refuses migration. File existence, -iteration order, path spelling, modification time, and physical file identity -do not enter the digest. - -The exact one-segment, one-catalog input and its canonical entries are frozen in -the version-2 corpus -[`inventory.tsv`](../../../conformance/segment-store/v2/inventory.tsv). - -`StoreMigrationInventoryEntry` derives canonical bytes only from admitted -artifacts. `StoreMigrationInventoryHasher` requires the bounded entry count -before streaming, retains only the preceding entry, refuses duplicate or -out-of-order evidence, and reproduces the frozen digest. - -`FilesystemStoreMigrationInventoryReader` retains exclusive writer authority -and pinned capabilities for both immutable pools. It inventories every regular -entry, including artifacts not reachable from the current publication head, -and reproduces the frozen digest without retaining every artifact body at -once. `FilesystemStoreMigrationAuthority` combines that digest with an -identity-stable fixed-width `HEAD`, its selected admitted catalog, the exact -version-1 root namespace, and the admitted physical root coordinates. Its -`verify_current` operation repeats the complete observation and refuses any -different canonical intent before mutation. Filesystem migration storage that -invokes this verification immediately before its first namespace mutation -remains in progress. diff --git a/docs/formats/segment-store-v2/migration-recovery.md b/docs/formats/segment-store-v2/migration-recovery.md deleted file mode 100644 index ea4124d3..00000000 --- a/docs/formats/segment-store-v2/migration-recovery.md +++ /dev/null @@ -1,102 +0,0 @@ -# Migration Protocol and Recovery - -This page owns the ordered one-way migration protocol and partial-migration -recovery for `keep.segment-store/v2`. The root namespace, format marker, reader -fence, and migration record grammars are owned by -[Migration and recovery](recovery.md); the exact process-death boundaries are -owned by [Migration crash points](migration-crash.md). - -## One-way migration protocol - -Migration performs these ordered steps: - -1. Admit and completely recover the exact version-1 store. -2. Revalidate its head, catalog, pools, root identity, and writer authority. -3. Publish `migration.intent` from `migration.intent.next` through the - no-replacement fixed-stage protocol. -4. Create and verify persistent `reader.lock`. -5. Create the exact `retention`, `retention/roots`, - `retention/manifests`, `gc`, `recovery`, and - `recovery/dispositions` directories. -6. Synchronize every created parent and the store root. -7. Publish `FORMAT` from `FORMAT.next` through the fixed-stage protocol. -8. Reopen and verify the complete version-2 view. -9. Publish `migration.receipt` from `migration.receipt.next` through the - fixed-stage protocol. - -The [migration crash-point specification](migration-crash.md) owns -`KEEP-CRASH-053` through `KEEP-CRASH-073`. -Migration never rewrites or deletes admitted version-1 immutable bytes and -provides no automatic downgrade. - -`FilesystemStoreMigrationAuthority` retains the writer lock and pinned root -and pools. It admits the version-1 namespace, Linux root identity, `HEAD`, -complete immutable-pool inventory, and selected catalog. Before mutation, it -requires the same canonical intent. Its port retains fixed-record handles, -verifies bytes and inode identity at each publication boundary, and admits only -ordered prefixes. It reopens the complete view before receipt staging and -leaves all version-1 immutable bytes untouched. -Version-1 admission refuses after a migration stage, `migration.intent`, -`reader.lock`, `FORMAT`, or version-2 directory exists. After durable intent, -only version-2 migration recovery may continue. - -## Partial migration recovery - -The migration recovery boundary admits only these ordered prefixes: - -| State | Required response | -| --- | --- | -| no migration artifact | admit exact version 1 | -| intent stage only | finalize an exact stage or explicitly discard an incomplete pre-effect stage | -| durable intent only | verify intent and continue | -| intent plus a canonical prefix of v2 names | verify each name and continue | -| complete v2 shape without marker | verify directories and write marker | -| marker without receipt | reopen full v2 view and publish receipt | -| exact receipt with optional exact receipt stage | clean the stage and admit complete migration | - -Every row also requires the admission checks in -[Executable recovery boundary](#executable-recovery-boundary). An intent stage -surviving a namespace effect, or a marker stage surviving a receipt effect, -refuses as `StageAfterEffect`. When a stage and canonical target both exist, -exact bytes and one shared device/inode identity are required before resumption. - -A partial migration retry revalidates intent and existing bytes, resumes at the -first absent canonical step, and never replaces an entry. A missing predecessor, -changed version-1 coordinate, out-of-order name, wrong kind or bytes, conflicting -receipt, unknown entry, or changed root identity is unrecoverable ambiguity. - -Death before durable intent leaves v1 plus at most its non-authoritative stage. -Death after durable intent leaves recovery-required v2 migration state. - -## Executable recovery boundary - -`FilesystemStoreMigrationAuthority::reopen_for_recovery` reacquires writer -authority without granting version-1 publication admission. -`recover_store_migration` revalidates the caller's freshly derived current -intent, then observes bounded residue and applies -`plan_store_migration_recovery` before adopting records or discarding stages. -Nested directory membership is checked before mutating recovery. An intent -stage surviving namespace creation, or a marker stage surviving receipt -publication, is refused as `StageAfterEffect`. - -When both a stage and its canonical target exist, adoption verifies they share -the same device and inode as well as exact bytes before any forward phase, -including directory synchronization. - -The receipt retains the exact observed namespace prefix and bound intent -digest, names the admitted plan, and lists the executed forward phases -through `executed_phases()`. The plan records the earliest unproven boundary; -it does not infer which synchronization calls completed before process death. -For `VersionOne`, `intent_digest()` returns `None`: no migration intent was -admitted. Every other plan reports the intent used by recovery. An incomplete -pre-effect intent stage has no complete persisted intent, so its discard path -uses the freshly verified current intent. -Restart compares device and inode identity; mount identity is same-process -evidence. Whenever an exact intent survives, recovery continues with its -persisted bytes. - -The filesystem laws cover every forward prefix, every strict byte-prefix truncation of all three stages and unchanged version-1 bytes. The [restart ambiguity matrix](../../testing-evidence/migration-restart-matrix.md) covers corrupt records, contradictory and byte-equal substituted stages, invalid ordering, copied-root identity, immutable-pool and current-head damage, changed inventory, foreign receipts, unknown names and wrong kinds. Each new refusal retains a complete before/after witness of names, file identities and bytes and checks the existing typed failure boundary. The evidence record distinguishes kernel/filesystem behavior from platform-admission, process-death and power-loss claims. - -## Completed namespace admission - -After a plan selects `Complete`, recovery calls `StoreMigrationRecoveryStorage::verify_complete` before returning its receipt. The filesystem implementation applies existing version-two namespace admission: reserved GC/recovery entries must match the current protocol, while owned retention state is permitted. A refusal retains the original cause under `StoreMigrationRecoveryError::Observation` and `FilesystemMigrationRecoveryRefusal::NamespacePreflight`, before any recovery effect. This is namespace admission, not verification or recovery of retained content; retention remains the owner of those records and stages. diff --git a/docs/formats/segment-store-v2/rationale.md b/docs/formats/segment-store-v2/rationale.md index 3032743c..00fec31e 100644 --- a/docs/formats/segment-store-v2/rationale.md +++ b/docs/formats/segment-store-v2/rationale.md @@ -7,27 +7,33 @@ authoritative for the cross-cutting retention and GC decision. Extending the version-1 root shape was rejected. Version 1 deliberately refuses unknown entries, so treating new retention state as optional would weaken its -admission law and make old readers misclassify a mutated store. A durable -migration intent creates an explicit authority boundary. +admission law and make old readers misclassify a mutated store. -Direct version-2 initialization was rejected for this version. Requiring one -admitted version-1 predecessor gives migration, compatibility, and recovery one -starting law instead of defining a second initialization protocol without a -consumer requirement. +## Initialize directly instead of migrating -## Stage fixed migration records +The first revision entered version 2 only by migrating a version-1 store. +ADR-0010 replaced that with direct initialization: nothing was released and +no store of value existed, and carrying a migration made every capability +exist twice. After the first release, a format change ships a separately +invoked upgrade tool that writes a new store and never modifies the source; +see the ADR. -Writing `migration.intent`, `FORMAT`, or `migration.receipt` in place was -rejected because process death can expose partial canonical bytes. Exact -`.next` stages make incomplete bytes non-authoritative and publish each -canonical fixed record through an immutable no-replacement link. +## Commit by linking the marker last -## Preserve version-1 immutable bytes +Writing `FORMAT` in place was rejected because process death can expose +partial canonical bytes. `FORMAT.next` is written and synchronized before any +directory exists, so its presence is the evidence of an initialization in +progress, and it is linked to `FORMAT` without replacement only after the +complete namespace is durable. Its removal follows the commit +synchronization, so a root without `FORMAT.next` can only be observed after +the commit is durable. A separate commit record was rejected: it would add a +second fixed record to admit with no state the stage does not already prove. +Recording the root directory's identity was rejected by ADR-0010. -Re-encoding segments, catalogs, or publication heads during migration was +## Keep version-1 grammars byte for byte + +Re-encoding segments, catalogs, or publication heads in version 2 was rejected. Their bytes are already canonical and independently evidenced. -Preservation narrows migration to new authority and namespace state and permits -byte-for-byte rollback analysis without promising an automatic downgrade. ## Keep namespace bytes out of paths @@ -82,12 +88,13 @@ order avoids lock inversion. Publication does not take the reader lock because it deletes no published immutable segment. Readers therefore double-collect both mutable heads around transitive admission and reject a mixed view. -## Derive logical store identity +## Persist no store identifier -Random or physical-location store identifiers were rejected. A deterministic -digest of the admitted version-1 catalog, immutable pools, and target format -definition gives byte-identical stores one logical identity while the migration -intent separately binds physical coordinates for in-place recovery. +Random or physical-location store identifiers were rejected. The migration +intent once carried a deterministic, content-derived identifier; it was +deleted with migration, and initialization records none. Two stores with the +same bytes are indistinguishable by design; telling an intended store from a +valid stranger needs an expectation the caller keeps, as ADR-0010 states. ## Reserve GC names but refuse their state @@ -97,32 +104,14 @@ also rejected. Version 2 reserves the names, while their presence remains an unsupported mandatory state until issue #21 supplies complete byte, parser, crash, recovery, corruption, and fuzz evidence. -## Compare restart-stable root coordinates on reopen - -The migration intent records the root's device, mount, and inode identity. -Only the first and last survive a remount: `statx.stx_mnt_id` is a mount -instance, not a volume. Comparing it on reopen made a correctly remounted -store refuse and would have made partial-prefix migration recovery reject -the store's own intent after a reboot. - -Reopen therefore compares device and inode only, and future restart recovery -must apply the same rule. Meanwhile, -the migrating process still compares all three against its own observation. -The intent bytes are unchanged. - -Rejected: removing the mount field. The definition digest covers the intent -layout, so the change would re-derive the format marker, the receipt, and -every version-2 fixture for no gain in restart safety. Rejected: the -filesystem UUID as the device coordinate in this change. That would require -separate platform admission, compatibility, and format decisions. Device -renumbering remains a refusal; this change does not introduce re-admission or -silently substitute a different identity coordinate. - -## Verify completed migration before reporting completion - -An exact receipt proves the migration records agree; it does not prove the current reserved namespace still admits. Recovery therefore invokes the read-only `StoreMigrationRecoveryStorage::verify_complete` capability after planning selects `Complete` and before returning success. The filesystem adapter reuses version-two namespace admission, retaining the original cause through `Observation` and `NamespacePreflight`. Existing corruption/planning refusals retain priority because this check follows planning. No namespace effect, synchronization or retention recovery is initiated. +## Bind no store to its root directory -Completed stores may contain published retention roots, manifests and heads. Reusing the partial-migration empty-directory preflight would reject valid post-migration state, so completion uses the existing version-two admission policy instead. Migration completion verifies its root/reserved directory contract; retention owns the interpretation of retention artifacts and stages. This adds a required method to the public recovery storage port; external implementations must implement equivalent effect-free admission. On-disk formats and identities are unchanged. +The migration intent recorded the root's device, mount, and inode, and reopen +compared device and inode. That binding refused every backup restore and copy +while proving less than it seemed: an intact store renamed into the path, or +an older snapshot restored into the same inode, still admitted. ADR-0010 +removed it with migration. Live checks remain: lock-file identity, the +same-process protocol-directory binding, and no-follow capability access. ## Gate catalog publication on every retained closure diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index f2404793..5eb45c3b 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -1,21 +1,18 @@ -# Migration and Recovery +# Namespace, Format Marker, and Recovery -This page owns version-2 filesystem migration and recovery. -[Migration protocol and recovery](migration-recovery.md) owns the ordered -one-way migration protocol and partial-migration recovery. +This page owns the exact version-2 root namespace, the `FORMAT` marker, the +reader fence, the GC and recovery-disposition reservation, and the laws that +make a store independent of the directory it lives in. +[Direct initialization](initialization.md) owns how a store comes into +existence; every version-2 store is created by direct initialization. ## Exact filesystem namespace -Version 2 preserves version-1 files and directories; it adds: +Version 2 keeps the version-1 files and directories and adds: ```text reader.lock FORMAT -migration.intent -migration.intent.next -migration.receipt -migration.receipt.next -FORMAT.next retention/HEAD retention/head.next retention/root.next @@ -28,14 +25,23 @@ recovery/disposition.next recovery/dispositions/.receipt ``` -`retention/HEAD`, every fixed `.next` stage, `gc/intent`, and `gc/receipt` are -optional according to the exact state tables below. The version-1 catalog -`head.next` stays a version-1 file: writer admission refuses it as recovery -state, while readers and version-2 catalog-stage recovery admit it, as -[catalog publication and stage recovery](catalog-publication.md) specifies. Immutable-pool coordinates -are data-dependent but canonically named. Every other root or -protocol-directory entry is an unknown entry and unrecoverable ambiguity. -Operations are capability-relative and never follow links. +Catalog `HEAD` is optional: an initialized store holds none until its first +catalog publication. `retention/HEAD`, every fixed `.next` stage, `gc/intent`, +and `gc/receipt` are optional according to the exact state tables of their +pages. The version-1 catalog `head.next` stays a version-1 file: writer +admission refuses it as recovery state, while readers and version-2 +catalog-stage recovery admit it, as +[catalog publication and stage recovery](catalog-publication.md) specifies. +`FORMAT.next` exists only while initialization has not committed; every +admission refuses it as `InitializationIncomplete`, and a root without `FORMAT` +refuses as `MissingFormatMarker`. Immutable-pool coordinates are +data-dependent but canonically named. Every other root or protocol-directory +entry is an unknown entry and unrecoverable ambiguity. Operations are +capability-relative and never follow links. + +Version-1 admission refuses every root that holds `FORMAT`, `FORMAT.next`, +`reader.lock`, `retention`, `gc`, or `recovery`. Version-1 recovery refuses +the same entries before it pins a pool. ## Format marker @@ -53,23 +59,20 @@ Operations are capability-relative and never follow links. | 64 | 32 | checksum | BLAKE3-256 over bytes `0..64` | The definition and checksum domains are `keep.segment-store-definition/v2\0` and -`keep.segment-store-marker-checksum/v2\0`. A missing marker is version 1 only -when the exact version-1 namespace admits. An unsupported, corrupt, +`keep.segment-store-marker-checksum/v2\0`. An unsupported, corrupt, substituted, or same-name/different-digest marker refuses. The format-definition digest is BLAKE3-256 of its domain followed by the exact corpus `definition.tsv` bytes. The format-marker digest is BLAKE3-256 of -`keep.store-format-marker/v2\0` followed by all 96 marker bytes. - -`CanonicalStoreFormatMarker` produces the registered marker; `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. -`CanonicalStoreMigrationIntent` retains typed intent coordinates; `CanonicalStoreMigrationReceipt` binds completion; admitted record types verify both. -`StoreMigrationStorage` names all 21 durability capabilities; `execute_store_migration` verifies current authority first and returns only after final synchronization. -`FilesystemStoreMigrationInventoryReader` inventories version-1 bytes under -retained writer authority. The fresh writer executes once; separate migration -recovery plans and executes the lawful remaining suffix after process death. -Version-1 reopen and version-1 recovery both refuse a migrated root. -Version-2 catalog-stage recovery admits it through -`FilesystemVersionTwoRecoveryAdmission`. +`keep.store-format-marker/v2\0` followed by all 96 marker bytes. The marker +identifies the format contract, not the software release that wrote the store. + +`CanonicalStoreFormatMarker` produces the one registered marker; +`AdmittedStoreFormatMarker` admits its framing, checksum, definition, and +namespace bound. Writer admission, catalog-stage recovery, recovery inventory, +and reader admission each read `FORMAT` no-follow, bounded to its exact length, +and refuse a trailing byte. `FilesystemPlatformAdmissionError::FormatMarker` +carries the decoder's typed refusal as its source. ## Reader fence @@ -80,6 +83,8 @@ A version-2 reader acquires a kernel-managed shared lock on `reader.lock` before opening catalog `HEAD` or `retention/HEAD`. The returned `ReaderFence` owns that lock for the complete snapshot lifetime. Close, drop, or process death releases only the kernel lock and never deletes the persistent file. +A reader of a committed store with no catalog `HEAD` refuses as +`FilesystemRetentionSnapshotError::Unpublished` once it holds the fence. GC acquires the store writer authority and then an exclusive `reader.lock`, in that fixed order. New readers wait and existing readers drain before GC @@ -87,138 +92,55 @@ revalidation or physical deletion. Catalog and retention publication may proceed beside readers because they publish immutable successors and delete no published segment. -## Migration records - -Migration is a one-way explicit migration under exclusive writer authority. - -`migration.intent` is exactly 256 bytes: - -| Offset | Width | Field | Canonical value | -| ---: | ---: | --- | --- | -| 0 | 16 | magic | `KEEP:MIG:INT2\0\0\0` | -| 16 | 2 | version | `2` | -| 18 | 2 | record length | `256` | -| 20 | 4 | flags | `0` | -| 24 | 8 | catalog generation named by version-1 `HEAD` | positive | -| 32 | 8 | catalog length named by version-1 `HEAD` | exact admitted length | -| 40 | 32 | catalog digest named by version-1 `HEAD` | exact admitted digest | -| 72 | 32 | predecessor catalog digest | zero for generation 1 | -| 104 | 32 | immutable-pool inventory digest | canonical complete inventory | -| 136 | 8 | root device identity | admitted Linux `dev_t` | -| 144 | 8 | root mount identity | admitted Linux `statx.stx_mnt_id` | -| 152 | 8 | root file identity | admitted Linux `statx.stx_ino` | -| 160 | 32 | target format-definition digest | exact registered v2 digest | -| 192 | 32 | new store identifier | deterministic derivation below | -| 224 | 32 | checksum | BLAKE3-256 over bytes `0..224` | - -The checksum domain is `keep.store-migration-intent-checksum/v2\0`. The -receipt's intent digest is BLAKE3-256 of -`keep.store-migration-intent/v2\0` followed by all 256 intent bytes. - -The [migration inventory](migration-inventory.md) defines its domain and law: -each migration inventory entry is exactly 56 bytes, and the fixed maximum is -2,097,152 entries. The intent therefore binds the exact catalog generation, -length, and digest named by the admitted version-1 `HEAD`. - -On Linux, the root device coordinate is `dev_t`, reconstructed from -`statx.stx_dev_major` and `statx.stx_dev_minor`; mount and file use -`statx.stx_mnt_id` and `statx.stx_ino`. Each is big-endian `u64`. - -The deterministically derived store identifier is: - -```text -BLAKE3-256("keep.store-identifier/v2\0" || - catalog-generation-u64 || - catalog-length-u64 || - catalog-digest || - predecessor-catalog-digest || - immutable-pool-inventory-digest || - target-format-definition-digest) -``` - -Integer fields use their fixed-width big-endian bytes. Root device, mount, file -identity, caller identity, path, and time do not enter the identifier. The -migration intent separately binds the physical root coordinates so in-place -recovery refuses a substituted store. +## Portable stores + +A store is not bound to the directory where it was created. The store records +no device, inode, mount, or path, and admission compares the root against +nothing persisted. A complete store copied while nothing was writing to it, +or taken from a consistent filesystem snapshot, opens on any admitted +filesystem after normal admission, and the opening process acquires fresh +writer or reader authority. + +Live authority is unchanged: + +- every access is capability-relative and no-follow; +- the writer lock and the reader fence verify their own lock files' identity; +- within one process, a protocol name must still identify the directory the + same process admitted + ([recovery directory binding](../../adr/retention-recovery-directory-binding.md)), + so a lock file or protocol directory replaced during an operation refuses; + and +- every content, closure, catalog, retention, and recovery validation still + applies, so a corrupt or incomplete copy refuses exactly as the original + would. + +A copy and its original are two stores. Each takes its own locks, and their +histories diverge independently while unchanged content keeps its identity. +Copies are not replication and are never coordinated. + +Admission does not detect an unintended but valid store, or an older valid +state of the intended one. Detecting either needs an expectation the caller +supplies and keeps outside the checked state, such as an exact catalog +generation and digest. Keep does not claim either check. ### Root identity across restart -The three root coordinates the intent records do not have the same lifetime. -`statx.stx_mnt_id` names a mount instance: it can change when a filesystem -is unmounted and mounted again or after a reboot. A restart path must not -require the historical mount identifier. The device and inode coordinates -name the volume and the root directory and survive remounts on the admitted -platform. - -The Linux [statx reference](https://man7.org/linux/man-pages/man2/statx.2.html) -defines the mount identifier separately from device and inode coordinates. -The lifetime distinction above is the reason Keep does not use it as restart -authority. - -The restart-stable root identity is therefore the pair `(device, file)`: - -- `FilesystemStoreMigrationAuthority` compares all three coordinates, but - only against the observation it made itself when it opened the root in the - same process; that comparison catches a root swapped underneath a running - migration and never crosses a restart. -- `FilesystemVersionTwoAdmission::reopen` compares device and - file only and refuses with `RootIdentityChanged { coordinate: Device | File, - .. }`. A remounted store admits; a store copied to another device or - restored into a different directory refuses. - -Partial-prefix migration recovery uses the same restart comparison; its -authority and resumption protocol are specified in -[the executable recovery boundary](migration-recovery.md#executable-recovery-boundary). - -The mount coordinate stays in the record as the migration-time observation. -It remains evidence for same-process migration checks, not restart authority. - -Limit: `dev_t` is stable across reboots only while the block device keeps its -major and minor numbers. A device-mapper or hot-plug renumbering makes a -correct store refuse with `RootIdentityChanged { coordinate: Device, .. }`; -version 2 defines no re-admission for that case, and a successor coordinate -(the filesystem UUID) is the rationale's recorded alternative if it proves -necessary. - -`migration.receipt` is exactly 256 bytes: - -| Offset | Width | Field | Canonical value | -| ---: | ---: | --- | --- | -| 0 | 16 | magic | `KEEP:MIG:REC2\0\0\0` | -| 16 | 2 | version | `2` | -| 18 | 2 | record length | `256` | -| 20 | 4 | flags | `0` | -| 24 | 32 | migration-intent digest | exact durable intent | -| 56 | 32 | store identifier | exact intent value | -| 88 | 32 | format-marker digest | exact verified marker | -| 120 | 32 | initial retention-state digest | exact no-payload digest below | -| 152 | 32 | initial GC-state digest | exact no-payload digest below | -| 184 | 32 | disposition namespace digest | exact no-payload digest below | -| 216 | 8 | completed synchronization mask | every mandatory bit set | -| 224 | 32 | checksum | BLAKE3-256 over bytes `0..224` | - -Its checksum domain is `keep.store-migration-receipt-checksum/v2\0`. Unknown -synchronization bits, a missing mandatory bit, or any mismatch with the intent -refuses. - -The three initial-state fields are the no-payload digests -`BLAKE3-256("keep.initial-retention-state/v2\0")`, -`BLAKE3-256("keep.initial-gc-state/v2\0")`, and -`BLAKE3-256("keep.empty-disposition-set/v2\0")`. At completed migration, -absence of `retention/HEAD` is the canonical empty retention state only while -all retention stages and pools are empty. Any retention artifact routes through -recovery instead. Direct version-2 initialization is undefined. - -The exact offsets and fixtures are requirement `KEEP-MIGRATION-002`. The fresh -writer emits only those canonical records; success is not restart evidence. -A migrated store is admitted for forward publication. Partial-prefix recovery -now plans and executes the lawful remaining migration suffix under writer -authority; the 68-case migration process-death matrix supplies restart evidence. -Broader hostile restart and compatibility coverage remain tracked in #111 and -issue #112. +Earlier revisions bound a migrated store to its root directory's device and +inode and refused a reopen or fenced snapshot whose root disagreed +(`RootIdentityChanged`). That binding lived in `migration.intent`, which no +longer exists, and is retired by +[ADR-0010](https://github.com/flyingrobots/keep/pull/199). Nothing replaces +it: the laws in [Portable stores](#portable-stores) apply instead. The +[retirement ledger](retired-identifiers.md) records the requirements the +binding satisfied. ## Retention publication recovery [Retention publication recovery](retention-recovery.md) owns fixed-stage classification, synchronization before publication, exact retained evidence, and the `KEEP-CRASH-036` through `052` process-death boundaries. `GcRetirementIntent`, `GcRetirementReceipt`, and `RecoveryDispositionReceipt` are owned by the [GC specification](gc.md). Until issue #21 implements them, any such artifact is unsupported and refuses. + +Initialization recovery is part of the initialization protocol: an interrupted +call is resumed by calling `FilesystemVersionTwoAdmission::initialize` again, +which classifies the root and replays the idempotent remaining phases. Its +process death states are specified in [Direct initialization](initialization.md). diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 9625ad25..3ea000ad 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -18,7 +18,7 @@ case is not evidence. | `KEEP-RETENTION-007` | Complete-stage recovery preserves canonical history; incomplete stages require disposition before mutation | Complete publication-prefix recovery and reader-state laws remain; incomplete direct/publication recovery and process-death laws now require typed refusal and preserved bytes. See [landing ledger](../../testing-evidence/retention-landing.md). | Implemented under the bounded scope merged in #99; automatic incomplete-stage disposition deferred to #155 | | `KEEP-RETENTION-008` | Readers double-collect catalog and retention heads and bind one complete catalog, manifest, and root-generation view under a `ReaderFence` | `ReaderFence` holds a shared kernel lock on a verified zero-length `reader.lock`; `collect_retention_view` accepts a view only when both head coordinates agree before and after loading and refuses an exhausted attempt limit (`retention_view_collector_tests`); `FilesystemRetentionSnapshot` binds the catalog snapshot, retention head, and manifest under the fence and verifies each selected root on demand while the fence is held, refusing a substituted root and a replaced fence, and two readers share the fence while an exclusive lock waits (`filesystem_retention_snapshot_tests`); Linux public snapshot process laws prove live-reader exclusion, SIGKILL fence release with persistent inode preservation, and kernel-queued new-reader exclusion until collection releases the fence (`tests/reader_fence_process.rs`; [evidence](../../testing-evidence/reader-fence-process.md)) | Implemented | | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | -| `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | all three-operation histories over initial publications of two namespaces, successor, release, restore, byte-identical retry and stale initial (343 histories, each in a fresh migrated store) compare fenced namespace generations, anchor sets and liveness against an operation-derived model after every step; exact typed refusals remain checked. The count describes exploration, not correctness. See [release/restore evidence](../../testing-evidence/retention-release-restore-model.md). Core architecture checks remain separate static evidence. | Implemented; release/restore model coverage added for #128 | +| `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | all three-operation histories over initial publications of two namespaces, successor, release, restore, byte-identical retry and stale initial (343 histories, each in a fresh initialized store) compare fenced namespace generations, anchor sets and liveness against an operation-derived model after every step; exact typed refusals remain checked. The count describes exploration, not correctness. See [release/restore evidence](../../testing-evidence/retention-release-restore-model.md). Core architecture checks remain separate static evidence. | Implemented; release/restore model coverage added for #128 | | `KEEP-RETENTION-011` | Catalog publication over retention state admits the current retention head, manifest, and every manifest root under writer authority; verifies every retained closure against the candidate before any catalog storage call; and refuses `HEAD` replacement if the retention head changed | port-level laws in `tests/retained_catalog_publication.rs`: dropped and substituted members refuse with the exact missing identity and no catalog stage; every manifest root is verified wherever it sorts; absent, corrupt, substituted, and contradictory head, manifest, and root evidence refuse with typed expected and observed state; a head changed, removed, or published before commit refuses after `head.next` verification without `replace_head`; an exhaustive 24-case candidate-by-root-set model and a repeat-run determinism check | Implemented at the storage port; the filesystem version-2 publisher and version-2 next-head finalization use it (#198, `KEEP-RETENTION-012` and `013`) | | `KEEP-RETENTION-012` | Under version-2 writer authority, one staged segment and a catalog successor publish only through the retained-closure gate, with no-follow bounded retention reads, head absence admitted only when uncontradicted, fixed retention stages refused, and no reader fence taken | `FilesystemVersionTwoCatalogPublisher`; `tests/version_two_catalog_writer.rs`: a preserving successor publishes, reopens, and reads both blobs while the retention root still verifies; a pinned reader view keeps its generation across publication; a candidate that drops retained content, a contradicted head absence, and a fixed retention stage each refuse with the exact typed error and an unchanged store witness; runtime RED for the contradiction and stage checks | Implemented for one staged segment per publication; segment rollover is #121 | | `KEEP-RETENTION-013` | Version-2 segment and catalog stage recovery admits version-2 root entries as known, reuses the version-1 executors, and finalizes a staged `head.next` only after the retained-closure gate admits it | `FilesystemVersionTwoRecoveryAdmission`, `classify_version_two_recovery_names`, and the `open_version_two` constructors; `tests/version_two_catalog_writer.rs` covers classification, completion, discard, reusable-prefix segment resume, and finalization that preserves or would lose retained content (runtime RED with the gate result ignored); `filesystem_version_two_catalog_recovery_tests` recovers injected interruptions after segment link, catalog-stage creation, and `head.next` synchronization | Implemented | @@ -27,23 +27,37 @@ case is not evidence. -## Migration +## Format marker | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | -| `KEEP-MIGRATION-001` | Exact version-1 stores remain admitted until a durable migration artifact exists | `tests/store_migration_recovery.rs` admits the artifact-free version-one observation; `src/adapters/store_migration/filesystem_migration_recovery_tests.rs` verifies every forward prefix under a fresh authority | Implemented; compatibility expansion in #112 | -| `KEEP-MIGRATION-002` | Format marker, intent, and receipt have complete fixed byte tables, named domains, bounds, checksums, deterministic store identity, and exact initial-state digests | exact admission in `tests/store_format_marker.rs`, `tests/store_migration_intent.rs`, and `tests/store_migration_receipt.rs`; canonical construction in `tests/store_migration_intent_encoding.rs` and `tests/store_migration_receipt_encoding.rs`; seeded `migration_format` fuzz target | Implemented | -| `KEEP-MIGRATION-003` | Migration revalidates version-1 head, catalog, pools, root identity (all three coordinates within the migrating process; device and file across restart), and writer authority before mutation | bounded canonical pool inventory in `tests/store_migration_inventory.rs`; writer-locked filesystem pool admission in `filesystem_inventory_*_tests`; exact authority observation and drift refusal in `filesystem_migration_authority_tests`; verification-first execution in `tests/store_migration_execution.rs`; fresh filesystem integration and post-publication drift refusal in `filesystem_migration_storage_tests`; a version-one store still holding a retained stage refuses before the intent is observed in `filesystem_migration_storage_tests` | Implemented | -| `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently under writer authority, comparing only the restart-stable root coordinates (device and file) against the persisted intent | restart-stable coordinate laws in `filesystem_version_two_admission_tests` and complete reopen laws in `filesystem_migration_remount_tests`; `tests/store_migration_recovery.rs` and `tests/store_migration_recovery_order.rs` freeze planner and ordering laws; `src/adapters/store_migration/filesystem_migration_recovery_tests.rs` covers every forward prefix; `src/adapters/store_migration/filesystem_migration_recovery_truncation_tests.rs` covers every strict fixed-stage truncation | Implemented in #108 | -| `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, conflicting, or changed evidence is unrecoverable ambiguity | Forward-execution laws remain. Filesystem restart record, pair, ordering, root, pool and namespace laws assert exact existing refusal boundaries and preserve complete names, device/inode identities and bytes; see [restart matrix](../../testing-evidence/migration-restart-matrix.md) for scenarios, calibration, diagnostic limits and validation ownership. | Evidence delivered in merged PR #161 (#111), including complete-state namespace refusal; see the restart matrix for bounded coverage and separate diagnostic obligations | -| `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | exact segment, catalog, and head witnesses in `src/adapters/store_migration/filesystem_migration_storage_tests.rs`, `src/adapters/store_migration/filesystem_migration_recovery_tests.rs`, and `src/adapters/store_migration/filesystem_migration_recovery_truncation_tests.rs`; subprocess restart witnesses in `cargo xtask durability-crash-matrix --sequence migration` | Implemented in #108 | -| `KEEP-MIGRATION-007` | Process death around every intent stage, canonical link, namespace prefix, marker stage, receipt stage, cleanup, and synchronization boundary reaches a documented lawful state | ordered phases and capabilities in `tests/store_migration_phase.rs` and `tests/store_migration_storage.rs`; exact phase-failure execution in `tests/store_migration_execution.rs`; production 21-phase forward execution in `filesystem_migration_storage_tests`; `cargo xtask durability-crash-matrix --sequence migration` runs 68 production subprocess cases at `KEEP-CRASH-053..=073`, debug and release | Implemented in #108 | -| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact after migration begins | Every forward-prefix compatibility law preserves v1 HEAD/catalog/segment bytes and refuses v1 authority after migration effects; public decoder laws pin unsupported versions and every mandatory flag bit; bounded seeded migration parser/recovery-planner fuzzing explores valid and malformed transitions. Existing jointly bound version-two admission and root-identity laws remain. See [compatibility evidence](../../testing-evidence/migration-compatibility-fuzz.md) for coverage, calibration and limits. | Implemented | +| `KEEP-FORMAT-001` | `FORMAT` has a complete fixed byte table, named domains, a checksum, the registered definition digest, and exact admission; every admission reads it no-follow, bounded, and refuses trailing bytes | `tests/store_format_marker.rs`; independent golden oracle `xtask/tests/retention_store_v2_format_oracle.rs`; corrupt and oversized marker refusals in `filesystem_version_two_admission_tests` and `verification_store_admission_tests`; seeded `format_marker` fuzz target | Implemented (#198); marker obligation of retired `KEEP-MIGRATION-002` | +## Initialization + + + +| ID | Requirement | Evidence | Status | +| --- | --- | --- | --- | +| `KEEP-INITIALIZATION-001` | Direct initialization creates exactly the committed namespace under exclusive writer authority, links `FORMAT` from a durable `FORMAT.next` without replacement, and returns authority only after the commit and cleanup synchronizations and complete admission | `a_fresh_root_commits_to_exactly_the_initialized_namespace` in `initialization_resume_tests` | Implemented (#198) | +| `KEEP-INITIALIZATION-002` | Every interrupted state classifies as exactly one prefix and resumes idempotently; every other nonempty root, including a committed one, refuses with typed evidence before any mutation, writer-lock creation included | `every_interrupted_boundary_classifies_and_resumes_to_one_store` (every boundary of a recorded run, against a model of the protocol table) and the incomplete-stage and sync-failure laws in `initialization_resume_tests`; foreign, out-of-order, stray-retention, foreign-stage, nonempty-lock, linked-directory, committed, and copied-marker refusals with unchanged witnesses in `initialization_prefix_tests` | Implemented (#198) | +| `KEEP-INITIALIZATION-003` | Admission requires a committed marker: writers and readers refuse `FORMAT.next` as `InitializationIncomplete` and a missing `FORMAT` as `MissingFormatMarker`; a committed store without catalog `HEAD` admits writers, refuses readers as `Unpublished`, and its first publication proves both absences instead of assuming them | `filesystem_version_two_admission_tests` | Implemented (#198) | +| `KEEP-INITIALIZATION-004` | Process death before, during, and after every initialization boundary reaches a documented state that `initialize` recovers | `cargo xtask durability-crash-matrix --sequence version-two-initialization` runs 41 cases at `KEEP-CRASH-081..=091`, checks the exact inventory, marker bytes and inode, admission result, and resumed state, then publishes, retains, and reconstructs the first blob | Implemented (#198); process death only, not power loss | +| `KEEP-INITIALIZATION-005` | Concurrent initialization attempts have exactly one initializer: the other refuses on writer authority and, after commit, as already initialized | `concurrent_attempts_have_exactly_one_initializer` in `initialization_resume_tests` | Implemented (#198) | + + + +## Retired identifiers + +The `KEEP-MIGRATION-001` to `-008` requirements and the `KEEP-CRASH-053` to +`-073` process-death rows were retired with migration. They are never reused. +The [retirement ledger](retired-identifiers.md) maps each one to its last +evidence and states where its obligation went. + ## Garbage collection reservation @@ -57,15 +71,15 @@ case is not evidence. ## Compatibility and nonclaims -- Version 2 preserves exact version-1 segment, catalog, and publication-head - bytes. -- Migration is one-way and provides no downgrade. +- Version 2 keeps the version-1 segment, catalog, and publication-head + grammars byte for byte. A version-1 store is not converted: version-2 code + refuses it, and no migration or downgrade exists. - Retention evidence proves a bounded physical reconstruction claim, not application meaning, causal ownership, future policy, or secure erasure. - A fresh forward writer is not proof that version 2 is restart-safe or production-admitted; partial-prefix recovery and crash evidence remain mandatory. This applies to retention publication exactly as it applies to - migration: the filesystem publication writer never continues a retained + initialization: the filesystem publication writer never continues a retained stage; it recovers complete evidence first, preserving incomplete stages for disposition, finalizing a complete head, and refusing a complete orphan until explicit disposition. A stage left behind by a failed write is recovery evidence like @@ -73,5 +87,7 @@ case is not evidence. - Publication binds this store's catalog `HEAD` to the verified closure and reopens the head-selected catalog and its segments under authority. Forward publication and recovery both replay the bounded live closure before effects. -- Benchmarks are required before performance-sensitive retention or migration - optimization. +- Admission does not detect an unintended but valid store or an older valid + state; a copy of a store is a second, independent store. +- Benchmarks are required before performance-sensitive retention or + initialization optimization. diff --git a/docs/formats/segment-store-v2/retention-publication.md b/docs/formats/segment-store-v2/retention-publication.md index a3750d8c..ddae0ac3 100644 --- a/docs/formats/segment-store-v2/retention-publication.md +++ b/docs/formats/segment-store-v2/retention-publication.md @@ -3,7 +3,7 @@ This page owns closure admission and the generation transition for `keep.segment-store/v2`. The record grammars it publishes are owned by [Retention records](retention.md); restart behaviour for a retained stage is -owned by [Migration and recovery](recovery.md). +owned by [Retention publication recovery](retention-recovery.md). ## Closure admission diff --git a/docs/formats/segment-store-v2/retention-recovery.md b/docs/formats/segment-store-v2/retention-recovery.md index c2070841..65622a8a 100644 --- a/docs/formats/segment-store-v2/retention-recovery.md +++ b/docs/formats/segment-store-v2/retention-recovery.md @@ -64,7 +64,7 @@ Observation and planning refusals initiate no recovery mutation. Execution failu Recovery stops immediately after an execution error and drops its execution context. Another attempt must observe and plan again. Complete-stage cleanup retains the opened source handle, verifies its observed identity and exact bytes as well as the immutable pool target before unlink, and checks source absence and surviving pool evidence afterwards. Successful unlink intentionally removes the original stage pathname; the preservation guarantee concerns verified pool evidence under the supported mutation model. A later verification or synchronization failure reports the completed removal with unconfirmed durability. -The supported mutation model is cooperating writers under Keep authority in a managed namespace. The writer lock supplies no isolation guarantee against arbitrary concurrent out-of-band namespace mutation. Existing no-follow, identity, exact-byte, namespace and corruption checks remain required and observed substitutions refuse. Open handles and metadata checks do not make pathname unlink or rename conditional on inode identity. The separate [migration discard path](migration-crash.md#fixed-stage-law) has its own contract and is unchanged by this retention scope decision. +The supported mutation model is cooperating writers under Keep authority in a managed namespace. The writer lock supplies no isolation guarantee against arbitrary concurrent out-of-band namespace mutation. Existing no-follow, identity, exact-byte, namespace and corruption checks remain required and observed substitutions refuse. Open handles and metadata checks do not make pathname unlink or rename conditional on inode identity. The separate [incomplete marker-stage discard](initialization.md#states-and-results) belongs to initialization and is unchanged by this retention scope decision. Complete-stage recovery and incomplete-stage preservation are implemented in this branch. The [landing ledger](../../testing-evidence/retention-landing.md) tracks remaining execution-failure reporting and exact-head acceptance for [PR #99](https://github.com/flyingrobots/keep/pull/99). Automatic incomplete-stage disposal and its stronger prefix-completion contract are explicitly deferred; historical discard-success evidence is not a claim of current behavior. diff --git a/docs/formats/segment-store-v2/retired-identifiers.md b/docs/formats/segment-store-v2/retired-identifiers.md new file mode 100644 index 00000000..24bc0502 --- /dev/null +++ b/docs/formats/segment-store-v2/retired-identifiers.md @@ -0,0 +1,103 @@ +# Retired Identifiers + +This ledger records requirement and crash identifiers that no longer name +anything in the code. A retired identifier is never reused, even for a +similar obligation; a new obligation takes a new identifier. + +Each row names the last commit on `main` that carried the identifier's +evidence, and where its obligation went: it either disappeared with the +subject it constrained, or moved to a current identifier. + +## Migration (ADR-0010, #198) + +[ADR-0010](https://github.com/flyingrobots/keep/pull/199) replaced the +version-1 to version-2 migration with direct initialization and deleted the +migration in the same change. Nothing had been released and no store with +data of value existed, so no conversion replaces it. +[#108](https://github.com/flyingrobots/keep/issues/108) (migration recovery) +is retired with it. + +### Requirements + + + +| ID | Obligation | Last evidence on `main` | Disposition | +| --- | --- | --- | --- | +| `KEEP-MIGRATION-001` | Version-1 stores stay admitted until a durable migration artifact exists | `tests/store_migration_recovery.rs` at `a58f237`; migration recovery laws under `src/adapters/store_migration/` at `6e7e2d6` | Disappeared: no migration artifact exists. Version-2 code refuses version-1 roots under `KEEP-RETENTION-014` until version-1 admission is deleted | +| `KEEP-MIGRATION-002` | Marker, intent, and receipt byte tables, domains, bounds, checksums, store identity, initial-state digests | `tests/store_format_marker.rs` at `aabfb89`; `tests/store_migration_intent.rs` at `fc20097`; `tests/store_migration_receipt.rs` at `94092ac`; `migration_format` fuzz target at `85919fb` | Marker part moved to `KEEP-FORMAT-001`. Intent, receipt, store identifier, and initial-state digests disappeared with their records | +| `KEEP-MIGRATION-003` | Migration revalidates the version-1 head, catalog, pools, root identity, and writer authority before mutation | `tests/store_migration_inventory.rs` at `89429a5`; `tests/store_migration_execution.rs` at `3fc9664`; authority laws at `6e7e2d6` | Disappeared: there is no version-1 source to revalidate. The persisted root-identity binding is retired by ADR-0010; live checks are `KEEP-PORTABILITY-003` | +| `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently, comparing restart-stable root coordinates | `tests/store_migration_recovery.rs` at `a58f237`; prefix and truncation laws at `6e7e2d6` | Prefix continuation moved to `KEEP-INITIALIZATION-002`. The root-coordinate comparison disappeared (ADR-0010) | +| `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, or changed evidence is unrecoverable ambiguity | restart matrix laws at `6e7e2d6`; `docs/testing-evidence/migration-restart-matrix.md` at `e206298` | Moved to `KEEP-INITIALIZATION-002`: a root that is not an initialization prefix refuses unchanged | +| `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | storage and recovery witnesses at `6e7e2d6`; migration crash restart witnesses at `c2414bf` | Disappeared: nothing converts a store | +| `KEEP-MIGRATION-007` | Process death at every migration boundary reaches a documented lawful state | `cargo xtask durability-crash-matrix --sequence migration` at `c2414bf` | Moved to `KEEP-INITIALIZATION-004` under new crash identifiers | +| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact | `tests/store_migration_compatibility.rs` at `85919fb`; `docs/testing-evidence/migration-compatibility-fuzz.md` at `1c886fd` | The version-2 part continues under `KEEP-RETENTION-014` until version-1 admission is deleted; the partial-migration part disappeared | + + + +The persisted reader root-identity requirement recorded in +[Retention reader root identity](../../adr/retention-reader-root-identity.md) +is superseded by ADR-0010 and replaced by the `KEEP-PORTABILITY-*` laws. Its +last evidence on `main` is +`src/adapters/retention/filesystem_retention_snapshot_identity_tests.rs` at +`5c32126`. + +### Crash identifiers + +`KEEP-CRASH-053` to `-073` named the 21 migration boundaries. The last +evidence for every row is the migration crash child and restart model at +`abbfe30` and `c2414bf`, and the transition ledger at `c2414bf`. + + + +| ID | Migration boundary | Disposition | +| --- | --- | --- | +| `KEEP-CRASH-053` | write `migration.intent.next` | Disappeared | +| `KEEP-CRASH-054` | synchronize the intent stage | Disappeared | +| `KEEP-CRASH-055` | link `migration.intent` | Disappeared | +| `KEEP-CRASH-056` | synchronize the root after the intent | Disappeared | +| `KEEP-CRASH-057` | remove the intent stage | Disappeared | +| `KEEP-CRASH-058` | synchronize the root after intent cleanup | Disappeared | +| `KEEP-CRASH-059` | admit the reader fence | Moved to `KEEP-CRASH-085` | +| `KEEP-CRASH-060` | admit the namespace prefix | Moved to `KEEP-CRASH-086` | +| `KEEP-CRASH-061` | synchronize the root after the namespace | Folded into `KEEP-CRASH-086`: every directory now synchronizes its parent | +| `KEEP-CRASH-062` | write `FORMAT.next` | Moved to `KEEP-CRASH-082` | +| `KEEP-CRASH-063` | synchronize the marker stage | Moved to `KEEP-CRASH-083` | +| `KEEP-CRASH-064` | link `FORMAT` | Moved to `KEEP-CRASH-087` | +| `KEEP-CRASH-065` | synchronize the root after the marker | Moved to `KEEP-CRASH-088`, now the commit point | +| `KEEP-CRASH-066` | remove the marker stage | Moved to `KEEP-CRASH-089` | +| `KEEP-CRASH-067` | synchronize the root after marker cleanup | Moved to `KEEP-CRASH-090` | +| `KEEP-CRASH-068` | write `migration.receipt.next` | Disappeared: the marker link is the commit point | +| `KEEP-CRASH-069` | synchronize the receipt stage | Disappeared | +| `KEEP-CRASH-070` | link `migration.receipt` | Disappeared | +| `KEEP-CRASH-071` | synchronize the root after the receipt | Disappeared | +| `KEEP-CRASH-072` | remove the receipt stage | Disappeared | +| `KEEP-CRASH-073` | synchronize the root after receipt cleanup | Disappeared | + + + +"Moved" means the obligation continues at a boundary of the +[initialization protocol](initialization.md); the boundary is new code with a +new identifier, not the migration boundary renamed. `KEEP-CRASH-084` +(the root synchronization after the marker stage), `-081` (the writer lock), +and `-091` (discarding an incomplete stage inside recovery) have no migration +predecessor. `KEEP-CRASH-078` to `-080` are reserved by version-2 catalog +recovery and are not assigned here. + +### Code and evidence removed + +- `src/adapters/store_migration/`: the migration records, inventory, + authority, recovery planner and executor, and their unit laws. +- `tests/store_migration_*`, `tests/migration_descriptor_exhaustion.rs`, and + the migration-completion and reader root-identity unit laws. +- The `migration_format` fuzz target and its seeds, replaced by + `format_marker` for the marker decoder that remains. +- The migration crash child, restart model, and transition rows. +- `conformance/segment-store/v2/migration-intent.hex`, + `migration-receipt.hex`, `inventory.tsv`, and `migration-source.tsv`, and + the migration rows of `definition.tsv` and `artifacts.tsv`. +- `docs/formats/segment-store-v2/migration-crash.md`, + `migration-inventory.md`, and `migration-recovery.md`, and the migration + evidence records under `docs/testing-evidence/`. + +PR [#191](https://github.com/flyingrobots/keep/pull/191) proposed typed +refusals at the migration storage boundaries; those parts are moot. diff --git a/docs/invariants/authenticated-reconstruction/README.md b/docs/invariants/authenticated-reconstruction/README.md index 7030352f..38fdc508 100644 --- a/docs/invariants/authenticated-reconstruction/README.md +++ b/docs/invariants/authenticated-reconstruction/README.md @@ -221,7 +221,7 @@ An operation claiming durable logical reconstruction must additionally: `DurableStore::open` returns a result and fixes an absolute locator at construction; changing the process working directory later cannot select a different store through that handle. Failure to resolve the locator preserves the original I/O cause through `DurableStoreError::Locator`, before store admission or output. -Snapshot admission requires the existing writable, case-sensitive local Linux ext4 profile across the root and present version-two protocol directories, retaining the admitted root capability through collection without acquiring writer authority. Valid migration records on an unsupported filesystem do not admit a readable snapshot. +Snapshot admission requires the existing writable, case-sensitive local Linux ext4 profile across the root and present version-two protocol directories, retaining the admitted root capability through collection without acquiring writer authority. A valid `FORMAT` marker on an unsupported filesystem does not admit a readable snapshot. Every selected retention root must match its manifest entry's namespace as well as its root generation and digest; a canonical foreign-namespace root refuses with preserved expected and observed namespace coordinates before a durable snapshot or caller output is exposed. diff --git a/docs/testing-evidence/current-durable-surfaces.md b/docs/testing-evidence/current-durable-surfaces.md index cceee735..bf1fee5c 100644 --- a/docs/testing-evidence/current-durable-surfaces.md +++ b/docs/testing-evidence/current-durable-surfaces.md @@ -8,7 +8,7 @@ The initial reconciliation used main integration `6051abb25a9fd33ae7ee0de5614514 | Claim | Owning source or evidence | Documentation disposition | | --- | --- | --- | -| Migration can resume admitted residue under writer authority | [Migration recovery](../../src/adapters/store_migration/filesystem_migration_recovery.rs), [requirements](../formats/segment-store-v2/requirements.md#migration) | Preserve the implemented migration claim and bound additional coverage by the [restart matrix](migration-restart-matrix.md) and [compatibility/fuzz evidence](migration-compatibility-fuzz.md), now merged through #161/#162. These finite receipts do not establish exhaustive hostile-state coverage. | +| Migration can resume admitted residue under writer authority | Retired with migration by ADR-0010 (#198); see the [retirement ledger](../formats/segment-store-v2/retired-identifiers.md) | Superseded: stores are created by [direct initialization](../formats/segment-store-v2/initialization.md), which resumes its own interrupted states. | | Retention recovery executes complete-stage plans and preserves incomplete stages | [Filesystem recovery](../../src/adapters/retention/filesystem_retention_recovery.rs), [planner](../../src/adapters/retention/recovery_planner.rs), [accepted landing evidence](retention-landing.md) | Remove obsolete absence and pending-#99 claims; keep incomplete-stage disposition deferred to #155. | | Execution failure is distinct from pre-effect refusal | [Execution](../../src/adapters/retention/recovery_execution.rs), [progress](../../src/adapters/retention/retention_storage_progress.rs), [contract](../formats/segment-store-v2/retention-recovery.md) | Preserve typed causes, known/uncertain effects, and durability distinctions. | | Retention snapshots bind a fenced view and admit selected roots | [Snapshot](../../src/adapters/retention/filesystem_retention_snapshot.rs), [collector](../../src/adapters/retention/retention_view_collector.rs) | Keep the snapshot contract distinct from the now-delivered `DurableStore` composition in #164. [Reader-fence process evidence](reader-fence-process.md) records the merged #160 lifecycle laws, not power-loss guarantees. | diff --git a/docs/testing-evidence/migration-clone-failure.md b/docs/testing-evidence/migration-clone-failure.md deleted file mode 100644 index 12dd4a22..00000000 --- a/docs/testing-evidence/migration-clone-failure.md +++ /dev/null @@ -1,21 +0,0 @@ -# Migration capability-clone failure - -Change kind: bug fix. Owner: `@flyingrobots`. Oracle: a failed root capability duplication reports `FilesystemMigrationAuthorityError::Namespace` with the original operating-system cause; root identity observation has its separate `RootIdentity` boundary. - -At base `84f5861`, the repository-task migration constructor used the unchecked-admission helper and labeled either failure `Platform`, despite performing no platform admission. The original review referred to an older location and a `RootIdentity` wrapper; the current defect was verified at the moved repository-task constructor. - -RED commit `50d0d0b` adds a Linux child-process regression that creates a valid version-one store, reacquires its real writer lock, fills a child-owned descriptor table, then invokes the public repository-task migration constructor. The kernel returns `EMFILE` during root capability cloning; the unfixed code reports `Platform` and fails the named namespace assertion. - -The child runs with a 64-descriptor limit and a 20-second deadline through the Docker image's shell and timeout utility. The parent retains its descriptor limits. Descriptors are released before inspecting the returned error, and no test mutates global process limits in the shared test runner. - -The fix performs capability cloning and lenient identity observation separately, preserving each source at its proper boundary, and releases the temporary cloned root before opening inventory capabilities as the previous helper did. - -The physical clone failure is exercised directly; identity-probe failure mapping is verified by code inspection, not claimed as a separately injected kernel fault. This evidence does not broaden the repository-task API into production platform admission. - -A copied-source mutation retained the `Namespace` variant but reconstructed the I/O error from its kind, losing `EMFILE`; the same assertion failed, independently calibrating source preservation in addition to the original wrong-boundary RED. - -The migration suite passed in debug and release. After explicitly restoring the temporary root handle's original lifetime, the focused regression passed again in both modes, with both workspace Clippy feature configurations, formatting and source-structure checks. Markdown validation passed separately. - -The descriptor law now lives in the dedicated `migration_descriptor_exhaustion` integration binary, so spawning its child cannot inherit descriptors from concurrent library tests. A full workspace run caught the original source-layout violation; the guard was retained and the test moved. Setup uses the public repository initialization API to obtain writer authority over a fresh version-one namespace before exhausting descriptors. - -Replay uses `cargo test --test migration_descriptor_exhaustion --all-features`, adding `--release` for optimized execution, inside copied Linux Docker. The fault schedule is deterministic descriptor exhaustion after writer-lock acquisition. Other targets do not run this Linux-specific regression. diff --git a/docs/testing-evidence/migration-compatibility-fuzz.md b/docs/testing-evidence/migration-compatibility-fuzz.md deleted file mode 100644 index b05cfb46..00000000 --- a/docs/testing-evidence/migration-compatibility-fuzz.md +++ /dev/null @@ -1,50 +0,0 @@ -# Migration compatibility and recovery fuzz evidence - -This change addresses #112 and KEEP-MIGRATION-008 from base main `6051abb25a9fd33ae7ee0de5614514b709a4d82a`. Change kind: missing runtime verification and stronger fuzz exploration, with no production behavior change. Owner: `@flyingrobots`. This branch is independent of the #111 restart ambiguity PR and does not claim mainline integration before merging. - -## Claims and owners - -`filesystem_migration_compatibility_tests::every_migration_prefix_preserves_v1_bytes_but_refuses_v1_authority` executes every complete forward-phase prefix in fresh filesystem storage. Before migration begins, version-one authority reopens and verifies the same intent. After any migration effect, reopening requires the existing exact `FilesystemPlatformAdmissionError::Namespace` / `InvalidData` refusal. After the attempt, every original version-one HEAD, segment and catalog name/byte pair must remain unchanged. Existing recovery-prefix and process-death laws retain responsibility for interrupted operations and restart recovery. - -`tests/store_migration_compatibility.rs` admits the golden migration records through public decoders and tests unsupported versions and every single mandatory flag bit in marker, intent and receipt. Expected coordinates come from the format's big-endian version and flags fields; precise `UnsupportedVersion` and `UnsupportedFlags` precede checksum refusal. Existing `tests/segment_header.rs`, `tests/segment_header/mutation_laws.rs`, `tests/catalog/header_laws.rs` and `tests/publication_head.rs` own version-one grammar/version/flag refusals. - -`migration_format` retains record parser fuzzing and adds bounded `plan_store_migration_recovery` exploration. Its properties require version-one success exactly when no migration evidence exists, precise effects-before-intent refusal, and complete success only with a full namespace, no stages and jointly admitted records. The complete-record property reuses product decoders; it checks agreement between planner and admission, not an independent implementation of the binary grammar. Other planner outcomes still receive robustness exploration rather than a complete reference-model oracle. - -## Corpus and bounds - -Checked-in seed recipes cover record admission, malformed marker version, unsupported intent flags, corrupt receipt checksum, valid migration prefixes, complete migration, corrupt intent, an effect before durable intent, a stage surviving an effect, a namespace hole and a receipt preceding its marker. They are materialized by `cargo xtask prepare-fuzz-corpus` and exercised through the existing migration target in smoke and scheduled workflows. No seed-count assertion establishes correctness. - -Recovery selector 3 is a fuzz-only envelope, not a durable format: a 256-byte expected intent, record-presence byte, namespace-presence byte and six length-prefixed record payloads. The envelope lengths use little-endian u16; embedded product records retain their canonical big-endian fields. Each payload is limited to 513 bytes, allowing overlong records while bounding payload allocation to 3,078 bytes. A complete envelope consumes at most 3,349 bytes including its selector; trailing fuzz-input bytes are ignored. This is an exploration bound, not a new storage limit. The checked-in campaign also enforces its own input, timeout and RSS bounds. - -The fixed local run uses pinned nightly `nightly-2026-07-24`, cargo-fuzz 0.13.2, seed 112, 20,000 executions, a 4,096-byte input cap, five-second per-input timeout and 1,024 MiB RSS limit. The named starting corpus is archived before launch; generated descendants are retained separately from the checked-in recipes. Replay `cargo +nightly-2026-07-24 fuzz run migration_format -- -runs=20000 -seed=112 -max_len=4096 -timeout=5 -rss_limit_mb=1024`. The instrumented binary exports `__asan_init`, and the release fuzz calibration demonstrates active assertions. No product crash or new real counterexample was found in that bounded run; it is not an exhaustive correctness claim. - -## Calibration and validation - -Separate copied-source/build mutations produce behavioral RED: admitting migrating namespaces as v1 incorrectly succeeds at the first migration prefix; damaging HEAD after the final migration phase fails byte preservation; bypassing marker version/flag checks changes the precise failures to checksum errors; forcing version-one success for a retained intent stage fails the fuzz oracle on its named seed. Mutations are excluded from the candidate. The initial test-authoring mistake using little-endian field patches and compiler/lint corrections are retained in logs but are not product RED evidence. - -Focused runtime tests pass in debug and release, and the bounded seeded fuzz run passes. The PR records immutable candidate coordinates, full workspace checks, migration crash campaign results and hosted CI; no earlier SHA's green result transfers automatically. Stable Rust is pinned to 1.96.0. Rust runs use copied Docker sources on Linux aarch64, not a writable host checkout. - -The filesystem law is medium-size and uses repository-only platform admission to isolate compatibility; it does not prove production platform eligibility. Decoder examples are small. Fuzzing invokes parser/planner runtime, not filesystem recovery execution, power-loss simulation or arbitrary writer interleavings. Existing platform, crash and recovery owners retain those claims. Ordinary-test resource enforcement gaps remain as disclosed in the testing enforcement profile. - -The seed materialization tool test no longer freezes global or per-target case counts. Mainline #172 now supplies the stronger materialization witness: a named emitted partial-seal counterexample reaches the real recovery classifier and produces its exact typed refusal, followed by deterministic repeated materialization. The integration preserves that witness instead of restoring the weaker nonempty-output assertion; migration product confidence comes from actual decoder, planner and filesystem outcomes. Deletion criterion: incidental inventory totals did not guard a product contract and rejected legitimate added exploration. Retire the new laws only when their compatibility contract disappears or stronger evidence subsumes it. - -## Landing calibration closure - -The fresh review of merged candidate `6e7e2d6a9c965f0240e006c7d45a5bc3c7010d37`, tracked tree `437bc1971fc8a98f2d039e446b3f5008979974f9`, found that the original Marker failures stopped execution before the independent Intent/Receipt checks, and the original VersionOne control did not reach the other fuzz properties. This is a proof gap, not a product bug. One bounded batch now demonstrates the distinct assertions below without changing their expected outcomes or the candidate's runtime code. - -| Control | Violated observation and intended runtime failure | Receipt | -| --- | --- | --- | -| [Intent header admission](migration-compatibility-fuzz/intent.patch) | Skip only Intent version/flag checks; unchanged Marker admission passes, then the Intent assertions observe checksum failures instead of exact version/flag refusals. | [RED](migration-compatibility-fuzz/intent-red.txt) | -| [Receipt header admission](migration-compatibility-fuzz/receipt.patch) | Skip only Receipt version/flag checks; earlier Marker/Intent assertions pass, and Receipt's exact refusals fail. | [RED](migration-compatibility-fuzz/receipt-red.txt) | -| [Pre-intent effect coordinate](migration-compatibility-fuzz/pre-intent.patch) | After the actual planner runs, replace its Namespace effect refusal with Marker; the unchanged VersionOne property passes, then the exact pre-intent assertion fails. | [RED](migration-compatibility-fuzz/pre-intent-red.txt) | -| [Complete namespace](migration-compatibility-fuzz/namespace.patch) | After actual Complete planning, remove the observed reader fence; the full-namespace assertion fails. | [RED](migration-compatibility-fuzz/namespace-red.txt) | -| [Complete stage absence](migration-compatibility-fuzz/stage.patch) | After actual Complete planning, add observed staged evidence; the namespace assertion passes, then stage absence fails. | [RED](migration-compatibility-fuzz/stage-red.txt) | -| [Complete record admission](migration-compatibility-fuzz/records.patch) | After actual Complete planning, replace the receipt observation with invalid bytes; earlier properties pass, then joint record admission fails. | [RED](migration-compatibility-fuzz/records-red.txt) | - -The four fuzz controls calibrate the unchanged oracles by perturbing observations after calling the real planner. They are deliberately inconsistent observations, not claims of production planner defects or real on-disk counterexamples. The earlier filesystem authority, byte-preservation, Marker and VersionOne controls remain valid for their original scope. No mutation score or one-control-per-bit claim is made. An initial launcher expected the wrong source-path prefix in the fuzz panic text and stopped after the valid pre-intent failure; the raw assertion failure is retained, the log-matching prefix was corrected, and only the three remaining controls were launched. That launcher mismatch is not runtime RED. - -Apply one patch at a time with `git apply --unidiff-zero` to a fresh copy of the source above. For Intent/Receipt, run `cargo test --all-features --locked --test store_migration_compatibility`. For fuzz controls, first materialize the unchanged candidate's named seeds with `cargo xtask prepare-fuzz-corpus`, then run `cargo +nightly-2026-07-24 fuzz run migration_format -- -runs=1 -seed=112 -max_len=4096 -timeout=5 -rss_limit_mb=1024`. Use `recovery-effect-before-intent` for the pre-intent control and `recovery-complete` for the other three. Each experiment uses its own copied source/build directory; no mutation enters the PR. - -The unchanged candidate then passes [public decoder debug/release GREEN](migration-compatibility-fuzz/restored-headers-green.txt) and a fresh [20,000-run seeded fuzz GREEN](migration-compatibility-fuzz/restored-fuzz-green.txt). The latter's named starting corpus is archived before launch; generated descendants are separate retained execution artifacts. Both receipts pin the original tracked tree. Full copied-Docker validation also completes successfully at that tree, and all four hosted jobs in run `37155153324` pass. Final evidence-only successors still require their own hosted/static checks and exact-head review; those are recorded in the PR. - -Committed logs normalize only isolated container source/build prefixes and line-end/trailing-empty whitespace. Original raw logs, exact variants, source archive, starting corpus and traced launchers remain retained by the author. This closure adds no runtime behavior, exploration scope, physical-power-loss evidence or resource-enforcement claim beyond the boundaries above. diff --git a/docs/testing-evidence/migration-restart-matrix.md b/docs/testing-evidence/migration-restart-matrix.md deleted file mode 100644 index a8f7c5ad..00000000 --- a/docs/testing-evidence/migration-restart-matrix.md +++ /dev/null @@ -1,69 +0,0 @@ -# Migration restart ambiguity matrix - -This closure ledger records #111 and KEEP-MIGRATION-005. Change kinds: missing runtime verification and the completed-namespace refusal bug fix documented below. The original test matrix alone changed no production behavior. Owner: `@flyingrobots`. Base: main `6051abb25a9fd33ae7ee0de5614514b709a4d82a`. Implementation and verification in this branch do not constitute mainline integration; the PR records the exact candidate, required checks and integration status. - -## Contract coverage - -| Invariant | Filesystem restart evidence | Disposition | -| --- | --- | --- | -| Corrupt intent, marker and receipt refuse | `filesystem_migration_restart_record_tests` damages each staged/canonical record checksum and requires the exact record-specific decoder variant and expected/observed checksum bytes. | Implemented. Existing codec corruption/fuzz owners retain exhaustive field grammar coverage. | -| Overlong stages are not disposed of as partial records | Record laws require exact `StageOverlong` stage and length for all three stages. | Implemented; existing strict-prefix success tests remain. | -| A receipt must bind its own intent | A valid checksummed receipt from another migrated root refuses with `ReceiptUndecodable` / `IntentDigestMismatch` and both digest coordinates. | Implemented. | -| Canonical/stage pairs bind one inode and identical bytes | `filesystem_migration_restart_pair_tests` replaces each linked stage with equal bytes at another inode, requiring `Adoption` / `ExactRecordRefusal::KindLengthOrIdentity`; contradictory bytes require exact `StageDiffers`. | Implemented; removing inode comparison incorrectly admits the substituted stage and fails the law. | -| Effects require durable intent and prior stage cleanup | `filesystem_migration_restart_order_tests` pins `EffectBeforeIntent` and `StageAfterEffect` with exact stage/effect coordinates through reopened authority. | Implemented. | -| Namespace holes, marker before namespace and receipt before marker refuse | Ordering laws cover filesystem-realizable holes with exact absent/present coordinates and both record-order refusals. | Implemented. An absent parent with a present child is not a realizable filesystem case. | -| Persisted root identity is binding | `filesystem_migration_restart_root_tests` copies the complete interrupted store into a different root and requires `IntentDiffers`. | Implemented; removing the root-file comparison incorrectly recovers the copy. | -| Immutable pools and current HEAD remain authoritative | `filesystem_migration_restart_pool_tests` damages segment/catalog/HEAD magic with exact decoder coordinates, substitutes a valid segment under the wrong digest name, and adds a valid orphan that changes the inventory bound by intent. | Implemented, including precise segment digest mismatch and `IntentDiffers`. | -| Unknown names refuse | Pool laws require exact pool, raw name and canonical-width refusal; namespace laws cover root and every nested migration directory. | Implemented; original root `Namespace` / `InvalidData` and nested `Adoption` / `NamespacePreflight` boundaries remain. | -| Wrong kinds and payload-bearing fences refuse | Namespace laws replace each fixed record/fence with directories and symbolic links, replace protocol directories with files, and require the exact nonempty fence kind/length refusal. | Implemented; existing diagnostic shapes are preserved. | -| Refusal preserves evidence | Every new scenario records root and descendants before reopening and after refusal: names, device/inode, file bytes, directory presence and link targets. Unknown witness shapes fail. | Implemented; deleting HEAD during observation fails the preservation assertion. | -| Success coverage and definition of done remain intact | Existing forward-prefix recovery, strict stage truncation, remount and process-death owners remain. The requirements ledger and normative recovery page link this matrix. | Full validation and exact-head hosted results belong to the PR receipt. Original roadmap checkboxes remain unchanged; merge is separately required. | - -Root namespace admission currently exposes a typed `FilesystemMigrationAuthorityError::Namespace` boundary with `InvalidData`, while its deepest namespace diagnostic is a textual I/O payload. Nested preflight exposes the existing typed `NamespacePreflight` wrapper. These tests pin those existing public failure shapes; this change does not claim to add finer typed namespace coordinates or complete the separate #110 diagnostic audit. - -## Calibration and observed results - -The covered runtime matrix passes in debug and release against the unmodified product. Separately copied source/build mutations demonstrate meaningful RED outcomes: omitted root-file comparison admits a copied root; omitted exact-record inode comparison admits a substituted stage; deleting HEAD during observation fails the preservation witness; skipping recovery namespace preflight changes an unknown-entry refusal from `Adoption` to the later `Resumption` boundary; omitting inventory and derived store-identifier comparison incorrectly recovers changed inventory. None of these mutations is included in the branch. - -Removing only the direct inventory comparison survives because the independently checked store identifier also binds that inventory. This survivor is retained as evidence of redundant protection, not reported as a weak test or hidden behind a mutation score. Removing both parts of that binding makes the orphan law fail with `ambiguous migration restarted successfully`. - -These are calibrations of missing verification, not claimed production bug fixes with fabricated parent RED. The existing single corrupt-intent test was removed under the recorded deletion criterion “subsumed by stronger evidence”: the new record matrix preserves its exact checksum refusal, adds checksum coordinates, and replaces its entry-count-only witness with all names, identities and bytes. Remaining risk lives in the new record law and the complete witness, not in a count assertion. - -## Execution and limits - -Replay `cargo test --lib --all-features --locked filesystem_migration_restart` and its `--release` variant. Tests run in copied Docker sources with pinned Rust 1.96.0 on Linux aarch64. The laws are medium-size real-filesystem experiments using repository-only platform admission to isolate migration semantics. They execute a named forward prefix, drop writer authority, alter owned evidence, then reopen and derive current intent before recovery. This is deterministic restart semantics, not a process-death or physical power-loss campaign; existing crash campaigns retain those claims. - -Schedules are the named forward phase and scenario tuple in each law; no random seed is involved. Each scenario owns fresh storage. Raw execution and calibration logs are retained in the author's issue-111 scratch evidence. The PR records immutable candidate coordinates and actual full validation results; compiler/setup failures are not behavioral RED evidence. - -Before/after equality does not independently prove absence of a transient write restored before observation. The tests combine preserved-state witnesses with the exact refusal boundary; source inspection places observation/planning/adoption refusal before resumed mutation. They do not establish isolation against arbitrary concurrent out-of-band writers. Ordinary-test resource ceilings and exhaustive mutation adequacy are not claimed; the enforcement profile's existing gaps remain. Retire these laws only if the corresponding contract disappears or stronger restart evidence subsumes it. - -## Historical landing counterexample: complete-plan namespace admission - -Landing review on `ff6f5be4b98e985e00601ab3e854a95e87aa9b34` (tree `de06577722b63ba09d425f31dfa7a1acd647e128`) exposed a limit that prevented closing #111 at that head; the correction and remaining acceptance gates are recorded below. After `RemoveReceiptStage`, adding `gc/unexpected` still permits recovery to return success. The new nested-namespace matrix currently tests the earlier `AdmitNamespacePrefix` state. Resumed recovery's pre-mutation refusal remains supported; complete-plan admission is a separate unfulfilled obligation. - -The [isolated probe patch](migration-restart-matrix/complete-namespace-probe.patch) selects only `gc` and the complete prefix in the existing namespace law, leaving production code and its refusal expectation unchanged. Its [runtime RED receipt](migration-restart-matrix/complete-namespace-probe-red.txt) records successful compilation and the executed law failing with `ambiguous migration restarted successfully` (exit 101). Replay in a fresh copy using `git apply --unidiff-zero`, then `cargo test --lib --all-features --locked unknown_root_and_nested_entries_preserve_restart_evidence -- --nocapture`. Execution used Rust 1.96.0, Linux aarch64, actual ext4 scratch and isolated source/build directories. Committed output normalizes only container path prefixes and trailing empty lines; original output and full variant are retained by the author. - -The `Complete` branch returns before `adopt_residue` and nested namespace preflight. Residue observation checks directory presence and kinds, not unknown nested membership. This evidence shows an accepted namespace contradiction, not a recovery mutation or byte substitution. Closure requires complete-plan admission consistent with its owning contract, permanent refusal coverage and preservation of lawful post-migration retention state. Do not indiscriminately apply the incomplete-prefix empty-pool rule to completed stores. A narrower acceptance scope needs explicit maintainer approval; changing prose alone does not close the original requirement. - -## Complete-plan correction under validation - -Regression commit `6351c7c` adds `completed_migration_refuses_unknown_reserved_entries_without_effects`. Its [observed RED](migration-restart-matrix/complete-regression-red.txt) ran with production code from `a8fda647bc67271ec7895577eab55f4b3a5e0085` and only the regression test added; it failed because recovery succeeded. The first source-copy command had a shell variable naming error and ran no test; the corrected isolated copy ran the regression and exited 101. Neither setup trouble nor an absent API is counted as RED. - -The correction invokes the new read-only recovery-port capability `verify_complete` only after the planner accepts the records as `Complete`. The filesystem implementation uses existing version-two namespace admission, with precise `Observation` / `NamespacePreflight` refusal before effects. The new law covers reserved GC, recovery and disposition names with the complete preserved-state witness. `completed_migration_preserves_published_retention_state` performs real retention publication, retries migration, requires a Complete receipt with no forward phases, and reads the exact retained bytes through a public snapshot. The [focused GREEN receipt](migration-restart-matrix/complete-fix-green.txt) records both new laws passing in debug/release and the existing restart matrix passing with the correction. - -This changes production refusal behavior and adds a required method for external `StoreMigrationRecoveryStorage` implementors. Retention content/stage semantics remain owned by retention; complete migration admission neither disposes stages nor certifies their content. No on-disk format, identity or durability ordering changes. The earlier counterexample remains historical evidence, not an unresolved claim about the corrected code. Full stable-candidate validation, the independent review's diagnostic calibration batch and exact-head approval are still required; KEEP-MIGRATION-005 remains in progress until those gates close. - -## Diagnostic calibration closure - -On fixed source `a3ea6c3e2ac58ca82a5c26f43d902ba1ed883be7`, tree `13a6148d8b4ca1fc28b8342cd6a06bc46b9bb2a5`, one finite negative-control batch demonstrates the distinct diagnostic assertions identified by independent review. The real restart runs first, and the unchanged complete witness comparison must pass. Only then does the copied fixture replace the observed error; expected values and assertion code are unchanged. This is oracle calibration at the observation boundary, not a claim that production emitted these artificial errors or that every production guard was separately mutated. - -| Control | Intended failing observation | Evidence | -| --- | --- | --- | -| Wrong diagnostics after real restart | Record checksum/length/foreign receipt, contradictory pair, ordering, HEAD/pool corruption, substituted segment, canonical pool name and root kind/no-follow refusal assertions reach their own failures. The pool identity is preserved here so its deeper decoder assertion executes. | [Patch](migration-restart-matrix/diagnostic.patch), [RED](migration-restart-matrix/diagnostic-red.txt) | -| Preserved outer recovery boundary, wrong inner cause | Root admission remains unchanged so nested cases execute. `Adoption` and `Observation` remain correctly typed while their inner causes change, failing the nested namespace, directory-kind, reader-fence and new completed-state diagnostic checks. | [Patch](migration-restart-matrix/nested-diagnostic.patch), [RED](migration-restart-matrix/nested-diagnostic-red.txt) | -| Wrong immutable-pool identity | Preserve the underlying artifact failure but report the other pool, reaching the pool-identity assertion before decoder checks. | [Patch](migration-restart-matrix/pool-diagnostic.patch), [RED](migration-restart-matrix/pool-diagnostic-red.txt) | -| Incorrect empty-pool completion policy | Substitute partial-migration preflight for completed namespace admission; the new law refuses genuine published retention state instead of returning Complete. | [Patch](migration-restart-matrix/lawful-retention.patch), [RED](migration-restart-matrix/lawful-retention-red.txt) | - -Each accepted control compiled, executed its named runtime law and exited 101. The initial pool control failed compilation on unused imports and is excluded as RED evidence; its corrected isolated build reaches the intended assertion. The first nested control stopped at the outer boundary, so the corrected control preserves that boundary and reaches the inner checks. Original variants and all attempted logs remain retained with the traced launchers; neither rejected attempt is represented as accepted calibration. - -Replay each patch independently in a fresh copied source with `git apply --unidiff-zero` and a separate build directory. Use `cargo test --lib --all-features --locked filesystem_migration_restart -- --nocapture` for the diagnostic control; replace the filter with `filesystem_migration_restart_namespace_tests`, `damaged_immutable_pool_bytes_refuse_restart_before_effects` or `completed_migration_preserves_published_retention_state` respectively for the remaining controls. The [restored GREEN receipt](migration-restart-matrix/calibration-restored-green.txt) records the unchanged fixed tree and both debug/release execution of the restart and completion laws. Logs normalize only isolated container path prefixes and trailing empty lines. Prior root/inode/inventory/preflight/preservation calibrations remain credited; this batch does not create a per-coordinate or per-row adequacy claim. diff --git a/docs/testing-evidence/retention-reader-root-identity.md b/docs/testing-evidence/retention-reader-root-identity.md index 7adaed80..6d20b627 100644 --- a/docs/testing-evidence/retention-reader-root-identity.md +++ b/docs/testing-evidence/retention-reader-root-identity.md @@ -1,5 +1,7 @@ # Retention reader root-identity evidence +Superseded (#198): ADR-0010 deleted migration and the persisted root binding this evidence tested; the law and its test were removed. The [retirement ledger](../formats/segment-store-v2/retired-identifiers.md) records the last evidence. This page is kept as history. + Change kind: bug fix. Subject: Keep runtime fenced snapshot admission. Size: medium. Oracle: mutually consistent migration records are admissible for reading only when their restart-stable device and inode binding names the opened root; identity diagnostics preserve the bound and observed values. Regression commit `5c32126`, based on `5d73016`, transplants the complete migration-record set from one controlled migrated directory into a second directory on the same filesystem, then invokes `FilesystemRetentionSnapshot::load`. The unfixed reader returned a snapshot and the regression was observed RED with `reader accepted migration records naming another physical root`. This was a runtime acceptance failure rather than setup or source-spelling evidence. diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index c8779611..6a30ab49 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -104,22 +104,22 @@ doc = false bench = false [[bin]] -name = "golden_protocol" -path = "fuzz_targets/golden_protocol.rs" +name = "format_marker" +path = "fuzz_targets/format_marker.rs" test = false doc = false bench = false [[bin]] -name = "layout_record" -path = "fuzz_targets/layout_record.rs" +name = "golden_protocol" +path = "fuzz_targets/golden_protocol.rs" test = false doc = false bench = false [[bin]] -name = "migration_format" -path = "fuzz_targets/migration_format.rs" +name = "layout_record" +path = "fuzz_targets/layout_record.rs" test = false doc = false bench = false diff --git a/fuzz/README.md b/fuzz/README.md index 8c4d554b..52bf7f36 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -70,7 +70,7 @@ retention-manifest, and retention-head decoders. The canonical one-root generation keeps mutations inside framing, semantic, ordering, checksum, and digest validation; every admitted value must retain its exact input bytes. -The `migration_format` seeds select public migration record decoders and bounded recovery planning. Receipt seeds carry exact marker and intent dependencies; malformed seeds exercise version, mandatory-flag and checksum refusals. Recovery seeds include valid prefixes and contradictory transition evidence. Properties require v1 admission only without migration evidence, precise pre-intent effect refusal, and complete success only with a full namespace and jointly admitted records. See [migration compatibility evidence](../docs/testing-evidence/migration-compatibility-fuzz.md) for the fuzz-only envelope, bounds, replay and oracle limits. +The `format_marker` seeds select the public `FORMAT` marker decoder: the canonical marker, an unsupported version, and a corrupt checksum. Every admitted input must be exactly the one registered marker and retain its exact bytes. The `segment_format` seeds select the public segment-header, record-header, complete-record, seal, and complete-segment boundaries. Canonical empty, diff --git a/fuzz/fuzz_targets/format_marker.rs b/fuzz/fuzz_targets/format_marker.rs new file mode 100644 index 00000000..c7563451 --- /dev/null +++ b/fuzz/fuzz_targets/format_marker.rs @@ -0,0 +1,17 @@ +#![no_main] + +//! This target owns `FORMAT` marker parser fuzzing. + +use keep::{AdmittedStoreFormatMarker, CanonicalStoreFormatMarker}; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|bytes: &[u8]| { + if let Ok(marker) = AdmittedStoreFormatMarker::decode(bytes) { + assert_eq!(marker.encoded(), bytes); + assert_eq!( + bytes, + CanonicalStoreFormatMarker::version_two().encoded(), + "exactly one marker is registered" + ); + } +}); diff --git a/fuzz/fuzz_targets/migration_format.rs b/fuzz/fuzz_targets/migration_format.rs deleted file mode 100644 index fa5b7672..00000000 --- a/fuzz/fuzz_targets/migration_format.rs +++ /dev/null @@ -1,58 +0,0 @@ -#![no_main] - -//! This target owns canonical store-migration record parser fuzzing. - -#[path = "migration_format/recovery.rs"] -pub mod recovery; - -use keep::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, -}; -use libfuzzer_sys::fuzz_target; - -const MARKER_BYTES: usize = 96; -const INTENT_BYTES: usize = 256; - -fuzz_target!(|bytes: &[u8]| { - let Some((&selector, input)) = bytes.split_first() else { - return; - }; - match selector { - 0 => marker(input), - 1 => intent(input), - 3 => { - let _ = recovery::exercise(input); - } - _ => receipt(input), - } -}); - -fn marker(input: &[u8]) { - if let Ok(marker) = AdmittedStoreFormatMarker::decode(input) { - assert_eq!(marker.encoded(), input); - } -} - -fn intent(input: &[u8]) { - if let Ok(intent) = AdmittedStoreMigrationIntent::decode(input) { - assert_eq!(intent.encoded(), input); - } -} - -fn receipt(input: &[u8]) { - let Some((marker_bytes, remainder)) = input.split_at_checked(MARKER_BYTES) else { - return; - }; - let Some((intent_bytes, receipt_bytes)) = remainder.split_at_checked(INTENT_BYTES) else { - return; - }; - let (Ok(marker), Ok(intent)) = ( - AdmittedStoreFormatMarker::decode(marker_bytes), - AdmittedStoreMigrationIntent::decode(intent_bytes), - ) else { - return; - }; - if let Ok(receipt) = AdmittedStoreMigrationReceipt::decode(receipt_bytes, &intent, &marker) { - assert_eq!(receipt.encoded(), receipt_bytes); - } -} diff --git a/fuzz/fuzz_targets/migration_format/recovery.rs b/fuzz/fuzz_targets/migration_format/recovery.rs deleted file mode 100644 index b04dd33b..00000000 --- a/fuzz/fuzz_targets/migration_format/recovery.rs +++ /dev/null @@ -1,115 +0,0 @@ -//! Bounded recovery-residue fuzz input and specified success/refusal properties. - -use keep::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, - StoreMigrationEffect, StoreMigrationRecoveryAmbiguity, StoreMigrationRecoveryPlan, - StoreMigrationResidue, plan_store_migration_recovery, -}; - -const INTENT_BYTES: usize = 256; -const MAX_RECORD_BYTES: usize = 513; - -pub(super) fn exercise(input: &[u8]) -> Option<()> { - let (expected_bytes, input) = input.split_at_checked(INTENT_BYTES)?; - let expected = AdmittedStoreMigrationIntent::decode(expected_bytes).ok()?; - let (&records, input) = input.split_first()?; - let (&namespace, mut input) = input.split_first()?; - let residue = StoreMigrationResidue { - intent_stage: present(records & 1, record(&mut input)?), - intent: present(records & 2, record(&mut input)?), - marker_stage: present(records & 4, record(&mut input)?), - marker: present(records & 8, record(&mut input)?), - receipt_stage: present(records & 16, record(&mut input)?), - receipt: present(records & 32, record(&mut input)?), - reader_fence: namespace & 1 != 0, - namespace_prefix: [2, 4, 8, 16, 32, 64].map(|bit| namespace & bit != 0), - }; - let result = plan_store_migration_recovery(&expected, &residue); - assert_version_one(&residue, &result); - assert_pre_intent_effects(&residue, &result); - if matches!(result, Ok(StoreMigrationRecoveryPlan::Complete)) { - assert_complete(&residue); - } - Some(()) -} - -fn record<'a>(input: &mut &'a [u8]) -> Option<&'a [u8]> { - let (length, remainder) = input.split_at_checked(2)?; - let length = usize::from(u16::from_le_bytes(length.try_into().ok()?)); - if length > MAX_RECORD_BYTES { - return None; - } - let (bytes, remainder) = remainder.split_at_checked(length)?; - *input = remainder; - Some(bytes) -} - -fn present(presence: u8, bytes: &[u8]) -> Option> { - (presence != 0).then(|| bytes.to_vec()) -} - -type Plan = Result; - -fn assert_version_one(residue: &StoreMigrationResidue, result: &Plan) { - let no_migration_evidence = residue.intent.is_none() - && residue.intent_stage.is_none() - && !residue.reader_fence - && !residue.namespace_prefix.contains(&true) - && residue.marker.is_none() - && residue.marker_stage.is_none() - && residue.receipt.is_none() - && residue.receipt_stage.is_none(); - assert_eq!( - matches!(result, Ok(StoreMigrationRecoveryPlan::VersionOne)), - no_migration_evidence, - "version-one admission requires absence of all migration evidence" - ); -} - -fn assert_pre_intent_effects(residue: &StoreMigrationResidue, result: &Plan) { - if residue.intent.is_some() { - return; - } - let effect = if residue.reader_fence || residue.namespace_prefix.contains(&true) { - Some(StoreMigrationEffect::Namespace) - } else if residue.marker.is_some() || residue.marker_stage.is_some() { - Some(StoreMigrationEffect::Marker) - } else if residue.receipt.is_some() || residue.receipt_stage.is_some() { - Some(StoreMigrationEffect::Receipt) - } else { - None - }; - if let Some(expected) = effect { - assert!( - matches!(result, Err(StoreMigrationRecoveryAmbiguity::EffectBeforeIntent { effect }) if *effect == expected), - "an effect before durable intent must retain its precise refusal: {result:?}" - ); - } -} - -fn assert_complete(residue: &StoreMigrationResidue) { - assert!( - residue.reader_fence && residue.namespace_prefix.iter().all(|present| *present), - "complete migration needs the entire namespace" - ); - assert!( - residue.intent_stage.is_none() - && residue.marker_stage.is_none() - && residue.receipt_stage.is_none(), - "complete migration must not leave staged evidence" - ); - let records = residue - .intent - .as_deref() - .zip(residue.marker.as_deref()) - .zip(residue.receipt.as_deref()); - let admitted = records.and_then(|((intent, marker), receipt)| { - let intent = AdmittedStoreMigrationIntent::decode(intent).ok()?; - let marker = AdmittedStoreFormatMarker::decode(marker).ok()?; - AdmittedStoreMigrationReceipt::decode(receipt, &intent, &marker).ok() - }); - assert!( - admitted.is_some(), - "complete migration needs jointly admitted records" - ); -} diff --git a/src/adapters/admitted_segment.rs b/src/adapters/admitted_segment.rs index 862e8a2c..f800819d 100644 --- a/src/adapters/admitted_segment.rs +++ b/src/adapters/admitted_segment.rs @@ -51,10 +51,6 @@ impl<'a> AdmittedSegment<'a> { self.seal.digest() } - pub(super) const fn segment_length(&self) -> u64 { - self.seal.segment_length() - } - /// Returns a revalidating iterator over records in physical order. #[must_use] pub const fn records(&self) -> SegmentRecords<'a> { diff --git a/src/adapters/durable/rationale.md b/src/adapters/durable/rationale.md index 7b0022a5..b91478ab 100644 --- a/src/adapters/durable/rationale.md +++ b/src/adapters/durable/rationale.md @@ -16,7 +16,7 @@ Refusal and operational failure remain distinct typed sources. Missing logical c The writer lock and reader fence coordinate cooperating Keep operations in a managed namespace. They do not isolate arbitrary concurrent raw filesystem mutation. Existing exact-byte, identity, namespace and corruption checks remain in force, including selected-root re-admission during lookup. -Readers reuse version-two platform admission before consuming migration records and retain the admitted directory capability through collection. Consistent record bytes alone cannot establish the filesystem semantics assumed by the shared fence and immutable pools. Reusing writer authority for this check was rejected because readers must coexist with a writer; the platform check itself acquires no writer lock. Unsupported filesystems now refuse through the existing admission error boundary. +Readers reuse version-two platform admission before consuming the `FORMAT` marker and retain the admitted directory capability through collection. Consistent record bytes alone cannot establish the filesystem semantics assumed by the shared fence and immutable pools. Reusing writer authority for this check was rejected because readers must coexist with a writer; the platform check itself acquires no writer lock. Unsupported filesystems now refuse through the existing admission error boundary. Platform admission performs a blocking root-directory synchronization before acquiring the shared fence. A synchronization failure preserves its original I/O cause under reader `Admission`, nested under the durable snapshot error. This probe does not publish content, flush caller output or grant a new content-durability guarantee. An existing snapshot read does not repeat this platform probe; a convenience read opens a new snapshot and therefore pays it again. diff --git a/src/adapters/durable/store.rs b/src/adapters/durable/store.rs index 98950acc..ee4a23fe 100644 --- a/src/adapters/durable/store.rs +++ b/src/adapters/durable/store.rs @@ -11,7 +11,7 @@ use super::{ use crate::adapters::{CatalogRestartPolicy, ReaderAttemptLimit}; use crate::{BlobId, ByteRange, LayoutId}; -/// One migrated version-two store to read from. +/// One committed version-two store to read from. /// /// The handle holds no fence and no view; every read pins a fresh /// [`DurableSnapshot`] unless the caller pins one with [`Self::snapshot`] @@ -27,7 +27,7 @@ use crate::{BlobId, ByteRange, LayoutId}; /// /// # Example /// -/// On Linux, read an already migrated version-two store with a retained blob. +/// On Linux, read an initialized version-two store with a retained blob. /// The caller owns output visibility: a failed write can leave an untrusted /// prefix. The receipt names the view; it does not extend retention after the /// snapshot is dropped. This example limits aggregate catalog-selected segment bytes; diff --git a/src/adapters/exports.rs b/src/adapters/exports.rs index ab8aaebd..0635a404 100644 --- a/src/adapters/exports.rs +++ b/src/adapters/exports.rs @@ -111,12 +111,19 @@ pub use super::segment_write_error::SegmentWriteError; pub use super::segment_write_phase::{SegmentDurabilityPhase, SegmentWritePhase}; pub use super::staged_segment::StagedSegment; pub use super::storage_profile_id_text_error::StorageProfileIdParseError; +pub use super::store_format::*; pub use super::store_initialization::initialize_store; pub use super::store_initialization_error::StoreInitializationError; pub use super::store_initialization_phase::StoreInitializationPhase; pub use super::store_initialization_receipt::StoreInitializationReceipt; pub use super::store_initialization_storage::StoreInitializationStorage; -pub use super::store_migration::*; +#[cfg(any(test, feature = "repository-tasks"))] +pub use super::version_two_initialization::VersionTwoInitializationObserver; +pub use super::version_two_initialization::{ + VersionTwoInitialization, VersionTwoInitializationBoundary, VersionTwoInitializationDirectory, + VersionTwoInitializationError, VersionTwoInitializationPhase, VersionTwoInitializationPrefix, + VersionTwoInitializationRefusal, +}; pub use super::writer_lock_acquire_error::WriterLockAcquireError; pub use super::writer_lock_acquire_phase::WriterLockAcquirePhase; pub use crate::segment_digest::SegmentDigest; diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index ad554871..c0b7ef21 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -88,7 +88,7 @@ impl FilesystemCatalogPublisher { policy: CatalogRestartPolicy, ) -> io::Result { Self::open( - FilesystemPlatformAdmission::unchecked_for_tests(lock)?, + FilesystemPlatformAdmission::unchecked_for_tests(lock), policy, ) } diff --git a/src/adapters/filesystem_exact_record.rs b/src/adapters/filesystem_exact_record.rs index 44707fab..ca507be1 100644 --- a/src/adapters/filesystem_exact_record.rs +++ b/src/adapters/filesystem_exact_record.rs @@ -221,36 +221,6 @@ pub(super) fn open_regular(directory: &Dir, name: &str) -> io::Result { open_read(directory, name) } -/// Reads at most `bound` bytes of the regular file `name`, or `None` if absent. -/// -/// Residue observers use this to see an incomplete, exact, or overlong record -/// as it is; a present entry that is not a regular file refuses by kind. -pub(super) fn read_bounded_optional( - directory: &Dir, - name: &str, - bound: usize, -) -> Result>, ExactRecordError> { - match directory.symlink_metadata(name) { - Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), - Err(source) => return Err(source.into()), - Ok(metadata) if !metadata.is_file() => { - return Err(ExactRecordRefusal::KindOrLength.into()); - } - Ok(_) => {} - } - let file = match open_read(directory, name) { - Ok(file) => file, - Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(None), - Err(source) => return Err(source.into()), - }; - if !file.metadata()?.is_file() { - return Err(ExactRecordRefusal::KindOrLength.into()); - } - let mut bytes = Vec::new(); - file.take(exact_length(bound)?).read_to_end(&mut bytes)?; - Ok(Some(bytes)) -} - /// Opens a record read-only without following links or blocking. pub(super) fn open_read(directory: &Dir, name: &str) -> io::Result { let mut options = OpenOptions::new(); diff --git a/src/adapters/filesystem_format_marker.rs b/src/adapters/filesystem_format_marker.rs new file mode 100644 index 00000000..bf86aad8 --- /dev/null +++ b/src/adapters/filesystem_format_marker.rs @@ -0,0 +1,44 @@ +//! This module owns admission of the committed `FORMAT` marker record. + +use std::io; + +use cap_std::fs::Dir; + +use super::VersionTwoRecordRefusal as Refusal; +use super::filesystem_exact_record::{self as exact_record, ExactRecordError, ExactRecordRefusal}; +use super::{AdmittedStoreFormatMarker, FORMAT_MARKER_LENGTH}; + +const MARKER_NAME: &str = "FORMAT"; + +/// Reads and admits `FORMAT` as the one registered version-two marker. +/// +/// The record is reopened without following links, bounded to its exact +/// canonical length, and decoded. Writer authority over a version-two root +/// must not be returned before this admission succeeds. Every refusal is an +/// `InvalidData` error whose source is a +/// [`VersionTwoRecordRefusal`](super::VersionTwoRecordRefusal). Nothing about +/// the root directory's own identity is compared: the marker binds the format +/// contract, not a location. +pub(super) fn admit(root: &Dir) -> io::Result<()> { + let bytes = exact_record::read_exact_regular(root, MARKER_NAME, FORMAT_MARKER_LENGTH) + .map_err(refusal)?; + AdmittedStoreFormatMarker::decode(&bytes) + .map(|_marker| ()) + .map_err(|source| Refusal::Marker { source }.into_io()) +} + +/// Maps a shared exact-record failure onto the marker's refusal. +fn refusal(error: ExactRecordError) -> io::Error { + match error { + ExactRecordError::Io(source) => source, + ExactRecordError::Refused(refusal) => match refusal { + ExactRecordRefusal::LengthOverflow => Refusal::LengthOverflow { name: MARKER_NAME }, + ExactRecordRefusal::TrailingBytes => Refusal::TrailingBytes { name: MARKER_NAME }, + ExactRecordRefusal::KindOrLength + | ExactRecordRefusal::KindLengthOrIdentity + | ExactRecordRefusal::Bytes + | ExactRecordRefusal::RemainedVisible => Refusal::KindOrLength { name: MARKER_NAME }, + } + .into_io(), + } +} diff --git a/src/adapters/filesystem_initialization_namespace.rs b/src/adapters/filesystem_initialization_namespace.rs index 39564d9b..d216eb1b 100644 --- a/src/adapters/filesystem_initialization_namespace.rs +++ b/src/adapters/filesystem_initialization_namespace.rs @@ -22,29 +22,24 @@ const PUBLISHED_NAMES: [&str; 5] = [ CATALOGS_NAME, HEAD_NAME, ]; -const VERSION_TWO_MARKERS: [&str; 10] = [ +const VERSION_TWO_MARKERS: [&str; 6] = [ "reader.lock", "FORMAT", "FORMAT.next", - "migration.intent", - "migration.intent.next", - "migration.receipt", - "migration.receipt.next", "retention", "gc", "recovery", ]; const READER_LOCK_NAME: &str = "reader.lock"; const MARKER_NAME: &str = "FORMAT"; -const INTENT_NAME: &str = "migration.intent"; -const RECEIPT_NAME: &str = "migration.receipt"; +const MARKER_STAGE_NAME: &str = "FORMAT.next"; const RETENTION_NAME: &str = "retention"; const GC_NAME: &str = "gc"; const RECOVERY_NAME: &str = "recovery"; const ROOTS_NAME: &str = "roots"; const MANIFESTS_NAME: &str = "manifests"; const DISPOSITIONS_NAME: &str = "dispositions"; -const VERSION_TWO_NAMES: [&str; 12] = [ +const VERSION_TWO_NAMES: [&str; 10] = [ LOCK_NAME, STAGING_NAME, SEGMENTS_NAME, @@ -52,14 +47,12 @@ const VERSION_TWO_NAMES: [&str; 12] = [ HEAD_NAME, READER_LOCK_NAME, MARKER_NAME, - INTENT_NAME, - RECEIPT_NAME, RETENTION_NAME, GC_NAME, RECOVERY_NAME, ]; const NEXT_HEAD_NAME: &str = "head.next"; -const VERSION_TWO_STAGED_NAMES: [&str; 13] = [ +const VERSION_TWO_STAGED_NAMES: [&str; 11] = [ LOCK_NAME, STAGING_NAME, SEGMENTS_NAME, @@ -68,8 +61,6 @@ const VERSION_TWO_STAGED_NAMES: [&str; 13] = [ NEXT_HEAD_NAME, READER_LOCK_NAME, MARKER_NAME, - INTENT_NAME, - RECEIPT_NAME, RETENTION_NAME, GC_NAME, RECOVERY_NAME, @@ -92,15 +83,15 @@ pub(super) fn admit_published(directory: &Dir) -> io::Result<()> { admit_membership(directory, &PUBLISHED_NAMES) } -/// Refuses version-two residue before version-one recovery touches a pool. +/// Refuses version-two state before version-one recovery touches a pool. /// /// Recovery may open a store at any lawful point of its version-one lifecycle, /// including before first publication and with `head.next` retained, and the /// recovery inventory already classifies every unknown root entry as -/// unexpected. What that classification cannot express is that a root has left -/// version one entirely: a format marker, reader fence, migration record or -/// stage, or a `retention`, `gc`, or `recovery` directory means version-one -/// recovery must refuse before pinning anything. +/// unexpected. What that classification cannot express is that a root is a +/// version-two store: a format marker or its stage, a reader fence, or a +/// `retention`, `gc`, or `recovery` directory means version-one recovery must +/// refuse before pinning anything. pub(super) fn admit_recoverable(directory: &Dir) -> io::Result<()> { for entry in directory.entries()? { let name = entry?.file_name(); @@ -111,18 +102,21 @@ pub(super) fn admit_recoverable(directory: &Dir) -> io::Result<()> { Ok(()) } -/// Admits the exact completely migrated version-2 root namespace. +/// Admits the exact committed version-2 root namespace. /// -/// Every version-1 published entry, the persistent reader fence, the format -/// marker, both migration records, and all three protocol directories must be -/// present. Any other entry is unrecoverable ambiguity. +/// `writer.lock`, `staging`, `segments`, `catalogs`, the persistent reader +/// fence, the format marker, and all three protocol directories must be +/// present; catalog `HEAD` is optional, because an initialized store holds +/// none until its first publication. A root that still holds `FORMAT.next` +/// refuses as `InitializationIncomplete` and a root without `FORMAT` as +/// `MissingFormatMarker`. Any other entry is unrecoverable ambiguity. pub(super) fn admit_version_two(directory: &Dir) -> io::Result<()> { admit_version_two_required(directory)?; admit_membership(directory, &VERSION_TWO_NAMES)?; admit_version_two_protocol_directories(directory) } -/// Admits a completely migrated version-2 root that may retain a catalog `head.next`. +/// Admits a committed version-2 root that may retain a catalog `head.next`. /// /// `head.next` is the version-1 catalog-protocol head stage, which version 2 /// preserves unchanged. A reader selects only `HEAD`, so a retained or @@ -138,28 +132,31 @@ pub(super) fn admit_version_two_with_catalog_stage(directory: &Dir) -> io::Resul } fn admit_version_two_required(directory: &Dir) -> io::Result<()> { + admit_committed(directory)?; + admit_version_two_layout(directory) +} + +/// Admits every committed root entry by kind; catalog `HEAD` is optional. +fn admit_version_two_layout(directory: &Dir) -> io::Result<()> { admit_required_file(directory, LOCK_NAME)?; admit_required_directory(directory, STAGING_NAME)?; admit_required_directory(directory, SEGMENTS_NAME)?; admit_required_directory(directory, CATALOGS_NAME)?; - admit_required_file(directory, HEAD_NAME)?; + admit_optional_file(directory, HEAD_NAME)?; admit_required_file(directory, READER_LOCK_NAME)?; admit_required_file(directory, MARKER_NAME)?; - admit_required_file(directory, INTENT_NAME)?; - admit_required_file(directory, RECEIPT_NAME)?; admit_required_directory(directory, RETENTION_NAME)?; admit_required_directory(directory, GC_NAME)?; admit_required_directory(directory, RECOVERY_NAME) } -/// Admits the nested version-2 protocol directories the migration writer left. +/// Admits the nested version-2 protocol directories initialization created. /// /// `retention` must carry both immutable pools (its head and stages belong to /// retention publication); `gc` must be empty until `KEEP-GC-001` implements -/// its records; `recovery` must hold exactly an empty `dispositions`. This is -/// the same membership `verify_prefix_directories` requires at the end of -/// migration, so a root that drifted after migration refuses here rather than -/// as a later pinning failure. +/// its records; `recovery` must hold exactly an empty `dispositions`. A root +/// that drifted after initialization refuses here rather than as a later +/// pinning failure. fn admit_version_two_protocol_directories(directory: &Dir) -> io::Result<()> { let retention = directory.open_dir_nofollow(RETENTION_NAME)?; admit_required_directory(&retention, ROOTS_NAME)?; @@ -236,34 +233,34 @@ fn ambiguous_namespace() -> io::Error { FilesystemNamespaceRefusal::UnexpectedEntry.into_io() } -/// Admits a published version-1 root carrying any subset of migration -/// residue, for migration recovery only. +/// Admits a committed root whose `FORMAT.next` cleanup was interrupted. /// -/// The five published names are required with their kinds; every migration -/// record, stage, the reader fence, and the three protocol directories are -/// optional but must have their kinds; anything else refuses. Which subsets -/// are lawful is the recovery planner's decision, not this admission's. -pub(super) fn admit_migrating(directory: &Dir) -> io::Result<()> { - admit_required_file(directory, LOCK_NAME)?; - admit_required_directory(directory, STAGING_NAME)?; - admit_required_directory(directory, SEGMENTS_NAME)?; - admit_required_directory(directory, CATALOGS_NAME)?; - admit_required_file(directory, HEAD_NAME)?; - for name in [ - READER_LOCK_NAME, - MARKER_NAME, - "FORMAT.next", - INTENT_NAME, - "migration.intent.next", - RECEIPT_NAME, - "migration.receipt.next", - ] { - admit_optional_file(directory, name)?; +/// Initialization uses this law only after it has proven that `FORMAT` and +/// `FORMAT.next` name one canonical marker inode. Every committed entry is +/// admitted as [`admit_version_two_with_catalog_stage`] admits it, and +/// `FORMAT.next` is additionally required. +pub(super) fn admit_version_two_marker_linked(directory: &Dir) -> io::Result<()> { + admit_required_file(directory, MARKER_STAGE_NAME)?; + admit_version_two_layout(directory)?; + admit_optional_file(directory, NEXT_HEAD_NAME)?; + let mut allowed: Vec<&str> = VERSION_TWO_STAGED_NAMES.to_vec(); + allowed.push(MARKER_STAGE_NAME); + admit_membership(directory, &allowed)?; + admit_version_two_protocol_directories(directory) +} + +/// Refuses an uncommitted root with a typed reason before any other law. +fn admit_committed(directory: &Dir) -> io::Result<()> { + match directory.symlink_metadata(MARKER_STAGE_NAME) { + Ok(_) => return Err(FilesystemNamespaceRefusal::InitializationIncomplete.into_io()), + Err(source) if source.kind() == io::ErrorKind::NotFound => {} + Err(source) => return Err(source), } - for name in [RETENTION_NAME, GC_NAME, RECOVERY_NAME] { - admit_optional_directory(directory, name)?; + match directory.symlink_metadata(MARKER_NAME) { + Ok(_) => Ok(()), + Err(source) if source.kind() == io::ErrorKind::NotFound => { + Err(FilesystemNamespaceRefusal::MissingFormatMarker.into_io()) + } + Err(source) => Err(source), } - let mut allowed: Vec<&str> = PUBLISHED_NAMES.to_vec(); - allowed.extend_from_slice(&VERSION_TWO_MARKERS); - admit_membership(directory, &allowed) } diff --git a/src/adapters/filesystem_namespace_refusal.rs b/src/adapters/filesystem_namespace_refusal.rs index 2151da4a..36f27d4f 100644 --- a/src/adapters/filesystem_namespace_refusal.rs +++ b/src/adapters/filesystem_namespace_refusal.rs @@ -39,6 +39,11 @@ impl FilesystemEntryKind { pub enum FilesystemNamespaceRefusal { /// An observed entry is outside the admitted protocol membership. UnexpectedEntry, + /// The root holds no `FORMAT` marker, so it is not a committed version-two store. + MissingFormatMarker, + /// The root still holds `FORMAT.next`: initialization has not committed. + /// Resume it with `FilesystemVersionTwoAdmission::initialize`. + InitializationIncomplete, /// A no-follow metadata observation established the wrong entry kind. WrongKind { /// Kind required by the protocol. @@ -52,6 +57,10 @@ impl fmt::Display for FilesystemNamespaceRefusal { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::UnexpectedEntry => formatter.write_str("unexpected protocol namespace entry"), + Self::MissingFormatMarker => formatter.write_str("the root holds no FORMAT marker"), + Self::InitializationIncomplete => { + formatter.write_str("initialization has not committed: FORMAT.next is present") + } Self::WrongKind { expected, observed } => write!( formatter, "protocol entry kind is {observed:?}, expected {expected:?}" diff --git a/src/adapters/filesystem_platform_admission.rs b/src/adapters/filesystem_platform_admission.rs index 556360ad..dc296494 100644 --- a/src/adapters/filesystem_platform_admission.rs +++ b/src/adapters/filesystem_platform_admission.rs @@ -1,7 +1,6 @@ //! This module owns proof that a filesystem root passed platform admission. use super::FilesystemWriterLock; -use super::filesystem_root_identity::FilesystemRootIdentity; /// Exclusive writer authority over a platform-admitted filesystem root. /// @@ -10,23 +9,16 @@ use super::filesystem_root_identity::FilesystemRootIdentity; #[must_use] pub struct FilesystemPlatformAdmission { lock: FilesystemWriterLock, - root_identity: FilesystemRootIdentity, } impl FilesystemPlatformAdmission { - pub(super) const fn initialized( - lock: FilesystemWriterLock, - root_identity: FilesystemRootIdentity, - ) -> Self { - Self { - lock, - root_identity, - } + pub(super) const fn initialized(lock: FilesystemWriterLock) -> Self { + Self { lock } } #[cfg(test)] - pub(super) fn unchecked_for_tests(lock: FilesystemWriterLock) -> std::io::Result { - Self::unchecked(lock) + pub(super) const fn unchecked_for_tests(lock: FilesystemWriterLock) -> Self { + Self::initialized(lock) } #[cfg(feature = "repository-tasks")] @@ -35,25 +27,11 @@ impl FilesystemPlatformAdmission { // descriptor of that same pinned directory, without an ambient reopen. let pinned = lock.clone_directory()?; let directory = super::sync_capable_directory::open(&pinned, ".")?; - let root_identity = super::filesystem_platform_profile::admit_locked_root(&directory)?; - Ok(Self::initialized(lock, root_identity)) + super::filesystem_platform_profile::admit_locked_root(&directory)?; + Ok(Self::initialized(lock)) } pub(super) fn into_lock(self) -> FilesystemWriterLock { self.lock } - - pub(super) fn into_parts(self) -> (FilesystemWriterLock, FilesystemRootIdentity) { - (self.lock, self.root_identity) - } - - /// Grants authority without platform admission only for private unit - /// tests; the identity probe tolerates a kernel that reports no mount - /// identity so the bypass does not require `STATX_MNT_ID`. - #[cfg(test)] - fn unchecked(lock: FilesystemWriterLock) -> std::io::Result { - let directory = lock.clone_directory()?; - let root_identity = super::filesystem_platform_profile::root_identity_lenient(&directory)?; - Ok(Self::initialized(lock, root_identity)) - } } diff --git a/src/adapters/filesystem_platform_admission_error.rs b/src/adapters/filesystem_platform_admission_error.rs index 62793eb1..ad906ab8 100644 --- a/src/adapters/filesystem_platform_admission_error.rs +++ b/src/adapters/filesystem_platform_admission_error.rs @@ -4,7 +4,7 @@ use std::error::Error; use std::fmt; use std::io; -use super::{StoreRootIdentityCoordinate, WriterLockAcquireError}; +use super::WriterLockAcquireError; /// Failure to reacquire writer authority over one published filesystem store. #[derive(Debug)] @@ -25,23 +25,11 @@ pub enum FilesystemPlatformAdmissionError { /// Preserved namespace-admission failure. source: io::Error, }, - /// A version-two migration record failed exact or joint admission. - MigrationRecord { + /// The version-two `FORMAT` marker failed exact admission. + FormatMarker { /// Preserved record-admission failure. source: io::Error, }, - /// The reopened root's physical identity is not the one the migration intent bound. - /// - /// Reopen compares the restart-stable device and file coordinates only; a - /// remounted store admits, so this variant never names `Mount`. - RootIdentityChanged { - /// The coordinate that disagreed. - coordinate: StoreRootIdentityCoordinate, - /// The value bound into `migration.intent`. - expected: u64, - /// The value observed on reopen. - observed: u64, - }, } impl fmt::Display for FilesystemPlatformAdmissionError { @@ -50,10 +38,7 @@ impl fmt::Display for FilesystemPlatformAdmissionError { Self::Platform { .. } => "published store platform admission failed", Self::WriterLock { .. } => "published store writer-lock acquisition failed", Self::Namespace { .. } => "published store namespace admission failed", - Self::MigrationRecord { .. } => "version-two migration record admission failed", - Self::RootIdentityChanged { .. } => { - "reopened root identity disagrees with the migration intent" - } + Self::FormatMarker { .. } => "version-two format marker admission failed", }) } } @@ -63,8 +48,7 @@ impl Error for FilesystemPlatformAdmissionError { match self { Self::Platform { source } | Self::Namespace { source } - | Self::MigrationRecord { source } => Some(source), - Self::RootIdentityChanged { .. } => None, + | Self::FormatMarker { source } => Some(source), Self::WriterLock { source } => Some(source), } } diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index 896ec543..64aea792 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -5,13 +5,11 @@ use std::path::Path; use cap_std::fs::Dir; -use super::filesystem_root_identity::FilesystemRootIdentity; - #[cfg(target_os = "linux")] const PROTOCOL_DIRECTORIES: [&str; 3] = ["staging", "segments", "catalogs"]; /// Every version-two protocol directory, including nested immutable pools. /// -/// A migrated root receives writer authority only when each of these shares +/// A version-two root receives writer authority only when each of these shares /// the root's filesystem type, device, and mount identity and is not /// casefolded or read-only. #[cfg(target_os = "linux")] @@ -58,13 +56,12 @@ pub(super) fn open(store_root: &Path) -> io::Result { /// Applies the production profile to the root already pinned by writer authority. #[cfg(all(target_os = "linux", feature = "repository-tasks"))] -pub(super) fn admit_locked_root(directory: &Dir) -> io::Result { - admit_linux_profile(directory, &PROTOCOL_DIRECTORIES)?; - root_identity(directory) +pub(super) fn admit_locked_root(directory: &Dir) -> io::Result<()> { + admit_linux_profile(directory, &PROTOCOL_DIRECTORIES) } #[cfg(all(not(target_os = "linux"), feature = "repository-tasks"))] -pub(super) fn admit_locked_root(_directory: &Dir) -> io::Result { +pub(super) fn admit_locked_root(_directory: &Dir) -> io::Result<()> { Err(io::Error::new( io::ErrorKind::Unsupported, "catalog publication requires the admitted Linux ext4 profile", @@ -166,115 +163,6 @@ fn linux_directory_properties(file: &std::fs::File) -> io::Result Option { - match (policy, reported) { - (_, true) => Some(mount_id), - (MountIdentityPolicy::Required, false) => None, - #[cfg(any(test, feature = "repository-tasks"))] - (MountIdentityPolicy::Lenient, false) => Some(0), - } -} - -#[cfg(target_os = "linux")] -pub(super) fn root_identity(directory: &Dir) -> io::Result { - let file = directory.try_clone()?.into_std_file(); - linux_file_identity(&file, MountIdentityPolicy::Required) -} - -/// Probes root identity for the test and repository-task admission bypass. -/// -/// Identical to [`root_identity`] except that an unreported mount identity is -/// recorded as zero instead of refusing; production admission never uses it. -#[cfg(all(target_os = "linux", any(test, feature = "repository-tasks")))] -pub(super) fn root_identity_lenient(directory: &Dir) -> io::Result { - let file = directory.try_clone()?.into_std_file(); - linux_file_identity(&file, MountIdentityPolicy::Lenient) -} - -#[cfg(target_os = "linux")] -fn linux_file_identity( - file: &std::fs::File, - policy: MountIdentityPolicy, -) -> io::Result { - use rustix::fs::{AtFlags, StatxFlags, statx}; - - let requested = StatxFlags::BASIC_STATS | StatxFlags::MNT_ID; - let status = statx(file, ".", AtFlags::empty(), requested)?; - let observed = StatxFlags::from_bits_retain(status.stx_mask); - if !observed.contains(StatxFlags::BASIC_STATS) { - return Err(unsupported_linux_profile()); - } - let reported = observed.contains(StatxFlags::MNT_ID); - let mount_id = admit_mount_identity(policy, reported, status.stx_mnt_id) - .ok_or_else(unsupported_linux_profile)?; - Ok(linux_root_identity( - status.stx_dev_major, - status.stx_dev_minor, - mount_id, - status.stx_ino, - )) -} - -#[cfg(target_os = "linux")] -fn linux_root_identity( - device_major: u32, - device_minor: u32, - mount_id: u64, - inode: u64, -) -> FilesystemRootIdentity { - let device = rustix::fs::makedev(device_major, device_minor); - FilesystemRootIdentity::new(device, mount_id, inode) -} - -#[cfg(all(not(target_os = "linux"), any(test, feature = "repository-tasks")))] -pub(super) fn root_identity(directory: &Dir) -> io::Result { - use cap_fs_ext::MetadataExt; - - let metadata = directory.dir_metadata()?; - Ok(FilesystemRootIdentity::new( - metadata.dev(), - metadata.dev(), - metadata.ino(), - )) -} - -#[cfg(all(not(target_os = "linux"), any(test, feature = "repository-tasks")))] -pub(super) fn root_identity_lenient(directory: &Dir) -> io::Result { - root_identity(directory) -} - -#[cfg(all(not(target_os = "linux"), not(any(test, feature = "repository-tasks"))))] -pub(super) fn root_identity(_directory: &Dir) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "filesystem root identity currently requires the admitted Linux ext4 profile", - )) -} - #[cfg(target_os = "linux")] fn admit_linux_properties( filesystem_type: rustix::fs::FsWord, @@ -322,7 +210,3 @@ fn unsupported_linux_profile() -> io::Error { #[cfg(all(test, target_os = "linux"))] #[path = "filesystem_platform_profile_tests.rs"] mod tests; - -#[cfg(test)] -#[path = "filesystem_platform_profile_policy_tests.rs"] -mod policy_tests; diff --git a/src/adapters/filesystem_platform_profile_policy_tests.rs b/src/adapters/filesystem_platform_profile_policy_tests.rs deleted file mode 100644 index 9bcd63f5..00000000 --- a/src/adapters/filesystem_platform_profile_policy_tests.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Mount-identity policy laws shared by every platform's test build. - -use super::{MountIdentityPolicy, admit_mount_identity}; - -#[test] -fn production_identity_requires_a_reported_mount_identity() { - assert_eq!( - admit_mount_identity(MountIdentityPolicy::Required, true, 7), - Some(7) - ); - assert_eq!( - admit_mount_identity(MountIdentityPolicy::Required, false, 7), - None - ); -} - -#[test] -fn bypass_identity_records_an_unreported_mount_identity_as_zero() { - assert_eq!( - admit_mount_identity(MountIdentityPolicy::Lenient, true, 7), - Some(7) - ); - assert_eq!( - admit_mount_identity(MountIdentityPolicy::Lenient, false, 7), - Some(0) - ); -} diff --git a/src/adapters/filesystem_platform_profile_tests.rs b/src/adapters/filesystem_platform_profile_tests.rs index 13f97f36..09082761 100644 --- a/src/adapters/filesystem_platform_profile_tests.rs +++ b/src/adapters/filesystem_platform_profile_tests.rs @@ -2,7 +2,7 @@ use super::{ LinuxDirectoryProperties, PROTOCOL_DIRECTORIES, VERSION_TWO_PROTOCOL_DIRECTORIES, - admit_linux_child_properties, admit_linux_properties, linux_root_identity, + admit_linux_child_properties, admit_linux_properties, }; use rustix::fs::{NFS_SUPER_MAGIC, StatVfsMountFlags}; @@ -48,14 +48,6 @@ fn every_protocol_child_must_share_the_root_filesystem_and_mount() { assert_unsupported(&admit_linux_child_properties(root, foreign_format)); } -#[test] -fn root_identity_uses_linux_device_mount_and_inode_coordinates() { - let identity = linux_root_identity(8, 1, 41, 73); - assert_eq!(identity.device(), rustix::fs::makedev(8, 1)); - assert_eq!(identity.mount(), 41); - assert_eq!(identity.file(), 73); -} - fn assert_unsupported(result: &std::io::Result<()>) { assert!(matches!( result, diff --git a/src/adapters/filesystem_recovery_inventory_reader.rs b/src/adapters/filesystem_recovery_inventory_reader.rs index 4acc2b10..99c776a8 100644 --- a/src/adapters/filesystem_recovery_inventory_reader.rs +++ b/src/adapters/filesystem_recovery_inventory_reader.rs @@ -71,12 +71,12 @@ impl FilesystemRecoveryInventoryReader { Self::from_root(root) } - /// Opens one migrated version-two store for read-only stage inventory. + /// Opens one committed version-two store for read-only stage inventory. /// /// Admission is the version-two recovery law: the production platform for /// every version-two protocol directory, the exact namespace with an - /// optional retained catalog `head.next`, the jointly admitted migration - /// records, and the restart-stable root identity. No writer lock is taken + /// optional retained catalog `head.next`, and the canonical `FORMAT` marker. + /// No writer lock is taken /// and no protocol state is mutated. Classify the resulting inventory with /// [`classify_version_two_recovery_names`](crate::classify_version_two_recovery_names). /// @@ -96,7 +96,7 @@ impl FilesystemRecoveryInventoryReader { Self::pin(root) } - /// Opens a migrated root without platform admission for repository tasks. + /// Opens a committed root without platform admission for repository tasks. /// /// Every namespace, record, and identity law of [`Self::open_version_two`] /// still applies. diff --git a/src/adapters/filesystem_recovery_next_head_finalizer.rs b/src/adapters/filesystem_recovery_next_head_finalizer.rs index cf66241f..eac78150 100644 --- a/src/adapters/filesystem_recovery_next_head_finalizer.rs +++ b/src/adapters/filesystem_recovery_next_head_finalizer.rs @@ -58,7 +58,7 @@ impl FilesystemRecoveryNextHeadFinalizer { .map_err(Into::into) } - /// Pins a migrated version-two store for gated head finalization. + /// Pins a committed version-two store for gated head finalization. /// /// The call performs no protocol mutation and reads no record. Execution /// verifies every retained closure named by the current retention head diff --git a/src/adapters/filesystem_recovery_segment_resumer.rs b/src/adapters/filesystem_recovery_segment_resumer.rs index 435e2788..98b5d153 100644 --- a/src/adapters/filesystem_recovery_segment_resumer.rs +++ b/src/adapters/filesystem_recovery_segment_resumer.rs @@ -35,7 +35,7 @@ impl FilesystemRecoverySegmentResumer { .map_err(Into::into) } - /// Pins a migrated version-two store for explicit segment continuation. + /// Pins a committed version-two store for explicit segment continuation. /// /// The call performs no protocol mutation. /// diff --git a/src/adapters/filesystem_recovery_stage_completer.rs b/src/adapters/filesystem_recovery_stage_completer.rs index 6d7e9ab6..db5f6860 100644 --- a/src/adapters/filesystem_recovery_stage_completer.rs +++ b/src/adapters/filesystem_recovery_stage_completer.rs @@ -33,7 +33,7 @@ impl FilesystemRecoveryStageCompleter { .map_err(Into::into) } - /// Pins a migrated version-two store for explicit stage completion. + /// Pins a committed version-two store for explicit stage completion. /// /// The call performs no protocol mutation. Completion links a stage into /// an immutable pool and never changes `HEAD`. diff --git a/src/adapters/filesystem_recovery_stage_discarder.rs b/src/adapters/filesystem_recovery_stage_discarder.rs index c3db169d..e50249c9 100644 --- a/src/adapters/filesystem_recovery_stage_discarder.rs +++ b/src/adapters/filesystem_recovery_stage_discarder.rs @@ -75,7 +75,7 @@ impl FilesystemRecoveryStageDiscarder { Self::from_root(root) } - /// Pins a migrated version-two store for explicit stage discard. + /// Pins a committed version-two store for explicit stage discard. /// /// The admission already holds writer authority over a root that passed /// version-two recovery admission; this call pins the catalog-protocol diff --git a/src/adapters/filesystem_root_identity.rs b/src/adapters/filesystem_root_identity.rs deleted file mode 100644 index 36af8bba..00000000 --- a/src/adapters/filesystem_root_identity.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! This module owns one admitted physical filesystem-root coordinate. - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) struct FilesystemRootIdentity { - device: u64, - mount: u64, - file: u64, -} - -impl FilesystemRootIdentity { - #[cfg(any( - target_os = "linux", - all(not(target_os = "linux"), any(test, feature = "repository-tasks")) - ))] - pub(super) const fn new(device: u64, mount: u64, file: u64) -> Self { - Self { - device, - mount, - file, - } - } - - pub(super) const fn device(self) -> u64 { - self.device - } - - pub(super) const fn mount(self) -> u64 { - self.mount - } - - pub(super) const fn file(self) -> u64 { - self.file - } -} diff --git a/src/adapters/filesystem_store_initializer.rs b/src/adapters/filesystem_store_initializer.rs index ff2fb3a6..e122eeb2 100644 --- a/src/adapters/filesystem_store_initializer.rs +++ b/src/adapters/filesystem_store_initializer.rs @@ -83,17 +83,7 @@ fn initialize_storage( let lock = storage.into_lock().map_err(|source| { StoreInitializationError::io(StoreInitializationPhase::OpenAndLockWriterFile, source) })?; - let directory = lock.clone_directory().map_err(|source| { - StoreInitializationError::io(StoreInitializationPhase::AdmitPlatform, source) - })?; - let root_identity = - filesystem_platform_profile::root_identity(&directory).map_err(|source| { - StoreInitializationError::io(StoreInitializationPhase::AdmitPlatform, source) - })?; - Ok(FilesystemPlatformAdmission::initialized( - lock, - root_identity, - )) + Ok(FilesystemPlatformAdmission::initialized(lock)) } fn reopen_root( @@ -113,10 +103,5 @@ fn admit_reopened( .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; admit_namespace(&directory) .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; - let root_identity = filesystem_platform_profile::root_identity(&directory) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; - Ok(FilesystemPlatformAdmission::initialized( - lock, - root_identity, - )) + Ok(FilesystemPlatformAdmission::initialized(lock)) } diff --git a/src/adapters/filesystem_version_two_admission.rs b/src/adapters/filesystem_version_two_admission.rs index 2d2d2f58..9579c24a 100644 --- a/src/adapters/filesystem_version_two_admission.rs +++ b/src/adapters/filesystem_version_two_admission.rs @@ -7,24 +7,31 @@ use cap_fs_ext::DirExt; use cap_std::ambient_authority; use cap_std::fs::Dir; -use super::filesystem_root_identity::FilesystemRootIdentity; -pub(super) use super::filesystem_version_two_records::BoundRootIdentity; +#[cfg(any(test, feature = "repository-tasks"))] +use super::VersionTwoInitializationObserver; +use super::version_two_initialization::{self as initialization, PhaseObserver}; use super::{ - FilesystemPlatformAdmissionError, FilesystemWriterLock, StoreRootIdentityCoordinate, - filesystem_initialization_namespace, filesystem_platform_profile, - filesystem_version_two_records, + FilesystemPlatformAdmissionError, FilesystemWriterLock, VersionTwoInitialization, + VersionTwoInitializationError, filesystem_format_marker, filesystem_initialization_namespace, + filesystem_platform_profile, }; -/// Exclusive writer authority over a completely migrated version-two root. +/// Exclusive writer authority over a committed version-two root. /// /// This type is deliberately distinct from /// [`FilesystemPlatformAdmission`](super::FilesystemPlatformAdmission): a /// version-one publisher cannot consume it, so version-one catalog publication -/// can never run against a migrated root and leave residue no adapter admits. -/// Fields are private so only version-two admission can create values. The -/// admitted `retention`, `roots`, and `manifests` capabilities are retained, -/// so the authority built from this value operates on the directories that -/// passed admission and not on whatever those names resolve to later. +/// can never run against a version-two root and leave residue no adapter +/// admits. Fields are private so only version-two admission and +/// initialization can create values. The admitted `retention`, `roots`, and +/// `manifests` capabilities are retained, so the authority built from this +/// value operates on the directories that passed admission and not on whatever +/// those names resolve to later. +/// +/// Admission binds the store to no directory: nothing persisted records the +/// root's device, inode, mount, or path, so a quiescent copy opens wherever +/// it is placed. Live authority is the writer lock and the pinned +/// capabilities of this process. #[must_use] pub struct FilesystemVersionTwoAdmission { lock: FilesystemWriterLock, @@ -34,31 +41,93 @@ pub struct FilesystemVersionTwoAdmission { } impl FilesystemVersionTwoAdmission { - /// Reacquires writer authority over one completely migrated version-two store. + /// Initializes a version-two store at `store_root`, or resumes an interrupted initialization. + /// + /// The root must exist. An empty root, or a recoverable prefix that an + /// interrupted call left, becomes a committed store with no catalog + /// `HEAD`; every other root is refused before it is mutated. The call + /// admits the production platform, holds exclusive writer authority from + /// its first mutation, and returns only after the commit point, the + /// removal of `FORMAT.next`, the final root synchronization, and complete + /// admission of the committed root. It is synchronous, allocates no + /// content-sized memory, and may block on filesystem I/O. + /// + /// # Errors + /// + /// Returns [`VersionTwoInitializationError`] with the exact platform, + /// writer-lock, refusal, phase, or admission failure. A committed store + /// refuses as + /// [`AlreadyInitialized`](super::VersionTwoInitializationRefusal::AlreadyInitialized); + /// a concurrent attempt refuses with a busy writer lock. + pub fn initialize( + store_root: &Path, + ) -> Result { + let root = filesystem_platform_profile::open_version_two(store_root) + .map_err(|source| VersionTwoInitializationError::Platform { source })?; + initialize_root(&root, &mut |_, _| Ok(())) + } + + /// Initializes without platform admission for repository tasks. + /// + /// Every protocol law after platform admission still applies. Production + /// callers use [`Self::initialize`]. + /// + /// # Errors + /// + /// Returns [`VersionTwoInitializationError`] exactly as [`Self::initialize`] + /// does after platform admission. + #[cfg(any(test, feature = "repository-tasks"))] + pub fn initialize_unchecked_for_repository_tasks( + store_root: &Path, + ) -> Result { + initialize_root(&open_unchecked(store_root)?, &mut |_, _| Ok(())) + } + + /// Initializes without platform admission, notifying `observer` around each phase. + /// + /// The crash matrix blocks inside the observer for process death; an + /// observer error fails the phase at that boundary. + /// + /// # Errors + /// + /// Returns [`VersionTwoInitializationError`] as + /// [`Self::initialize_unchecked_for_repository_tasks`] does, or the + /// observer's error at its phase. + #[cfg(any(test, feature = "repository-tasks"))] + #[doc(hidden)] + pub fn initialize_observed_for_repository_tasks( + store_root: &Path, + mut observer: VersionTwoInitializationObserver, + ) -> Result { + initialize_root(&open_unchecked(store_root)?, &mut *observer) + } + + /// Reacquires writer authority over one committed version-two store. /// /// The call mutates no protocol state. It admits the production platform /// for every version-two protocol directory, acquires the existing writer - /// lock, requires the exact version-two root namespace, and jointly admits - /// the marker, intent, and receipt records. Retention adapters perform - /// content-level validation under the returned authority. The synchronous - /// call may block on filesystem I/O. + /// lock, requires the exact committed root namespace, and admits the + /// canonical `FORMAT` marker. A store with no catalog `HEAD` admits. + /// Retention adapters perform content-level validation under the + /// returned authority. The synchronous call may block on filesystem I/O. /// /// # Errors /// /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, - /// writer-lock, namespace, or migration-record boundary and preserved - /// source. + /// writer-lock, namespace, or format-marker boundary and preserved source. + /// A root whose initialization has not committed refuses at the namespace + /// boundary. pub fn reopen(store_root: &Path) -> Result { let root = filesystem_platform_profile::open_version_two(store_root) .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; Self::admit(root) } - /// Reopens a migrated root without platform admission for repository tasks. + /// Reopens a committed root without platform admission for repository tasks. /// /// The crash matrix and other repository tools run on hosts outside the - /// admitted Linux profile; every namespace, record, and identity law still - /// applies. Production callers use [`Self::reopen`]. + /// admitted Linux profile; every namespace and record law still applies. + /// Production callers use [`Self::reopen`]. /// /// # Errors /// @@ -79,32 +148,62 @@ impl FilesystemVersionTwoAdmission { } fn admit(root: Dir) -> Result { - let (lock, retention, roots, manifests) = - admit_locked(root, filesystem_initialization_namespace::admit_version_two)?; - Ok(Self { + admit_locked(root, filesystem_initialization_namespace::admit_version_two) + .map(Self::from_parts) + } + + fn from_parts((lock, retention, roots, manifests): AdmittedVersionTwoParts) -> Self { + Self { lock, retention, roots, manifests, - }) + } } } /// Writer authority and the three pinned retention capabilities of one admitted root. pub(super) type AdmittedVersionTwoParts = (FilesystemWriterLock, Dir, Dir, Dir); +#[cfg(any(test, feature = "repository-tasks"))] +fn open_unchecked(store_root: &Path) -> Result { + Dir::open_ambient_dir(store_root, ambient_authority()) + .map_err(|source| VersionTwoInitializationError::Platform { source }) +} + +/// Runs the initialization protocol, then admits the committed root under its lock. +fn initialize_root( + root: &Dir, + observer: PhaseObserver<'_>, +) -> Result { + let (lock, found) = initialization::initialize(root, observer)?; + let parts = admit_under_lock(lock, filesystem_initialization_namespace::admit_version_two) + .map_err(|source| VersionTwoInitializationError::Admission { source })?; + Ok(VersionTwoInitialization::new( + FilesystemVersionTwoAdmission::from_parts(parts), + found, + )) +} + /// Acquires writer authority, then admits the root under `namespace`. /// /// Forward writers pass the exact version-two namespace law; catalog-stage /// recovery passes the law that also admits a retained `head.next`. Every -/// other check is identical: the marker, intent, and receipt records, the -/// restart-stable root identity, and the pinned retention directories. +/// other check is identical: the `FORMAT` marker and the pinned retention +/// directories. pub(super) fn admit_locked( root: Dir, namespace: fn(&Dir) -> std::io::Result<()>, ) -> Result { let lock = FilesystemWriterLock::try_acquire_in(root) .map_err(|source| FilesystemPlatformAdmissionError::WriterLock { source })?; + admit_under_lock(lock, namespace) +} + +fn admit_under_lock( + lock: FilesystemWriterLock, + namespace: fn(&Dir) -> std::io::Result<()>, +) -> Result { let directory = lock .clone_directory() .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; @@ -115,18 +214,15 @@ pub(super) fn admit_locked( Ok((lock, retention, roots, manifests)) } -/// Admits the namespace, migration records, and root identity without mutation. +/// Admits the namespace and the `FORMAT` marker without mutation. pub(super) fn admit_root( directory: &Dir, namespace: fn(&Dir) -> std::io::Result<()>, ) -> Result<(), FilesystemPlatformAdmissionError> { namespace(directory) .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; - let observed = filesystem_platform_profile::root_identity(directory) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; - let bound = filesystem_version_two_records::admit(directory) - .map_err(|source| FilesystemPlatformAdmissionError::MigrationRecord { source })?; - require_root_identity(bound, observed) + filesystem_format_marker::admit(directory) + .map_err(|source| FilesystemPlatformAdmissionError::FormatMarker { source }) } /// Pins one admitted protocol directory without following links. @@ -135,38 +231,3 @@ fn pin(parent: &Dir, name: &str) -> Result Result<(), FilesystemPlatformAdmissionError> { - for (coordinate, expected, actual) in [ - ( - StoreRootIdentityCoordinate::Device, - bound.device(), - observed.device(), - ), - ( - StoreRootIdentityCoordinate::File, - bound.file(), - observed.file(), - ), - ] { - if expected != actual { - return Err(FilesystemPlatformAdmissionError::RootIdentityChanged { - coordinate, - expected, - observed: actual, - }); - } - } - Ok(()) -} diff --git a/src/adapters/filesystem_version_two_record_refusal.rs b/src/adapters/filesystem_version_two_record_refusal.rs index 21711cff..e8927272 100644 --- a/src/adapters/filesystem_version_two_record_refusal.rs +++ b/src/adapters/filesystem_version_two_record_refusal.rs @@ -4,15 +4,13 @@ use std::error::Error; use std::fmt; use std::io; -use super::{ - StoreFormatMarkerDecodeError, StoreMigrationIntentDecodeError, StoreMigrationReceiptDecodeError, -}; +use super::StoreFormatMarkerDecodeError; -/// Why `FORMAT`, `migration.intent`, or `migration.receipt` refused admission. +/// Why the `FORMAT` record refused admission. /// /// Carried as the source of the `InvalidData` error behind -/// [`FilesystemPlatformAdmissionError::MigrationRecord`](super::FilesystemPlatformAdmissionError::MigrationRecord), -/// so a caller can tell which record refused and recover the decoder's own +/// [`FilesystemPlatformAdmissionError::FormatMarker`](super::FilesystemPlatformAdmissionError::FormatMarker), +/// so a caller can recover the decoder's own /// diagnosis instead of a rendered string. #[derive(Debug)] #[non_exhaustive] @@ -37,16 +35,6 @@ pub enum VersionTwoRecordRefusal { /// The exact decode refusal. source: StoreFormatMarkerDecodeError, }, - /// `migration.intent` did not decode as a canonical intent. - Intent { - /// The exact decode refusal. - source: StoreMigrationIntentDecodeError, - }, - /// `migration.receipt` did not admit against the decoded intent and marker. - Receipt { - /// The exact admission refusal. - source: StoreMigrationReceiptDecodeError, - }, } impl VersionTwoRecordRefusal { @@ -78,12 +66,6 @@ impl fmt::Display for VersionTwoRecordRefusal { Self::Marker { .. } => { formatter.write_str("version-two record FORMAT refused admission") } - Self::Intent { .. } => { - formatter.write_str("version-two record migration.intent refused admission") - } - Self::Receipt { .. } => { - formatter.write_str("version-two record migration.receipt refused admission") - } } } } @@ -92,8 +74,6 @@ impl Error for VersionTwoRecordRefusal { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::Marker { source } => Some(source), - Self::Intent { source } => Some(source), - Self::Receipt { source } => Some(source), Self::LengthOverflow { .. } | Self::KindOrLength { .. } | Self::TrailingBytes { .. } => None, diff --git a/src/adapters/filesystem_version_two_records.rs b/src/adapters/filesystem_version_two_records.rs deleted file mode 100644 index acb2e1f7..00000000 --- a/src/adapters/filesystem_version_two_records.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! This module owns joint admission of the three fixed version-two migration records. - -use std::io; - -use cap_std::fs::Dir; - -use super::VersionTwoRecordRefusal as Refusal; -use super::filesystem_exact_record::{self as exact_record, ExactRecordError, ExactRecordRefusal}; -use super::store_migration::{ - FORMAT_MARKER_LENGTH, MIGRATION_INTENT_LENGTH, MIGRATION_RECEIPT_LENGTH, -}; -use super::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, -}; - -const MARKER_NAME: &str = "FORMAT"; -const INTENT_NAME: &str = "migration.intent"; -const RECEIPT_NAME: &str = "migration.receipt"; - -/// Root identity coordinates bound into an admitted `migration.intent`. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) struct BoundRootIdentity { - device: u64, - file: u64, -} - -impl BoundRootIdentity { - /// Binds the two restart-stable root coordinates the intent persists. - /// - /// The intent also records the mount identity observed during migration, - /// but `statx.stx_mnt_id` can change across mount instances and reboots, so - /// reopen does not compare it; the migration authority compares it only - /// inside the process that observed it. - pub(super) const fn new(device: u64, file: u64) -> Self { - Self { device, file } - } - - pub(super) const fn device(self) -> u64 { - self.device - } - - pub(super) const fn file(self) -> u64 { - self.file - } -} - -/// Reads and jointly admits `FORMAT`, `migration.intent`, and `migration.receipt`. -/// -/// Each record is reopened without following links, bounded to its exact -/// canonical length, and decoded. The receipt is admitted only against the -/// decoded intent and marker, so a record set that is individually -/// well-formed but mutually inconsistent refuses. Writer authority over a -/// version-two root must not be returned before this admission succeeds. Every -/// refusal is an `InvalidData` error whose source is a -/// [`VersionTwoRecordRefusal`](super::VersionTwoRecordRefusal). The intent's -/// bound root coordinates are returned for identity comparison. -pub(super) fn admit(root: &Dir) -> io::Result { - let marker_bytes = read_exact(root, MARKER_NAME, FORMAT_MARKER_LENGTH)?; - let intent_bytes = read_exact(root, INTENT_NAME, MIGRATION_INTENT_LENGTH)?; - let receipt_bytes = read_exact(root, RECEIPT_NAME, MIGRATION_RECEIPT_LENGTH)?; - let marker = AdmittedStoreFormatMarker::decode(&marker_bytes) - .map_err(|source| Refusal::Marker { source }.into_io())?; - let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes) - .map_err(|source| Refusal::Intent { source }.into_io())?; - let _receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &intent, &marker) - .map_err(|source| Refusal::Receipt { source }.into_io())?; - Ok(BoundRootIdentity::new( - intent.root_device_identity().get(), - intent.root_file_identity().get(), - )) -} - -/// Reads one required record, mapping shared refusals onto this record's refusal. -fn read_exact(root: &Dir, name: &'static str, length: usize) -> io::Result> { - exact_record::read_exact_regular(root, name, length).map_err(|error| match error { - ExactRecordError::Io(source) => source, - ExactRecordError::Refused(refusal) => match refusal { - ExactRecordRefusal::LengthOverflow => Refusal::LengthOverflow { name }, - ExactRecordRefusal::TrailingBytes => Refusal::TrailingBytes { name }, - ExactRecordRefusal::KindOrLength - | ExactRecordRefusal::KindLengthOrIdentity - | ExactRecordRefusal::Bytes - | ExactRecordRefusal::RemainedVisible => Refusal::KindOrLength { name }, - } - .into_io(), - }) -} diff --git a/src/adapters/filesystem_version_two_recovery_admission.rs b/src/adapters/filesystem_version_two_recovery_admission.rs index 62561f31..582c8bef 100644 --- a/src/adapters/filesystem_version_two_recovery_admission.rs +++ b/src/adapters/filesystem_version_two_recovery_admission.rs @@ -12,34 +12,33 @@ use super::{ filesystem_platform_profile, }; -/// Exclusive writer authority over a migrated version-two root for stage recovery. +/// Exclusive writer authority over a committed version-two root for stage recovery. /// /// Admission is the version-two writer admission with one difference: the /// root may retain the catalog `head.next` that an interrupted publication /// left. Every other law is the same: the production platform, the writer -/// lock, the exact namespace with its protocol directories, the jointly -/// admitted marker, intent, and receipt records, and the restart-stable root -/// identity. The admitted `retention`, `roots`, and `manifests` capabilities +/// lock, the exact namespace with its protocol directories, and the canonical +/// `FORMAT` marker. The admitted `retention`, `roots`, and `manifests` capabilities /// are retained so recovery finalization verifies retained closures against /// the directories that passed admission. /// /// Only the version-two recovery constructors consume this value; version-one /// recovery keeps refusing a version-two root, and this admission refuses a -/// version-one root because the marker and migration records are required. +/// version-one root because the marker is required. #[must_use] pub struct FilesystemVersionTwoRecoveryAdmission { parts: AdmittedVersionTwoParts, } impl FilesystemVersionTwoRecoveryAdmission { - /// Acquires writer authority over one migrated store for stage recovery. + /// Acquires writer authority over one committed store for stage recovery. /// /// The call mutates no protocol state and may block on filesystem I/O. /// /// # Errors /// /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, - /// writer-lock, namespace, or migration-record boundary and preserved + /// writer-lock, namespace, or format-marker boundary and preserved /// source. pub fn reopen(store_root: &Path) -> Result { let root = filesystem_platform_profile::open_version_two(store_root) @@ -47,9 +46,9 @@ impl FilesystemVersionTwoRecoveryAdmission { Self::admit(root) } - /// Reopens a migrated root without platform admission for repository tasks. + /// Reopens a committed root without platform admission for repository tasks. /// - /// Every namespace, record, and identity law still applies. Production + /// Every namespace and record law still applies. Production /// callers use [`Self::reopen`]. /// /// # Errors diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index aa2f2f88..01d8a46e 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -91,6 +91,7 @@ mod filesystem_catalog_snapshot; mod filesystem_catalog_stages; mod filesystem_catalog_storage; mod filesystem_exact_record; +mod filesystem_format_marker; mod filesystem_initialization_namespace; mod filesystem_initialization_storage; mod filesystem_namespace_refusal; @@ -133,7 +134,6 @@ mod filesystem_recovery_stage_materialization; mod filesystem_recovery_stage_sync; #[cfg(all(test, unix))] mod filesystem_recovery_stage_tests; -mod filesystem_root_identity; mod filesystem_segment_stage; #[cfg(test)] mod filesystem_segment_stage_tests; @@ -150,7 +150,6 @@ mod filesystem_version_two_catalog_recovery_tests; mod filesystem_version_two_catalog_stages; mod filesystem_version_two_catalog_storage; mod filesystem_version_two_record_refusal; -mod filesystem_version_two_records; mod filesystem_version_two_recovery_admission; mod filesystem_writer_lock; mod framed_blake3; @@ -249,12 +248,12 @@ mod segment_write_phase; mod staged_segment; mod storage_profile_id_text; mod storage_profile_id_text_error; +mod store_format; mod store_initialization; mod store_initialization_error; mod store_initialization_phase; mod store_initialization_receipt; mod store_initialization_storage; -mod store_migration; mod sync_capable_directory; #[cfg(test)] #[path = "../../tests/support/mod.rs"] @@ -264,6 +263,7 @@ mod verification_decode_error; mod verification_error; mod verification_failure_class; mod verification_ingress; +mod version_two_initialization; mod writer_lock_acquire_error; mod writer_lock_acquire_phase; diff --git a/src/adapters/recovery/recovery_entry_role.rs b/src/adapters/recovery/recovery_entry_role.rs index c54450b9..05b4d50d 100644 --- a/src/adapters/recovery/recovery_entry_role.rs +++ b/src/adapters/recovery/recovery_entry_role.rs @@ -38,10 +38,6 @@ pub enum RecoveryEntryRole { ReaderFence, /// Version-two format marker `FORMAT`. FormatMarker, - /// Version-two migration intent record. - MigrationIntent, - /// Version-two migration receipt record. - MigrationReceipt, /// Version-two `retention` protocol directory, admitted by retention recovery. RetentionDirectory, /// Version-two reserved `gc` protocol directory. diff --git a/src/adapters/recovery/recovery_name_classification.rs b/src/adapters/recovery/recovery_name_classification.rs index fe7bde87..f1b7c27c 100644 --- a/src/adapters/recovery/recovery_name_classification.rs +++ b/src/adapters/recovery/recovery_name_classification.rs @@ -23,13 +23,14 @@ pub fn classify_recovery_names( classify_inventory(inventory, StoreLayout::VersionOne) } -/// Classifies one complete migrated version-two inventory without opening bytes. +/// Classifies one complete committed version-two inventory without opening bytes. /// /// The version-one laws apply unchanged, and the version-two root entries -/// `reader.lock`, `FORMAT`, `migration.intent`, `migration.receipt`, -/// `retention`, `gc`, and `recovery` are classified as known roles, never as -/// unexpected or ignored. Each of them and the published `HEAD` is required. -/// Migration stages are unknown: a completely migrated root never holds one. +/// `reader.lock`, `FORMAT`, `retention`, `gc`, and `recovery` are classified +/// as known roles, never as unexpected or ignored. Each of them is required; +/// catalog `HEAD` is not, because the first publication of an initialized +/// store can be interrupted before it exists. `FORMAT.next` is unknown: a +/// committed root never holds it. /// The contents of `retention`, `gc`, and `recovery` are not inventoried here; /// version-two admission and retention recovery own them. /// @@ -129,8 +130,6 @@ fn classify_version_two_root(name: &[u8]) -> Result Ok(RecoveryEntryRole::ReaderFence), b"FORMAT" => Ok(RecoveryEntryRole::FormatMarker), - b"migration.intent" => Ok(RecoveryEntryRole::MigrationIntent), - b"migration.receipt" => Ok(RecoveryEntryRole::MigrationReceipt), b"retention" => Ok(RecoveryEntryRole::RetentionDirectory), b"gc" => Ok(RecoveryEntryRole::GcDirectory), b"recovery" => Ok(RecoveryEntryRole::RecoveryDirectory), @@ -152,16 +151,13 @@ enum NameFailure { } /// Required root entries in report order; version one requires the prefix. -const REQUIRED: [RecoveryRequiredEntry; 12] = [ +const REQUIRED: [RecoveryRequiredEntry; 9] = [ RecoveryRequiredEntry::WriterLock, RecoveryRequiredEntry::StagingDirectory, RecoveryRequiredEntry::SegmentPoolDirectory, RecoveryRequiredEntry::CatalogPoolDirectory, - RecoveryRequiredEntry::CurrentHead, RecoveryRequiredEntry::ReaderFence, RecoveryRequiredEntry::FormatMarker, - RecoveryRequiredEntry::MigrationIntent, - RecoveryRequiredEntry::MigrationReceipt, RecoveryRequiredEntry::RetentionDirectory, RecoveryRequiredEntry::GcDirectory, RecoveryRequiredEntry::RecoveryDirectory, @@ -208,15 +204,13 @@ const fn required_entry(role: RecoveryEntryRole) -> Option { Some(RecoveryRequiredEntry::CatalogPoolDirectory) } - RecoveryEntryRole::CurrentHead => Some(RecoveryRequiredEntry::CurrentHead), RecoveryEntryRole::ReaderFence => Some(RecoveryRequiredEntry::ReaderFence), RecoveryEntryRole::FormatMarker => Some(RecoveryRequiredEntry::FormatMarker), - RecoveryEntryRole::MigrationIntent => Some(RecoveryRequiredEntry::MigrationIntent), - RecoveryEntryRole::MigrationReceipt => Some(RecoveryRequiredEntry::MigrationReceipt), RecoveryEntryRole::RetentionDirectory => Some(RecoveryRequiredEntry::RetentionDirectory), RecoveryEntryRole::GcDirectory => Some(RecoveryRequiredEntry::GcDirectory), RecoveryEntryRole::RecoveryDirectory => Some(RecoveryRequiredEntry::RecoveryDirectory), - RecoveryEntryRole::NextHeadStage + RecoveryEntryRole::CurrentHead + | RecoveryEntryRole::NextHeadStage | RecoveryEntryRole::SegmentStage | RecoveryEntryRole::CatalogStage | RecoveryEntryRole::ImmutableSegment { .. } diff --git a/src/adapters/recovery/recovery_required_entry.rs b/src/adapters/recovery/recovery_required_entry.rs index dcb45ee0..550fd7ec 100644 --- a/src/adapters/recovery/recovery_required_entry.rs +++ b/src/adapters/recovery/recovery_required_entry.rs @@ -5,7 +5,8 @@ use std::fmt; /// One fixed entry required in an initialized store root. /// /// The first four are required in every root. The remaining entries are -/// required only when a migrated version-two root is classified. +/// required only when a version-two root is classified. Catalog `HEAD` is +/// required by neither: a store holds none until its first publication. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RecoveryRequiredEntry { /// Persistent `writer.lock`. @@ -16,16 +17,10 @@ pub enum RecoveryRequiredEntry { SegmentPoolDirectory, /// `catalogs` directory. CatalogPoolDirectory, - /// Published catalog `HEAD`, required in a version-two root. - CurrentHead, /// Persistent `reader.lock`. ReaderFence, /// `FORMAT` marker. FormatMarker, - /// `migration.intent` record. - MigrationIntent, - /// `migration.receipt` record. - MigrationReceipt, /// `retention` directory. RetentionDirectory, /// `gc` directory. @@ -41,11 +36,8 @@ impl fmt::Display for RecoveryRequiredEntry { Self::StagingDirectory => "staging", Self::SegmentPoolDirectory => "segments", Self::CatalogPoolDirectory => "catalogs", - Self::CurrentHead => "HEAD", Self::ReaderFence => "reader.lock", Self::FormatMarker => "FORMAT", - Self::MigrationIntent => "migration.intent", - Self::MigrationReceipt => "migration.receipt", Self::RetentionDirectory => "retention", Self::GcDirectory => "gc", Self::RecoveryDirectory => "recovery", diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 344f3074..2f930cb1 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -119,8 +119,6 @@ mod filesystem_retention_snapshot_error; #[cfg(test)] mod filesystem_retention_snapshot_error_tests; #[cfg(test)] -mod filesystem_retention_snapshot_identity_tests; -#[cfg(test)] mod filesystem_retention_snapshot_tests; mod filesystem_retention_stage; mod filesystem_retention_storage; @@ -306,6 +304,3 @@ pub use verified_closure::VerifiedRetentionClosure; pub(super) use closure_profile_error::map as profile_verification_refusal; pub use verification_view_collector::collect_verification_view; - -#[cfg(all(test, feature = "repository-tasks"))] -mod filesystem_retention_migration_completion_tests; diff --git a/src/adapters/retention/filesystem_recovery_admission_tests.rs b/src/adapters/retention/filesystem_recovery_admission_tests.rs index d3666081..8de6fcec 100644 --- a/src/adapters/retention/filesystem_recovery_admission_tests.rs +++ b/src/adapters/retention/filesystem_recovery_admission_tests.rs @@ -1,8 +1,8 @@ -//! Version-one recovery adapters refuse a migrated version-two root. +//! Version-one recovery adapters refuse a version-two root. use std::error::Error; -use super::filesystem_retention_test_fixture::migrated_store; +use super::filesystem_retention_test_fixture::initialized_store; use crate::adapters::{ FilesystemRecoveryInventoryReader, FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, RecoveryInventoryError, RecoveryInventoryOperation, @@ -10,12 +10,12 @@ use crate::adapters::{ }; #[test] -fn recovery_inventory_reader_refuses_a_migrated_root() -> Result<(), Box> { - let sandbox = migrated_store("recovery-admission-inventory-migrated")?; +fn recovery_inventory_reader_refuses_a_version_two_root() -> Result<(), Box> { + let sandbox = initialized_store("recovery-admission-inventory-version-two")?; let error = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path()) .err() - .ok_or("version-one recovery inventory opened a migrated root")?; + .ok_or("version-one recovery inventory opened a version-two root")?; assert!(matches!( error, @@ -29,12 +29,12 @@ fn recovery_inventory_reader_refuses_a_migrated_root() -> Result<(), Box Result<(), Box> { - let sandbox = migrated_store("recovery-admission-discarder-migrated")?; +fn recovery_stage_discarder_refuses_a_version_two_root() -> Result<(), Box> { + let sandbox = initialized_store("recovery-admission-discarder-version-two")?; let error = FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(sandbox.path()) .err() - .ok_or("version-one recovery discarder opened a migrated root")?; + .ok_or("version-one recovery discarder opened a version-two root")?; assert!(matches!( error, diff --git a/src/adapters/retention/filesystem_retention_fifo_tests.rs b/src/adapters/retention/filesystem_retention_fifo_tests.rs index 2aa951ab..1f219b2d 100644 --- a/src/adapters/retention/filesystem_retention_fifo_tests.rs +++ b/src/adapters/retention/filesystem_retention_fifo_tests.rs @@ -9,7 +9,7 @@ use std::thread; use std::time::Duration; use super::filesystem_retention_test_fixture::{ - ROOT_HEX, fixture, initial_preparation, manifest_pool_path, migrated_store, open_authority, + ROOT_HEX, fixture, initial_preparation, initialized_store, manifest_pool_path, open_authority, }; use crate::adapters::FilesystemVersionTwoAdmission; use crate::execute_retention_publication; @@ -29,7 +29,7 @@ fn a_fifo_at_the_retention_head_refuses_instead_of_blocking() -> Result<(), Box< #[test] fn a_fifo_at_the_format_marker_refuses_instead_of_blocking() -> Result<(), Box> { - let sandbox = migrated_store("filesystem-retention-fifo-marker")?; + let sandbox = initialized_store("filesystem-retention-fifo-marker")?; std::fs::remove_file(sandbox.path().join("FORMAT"))?; make_fifo(&sandbox.path().join("FORMAT"))?; let path = sandbox.path().to_path_buf(); diff --git a/src/adapters/retention/filesystem_retention_migration_completion_tests.rs b/src/adapters/retention/filesystem_retention_migration_completion_tests.rs deleted file mode 100644 index 1b5ed751..00000000 --- a/src/adapters/retention/filesystem_retention_migration_completion_tests.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! Completed migration admits live retention state without taking its ownership. - -use std::error::Error; - -use super::filesystem_retention_test_fixture::{ - ROOT_HEX, fixture, initial_preparation, open_authority, -}; -use crate::{ - CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemRetentionSnapshot, - FilesystemStoreMigrationAuthority, ReaderAttemptLimit, SegmentReadPolicy, - StoreMigrationRecoveryPlan, execute_retention_publication, recover_store_migration, -}; - -// Size: medium. Oracle: migration completion preserves a successfully published retained root. -// Delete only if migration completion cannot be retried or stronger admission laws subsume it. -#[test] -fn completed_migration_preserves_published_retention_state() -> Result<(), Box> { - let (store, mut publisher) = open_authority("completed-migration-retention")?; - let root_bytes = fixture(ROOT_HEX)?; - let preparation = initial_preparation(&root_bytes)?; - let _published = execute_retention_publication(&mut publisher, &preparation)?; - drop(publisher); - let mut migration = - FilesystemStoreMigrationAuthority::reopen_for_recovery_unchecked_for_repository_tasks( - store.path(), - SegmentReadPolicy::MAXIMUM, - )?; - let intent = migration.observe_intent()?; - let receipt = recover_store_migration(&mut migration, &intent)?; - assert_eq!(receipt.plan(), StoreMigrationRecoveryPlan::Complete); - assert_eq!(receipt.executed_phases().next(), None); - drop(migration); - let policy = CatalogRestartPolicy::new( - SegmentReadPolicy::MAXIMUM, - CatalogRestartByteLimit::new(1_048_576)?, - ); - let view = - FilesystemRetentionSnapshot::load(store.path(), policy, ReaderAttemptLimit::DEFAULT)?; - let namespace = super::AdmittedRetentionRoot::decode(&root_bytes)? - .root() - .namespace() - .digest(); - let retained = view - .retained_root(namespace)? - .ok_or("published root disappeared")?; - assert_eq!(&*retained, root_bytes.as_slice()); - Ok(()) -} diff --git a/src/adapters/retention/filesystem_retention_snapshot.rs b/src/adapters/retention/filesystem_retention_snapshot.rs index 13e9e797..6332d99b 100644 --- a/src/adapters/retention/filesystem_retention_snapshot.rs +++ b/src/adapters/retention/filesystem_retention_snapshot.rs @@ -16,12 +16,10 @@ use super::{ use crate::adapters::filesystem_exact_record::{ self as exact_record, ExactRecordError, ExactRecordRefusal, }; -use crate::adapters::filesystem_platform_profile::root_identity; -use crate::adapters::filesystem_version_two_admission::require_root_identity; use crate::adapters::{ CatalogRestartError, CatalogRestartPolicy, ChecksummedPublicationHead, - FilesystemCatalogSnapshot, filesystem_initialization_namespace, filesystem_platform_profile, - filesystem_version_two_records, publication_head_decoder, + FilesystemCatalogSnapshot, filesystem_format_marker, filesystem_initialization_namespace, + filesystem_platform_profile, publication_head_decoder, }; use crate::{RetentionHead, RetentionManifest, RetentionNamespaceDigest}; @@ -109,8 +107,8 @@ impl RetentionViewSource for Source { impl FilesystemRetentionSnapshot { /// Admits the production filesystem profile and version-two root, acquires the reader fence, and /// double-collects one consistent view within `limit` attempts. - /// Admission requires the opened directory's restart-stable device and - /// inode to match the jointly admitted migration records before fencing. + /// Admission requires the committed namespace and the canonical `FORMAT` + /// marker before fencing; nothing binds the store to the directory it is in. /// /// The call takes no writer authority and performs no namespace writes. /// Platform admission synchronizes the opened root directory; failure is @@ -118,8 +116,10 @@ impl FilesystemRetentionSnapshot { /// while collection holds the fence exclusively. /// Platform admission requires the existing local writable, case-sensitive /// Linux ext4 profile across every present version-two protocol directory. - /// The same opened root capability is retained through namespace, migration - /// identity, fence, and coordinate admission; the ambient path is not reopened. + /// The same opened root capability is retained through namespace, marker, + /// fence, and coordinate admission; the ambient path is not reopened. A + /// committed store with no catalog `HEAD` refuses as `Unpublished` once the + /// fence is held. /// A retained or in-flight catalog `head.next` is admitted and ignored: the /// view selects only `HEAD`, so catalog publication runs beside readers. /// @@ -151,13 +151,9 @@ impl FilesystemRetentionSnapshot { .map_err(|source| Error::Admission { source })?; filesystem_initialization_namespace::admit_version_two_with_catalog_stage(&root) .map_err(|source| Error::Admission { source })?; - let bound = filesystem_version_two_records::admit(&root) - .map_err(|source| Error::Admission { source })?; - let observed = root_identity(&root).map_err(|source| Error::Admission { source })?; - require_root_identity(bound, observed).map_err(|source| Error::Admission { - source: io::Error::new(io::ErrorKind::InvalidData, source), - })?; + filesystem_format_marker::admit(&root).map_err(|source| Error::Admission { source })?; let fence = ReaderFence::acquire(&root).map_err(|source| Error::Fence { source })?; + require_published(&root)?; let retention = root .open_dir_nofollow(pool_name::RETENTION) .map_err(|source| Error::Admission { source })?; @@ -321,3 +317,15 @@ fn selected_namespace_directory( crate::adapters::filesystem_namespace_refusal::require_directory(roots, &name)?; roots.open_dir_nofollow(name) } + +/// Refuses a committed store that has not published its first catalog. +/// +/// Checked under the fence, after admission: absence of `HEAD` is a defined +/// state of an initialized store, reported as such instead of as a failed read. +fn require_published(root: &Dir) -> Result<(), Error> { + match root.symlink_metadata(HEAD_NAME) { + Ok(_) => Ok(()), + Err(source) if source.kind() == io::ErrorKind::NotFound => Err(Error::Unpublished), + Err(source) => Err(Error::Admission { source }), + } +} diff --git a/src/adapters/retention/filesystem_retention_snapshot_coordinate_tests.rs b/src/adapters/retention/filesystem_retention_snapshot_coordinate_tests.rs index 35ff35cd..45127e79 100644 --- a/src/adapters/retention/filesystem_retention_snapshot_coordinate_tests.rs +++ b/src/adapters/retention/filesystem_retention_snapshot_coordinate_tests.rs @@ -6,7 +6,7 @@ use std::io; use std::num::NonZeroU32; use super::{Source, collect_retention_view, pool_name}; -use crate::adapters::retention::filesystem_retention_test_fixture::migrated_store; +use crate::adapters::retention::filesystem_retention_test_fixture::initialized_store; use crate::adapters::retention::{ ReaderAttemptLimit, ReaderFence, RetentionViewCoordinates, RetentionViewError, RetentionViewSource, @@ -41,7 +41,7 @@ impl io::Result<()>> RetentionViewSource for RewriteHead { // Delete when double collection is removed or a stronger filesystem law subsumes this schedule. #[test] fn a_changed_catalog_length_refuses_the_loaded_reader_view() -> Result<(), Box> { - let sandbox = migrated_store("reader-changed-catalog-length")?; + let sandbox = initialized_store("reader-changed-catalog-length")?; let root = Dir::open_ambient_dir(sandbox.path(), cap_std::ambient_authority())?; let _fence = ReaderFence::acquire(&root)?; let retention = root.open_dir_nofollow(pool_name::RETENTION)?; diff --git a/src/adapters/retention/filesystem_retention_snapshot_error.rs b/src/adapters/retention/filesystem_retention_snapshot_error.rs index 507af461..d04eae78 100644 --- a/src/adapters/retention/filesystem_retention_snapshot_error.rs +++ b/src/adapters/retention/filesystem_retention_snapshot_error.rs @@ -16,6 +16,11 @@ pub enum FilesystemRetentionSnapshotError { /// The exact platform I/O, namespace, or record failure. source: io::Error, }, + /// The store is initialized but has published no catalog `HEAD`. + /// + /// This is a defined state of a committed store, observed under the + /// reader fence, not a failure to read one. + Unpublished, /// The reader fence could not be acquired. Fence { /// The exact filesystem failure. @@ -42,6 +47,7 @@ impl fmt::Display for FilesystemRetentionSnapshotError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { Self::Admission { .. } => "version-two reader admission refused", + Self::Unpublished => "the store has published no catalog", Self::Fence { .. } => "reader fence acquisition failed", Self::View { .. } => "reader view collection refused", Self::Catalog { .. } => "catalog snapshot refused", @@ -58,6 +64,7 @@ impl Error for FilesystemRetentionSnapshotError { } Self::View { source } => Some(source), Self::Catalog { source } => Some(source), + Self::Unpublished => None, } } } diff --git a/src/adapters/retention/filesystem_retention_snapshot_error_tests.rs b/src/adapters/retention/filesystem_retention_snapshot_error_tests.rs index cac386d0..9a8a8fbd 100644 --- a/src/adapters/retention/filesystem_retention_snapshot_error_tests.rs +++ b/src/adapters/retention/filesystem_retention_snapshot_error_tests.rs @@ -5,7 +5,7 @@ use std::fs; use std::io; use std::path::Path; -use super::filesystem_retention_test_fixture::{CATALOG_NAME, SEGMENT_NAME, migrated_store}; +use super::filesystem_retention_test_fixture::{CATALOG_NAME, SEGMENT_NAME, initialized_store}; use super::{ FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError as SnapshotError, ReaderAttemptLimit, RetentionViewError, @@ -20,7 +20,7 @@ use crate::adapters::{ // Delete when the reader API is removed or stronger public laws subsume these cases. #[test] fn a_missing_selected_catalog_reports_its_exact_catalog_refusal() -> Result<(), Box> { - let sandbox = migrated_store("reader-error-missing-catalog")?; + let sandbox = initialized_store("reader-error-missing-catalog")?; fs::remove_file(sandbox.path().join("catalogs").join(CATALOG_NAME))?; let error = refusal(sandbox.path())?; @@ -45,7 +45,7 @@ fn a_missing_selected_catalog_reports_its_exact_catalog_refusal() -> Result<(), // Size: medium. Oracle and deletion criterion: the public catalog-boundary law above. #[test] fn a_missing_selected_segment_reports_its_exact_catalog_refusal() -> Result<(), Box> { - let sandbox = migrated_store("reader-error-missing-segment")?; + let sandbox = initialized_store("reader-error-missing-segment")?; fs::remove_file(sandbox.path().join("segments").join(SEGMENT_NAME))?; let error = refusal(sandbox.path())?; @@ -62,7 +62,7 @@ fn a_missing_selected_segment_reports_its_exact_catalog_refusal() -> Result<(), // Size: medium. Oracle and deletion criterion: the public catalog-boundary law above. #[test] fn a_corrupt_selected_catalog_reports_its_decoder_refusal() -> Result<(), Box> { - let sandbox = migrated_store("reader-error-corrupt-catalog")?; + let sandbox = initialized_store("reader-error-corrupt-catalog")?; let path = sandbox.path().join("catalogs").join(CATALOG_NAME); let mut bytes = fs::read(&path)?; let checksum = bytes @@ -92,7 +92,7 @@ fn a_corrupt_selected_catalog_reports_its_decoder_refusal() -> Result<(), Box Result<(), Box> { - let sandbox = migrated_store("reader-error-corrupt-head")?; + let sandbox = initialized_store("reader-error-corrupt-head")?; let path = sandbox.path().join("HEAD"); let mut bytes = fs::read(&path)?; *bytes.last_mut().ok_or("HEAD absent")? ^= 1; diff --git a/src/adapters/retention/filesystem_retention_snapshot_identity_tests.rs b/src/adapters/retention/filesystem_retention_snapshot_identity_tests.rs deleted file mode 100644 index 0570214e..00000000 --- a/src/adapters/retention/filesystem_retention_snapshot_identity_tests.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! These laws own reader admission against migration-bound physical root identity. - -use std::error::Error; -use std::fs; -use std::os::unix::fs::MetadataExt; - -use super::filesystem_retention_test_fixture::migrated_store; -use super::{FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, ReaderAttemptLimit}; -use crate::adapters::{ - CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemPlatformAdmissionError, - SegmentReadPolicy, StoreRootIdentityCoordinate, -}; - -// Size: medium. Oracle: a reader must admit the root named by migration.intent. -// Delete only with migration binding or a stronger public reader-boundary law. -#[test] -fn a_reader_refuses_migration_records_bound_to_another_root() -> Result<(), Box> { - let donor = migrated_store("reader-migration-identity-donor")?; - let recipient = migrated_store("reader-migration-identity-recipient")?; - for name in ["FORMAT", "migration.intent", "migration.receipt"] { - fs::copy(donor.path().join(name), recipient.path().join(name))?; - } - let expected = fs::metadata(donor.path())?.ino(); - let observed = fs::metadata(recipient.path())?.ino(); - let policy = CatalogRestartPolicy::new( - SegmentReadPolicy::MAXIMUM, - CatalogRestartByteLimit::new(1_048_576)?, - ); - - let result = - FilesystemRetentionSnapshot::load(recipient.path(), policy, ReaderAttemptLimit::DEFAULT); - - let error = match result { - Err(FilesystemRetentionSnapshotError::Admission { source }) => source, - Ok(_) => { - return Err("reader accepted migration records naming another physical root".into()); - } - Err(other) => { - return Err(format!("reader root identity must refuse at admission: {other:?}").into()); - } - }; - let refusal = error - .get_ref() - .and_then(|source| source.downcast_ref::()); - assert!( - matches!(refusal, Some(FilesystemPlatformAdmissionError::RootIdentityChanged { - coordinate: StoreRootIdentityCoordinate::File, - expected: actual_expected, observed: actual_observed, - }) if *actual_expected == expected && *actual_observed == observed), - "reader must preserve exact bound inode {expected} and observed inode {observed}: {error:?}" - ); - Ok(()) -} diff --git a/src/adapters/retention/filesystem_retention_snapshot_pinning_tests.rs b/src/adapters/retention/filesystem_retention_snapshot_pinning_tests.rs index 7f5a8a58..61b51832 100644 --- a/src/adapters/retention/filesystem_retention_snapshot_pinning_tests.rs +++ b/src/adapters/retention/filesystem_retention_snapshot_pinning_tests.rs @@ -5,7 +5,7 @@ use std::fs; use super::{Source, collect_retention_view, pool_name}; use crate::adapters::filesystem_test_sandbox::TestDirectory; -use crate::adapters::retention::filesystem_retention_test_fixture::migrated_store; +use crate::adapters::retention::filesystem_retention_test_fixture::initialized_store; use crate::adapters::retention::{ReaderAttemptLimit, RetentionViewSource}; use crate::adapters::{CatalogRestartByteLimit, CatalogRestartPolicy, SegmentReadPolicy}; use cap_fs_ext::DirExt; @@ -16,7 +16,7 @@ use cap_std::fs::Dir; // Delete only when pinned reader collection is removed or a stronger law subsumes it. #[test] fn replacing_the_ambient_root_preserves_the_pinned_catalog() -> Result<(), Box> { - let sandbox = migrated_store("reader-catalog-path-pinning")?; + let sandbox = initialized_store("reader-catalog-path-pinning")?; let moved = TestDirectory::create("reader-catalog-path-pinning-moved")?; let root = Dir::open_ambient_dir(sandbox.path(), cap_std::ambient_authority())?; let retention = root.open_dir_nofollow(pool_name::RETENTION)?; diff --git a/src/adapters/retention/filesystem_retention_snapshot_tests.rs b/src/adapters/retention/filesystem_retention_snapshot_tests.rs index abbff4ac..71ae97f0 100644 --- a/src/adapters/retention/filesystem_retention_snapshot_tests.rs +++ b/src/adapters/retention/filesystem_retention_snapshot_tests.rs @@ -1,4 +1,4 @@ -//! Reader fence and fenced snapshot laws over migrated stores. +//! Reader fence and fenced snapshot laws over initialized stores. use std::error::Error; use std::fs; @@ -8,7 +8,7 @@ use rustix::fs::{FlockOperation, flock}; use rustix::io::Errno; use super::filesystem_retention_test_fixture::{ - ROOT_HEX, fixture, initial_preparation, migrated_store, open_authority, + ROOT_HEX, fixture, initial_preparation, initialized_store, open_authority, }; use super::{ AdmittedRetentionRoot, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, @@ -27,9 +27,9 @@ fn policy() -> Result> { } #[test] -fn a_migrated_store_snapshot_binds_the_catalog_and_no_retention_head() -> Result<(), Box> -{ - let sandbox = migrated_store("filesystem-retention-snapshot-empty")?; +fn an_initialized_store_snapshot_binds_the_catalog_and_no_retention_head() +-> Result<(), Box> { + let sandbox = initialized_store("filesystem-retention-snapshot-empty")?; let snapshot = FilesystemRetentionSnapshot::load(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; assert_eq!(snapshot.catalog().generation().get(), 1); @@ -112,7 +112,7 @@ fn root_checksum_damage_reports_its_exact_mismatch() -> Result<(), Box Result<(), Box> { - let sandbox = migrated_store("filesystem-retention-snapshot-fence")?; + let sandbox = initialized_store("filesystem-retention-snapshot-fence")?; let root = Dir::open_ambient_dir(sandbox.path(), cap_std::ambient_authority())?; let first = ReaderFence::acquire(&root)?; let second = ReaderFence::acquire(&root)?; @@ -135,7 +135,7 @@ fn readers_share_the_fence_and_collection_cannot_take_it_exclusively() -> Result // Size: medium. Oracle: the reader.lock protocol requires an empty regular file. // Delete when that protocol is removed or a stronger public-reader law subsumes this case. fn a_non_empty_reader_lock_refuses_the_fence() -> Result<(), Box> { - let sandbox = migrated_store("filesystem-retention-snapshot-bad-fence")?; + let sandbox = initialized_store("filesystem-retention-snapshot-bad-fence")?; fs::write(sandbox.path().join("reader.lock"), b"not empty")?; let error = FilesystemRetentionSnapshot::load(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT) diff --git a/src/adapters/retention/filesystem_retention_storage_tests.rs b/src/adapters/retention/filesystem_retention_storage_tests.rs index b1718c01..d22d3e37 100644 --- a/src/adapters/retention/filesystem_retention_storage_tests.rs +++ b/src/adapters/retention/filesystem_retention_storage_tests.rs @@ -13,17 +13,17 @@ use super::{RetentionPublicationError, RetentionPublicationOutcome, RetentionPub use crate::execute_retention_publication; #[test] -fn complete_publication_preserves_migrated_bytes_and_publishes_exact_retention_prefix() +fn complete_publication_preserves_committed_bytes_and_publishes_exact_retention_prefix() -> Result<(), Box> { let (sandbox, mut authority) = open_authority("filesystem-retention-complete")?; - let before = migrated_witness(sandbox.path())?; + let before = committed_witness(sandbox.path())?; let root_bytes = fixture(ROOT_HEX)?; let preparation = initial_preparation(&root_bytes)?; let receipt = execute_retention_publication(&mut authority, &preparation)?; assert_eq!(receipt.outcome(), RetentionPublicationOutcome::Published); - assert_eq!(migrated_witness(sandbox.path())?, before); + assert_eq!(committed_witness(sandbox.path())?, before); assert_eq!(fs::read(head_path(sandbox.path()))?, fixture(HEAD_HEX)?); assert_eq!( fs::read(root_pool_path(sandbox.path(), preparation.candidate()))?, @@ -142,9 +142,9 @@ fn assert_stages_absent(root: &Path) -> Result<(), Box> { Ok(()) } -fn migrated_witness(root: &Path) -> io::Result)>> { +fn committed_witness(root: &Path) -> io::Result)>> { let mut witness = Vec::new(); - for name in ["HEAD", "FORMAT", "migration.intent", "migration.receipt"] { + for name in ["HEAD", "FORMAT"] { witness.push((PathBuf::from(name), fs::read(root.join(name))?)); } for pool in ["segments", "catalogs"] { diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs index 6fe05d01..bbf9c173 100644 --- a/src/adapters/retention/filesystem_retention_test_fixture.rs +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -1,4 +1,4 @@ -//! This test module owns one migrated version-2 retention publication fixture. +//! This test module owns one initialized version-2 retention publication fixture. use std::collections::BTreeSet; use std::error::Error; @@ -20,8 +20,8 @@ use crate::adapters::populated_store::{self, FirstCatalogRecords, StoreAdmission use crate::adapters::test_support::decode_hex; use crate::adapters::{ AdmittedCatalog, AdmittedSegment, CatalogSnapshot, ChecksummedCatalog, - ChecksummedPublicationHead, FilesystemPlatformAdmission, FilesystemVersionTwoAdmission, - SegmentReadPolicy, SegmentRecordLimit, + ChecksummedPublicationHead, FilesystemVersionTwoAdmission, SegmentReadPolicy, + SegmentRecordLimit, }; use crate::{ RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionRoot, @@ -50,15 +50,15 @@ pub(super) const SEGMENT_NAME: &str = pub(super) const CATALOG_NAME: &str = "0000000000000001-0b7cad1b6de663d34beacbc214db7497f2e36ab6b08dfbd5febbc8d06a418811.cat"; -/// Builds one migrated version-2 store and pins its retention authority. +/// Builds one initialized version-2 store and pins its retention authority. /// -/// The fixture publishes the exact bundle version-1 corpus, executes the -/// complete forward migration, releases writer authority, then reopens the -/// admitted root for retention publication. +/// The fixture initializes the store, publishes the exact bundle corpus as +/// its first catalog through the version-two writer, releases writer +/// authority, then reopens the admitted root for retention publication. pub(super) fn open_authority( name: &str, ) -> Result<(TestDirectory, FilesystemRetentionPublicationAuthority), Box> { - let sandbox = migrated_store(name)?; + let sandbox = initialized_store(name)?; let admission = FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; let authority = FilesystemRetentionPublicationAuthority::open(admission)?; @@ -246,14 +246,10 @@ fn hex(bytes: &[u8; 32]) -> String { /// The shared builder's route on hosts outside the admitted platform profile. const UNCHECKED: StoreAdmission = StoreAdmission { initialize: |root| { - Ok(FilesystemPlatformAdmission::initialize_unchecked_for_tests( - root, - )?) - }, - reopen_version_one: |root| { - Ok(FilesystemPlatformAdmission::reopen_unchecked_for_tests( - root, - )?) + Ok( + FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(root)? + .into_admission(), + ) }, reopen_version_two: |root| { Ok(FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(root)?) @@ -261,7 +257,7 @@ const UNCHECKED: StoreAdmission = StoreAdmission { }; /// Builds one version-2 store holding the frozen bundle, with writer authority released. -pub(super) fn migrated_store(name: &str) -> Result> { +pub(super) fn initialized_store(name: &str) -> Result> { let sandbox = TestDirectory::create(name)?; populate_bundle(sandbox.path())?; Ok(sandbox) diff --git a/src/adapters/retention/filesystem_retention_verification.rs b/src/adapters/retention/filesystem_retention_verification.rs index 6447874c..e98f2441 100644 --- a/src/adapters/retention/filesystem_retention_verification.rs +++ b/src/adapters/retention/filesystem_retention_verification.rs @@ -9,10 +9,10 @@ use super::{AdmittedRetentionRoot, RetentionSelectedRootRefusal}; use crate::adapters::filesystem_exact_record::ExactRecordError; use crate::adapters::{verification_admission, verification_ingress}; use crate::{ - CatalogRestartPolicy, FilesystemPlatformAdmissionError, FilesystemRetentionSnapshot, - FilesystemRetentionSnapshotError, ReaderAttemptLimit, RetentionNamespaceDigest, - RetentionRootDecodeError, VerificationDepth, VerificationError, VerificationRefusal, - VerificationReport, VerificationSource, VerificationSubject, VersionTwoRecordRefusal, + CatalogRestartPolicy, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, + ReaderAttemptLimit, RetentionNamespaceDigest, RetentionRootDecodeError, VerificationDepth, + VerificationError, VerificationRefusal, VerificationReport, VerificationSource, + VerificationSubject, VersionTwoRecordRefusal, }; use std::io; @@ -115,6 +115,14 @@ fn view_error(source: FilesystemRetentionSnapshotError) -> VerificationError { if let FilesystemRetentionSnapshotError::Catalog { source } = source { return verification_ingress::catalog_error(source); } + if matches!(source, FilesystemRetentionSnapshotError::Unpublished) { + return VerificationError::Refused { + refusal: VerificationRefusal::Missing { + subject: VerificationSubject::PublishedView, + }, + source: Some(Box::new(VerificationSource::Retention(source))), + }; + } if matches!(&source, FilesystemRetentionSnapshotError::Admission { source } if admission_is_corrupt(source)) { return VerificationError::Refused { @@ -142,21 +150,10 @@ fn admission_is_corrupt(error: &io::Error) -> bool { VersionTwoRecordRefusal::LengthOverflow { .. } => false, VersionTwoRecordRefusal::KindOrLength { .. } | VersionTwoRecordRefusal::TrailingBytes { .. } - | VersionTwoRecordRefusal::Marker { .. } - | VersionTwoRecordRefusal::Intent { .. } - | VersionTwoRecordRefusal::Receipt { .. } => true, + | VersionTwoRecordRefusal::Marker { .. } => true, }; } - match source.downcast_ref::() { - Some(FilesystemPlatformAdmissionError::RootIdentityChanged { .. }) => true, - Some( - FilesystemPlatformAdmissionError::Platform { .. } - | FilesystemPlatformAdmissionError::WriterLock { .. } - | FilesystemPlatformAdmissionError::Namespace { .. } - | FilesystemPlatformAdmissionError::MigrationRecord { .. }, - ) - | None => false, - } + false } fn root_error( diff --git a/src/adapters/retention/filesystem_verification_law_tests.rs b/src/adapters/retention/filesystem_verification_law_tests.rs index 405c9f47..5be720e3 100644 --- a/src/adapters/retention/filesystem_verification_law_tests.rs +++ b/src/adapters/retention/filesystem_verification_law_tests.rs @@ -1,4 +1,4 @@ -//! Public verification outcomes over owned migrated filesystem witnesses. +//! Public verification outcomes over owned initialized filesystem witnesses. use super::filesystem_retention_test_fixture::{ ROOT_HEX, fixture, initial_preparation, open_authority, retention_witness, root_pool_path, diff --git a/src/adapters/retention/filesystem_version_two_admission_tests.rs b/src/adapters/retention/filesystem_version_two_admission_tests.rs index 98fe3a28..cfaec3ea 100644 --- a/src/adapters/retention/filesystem_version_two_admission_tests.rs +++ b/src/adapters/retention/filesystem_version_two_admission_tests.rs @@ -1,26 +1,25 @@ -//! Version-two reopen laws: writer authority requires jointly admitted migration records. +//! Version-two reopen laws: writer authority requires a committed namespace and the canonical marker. use std::error::Error; use std::fs; use super::filesystem_retention_test_fixture::{ - ROOT_HEX, fixture, initial_preparation, migrated_store, open_authority, refusal, + ROOT_HEX, fixture, initial_preparation, initialized_store, open_authority, refusal, }; use super::{ FilesystemRetentionPublicationAuthority, RetentionCurrentStateRefusal, RetentionPublicationStorage, }; -use crate::adapters::filesystem_root_identity::FilesystemRootIdentity; -use crate::adapters::filesystem_version_two_admission::{BoundRootIdentity, require_root_identity}; +use crate::adapters::filesystem_test_sandbox::TestDirectory; use crate::adapters::{ - FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, StoreRootIdentityCoordinate, + FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, VersionTwoRecordRefusal, }; use crate::execute_retention_publication; #[test] fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box> { - let sandbox = migrated_store("version-two-admission-corrupt-marker")?; + let sandbox = initialized_store("version-two-admission-corrupt-marker")?; let marker = sandbox.path().join("FORMAT"); let mut bytes = fs::read(&marker)?; *bytes @@ -33,7 +32,7 @@ fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box Result<(), Box Result<(), Box> { - let sandbox = migrated_store("version-two-admission-oversized-marker")?; + let sandbox = initialized_store("version-two-admission-oversized-marker")?; let marker = sandbox.path().join("FORMAT"); let mut bytes = fs::read(&marker)?; bytes.push(0); @@ -60,33 +59,14 @@ fn version_two_reopen_refuses_an_oversized_format_marker() -> Result<(), Box Result<(), Box> { - let sandbox = migrated_store("version-two-admission-receipt-disagrees")?; - let intent = fs::read(sandbox.path().join("migration.intent"))?; - fs::write(sandbox.path().join("migration.receipt"), &intent)?; - - let error = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path()) - .err() - .ok_or("receipt disagreeing with its intent was unexpectedly admitted")?; - - assert!(matches!( - error, - FilesystemPlatformAdmissionError::MigrationRecord { .. } - )); - Ok(()) -} - -#[test] -fn version_two_reopen_admits_exact_migration_records() -> Result<(), Box> { - let sandbox = migrated_store("version-two-admission-exact")?; +fn version_two_reopen_admits_a_canonical_marker() -> Result<(), Box> { + let sandbox = initialized_store("version-two-admission-exact")?; let admission = FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; @@ -99,7 +79,7 @@ fn version_two_reopen_admits_exact_migration_records() -> Result<(), Box Result<(), Box> { - let sandbox = migrated_store("version-two-admission-aliased-gc")?; + let sandbox = initialized_store("version-two-admission-aliased-gc")?; let alias_target = sandbox.path().join("elsewhere"); fs::create_dir(&alias_target)?; fs::remove_dir(sandbox.path().join("gc"))?; @@ -118,8 +98,8 @@ fn production_version_two_reopen_refuses_an_aliased_protocol_directory() #[cfg(target_os = "linux")] #[test] -fn production_version_two_reopen_admits_an_exact_migrated_store() -> Result<(), Box> { - let sandbox = migrated_store("version-two-admission-production-exact")?; +fn production_version_two_reopen_admits_an_exact_initialized_store() -> Result<(), Box> { + let sandbox = initialized_store("version-two-admission-production-exact")?; let admission = FilesystemVersionTwoAdmission::reopen(sandbox.path())?; @@ -127,41 +107,11 @@ fn production_version_two_reopen_admits_an_exact_migrated_store() -> Result<(), Ok(()) } -/// A remount changes `statx.stx_mnt_id` but neither the device nor the root -/// inode; reopen must admit that store. A different inode on the same device -/// or a different device is a relocated or restored store and refuses. -#[test] -fn reopen_admits_a_remounted_root_and_refuses_a_moved_one() { - let bound = BoundRootIdentity::new(1, 3); - - assert!(require_root_identity(bound, FilesystemRootIdentity::new(1, 2, 3)).is_ok()); - assert!( - require_root_identity(bound, FilesystemRootIdentity::new(1, 7, 3)).is_ok(), - "a remounted root (same device and inode, new mount id) must reopen" - ); - assert!(matches!( - require_root_identity(bound, FilesystemRootIdentity::new(1, 2, 4)), - Err(FilesystemPlatformAdmissionError::RootIdentityChanged { - coordinate: StoreRootIdentityCoordinate::File, - expected: 3, - observed: 4, - }) - )); - assert!(matches!( - require_root_identity(bound, FilesystemRootIdentity::new(9, 2, 3)), - Err(FilesystemPlatformAdmissionError::RootIdentityChanged { - coordinate: StoreRootIdentityCoordinate::Device, - expected: 1, - observed: 9, - }) - )); -} - fn refuses_namespace( name: &str, mutate: impl FnOnce(&std::path::Path) -> Result<(), Box>, ) -> Result<(), Box> { - let sandbox = migrated_store(name)?; + let sandbox = initialized_store(name)?; mutate(sandbox.path())?; let error = @@ -244,3 +194,57 @@ fn a_protocol_directory_replaced_after_reopen_is_neither_opened_nor_published_in )); Ok(()) } + +#[test] +fn an_initialized_store_without_a_published_catalog_reopens() -> Result<(), Box> { + let sandbox = TestDirectory::create("version-two-admission-unpublished")?; + drop(FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(sandbox.path())?); + assert!(!sandbox.path().join("HEAD").exists()); + + let admission = + FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; + + drop(admission); + Ok(()) +} + +#[test] +fn reopen_refuses_a_root_whose_marker_stage_remains() -> Result<(), Box> { + let sandbox = initialized_store("version-two-admission-marker-stage")?; + fs::hard_link( + sandbox.path().join("FORMAT"), + sandbox.path().join("FORMAT.next"), + )?; + + let error = reopen_namespace_refusal(sandbox.path())?; + + assert_eq!(error, FilesystemNamespaceRefusal::InitializationIncomplete); + Ok(()) +} + +#[test] +fn reopen_refuses_a_root_without_a_marker() -> Result<(), Box> { + let sandbox = initialized_store("version-two-admission-no-marker")?; + fs::remove_file(sandbox.path().join("FORMAT"))?; + + let error = reopen_namespace_refusal(sandbox.path())?; + + assert_eq!(error, FilesystemNamespaceRefusal::MissingFormatMarker); + Ok(()) +} + +fn reopen_namespace_refusal( + root: &std::path::Path, +) -> Result> { + let error = FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(root) + .err() + .ok_or("an uncommitted root was unexpectedly admitted")?; + let FilesystemPlatformAdmissionError::Namespace { source } = error else { + return Err(format!("expected a namespace refusal, got {error:?}").into()); + }; + source + .get_ref() + .and_then(|cause| cause.downcast_ref::()) + .copied() + .ok_or_else(|| "namespace refusal carried no typed cause".into()) +} diff --git a/src/adapters/retention/reader_fence_tests.rs b/src/adapters/retention/reader_fence_tests.rs index b89fbfb8..8fd85572 100644 --- a/src/adapters/retention/reader_fence_tests.rs +++ b/src/adapters/retention/reader_fence_tests.rs @@ -5,7 +5,7 @@ use std::fs; use std::io; use super::ReaderFence; -use crate::adapters::retention::filesystem_retention_test_fixture::migrated_store; +use crate::adapters::retention::filesystem_retention_test_fixture::initialized_store; use cap_std::fs::Dir; // Size: medium (owned filesystem). Oracle: a fence must lock the same inode @@ -13,7 +13,7 @@ use cap_std::fs::Dir; // Delete when reader fencing is removed or stronger scheduling evidence subsumes this law. #[test] fn replacing_the_reader_lock_during_acquisition_refuses_the_fence() -> Result<(), Box> { - let sandbox = migrated_store("reader-fence-replaced-during-acquisition")?; + let sandbox = initialized_store("reader-fence-replaced-during-acquisition")?; let root = Dir::open_ambient_dir(sandbox.path(), cap_std::ambient_authority())?; let path = sandbox.path().join("reader.lock"); diff --git a/src/adapters/retention/reader_platform_law_tests.rs b/src/adapters/retention/reader_platform_law_tests.rs index b09afb26..9fbebc6d 100644 --- a/src/adapters/retention/reader_platform_law_tests.rs +++ b/src/adapters/retention/reader_platform_law_tests.rs @@ -1,7 +1,7 @@ //! Reader platform admission at the public snapshot boundaries. //! //! Size: medium. Oracle: production readers require the existing writable, -//! case-sensitive local ext4 profile, independently of valid migration bytes. +//! case-sensitive local ext4 profile, independently of valid store bytes. //! The negative fixture deliberately uses test-only writer admission on tmpfs. //! Delete when the supported reader profile changes or stronger public laws subsume it. @@ -10,7 +10,7 @@ use std::fs; use std::io; use std::path::PathBuf; -use super::filesystem_retention_test_fixture::{migrated_store, populate_bundle}; +use super::filesystem_retention_test_fixture::{initialized_store, populate_bundle}; use crate::{ CatalogRestartByteLimit, CatalogRestartPolicy, DurableSnapshot, DurableStoreError, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, FilesystemWriterLock, @@ -23,7 +23,7 @@ fn a_canonical_tmpfs_store_refuses_the_public_reader_profile() -> Result<(), Box let refusal = FilesystemRetentionSnapshot::load(&store.0, policy()?, ReaderAttemptLimit::DEFAULT) .err() - .ok_or("valid migration bytes admitted an unsupported reader filesystem")?; + .ok_or("valid store bytes admitted an unsupported reader filesystem")?; assert!( matches!(&refusal, FilesystemRetentionSnapshotError::Admission { source } if source.kind() == io::ErrorKind::Unsupported), @@ -49,7 +49,7 @@ fn a_canonical_tmpfs_store_refuses_durable_snapshot_admission() -> Result<(), Bo #[test] fn reader_platform_admission_does_not_acquire_writer_authority() -> Result<(), Box> { - let store = migrated_store("reader-platform-existing-writer")?; + let store = initialized_store("reader-platform-existing-writer")?; let writer = FilesystemWriterLock::try_acquire(store.path())?; let snapshot = diff --git a/src/adapters/retention/retention_model_tests.rs b/src/adapters/retention/retention_model_tests.rs index 1c09a1d4..11f7f38a 100644 --- a/src/adapters/retention/retention_model_tests.rs +++ b/src/adapters/retention/retention_model_tests.rs @@ -378,7 +378,7 @@ fn verify(store: &Store, model: &Model, schedule: [Operation; 3]) -> Result<(), } /// Runs every three-operation sequence that starts with `first` in a fresh -/// migrated store each, checking the fenced view against the model after +/// initialized store each, checking the fenced view against the model after /// every step. fn run_sequences(first: Operation, label: &str) -> Result<(), Box> { let template = fixture(ROOT_HEX)?; diff --git a/src/adapters/retention/verification_namespace_law_tests.rs b/src/adapters/retention/verification_namespace_law_tests.rs index 5665d10d..cfba189a 100644 --- a/src/adapters/retention/verification_namespace_law_tests.rs +++ b/src/adapters/retention/verification_namespace_law_tests.rs @@ -3,7 +3,7 @@ //! Size: medium. Oracle: canonical protocol membership and no-follow entry kinds. //! Delete if stronger public namespace laws subsume these exact boundaries. -use super::filesystem_retention_test_fixture::migrated_store; +use super::filesystem_retention_test_fixture::initialized_store; use crate::{ CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemEntryKind, FilesystemNamespaceRefusal, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, ReaderAttemptLimit, @@ -16,7 +16,7 @@ type ResultOf = Result>; #[test] fn unexpected_store_entries_are_published_view_corruption() -> ResultOf<()> { - let store = migrated_store("verification-unexpected-entry")?; + let store = initialized_store("verification-unexpected-entry")?; let path = store.path().join("unexpected-entry"); fs::write(&path, b"preserve unexpected evidence")?; assert_corrupt(store.path(), FilesystemNamespaceRefusal::UnexpectedEntry)?; @@ -26,7 +26,7 @@ fn unexpected_store_entries_are_published_view_corruption() -> ResultOf<()> { #[test] fn required_directory_files_are_published_view_corruption() -> ResultOf<()> { - let store = migrated_store("verification-required-directory-file")?; + let store = initialized_store("verification-required-directory-file")?; let path = store.path().join("gc"); fs::remove_dir(&path)?; fs::write(&path, b"preserve wrong kind")?; @@ -43,7 +43,7 @@ fn required_directory_files_are_published_view_corruption() -> ResultOf<()> { #[test] fn required_directory_symlinks_are_published_view_corruption() -> ResultOf<()> { - let store = migrated_store("verification-required-directory-symlink")?; + let store = initialized_store("verification-required-directory-symlink")?; let path = store.path().join("gc"); let target = store.path().join("retention"); fs::remove_dir(&path)?; diff --git a/src/adapters/retention/verification_store_admission_tests.rs b/src/adapters/retention/verification_store_admission_tests.rs index a18a43ea..c6a156a9 100644 --- a/src/adapters/retention/verification_store_admission_tests.rs +++ b/src/adapters/retention/verification_store_admission_tests.rs @@ -1,15 +1,12 @@ //! Typed store contradictions stay distinct from inconclusive admission failures. -use super::filesystem_retention_test_fixture::{migrated_store, retention_witness}; +use super::filesystem_retention_test_fixture::initialized_store; use crate::{ - CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemPlatformAdmissionError, - FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, ReaderAttemptLimit, - SegmentReadPolicy, StoreFormatMarkerDecodeError, StoreMigrationIntentDecodeError, - StoreMigrationReceiptDecodeError, StoreRootIdentityCoordinate, VerificationError, - VerificationObservation, VerificationRefusal, VerificationSource, VerificationSubject, - VersionTwoRecordRefusal, + CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemRetentionSnapshot, + FilesystemRetentionSnapshotError, ReaderAttemptLimit, SegmentReadPolicy, + StoreFormatMarkerDecodeError, VerificationError, VerificationObservation, VerificationRefusal, + VerificationSource, VerificationSubject, VersionTwoRecordRefusal, }; -use std::os::unix::fs::MetadataExt; use std::{error::Error, fs, io}; type ResultOf = Result>; @@ -17,79 +14,33 @@ type ResultOf = Result>; // Size: medium. Oracle: bad literal record magic proves a content contradiction. // Delete if a stronger public loader matrix preserves these typed causes. #[test] -fn malformed_store_records_are_corrupt_with_the_original_magic() -> ResultOf<()> { - for name in ["FORMAT", "migration.intent", "migration.receipt"] { - let store = migrated_store("verification-malformed-store-record")?; - let path = store.path().join(name); - let mut bytes = fs::read(&path)?; - *bytes.first_mut().ok_or("empty migration record")? ^= 1; - let expected: [u8; 16] = bytes.get(..16).ok_or("missing magic")?.try_into()?; - fs::write(&path, &bytes)?; - let source = corrupt_cause( - FilesystemRetentionSnapshot::load_for_verification( - store.path(), - policy()?, - ReaderAttemptLimit::DEFAULT, - ) - .err() - .ok_or("bad magic admitted")?, - )?; - let typed = source - .get_ref() - .and_then(|source| source.downcast_ref::()); - let observed = match typed { - Some(VersionTwoRecordRefusal::Marker { - source: StoreFormatMarkerDecodeError::InvalidMagic { observed }, - }) if name == "FORMAT" => observed, - Some(VersionTwoRecordRefusal::Intent { - source: StoreMigrationIntentDecodeError::InvalidMagic { observed }, - }) if name == "migration.intent" => observed, - Some(VersionTwoRecordRefusal::Receipt { - source: StoreMigrationReceiptDecodeError::InvalidMagic { observed }, - }) if name == "migration.receipt" => observed, - _ => return Err(format!("{name} lost its exact decode refusal: {source:?}").into()), - }; - assert_eq!(*observed, expected, "original magic for {name}"); - assert_eq!(fs::read(path)?, bytes, "refusal must preserve {name}"); - } - Ok(()) -} - -// Size: medium. Oracle: copied migration records still name the donor's inode. -// Delete if migration identity binding is removed or a stronger loader law subsumes it. -#[test] -fn a_foreign_migration_binding_is_corruption_with_exact_root_coordinates() -> ResultOf<()> { - let donor = migrated_store("verification-identity-donor")?; - let recipient = migrated_store("verification-identity-recipient")?; - for name in ["FORMAT", "migration.intent", "migration.receipt"] { - fs::copy(donor.path().join(name), recipient.path().join(name))?; - } - let expected = fs::metadata(donor.path())?.ino(); - let observed = fs::metadata(recipient.path())?.ino(); - let before = retention_witness(recipient.path())?; +fn a_malformed_format_marker_is_corrupt_with_the_original_magic() -> ResultOf<()> { + let store = initialized_store("verification-malformed-store-record")?; + let path = store.path().join("FORMAT"); + let mut bytes = fs::read(&path)?; + *bytes.first_mut().ok_or("empty format marker")? ^= 1; + let expected: [u8; 16] = bytes.get(..16).ok_or("missing magic")?.try_into()?; + fs::write(&path, &bytes)?; let source = corrupt_cause( FilesystemRetentionSnapshot::load_for_verification( - recipient.path(), + store.path(), policy()?, ReaderAttemptLimit::DEFAULT, ) .err() - .ok_or("foreign binding admitted")?, + .ok_or("bad magic admitted")?, )?; let typed = source .get_ref() - .and_then(|source| source.downcast_ref::()); - assert!( - matches!(typed, Some(FilesystemPlatformAdmissionError::RootIdentityChanged { - coordinate: StoreRootIdentityCoordinate::File, expected: actual_expected, observed: actual_observed - }) if *actual_expected == expected && *actual_observed == observed), - "original binding coordinates required: {source:?}" - ); - assert_eq!( - retention_witness(recipient.path())?, - before, - "refusal must preserve the store" - ); + .and_then(|source| source.downcast_ref::()); + let Some(VersionTwoRecordRefusal::Marker { + source: StoreFormatMarkerDecodeError::InvalidMagic { observed }, + }) = typed + else { + return Err(format!("FORMAT lost its exact decode refusal: {source:?}").into()); + }; + assert_eq!(*observed, expected, "original magic"); + assert_eq!(fs::read(path)?, bytes, "refusal must preserve FORMAT"); Ok(()) } diff --git a/src/adapters/store_format.rs b/src/adapters/store_format.rs new file mode 100644 index 00000000..49a0e1c9 --- /dev/null +++ b/src/adapters/store_format.rs @@ -0,0 +1,21 @@ +//! This module owns the canonical `FORMAT` marker and the format-definition digest. +//! +//! The marker is the store's only format-identity record: its magic, version, +//! and definition digest name the format contract, not the software release +//! that wrote a store. + +mod admitted_format_marker; +mod canonical_format_marker; +mod format_definition_digest; +mod format_marker_decode_error; +mod format_marker_decode_error_display; +mod format_marker_decoder; +mod format_marker_digest; +mod format_marker_encoder; + +pub use admitted_format_marker::AdmittedStoreFormatMarker; +pub use canonical_format_marker::CanonicalStoreFormatMarker; +pub use format_definition_digest::StoreFormatDefinitionDigest; +pub use format_marker_decode_error::StoreFormatMarkerDecodeError; +pub(in crate::adapters) use format_marker_decoder::ENCODED_LENGTH as FORMAT_MARKER_LENGTH; +pub use format_marker_digest::StoreFormatMarkerDigest; diff --git a/src/adapters/store_migration/admitted_format_marker.rs b/src/adapters/store_format/admitted_format_marker.rs similarity index 100% rename from src/adapters/store_migration/admitted_format_marker.rs rename to src/adapters/store_format/admitted_format_marker.rs diff --git a/src/adapters/store_migration/canonical_format_marker.rs b/src/adapters/store_format/canonical_format_marker.rs similarity index 100% rename from src/adapters/store_migration/canonical_format_marker.rs rename to src/adapters/store_format/canonical_format_marker.rs diff --git a/src/adapters/store_migration/format_definition_digest.rs b/src/adapters/store_format/format_definition_digest.rs similarity index 74% rename from src/adapters/store_migration/format_definition_digest.rs rename to src/adapters/store_format/format_definition_digest.rs index f9b29bbf..c7a1a298 100644 --- a/src/adapters/store_migration/format_definition_digest.rs +++ b/src/adapters/store_format/format_definition_digest.rs @@ -8,9 +8,9 @@ pub struct StoreFormatDefinitionDigest([u8; 32]); impl StoreFormatDefinitionDigest { /// Digest of the frozen `keep.segment-store/v2` definition. pub const VERSION_TWO: Self = Self([ - 0x32, 0x38, 0x1f, 0x1a, 0xc3, 0x32, 0xd1, 0x27, 0x7a, 0x7e, 0x1f, 0xaf, 0x8f, 0x11, 0x57, - 0x69, 0x93, 0xcb, 0x55, 0xb7, 0xe8, 0x5d, 0x2a, 0x11, 0x0b, 0x74, 0xdc, 0x9c, 0x3b, 0x87, - 0x34, 0x27, + 0xf1, 0xa0, 0xce, 0x10, 0xc0, 0xd3, 0x63, 0xd1, 0x45, 0xf8, 0xa6, 0x93, 0xea, 0x22, 0x65, + 0x1e, 0x47, 0xcf, 0x35, 0x6c, 0xaa, 0x14, 0xf1, 0xef, 0x4f, 0x89, 0x58, 0xd8, 0xb2, 0x9f, + 0x36, 0x1d, ]); /// Returns the raw digest bytes. diff --git a/src/adapters/store_migration/format_marker_decode_error.rs b/src/adapters/store_format/format_marker_decode_error.rs similarity index 100% rename from src/adapters/store_migration/format_marker_decode_error.rs rename to src/adapters/store_format/format_marker_decode_error.rs diff --git a/src/adapters/store_migration/format_marker_decode_error_display.rs b/src/adapters/store_format/format_marker_decode_error_display.rs similarity index 100% rename from src/adapters/store_migration/format_marker_decode_error_display.rs rename to src/adapters/store_format/format_marker_decode_error_display.rs diff --git a/src/adapters/store_migration/format_marker_decoder.rs b/src/adapters/store_format/format_marker_decoder.rs similarity index 100% rename from src/adapters/store_migration/format_marker_decoder.rs rename to src/adapters/store_format/format_marker_decoder.rs diff --git a/src/adapters/store_migration/format_marker_digest.rs b/src/adapters/store_format/format_marker_digest.rs similarity index 100% rename from src/adapters/store_migration/format_marker_digest.rs rename to src/adapters/store_format/format_marker_digest.rs diff --git a/src/adapters/store_migration/format_marker_encoder.rs b/src/adapters/store_format/format_marker_encoder.rs similarity index 100% rename from src/adapters/store_migration/format_marker_encoder.rs rename to src/adapters/store_format/format_marker_encoder.rs diff --git a/src/adapters/store_migration.rs b/src/adapters/store_migration.rs deleted file mode 100644 index 4088b3e4..00000000 --- a/src/adapters/store_migration.rs +++ /dev/null @@ -1,191 +0,0 @@ -//! Canonical version-2 store migration record adapters. - -mod admitted_format_marker; -mod admitted_migration_intent; -mod admitted_migration_receipt; -mod canonical_format_marker; -mod canonical_migration_intent; -mod canonical_migration_receipt; -mod empty_disposition_set_digest; -mod filesystem_inventory_catalog_errors; -mod filesystem_inventory_catalogs; -#[cfg(test)] -mod filesystem_inventory_catalogs_refusal_tests; -#[cfg(test)] -mod filesystem_inventory_catalogs_test_fixture; -#[cfg(test)] -mod filesystem_inventory_catalogs_tests; -mod filesystem_inventory_directory; -mod filesystem_inventory_error; -mod filesystem_inventory_error_display; -mod filesystem_inventory_file; -#[cfg(test)] -mod filesystem_inventory_file_tests; -mod filesystem_inventory_names; -#[cfg(test)] -mod filesystem_inventory_names_tests; -mod filesystem_inventory_reader; -#[cfg(test)] -mod filesystem_inventory_reader_tests; -mod filesystem_inventory_segments; -#[cfg(test)] -mod filesystem_inventory_segments_refusal_tests; -#[cfg(test)] -mod filesystem_inventory_segments_test_fixture; -#[cfg(test)] -mod filesystem_inventory_segments_tests; -mod filesystem_migration_authority; -mod filesystem_migration_authority_error; -mod filesystem_migration_authority_error_display; -#[cfg(test)] -mod filesystem_migration_authority_tests; -mod filesystem_migration_authority_validation; -mod filesystem_migration_fixed_artifact; -mod filesystem_migration_namespace; -mod filesystem_migration_namespace_directory; -mod filesystem_migration_reader_fence; -mod filesystem_migration_recovery; -mod filesystem_migration_recovery_refusal; -pub use filesystem_migration_recovery_refusal::{ - FilesystemMigrationRecoveryRefusal, FilesystemMigrationResidueKind, -}; -#[cfg(test)] -mod filesystem_migration_compatibility_tests; -#[cfg(test)] -mod filesystem_migration_current_recovery_tests; -#[cfg(test)] -mod filesystem_migration_pair_admission_tests; -#[cfg(test)] -mod filesystem_migration_receipt_evidence_tests; -#[cfg(test)] -mod filesystem_migration_recovery_tests; -#[cfg(test)] -mod filesystem_migration_recovery_truncation_tests; -#[cfg(test)] -mod filesystem_migration_remount_tests; -#[cfg(feature = "repository-tasks")] -mod filesystem_migration_repository_tasks; -mod filesystem_migration_residue; -#[cfg(test)] -mod filesystem_migration_residue_kind_tests; -#[cfg(all(test, unix))] -mod filesystem_migration_restart_namespace_tests; -#[cfg(all(test, unix))] -mod filesystem_migration_restart_order_tests; -#[cfg(all(test, unix))] -mod filesystem_migration_restart_pair_tests; -#[cfg(all(test, unix))] -mod filesystem_migration_restart_pool_tests; -#[cfg(all(test, unix))] -mod filesystem_migration_restart_record_tests; -#[cfg(all(test, unix))] -mod filesystem_migration_restart_root_tests; -#[cfg(all(test, unix))] -mod filesystem_migration_restart_test_fixture; -mod filesystem_migration_storage; -#[cfg(test)] -mod filesystem_migration_storage_tests; -#[cfg(test)] -mod filesystem_migration_test_fixture; -mod format_definition_digest; -mod format_marker_decode_error; -mod format_marker_decode_error_display; -mod format_marker_decoder; -mod format_marker_digest; -mod format_marker_encoder; -mod immutable_pool_inventory_digest; -mod initial_gc_state_digest; -mod initial_retention_state_digest; -mod migration_catalog_admission; -mod migration_catalog_coordinates; -mod migration_catalog_plan; -mod migration_catalog_records; -mod migration_error; -mod migration_execution; -mod migration_intent_decode_error; -mod migration_intent_decode_error_display; -mod migration_intent_decoder; -mod migration_intent_digest; -mod migration_intent_encoder; -mod migration_intent_format; -mod migration_inventory_entry; -mod migration_inventory_entry_count; -mod migration_inventory_entry_count_error; -mod migration_inventory_error; -mod migration_inventory_hasher; -mod migration_phase; -mod migration_receipt_decode_error; -mod migration_receipt_decode_error_display; -mod migration_receipt_decoder; -mod migration_receipt_encoder; -mod migration_receipt_format; -mod migration_receipt_initial_state; -mod migration_record_bytes; -mod migration_stage_decode_error; -pub use migration_stage_decode_error::StoreMigrationStageDecodeError; -mod migration_namespace_prefix; -mod migration_recovery_ambiguity; -mod migration_recovery_ambiguity_display; -mod migration_recovery_execution; -mod migration_recovery_plan; -pub use migration_namespace_prefix::StoreMigrationNamespacePrefix; -mod migration_recovery_planner; -mod migration_recovery_residue; -mod migration_recovery_storage; -mod migration_resumption; -mod migration_storage; -mod migration_synchronization_mask; -mod store_identifier; -mod store_root_identity; - -pub use admitted_format_marker::AdmittedStoreFormatMarker; -pub use admitted_migration_intent::AdmittedStoreMigrationIntent; -pub use admitted_migration_receipt::AdmittedStoreMigrationReceipt; -pub use canonical_format_marker::CanonicalStoreFormatMarker; -pub use canonical_migration_intent::CanonicalStoreMigrationIntent; -pub use canonical_migration_receipt::CanonicalStoreMigrationReceipt; -pub use empty_disposition_set_digest::EmptyDispositionSetDigest; -pub use filesystem_inventory_error::{ - FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, - MigrationInventoryNamespace, MigrationInventoryPool, -}; -pub use filesystem_inventory_reader::FilesystemStoreMigrationInventoryReader; -pub use filesystem_migration_authority::FilesystemStoreMigrationAuthority; -pub use filesystem_migration_authority_error::{ - FilesystemMigrationAuthorityArtifact, FilesystemMigrationAuthorityError, - StoreRootIdentityCoordinate, -}; -pub use format_definition_digest::StoreFormatDefinitionDigest; -pub use format_marker_decode_error::StoreFormatMarkerDecodeError; -pub(super) use format_marker_decoder::ENCODED_LENGTH as FORMAT_MARKER_LENGTH; -pub use format_marker_digest::StoreFormatMarkerDigest; -pub use immutable_pool_inventory_digest::ImmutablePoolInventoryDigest; -pub use initial_gc_state_digest::InitialGcStateDigest; -pub use initial_retention_state_digest::InitialRetentionStateDigest; -pub use migration_error::StoreMigrationError; -pub use migration_execution::execute_store_migration; -pub use migration_intent_decode_error::StoreMigrationIntentDecodeError; -pub use migration_intent_digest::StoreMigrationIntentDigest; -pub(super) use migration_intent_format::ENCODED_LENGTH as MIGRATION_INTENT_LENGTH; -pub use migration_inventory_entry::StoreMigrationInventoryEntry; -pub use migration_inventory_entry_count::StoreMigrationInventoryEntryCount; -pub use migration_inventory_entry_count_error::StoreMigrationInventoryEntryCountError; -pub use migration_inventory_error::StoreMigrationInventoryError; -pub use migration_inventory_hasher::StoreMigrationInventoryHasher; -pub use migration_phase::StoreMigrationPhase; -pub use migration_receipt_decode_error::StoreMigrationReceiptDecodeError; -pub(super) use migration_receipt_format::ENCODED_LENGTH as MIGRATION_RECEIPT_LENGTH; -pub use migration_recovery_ambiguity::{StoreMigrationEffect, StoreMigrationRecoveryAmbiguity}; -pub use migration_recovery_execution::{ - StoreMigrationRecoveryError, StoreMigrationRecoveryReceipt, recover_store_migration, -}; -pub use migration_recovery_plan::{StoreMigrationFixedStage, StoreMigrationRecoveryPlan}; -pub use migration_recovery_planner::plan_store_migration_recovery; -pub use migration_recovery_residue::{MIGRATION_NAMESPACE_PREFIX, StoreMigrationResidue}; -pub use migration_recovery_storage::StoreMigrationRecoveryStorage; -pub use migration_storage::StoreMigrationStorage; -pub use migration_synchronization_mask::MigrationSynchronizationMask; -pub use store_identifier::StoreIdentifier; -pub use store_root_identity::{ - StoreRootDeviceIdentity, StoreRootFileIdentity, StoreRootMountIdentity, -}; diff --git a/src/adapters/store_migration/admitted_migration_intent.rs b/src/adapters/store_migration/admitted_migration_intent.rs deleted file mode 100644 index fa625b1f..00000000 --- a/src/adapters/store_migration/admitted_migration_intent.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! This boundary module owns admitted store-migration intent evidence. - -use super::{ - ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, - StoreMigrationIntentDecodeError, StoreMigrationIntentDigest, StoreRootDeviceIdentity, - StoreRootFileIdentity, StoreRootMountIdentity, migration_intent_decoder, -}; -use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; - -/// Semantic fields admitted from one canonical migration intent. -#[derive(Clone, Debug, Eq, PartialEq)] -pub(super) struct StoreMigrationIntentFields { - pub(super) catalog_generation: CatalogGeneration, - pub(super) catalog_length: CatalogLength, - pub(super) catalog_digest: CatalogDigest, - pub(super) predecessor_catalog_digest: Option, - pub(super) inventory_digest: ImmutablePoolInventoryDigest, - pub(super) root_device_identity: StoreRootDeviceIdentity, - pub(super) root_mount_identity: StoreRootMountIdentity, - pub(super) root_file_identity: StoreRootFileIdentity, - pub(super) target_definition_digest: StoreFormatDefinitionDigest, - pub(super) store_identifier: StoreIdentifier, -} - -/// Borrowed canonical version-2 store-migration intent. -/// -/// Admission proves record framing, integrity, internal generation laws, the -/// registered target definition, and deterministic store identity. It does not -/// prove that the named catalog, inventory, or physical root is current. -#[must_use] -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct AdmittedStoreMigrationIntent<'encoded> { - encoded: &'encoded [u8], - fields: StoreMigrationIntentFields, - digest: StoreMigrationIntentDigest, -} - -impl<'encoded> AdmittedStoreMigrationIntent<'encoded> { - /// Decodes and admits one exact canonical migration intent. - /// - /// # Errors - /// - /// Returns [`StoreMigrationIntentDecodeError`] for invalid framing, - /// integrity, catalog coordinates, predecessor state, definition identity, - /// or store identity. - pub fn decode(encoded: &'encoded [u8]) -> Result { - migration_intent_decoder::decode(encoded) - } - - /// Returns the exact borrowed canonical bytes. - #[must_use] - pub const fn encoded(&self) -> &'encoded [u8] { - self.encoded - } - - /// Returns the positive catalog generation named by the intent. - pub const fn catalog_generation(&self) -> CatalogGeneration { - self.fields.catalog_generation - } - - /// Returns the exact admitted catalog byte length. - pub const fn catalog_length(&self) -> CatalogLength { - self.fields.catalog_length - } - - /// Returns the catalog digest named by the intent. - pub const fn catalog_digest(&self) -> CatalogDigest { - self.fields.catalog_digest - } - - /// Returns the generation-relative predecessor digest. - pub const fn predecessor_catalog_digest(&self) -> Option { - self.fields.predecessor_catalog_digest - } - - /// Returns the immutable-pool inventory digest named by the intent. - pub const fn inventory_digest(&self) -> ImmutablePoolInventoryDigest { - self.fields.inventory_digest - } - - /// Returns the serialized root device coordinate named by the intent. - pub const fn root_device_identity(&self) -> StoreRootDeviceIdentity { - self.fields.root_device_identity - } - - /// Returns the serialized root mount coordinate named by the intent. - pub const fn root_mount_identity(&self) -> StoreRootMountIdentity { - self.fields.root_mount_identity - } - - /// Returns the serialized root file coordinate named by the intent. - pub const fn root_file_identity(&self) -> StoreRootFileIdentity { - self.fields.root_file_identity - } - - /// Returns the registered target format-definition digest. - pub const fn target_definition_digest(&self) -> StoreFormatDefinitionDigest { - self.fields.target_definition_digest - } - - /// Returns the deterministic logical store identity. - pub const fn store_identifier(&self) -> StoreIdentifier { - self.fields.store_identifier - } - - /// Returns the domain-separated identity of all intent bytes. - pub const fn digest(&self) -> StoreMigrationIntentDigest { - self.digest - } - - pub(super) const fn admitted( - encoded: &'encoded [u8], - fields: StoreMigrationIntentFields, - digest: StoreMigrationIntentDigest, - ) -> Self { - Self { - encoded, - fields, - digest, - } - } -} diff --git a/src/adapters/store_migration/admitted_migration_receipt.rs b/src/adapters/store_migration/admitted_migration_receipt.rs deleted file mode 100644 index 6abf88c9..00000000 --- a/src/adapters/store_migration/admitted_migration_receipt.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! This boundary module owns admitted store-migration completion evidence. - -use super::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, EmptyDispositionSetDigest, - InitialGcStateDigest, InitialRetentionStateDigest, MigrationSynchronizationMask, - StoreFormatMarkerDigest, StoreIdentifier, StoreMigrationIntentDigest, - StoreMigrationReceiptDecodeError, migration_receipt_decoder, -}; - -/// Semantic fields admitted from one canonical migration receipt. -#[derive(Clone, Debug, Eq, PartialEq)] -pub(super) struct StoreMigrationReceiptFields { - pub(super) intent_digest: StoreMigrationIntentDigest, - pub(super) store_identifier: StoreIdentifier, - pub(super) format_marker_digest: StoreFormatMarkerDigest, - pub(super) initial_retention_state_digest: InitialRetentionStateDigest, - pub(super) initial_gc_state_digest: InitialGcStateDigest, - pub(super) empty_disposition_set_digest: EmptyDispositionSetDigest, - pub(super) synchronization_mask: MigrationSynchronizationMask, -} - -/// Borrowed canonical version-2 store-migration receipt. -/// -/// Admission proves record integrity, exact binding to caller-supplied admitted -/// intent and marker evidence, registered empty-state digests, and the complete -/// synchronization mask. It does not prove that filesystem transitions -/// actually occurred; production recovery must establish that separately. -#[must_use] -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct AdmittedStoreMigrationReceipt<'encoded> { - encoded: &'encoded [u8], - fields: StoreMigrationReceiptFields, -} - -impl<'encoded> AdmittedStoreMigrationReceipt<'encoded> { - /// Decodes and admits one exact receipt bound to `intent` and `marker`. - /// - /// # Errors - /// - /// Returns [`StoreMigrationReceiptDecodeError`] for invalid framing, - /// integrity, binding, registered state digest, or synchronization bits. - pub fn decode( - encoded: &'encoded [u8], - intent: &AdmittedStoreMigrationIntent<'_>, - marker: &AdmittedStoreFormatMarker<'_>, - ) -> Result { - migration_receipt_decoder::decode(encoded, intent, marker) - } - - /// Returns the exact borrowed canonical bytes. - #[must_use] - pub const fn encoded(&self) -> &'encoded [u8] { - self.encoded - } - - /// Returns the exact bound migration-intent digest. - pub const fn intent_digest(&self) -> StoreMigrationIntentDigest { - self.fields.intent_digest - } - - /// Returns the exact bound logical store identity. - pub const fn store_identifier(&self) -> StoreIdentifier { - self.fields.store_identifier - } - - /// Returns the exact bound format-marker digest. - pub const fn format_marker_digest(&self) -> StoreFormatMarkerDigest { - self.fields.format_marker_digest - } - - /// Returns the registered empty retention-state digest. - pub const fn initial_retention_state_digest(&self) -> InitialRetentionStateDigest { - self.fields.initial_retention_state_digest - } - - /// Returns the registered empty garbage-collection-state digest. - pub const fn initial_gc_state_digest(&self) -> InitialGcStateDigest { - self.fields.initial_gc_state_digest - } - - /// Returns the registered empty recovery-disposition-set digest. - pub const fn empty_disposition_set_digest(&self) -> EmptyDispositionSetDigest { - self.fields.empty_disposition_set_digest - } - - /// Returns the complete admitted synchronization mask. - pub const fn synchronization_mask(&self) -> MigrationSynchronizationMask { - self.fields.synchronization_mask - } - - pub(super) const fn admitted( - encoded: &'encoded [u8], - fields: StoreMigrationReceiptFields, - ) -> Self { - Self { encoded, fields } - } -} diff --git a/src/adapters/store_migration/canonical_migration_intent.rs b/src/adapters/store_migration/canonical_migration_intent.rs deleted file mode 100644 index 3478a4f1..00000000 --- a/src/adapters/store_migration/canonical_migration_intent.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! This boundary module owns canonical owned migration-intent bytes. - -use super::admitted_migration_intent::StoreMigrationIntentFields; -use super::migration_catalog_coordinates::MigrationCatalogCoordinates; -use super::store_root_identity::StoreRootIdentities; -use super::{ - ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, - StoreMigrationIntentDigest, StoreRootDeviceIdentity, StoreRootFileIdentity, - StoreRootMountIdentity, migration_intent_encoder, migration_intent_format, -}; -use crate::{CatalogDigest, CatalogGeneration, CatalogLength, CatalogSnapshot}; - -/// Owned canonical version-2 store-migration intent. -/// -/// Construction preserves admitted catalog coordinates and serializes the -/// supplied inventory and physical-root coordinates. It does not prove that -/// the inventory or physical root remains current. -#[must_use] -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct CanonicalStoreMigrationIntent { - encoded: [u8; migration_intent_format::ENCODED_LENGTH], - fields: StoreMigrationIntentFields, - digest: StoreMigrationIntentDigest, -} - -impl CanonicalStoreMigrationIntent { - /// Owns the exact bytes and identities of an admitted intent. - pub const fn from_admitted(intent: &super::AdmittedStoreMigrationIntent<'_>) -> Self { - let mut encoded = [0_u8; migration_intent_format::ENCODED_LENGTH]; - encoded.copy_from_slice(intent.encoded()); - Self { - encoded, - fields: StoreMigrationIntentFields { - catalog_generation: intent.catalog_generation(), - catalog_length: intent.catalog_length(), - catalog_digest: intent.catalog_digest(), - predecessor_catalog_digest: intent.predecessor_catalog_digest(), - inventory_digest: intent.inventory_digest(), - root_device_identity: intent.root_device_identity(), - root_mount_identity: intent.root_mount_identity(), - root_file_identity: intent.root_file_identity(), - target_definition_digest: intent.target_definition_digest(), - store_identifier: intent.store_identifier(), - }, - digest: intent.digest(), - } - } - - /// Constructs one canonical intent from typed migration coordinates. - pub fn from_snapshot( - snapshot: &CatalogSnapshot<'_, '_, '_>, - inventory_digest: ImmutablePoolInventoryDigest, - root_device_identity: StoreRootDeviceIdentity, - root_mount_identity: StoreRootMountIdentity, - root_file_identity: StoreRootFileIdentity, - ) -> Self { - migration_intent_encoder::encode( - MigrationCatalogCoordinates::from_snapshot(snapshot), - inventory_digest, - StoreRootIdentities::new( - root_device_identity, - root_mount_identity, - root_file_identity, - ), - ) - } - - pub(super) fn from_coordinates( - catalog: MigrationCatalogCoordinates, - inventory_digest: ImmutablePoolInventoryDigest, - roots: StoreRootIdentities, - ) -> Self { - migration_intent_encoder::encode(catalog, inventory_digest, roots) - } - - /// Returns the exact canonical intent bytes. - pub const fn encoded(&self) -> &[u8] { - &self.encoded - } - - /// Returns the domain-separated identity of all intent bytes. - pub const fn digest(&self) -> StoreMigrationIntentDigest { - self.digest - } - - /// Returns the positive catalog generation named by the intent. - pub const fn catalog_generation(&self) -> CatalogGeneration { - self.fields.catalog_generation - } - - /// Returns the exact catalog byte length named by the intent. - pub const fn catalog_length(&self) -> CatalogLength { - self.fields.catalog_length - } - - /// Returns the catalog digest named by the intent. - pub const fn catalog_digest(&self) -> CatalogDigest { - self.fields.catalog_digest - } - - /// Returns the generation-relative predecessor digest. - pub const fn predecessor_catalog_digest(&self) -> Option { - self.fields.predecessor_catalog_digest - } - - /// Returns the immutable-pool inventory digest named by the intent. - pub const fn inventory_digest(&self) -> ImmutablePoolInventoryDigest { - self.fields.inventory_digest - } - - /// Returns the serialized root device coordinate. - pub const fn root_device_identity(&self) -> StoreRootDeviceIdentity { - self.fields.root_device_identity - } - - /// Returns the serialized root mount coordinate. - pub const fn root_mount_identity(&self) -> StoreRootMountIdentity { - self.fields.root_mount_identity - } - - /// Returns the serialized root file coordinate. - pub const fn root_file_identity(&self) -> StoreRootFileIdentity { - self.fields.root_file_identity - } - - /// Returns the registered target format-definition digest. - pub const fn target_definition_digest(&self) -> StoreFormatDefinitionDigest { - self.fields.target_definition_digest - } - - /// Returns the deterministic logical store identity. - pub const fn store_identifier(&self) -> StoreIdentifier { - self.fields.store_identifier - } - - pub(super) const fn admitted( - encoded: [u8; migration_intent_format::ENCODED_LENGTH], - fields: StoreMigrationIntentFields, - digest: StoreMigrationIntentDigest, - ) -> Self { - Self { - encoded, - fields, - digest, - } - } -} diff --git a/src/adapters/store_migration/canonical_migration_receipt.rs b/src/adapters/store_migration/canonical_migration_receipt.rs deleted file mode 100644 index 7c40d866..00000000 --- a/src/adapters/store_migration/canonical_migration_receipt.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! This boundary module owns canonical owned migration-receipt bytes. - -use super::{ - CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, migration_receipt_encoder, - migration_receipt_format, -}; - -/// Owned canonical version-2 store-migration completion receipt. -/// -/// Construction binds canonical artifacts and the registered complete initial -/// state. It does not prove that the named filesystem transitions occurred. -#[must_use] -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct CanonicalStoreMigrationReceipt { - encoded: [u8; migration_receipt_format::ENCODED_LENGTH], -} - -impl CanonicalStoreMigrationReceipt { - /// Constructs the one complete receipt for `intent` and `marker`. - pub fn from_canonical( - intent: &CanonicalStoreMigrationIntent, - marker: &CanonicalStoreFormatMarker, - ) -> Self { - migration_receipt_encoder::encode(intent, marker) - } - - /// Returns the exact canonical receipt bytes. - pub const fn encoded(&self) -> &[u8] { - &self.encoded - } - - pub(super) const fn admitted(encoded: [u8; migration_receipt_format::ENCODED_LENGTH]) -> Self { - Self { encoded } - } -} diff --git a/src/adapters/store_migration/empty_disposition_set_digest.rs b/src/adapters/store_migration/empty_disposition_set_digest.rs deleted file mode 100644 index ad59041f..00000000 --- a/src/adapters/store_migration/empty_disposition_set_digest.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! This module owns the registered empty recovery-disposition-set identity. - -/// Registered identity of an empty version-2 recovery-disposition set. -#[must_use] -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct EmptyDispositionSetDigest([u8; 32]); - -impl EmptyDispositionSetDigest { - /// Returns the exact digest bytes. - #[must_use] - pub const fn as_bytes(&self) -> &[u8; 32] { - &self.0 - } - - pub(super) const fn from_hash(hash: [u8; 32]) -> Self { - Self(hash) - } -} diff --git a/src/adapters/store_migration/filesystem_inventory_catalog_errors.rs b/src/adapters/store_migration/filesystem_inventory_catalog_errors.rs deleted file mode 100644 index 7a579e2b..00000000 --- a/src/adapters/store_migration/filesystem_inventory_catalog_errors.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! This module owns filesystem migration catalog-inventory error translation. - -use super::filesystem_inventory_error::{ - FilesystemMigrationInventoryError, MigrationInventoryPool, -}; -use super::filesystem_inventory_file::FilesystemInventoryFileError; -use super::migration_catalog_admission::MigrationCatalogAdmissionError; -use crate::adapters::{ - CatalogAdmissionError, CatalogRestartError, RecoveryEntryName, SegmentDigest, -}; - -const POOL: MigrationInventoryPool = MigrationInventoryPool::Catalogs; - -pub(super) fn admission( - name: &RecoveryEntryName, - source: MigrationCatalogAdmissionError, -) -> FilesystemMigrationInventoryError { - match source { - MigrationCatalogAdmissionError::Catalog(source) => catalog_admission(name, *source), - MigrationCatalogAdmissionError::SegmentSource { digest, source } => { - referenced_segment(digest, source) - } - MigrationCatalogAdmissionError::SegmentCoordinate { expected, observed } => { - FilesystemMigrationInventoryError::ReferencedSegment { - digest: expected, - source: Box::new(CatalogRestartError::SegmentCoordinate { expected, observed }), - } - } - } -} - -fn catalog_admission( - name: &RecoveryEntryName, - source: CatalogAdmissionError, -) -> FilesystemMigrationInventoryError { - match source { - CatalogAdmissionError::MissingSegment { digest } => { - FilesystemMigrationInventoryError::ReferencedSegment { - digest, - source: Box::new(CatalogRestartError::CatalogAdmission { - source: Box::new(CatalogAdmissionError::MissingSegment { digest }), - }), - } - } - CatalogAdmissionError::Segment { digest, source } => { - FilesystemMigrationInventoryError::ReferencedSegment { - digest, - source: Box::new(CatalogRestartError::Segment { - expected: digest, - source, - }), - } - } - source => artifact( - name, - CatalogRestartError::CatalogAdmission { - source: Box::new(source), - }, - ), - } -} - -pub(super) fn catalog_file( - name: &RecoveryEntryName, - source: FilesystemInventoryFileError, -) -> FilesystemMigrationInventoryError { - match source { - FilesystemInventoryFileError::Artifact(source) => { - FilesystemMigrationInventoryError::Artifact { - pool: POOL, - name: name.clone(), - source, - } - } - FilesystemInventoryFileError::Changed => { - FilesystemMigrationInventoryError::ArtifactChanged { - pool: POOL, - name: name.clone(), - } - } - } -} - -fn referenced_segment( - digest: SegmentDigest, - source: FilesystemInventoryFileError, -) -> FilesystemMigrationInventoryError { - match source { - FilesystemInventoryFileError::Artifact(source) => { - FilesystemMigrationInventoryError::ReferencedSegment { digest, source } - } - FilesystemInventoryFileError::Changed => { - FilesystemMigrationInventoryError::ReferencedSegmentChanged { digest } - } - } -} - -pub(super) fn artifact( - name: &RecoveryEntryName, - source: CatalogRestartError, -) -> FilesystemMigrationInventoryError { - FilesystemMigrationInventoryError::Artifact { - pool: POOL, - name: name.clone(), - source: Box::new(source), - } -} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs.rs b/src/adapters/store_migration/filesystem_inventory_catalogs.rs deleted file mode 100644 index ebed9d8c..00000000 --- a/src/adapters/store_migration/filesystem_inventory_catalogs.rs +++ /dev/null @@ -1,179 +0,0 @@ -//! This module owns complete filesystem migration catalog-pool admission. - -use std::collections::TryReserveError; - -use cap_std::fs::Dir; - -use super::StoreMigrationInventoryEntry; -use super::filesystem_inventory_catalog_errors; -use super::filesystem_inventory_error::{ - FilesystemMigrationInventoryError, MigrationInventoryPool, -}; -use super::filesystem_inventory_file::{ - self, FilesystemInventoryFileError, FilesystemInventoryFilePolicy, -}; -use super::filesystem_inventory_names; -use super::filesystem_inventory_segments::FilesystemMigrationSegmentInventory; -use super::migration_catalog_admission::{self, MigrationSegmentLoadError}; -use crate::CatalogLength; -use crate::adapters::recovery::recovery_pool_name; -use crate::adapters::{ - CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, ChecksummedCatalog, - RecoveryEntryName, SegmentDigest, SegmentReadPolicy, physical_pool_name, -}; - -const POOL: MigrationInventoryPool = MigrationInventoryPool::Catalogs; - -pub(super) struct FilesystemMigrationCatalogInventory { - entries: Vec, - names: Vec, - remaining: u32, -} - -impl FilesystemMigrationCatalogInventory { - pub(super) fn entries(&self) -> &[StoreMigrationInventoryEntry] { - &self.entries - } - - pub(super) const fn len(&self) -> usize { - self.entries.len() - } - - pub(super) fn verify_names( - &self, - directory: &Dir, - ) -> Result<(), FilesystemMigrationInventoryError> { - filesystem_inventory_names::verify(directory, POOL, self.remaining, &self.names) - } -} - -pub(super) fn read( - catalogs: &Dir, - segments: &Dir, - admitted_segments: &FilesystemMigrationSegmentInventory, - remaining: u32, - policy: SegmentReadPolicy, -) -> Result { - let names = filesystem_inventory_names::read(catalogs, POOL, remaining)?; - let capacity = names.len(); - let entry_count = u64::try_from(capacity) - .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool: POOL })?; - let mut entries = reserve(capacity, entry_count)?; - for name in &names { - entries.push(admit(catalogs, segments, admitted_segments, name, policy)?); - } - entries.sort_unstable(); - Ok(FilesystemMigrationCatalogInventory { - entries, - names, - remaining, - }) -} - -fn reserve( - capacity: usize, - entry_count: u64, -) -> Result, FilesystemMigrationInventoryError> { - let mut values = Vec::new(); - values - .try_reserve_exact(capacity) - .map_err(|source| allocation(entry_count, source))?; - Ok(values) -} - -const fn allocation( - entry_count: u64, - source: TryReserveError, -) -> FilesystemMigrationInventoryError { - FilesystemMigrationInventoryError::Allocation { - pool: POOL, - entry_count, - source, - } -} - -fn admit( - catalogs: &Dir, - segments: &Dir, - admitted_segments: &FilesystemMigrationSegmentInventory, - name: &RecoveryEntryName, - policy: SegmentReadPolicy, -) -> Result { - let (generation, digest) = recovery_pool_name::catalog(name).map_err(|source| { - FilesystemMigrationInventoryError::Name { - pool: POOL, - name: name.clone(), - source, - } - })?; - let encoded = read_catalog(catalogs, name, generation, digest)?; - let catalog = ChecksummedCatalog::decode(&encoded).map_err(|source| { - filesystem_inventory_catalog_errors::artifact(name, CatalogRestartError::Catalog { source }) - })?; - require_coordinate(name, generation, digest, catalog)?; - let admitted = migration_catalog_admission::admit(catalog, policy, |required| { - load_segment(segments, admitted_segments, required) - }) - .map_err(|source| filesystem_inventory_catalog_errors::admission(name, source))?; - Ok(StoreMigrationInventoryEntry::from_migration_catalog( - &admitted, - )) -} - -fn read_catalog( - directory: &Dir, - name: &RecoveryEntryName, - generation: crate::CatalogGeneration, - digest: crate::CatalogDigest, -) -> Result, FilesystemMigrationInventoryError> { - let canonical_name = physical_pool_name::catalog(generation, digest); - let policy = FilesystemInventoryFilePolicy::new( - CatalogRestartArtifact::Catalog, - CatalogRestartPhase::OpenCatalog, - CatalogRestartPhase::ReadCatalog, - CatalogLength::MAXIMUM.get(), - ); - filesystem_inventory_file::read(directory, &canonical_name, policy) - .map_err(|source| filesystem_inventory_catalog_errors::catalog_file(name, source)) -} - -fn require_coordinate( - name: &RecoveryEntryName, - generation: crate::CatalogGeneration, - digest: crate::CatalogDigest, - catalog: ChecksummedCatalog<'_>, -) -> Result<(), FilesystemMigrationInventoryError> { - if catalog.generation() == generation && catalog.digest() == digest { - return Ok(()); - } - Err(filesystem_inventory_catalog_errors::artifact( - name, - CatalogRestartError::CatalogCoordinate { - expected_generation: generation, - observed_generation: catalog.generation(), - expected_length: catalog.length(), - observed_length: catalog.length(), - expected_digest: digest, - observed_digest: catalog.digest(), - }, - )) -} - -fn load_segment( - directory: &Dir, - admitted: &FilesystemMigrationSegmentInventory, - digest: SegmentDigest, -) -> Result, MigrationSegmentLoadError> { - if !admitted.contains(digest) { - return Err(MigrationSegmentLoadError::Missing); - } - let name = physical_pool_name::segment(digest); - let policy = FilesystemInventoryFilePolicy::new( - CatalogRestartArtifact::Segment { digest }, - CatalogRestartPhase::OpenSegment, - CatalogRestartPhase::ReadSegment, - crate::adapters::segment_header::MAXIMUM_SEGMENT_LENGTH, - ); - filesystem_inventory_file::read(directory, &name, policy) - .map_err(MigrationSegmentLoadError::Source) -} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs b/src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs deleted file mode 100644 index 8676d40e..00000000 --- a/src/adapters/store_migration/filesystem_inventory_catalogs_refusal_tests.rs +++ /dev/null @@ -1,184 +0,0 @@ -//! Filesystem migration catalog-pool refusal and orphan laws. - -use std::error::Error; -use std::fs; -use std::io; - -use super::filesystem_inventory_catalogs; -use super::filesystem_inventory_catalogs_test_fixture::{ - CatalogPoolFixture, empty_segment_bytes, maximum_policy, -}; -use super::filesystem_inventory_error::FilesystemMigrationInventoryError; -use super::filesystem_inventory_segments; -use crate::adapters::{ - AdmittedSegment, CatalogAdmissionError, CatalogRestartError, physical_pool_name, -}; - -#[test] -fn unrelated_orphan_segment_remains_in_exact_pool_inventory() -> Result<(), Box> { - let fixture = CatalogPoolFixture::create("migration-catalog-orphan-segment")?; - let orphan_bytes = empty_segment_bytes()?; - let orphan = AdmittedSegment::decode(&orphan_bytes, maximum_policy())?; - fs::write( - fixture - .segments_path() - .join(physical_pool_name::segment(orphan.digest())), - &orphan_bytes, - )?; - let segments_directory = fixture.open_segments()?; - let catalogs_directory = fixture.open_catalogs()?; - let segments = filesystem_inventory_segments::read(&segments_directory, 2, maximum_policy())?; - - let catalogs = filesystem_inventory_catalogs::read( - &catalogs_directory, - &segments_directory, - &segments, - 1, - maximum_policy(), - )?; - assert_eq!(catalogs.entries().len(), 1); - drop(catalogs_directory); - drop(segments_directory); - fixture.remove()?; - Ok(()) -} - -#[test] -fn catalog_missing_its_segment_refuses_inventory() -> Result<(), Box> { - let fixture = CatalogPoolFixture::create("migration-catalog-missing-segment")?; - let missing = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?.digest(); - remove_fixture_segment(&fixture)?; - let segments_directory = fixture.open_segments()?; - let catalogs_directory = fixture.open_catalogs()?; - let segments = filesystem_inventory_segments::read(&segments_directory, 0, maximum_policy())?; - - let error = require_error(filesystem_inventory_catalogs::read( - &catalogs_directory, - &segments_directory, - &segments, - 1, - maximum_policy(), - ))?; - let FilesystemMigrationInventoryError::ReferencedSegment { digest, source } = error else { - return Err(io::Error::other("missing segment returned wrong refusal").into()); - }; - assert_eq!(digest, missing); - assert!(matches!( - source.as_ref(), - CatalogRestartError::CatalogAdmission { - source - } if matches!( - source.as_ref(), - CatalogAdmissionError::MissingSegment { digest } if *digest == missing - ) - )); - drop(catalogs_directory); - drop(segments_directory); - fixture.remove()?; - Ok(()) -} - -#[test] -fn segment_corruption_after_pool_admission_refuses_catalog() -> Result<(), Box> { - let fixture = CatalogPoolFixture::create("migration-catalog-segment-corruption")?; - let segments_directory = fixture.open_segments()?; - let catalogs_directory = fixture.open_catalogs()?; - let segments = filesystem_inventory_segments::read(&segments_directory, 1, maximum_policy())?; - let segment = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; - fs::write( - fixture - .segments_path() - .join(physical_pool_name::segment(segment.digest())), - b"corrupt", - )?; - - let error = require_error(filesystem_inventory_catalogs::read( - &catalogs_directory, - &segments_directory, - &segments, - 1, - maximum_policy(), - ))?; - let (digest, source) = match error { - FilesystemMigrationInventoryError::ReferencedSegment { digest, source } => (digest, source), - other => { - return Err(io::Error::other(format!( - "corrupt referenced segment returned wrong refusal: {other:?}" - )) - .into()); - } - }; - assert_eq!(digest, segment.digest()); - assert!(matches!( - source.as_ref(), - CatalogRestartError::Segment { - expected, - .. - } if *expected == segment.digest() - )); - drop(catalogs_directory); - drop(segments_directory); - fixture.remove()?; - Ok(()) -} - -#[test] -fn valid_segment_substitution_names_the_referenced_coordinate() -> Result<(), Box> { - let fixture = CatalogPoolFixture::create("migration-catalog-segment-substitution")?; - let segments_directory = fixture.open_segments()?; - let catalogs_directory = fixture.open_catalogs()?; - let segments = filesystem_inventory_segments::read(&segments_directory, 1, maximum_policy())?; - let expected = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; - let replacement_bytes = empty_segment_bytes()?; - let replacement = AdmittedSegment::decode(&replacement_bytes, maximum_policy())?; - assert_ne!(expected.digest(), replacement.digest()); - fs::write( - fixture - .segments_path() - .join(physical_pool_name::segment(expected.digest())), - &replacement_bytes, - )?; - - let error = require_error(filesystem_inventory_catalogs::read( - &catalogs_directory, - &segments_directory, - &segments, - 1, - maximum_policy(), - ))?; - let FilesystemMigrationInventoryError::ReferencedSegment { digest, source } = error else { - return Err(io::Error::other("segment substitution returned wrong refusal").into()); - }; - assert_eq!(digest, expected.digest()); - assert!(matches!( - source.as_ref(), - CatalogRestartError::SegmentCoordinate { - expected: expected_digest, - observed - } if *expected_digest == expected.digest() && *observed == replacement.digest() - )); - drop(catalogs_directory); - drop(segments_directory); - fixture.remove()?; - Ok(()) -} - -fn remove_fixture_segment(fixture: &CatalogPoolFixture) -> Result<(), Box> { - let segment = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; - fs::remove_file( - fixture - .segments_path() - .join(physical_pool_name::segment(segment.digest())), - )?; - Ok(()) -} - -fn require_error( - result: Result, -) -> Result { - result.map_or_else(Ok, |_value| { - Err(io::Error::other( - "filesystem migration catalog inventory unexpectedly succeeded", - )) - }) -} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs b/src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs deleted file mode 100644 index c9a45c3b..00000000 --- a/src/adapters/store_migration/filesystem_inventory_catalogs_test_fixture.rs +++ /dev/null @@ -1,109 +0,0 @@ -//! Deterministic filesystem migration catalog-pool fixture. - -use std::error::Error; -use std::fs; -use std::path::{Path, PathBuf}; - -use cap_std::ambient_authority; -use cap_std::fs::Dir; - -use crate::LayoutEntryLimit; -use crate::adapters::filesystem_test_sandbox::TestDirectory; -use crate::adapters::test_support::decode_hex; -use crate::adapters::{ - AdmittedSegment, ChecksummedCatalog, SegmentReadPolicy, SegmentRecordLimit, physical_pool_name, -}; - -const SEGMENT_HEX: &str = - include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); -const CATALOG_HEX: &str = - include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); -const EMPTY_SEGMENT_HEX: &str = - include_str!("../../../conformance/segment-store/v1/empty-segment.hex"); - -pub(super) struct CatalogPoolFixture { - sandbox: TestDirectory, - segment_bytes: Vec, - catalog_bytes: Vec, -} - -impl CatalogPoolFixture { - pub(super) fn create(name: &str) -> Result> { - let sandbox = TestDirectory::create(name)?; - fs::create_dir(sandbox.path().join("segments"))?; - fs::create_dir(sandbox.path().join("catalogs"))?; - let segment_bytes = decode_hex(SEGMENT_HEX.trim())?; - let catalog_bytes = decode_hex(CATALOG_HEX.trim())?; - let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; - let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; - fs::write( - sandbox - .path() - .join("segments") - .join(physical_pool_name::segment(segment.digest())), - &segment_bytes, - )?; - fs::write( - sandbox - .path() - .join("catalogs") - .join(physical_pool_name::catalog( - catalog.generation(), - catalog.digest(), - )), - &catalog_bytes, - )?; - Ok(Self { - sandbox, - segment_bytes, - catalog_bytes, - }) - } - - pub(super) fn path(&self) -> &Path { - self.sandbox.path() - } - - pub(super) fn segments_path(&self) -> PathBuf { - self.path().join("segments") - } - - pub(super) fn catalogs_path(&self) -> PathBuf { - self.path().join("catalogs") - } - - pub(super) fn open_segments(&self) -> Result> { - Ok(Dir::open_ambient_dir( - self.segments_path(), - ambient_authority(), - )?) - } - - pub(super) fn open_catalogs(&self) -> Result> { - Ok(Dir::open_ambient_dir( - self.catalogs_path(), - ambient_authority(), - )?) - } - - pub(super) fn segment_bytes(&self) -> &[u8] { - &self.segment_bytes - } - - pub(super) fn catalog_bytes(&self) -> &[u8] { - &self.catalog_bytes - } - - pub(super) fn remove(self) -> Result<(), Box> { - self.sandbox.remove()?; - Ok(()) - } -} - -pub(super) const fn maximum_policy() -> SegmentReadPolicy { - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) -} - -pub(super) fn empty_segment_bytes() -> Result, Box> { - Ok(decode_hex(EMPTY_SEGMENT_HEX.trim())?) -} diff --git a/src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs b/src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs deleted file mode 100644 index 08aff5ba..00000000 --- a/src/adapters/store_migration/filesystem_inventory_catalogs_tests.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Filesystem migration catalog-pool admission laws. - -use std::error::Error; - -use super::StoreMigrationInventoryEntry; -use super::filesystem_inventory_catalogs; -use super::filesystem_inventory_catalogs_test_fixture::{CatalogPoolFixture, maximum_policy}; -use super::filesystem_inventory_segments; -use crate::adapters::{AdmittedSegment, ChecksummedCatalog}; - -#[test] -fn every_catalog_binds_exact_pool_segment_records() -> Result<(), Box> { - let fixture = CatalogPoolFixture::create("migration-catalog-inventory")?; - let segments_directory = fixture.open_segments()?; - let catalogs_directory = fixture.open_catalogs()?; - let segments = filesystem_inventory_segments::read(&segments_directory, 1, maximum_policy())?; - let catalogs = filesystem_inventory_catalogs::read( - &catalogs_directory, - &segments_directory, - &segments, - 1, - maximum_policy(), - )?; - let segment = AdmittedSegment::decode(fixture.segment_bytes(), maximum_policy())?; - let admitted = ChecksummedCatalog::decode(fixture.catalog_bytes())?.admit(&[segment])?; - - assert_eq!( - catalogs.entries(), - &[StoreMigrationInventoryEntry::from_catalog(&admitted)] - ); - drop(catalogs_directory); - drop(segments_directory); - fixture.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_inventory_directory.rs b/src/adapters/store_migration/filesystem_inventory_directory.rs deleted file mode 100644 index e48f836f..00000000 --- a/src/adapters/store_migration/filesystem_inventory_directory.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! This module owns pinned migration pool-directory identity. - -use cap_std::fs::Dir; - -use crate::adapters::filesystem_exact_record::EntryIdentity; - -use super::filesystem_inventory_error::{ - FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, - MigrationInventoryNamespace, MigrationInventoryPool, -}; -use crate::adapters::sync_capable_directory; - -pub(super) struct PinnedMigrationPoolDirectory { - pool: MigrationInventoryPool, - name: &'static str, - identity: EntryIdentity, - directory: Dir, -} - -impl PinnedMigrationPoolDirectory { - pub(super) fn open( - root: &Dir, - pool: MigrationInventoryPool, - name: &'static str, - ) -> Result { - let directory = sync_capable_directory::open(root, name).map_err(|source| { - FilesystemMigrationInventoryError::Io { - namespace: MigrationInventoryNamespace::from(pool), - operation: FilesystemMigrationInventoryOperation::OpenPool, - source, - } - })?; - let identity = EntryIdentity::of_directory(&directory).map_err(|source| { - FilesystemMigrationInventoryError::Io { - namespace: MigrationInventoryNamespace::from(pool), - operation: FilesystemMigrationInventoryOperation::OpenPool, - source, - } - })?; - Ok(Self { - pool, - name, - identity, - directory, - }) - } - - pub(super) fn verify(&self, root: &Dir) -> Result<(), FilesystemMigrationInventoryError> { - let handle = EntryIdentity::of_directory(&self.directory).map_err(|source| { - FilesystemMigrationInventoryError::Io { - namespace: MigrationInventoryNamespace::from(self.pool), - operation: FilesystemMigrationInventoryOperation::VerifyPool, - source, - } - })?; - let metadata = root.symlink_metadata(self.name).map_err(|source| { - FilesystemMigrationInventoryError::Io { - namespace: MigrationInventoryNamespace::from(self.pool), - operation: FilesystemMigrationInventoryOperation::VerifyPool, - source, - } - })?; - let current = EntryIdentity::from(&metadata); - if metadata.is_dir() && handle == self.identity && current == self.identity { - Ok(()) - } else { - Err(FilesystemMigrationInventoryError::NamespaceChanged { pool: self.pool }) - } - } - - pub(super) const fn directory(&self) -> &Dir { - &self.directory - } -} diff --git a/src/adapters/store_migration/filesystem_inventory_error.rs b/src/adapters/store_migration/filesystem_inventory_error.rs deleted file mode 100644 index df903b26..00000000 --- a/src/adapters/store_migration/filesystem_inventory_error.rs +++ /dev/null @@ -1,155 +0,0 @@ -//! This boundary module owns filesystem migration-inventory failures. - -use std::collections::TryReserveError; -use std::io; - -use super::super::{CatalogRestartError, RecoveryEntryName, RecoveryPoolNameError, SegmentDigest}; - -/// Immutable version-1 pool selected during migration inventory. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum MigrationInventoryPool { - /// The `segments` immutable pool. - Segments, - /// The `catalogs` immutable pool. - Catalogs, -} - -/// Pinned filesystem namespace observed during migration inventory. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum MigrationInventoryNamespace { - /// The writer-authorized store root. - Root, - /// The `segments` immutable pool. - Segments, - /// The `catalogs` immutable pool. - Catalogs, -} - -impl From for MigrationInventoryNamespace { - fn from(pool: MigrationInventoryPool) -> Self { - match pool { - MigrationInventoryPool::Segments => Self::Segments, - MigrationInventoryPool::Catalogs => Self::Catalogs, - } - } -} - -/// Capability-relative directory operation attempted during inventory. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum FilesystemMigrationInventoryOperation { - /// Clone the writer-authorized root capability. - CloneRoot, - /// Open one immutable-pool directory without following links. - OpenPool, - /// Revalidate one pinned immutable-pool directory. - VerifyPool, - /// Count entries under the pinned directory capability. - CountEntries, - /// Read exact raw entry names under the pinned directory capability. - ReadEntryNames, -} - -/// Failure to derive exact migration inventory from immutable filesystem pools. -#[derive(Debug)] -pub enum FilesystemMigrationInventoryError { - /// One capability-relative directory operation failed. - Io { - /// Namespace being observed. - namespace: MigrationInventoryNamespace, - /// Exact failed operation. - operation: FilesystemMigrationInventoryOperation, - /// Preserved filesystem source. - source: io::Error, - }, - /// A pinned immutable-pool directory changed identity. - NamespaceChanged { - /// Pool whose canonical directory entry changed. - pool: MigrationInventoryPool, - }, - /// Exact raw pool membership changed during artifact admission. - EntriesChanged { - /// Pool whose entry-name set changed. - pool: MigrationInventoryPool, - }, - /// A pool exceeded the remaining inventory entry budget. - EntryLimitExceeded { - /// Pool being observed. - pool: MigrationInventoryPool, - /// Remaining entry budget. - maximum: u32, - /// Smallest count observed before enumeration stopped. - observed_at_least: u64, - }, - /// The directory entry count changed between bounded passes. - EntryCountChanged { - /// Pool being observed. - pool: MigrationInventoryPool, - /// Count established by the first pass. - expected: u64, - /// Count established by the second pass. - observed: u64, - }, - /// A host entry count did not fit the protocol representation. - EntryCountHostWidth { - /// Pool being observed. - pool: MigrationInventoryPool, - }, - /// Memory could not retain the bounded semantic inventory. - Allocation { - /// Pool being observed. - pool: MigrationInventoryPool, - /// Exact number of entries requested. - entry_count: u64, - /// Preserved allocation source. - source: TryReserveError, - }, - /// One immutable-pool name was not canonical. - Name { - /// Pool containing the entry. - pool: MigrationInventoryPool, - /// Exact raw name that was refused. - name: RecoveryEntryName, - /// Preserved canonical-name refusal. - source: RecoveryPoolNameError, - }, - /// One named immutable artifact could not be completely admitted. - Artifact { - /// Pool containing the artifact. - pool: MigrationInventoryPool, - /// Exact raw canonical name. - name: RecoveryEntryName, - /// Preserved artifact refusal. - source: Box, - }, - /// A canonical artifact changed identity while it was being admitted. - ArtifactChanged { - /// Pool containing the artifact. - pool: MigrationInventoryPool, - /// Exact raw canonical name. - name: RecoveryEntryName, - }, - /// A catalog-referenced segment could not be completely admitted. - ReferencedSegment { - /// Exact segment coordinate required by the catalog. - digest: SegmentDigest, - /// Preserved segment artifact refusal. - source: Box, - }, - /// A catalog-referenced segment changed identity during admission. - ReferencedSegmentChanged { - /// Exact segment coordinate required by the catalog. - digest: SegmentDigest, - }, - /// Combined pool count arithmetic could not be represented. - EntryCountArithmetic, - /// Combined pool count violated the canonical inventory bound. - EntryCount { - /// Preserved canonical count refusal. - source: super::StoreMigrationInventoryEntryCountError, - }, - /// Canonical entry streaming refused the observed inventory. - Canonical { - /// Preserved canonical inventory refusal. - source: Box, - }, -} diff --git a/src/adapters/store_migration/filesystem_inventory_error_display.rs b/src/adapters/store_migration/filesystem_inventory_error_display.rs deleted file mode 100644 index aa955cfd..00000000 --- a/src/adapters/store_migration/filesystem_inventory_error_display.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! This module owns filesystem migration-inventory error presentation. - -use std::error::Error; -use std::fmt; - -use super::filesystem_inventory_error::{ - FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, - MigrationInventoryNamespace, MigrationInventoryPool, -}; - -impl fmt::Display for MigrationInventoryPool { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(match self { - Self::Segments => "segments", - Self::Catalogs => "catalogs", - }) - } -} - -impl fmt::Display for FilesystemMigrationInventoryOperation { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(match self { - Self::CloneRoot => "clone writer-authorized root", - Self::OpenPool => "open pool capability", - Self::VerifyPool => "verify pool capability", - Self::CountEntries => "count entries", - Self::ReadEntryNames => "read entry names", - }) - } -} - -impl fmt::Display for MigrationInventoryNamespace { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(match self { - Self::Root => "root", - Self::Segments => "segments", - Self::Catalogs => "catalogs", - }) - } -} - -impl fmt::Display for FilesystemMigrationInventoryError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io { - namespace, - operation, - .. - } => write!( - formatter, - "migration inventory failed to {operation} in {namespace}" - ), - Self::NamespaceChanged { pool } => { - write!( - formatter, - "migration inventory namespace changed for {pool}" - ) - } - Self::EntriesChanged { pool } => { - write!(formatter, "migration inventory entries changed for {pool}") - } - Self::EntryLimitExceeded { - pool, - maximum, - observed_at_least, - } => write!( - formatter, - "migration inventory observed at least {observed_at_least} entries in {pool}, \ - exceeding remaining limit {maximum}" - ), - Self::EntryCountChanged { - pool, - expected, - observed, - } => write!( - formatter, - "migration inventory entry count changed in {pool}: expected {expected}, \ - observed {observed}" - ), - Self::EntryCountHostWidth { pool } => { - write!( - formatter, - "migration inventory count does not fit for {pool}" - ) - } - Self::Allocation { - pool, entry_count, .. - } => write!( - formatter, - "migration inventory could not retain {entry_count} entries for {pool}" - ), - Self::Name { pool, name, .. } => write!( - formatter, - "migration inventory found noncanonical name {:?} in {pool}", - name.as_bytes() - ), - Self::Artifact { pool, name, .. } => write!( - formatter, - "migration inventory could not admit artifact {:?} in {pool}", - name.as_bytes() - ), - Self::ArtifactChanged { pool, name } => write!( - formatter, - "migration inventory artifact {:?} changed identity in {pool}", - name.as_bytes() - ), - Self::ReferencedSegment { digest, .. } => write!( - formatter, - "migration inventory could not admit catalog segment {digest:?}" - ), - Self::ReferencedSegmentChanged { digest } => write!( - formatter, - "migration inventory catalog segment {digest:?} changed identity" - ), - Self::EntryCountArithmetic => { - formatter.write_str("migration inventory entry count overflowed") - } - Self::EntryCount { .. } => { - formatter.write_str("migration inventory entry count was refused") - } - Self::Canonical { .. } => { - formatter.write_str("migration inventory canonical streaming failed") - } - } - } -} - -impl Error for FilesystemMigrationInventoryError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Io { source, .. } => Some(source), - Self::Allocation { source, .. } => Some(source), - Self::Name { source, .. } => Some(source), - Self::Artifact { source, .. } | Self::ReferencedSegment { source, .. } => Some(source), - Self::EntryCount { source } => Some(source), - Self::Canonical { source } => Some(source), - Self::EntryLimitExceeded { .. } - | Self::EntryCountChanged { .. } - | Self::EntryCountHostWidth { .. } - | Self::ArtifactChanged { .. } - | Self::ReferencedSegmentChanged { .. } - | Self::NamespaceChanged { .. } - | Self::EntriesChanged { .. } - | Self::EntryCountArithmetic => None, - } - } -} diff --git a/src/adapters/store_migration/filesystem_inventory_file.rs b/src/adapters/store_migration/filesystem_inventory_file.rs deleted file mode 100644 index 70080165..00000000 --- a/src/adapters/store_migration/filesystem_inventory_file.rs +++ /dev/null @@ -1,128 +0,0 @@ -//! This module owns identity-stable migration artifact reads. - -use cap_fs_ext::MetadataExt; -use cap_std::fs::{Dir, File, Metadata}; - -use crate::adapters::{ - CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, catalog_restart_io, -}; - -pub(super) enum FilesystemInventoryFileError { - Artifact(Box), - Changed, -} - -#[derive(Clone, Copy)] -pub(super) struct FilesystemInventoryFilePolicy { - artifact: CatalogRestartArtifact, - open_phase: CatalogRestartPhase, - read_phase: CatalogRestartPhase, - maximum_length: u64, -} - -impl FilesystemInventoryFilePolicy { - pub(super) const fn new( - artifact: CatalogRestartArtifact, - open_phase: CatalogRestartPhase, - read_phase: CatalogRestartPhase, - maximum_length: u64, - ) -> Self { - Self { - artifact, - open_phase, - read_phase, - maximum_length, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct FileIdentity { - device: u64, - inode: u64, - length: u64, -} - -impl FileIdentity { - fn read(file: &File, phase: CatalogRestartPhase) -> Result { - file.metadata() - .map(|metadata| Self::from(&metadata)) - .map_err(|source| CatalogRestartError::io(phase, source)) - } -} - -impl From<&Metadata> for FileIdentity { - fn from(metadata: &Metadata) -> Self { - Self { - device: metadata.dev(), - inode: metadata.ino(), - length: metadata.len(), - } - } -} - -pub(super) fn read( - directory: &Dir, - name: &str, - policy: FilesystemInventoryFilePolicy, -) -> Result, FilesystemInventoryFileError> { - read_with(directory, name, policy, || {}) -} - -pub(super) fn read_with( - directory: &Dir, - name: &str, - policy: FilesystemInventoryFilePolicy, - before_verify: F, -) -> Result, FilesystemInventoryFileError> -where - F: FnOnce(), -{ - let (file, length) = - catalog_restart_io::open_regular(directory, name, policy.artifact, policy.open_phase) - .map_err(artifact_error)?; - if length > policy.maximum_length { - return Err(FilesystemInventoryFileError::Artifact(Box::new( - CatalogRestartError::Length { - artifact: policy.artifact, - minimum: 0, - maximum: policy.maximum_length, - observed: length, - }, - ))); - } - let identity = FileIdentity::read(&file, policy.read_phase).map_err(artifact_error)?; - let retained = file - .try_clone() - .map_err(|source| CatalogRestartError::io(policy.read_phase, source)) - .map_err(artifact_error)?; - let encoded = catalog_restart_io::read_exact(file, policy.artifact, policy.read_phase, length) - .map_err(artifact_error)?; - before_verify(); - verify(directory, name, &retained, identity, policy.read_phase)?; - Ok(encoded) -} - -fn verify( - directory: &Dir, - name: &str, - file: &File, - identity: FileIdentity, - phase: CatalogRestartPhase, -) -> Result<(), FilesystemInventoryFileError> { - let handle = FileIdentity::read(file, phase).map_err(artifact_error)?; - let metadata = directory - .symlink_metadata(name) - .map_err(|source| CatalogRestartError::io(phase, source)) - .map_err(artifact_error)?; - let current = FileIdentity::from(&metadata); - if metadata.is_file() && handle == identity && current == identity { - Ok(()) - } else { - Err(FilesystemInventoryFileError::Changed) - } -} - -fn artifact_error(source: CatalogRestartError) -> FilesystemInventoryFileError { - FilesystemInventoryFileError::Artifact(Box::new(source)) -} diff --git a/src/adapters/store_migration/filesystem_inventory_file_tests.rs b/src/adapters/store_migration/filesystem_inventory_file_tests.rs deleted file mode 100644 index 57359501..00000000 --- a/src/adapters/store_migration/filesystem_inventory_file_tests.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! Identity-stable migration artifact read laws. - -use std::error::Error; -use std::fs; -use std::io; - -use cap_std::ambient_authority; -use cap_std::fs::Dir; - -use super::filesystem_inventory_file::{ - self, FilesystemInventoryFileError, FilesystemInventoryFilePolicy, -}; -use crate::adapters::filesystem_test_sandbox::TestDirectory; -use crate::adapters::{CatalogRestartArtifact, CatalogRestartPhase}; - -#[test] -fn replacement_after_read_refuses_the_opened_artifact() -> Result<(), Box> { - let sandbox = TestDirectory::create("migration-artifact-replacement")?; - let name = "artifact"; - fs::write(sandbox.path().join(name), b"old bytes")?; - let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; - - let result = filesystem_inventory_file::read_with(&directory, name, policy(9), || { - replace(sandbox.path(), name); - }); - assert!(matches!(result, Err(FilesystemInventoryFileError::Changed))); - drop(directory); - sandbox.remove()?; - Ok(()) -} - -fn replace(root: &std::path::Path, name: &str) { - let renamed = fs::rename(root.join(name), root.join("replaced")); - let written = fs::write(root.join(name), b"old bytes"); - assert!(renamed.is_ok(), "test replacement rename failed"); - assert!(written.is_ok(), "test replacement write failed"); -} - -#[test] -fn regular_file_read_returns_exact_bytes() -> Result<(), Box> { - let sandbox = TestDirectory::create("migration-artifact-read")?; - let name = "artifact"; - fs::write(sandbox.path().join(name), b"exact")?; - let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; - - let bytes = filesystem_inventory_file::read(&directory, name, policy(5)).map_err(file_error)?; - assert_eq!(bytes, b"exact"); - drop(directory); - sandbox.remove()?; - Ok(()) -} - -fn file_error(error: FilesystemInventoryFileError) -> io::Error { - match error { - FilesystemInventoryFileError::Artifact(source) => io::Error::other(source), - FilesystemInventoryFileError::Changed => io::Error::other("artifact changed"), - } -} - -const fn policy(maximum_length: u64) -> FilesystemInventoryFilePolicy { - FilesystemInventoryFilePolicy::new( - CatalogRestartArtifact::Catalog, - CatalogRestartPhase::OpenCatalog, - CatalogRestartPhase::ReadCatalog, - maximum_length, - ) -} diff --git a/src/adapters/store_migration/filesystem_inventory_names.rs b/src/adapters/store_migration/filesystem_inventory_names.rs deleted file mode 100644 index ce76dcdc..00000000 --- a/src/adapters/store_migration/filesystem_inventory_names.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! This module owns deterministic bounded migration pool-name scans. - -use cap_std::fs::Dir; - -use super::filesystem_inventory_error::{ - FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, - MigrationInventoryNamespace, MigrationInventoryPool, -}; -use crate::adapters::{RecoveryEntryName, filesystem_recovery_inventory_scan}; - -pub(super) fn read( - directory: &Dir, - pool: MigrationInventoryPool, - remaining: u32, -) -> Result, FilesystemMigrationInventoryError> { - let expected = - filesystem_recovery_inventory_scan::count_entries(directory, u64::from(remaining)) - .map_err(|source| FilesystemMigrationInventoryError::Io { - namespace: MigrationInventoryNamespace::from(pool), - operation: FilesystemMigrationInventoryOperation::CountEntries, - source, - })?; - if expected > u64::from(remaining) { - return Err(FilesystemMigrationInventoryError::EntryLimitExceeded { - pool, - maximum: remaining, - observed_at_least: expected, - }); - } - let mut names = filesystem_recovery_inventory_scan::read_entry_names(directory, expected) - .map_err(|source| FilesystemMigrationInventoryError::Io { - namespace: MigrationInventoryNamespace::from(pool), - operation: FilesystemMigrationInventoryOperation::ReadEntryNames, - source, - })?; - let observed = u64::try_from(names.len()) - .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool })?; - if observed != expected { - return Err(FilesystemMigrationInventoryError::EntryCountChanged { - pool, - expected, - observed, - }); - } - names.sort_unstable(); - Ok(names) -} - -pub(super) fn verify( - directory: &Dir, - pool: MigrationInventoryPool, - remaining: u32, - expected: &[RecoveryEntryName], -) -> Result<(), FilesystemMigrationInventoryError> { - let observed = read(directory, pool, remaining)?; - if observed == expected { - Ok(()) - } else { - Err(FilesystemMigrationInventoryError::EntriesChanged { pool }) - } -} diff --git a/src/adapters/store_migration/filesystem_inventory_names_tests.rs b/src/adapters/store_migration/filesystem_inventory_names_tests.rs deleted file mode 100644 index 729b0cc0..00000000 --- a/src/adapters/store_migration/filesystem_inventory_names_tests.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! Migration pool-name revalidation laws. - -use std::error::Error; -use std::fs; - -use cap_std::ambient_authority; -use cap_std::fs::Dir; - -use super::filesystem_inventory_error::{ - FilesystemMigrationInventoryError, MigrationInventoryPool, -}; -use super::filesystem_inventory_names; -use crate::adapters::filesystem_test_sandbox::TestDirectory; - -#[test] -fn membership_change_after_scan_refuses_revalidation() -> Result<(), Box> { - let sandbox = TestDirectory::create("migration-name-revalidation")?; - fs::write(sandbox.path().join("first"), b"one")?; - let directory = Dir::open_ambient_dir(sandbox.path(), ambient_authority())?; - let expected = - filesystem_inventory_names::read(&directory, MigrationInventoryPool::Segments, 2)?; - fs::write(sandbox.path().join("second"), b"two")?; - - let error = filesystem_inventory_names::verify( - &directory, - MigrationInventoryPool::Segments, - 2, - &expected, - ) - .err() - .ok_or("changed membership unexpectedly passed")?; - assert!(matches!( - error, - FilesystemMigrationInventoryError::EntriesChanged { - pool: MigrationInventoryPool::Segments - } - )); - drop(directory); - sandbox.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_inventory_reader.rs b/src/adapters/store_migration/filesystem_inventory_reader.rs deleted file mode 100644 index 2519a9ce..00000000 --- a/src/adapters/store_migration/filesystem_inventory_reader.rs +++ /dev/null @@ -1,196 +0,0 @@ -//! This module owns writer-locked filesystem migration pool inventory. - -use cap_std::fs::Dir; - -use super::filesystem_inventory_catalogs; -use super::filesystem_inventory_catalogs::FilesystemMigrationCatalogInventory; -use super::filesystem_inventory_directory::PinnedMigrationPoolDirectory; -use super::filesystem_inventory_error::{ - FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, - MigrationInventoryNamespace, MigrationInventoryPool, -}; -use super::filesystem_inventory_segments; -use super::filesystem_inventory_segments::FilesystemMigrationSegmentInventory; -use super::{ - ImmutablePoolInventoryDigest, StoreMigrationInventoryEntryCount, StoreMigrationInventoryHasher, -}; -use crate::adapters::filesystem_root_identity::FilesystemRootIdentity; -use crate::adapters::{FilesystemPlatformAdmission, FilesystemWriterLock, SegmentReadPolicy}; - -const SEGMENTS_NAME: &str = "segments"; -const CATALOGS_NAME: &str = "catalogs"; - -/// Writer-authorized reader for one exact version-1 immutable-pool inventory. -/// -/// The reader retains the writer lock and pinned root, segment-pool, and -/// catalog-pool capabilities. It performs no protocol mutation. -#[must_use] -pub struct FilesystemStoreMigrationInventoryReader { - root: Dir, - segments: PinnedMigrationPoolDirectory, - catalogs: PinnedMigrationPoolDirectory, - policy: SegmentReadPolicy, - root_identity: FilesystemRootIdentity, - _lock: FilesystemWriterLock, -} - -impl FilesystemStoreMigrationInventoryReader { - /// Pins both immutable pools under admitted exclusive writer authority. - /// - /// The synchronous call performs bounded capability-relative filesystem - /// I/O and allocates no content-sized memory. - /// - /// # Errors - /// - /// Returns [`FilesystemMigrationInventoryError`] when the root capability - /// cannot be cloned or either canonical pool is missing, linked, replaced, - /// or not a directory. - pub fn open( - admission: FilesystemPlatformAdmission, - policy: SegmentReadPolicy, - ) -> Result { - let (lock, root_identity) = admission.into_parts(); - Self::open_locked(lock, root_identity, policy) - } - - /// Pins both immutable pools under an already-held writer lock. - /// - /// Migration recovery uses this so a root carrying migration residue never - /// mints a version-1 platform admission the version-1 publisher could - /// consume. - pub(super) fn open_locked( - lock: FilesystemWriterLock, - root_identity: FilesystemRootIdentity, - policy: SegmentReadPolicy, - ) -> Result { - let root = - lock.clone_directory() - .map_err(|source| FilesystemMigrationInventoryError::Io { - namespace: MigrationInventoryNamespace::Root, - operation: FilesystemMigrationInventoryOperation::CloneRoot, - source, - })?; - let segments = PinnedMigrationPoolDirectory::open( - &root, - MigrationInventoryPool::Segments, - SEGMENTS_NAME, - )?; - let catalogs = PinnedMigrationPoolDirectory::open( - &root, - MigrationInventoryPool::Catalogs, - CATALOGS_NAME, - )?; - Ok(Self { - root, - segments, - catalogs, - policy, - root_identity, - _lock: lock, - }) - } - - /// Derives the exact bounded canonical inventory digest. - /// - /// Every segment and catalog pool entry is named canonically, opened - /// without following links, read under the fixed format bound, verified - /// against its physical coordinate, and completely admitted. Catalog - /// record bindings reopen only referenced admitted segment coordinates, so - /// peak content allocation is bounded by one catalog and one segment. - /// - /// The synchronous call may block on filesystem I/O and performs no - /// protocol mutation. - /// - /// # Errors - /// - /// Returns [`FilesystemMigrationInventoryError`] for namespace drift, - /// count drift or overflow, noncanonical names, linked or changed - /// artifacts, malformed content, catalog binding failure, allocation - /// refusal, or canonical digest-stream refusal. - pub fn read(&self) -> Result { - self.verify_directories()?; - let (segments, catalogs) = self.read_pools()?; - let digest = hash_inventory(&segments, &catalogs)?; - segments.verify_names(self.segments.directory())?; - catalogs.verify_names(self.catalogs.directory())?; - self.verify_directories()?; - Ok(digest) - } - - fn read_pools( - &self, - ) -> Result< - ( - FilesystemMigrationSegmentInventory, - FilesystemMigrationCatalogInventory, - ), - FilesystemMigrationInventoryError, - > { - let maximum = StoreMigrationInventoryEntryCount::MAXIMUM; - let segments = - filesystem_inventory_segments::read(self.segments.directory(), maximum, self.policy)?; - let segment_count = host_count(segments.len(), MigrationInventoryPool::Segments)?; - let remaining = maximum - .checked_sub(segment_count) - .ok_or(FilesystemMigrationInventoryError::EntryCountArithmetic)?; - let catalogs = filesystem_inventory_catalogs::read( - self.catalogs.directory(), - self.segments.directory(), - &segments, - remaining, - self.policy, - )?; - Ok((segments, catalogs)) - } - - fn verify_directories(&self) -> Result<(), FilesystemMigrationInventoryError> { - self.segments.verify(&self.root)?; - self.catalogs.verify(&self.root) - } - - pub(super) const fn root(&self) -> &Dir { - &self.root - } - - pub(super) const fn catalogs(&self) -> &Dir { - self.catalogs.directory() - } - - pub(super) const fn root_identity(&self) -> FilesystemRootIdentity { - self.root_identity - } -} - -fn hash_inventory( - segments: &FilesystemMigrationSegmentInventory, - catalogs: &FilesystemMigrationCatalogInventory, -) -> Result { - let segment_count = host_count(segments.len(), MigrationInventoryPool::Segments)?; - let catalog_count = host_count(catalogs.len(), MigrationInventoryPool::Catalogs)?; - let total = segment_count - .checked_add(catalog_count) - .ok_or(FilesystemMigrationInventoryError::EntryCountArithmetic)?; - let count = StoreMigrationInventoryEntryCount::new(total) - .map_err(|source| FilesystemMigrationInventoryError::EntryCount { source })?; - let mut hasher = StoreMigrationInventoryHasher::new(count); - for entry in segments.entries().iter().chain(catalogs.entries()) { - hasher.push(*entry).map_err(canonical_error)?; - } - hasher.finish().map_err(canonical_error) -} - -fn canonical_error( - source: super::StoreMigrationInventoryError, -) -> FilesystemMigrationInventoryError { - FilesystemMigrationInventoryError::Canonical { - source: Box::new(source), - } -} - -fn host_count( - count: usize, - pool: MigrationInventoryPool, -) -> Result { - u32::try_from(count) - .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool }) -} diff --git a/src/adapters/store_migration/filesystem_inventory_reader_tests.rs b/src/adapters/store_migration/filesystem_inventory_reader_tests.rs deleted file mode 100644 index 50874315..00000000 --- a/src/adapters/store_migration/filesystem_inventory_reader_tests.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! Writer-locked filesystem migration inventory laws. - -use std::error::Error; -use std::fs; - -use super::filesystem_inventory_reader::FilesystemStoreMigrationInventoryReader; -use super::{FilesystemMigrationInventoryError, MigrationInventoryPool}; -use crate::adapters::filesystem_test_sandbox::TestDirectory; -use crate::adapters::test_support::decode_hex; -use crate::adapters::{ - AdmittedSegment, ChecksummedCatalog, FilesystemPlatformAdmission, physical_pool_name, -}; - -const SEGMENT_HEX: &str = - include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); -const CATALOG_HEX: &str = - include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); -const INVENTORY_DIGEST: &str = "40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9"; - -#[test] -fn writer_locked_pools_reproduce_the_frozen_inventory_digest() -> Result<(), Box> { - let sandbox = TestDirectory::create("migration-inventory-reader")?; - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - let segment_bytes = decode_hex(SEGMENT_HEX.trim())?; - let catalog_bytes = decode_hex(CATALOG_HEX.trim())?; - let policy = super::filesystem_inventory_catalogs_test_fixture::maximum_policy(); - let segment = AdmittedSegment::decode(&segment_bytes, policy)?; - let catalog = ChecksummedCatalog::decode(&catalog_bytes)?; - fs::write( - sandbox - .path() - .join("segments") - .join(physical_pool_name::segment(segment.digest())), - &segment_bytes, - )?; - fs::write( - sandbox - .path() - .join("catalogs") - .join(physical_pool_name::catalog( - catalog.generation(), - catalog.digest(), - )), - &catalog_bytes, - )?; - - let reader = FilesystemStoreMigrationInventoryReader::open(admission, policy)?; - let digest = reader.read()?; - assert_eq!(digest.as_bytes().as_slice(), decode_hex(INVENTORY_DIGEST)?); - drop(reader); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn replaced_pool_directory_refuses_before_artifact_reads() -> Result<(), Box> { - let sandbox = TestDirectory::create("migration-inventory-directory-replacement")?; - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - let policy = super::filesystem_inventory_catalogs_test_fixture::maximum_policy(); - let reader = FilesystemStoreMigrationInventoryReader::open(admission, policy)?; - fs::rename( - sandbox.path().join("segments"), - sandbox.path().join("segments-replaced"), - )?; - fs::create_dir(sandbox.path().join("segments"))?; - - let error = reader - .read() - .err() - .ok_or("replaced segment pool unexpectedly passed")?; - assert!(matches!( - error, - FilesystemMigrationInventoryError::NamespaceChanged { - pool: MigrationInventoryPool::Segments - } - )); - drop(reader); - sandbox.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_inventory_segments.rs b/src/adapters/store_migration/filesystem_inventory_segments.rs deleted file mode 100644 index 9aa77226..00000000 --- a/src/adapters/store_migration/filesystem_inventory_segments.rs +++ /dev/null @@ -1,184 +0,0 @@ -//! This module owns complete filesystem migration segment-pool admission. - -use std::collections::TryReserveError; - -use cap_std::fs::Dir; - -use super::StoreMigrationInventoryEntry; -use super::filesystem_inventory_error::{ - FilesystemMigrationInventoryError, MigrationInventoryPool, -}; -use super::filesystem_inventory_file::{ - self, FilesystemInventoryFileError, FilesystemInventoryFilePolicy, -}; -use super::filesystem_inventory_names; -use crate::adapters::recovery::recovery_pool_name; -use crate::adapters::segment_header::MAXIMUM_SEGMENT_LENGTH; -use crate::adapters::{ - AdmittedSegment, CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, - RecoveryEntryName, SegmentDigest, SegmentReadPolicy, physical_pool_name, -}; - -const POOL: MigrationInventoryPool = MigrationInventoryPool::Segments; -pub(super) struct FilesystemMigrationSegmentInventory { - entries: Vec, - digests: Vec, - names: Vec, - remaining: u32, -} - -impl FilesystemMigrationSegmentInventory { - pub(super) fn entries(&self) -> &[StoreMigrationInventoryEntry] { - &self.entries - } - - pub(super) fn contains(&self, digest: SegmentDigest) -> bool { - self.digests.binary_search(&digest).is_ok() - } - - pub(super) const fn len(&self) -> usize { - self.entries.len() - } - - pub(super) fn verify_names( - &self, - directory: &Dir, - ) -> Result<(), FilesystemMigrationInventoryError> { - filesystem_inventory_names::verify(directory, POOL, self.remaining, &self.names) - } -} - -pub(super) fn read( - directory: &Dir, - remaining: u32, - policy: SegmentReadPolicy, -) -> Result { - let names = filesystem_inventory_names::read(directory, POOL, remaining)?; - let capacity = names.len(); - let entry_count = u64::try_from(capacity) - .map_err(|_source| FilesystemMigrationInventoryError::EntryCountHostWidth { pool: POOL })?; - let mut entries = reserve(capacity, entry_count)?; - let mut digests = reserve(capacity, entry_count)?; - for name in &names { - let (entry, digest) = admit(directory, name, policy)?; - entries.push(entry); - digests.push(digest); - } - entries.sort_unstable(); - digests.sort_unstable(); - Ok(FilesystemMigrationSegmentInventory { - entries, - digests, - names, - remaining, - }) -} - -fn reserve( - capacity: usize, - entry_count: u64, -) -> Result, FilesystemMigrationInventoryError> { - let mut values = Vec::new(); - values - .try_reserve_exact(capacity) - .map_err(|source| allocation(entry_count, source))?; - Ok(values) -} - -const fn allocation( - entry_count: u64, - source: TryReserveError, -) -> FilesystemMigrationInventoryError { - FilesystemMigrationInventoryError::Allocation { - pool: POOL, - entry_count, - source, - } -} - -fn admit( - directory: &Dir, - name: &RecoveryEntryName, - policy: SegmentReadPolicy, -) -> Result<(StoreMigrationInventoryEntry, SegmentDigest), FilesystemMigrationInventoryError> { - let expected = parse_name(name)?; - let encoded = read_encoded(directory, name, expected)?; - let segment = AdmittedSegment::decode(&encoded, policy).map_err(|source| { - artifact_error( - name, - CatalogRestartError::Segment { - expected, - source: Box::new(source), - }, - ) - })?; - if segment.digest() != expected { - return Err(artifact_error( - name, - CatalogRestartError::SegmentCoordinate { - expected, - observed: segment.digest(), - }, - )); - } - Ok(( - StoreMigrationInventoryEntry::from_segment(&segment), - segment.digest(), - )) -} - -fn parse_name( - name: &RecoveryEntryName, -) -> Result { - recovery_pool_name::segment(name).map_err(|source| FilesystemMigrationInventoryError::Name { - pool: POOL, - name: name.clone(), - source, - }) -} - -fn read_encoded( - directory: &Dir, - name: &RecoveryEntryName, - expected: SegmentDigest, -) -> Result, FilesystemMigrationInventoryError> { - let canonical_name = physical_pool_name::segment(expected); - let artifact = CatalogRestartArtifact::Segment { digest: expected }; - filesystem_inventory_file::read( - directory, - &canonical_name, - FilesystemInventoryFilePolicy::new( - artifact, - CatalogRestartPhase::OpenSegment, - CatalogRestartPhase::ReadSegment, - MAXIMUM_SEGMENT_LENGTH, - ), - ) - .map_err(|source| file_error(name, source)) -} - -fn artifact_error( - name: &RecoveryEntryName, - source: CatalogRestartError, -) -> FilesystemMigrationInventoryError { - FilesystemMigrationInventoryError::Artifact { - pool: POOL, - name: name.clone(), - source: Box::new(source), - } -} - -fn file_error( - name: &RecoveryEntryName, - source: FilesystemInventoryFileError, -) -> FilesystemMigrationInventoryError { - match source { - FilesystemInventoryFileError::Artifact(source) => artifact_error(name, *source), - FilesystemInventoryFileError::Changed => { - FilesystemMigrationInventoryError::ArtifactChanged { - pool: POOL, - name: name.clone(), - } - } - } -} diff --git a/src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs b/src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs deleted file mode 100644 index 13398443..00000000 --- a/src/adapters/store_migration/filesystem_inventory_segments_refusal_tests.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! Filesystem migration segment-pool refusal laws. - -use std::error::Error; -use std::fs; -use std::io; - -use super::filesystem_inventory_error::{ - FilesystemMigrationInventoryError, MigrationInventoryPool, -}; -use super::filesystem_inventory_segments; -use super::filesystem_inventory_segments_test_fixture::{ - SegmentPoolFixture, maximum_policy, one_zero_bytes, -}; -use crate::adapters::{ - AdmittedSegment, CatalogRestartError, CatalogRestartPhase, RecoveryPoolNameError, - physical_pool_name, -}; - -#[test] -fn noncanonical_segment_name_refuses_inventory() -> Result<(), Box> { - let fixture = SegmentPoolFixture::create("migration-segment-name-refusal")?; - let bytes = one_zero_bytes()?; - let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; - let canonical = physical_pool_name::segment(segment.digest()); - let stem = canonical - .strip_suffix(".seg") - .ok_or_else(|| io::Error::other("canonical segment name lost its suffix"))?; - fixture.write_named(&format!("{}.seg", stem.to_uppercase()), &bytes)?; - - let pool = fixture.open()?; - let error = require_error(filesystem_inventory_segments::read( - &pool, - 1, - maximum_policy(), - ))?; - assert!(matches!( - error, - FilesystemMigrationInventoryError::Name { - pool: MigrationInventoryPool::Segments, - source: RecoveryPoolNameError::UppercaseDigest, - .. - } - )); - drop(pool); - fixture.remove()?; - Ok(()) -} - -#[test] -fn corrupt_segment_bytes_refuse_inventory() -> Result<(), Box> { - let fixture = SegmentPoolFixture::create("migration-segment-corruption-refusal")?; - let mut bytes = one_zero_bytes()?; - let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; - let name = physical_pool_name::segment(segment.digest()); - let first = bytes - .first_mut() - .ok_or_else(|| io::Error::other("segment fixture is empty"))?; - *first ^= u8::MAX; - fixture.write_named(&name, &bytes)?; - - let pool = fixture.open()?; - let error = require_error(filesystem_inventory_segments::read( - &pool, - 1, - maximum_policy(), - ))?; - let FilesystemMigrationInventoryError::Artifact { - pool: observed_pool, - source, - .. - } = error - else { - return Err(io::Error::other("corrupt segment returned wrong refusal").into()); - }; - assert_eq!(observed_pool, MigrationInventoryPool::Segments); - assert!(matches!( - source.as_ref(), - CatalogRestartError::Segment { .. } - )); - drop(pool); - fixture.remove()?; - Ok(()) -} - -#[cfg(unix)] -#[test] -fn linked_segment_entry_refuses_inventory() -> Result<(), Box> { - use std::os::unix::fs::symlink; - - let fixture = SegmentPoolFixture::create("migration-segment-link-refusal")?; - let bytes = one_zero_bytes()?; - let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; - let name = physical_pool_name::segment(segment.digest()); - fs::write(fixture.path().join("linked-target"), &bytes)?; - symlink("../linked-target", fixture.pool_path().join(name))?; - - let pool = fixture.open()?; - let error = require_error(filesystem_inventory_segments::read( - &pool, - 1, - maximum_policy(), - ))?; - let FilesystemMigrationInventoryError::Artifact { - pool: observed_pool, - source, - .. - } = error - else { - return Err(io::Error::other("linked segment returned wrong refusal").into()); - }; - assert_eq!(observed_pool, MigrationInventoryPool::Segments); - assert!(matches!( - source.as_ref(), - CatalogRestartError::Io { - phase: CatalogRestartPhase::OpenSegment, - .. - } - )); - drop(pool); - fixture.remove()?; - Ok(()) -} - -#[test] -fn segment_pool_above_remaining_limit_refuses_before_names() -> Result<(), Box> { - let fixture = SegmentPoolFixture::create("migration-segment-limit-refusal")?; - fixture.write_named("unknown", b"not admitted")?; - - let pool = fixture.open()?; - let error = require_error(filesystem_inventory_segments::read( - &pool, - 0, - maximum_policy(), - ))?; - assert!(matches!( - error, - FilesystemMigrationInventoryError::EntryLimitExceeded { - pool: MigrationInventoryPool::Segments, - maximum: 0, - observed_at_least: 1, - } - )); - drop(pool); - fixture.remove()?; - Ok(()) -} - -fn require_error( - result: Result, -) -> Result { - result.map_or_else(Ok, |_value| { - Err(io::Error::other( - "filesystem migration segment inventory unexpectedly succeeded", - )) - }) -} diff --git a/src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs b/src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs deleted file mode 100644 index 7a399046..00000000 --- a/src/adapters/store_migration/filesystem_inventory_segments_test_fixture.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! Deterministic filesystem migration segment-pool fixture. - -use std::error::Error; -use std::fs; -use std::path::Path; - -use cap_std::ambient_authority; -use cap_std::fs::Dir; - -use crate::LayoutEntryLimit; -use crate::adapters::filesystem_test_sandbox::TestDirectory; -use crate::adapters::test_support::decode_hex; -use crate::adapters::{ - AdmittedSegment, SegmentDigest, SegmentReadPolicy, SegmentRecordLimit, physical_pool_name, -}; - -const SEGMENT_HEX: &str = - include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); -const EMPTY_SEGMENT_HEX: &str = - include_str!("../../../conformance/segment-store/v1/empty-segment.hex"); - -pub(super) struct SegmentPoolFixture { - sandbox: TestDirectory, -} - -impl SegmentPoolFixture { - pub(super) fn create(name: &str) -> Result> { - let sandbox = TestDirectory::create(name)?; - fs::create_dir(sandbox.path().join("segments"))?; - Ok(Self { sandbox }) - } - - pub(super) fn path(&self) -> &Path { - self.sandbox.path() - } - - pub(super) fn pool_path(&self) -> std::path::PathBuf { - self.path().join("segments") - } - - pub(super) fn open(&self) -> Result> { - Ok(Dir::open_ambient_dir( - self.pool_path(), - ambient_authority(), - )?) - } - - pub(super) fn write_segment(&self, bytes: &[u8]) -> Result> { - let segment = AdmittedSegment::decode(bytes, maximum_policy())?; - fs::write( - self.pool_path() - .join(physical_pool_name::segment(segment.digest())), - bytes, - )?; - Ok(segment.digest()) - } - - pub(super) fn write_named(&self, name: &str, bytes: &[u8]) -> Result<(), Box> { - fs::write(self.pool_path().join(name), bytes)?; - Ok(()) - } - - pub(super) fn remove(self) -> Result<(), Box> { - self.sandbox.remove()?; - Ok(()) - } -} - -pub(super) fn one_zero_bytes() -> Result, Box> { - Ok(decode_hex(SEGMENT_HEX.trim())?) -} - -pub(super) fn empty_bytes() -> Result, Box> { - Ok(decode_hex(EMPTY_SEGMENT_HEX.trim())?) -} - -pub(super) const fn maximum_policy() -> SegmentReadPolicy { - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) -} diff --git a/src/adapters/store_migration/filesystem_inventory_segments_tests.rs b/src/adapters/store_migration/filesystem_inventory_segments_tests.rs deleted file mode 100644 index caa92ac7..00000000 --- a/src/adapters/store_migration/filesystem_inventory_segments_tests.rs +++ /dev/null @@ -1,36 +0,0 @@ -//! Filesystem migration segment-pool admission laws. - -use std::error::Error; - -use super::StoreMigrationInventoryEntry; -use super::filesystem_inventory_segments; -use super::filesystem_inventory_segments_test_fixture::{ - SegmentPoolFixture, empty_bytes, maximum_policy, one_zero_bytes, -}; -use crate::adapters::AdmittedSegment; - -#[test] -fn every_canonical_segment_is_admitted_into_migration_inventory() -> Result<(), Box> { - let fixture = SegmentPoolFixture::create("migration-segment-inventory")?; - let first_bytes = one_zero_bytes()?; - let second_bytes = empty_bytes()?; - let first = AdmittedSegment::decode(&first_bytes, maximum_policy())?; - let second = AdmittedSegment::decode(&second_bytes, maximum_policy())?; - let _first_digest = fixture.write_segment(&first_bytes)?; - let _second_digest = fixture.write_segment(&second_bytes)?; - - let pool = fixture.open()?; - let inventory = filesystem_inventory_segments::read(&pool, 2, maximum_policy())?; - let mut expected = [ - StoreMigrationInventoryEntry::from_segment(&first), - StoreMigrationInventoryEntry::from_segment(&second), - ]; - expected.sort_unstable(); - - assert_eq!(inventory.entries(), expected.as_slice()); - assert!(inventory.contains(first.digest())); - assert!(inventory.contains(second.digest())); - drop(pool); - fixture.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_authority.rs b/src/adapters/store_migration/filesystem_migration_authority.rs deleted file mode 100644 index 44221757..00000000 --- a/src/adapters/store_migration/filesystem_migration_authority.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! This module owns exact writer-locked filesystem migration authority. - -use cap_fs_ext::DirExt; - -use super::filesystem_inventory_file::{self, FilesystemInventoryFilePolicy}; -use super::filesystem_migration_authority_error::{ - FilesystemMigrationAuthorityArtifact as Artifact, FilesystemMigrationAuthorityError as Error, - StoreRootIdentityCoordinate as RootCoordinate, -}; -use super::filesystem_migration_authority_validation::{ - artifact_error, require_root, verify_catalog, -}; -use super::filesystem_migration_fixed_artifact::FilesystemMigrationFixedStage; -use super::migration_catalog_coordinates::MigrationCatalogCoordinates; -use super::store_root_identity::StoreRootIdentities; -use super::{CanonicalStoreMigrationIntent, FilesystemStoreMigrationInventoryReader}; -use crate::adapters::{ - CatalogRestartArtifact, CatalogRestartPhase, ChecksummedCatalog, ChecksummedPublicationHead, - FilesystemPlatformAdmission, SegmentReadPolicy, filesystem_initialization_namespace, - filesystem_platform_profile, physical_pool_name, -}; - -const HEAD_NAME: &str = "HEAD"; -const STAGING_NAME: &str = "staging"; -const HEAD_LENGTH: u64 = 128; - -/// Exclusive authority to observe and migrate one pinned version-1 filesystem root. -/// -/// The authority retains the admitted writer lock and pinned root and immutable -/// pool capabilities for its entire lifetime. Observation performs no protocol -/// mutation. When passed to [`crate::execute_store_migration`], its -/// [`crate::StoreMigrationStorage`] implementation executes only the fresh -/// forward protocol from an exactly admitted version-1 root. It retains opened -/// fixed-record handles through final verification, performs synchronous -/// capability-relative I/O, and uses neither a network nor an asynchronous -/// runtime. Reopening a partial migration prefix remains a separate recovery -/// boundary. -#[must_use] -pub struct FilesystemStoreMigrationAuthority { - inventory: FilesystemStoreMigrationInventoryReader, - pub(super) namespace: MigrationNamespacePolicy, - pub(super) fixed_stage: Option, - pub(super) published_intent: Option, - pub(super) published_marker: Option, - pub(super) published_receipt: Option, -} - -impl FilesystemStoreMigrationAuthority { - /// Pins one admitted filesystem root for migration observation. - /// - /// This synchronous constructor opens pinned directory capabilities but - /// materializes no artifact bodies and performs no protocol mutation. - /// - /// # Errors - /// - /// Returns [`FilesystemMigrationAuthorityError`](super::FilesystemMigrationAuthorityError) - /// when the root capability cannot be cloned or either immutable pool - /// cannot be pinned without following links. - pub fn open( - admission: FilesystemPlatformAdmission, - policy: SegmentReadPolicy, - ) -> Result { - let inventory = FilesystemStoreMigrationInventoryReader::open(admission, policy) - .map_err(|source| Error::Inventory { source })?; - Ok(Self::with_policy( - inventory, - MigrationNamespacePolicy::Published, - )) - } - - pub(super) const fn with_policy( - inventory: FilesystemStoreMigrationInventoryReader, - namespace: MigrationNamespacePolicy, - ) -> Self { - Self { - inventory, - namespace, - fixed_stage: None, - published_intent: None, - published_marker: None, - published_receipt: None, - } - } - - /// Observes one canonical intent from exact current version-1 authority. - /// - /// The synchronous call admits the exact published root namespace, physical - /// root coordinate, fixed-width head, complete immutable-pool inventory, - /// and head-selected catalog. Peak content allocation is bounded by one - /// catalog and one segment in addition to the bounded semantic inventory. - /// - /// # Errors - /// - /// Returns [`FilesystemMigrationAuthorityError`](super::FilesystemMigrationAuthorityError) - /// at the exact namespace, root, artifact, coordinate, or inventory refusal. - pub fn observe_intent(&self) -> Result { - self.verify_namespace()?; - let roots = self.verify_root_identity()?; - let head_bytes = self.read_head()?; - let head = ChecksummedPublicationHead::decode(&head_bytes) - .map_err(|source| Error::Head { source })?; - let inventory_digest = self - .inventory - .read() - .map_err(|source| Error::Inventory { source })?; - let coordinates = self.read_catalog(head)?; - if self.read_head()? != head_bytes { - return Err(Error::HeadChanged); - } - self.verify_namespace()?; - let _current_roots = self.verify_root_identity()?; - Ok(CanonicalStoreMigrationIntent::from_coordinates( - coordinates, - inventory_digest, - roots, - )) - } - - /// Re-observes and compares every coordinate in one canonical intent. - /// - /// This has the same synchronous I/O and bounded-allocation behavior as - /// [`Self::observe_intent`] and performs no protocol mutation. - /// - /// # Errors - /// - /// Returns the exact observation refusal or - /// [`Error::IntentChanged`] with both intent - /// digests when current authority no longer reproduces `expected`. - pub fn verify_current(&self, expected: &CanonicalStoreMigrationIntent) -> Result<(), Error> { - let observed = self.observe_intent()?; - if &observed == expected { - Ok(()) - } else { - Err(Error::IntentChanged { - expected: expected.digest(), - observed: observed.digest(), - }) - } - } - - /// Admits the exact published version-one root and requires `staging` to - /// hold nothing. - /// - /// A retained `current.seg` or `current.cat` is version-one recovery - /// evidence. Migration never inventories `staging`, and once the version-two - /// markers exist the version-one recovery constructors refuse the root, so - /// migrating over a retained stage would strand a recoverable crash state. - /// Recovery must complete before the intent is observed. - fn verify_namespace(&self) -> Result<(), Error> { - let root = self.inventory.root(); - match self.namespace { - MigrationNamespacePolicy::Published => { - filesystem_initialization_namespace::admit_published(root) - } - MigrationNamespacePolicy::Migrating => { - filesystem_initialization_namespace::admit_migrating(root) - } - } - .map_err(|source| Error::Namespace { source })?; - let staging = root - .open_dir_nofollow(STAGING_NAME) - .map_err(|source| Error::Namespace { source })?; - let mut entries = staging - .entries() - .map_err(|source| Error::Namespace { source })?; - match entries.next().transpose() { - Ok(None) => Ok(()), - Ok(Some(_entry)) => Err(Error::Namespace { - source: std::io::Error::new( - std::io::ErrorKind::InvalidData, - "version-one staging holds a retained stage; recover it before migration", - ), - }), - Err(source) => Err(Error::Namespace { source }), - } - } - - fn verify_root_identity(&self) -> Result { - let expected = self.inventory.root_identity(); - let observed = filesystem_platform_profile::root_identity(self.inventory.root()) - .map_err(|source| Error::RootIdentity { source })?; - require_root(RootCoordinate::Device, expected.device(), observed.device())?; - require_root(RootCoordinate::Mount, expected.mount(), observed.mount())?; - require_root(RootCoordinate::File, expected.file(), observed.file())?; - Ok(StoreRootIdentities::from_filesystem(observed)) - } - - fn read_head(&self) -> Result, Error> { - let policy = FilesystemInventoryFilePolicy::new( - CatalogRestartArtifact::Head, - CatalogRestartPhase::OpenHead, - CatalogRestartPhase::ReadHead, - HEAD_LENGTH, - ); - filesystem_inventory_file::read(self.inventory.root(), HEAD_NAME, policy) - .map_err(|source| artifact_error(Artifact::Head, source)) - } - - fn read_catalog( - &self, - head: ChecksummedPublicationHead<'_>, - ) -> Result { - let artifact = Artifact::Catalog { - generation: head.generation(), - digest: head.catalog_digest(), - }; - let name = physical_pool_name::catalog(head.generation(), head.catalog_digest()); - let policy = FilesystemInventoryFilePolicy::new( - CatalogRestartArtifact::Catalog, - CatalogRestartPhase::OpenCatalog, - CatalogRestartPhase::ReadCatalog, - head.catalog_length().get(), - ); - let bytes = filesystem_inventory_file::read(self.inventory.catalogs(), &name, policy) - .map_err(|source| artifact_error(artifact, source))?; - let catalog = ChecksummedCatalog::decode(&bytes).map_err(|source| Error::Catalog { - generation: head.generation(), - digest: head.catalog_digest(), - source: Box::new(source), - })?; - verify_catalog(head, catalog) - } - - pub(super) const fn root(&self) -> &cap_std::fs::Dir { - self.inventory.root() - } -} - -/// Which root namespaces an authority admits when it observes. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum MigrationNamespacePolicy { - /// Exactly the published version-1 namespace: a fresh migration. - Published, - /// The published version-1 namespace plus any migration residue: recovery. - Migrating, -} diff --git a/src/adapters/store_migration/filesystem_migration_authority_error.rs b/src/adapters/store_migration/filesystem_migration_authority_error.rs deleted file mode 100644 index 3dfb37f0..00000000 --- a/src/adapters/store_migration/filesystem_migration_authority_error.rs +++ /dev/null @@ -1,134 +0,0 @@ -//! This boundary module owns filesystem migration-authority failures. - -use std::io; - -use super::{FilesystemMigrationInventoryError, StoreMigrationIntentDigest}; -use crate::adapters::{ - CatalogDecodeError, CatalogRestartError, PublicationHeadDecodeError, WriterLockAcquireError, -}; -use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; - -/// Published version-1 artifact observed while establishing migration authority. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum FilesystemMigrationAuthorityArtifact { - /// The mutable published `HEAD`. - Head, - /// The immutable catalog selected by `HEAD`. - Catalog { - /// Catalog generation named by `HEAD`. - generation: CatalogGeneration, - /// Catalog digest named by `HEAD`. - digest: CatalogDigest, - }, -} - -/// Physical store-root coordinate compared during migration admission. -/// -/// `Device` and `File` are restart-stable and are compared on every reopen of -/// a migrated store. `Mount` names a mount instance that can change across -/// unmount, remount, and reboot; only the migrating process compares it, and -/// only against the observation it made itself. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StoreRootIdentityCoordinate { - /// Platform device coordinate. - Device, - /// Platform mount coordinate; same-process evidence only. - Mount, - /// Platform file coordinate. - File, -} - -/// Failure to observe or revalidate exact filesystem migration authority. -#[derive(Debug)] -pub enum FilesystemMigrationAuthorityError { - /// The production platform refused the store root. - Platform { - /// Preserved platform source. - source: io::Error, - }, - /// The existing writer lock could not be acquired. - WriterLock { - /// Preserved lock refusal. - source: WriterLockAcquireError, - }, - /// Complete immutable-pool inventory could not be admitted. - Inventory { - /// Preserved inventory refusal. - source: FilesystemMigrationInventoryError, - }, - /// The exact published version-1 root namespace could not be admitted. - Namespace { - /// Preserved capability-relative filesystem source. - source: io::Error, - }, - /// The physical root identity could not be observed. - RootIdentity { - /// Preserved platform source. - source: io::Error, - }, - /// One physical root coordinate changed under retained authority. - RootIdentityChanged { - /// Coordinate that changed. - coordinate: StoreRootIdentityCoordinate, - /// Coordinate retained by platform admission. - expected: u64, - /// Coordinate observed immediately before migration. - observed: u64, - }, - /// One selected artifact could not be read completely. - Artifact { - /// Exact artifact being observed. - artifact: FilesystemMigrationAuthorityArtifact, - /// Preserved bounded-read refusal. - source: Box, - }, - /// One selected artifact changed physical identity while being read. - ArtifactChanged { - /// Exact artifact that changed. - artifact: FilesystemMigrationAuthorityArtifact, - }, - /// The published head bytes were malformed. - Head { - /// Preserved head-decoding refusal. - source: PublicationHeadDecodeError, - }, - /// The selected catalog bytes were malformed. - Catalog { - /// Catalog generation selected by the head. - generation: CatalogGeneration, - /// Catalog digest selected by the head. - digest: CatalogDigest, - /// Preserved catalog-decoding refusal. - source: Box, - }, - /// Head and selected catalog generation coordinates disagreed. - CatalogGeneration { - /// Generation required by the head. - expected: CatalogGeneration, - /// Generation observed in the catalog. - observed: CatalogGeneration, - }, - /// Head and selected catalog length coordinates disagreed. - CatalogLength { - /// Length required by the head. - expected: CatalogLength, - /// Length observed in the catalog. - observed: CatalogLength, - }, - /// Head and selected catalog digest coordinates disagreed. - CatalogDigest { - /// Digest required by the head. - expected: CatalogDigest, - /// Digest observed in the catalog. - observed: CatalogDigest, - }, - /// The mutable head changed during authority observation. - HeadChanged, - /// Re-observation did not reproduce the supplied canonical intent. - IntentChanged { - /// Intent authorized by the caller. - expected: StoreMigrationIntentDigest, - /// Intent derived from current filesystem authority. - observed: StoreMigrationIntentDigest, - }, -} diff --git a/src/adapters/store_migration/filesystem_migration_authority_error_display.rs b/src/adapters/store_migration/filesystem_migration_authority_error_display.rs deleted file mode 100644 index f794346b..00000000 --- a/src/adapters/store_migration/filesystem_migration_authority_error_display.rs +++ /dev/null @@ -1,121 +0,0 @@ -//! This module owns filesystem migration-authority error presentation. - -use std::error::Error; -use std::fmt; - -use super::filesystem_migration_authority_error::{ - FilesystemMigrationAuthorityArtifact, FilesystemMigrationAuthorityError, - StoreRootIdentityCoordinate, -}; - -impl fmt::Display for FilesystemMigrationAuthorityArtifact { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Head => formatter.write_str("HEAD"), - Self::Catalog { generation, digest } => { - write!(formatter, "catalog {generation:?}/{digest:?}") - } - } - } -} - -impl fmt::Display for StoreRootIdentityCoordinate { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(match self { - Self::Device => "device", - Self::Mount => "mount", - Self::File => "file", - }) - } -} - -impl fmt::Display for FilesystemMigrationAuthorityError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Platform { .. } => { - formatter.write_str("filesystem migration platform admission was refused") - } - Self::WriterLock { .. } => { - formatter.write_str("filesystem migration writer lock was refused") - } - Self::Inventory { .. } => { - formatter.write_str("filesystem migration inventory was refused") - } - Self::Namespace { .. } => { - formatter.write_str("filesystem migration root namespace was refused") - } - Self::RootIdentity { .. } => { - formatter.write_str("filesystem migration root identity could not be observed") - } - Self::RootIdentityChanged { - coordinate, - expected, - observed, - } => write!( - formatter, - "filesystem migration root {coordinate} changed: expected {expected}, observed \ - {observed}" - ), - Self::Artifact { artifact, .. } => { - write!(formatter, "filesystem migration could not read {artifact}") - } - Self::ArtifactChanged { artifact } => { - write!( - formatter, - "filesystem migration {artifact} changed identity" - ) - } - Self::Head { .. } => formatter.write_str("filesystem migration HEAD was malformed"), - Self::Catalog { - generation, digest, .. - } => write!( - formatter, - "filesystem migration catalog {generation:?}/{digest:?} was malformed" - ), - Self::CatalogGeneration { expected, observed } => write!( - formatter, - "filesystem migration catalog generation disagreed: expected {expected:?}, \ - observed {observed:?}" - ), - Self::CatalogLength { expected, observed } => write!( - formatter, - "filesystem migration catalog length disagreed: expected {expected:?}, observed \ - {observed:?}" - ), - Self::CatalogDigest { expected, observed } => write!( - formatter, - "filesystem migration catalog digest disagreed: expected {expected:?}, observed \ - {observed:?}" - ), - Self::HeadChanged => { - formatter.write_str("filesystem migration HEAD changed during observation") - } - Self::IntentChanged { expected, observed } => write!( - formatter, - "filesystem migration intent changed: expected {expected:?}, observed {observed:?}" - ), - } - } -} - -impl Error for FilesystemMigrationAuthorityError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Inventory { source } => Some(source), - Self::WriterLock { source } => Some(source), - Self::Platform { source } - | Self::Namespace { source } - | Self::RootIdentity { source } => Some(source), - Self::Artifact { source, .. } => Some(source), - Self::Head { source } => Some(source), - Self::Catalog { source, .. } => Some(source), - Self::RootIdentityChanged { .. } - | Self::ArtifactChanged { .. } - | Self::CatalogGeneration { .. } - | Self::CatalogLength { .. } - | Self::CatalogDigest { .. } - | Self::HeadChanged - | Self::IntentChanged { .. } => None, - } - } -} diff --git a/src/adapters/store_migration/filesystem_migration_authority_tests.rs b/src/adapters/store_migration/filesystem_migration_authority_tests.rs deleted file mode 100644 index 9750f86a..00000000 --- a/src/adapters/store_migration/filesystem_migration_authority_tests.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! Writer-locked filesystem migration authority laws. - -use std::error::Error; -use std::fs; - -use super::FilesystemMigrationAuthorityError; -use crate::adapters::test_support::decode_hex; -use crate::adapters::{AdmittedSegment, physical_pool_name}; - -use super::filesystem_migration_test_fixture::{maximum_policy, open_authority}; - -const EMPTY_SEGMENT_HEX: &str = - include_str!("../../../conformance/segment-store/v1/empty-segment.hex"); -const INVENTORY_DIGEST: &str = "40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9"; - -#[test] -fn exact_published_v1_authority_constructs_and_revalidates_one_intent() -> Result<(), Box> -{ - let (sandbox, authority) = open_authority("migration-authority-current")?; - let intent = authority.observe_intent()?; - authority.verify_current(&intent)?; - - assert_eq!(intent.catalog_generation().get(), 1); - assert_eq!( - intent.inventory_digest().as_bytes().as_slice(), - decode_hex(INVENTORY_DIGEST)? - ); - drop(authority); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn immutable_pool_drift_refuses_the_retained_intent() -> Result<(), Box> { - let (sandbox, authority) = open_authority("migration-authority-inventory-drift")?; - let intent = authority.observe_intent()?; - let bytes = decode_hex(EMPTY_SEGMENT_HEX.trim())?; - let segment = AdmittedSegment::decode(&bytes, maximum_policy())?; - fs::write( - sandbox - .path() - .join("segments") - .join(physical_pool_name::segment(segment.digest())), - bytes, - )?; - - let error = authority - .verify_current(&intent) - .err() - .ok_or("changed inventory unexpectedly retained authority")?; - assert!(matches!( - error, - FilesystemMigrationAuthorityError::IntentChanged { expected, observed } - if expected == intent.digest() && observed != expected - )); - drop(authority); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn version_two_namespace_evidence_refuses_before_mutation() -> Result<(), Box> { - let (sandbox, authority) = open_authority("migration-authority-v2-evidence")?; - let intent = authority.observe_intent()?; - fs::write(sandbox.path().join("FORMAT"), [])?; - - let error = authority - .verify_current(&intent) - .err() - .ok_or("version-two evidence unexpectedly retained authority")?; - assert!(matches!( - error, - FilesystemMigrationAuthorityError::Namespace { source } - if source.kind() == std::io::ErrorKind::InvalidData - )); - drop(authority); - sandbox.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_authority_validation.rs b/src/adapters/store_migration/filesystem_migration_authority_validation.rs deleted file mode 100644 index fe68a46e..00000000 --- a/src/adapters/store_migration/filesystem_migration_authority_validation.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! This module owns migration-authority coordinate validation. - -use super::filesystem_inventory_file::FilesystemInventoryFileError; -use super::filesystem_migration_authority_error::{ - FilesystemMigrationAuthorityArtifact as Artifact, FilesystemMigrationAuthorityError as Error, - StoreRootIdentityCoordinate as RootCoordinate, -}; -use super::migration_catalog_coordinates::MigrationCatalogCoordinates; -use crate::adapters::{ChecksummedCatalog, ChecksummedPublicationHead}; - -pub(super) const fn require_root( - coordinate: RootCoordinate, - expected: u64, - observed: u64, -) -> Result<(), Error> { - if expected == observed { - Ok(()) - } else { - Err(Error::RootIdentityChanged { - coordinate, - expected, - observed, - }) - } -} - -pub(super) fn artifact_error(artifact: Artifact, source: FilesystemInventoryFileError) -> Error { - match source { - FilesystemInventoryFileError::Artifact(source) => Error::Artifact { artifact, source }, - FilesystemInventoryFileError::Changed => Error::ArtifactChanged { artifact }, - } -} - -pub(super) fn verify_catalog( - head: ChecksummedPublicationHead<'_>, - catalog: ChecksummedCatalog<'_>, -) -> Result { - require_generation(head.generation(), catalog.generation())?; - require_length(head.catalog_length(), catalog.length())?; - require_digest(head.catalog_digest(), catalog.digest())?; - Ok(MigrationCatalogCoordinates::new( - catalog.generation(), - catalog.length(), - catalog.digest(), - catalog.previous_catalog_digest(), - )) -} - -fn require_generation( - expected: crate::CatalogGeneration, - observed: crate::CatalogGeneration, -) -> Result<(), Error> { - if expected == observed { - Ok(()) - } else { - Err(Error::CatalogGeneration { expected, observed }) - } -} - -fn require_length( - expected: crate::CatalogLength, - observed: crate::CatalogLength, -) -> Result<(), Error> { - if expected == observed { - Ok(()) - } else { - Err(Error::CatalogLength { expected, observed }) - } -} - -fn require_digest( - expected: crate::CatalogDigest, - observed: crate::CatalogDigest, -) -> Result<(), Error> { - if expected == observed { - Ok(()) - } else { - Err(Error::CatalogDigest { expected, observed }) - } -} diff --git a/src/adapters/store_migration/filesystem_migration_compatibility_tests.rs b/src/adapters/store_migration/filesystem_migration_compatibility_tests.rs deleted file mode 100644 index 948a2f94..00000000 --- a/src/adapters/store_migration/filesystem_migration_compatibility_tests.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! Migration preserves version-one bytes while irrevocably refusing version-one authority. - -use super::filesystem_migration_recovery_tests::version_one_witness; -use super::filesystem_migration_test_fixture::{maximum_policy, open_authority}; -use super::migration_resumption::{MigrationRecords, execute_phase}; -use super::{StoreMigrationPhase, StoreMigrationStorage}; -use crate::adapters::{FilesystemPlatformAdmission, FilesystemPlatformAdmissionError}; -use std::{error::Error, io}; - -// Size: medium. Oracle: KEEP-MIGRATION-008 forbids v1 authority after the first -// migration effect, while every original segment/catalog/head byte is preserved. -// Delete only if the one-way migration contract disappears or stronger runtime laws subsume it. -#[test] -fn every_migration_prefix_preserves_v1_bytes_but_refuses_v1_authority() -> Result<(), Box> -{ - for count in 0..=StoreMigrationPhase::ALL.len() { - let (store, mut authority) = - open_authority(&format!("migration-v1-compatibility-{count}"))?; - let before = version_one_witness(store.path())?; - let intent = authority.observe_intent()?; - StoreMigrationStorage::verify_current(&mut authority, &intent)?; - let records = MigrationRecords::for_intent(&intent); - for phase in StoreMigrationPhase::ALL.iter().take(count) { - execute_phase(&mut authority, *phase, &records)?; - } - drop(authority); - match ( - count, - FilesystemPlatformAdmission::reopen_unchecked_for_tests(store.path()), - ) { - (0, Ok(admission)) => { - let authority = - super::FilesystemStoreMigrationAuthority::open(admission, maximum_policy())?; - authority.verify_current(&intent)?; - } - (_, Err(FilesystemPlatformAdmissionError::Namespace { source })) if count > 0 => { - assert_eq!(source.kind(), io::ErrorKind::InvalidData, "prefix {count}"); - } - (_, result) => { - return Err(format!( - "prefix {count}: incorrect v1 authority outcome: {}", - result - .err() - .map_or_else(|| "admitted".to_owned(), |error| format!("{error:?}")) - ) - .into()); - } - } - assert_eq!( - version_one_witness(store.path())?, - before, - "prefix {count}: v1 immutable bytes changed" - ); - store.remove()?; - } - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_current_recovery_tests.rs b/src/adapters/store_migration/filesystem_migration_current_recovery_tests.rs deleted file mode 100644 index a5c981e2..00000000 --- a/src/adapters/store_migration/filesystem_migration_current_recovery_tests.rs +++ /dev/null @@ -1,50 +0,0 @@ -//! This module owns recovery's current-authority validation before admission. - -use std::error::Error; -use std::fs; -use std::io; - -use super::filesystem_migration_test_fixture::{maximum_policy, open_authority}; -use super::{ - FilesystemMigrationAuthorityError, FilesystemStoreMigrationAuthority, recover_store_migration, -}; - -#[test] -fn a_corrupt_current_head_cannot_receive_a_version_one_recovery_receipt() --> Result<(), Box> { - let (sandbox, authority) = open_authority("migration-recovery-current-head")?; - let expected = authority.observe_intent()?; - drop(authority); - let path = sandbox.path().join("HEAD"); - let mut bytes = fs::read(&path)?; - *bytes.last_mut().ok_or("HEAD is empty")? ^= 1; - fs::write(&path, &bytes)?; - let mut recovered = FilesystemStoreMigrationAuthority::reopen_for_recovery_unchecked_for_tests( - sandbox.path(), - maximum_policy(), - )?; - let error = recover_store_migration(&mut recovered, &expected) - .err() - .ok_or("corrupt HEAD received a version-one recovery receipt")?; - assert!(matches!( - error, - super::StoreMigrationRecoveryError::CurrentVerification { .. } - )); - let source = error - .source() - .and_then(|source| source.downcast_ref::()) - .ok_or("current-state failure lost its I/O boundary")?; - assert!(matches!( - source - .get_ref() - .and_then(|source| source.downcast_ref::()), - Some(FilesystemMigrationAuthorityError::Head { - source: crate::adapters::PublicationHeadDecodeError::ChecksumMismatch { .. } - }) - )); - assert_eq!(fs::read(&path)?, bytes); - assert!(!sandbox.path().join("migration.intent.next").exists()); - drop(recovered); - sandbox.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs b/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs deleted file mode 100644 index 5a9f52b7..00000000 --- a/src/adapters/store_migration/filesystem_migration_fixed_artifact.rs +++ /dev/null @@ -1,256 +0,0 @@ -//! This module owns exact fixed-record migration publication. - -use std::io::{self, Write}; - -use cap_std::fs::{Dir, File}; - -use super::filesystem_migration_recovery_refusal::{ - FilesystemMigrationRecoveryRefusal as Refusal, invalid, -}; -use super::{format_marker_decoder, migration_intent_format, migration_receipt_format}; -use crate::adapters::filesystem_catalog_artifact; -use crate::adapters::filesystem_exact_record::{ - self as exact_record, EntryIdentity, ExactRecordError, -}; - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum FilesystemMigrationFixedArtifact { - Intent, - Marker, - Receipt, -} - -impl FilesystemMigrationFixedArtifact { - pub(super) const fn stage_name(self) -> &'static str { - match self { - Self::Intent => "migration.intent.next", - Self::Marker => "FORMAT.next", - Self::Receipt => "migration.receipt.next", - } - } - - pub(super) const fn canonical_name(self) -> &'static str { - match self { - Self::Intent => "migration.intent", - Self::Marker => "FORMAT", - Self::Receipt => "migration.receipt", - } - } - - pub(super) const fn encoded_length(self) -> usize { - match self { - Self::Intent => migration_intent_format::ENCODED_LENGTH, - Self::Marker => format_marker_decoder::ENCODED_LENGTH, - Self::Receipt => migration_receipt_format::ENCODED_LENGTH, - } - } -} - -pub(super) struct FilesystemMigrationFixedStage { - artifact: FilesystemMigrationFixedArtifact, - expected: Box<[u8]>, - identity: EntryIdentity, - file: File, -} - -impl FilesystemMigrationFixedStage { - pub(super) fn create( - root: &Dir, - artifact: FilesystemMigrationFixedArtifact, - expected: &[u8], - ) -> io::Result { - require_length(artifact, expected)?; - let mut file = filesystem_catalog_artifact::create_exclusive(root, artifact.stage_name())?; - let identity = EntryIdentity::of_file(&file)?; - file.write_all(expected)?; - file.flush()?; - Ok(Self { - artifact, - expected: Box::from(expected), - identity, - file, - }) - } - - /// Creates the stage exclusively and writes only `expected[..end]`, - /// leaving an unsynchronized incomplete pre-effect stage behind. The - /// handle is dropped: repository crash tasks kill the process next. - #[cfg(feature = "repository-tasks")] - pub(super) fn create_prefix( - root: &Dir, - artifact: FilesystemMigrationFixedArtifact, - expected: &[u8], - end: usize, - ) -> io::Result<()> { - require_length(artifact, expected)?; - let prefix = expected - .get(..end) - .filter(|prefix| prefix.len() < expected.len()) - .ok_or_else(|| { - invalid(Refusal::StagePrefix { - complete_length: expected.len(), - observed: end, - }) - })?; - let mut file = filesystem_catalog_artifact::create_exclusive(root, artifact.stage_name())?; - file.write_all(prefix)?; - file.flush() - } - - pub(super) fn synchronize(&self, root: &Dir) -> io::Result<()> { - self.require_handle()?; - self.file.sync_all()?; - self.verify_stage(root) - } - - pub(super) fn link( - &self, - root: &Dir, - artifact: FilesystemMigrationFixedArtifact, - expected: &[u8], - ) -> io::Result<()> { - self.require_record(artifact, expected)?; - self.verify_stage(root)?; - exact_record::link_without_replacement( - root, - self.artifact.stage_name(), - root, - self.artifact.canonical_name(), - )?; - self.verify_linked_names(root) - } - - pub(super) fn remove(self, root: &Dir) -> io::Result { - self.verify_linked_names(root)?; - root.remove_file(self.artifact.stage_name())?; - exact_record::require_absent(root, self.artifact.stage_name()).map_err(migration_error)?; - self.verify_canonical(root)?; - Ok(self) - } - - pub(super) fn verify_linked(&self, root: &Dir) -> io::Result<()> { - self.verify_linked_names(root) - } - - pub(super) fn verify_canonical(&self, root: &Dir) -> io::Result<()> { - self.require_handle()?; - verify_named_record( - root, - self.artifact.canonical_name(), - &self.expected, - self.identity, - ) - } - - pub(super) const fn artifact(&self) -> FilesystemMigrationFixedArtifact { - self.artifact - } - - fn require_handle(&self) -> io::Result<()> { - let observed = EntryIdentity::of_file(&self.file)?; - if observed == self.identity { - Ok(()) - } else { - Err(invalid_data("migration stage handle changed identity")) - } - } - - fn require_record( - &self, - artifact: FilesystemMigrationFixedArtifact, - expected: &[u8], - ) -> io::Result<()> { - if self.artifact == artifact && self.expected.as_ref() == expected { - Ok(()) - } else { - Err(invalid_data("migration stage record disagreed")) - } - } - - fn verify_stage(&self, root: &Dir) -> io::Result<()> { - verify_named_record( - root, - self.artifact.stage_name(), - &self.expected, - self.identity, - ) - } - - fn verify_linked_names(&self, root: &Dir) -> io::Result<()> { - self.verify_stage(root)?; - verify_named_record( - root, - self.artifact.canonical_name(), - &self.expected, - self.identity, - ) - } -} - -fn verify_named_record( - root: &Dir, - name: &str, - expected: &[u8], - identity: EntryIdentity, -) -> io::Result<()> { - exact_record::verify_named(root, name, expected, identity).map_err(migration_error) -} - -/// Maps a shared exact-record failure onto this protocol's refusal messages. -fn migration_error(error: ExactRecordError) -> io::Error { - error.into_io() -} - -fn require_length(artifact: FilesystemMigrationFixedArtifact, expected: &[u8]) -> io::Result<()> { - if expected.len() == artifact.encoded_length() { - Ok(()) - } else { - Err(invalid(Refusal::RecordLength { - expected: artifact.encoded_length(), - observed: expected.len(), - })) - } -} - -fn invalid_data(message: &'static str) -> io::Error { - io::Error::new(io::ErrorKind::InvalidData, message) -} - -impl FilesystemMigrationFixedStage { - /// Reopens an existing exact stage by identity for a resumed migration. - pub(super) fn reopen_stage( - root: &Dir, - artifact: FilesystemMigrationFixedArtifact, - expected: &[u8], - ) -> io::Result { - Self::reopen(root, artifact, artifact.stage_name(), expected) - } - - /// Reopens an existing exact canonical record by identity, as the - /// published handle a resumed migration verifies against. - pub(super) fn reopen_canonical( - root: &Dir, - artifact: FilesystemMigrationFixedArtifact, - expected: &[u8], - ) -> io::Result { - Self::reopen(root, artifact, artifact.canonical_name(), expected) - } - - fn reopen( - root: &Dir, - artifact: FilesystemMigrationFixedArtifact, - name: &str, - expected: &[u8], - ) -> io::Result { - require_length(artifact, expected)?; - let file = exact_record::open_regular(root, name)?; - let identity = EntryIdentity::of_file(&file)?; - verify_named_record(root, name, expected, identity)?; - Ok(Self { - artifact, - expected: Box::from(expected), - identity, - file, - }) - } -} diff --git a/src/adapters/store_migration/filesystem_migration_namespace.rs b/src/adapters/store_migration/filesystem_migration_namespace.rs deleted file mode 100644 index 0343d15a..00000000 --- a/src/adapters/store_migration/filesystem_migration_namespace.rs +++ /dev/null @@ -1,307 +0,0 @@ -//! This module owns ordered version-2 migration namespace admission. - -use std::io; - -use cap_std::fs::Dir; - -use super::filesystem_migration_namespace_directory::{ - PinnedMigrationDirectory, ambiguous, exact_membership, optional_directory, - require_allowed_membership, require_directory, require_empty, require_exact_membership, - require_regular, required_directory, -}; -use super::filesystem_migration_reader_fence; -use super::{FORMAT_MARKER_LENGTH, MIGRATION_INTENT_LENGTH, MIGRATION_RECEIPT_LENGTH}; -use crate::adapters::publication_head_decoder::ENCODED_LENGTH as PUBLICATION_HEAD_LENGTH; - -const WRITER_LOCK: &str = "writer.lock"; -const STAGING: &str = "staging"; -const SEGMENTS: &str = "segments"; -const CATALOGS: &str = "catalogs"; -const HEAD: &str = "HEAD"; -const INTENT: &str = "migration.intent"; -const READER_LOCK: &str = "reader.lock"; -const RETENTION: &str = "retention"; -const ROOTS: &str = "roots"; -const MANIFESTS: &str = "manifests"; -const GC: &str = "gc"; -const RECOVERY: &str = "recovery"; -const DISPOSITIONS: &str = "dispositions"; - -const BEFORE_READER: [&str; 6] = [WRITER_LOCK, STAGING, SEGMENTS, CATALOGS, HEAD, INTENT]; -const AFTER_READER: [&str; 7] = [ - WRITER_LOCK, - STAGING, - SEGMENTS, - CATALOGS, - HEAD, - INTENT, - READER_LOCK, -]; -const PREFIX_ROOT: [&str; 10] = [ - WRITER_LOCK, - STAGING, - SEGMENTS, - CATALOGS, - HEAD, - INTENT, - READER_LOCK, - RETENTION, - GC, - RECOVERY, -]; -const MARKER_ROOT: [&str; 11] = [ - WRITER_LOCK, - STAGING, - SEGMENTS, - CATALOGS, - HEAD, - INTENT, - READER_LOCK, - RETENTION, - GC, - RECOVERY, - "FORMAT", -]; -const RECEIPT_ROOT: [&str; 12] = [ - WRITER_LOCK, - STAGING, - SEGMENTS, - CATALOGS, - HEAD, - INTENT, - READER_LOCK, - RETENTION, - GC, - RECOVERY, - "FORMAT", - "migration.receipt", -]; - -pub(super) fn admit_reader_fence(root: &Dir) -> io::Result<()> { - let before = exact_membership(root, &BEFORE_READER)?; - let after = exact_membership(root, &AFTER_READER)?; - if !before && !after { - return Err(ambiguous("reader-fence predecessor namespace disagreed")); - } - let file = if before { - filesystem_migration_reader_fence::create(root)? - } else { - filesystem_migration_reader_fence::open(root)? - }; - filesystem_migration_reader_fence::verify(root, &file)?; - file.sync_all()?; - filesystem_migration_reader_fence::verify(root, &file)?; - verify_reader_root(root) -} - -/// The number of directories the namespace prefix admits, in order. -pub(super) const PREFIX_DIRECTORY_COUNT: usize = 6; - -pub(super) fn admit_namespace_prefix(root: &Dir) -> io::Result<()> { - preflight_prefix(root)?; - let prefix = admit_directories(root, PREFIX_DIRECTORY_COUNT)?; - let (retention, roots, manifests, gc, recovery, dispositions) = prefix - .complete() - .ok_or_else(|| ambiguous("namespace prefix admission stopped short"))?; - roots.verify(retention.directory())?; - manifests.verify(retention.directory())?; - dispositions.verify(recovery.directory())?; - retention.verify(root)?; - gc.verify(root)?; - recovery.verify(root)?; - verify_namespace_prefix(root) -} - -/// Admits only the first `count` prefix directories, in protocol order, and -/// stops without the final verification. Repository crash tasks use this to -/// leave a store at an exact directory-prefix length. -#[cfg(feature = "repository-tasks")] -pub(super) fn admit_namespace_prefix_partially(root: &Dir, count: usize) -> io::Result<()> { - if count > PREFIX_DIRECTORY_COUNT { - return Err(ambiguous("namespace prefix count exceeds the protocol")); - } - preflight_prefix(root)?; - admit_directories(root, count).map(|_prefix| ()) -} - -/// The prefix directories admitted so far, each pinned by its handle. -#[derive(Default)] -struct AdmittedPrefix { - retention: Option, - roots: Option, - manifests: Option, - gc: Option, - recovery: Option, - dispositions: Option, -} - -type CompletePrefix<'a> = ( - &'a PinnedMigrationDirectory, - &'a PinnedMigrationDirectory, - &'a PinnedMigrationDirectory, - &'a PinnedMigrationDirectory, - &'a PinnedMigrationDirectory, - &'a PinnedMigrationDirectory, -); - -impl AdmittedPrefix { - fn complete(&self) -> Option> { - Some(( - self.retention.as_ref()?, - self.roots.as_ref()?, - self.manifests.as_ref()?, - self.gc.as_ref()?, - self.recovery.as_ref()?, - self.dispositions.as_ref()?, - )) - } -} - -/// Admits the first `count` prefix directories in the normative order: -/// `retention`, `retention/roots`, `retention/manifests`, `gc`, `recovery`, -/// `recovery/dispositions`. Each admission synchronizes its parent. -fn admit_directories(root: &Dir, count: usize) -> io::Result { - let mut prefix = AdmittedPrefix::default(); - for ordinal in 0..count { - match ordinal { - 0 => prefix.retention = Some(PinnedMigrationDirectory::admit(root, RETENTION)?), - 1 => prefix.roots = Some(admit_child(prefix.retention.as_ref(), ROOTS)?), - 2 => prefix.manifests = Some(admit_child(prefix.retention.as_ref(), MANIFESTS)?), - 3 => prefix.gc = Some(PinnedMigrationDirectory::admit(root, GC)?), - 4 => prefix.recovery = Some(PinnedMigrationDirectory::admit(root, RECOVERY)?), - _ => prefix.dispositions = Some(admit_child(prefix.recovery.as_ref(), DISPOSITIONS)?), - } - } - Ok(prefix) -} - -fn admit_child( - parent: Option<&PinnedMigrationDirectory>, - name: &'static str, -) -> io::Result { - let parent = parent.ok_or_else(|| ambiguous("namespace prefix parent was not admitted"))?; - PinnedMigrationDirectory::admit(parent.directory(), name) -} - -pub(super) fn verify_intent_root(root: &Dir) -> io::Result<()> { - require_v1_and_intent(root)?; - require_exact_membership(root, &BEFORE_READER) -} - -pub(super) fn verify_reader_root(root: &Dir) -> io::Result<()> { - require_v1_and_intent(root)?; - let reader = filesystem_migration_reader_fence::open(root)?; - filesystem_migration_reader_fence::verify(root, &reader)?; - require_exact_membership(root, &AFTER_READER) -} - -pub(super) fn verify_namespace_prefix(root: &Dir) -> io::Result<()> { - verify_prefix_directories(root)?; - require_exact_membership(root, &PREFIX_ROOT) -} - -pub(super) fn verify_namespace_contents(root: &Dir) -> io::Result<()> { - verify_prefix_directories(root) -} - -pub(super) fn verify_marker_view(root: &Dir) -> io::Result<()> { - verify_marker_contents(root)?; - require_exact_membership(root, &MARKER_ROOT) -} - -pub(super) fn verify_marker_contents(root: &Dir) -> io::Result<()> { - verify_prefix_directories(root)?; - require_regular(root, "FORMAT", Some(FORMAT_MARKER_LENGTH)) -} - -pub(super) fn verify_receipt_view(root: &Dir) -> io::Result<()> { - verify_prefix_directories(root)?; - require_regular(root, "FORMAT", Some(FORMAT_MARKER_LENGTH))?; - require_regular(root, "migration.receipt", Some(MIGRATION_RECEIPT_LENGTH))?; - require_exact_membership(root, &RECEIPT_ROOT) -} - -fn preflight_prefix(root: &Dir) -> io::Result<()> { - require_allowed_membership(root, &PREFIX_ROOT)?; - require_base_namespace(root)?; - preflight_recovery_directories(root) -} - -/// Checks nested migration residue before any stage adoption or removal. -pub(super) fn preflight_recovery_directories(root: &Dir) -> io::Result<()> { - let retention = optional_directory(root, RETENTION)?; - let gc = optional_directory(root, GC)?; - let recovery = optional_directory(root, RECOVERY)?; - let retention_complete = preflight_retention(retention.as_ref())?; - if gc.is_some() && !retention_complete { - return Err(ambiguous("gc appeared before the retention prefix")); - } - if recovery.is_some() && gc.is_none() { - return Err(ambiguous("recovery appeared before the gc prefix")); - } - if let Some(directory) = gc.as_ref() { - require_empty(directory.directory())?; - } - preflight_recovery(recovery.as_ref()) -} - -fn require_base_namespace(root: &Dir) -> io::Result<()> { - require_v1_and_intent(root)?; - let reader = filesystem_migration_reader_fence::open(root)?; - filesystem_migration_reader_fence::verify(root, &reader) -} - -fn require_v1_and_intent(root: &Dir) -> io::Result<()> { - require_regular(root, WRITER_LOCK, None)?; - require_directory(root, STAGING)?; - require_directory(root, SEGMENTS)?; - require_directory(root, CATALOGS)?; - require_regular(root, HEAD, Some(PUBLICATION_HEAD_LENGTH))?; - require_regular(root, INTENT, Some(MIGRATION_INTENT_LENGTH)) -} - -fn verify_prefix_directories(root: &Dir) -> io::Result<()> { - require_base_namespace(root)?; - let retention = required_directory(root, RETENTION)?; - let roots = required_directory(retention.directory(), ROOTS)?; - let manifests = required_directory(retention.directory(), MANIFESTS)?; - let gc = required_directory(root, GC)?; - let recovery = required_directory(root, RECOVERY)?; - let dispositions = required_directory(recovery.directory(), DISPOSITIONS)?; - require_empty(roots.directory())?; - require_empty(manifests.directory())?; - require_empty(gc.directory())?; - require_empty(dispositions.directory())?; - require_exact_membership(retention.directory(), &[ROOTS, MANIFESTS])?; - require_exact_membership(recovery.directory(), &[DISPOSITIONS]) -} - -fn preflight_retention(retention: Option<&PinnedMigrationDirectory>) -> io::Result { - let Some(retention) = retention else { - return Ok(false); - }; - require_allowed_membership(retention.directory(), &[ROOTS, MANIFESTS])?; - let roots = optional_directory(retention.directory(), ROOTS)?; - let manifests = optional_directory(retention.directory(), MANIFESTS)?; - if manifests.is_some() && roots.is_none() { - return Err(ambiguous("retention manifests appeared before roots")); - } - if let Some(directory) = roots.as_ref() { - require_empty(directory.directory())?; - } - if let Some(directory) = manifests.as_ref() { - require_empty(directory.directory())?; - } - Ok(roots.is_some() && manifests.is_some()) -} - -fn preflight_recovery(recovery: Option<&PinnedMigrationDirectory>) -> io::Result<()> { - let Some(recovery) = recovery else { - return Ok(()); - }; - require_allowed_membership(recovery.directory(), &[DISPOSITIONS])?; - if let Some(dispositions) = optional_directory(recovery.directory(), DISPOSITIONS)? { - require_empty(dispositions.directory())?; - } - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_namespace_directory.rs b/src/adapters/store_migration/filesystem_migration_namespace_directory.rs deleted file mode 100644 index 297dd829..00000000 --- a/src/adapters/store_migration/filesystem_migration_namespace_directory.rs +++ /dev/null @@ -1,178 +0,0 @@ -//! This module owns pinned migration namespace-directory admission. - -use std::ffi::OsStr; -use std::io; - -use cap_std::fs::Dir; - -use crate::adapters::filesystem_exact_record::EntryIdentity; - -use crate::adapters::{ - filesystem_catalog_artifact, filesystem_platform_profile, sync_capable_directory, -}; - -pub(super) struct PinnedMigrationDirectory { - name: &'static str, - identity: EntryIdentity, - directory: Dir, -} - -impl PinnedMigrationDirectory { - pub(super) fn admit(parent: &Dir, name: &'static str) -> io::Result { - let created = match parent.create_dir(name) { - Ok(()) => true, - Err(source) if source.kind() == io::ErrorKind::AlreadyExists => false, - Err(source) => return Err(source), - }; - let directory = sync_capable_directory::open(parent, name)?; - require_same_filesystem(parent, &directory)?; - let identity = EntryIdentity::from(&directory.dir_metadata()?); - let pinned = Self { - name, - identity, - directory, - }; - pinned.verify(parent)?; - if created { - filesystem_catalog_artifact::synchronize_directory(parent)?; - } - Ok(pinned) - } - - pub(super) fn verify(&self, parent: &Dir) -> io::Result<()> { - let handle = EntryIdentity::from(&self.directory.dir_metadata()?); - let current = sync_capable_directory::open(parent, self.name)?; - require_same_filesystem(parent, ¤t)?; - let current = EntryIdentity::from(¤t.dir_metadata()?); - let metadata = parent.symlink_metadata(self.name)?; - if metadata.is_dir() - && handle == self.identity - && current == self.identity - && EntryIdentity::from(&metadata) == handle - { - Ok(()) - } else { - Err(ambiguous("migration directory changed identity")) - } - } - - pub(super) const fn directory(&self) -> &Dir { - &self.directory - } -} - -pub(super) fn optional_directory( - parent: &Dir, - name: &'static str, -) -> io::Result> { - match parent.symlink_metadata(name) { - Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(None), - Err(source) => Err(source), - Ok(metadata) if metadata.is_dir() => { - let directory = sync_capable_directory::open(parent, name)?; - require_same_filesystem(parent, &directory)?; - let pinned = PinnedMigrationDirectory { - name, - identity: EntryIdentity::from(&metadata), - directory, - }; - pinned.verify(parent)?; - Ok(Some(pinned)) - } - Ok(_) => Err(ambiguous("migration namespace entry has the wrong kind")), - } -} - -fn require_same_filesystem(parent: &Dir, child: &Dir) -> io::Result<()> { - let parent = filesystem_platform_profile::root_identity(parent)?; - let child = filesystem_platform_profile::root_identity(child)?; - if parent.device() == child.device() && parent.mount() == child.mount() { - Ok(()) - } else { - Err(ambiguous( - "migration namespace crossed the admitted filesystem or mount", - )) - } -} - -pub(super) fn required_directory( - parent: &Dir, - name: &'static str, -) -> io::Result { - optional_directory(parent, name)? - .ok_or_else(|| ambiguous("required migration namespace directory was absent")) -} - -pub(super) fn require_directory(parent: &Dir, name: &str) -> io::Result<()> { - if parent.symlink_metadata(name)?.is_dir() { - Ok(()) - } else { - Err(ambiguous("required migration directory has the wrong kind")) - } -} - -pub(super) fn require_regular(parent: &Dir, name: &str, length: Option) -> io::Result<()> { - let metadata = parent.symlink_metadata(name)?; - let expected = length - .map(u64::try_from) - .transpose() - .map_err(|_source| ambiguous("required migration file length exceeded u64"))?; - if metadata.is_file() && expected.is_none_or(|expected| metadata.len() == expected) { - Ok(()) - } else { - Err(ambiguous( - "required migration file has the wrong kind or length", - )) - } -} - -pub(super) fn require_empty(directory: &Dir) -> io::Result<()> { - let mut entries = directory.entries()?; - if entries.next().transpose()?.is_none() { - Ok(()) - } else { - Err(ambiguous("new migration namespace was not empty")) - } -} - -pub(super) fn require_exact_membership(directory: &Dir, expected: &[&str]) -> io::Result<()> { - if exact_membership(directory, expected)? { - Ok(()) - } else { - Err(ambiguous("migration namespace membership disagreed")) - } -} - -pub(super) fn exact_membership(directory: &Dir, expected: &[&str]) -> io::Result { - let mut observed = Vec::with_capacity(expected.len()); - for entry in directory.entries()? { - if observed.len() == expected.len() { - return Ok(false); - } - observed.push(entry?.file_name()); - } - Ok(observed.len() == expected.len() - && observed.iter().all(|name| { - expected - .iter() - .any(|candidate| name == OsStr::new(candidate)) - })) -} - -pub(super) fn require_allowed_membership(directory: &Dir, allowed: &[&str]) -> io::Result<()> { - let mut observed = 0_usize; - for entry in directory.entries()? { - observed = observed - .checked_add(1) - .ok_or_else(|| ambiguous("migration namespace count overflowed"))?; - let name = entry?.file_name(); - if observed > allowed.len() || !allowed.iter().any(|candidate| name == *candidate) { - return Err(ambiguous("migration namespace contains an unknown entry")); - } - } - Ok(()) -} - -pub(super) fn ambiguous(message: &'static str) -> io::Error { - io::Error::new(io::ErrorKind::InvalidData, message) -} diff --git a/src/adapters/store_migration/filesystem_migration_pair_admission_tests.rs b/src/adapters/store_migration/filesystem_migration_pair_admission_tests.rs deleted file mode 100644 index e6be821c..00000000 --- a/src/adapters/store_migration/filesystem_migration_pair_admission_tests.rs +++ /dev/null @@ -1,66 +0,0 @@ -//! This module owns refusal of substituted stage/canonical pairs before resumption. - -use std::error::Error; -use std::fs; - -use super::filesystem_migration_recovery_tests::version_one_witness; -use super::filesystem_migration_test_fixture::{maximum_policy, open_authority}; -use super::migration_resumption::{MigrationRecords, execute_phase}; -use super::{ - FilesystemStoreMigrationAuthority, StoreMigrationPhase, StoreMigrationRecoveryError, - StoreMigrationStorage, recover_store_migration, -}; -use crate::adapters::filesystem_exact_record::{ExactRecordError, ExactRecordRefusal}; - -#[test] -fn substituted_canonical_pairs_refuse_during_adoption_before_forward_execution() --> Result<(), Box> { - for (count, canonical, stage) in [ - (3, "migration.intent", "migration.intent.next"), - (12, "FORMAT", "FORMAT.next"), - (18, "migration.receipt", "migration.receipt.next"), - ] { - refuse_pair(count, canonical, stage)?; - } - Ok(()) -} - -fn refuse_pair(count: usize, canonical: &str, stage: &str) -> Result<(), Box> { - let (sandbox, mut authority) = open_authority("migration-pair-admission")?; - let intent = authority.observe_intent()?; - StoreMigrationStorage::verify_current(&mut authority, &intent)?; - let records = MigrationRecords::for_intent(&intent); - for phase in StoreMigrationPhase::ALL.iter().take(count) { - execute_phase(&mut authority, *phase, &records)?; - } - drop(authority); - let bytes = fs::read(sandbox.path().join(canonical))?; - fs::remove_file(sandbox.path().join(canonical))?; - fs::write(sandbox.path().join(canonical), &bytes)?; - let witness = version_one_witness(sandbox.path())?; - let mut recovered = FilesystemStoreMigrationAuthority::reopen_for_recovery_unchecked_for_tests( - sandbox.path(), - maximum_policy(), - )?; - let expected = recovered.observe_intent()?; - let error = recover_store_migration(&mut recovered, &expected) - .err() - .ok_or("a substituted pair was admitted")?; - let StoreMigrationRecoveryError::Adoption { source } = error else { - return Err(format!("{canonical}: substitution reached a forward phase: {error}").into()); - }; - assert!(matches!( - source - .get_ref() - .and_then(|error| error.downcast_ref::()), - Some(ExactRecordError::Refused( - ExactRecordRefusal::KindLengthOrIdentity - )) - )); - assert_eq!(fs::read(sandbox.path().join(canonical))?, bytes); - assert_eq!(fs::read(sandbox.path().join(stage))?, bytes); - assert_eq!(version_one_witness(sandbox.path())?, witness); - drop(recovered); - sandbox.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_reader_fence.rs b/src/adapters/store_migration/filesystem_migration_reader_fence.rs deleted file mode 100644 index 7465fb12..00000000 --- a/src/adapters/store_migration/filesystem_migration_reader_fence.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! This module owns persistent migration reader-fence admission. - -use std::io; - -use cap_fs_ext::{FollowSymlinks, MetadataExt, OpenOptionsFollowExt, OpenOptionsSyncExt}; -use cap_std::fs::{Dir, File, Metadata, OpenOptions}; - -use super::filesystem_migration_namespace_directory::ambiguous; - -const READER_LOCK: &str = "reader.lock"; - -pub(super) fn create(root: &Dir) -> io::Result { - let mut options = options(); - options.create_new(true); - root.open_with(READER_LOCK, &options) -} - -pub(super) fn open(root: &Dir) -> io::Result { - root.open_with(READER_LOCK, &options()) -} - -pub(super) fn verify(root: &Dir, file: &File) -> io::Result<()> { - let handle = file.metadata()?; - let entry = root.symlink_metadata(READER_LOCK)?; - if handle.is_file() - && entry.is_file() - && handle.len() == 0 - && entry.len() == 0 - && FileIdentity::from(&handle) == FileIdentity::from(&entry) - { - Ok(()) - } else { - Err(ambiguous( - "reader fence kind, length, or identity disagreed", - )) - } -} - -fn options() -> OpenOptions { - let mut options = OpenOptions::new(); - options - .read(true) - .write(true) - .follow(FollowSymlinks::No) - .nonblock(true); - options -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct FileIdentity { - device: u64, - inode: u64, -} - -impl From<&Metadata> for FileIdentity { - fn from(metadata: &Metadata) -> Self { - Self { - device: metadata.dev(), - inode: metadata.ino(), - } - } -} diff --git a/src/adapters/store_migration/filesystem_migration_receipt_evidence_tests.rs b/src/adapters/store_migration/filesystem_migration_receipt_evidence_tests.rs deleted file mode 100644 index 355bde91..00000000 --- a/src/adapters/store_migration/filesystem_migration_receipt_evidence_tests.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! This module owns the law that recovery receipts preserve observed evidence. - -use std::error::Error; -use std::fs; - -use super::filesystem_migration_test_fixture::{maximum_policy, open_authority}; -use super::migration_resumption::{MigrationRecords, execute_phase}; -use super::{ - FilesystemStoreMigrationAuthority, StoreMigrationPhase, StoreMigrationStorage, - recover_store_migration, -}; - -const DIRECTORIES: [&str; 6] = [ - "retention", - "retention/roots", - "retention/manifests", - "gc", - "recovery", - "recovery/dispositions", -]; - -#[test] -fn recovery_receipts_distinguish_each_observed_namespace_prefix() -> Result<(), Box> { - let (sandbox, mut authority) = open_authority("migration-receipt-prefix-evidence")?; - let intent = authority.observe_intent()?; - StoreMigrationStorage::verify_current(&mut authority, &intent)?; - let records = MigrationRecords::for_intent(&intent); - for phase in StoreMigrationPhase::ALL.iter().take(7) { - execute_phase(&mut authority, *phase, &records)?; - } - drop(authority); - let mut previous = None; - for count in 0..=DIRECTORIES.len() { - for name in DIRECTORIES.iter().take(count) { - fs::create_dir(sandbox.path().join(name))?; - } - let mut recovered = - FilesystemStoreMigrationAuthority::reopen_for_recovery_unchecked_for_tests( - sandbox.path(), - maximum_policy(), - )?; - let expected = recovered.observe_intent()?; - let receipt = recover_store_migration(&mut recovered, &expected)?; - assert_eq!(receipt.intent_digest(), Some(intent.digest())); - assert_eq!( - receipt.observed_namespace_prefix().len(), - count.checked_add(1).ok_or("prefix overflow")? - ); - let names: Vec<_> = std::iter::once("reader.lock") - .chain(DIRECTORIES.into_iter().take(count)) - .collect(); - assert_eq!( - receipt - .observed_namespace_prefix() - .names() - .collect::>(), - names - ); - if let Some(previous) = previous { - assert_ne!( - receipt, previous, - "distinct observed prefixes collapsed at {count}" - ); - } - previous = Some(receipt); - let complete = recover_store_migration(&mut recovered, &expected)?; - assert_eq!(complete.intent_digest(), Some(intent.digest())); - assert_eq!(complete.observed_namespace_prefix().len(), 7); - assert_eq!(complete.executed_phases().count(), 0); - drop(recovered); - fs::remove_file(sandbox.path().join("FORMAT"))?; - fs::remove_file(sandbox.path().join("migration.receipt"))?; - for name in DIRECTORIES.iter().rev() { - fs::remove_dir(sandbox.path().join(name))?; - } - } - sandbox.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_recovery.rs b/src/adapters/store_migration/filesystem_migration_recovery.rs deleted file mode 100644 index 23151f43..00000000 --- a/src/adapters/store_migration/filesystem_migration_recovery.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! This module binds filesystem migration authority to the recovery port. - -use std::io; -use std::path::Path; - -use cap_std::fs::Dir; - -use super::FilesystemMigrationRecoveryRefusal as Refusal; -use super::filesystem_migration_authority::MigrationNamespacePolicy; -use super::filesystem_migration_authority_error::FilesystemMigrationAuthorityError as Error; -use super::filesystem_migration_fixed_artifact::{ - FilesystemMigrationFixedArtifact as FixedArtifact, FilesystemMigrationFixedStage, -}; -use super::filesystem_migration_recovery_refusal::{invalid, kind}; -use super::migration_resumption::MigrationRecords; -use super::{ - CanonicalStoreMigrationIntent, FilesystemStoreMigrationAuthority, - FilesystemStoreMigrationInventoryReader, StoreMigrationFixedStage, - StoreMigrationRecoveryStorage, StoreMigrationResidue, filesystem_migration_residue, -}; -use crate::adapters::filesystem_exact_record::{self as exact_record, ExactRecordError}; -use crate::adapters::{ - FilesystemWriterLock, SegmentReadPolicy, filesystem_catalog_artifact, - filesystem_initialization_namespace, filesystem_platform_profile, -}; - -impl FilesystemStoreMigrationAuthority { - /// Reacquires writer authority over a root that may carry migration residue. - /// - /// The call admits the production platform, acquires the existing writer - /// lock, and admits the published version-1 namespace plus any subset of - /// migration records, stages, the reader fence, and protocol directories. - /// It never produces a version-1 platform admission, so the version-1 - /// publisher can never run against a partly migrated root. The returned - /// authority observes with the migrating namespace policy and implements - /// [`StoreMigrationRecoveryStorage`]. It mutates nothing. - /// - /// # Errors - /// - /// Returns [`FilesystemMigrationAuthorityError`](Error) at the exact - /// platform, writer-lock, namespace, or pool refusal. - pub fn reopen_for_recovery( - store_root: &Path, - policy: SegmentReadPolicy, - ) -> Result { - let root = filesystem_platform_profile::open(store_root) - .map_err(|source| Error::Platform { source })?; - Self::recover_root(root, policy) - } - - #[cfg(test)] - pub(super) fn reopen_for_recovery_unchecked_for_tests( - store_root: &Path, - policy: SegmentReadPolicy, - ) -> Result { - let root = Dir::open_ambient_dir(store_root, cap_std::ambient_authority()) - .map_err(|source| Error::Platform { source })?; - Self::recover_root(root, policy) - } - - pub(super) fn recover_root(root: Dir, policy: SegmentReadPolicy) -> Result { - let lock = FilesystemWriterLock::try_acquire_in(root) - .map_err(|source| Error::WriterLock { source })?; - let directory = lock - .clone_directory() - .map_err(|source| Error::Namespace { source })?; - filesystem_initialization_namespace::admit_migrating(&directory) - .map_err(|source| Error::Namespace { source })?; - let root_identity = filesystem_platform_profile::root_identity(&directory) - .map_err(|source| Error::RootIdentity { source })?; - let inventory = - FilesystemStoreMigrationInventoryReader::open_locked(lock, root_identity, policy) - .map_err(|source| Error::Inventory { source })?; - Ok(Self::with_policy( - inventory, - MigrationNamespacePolicy::Migrating, - )) - } -} - -impl StoreMigrationRecoveryStorage for FilesystemStoreMigrationAuthority { - fn observe_residue(&mut self) -> io::Result { - filesystem_migration_residue::observe(self.root()) - } - - fn verify_complete(&mut self) -> io::Result<()> { - filesystem_initialization_namespace::admit_version_two(self.root()) - .map_err(|source| io::Error::new(source.kind(), Refusal::NamespacePreflight { source })) - } - - fn adopt_residue( - &mut self, - residue: &StoreMigrationResidue, - intent: &CanonicalStoreMigrationIntent, - ) -> io::Result<()> { - super::filesystem_migration_namespace::preflight_recovery_directories(self.root()) - .map_err(|source| { - io::Error::new(source.kind(), Refusal::NamespacePreflight { source }) - })?; - let records = MigrationRecords::for_intent(intent); - adopt_artifact( - self, - FixedArtifact::Intent, - residue.intent_stage.as_deref(), - residue.intent.as_deref(), - intent.encoded(), - )?; - adopt_artifact( - self, - FixedArtifact::Marker, - residue.marker_stage.as_deref(), - residue.marker.as_deref(), - records.marker.encoded(), - )?; - adopt_artifact( - self, - FixedArtifact::Receipt, - residue.receipt_stage.as_deref(), - residue.receipt.as_deref(), - records.receipt.encoded(), - ) - } - - fn discard_stage(&mut self, stage: StoreMigrationFixedStage) -> io::Result<()> { - let artifact = match stage { - StoreMigrationFixedStage::Intent => FixedArtifact::Intent, - StoreMigrationFixedStage::Marker => FixedArtifact::Marker, - StoreMigrationFixedStage::Receipt => FixedArtifact::Receipt, - }; - let root = self.root(); - exact_record::require_absent(root, artifact.canonical_name()).map_err(refusal)?; - let metadata = root.symlink_metadata(artifact.stage_name())?; - let complete = u64::try_from(artifact.encoded_length()).map_err(|source| { - invalid(Refusal::StageLengthOverflow { - length: artifact.encoded_length(), - source, - }) - })?; - if !metadata.is_file() || metadata.len() >= complete { - return Err(invalid(Refusal::StageNotIncomplete { - stage, - complete_length: complete, - observed_length: metadata.len(), - observed_kind: kind(&metadata), - })); - } - root.remove_file(artifact.stage_name())?; - exact_record::require_absent(root, artifact.stage_name()).map_err(refusal)?; - filesystem_catalog_artifact::synchronize_directory(root) - } -} - -/// Reopens the handles one artifact's residue implies: an exact stage becomes -/// the active stage (shared with its canonical target when both exist), a -/// canonical record alone becomes the published handle, and an incomplete -/// stage is left for the planner's discard. -fn adopt_artifact( - authority: &mut FilesystemStoreMigrationAuthority, - artifact: FixedArtifact, - stage: Option<&[u8]>, - canonical: Option<&[u8]>, - expected: &[u8], -) -> io::Result<()> { - let root = authority.root(); - if stage.is_some_and(|bytes| bytes == expected) { - if authority.fixed_stage.is_some() { - return Err(invalid(Refusal::MultipleExactStages)); - } - let reopened = FilesystemMigrationFixedStage::reopen_stage(root, artifact, expected)?; - if canonical.is_some() { - reopened.verify_linked(root)?; - } - authority.fixed_stage = Some(reopened); - return Ok(()); - } - if canonical.is_some() { - let reopened = FilesystemMigrationFixedStage::reopen_canonical(root, artifact, expected)?; - match artifact { - FixedArtifact::Intent => authority.published_intent = Some(reopened), - FixedArtifact::Marker => authority.published_marker = Some(reopened), - FixedArtifact::Receipt => authority.published_receipt = Some(reopened), - } - } - Ok(()) -} - -fn refusal(error: ExactRecordError) -> io::Error { - error.into_io() -} diff --git a/src/adapters/store_migration/filesystem_migration_recovery_refusal.rs b/src/adapters/store_migration/filesystem_migration_recovery_refusal.rs deleted file mode 100644 index 8da6e67c..00000000 --- a/src/adapters/store_migration/filesystem_migration_recovery_refusal.rs +++ /dev/null @@ -1,110 +0,0 @@ -//! This module owns typed filesystem migration recovery refusals. - -use std::error::Error; -use std::fmt; -use std::num::TryFromIntError; - -use super::StoreMigrationFixedStage; - -/// Kind observed at a fixed migration protocol name without following links. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum FilesystemMigrationResidueKind { - /// A regular file. - RegularFile, - /// A directory. - Directory, - /// A link, device, pipe, or another unsupported kind. - Other, -} - -/// Semantic failure retained as an I/O payload during migration recovery. -#[derive(Debug)] -#[non_exhaustive] -pub enum FilesystemMigrationRecoveryRefusal { - /// Nested migration directories contain invalid or out-of-order residue. - NamespacePreflight { - /// The original filesystem or namespace failure. - source: std::io::Error, - }, - /// The record's bounded observation length cannot be represented. - ResidueBoundOverflow { - /// The canonical record bound. - length: usize, - }, - /// A persistent reader fence is not an empty regular file. - ReaderFence { - /// The observed entry kind; an empty regular file is required. - observed_kind: FilesystemMigrationResidueKind, - /// The observed length; zero is required. - observed_length: u64, - }, - /// A directory-prefix name is not a directory. - NamespaceKind { - /// The observed kind. - observed_kind: FilesystemMigrationResidueKind, - }, - /// The canonical stage bound does not fit the filesystem length. - StageLengthOverflow { - /// The bound that failed conversion. - length: usize, - /// The original conversion failure. - source: TryFromIntError, - }, - /// Only an incomplete regular pre-effect stage can be discarded. - StageNotIncomplete { - /// The stage being considered. - stage: StoreMigrationFixedStage, - /// The first complete length, excluded from discard. - complete_length: u64, - /// The observed entry length. - observed_length: u64, - /// The observed entry kind. - observed_kind: FilesystemMigrationResidueKind, - }, - /// More than one complete fixed stage would be adopted simultaneously. - MultipleExactStages, - /// A crash hook requested a non-strict byte prefix. - StagePrefix { - /// The complete length, excluded from strict prefixes. - complete_length: usize, - /// The requested prefix length. - observed: usize, - }, - /// The supplied fixed record does not have its canonical length. - RecordLength { - /// The canonical length. - expected: usize, - /// The supplied length. - observed: usize, - }, -} - -impl fmt::Display for FilesystemMigrationRecoveryRefusal { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "migration recovery refused: {self:?}") - } -} - -impl Error for FilesystemMigrationRecoveryRefusal { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::NamespacePreflight { source } => Some(source), - Self::StageLengthOverflow { source, .. } => Some(source), - _ => None, - } - } -} - -pub(super) fn kind(metadata: &cap_std::fs::Metadata) -> FilesystemMigrationResidueKind { - if metadata.is_file() { - FilesystemMigrationResidueKind::RegularFile - } else if metadata.is_dir() { - FilesystemMigrationResidueKind::Directory - } else { - FilesystemMigrationResidueKind::Other - } -} - -pub(super) fn invalid(refusal: FilesystemMigrationRecoveryRefusal) -> std::io::Error { - std::io::Error::new(std::io::ErrorKind::InvalidData, refusal) -} diff --git a/src/adapters/store_migration/filesystem_migration_recovery_tests.rs b/src/adapters/store_migration/filesystem_migration_recovery_tests.rs deleted file mode 100644 index 95cc1c8b..00000000 --- a/src/adapters/store_migration/filesystem_migration_recovery_tests.rs +++ /dev/null @@ -1,159 +0,0 @@ -//! Filesystem migration recovery laws: every forward prefix recovers. - -use std::error::Error; -use std::fs; -use std::io; -use std::path::Path; - -use super::filesystem_migration_test_fixture::{maximum_policy, open_authority}; -use super::migration_resumption::{MigrationRecords, execute_phase}; -use super::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, - FilesystemStoreMigrationAuthority, StoreMigrationFixedStage, StoreMigrationPhase, - StoreMigrationRecoveryPlan, StoreMigrationStorage, recover_store_migration, -}; - -/// Runs the first `count` forward phases in-process, drops the writer, and -/// recovers under a fresh authority. Every prefix must end in exactly one -/// complete migration whose records admit, with every version-1 byte intact. -#[test] -fn every_forward_prefix_recovers_to_one_complete_migration() -> Result<(), Box> { - for count in 0..=StoreMigrationPhase::ALL.len() { - let name = format!("filesystem-migration-recovery-prefix-{count}"); - let (sandbox, mut authority) = open_authority(&name)?; - let intent = authority.observe_intent()?; - let witness = version_one_witness(sandbox.path())?; - StoreMigrationStorage::verify_current(&mut authority, &intent)?; - let records = MigrationRecords::for_intent(&intent); - for phase in StoreMigrationPhase::ALL.iter().take(count) { - execute_phase(&mut authority, *phase, &records)?; - } - drop(authority); - - let mut recovered = - FilesystemStoreMigrationAuthority::reopen_for_recovery_unchecked_for_tests( - sandbox.path(), - maximum_policy(), - )?; - let expected = recovered.observe_intent()?; - let receipt = recover_store_migration(&mut recovered, &expected) - .map_err(|error| format!("prefix {count}: {error}: {:?}", error.source()))?; - match receipt.plan() { - StoreMigrationRecoveryPlan::Resume { resume } - | StoreMigrationRecoveryPlan::DiscardStage { resume, .. } => { - assert_eq!(receipt.executed_phases().next(), Some(resume)); - assert_eq!( - receipt.executed_phases().last(), - Some(StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup) - ); - } - _ => assert_eq!(receipt.executed_phases().count(), 0), - } - drop(recovered); - - match receipt.plan() { - StoreMigrationRecoveryPlan::VersionOne => { - assert_eq!(count, 0, "only an untouched store admits version one"); - assert!(!sandbox.path().join("migration.intent").exists()); - } - StoreMigrationRecoveryPlan::Complete => { - // The receipt is canonical from phase 20 on; the final root - // synchronization leaves nothing a residue can observe. - assert!(count >= 20, "prefix {count} reported complete"); - } - StoreMigrationRecoveryPlan::Resume { .. } => { - assert!(receipt.published().is_some(), "prefix {count} resumed"); - assert_complete_migration(sandbox.path(), &intent)?; - } - StoreMigrationRecoveryPlan::DiscardStage { .. } => { - return Err(format!("prefix {count} needed a discard for an exact stage").into()); - } - } - if count > 0 { - assert_complete_migration(sandbox.path(), &intent)?; - } - assert_eq!( - version_one_witness(sandbox.path())?, - witness, - "prefix {count}" - ); - sandbox.remove()?; - } - Ok(()) -} - -#[test] -fn a_truncated_intent_stage_is_discarded_and_the_migration_completes() -> Result<(), Box> -{ - let (sandbox, mut authority) = open_authority("filesystem-migration-recovery-truncated")?; - let intent = authority.observe_intent()?; - StoreMigrationStorage::verify_current(&mut authority, &intent)?; - StoreMigrationStorage::write_intent_stage(&mut authority, &intent)?; - drop(authority); - let stage = sandbox.path().join("migration.intent.next"); - let mut bytes = fs::read(&stage)?; - bytes.truncate(100); - fs::write(&stage, &bytes)?; - - let mut recovered = FilesystemStoreMigrationAuthority::reopen_for_recovery_unchecked_for_tests( - sandbox.path(), - maximum_policy(), - )?; - let expected = recovered.observe_intent()?; - let receipt = recover_store_migration(&mut recovered, &expected)?; - drop(recovered); - - assert_eq!( - receipt.plan(), - StoreMigrationRecoveryPlan::DiscardStage { - stage: StoreMigrationFixedStage::Intent, - resume: StoreMigrationPhase::WriteIntentStage, - } - ); - assert_complete_migration(sandbox.path(), &intent)?; - sandbox.remove()?; - Ok(()) -} - -pub(super) fn assert_complete_migration( - root: &Path, - intent: &super::CanonicalStoreMigrationIntent, -) -> Result<(), Box> { - let intent_bytes = fs::read(root.join("migration.intent"))?; - let marker_bytes = fs::read(root.join("FORMAT"))?; - let receipt_bytes = fs::read(root.join("migration.receipt"))?; - let admitted = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; - let marker = AdmittedStoreFormatMarker::decode(&marker_bytes)?; - let _receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &admitted, &marker)?; - assert_eq!(admitted.encoded(), intent.encoded()); - for stage in [ - "migration.intent.next", - "FORMAT.next", - "migration.receipt.next", - ] { - assert!(!root.join(stage).exists(), "{stage} survived recovery"); - } - for directory in [ - "retention/roots", - "retention/manifests", - "gc", - "recovery/dispositions", - ] { - assert!(root.join(directory).is_dir(), "{directory} is absent"); - } - assert_eq!(fs::metadata(root.join("reader.lock"))?.len(), 0); - Ok(()) -} - -pub(super) fn version_one_witness(root: &Path) -> io::Result)>> { - let mut witness = vec![("HEAD".to_owned(), fs::read(root.join("HEAD"))?)]; - for pool in ["segments", "catalogs"] { - for entry in fs::read_dir(root.join(pool))? { - let entry = entry?; - let name = format!("{pool}/{}", entry.file_name().to_string_lossy()); - witness.push((name, fs::read(entry.path())?)); - } - } - witness.sort(); - Ok(witness) -} diff --git a/src/adapters/store_migration/filesystem_migration_recovery_truncation_tests.rs b/src/adapters/store_migration/filesystem_migration_recovery_truncation_tests.rs deleted file mode 100644 index 3a1c4ec0..00000000 --- a/src/adapters/store_migration/filesystem_migration_recovery_truncation_tests.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! This module owns exhaustive strict-stage-prefix recovery and preflight laws. - -use std::error::Error; -use std::fs; - -use super::filesystem_migration_recovery_tests::{assert_complete_migration, version_one_witness}; -use super::filesystem_migration_test_fixture::{maximum_policy, open_authority}; -use super::migration_resumption::{MigrationRecords, execute_phase}; -use super::{ - FilesystemMigrationRecoveryRefusal, FilesystemStoreMigrationAuthority, - StoreMigrationFixedStage, StoreMigrationPhase, StoreMigrationRecoveryError, - StoreMigrationRecoveryPlan, StoreMigrationStorage, recover_store_migration, -}; - -#[test] -fn every_strict_fixed_stage_prefix_is_discarded_without_changing_version_one_bytes() --> Result<(), Box> { - for (stage, phase, name, length) in [ - ( - StoreMigrationFixedStage::Intent, - StoreMigrationPhase::WriteIntentStage, - "migration.intent.next", - super::MIGRATION_INTENT_LENGTH, - ), - ( - StoreMigrationFixedStage::Marker, - StoreMigrationPhase::WriteMarkerStage, - "FORMAT.next", - super::FORMAT_MARKER_LENGTH, - ), - ( - StoreMigrationFixedStage::Receipt, - StoreMigrationPhase::WriteReceiptStage, - "migration.receipt.next", - super::MIGRATION_RECEIPT_LENGTH, - ), - ] { - for length in 0..length { - recover_truncation(stage, phase, name, length)?; - } - } - Ok(()) -} - -fn recover_truncation( - stage: StoreMigrationFixedStage, - phase: StoreMigrationPhase, - name: &str, - length: usize, -) -> Result<(), Box> { - let (sandbox, mut authority) = open_authority("migration-all-truncations")?; - let intent = authority.observe_intent()?; - let witness = version_one_witness(sandbox.path())?; - StoreMigrationStorage::verify_current(&mut authority, &intent)?; - let records = MigrationRecords::for_intent(&intent); - for current in StoreMigrationPhase::ALL { - execute_phase(&mut authority, current, &records)?; - if current == phase { - break; - } - } - drop(authority); - fs::OpenOptions::new() - .write(true) - .open(sandbox.path().join(name))? - .set_len(u64::try_from(length)?)?; - let mut recovered = FilesystemStoreMigrationAuthority::reopen_for_recovery_unchecked_for_tests( - sandbox.path(), - maximum_policy(), - )?; - let expected = recovered.observe_intent()?; - let receipt = recover_store_migration(&mut recovered, &expected)?; - assert_eq!( - receipt.plan(), - StoreMigrationRecoveryPlan::DiscardStage { - stage, - resume: phase - } - ); - drop(recovered); - assert_complete_migration(sandbox.path(), &intent)?; - assert_eq!(version_one_witness(sandbox.path())?, witness); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn unexpected_nested_residue_refuses_before_creating_a_marker_stage() -> Result<(), Box> -{ - let (sandbox, mut authority) = open_authority("migration-nested-refusal")?; - let intent = authority.observe_intent()?; - StoreMigrationStorage::verify_current(&mut authority, &intent)?; - let records = MigrationRecords::for_intent(&intent); - for phase in StoreMigrationPhase::ALL.iter().take(8) { - execute_phase(&mut authority, *phase, &records)?; - } - drop(authority); - fs::write(sandbox.path().join("gc/unexpected"), b"preserve")?; - let witness = version_one_witness(sandbox.path())?; - let mut recovered = FilesystemStoreMigrationAuthority::reopen_for_recovery_unchecked_for_tests( - sandbox.path(), - maximum_policy(), - )?; - let expected = recovered.observe_intent()?; - let error = recover_store_migration(&mut recovered, &expected) - .err() - .ok_or("unexpected nested residue admitted")?; - let StoreMigrationRecoveryError::Adoption { source } = error else { - return Err("nested residue was not rejected before adoption".into()); - }; - assert!(matches!( - source - .get_ref() - .and_then(|error| error.downcast_ref::()), - Some(FilesystemMigrationRecoveryRefusal::NamespacePreflight { .. }) - )); - assert!(!sandbox.path().join("FORMAT.next").exists()); - assert_eq!(fs::read(sandbox.path().join("gc/unexpected"))?, b"preserve"); - assert_eq!(version_one_witness(sandbox.path())?, witness); - drop(recovered); - sandbox.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_remount_tests.rs b/src/adapters/store_migration/filesystem_migration_remount_tests.rs deleted file mode 100644 index 3a9a3860..00000000 --- a/src/adapters/store_migration/filesystem_migration_remount_tests.rs +++ /dev/null @@ -1,131 +0,0 @@ -//! This module owns restart admission laws for changed root coordinates. - -use std::error::Error; -use std::fs; -use std::path::Path; - -use super::filesystem_migration_test_fixture::open_authority; -use super::migration_catalog_coordinates::MigrationCatalogCoordinates; -use super::store_root_identity::StoreRootIdentities; -use super::{ - AdmittedStoreMigrationIntent, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, - CanonicalStoreMigrationReceipt, StoreRootDeviceIdentity, StoreRootFileIdentity, - StoreRootIdentityCoordinate, StoreRootMountIdentity, execute_store_migration, -}; -use crate::adapters::filesystem_test_sandbox::TestDirectory; -use crate::adapters::{FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission}; - -#[test] -fn a_remounted_store_reopens_without_changing_its_catalog_head() -> Result<(), Box> { - let sandbox = migrated("migration-remount-admission")?; - let head = fs::read(sandbox.path().join("HEAD"))?; - let _coordinates = substitute_coordinate(&sandbox, StoreRootIdentityCoordinate::Mount)?; - let intent = fs::read(sandbox.path().join("migration.intent"))?; - let receipt = fs::read(sandbox.path().join("migration.receipt"))?; - - let admission = reopen(sandbox.path())?; - - assert_eq!(fs::read(sandbox.path().join("HEAD"))?, head); - assert_eq!(fs::read(sandbox.path().join("migration.intent"))?, intent); - assert_eq!(fs::read(sandbox.path().join("migration.receipt"))?, receipt); - drop(admission); - Ok(()) -} - -#[test] -fn device_and_inode_disagreement_refuse_with_exact_coordinates() -> Result<(), Box> { - for coordinate in [ - StoreRootIdentityCoordinate::Device, - StoreRootIdentityCoordinate::File, - ] { - let sandbox = migrated(&format!("migration-moved-root-{coordinate:?}"))?; - let head = fs::read(sandbox.path().join("HEAD"))?; - let (expected, observed) = substitute_coordinate(&sandbox, coordinate)?; - - let error = reopen(sandbox.path()) - .err() - .ok_or("a moved root was admitted")?; - - assert!(matches!( - error, - FilesystemPlatformAdmissionError::RootIdentityChanged { - coordinate: refused, expected: bound, observed: current, - } if refused == coordinate && bound == expected && current == observed - )); - assert_eq!(fs::read(sandbox.path().join("HEAD"))?, head); - } - Ok(()) -} - -fn migrated(name: &str) -> Result> { - let (sandbox, mut authority) = open_authority(name)?; - let intent = authority.observe_intent()?; - let _receipt = execute_store_migration(&mut authority, &intent)?; - drop(authority); - Ok(sandbox) -} - -fn substitute_coordinate( - sandbox: &TestDirectory, - coordinate: StoreRootIdentityCoordinate, -) -> Result<(u64, u64), Box> { - let bytes = fs::read(sandbox.path().join("migration.intent"))?; - let original = AdmittedStoreMigrationIntent::decode(&bytes)?; - let observed = match coordinate { - StoreRootIdentityCoordinate::Device => original.root_device_identity().get(), - StoreRootIdentityCoordinate::Mount => original.root_mount_identity().get(), - StoreRootIdentityCoordinate::File => original.root_file_identity().get(), - }; - let expected = observed - .checked_add(1) - .ok_or("fixture coordinate overflow")?; - let changed = |selected, value| { - if coordinate == selected { - expected - } else { - value - } - }; - let roots = StoreRootIdentities::new( - StoreRootDeviceIdentity::from_admitted(changed( - StoreRootIdentityCoordinate::Device, - original.root_device_identity().get(), - )), - StoreRootMountIdentity::from_admitted(changed( - StoreRootIdentityCoordinate::Mount, - original.root_mount_identity().get(), - )), - StoreRootFileIdentity::from_admitted(changed( - StoreRootIdentityCoordinate::File, - original.root_file_identity().get(), - )), - ); - let catalog = MigrationCatalogCoordinates::new( - original.catalog_generation(), - original.catalog_length(), - original.catalog_digest(), - original.predecessor_catalog_digest(), - ); - let intent = CanonicalStoreMigrationIntent::from_coordinates( - catalog, - original.inventory_digest(), - roots, - ); - let receipt = CanonicalStoreMigrationReceipt::from_canonical( - &intent, - &CanonicalStoreFormatMarker::version_two(), - ); - fs::write(sandbox.path().join("migration.intent"), intent.encoded())?; - fs::write(sandbox.path().join("migration.receipt"), receipt.encoded())?; - Ok((expected, observed)) -} - -#[cfg(target_os = "linux")] -fn reopen(root: &Path) -> Result { - FilesystemVersionTwoAdmission::reopen(root) -} - -#[cfg(not(target_os = "linux"))] -fn reopen(root: &Path) -> Result { - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(root) -} diff --git a/src/adapters/store_migration/filesystem_migration_repository_tasks.rs b/src/adapters/store_migration/filesystem_migration_repository_tasks.rs deleted file mode 100644 index e7579f3e..00000000 --- a/src/adapters/store_migration/filesystem_migration_repository_tasks.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! This module owns the migration authority's repository crash-task hooks. -//! -//! Repository process-death tasks drive the production migration protocol -//! from a store they built without platform admission, stop it at an exact -//! byte or directory prefix, and reopen it for recovery. Nothing here is a -//! separate protocol: each hook runs one production step short, or opens the -//! same authority without the Linux platform check. - -use std::io; -use std::path::Path; - -use cap_std::fs::Dir; - -use super::filesystem_migration_authority::MigrationNamespacePolicy; -use super::filesystem_migration_authority_error::FilesystemMigrationAuthorityError as Error; -use super::filesystem_migration_fixed_artifact::{ - FilesystemMigrationFixedArtifact as FixedArtifact, FilesystemMigrationFixedStage, -}; -use super::{ - FilesystemStoreMigrationAuthority, FilesystemStoreMigrationInventoryReader, - StoreMigrationFixedStage, filesystem_migration_namespace, -}; -use crate::adapters::{FilesystemPlatformAdmission, FilesystemWriterLock, SegmentReadPolicy}; - -impl FilesystemStoreMigrationAuthority { - /// Opens fresh migration authority over a store built without platform - /// admission, for repository process-death tasks. - /// - /// # Errors - /// - /// Returns [`Error::Namespace`] when the root capability cannot be cloned, - /// [`Error::RootIdentity`] when its identity cannot be observed, or the - /// inventory failures from [`Self::open`]. No platform admission is performed. - #[doc(hidden)] - pub fn open_unchecked_for_repository_tasks( - lock: FilesystemWriterLock, - policy: SegmentReadPolicy, - ) -> Result { - let root = lock - .clone_directory() - .map_err(|source| Error::Namespace { source })?; - let identity = crate::adapters::filesystem_platform_profile::root_identity_lenient(&root) - .map_err(|source| Error::RootIdentity { source })?; - drop(root); - let admission = FilesystemPlatformAdmission::initialized(lock, identity); - let inventory = FilesystemStoreMigrationInventoryReader::open(admission, policy) - .map_err(|source| Error::Inventory { source })?; - Ok(Self::with_policy( - inventory, - MigrationNamespacePolicy::Published, - )) - } - - /// Reacquires recovery authority without platform admission, for - /// repository process-death tasks; otherwise exactly - /// [`Self::reopen_for_recovery`]. - /// - /// # Errors - /// - /// Returns the same writer-lock, namespace, and pool failures as - /// [`Self::reopen_for_recovery`]. - #[doc(hidden)] - pub fn reopen_for_recovery_unchecked_for_repository_tasks( - store_root: &Path, - policy: SegmentReadPolicy, - ) -> Result { - let root = Dir::open_ambient_dir(store_root, cap_std::ambient_authority()) - .map_err(|source| Error::Platform { source })?; - Self::recover_root(root, policy) - } - - /// Creates `stage` and writes a strict prefix of `record`, leaving an - /// incomplete pre-effect stage exactly as process death during the write - /// would. - /// - /// # Errors - /// - /// Returns the exact length, prefix-bound, creation, or write failure. - #[doc(hidden)] - pub fn write_fixed_stage_prefix_for_repository_tasks( - &mut self, - stage: StoreMigrationFixedStage, - record: &[u8], - prefix: usize, - ) -> io::Result<()> { - let artifact = match stage { - StoreMigrationFixedStage::Intent => FixedArtifact::Intent, - StoreMigrationFixedStage::Marker => FixedArtifact::Marker, - StoreMigrationFixedStage::Receipt => FixedArtifact::Receipt, - }; - FilesystemMigrationFixedStage::create_prefix(self.root(), artifact, record, prefix) - } - - /// Admits only the first `count` namespace-prefix directories in protocol - /// order, each with its parent synchronized, and stops before the final - /// prefix verification. - /// - /// # Errors - /// - /// Returns the exact preflight or admission failure, or refuses a count - /// beyond the six-directory prefix. - #[doc(hidden)] - pub fn admit_namespace_prefix_for_repository_tasks(&mut self, count: usize) -> io::Result<()> { - filesystem_migration_namespace::admit_namespace_prefix_partially(self.root(), count) - } -} diff --git a/src/adapters/store_migration/filesystem_migration_residue.rs b/src/adapters/store_migration/filesystem_migration_residue.rs deleted file mode 100644 index 0552d5c4..00000000 --- a/src/adapters/store_migration/filesystem_migration_residue.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! This module owns observing migration residue on one pinned root. - -use std::io; - -use cap_fs_ext::DirExt; -use cap_std::fs::Dir; - -use super::FilesystemMigrationRecoveryRefusal as Refusal; -use super::filesystem_migration_recovery_refusal::{invalid, kind}; -use super::{ - FORMAT_MARKER_LENGTH, MIGRATION_INTENT_LENGTH, MIGRATION_RECEIPT_LENGTH, StoreMigrationResidue, -}; -use crate::adapters::filesystem_exact_record::{self as exact_record, ExactRecordError}; - -const READER_LOCK: &str = "reader.lock"; - -/// Observes every fixed migration name without mutation. -/// -/// Records and stages are read without following links and bounded to one -/// byte more than their canonical length, so an overlong file stays -/// distinguishable from an exact one. A wrong kind refuses. -pub(super) fn observe(root: &Dir) -> io::Result { - Ok(StoreMigrationResidue { - intent_stage: bounded_file(root, "migration.intent.next", MIGRATION_INTENT_LENGTH)?, - intent: bounded_file(root, "migration.intent", MIGRATION_INTENT_LENGTH)?, - reader_fence: reader_fence(root)?, - namespace_prefix: namespace_prefix(root)?, - marker_stage: bounded_file(root, "FORMAT.next", FORMAT_MARKER_LENGTH)?, - marker: bounded_file(root, "FORMAT", FORMAT_MARKER_LENGTH)?, - receipt_stage: bounded_file(root, "migration.receipt.next", MIGRATION_RECEIPT_LENGTH)?, - receipt: bounded_file(root, "migration.receipt", MIGRATION_RECEIPT_LENGTH)?, - }) -} - -fn bounded_file(root: &Dir, name: &str, length: usize) -> io::Result>> { - let bound = length - .checked_add(1) - .ok_or_else(|| invalid(Refusal::ResidueBoundOverflow { length }))?; - exact_record::read_bounded_optional(root, name, bound).map_err(ExactRecordError::into_io) -} - -fn reader_fence(root: &Dir) -> io::Result { - match root.symlink_metadata(READER_LOCK) { - Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(false), - Err(source) => Err(source), - Ok(metadata) if metadata.is_file() && metadata.len() == 0 => Ok(true), - Ok(metadata) => Err(invalid(Refusal::ReaderFence { - observed_kind: kind(&metadata), - observed_length: metadata.len(), - })), - } -} - -fn namespace_prefix(root: &Dir) -> io::Result<[bool; 6]> { - let retention = optional_directory(root, "retention")?; - let (roots, manifests) = match retention.as_ref() { - Some(retention) => ( - optional_directory(retention, "roots")?.is_some(), - optional_directory(retention, "manifests")?.is_some(), - ), - None => (false, false), - }; - let gc = optional_directory(root, "gc")?.is_some(); - let recovery = optional_directory(root, "recovery")?; - let dispositions = match recovery.as_ref() { - Some(recovery) => optional_directory(recovery, "dispositions")?.is_some(), - None => false, - }; - Ok([ - retention.is_some(), - roots, - manifests, - gc, - recovery.is_some(), - dispositions, - ]) -} - -fn optional_directory(parent: &Dir, name: &str) -> io::Result> { - match parent.symlink_metadata(name) { - Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(None), - Err(source) => Err(source), - Ok(metadata) if metadata.is_dir() => parent.open_dir_nofollow(name).map(Some), - Ok(metadata) => Err(invalid(Refusal::NamespaceKind { - observed_kind: kind(&metadata), - })), - } -} diff --git a/src/adapters/store_migration/filesystem_migration_residue_kind_tests.rs b/src/adapters/store_migration/filesystem_migration_residue_kind_tests.rs deleted file mode 100644 index 476db544..00000000 --- a/src/adapters/store_migration/filesystem_migration_residue_kind_tests.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! This module owns typed refusal of non-regular migration residue. - -#![cfg(unix)] - -use std::error::Error; -use std::fs; -use std::os::unix::fs::symlink; - -use super::StoreMigrationRecoveryStorage; -use super::filesystem_migration_test_fixture::open_authority; -use crate::adapters::filesystem_exact_record::{ExactRecordError, ExactRecordRefusal}; - -#[test] -fn symlink_residue_refuses_by_kind_before_reading_its_target() -> Result<(), Box> { - for name in [ - "migration.intent.next", - "migration.intent", - "FORMAT.next", - "FORMAT", - "migration.receipt.next", - "migration.receipt", - ] { - let (sandbox, mut authority) = open_authority("migration-residue-symlink")?; - let target = sandbox.path().join("HEAD"); - let before = fs::read(&target)?; - symlink("HEAD", sandbox.path().join(name))?; - let error = StoreMigrationRecoveryStorage::observe_residue(&mut authority) - .err() - .ok_or("symlink residue was admitted")?; - assert!( - matches!( - error - .get_ref() - .and_then(|source| source.downcast_ref::()), - Some(ExactRecordError::Refused(ExactRecordRefusal::KindOrLength)) - ), - "{name}: {error:?}" - ); - assert_eq!(fs::read(&target)?, before); - assert_eq!( - fs::read_link(sandbox.path().join(name))?, - std::path::Path::new("HEAD") - ); - drop(authority); - sandbox.remove()?; - } - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_restart_namespace_tests.rs b/src/adapters/store_migration/filesystem_migration_restart_namespace_tests.rs deleted file mode 100644 index cdc32415..00000000 --- a/src/adapters/store_migration/filesystem_migration_restart_namespace_tests.rs +++ /dev/null @@ -1,190 +0,0 @@ -//! Restart refuses unknown names and incorrect filesystem kinds without cleanup. - -use super::filesystem_migration_restart_test_fixture::{prefix, refusal}; -use super::{ - FilesystemMigrationAuthorityError as Authority, FilesystemMigrationRecoveryRefusal as Refusal, - FilesystemMigrationResidueKind as Kind, StoreMigrationPhase as Phase, - StoreMigrationRecoveryError as Recovery, -}; -use std::{error::Error, fs, io}; - -// Size: medium. Oracle: recovery admits only the closed migration namespace. -// Delete only if unknown names become explicitly admitted by a new protocol. -#[test] -fn unknown_root_and_nested_entries_preserve_restart_evidence() -> Result<(), Box> { - for parent in [ - "", - "retention", - "retention/roots", - "retention/manifests", - "gc", - "recovery", - "recovery/dispositions", - ] { - let store = prefix( - &format!("restart-unknown-{}", parent.replace('/', "-")), - Phase::AdmitNamespacePrefix, - )?; - fs::write( - store.path().join(parent).join("unexpected"), - b"retain this evidence", - )?; - let error = refusal(store.path())?; - if parent.is_empty() { - require_namespace(error.as_ref()); - } else { - let Some(Recovery::Adoption { source }) = error.downcast_ref::() else { - return Err(format!("{parent}: wrong refusal boundary: {error:?}").into()); - }; - assert!( - matches!(source.get_ref().and_then(|source| source.downcast_ref::()), - Some(Refusal::NamespacePreflight { source }) if source.kind() == io::ErrorKind::InvalidData), - "{parent}: {error:?}" - ); - } - store.remove()?; - } - Ok(()) -} - -// Size: medium. Oracle: fixed record and fence names are regular files, never directories. -// Delete only if the named filesystem kinds change by protocol decision. -#[test] -fn directory_substitution_at_each_fixed_file_refuses_restart() -> Result<(), Box> { - for name in [ - "migration.intent", - "migration.intent.next", - "FORMAT", - "FORMAT.next", - "migration.receipt", - "migration.receipt.next", - "reader.lock", - ] { - let store = prefix( - &format!("restart-wrong-file-kind-{name}"), - Phase::RemoveReceiptStage, - )?; - let path = store.path().join(name); - if path.exists() { - fs::remove_file(&path)?; - } - fs::create_dir(path)?; - let error = refusal(store.path())?; - require_namespace(error.as_ref()); - store.remove()?; - } - Ok(()) -} - -// Size: medium. Oracle: all protocol directories must remain real directories. -// Delete only if migration directory requirements are removed. -#[test] -fn regular_files_at_protocol_directory_names_refuse_restart() -> Result<(), Box> { - for name in [ - "retention", - "gc", - "recovery", - "retention/roots", - "retention/manifests", - "recovery/dispositions", - ] { - let store = prefix( - &format!("restart-wrong-dir-kind-{}", name.replace('/', "-")), - Phase::AdmitNamespacePrefix, - )?; - let path = store.path().join(name); - fs::remove_dir_all(&path)?; - fs::write(path, b"not a directory")?; - let error = refusal(store.path())?; - if name.contains('/') { - assert!( - matches!(error.downcast_ref::(), Some(Recovery::Observation { source }) - if matches!(source.get_ref().and_then(|source| source.downcast_ref::()), - Some(Refusal::NamespaceKind { observed_kind: Kind::RegularFile }))), - "{name}: {error:?}" - ); - } else { - require_namespace(error.as_ref()); - } - store.remove()?; - } - Ok(()) -} - -// Size: medium. Oracle: a reader fence carries no payload; observation names its exact length. -// Delete only if reader-lock payloads acquire defined protocol semantics. -#[test] -fn a_nonempty_reader_fence_preserves_its_exact_refusal() -> Result<(), Box> { - let store = prefix("restart-nonempty-reader-fence", Phase::AdmitReaderFence)?; - fs::write(store.path().join("reader.lock"), b"x")?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Recovery::Observation { source }) - if matches!(source.get_ref().and_then(|source| source.downcast_ref::()), - Some(Refusal::ReaderFence { observed_kind: Kind::RegularFile, observed_length: 1 }))), - "{error:?}" - ); - store.remove()?; - Ok(()) -} - -fn require_namespace(error: &(dyn Error + 'static)) { - assert!( - matches!(error.downcast_ref::(), Some(Authority::Namespace { source }) - if source.kind() == io::ErrorKind::InvalidData), - "namespace refusal lost its typed boundary: {error:?}" - ); -} - -// Size: medium. Oracle: migration fixed names never follow substituted symbolic links. -// Delete only if no-follow admission is explicitly replaced by another protocol. -#[test] -fn symbolic_links_at_fixed_record_names_refuse_without_following_them() -> Result<(), Box> -{ - for name in [ - "migration.intent", - "migration.intent.next", - "FORMAT", - "FORMAT.next", - "migration.receipt", - "migration.receipt.next", - "reader.lock", - ] { - let store = prefix( - &format!("restart-linked-record-{name}"), - Phase::RemoveReceiptStage, - )?; - let path = store.path().join(name); - if path.exists() { - fs::remove_file(&path)?; - } - std::os::unix::fs::symlink("HEAD", path)?; - let error = refusal(store.path())?; - require_namespace(error.as_ref()); - store.remove()?; - } - Ok(()) -} - -// Size: medium. Oracle: completed migration still admits only the reserved v2 namespace. -// Delete only if that namespace contract is removed or stronger restart laws subsume this law. -#[test] -fn completed_migration_refuses_unknown_reserved_entries_without_effects() --> Result<(), Box> { - for parent in ["gc", "recovery", "recovery/dispositions"] { - let store = prefix( - &format!("restart-complete-unknown-{}", parent.replace('/', "-")), - Phase::RemoveReceiptStage, - )?; - fs::write(store.path().join(parent).join("unexpected"), b"preserve")?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Recovery::Observation { source }) - if matches!(source.get_ref().and_then(|source| source.downcast_ref::()), - Some(Refusal::NamespacePreflight { source }) if source.kind() == io::ErrorKind::InvalidData)), - "completed migration admitted invalid {parent}: {error:?}" - ); - store.remove()?; - } - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_restart_order_tests.rs b/src/adapters/store_migration/filesystem_migration_restart_order_tests.rs deleted file mode 100644 index 4706ae15..00000000 --- a/src/adapters/store_migration/filesystem_migration_restart_order_tests.rs +++ /dev/null @@ -1,148 +0,0 @@ -//! Filesystem restart refusal at impossible migration orderings. - -use super::filesystem_migration_restart_test_fixture::{prefix, refusal}; -use super::{ - StoreMigrationEffect as Effect, StoreMigrationFixedStage as Stage, - StoreMigrationPhase as Phase, StoreMigrationRecoveryAmbiguity as Ambiguity, - StoreMigrationRecoveryError as Recovery, -}; -use std::{error::Error, fs}; - -// Size: medium. Oracle: no namespace, marker or receipt effect may precede durable intent. -// Delete only if the migration ordering contract disappears or stronger coverage subsumes it. -#[test] -fn effects_without_durable_intent_preserve_restart_evidence() -> Result<(), Box> { - for (name, effect) in [ - ("reader.lock", Effect::Namespace), - ("FORMAT", Effect::Marker), - ("migration.receipt", Effect::Receipt), - ] { - let store = prefix( - &format!("restart-before-intent-{name}"), - Phase::WriteIntentStage, - )?; - fs::write(store.path().join(name), [])?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Recovery::Ambiguity { - source: Ambiguity::EffectBeforeIntent { effect: observed } - }) if *observed == effect), - "{name}: {error:?}" - ); - store.remove()?; - } - Ok(()) -} - -// Size: medium. Oracle: later effects require cleanup of the previous stage. -// Delete only if the cleanup ordering requirement disappears. -#[test] -fn stages_surviving_later_effects_refuse_restart() -> Result<(), Box> { - for (phase, name, stage, effect) in [ - ( - Phase::LinkIntent, - "reader.lock", - Stage::Intent, - Effect::Namespace, - ), - (Phase::LinkIntent, "FORMAT", Stage::Intent, Effect::Marker), - ( - Phase::LinkIntent, - "migration.receipt", - Stage::Intent, - Effect::Receipt, - ), - ( - Phase::LinkMarker, - "migration.receipt", - Stage::Marker, - Effect::Receipt, - ), - ] { - let store = prefix(&format!("restart-stage-after-{phase:?}-{name}"), phase)?; - fs::write(store.path().join(name), [])?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Recovery::Ambiguity { - source: Ambiguity::StageAfterEffect { stage: observed_stage, effect: observed_effect } - }) if *observed_stage == stage && *observed_effect == effect), - "{name}: {error:?}" - ); - store.remove()?; - } - Ok(()) -} - -// Size: medium. Oracle: filesystem-realizable holes cannot be filled over later names. -// Delete only if the ordered namespace prefix is removed. -#[test] -fn namespace_holes_refuse_restart_at_the_exact_coordinates() -> Result<(), Box> { - for (name, absent, present) in [ - ("reader.lock", 0, 1), - ("retention", 1, 4), - ("retention/roots", 2, 3), - ("retention/manifests", 3, 4), - ("gc", 4, 5), - ] { - let store = prefix( - &format!("restart-namespace-hole-{absent}"), - Phase::AdmitNamespacePrefix, - )?; - let path = store.path().join(name); - if path.is_dir() { - fs::remove_dir_all(path)?; - } else { - fs::remove_file(path)?; - } - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Recovery::Ambiguity { - source: Ambiguity::NamespaceOutOfOrder { absent: observed_absent, present: observed_present } - }) if *observed_absent == absent && *observed_present == present), - "{name}: {error:?}" - ); - store.remove()?; - } - Ok(()) -} - -// Size: medium. Oracle: a marker cannot precede the complete namespace. -// Delete only if migration stops requiring the namespace before marker publication. -#[test] -fn a_marker_before_namespace_completion_preserves_evidence() -> Result<(), Box> { - let store = prefix("restart-marker-before-namespace", Phase::WriteMarkerStage)?; - fs::remove_dir_all(store.path().join("recovery"))?; - fs::remove_dir(store.path().join("gc"))?; - let error = refusal(store.path())?; - assert!( - matches!( - error.downcast_ref::(), - Some(Recovery::Ambiguity { - source: Ambiguity::MarkerBeforeNamespace - }) - ), - "{error:?}" - ); - store.remove()?; - Ok(()) -} - -// Size: medium. Oracle: a receipt requires the durable marker it binds. -// Delete only if receipts cease to bind the migration marker. -#[test] -fn a_receipt_without_its_marker_preserves_evidence() -> Result<(), Box> { - let store = prefix("restart-receipt-before-marker", Phase::WriteReceiptStage)?; - fs::remove_file(store.path().join("FORMAT"))?; - let error = refusal(store.path())?; - assert!( - matches!( - error.downcast_ref::(), - Some(Recovery::Ambiguity { - source: Ambiguity::ReceiptBeforeMarker - }) - ), - "{error:?}" - ); - store.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_restart_pair_tests.rs b/src/adapters/store_migration/filesystem_migration_restart_pair_tests.rs deleted file mode 100644 index 4876788a..00000000 --- a/src/adapters/store_migration/filesystem_migration_restart_pair_tests.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! Restart laws for fixed-stage identity and contradictory paired evidence. - -use super::filesystem_migration_restart_test_fixture::{prefix, refusal}; -use super::{ - StoreMigrationFixedStage as Stage, StoreMigrationPhase as Phase, - StoreMigrationRecoveryAmbiguity as Ambiguity, StoreMigrationRecoveryError as Recovery, -}; -use crate::adapters::filesystem_exact_record::{ExactRecordError, ExactRecordRefusal}; -use std::{error::Error, fs}; - -// Size: medium. Oracle: stage and canonical record must share one inode, not only bytes. -// Delete only if the fixed-stage protocol disappears or stronger restart laws subsume it. -#[test] -fn byte_equal_stage_substitution_refuses_before_restart_effects() -> Result<(), Box> { - for (name, phase) in [ - ("migration.intent.next", Phase::LinkIntent), - ("FORMAT.next", Phase::LinkMarker), - ("migration.receipt.next", Phase::LinkReceipt), - ] { - let store = prefix(&format!("restart-substitution-{name}"), phase)?; - let path = store.path().join(name); - let bytes = fs::read(&path)?; - fs::remove_file(&path)?; - fs::write(path, bytes)?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Recovery::Adoption { source }) - if matches!(source.get_ref().and_then(|source| source.downcast_ref::()), - Some(ExactRecordError::Refused(ExactRecordRefusal::KindLengthOrIdentity)))), - "{name}: {error:?}" - ); - store.remove()?; - } - Ok(()) -} - -// Size: medium. Oracle: contradictory stage/target bytes refuse rather than choose a winner. -// Delete only if paired migration evidence is no longer part of the protocol. -#[test] -fn conflicting_stage_bytes_preserve_both_records_on_restart() -> Result<(), Box> { - for (name, phase, stage) in [ - ("migration.intent.next", Phase::LinkIntent, Stage::Intent), - ("FORMAT.next", Phase::LinkMarker, Stage::Marker), - ("migration.receipt.next", Phase::LinkReceipt, Stage::Receipt), - ] { - let store = prefix(&format!("restart-conflict-{name}"), phase)?; - let path = store.path().join(name); - let mut bytes = fs::read(&path)?; - *bytes.last_mut().ok_or("empty stage")? ^= 1; - fs::remove_file(&path)?; - fs::write(path, bytes)?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Recovery::Ambiguity { - source: Ambiguity::StageDiffers { stage: observed } - }) if *observed == stage), - "{name}: {error:?}" - ); - store.remove()?; - } - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_restart_pool_tests.rs b/src/adapters/store_migration/filesystem_migration_restart_pool_tests.rs deleted file mode 100644 index a0caff57..00000000 --- a/src/adapters/store_migration/filesystem_migration_restart_pool_tests.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! Restart admission verifies immutable pools and current HEAD before recovery. - -use super::filesystem_migration_restart_test_fixture::{prefix, refusal}; -use super::{ - FilesystemMigrationAuthorityError as Authority, FilesystemMigrationInventoryError as Inventory, - MigrationInventoryPool as Pool, StoreMigrationPhase as Phase, -}; -use crate::adapters::{ - CatalogDecodeError, CatalogRestartError, PublicationHeadDecodeError, SegmentHeaderError, - SegmentReadError, -}; -use std::{ - error::Error, - fs, - path::{Path, PathBuf}, -}; - -// Size: medium. Oracle: malformed magic retains its exact decoder coordinates; -// restart cannot mutate a store whose immutable segment or catalog is corrupt. -// Delete only if immutable-pool verification is removed or stronger restart laws subsume it. -#[test] -fn damaged_immutable_pool_bytes_refuse_restart_before_effects() -> Result<(), Box> { - for (directory, pool) in [("segments", Pool::Segments), ("catalogs", Pool::Catalogs)] { - let store = prefix( - &format!("restart-damaged-{directory}"), - Phase::RemoveIntentStage, - )?; - let path = only_entry(&store.path().join(directory))?; - let (expected, observed) = damage_magic(&path)?; - let error = refusal(store.path())?; - let Some(Authority::Inventory { - source: - Inventory::Artifact { - pool: found, - source, - .. - }, - }) = error.downcast_ref::() - else { - return Err(format!("{directory}: wrong restart failure: {error:?}").into()); - }; - assert_eq!( - *found, pool, - "refusal must identify the damaged immutable pool" - ); - assert!( - match (pool, source.as_ref()) { - (Pool::Segments, CatalogRestartError::Segment { source, .. }) => - matches!(source.as_ref(), - SegmentReadError::Header { source: SegmentHeaderError::InvalidMagic { expected: found_expected, observed: found_observed } } - if *found_expected == expected && *found_observed == observed), - ( - Pool::Catalogs, - CatalogRestartError::Catalog { - source: CatalogDecodeError::InvalidMagic { observed: found }, - }, - ) => *found == observed, - _ => false, - }, - "{directory}: exact decoder failure lost: {error:?}" - ); - store.remove()?; - } - Ok(()) -} - -// Size: medium. Oracle: the freshly reopened current HEAD must decode before recovery. -// Delete only if migration no longer binds the version-one head. -#[test] -fn a_damaged_current_head_preserves_all_restart_evidence() -> Result<(), Box> { - let store = prefix("restart-damaged-current-head", Phase::RemoveIntentStage)?; - let (_expected, observed) = damage_magic(&store.path().join("HEAD"))?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Authority::Head { - source: PublicationHeadDecodeError::InvalidMagic { observed: found } - }) if *found == observed), - "HEAD refusal lost its decoder coordinate: {error:?}" - ); - store.remove()?; - Ok(()) -} - -fn only_entry(directory: &Path) -> Result> { - let mut entries = fs::read_dir(directory)?; - let path = entries.next().ok_or("fixture pool is empty")??.path(); - if entries.next().is_some() { - return Err("fixture pool is not a single artifact".into()); - } - Ok(path) -} - -type MagicCoordinates = ([u8; 16], [u8; 16]); - -fn damage_magic(path: &Path) -> Result> { - let mut bytes = fs::read(path)?; - let expected = bytes.get(..16).ok_or("no magic")?.try_into()?; - *bytes.first_mut().ok_or("empty record")? ^= 1; - let observed = bytes.get(..16).ok_or("no magic")?.try_into()?; - fs::write(path, bytes)?; - Ok((expected, observed)) -} - -// Size: medium. Oracle: valid alternate content cannot occupy another segment's name. -// Delete only if physical segment names no longer bind exact content. -#[test] -fn a_valid_segment_substitution_reports_both_digest_coordinates() -> Result<(), Box> { - use crate::adapters::{AdmittedSegment, SegmentReadPolicy}; - let store = prefix( - "restart-valid-segment-substitution", - Phase::RemoveIntentStage, - )?; - let path = only_entry(&store.path().join("segments"))?; - let original = fs::read(&path)?; - let replacement = super::filesystem_inventory_catalogs_test_fixture::empty_segment_bytes()?; - let expected = AdmittedSegment::decode(&original, SegmentReadPolicy::MAXIMUM)?.digest(); - let observed = AdmittedSegment::decode(&replacement, SegmentReadPolicy::MAXIMUM)?.digest(); - fs::write(&path, replacement)?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Authority::Inventory { - source: Inventory::Artifact { pool: Pool::Segments, source, .. } - }) if matches!(source.as_ref(), CatalogRestartError::SegmentCoordinate { expected: found_expected, observed: found_observed } - if *found_expected == expected && *found_observed == observed)), - "{error:?}" - ); - store.remove()?; - Ok(()) -} - -// Size: medium. Oracle: persisted intent binds the entire immutable inventory, including orphans. -// Delete only if inventory binding is explicitly removed from the migration protocol. -#[test] -fn adding_a_valid_orphan_cannot_reuse_the_previous_migration_intent() -> Result<(), Box> -{ - use super::{StoreMigrationRecoveryAmbiguity, StoreMigrationRecoveryError}; - use crate::adapters::{AdmittedSegment, SegmentReadPolicy, physical_pool_name}; - let store = prefix( - "restart-valid-orphan-inventory-change", - Phase::RemoveIntentStage, - )?; - let orphan = super::filesystem_inventory_catalogs_test_fixture::empty_segment_bytes()?; - let digest = AdmittedSegment::decode(&orphan, SegmentReadPolicy::MAXIMUM)?.digest(); - fs::write( - store - .path() - .join("segments") - .join(physical_pool_name::segment(digest)), - orphan, - )?; - let error = refusal(store.path())?; - assert!( - matches!( - error.downcast_ref::(), - Some(StoreMigrationRecoveryError::Ambiguity { - source: StoreMigrationRecoveryAmbiguity::IntentDiffers - }) - ), - "changed immutable inventory must not inherit old intent: {error:?}" - ); - store.remove()?; - Ok(()) -} - -// Size: medium. Oracle: canonical pool filenames have 68/85 bytes in segment-store/v1. -// Delete only if immutable pool naming changes by protocol decision. -#[test] -fn unknown_immutable_pool_names_refuse_with_the_exact_name() -> Result<(), Box> { - use crate::adapters::RecoveryPoolNameError; - for (directory, pool, width) in [ - ("segments", Pool::Segments, 68), - ("catalogs", Pool::Catalogs, 85), - ] { - let store = prefix( - &format!("restart-unknown-pool-{directory}"), - Phase::RemoveIntentStage, - )?; - fs::write( - store.path().join(directory).join("unknown"), - b"retained evidence", - )?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Authority::Inventory { - source: Inventory::Name { pool: found_pool, name, source: RecoveryPoolNameError::WrongLength { expected, observed: 7 } } - }) if *found_pool == pool && name.as_bytes() == b"unknown" && *expected == width), - "{directory}: {error:?}" - ); - store.remove()?; - } - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_restart_record_tests.rs b/src/adapters/store_migration/filesystem_migration_restart_record_tests.rs deleted file mode 100644 index b3a9b432..00000000 --- a/src/adapters/store_migration/filesystem_migration_restart_record_tests.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! Exact restart diagnostics and preserved filesystem evidence for damaged records. - -use super::filesystem_migration_restart_test_fixture::{prefix, refusal}; -use super::{ - StoreFormatMarkerDecodeError as MarkerDecode, StoreMigrationFixedStage as Stage, - StoreMigrationIntentDecodeError as IntentDecode, StoreMigrationPhase as Phase, - StoreMigrationReceiptDecodeError as ReceiptDecode, - StoreMigrationRecoveryAmbiguity as Ambiguity, StoreMigrationRecoveryError as Recovery, - StoreMigrationStageDecodeError as StageDecode, -}; -use std::{error::Error, fs}; - -// Size: medium. Oracle: the documented restart boundary retains the exact decoder -// checksum coordinates and preserves all names, inode identities and bytes. -// Delete only if migration records disappear or stronger restart laws subsume this matrix. -#[test] -fn checksum_damage_in_each_migration_record_preserves_restart_evidence() --> Result<(), Box> { - for (name, phase) in [ - ("migration.intent.next", Phase::WriteIntentStage), - ("migration.intent", Phase::RemoveIntentStage), - ("FORMAT.next", Phase::WriteMarkerStage), - ("FORMAT", Phase::RemoveMarkerStage), - ("migration.receipt.next", Phase::WriteReceiptStage), - ("migration.receipt", Phase::RemoveReceiptStage), - ] { - let store = prefix(&format!("restart-checksum-{name}"), phase)?; - let path = store.path().join(name); - let mut bytes = fs::read(&path)?; - let start = bytes.len().checked_sub(32).ok_or("no record checksum")?; - let expected: [u8; 32] = bytes.get(start..).ok_or("missing checksum")?.try_into()?; - *bytes.last_mut().ok_or("empty record")? ^= 1; - let observed: [u8; 32] = bytes.get(start..).ok_or("missing checksum")?.try_into()?; - fs::write(path, bytes)?; - let error = refusal(store.path())?; - assert_eq!( - checksum_coordinates(error.as_ref(), name), - Some((expected, observed)), - "{name}: {error:?}" - ); - store.remove()?; - } - Ok(()) -} - -fn checksum_coordinates(error: &(dyn Error + 'static), name: &str) -> Option<([u8; 32], [u8; 32])> { - let Recovery::Ambiguity { source } = error.downcast_ref::()? else { - return None; - }; - match (name, source) { - ( - "migration.intent", - Ambiguity::IntentUndecodable { - source: IntentDecode::ChecksumMismatch { expected, observed }, - }, - ) - | ( - "FORMAT", - Ambiguity::MarkerUndecodable { - source: MarkerDecode::ChecksumMismatch { expected, observed }, - }, - ) - | ( - "migration.receipt", - Ambiguity::ReceiptUndecodable { - source: ReceiptDecode::ChecksumMismatch { expected, observed }, - }, - ) - | ( - "migration.intent.next", - Ambiguity::StageUndecodable { - stage: Stage::Intent, - source: - StageDecode::Intent { - source: IntentDecode::ChecksumMismatch { expected, observed }, - }, - }, - ) - | ( - "FORMAT.next", - Ambiguity::StageUndecodable { - stage: Stage::Marker, - source: - StageDecode::Marker { - source: MarkerDecode::ChecksumMismatch { expected, observed }, - }, - }, - ) - | ( - "migration.receipt.next", - Ambiguity::StageUndecodable { - stage: Stage::Receipt, - source: - StageDecode::Receipt { - source: ReceiptDecode::ChecksumMismatch { expected, observed }, - }, - }, - ) => Some((*expected, *observed)), - _ => None, - } -} - -// Size: medium. Oracle: an overlong stage is not a discardable partial record. -// Delete only if stage framing disappears or stronger restart coverage subsumes it. -#[test] -fn overlong_stages_refuse_restart_without_disposal() -> Result<(), Box> { - for (name, phase, stage) in [ - ( - "migration.intent.next", - Phase::WriteIntentStage, - Stage::Intent, - ), - ("FORMAT.next", Phase::WriteMarkerStage, Stage::Marker), - ( - "migration.receipt.next", - Phase::WriteReceiptStage, - Stage::Receipt, - ), - ] { - let store = prefix(&format!("restart-overlong-{name}"), phase)?; - let path = store.path().join(name); - let mut bytes = fs::read(&path)?; - bytes.push(0); - let length = bytes.len(); - fs::write(path, bytes)?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Recovery::Ambiguity { - source: Ambiguity::StageOverlong { stage: observed_stage, observed } - }) if *observed_stage == stage && *observed == length), - "{name}: {error:?}" - ); - store.remove()?; - } - Ok(()) -} - -// Size: medium. Oracle: a checksummed receipt from another root cannot bind this intent. -// Delete only if receipts no longer bind the migration intent. -#[test] -fn a_valid_foreign_receipt_refuses_its_conflicting_intent_digest() -> Result<(), Box> { - use super::AdmittedStoreMigrationIntent; - let store = prefix("restart-foreign-receipt-local", Phase::RemoveReceiptStage)?; - let foreign = prefix("restart-foreign-receipt-other", Phase::RemoveReceiptStage)?; - let own_intent = fs::read(store.path().join("migration.intent"))?; - let other_intent = fs::read(foreign.path().join("migration.intent"))?; - let expected = AdmittedStoreMigrationIntent::decode(&own_intent)?.digest(); - let observed = AdmittedStoreMigrationIntent::decode(&other_intent)?.digest(); - fs::copy( - foreign.path().join("migration.receipt"), - store.path().join("migration.receipt"), - )?; - let error = refusal(store.path())?; - assert!( - matches!(error.downcast_ref::(), Some(Recovery::Ambiguity { - source: Ambiguity::ReceiptUndecodable { source: ReceiptDecode::IntentDigestMismatch { expected: found_expected, observed: found_observed } } - }) if found_expected == expected.as_bytes() && found_observed == observed.as_bytes()), - "{error:?}" - ); - store.remove()?; - foreign.remove()?; - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_restart_root_tests.rs b/src/adapters/store_migration/filesystem_migration_restart_root_tests.rs deleted file mode 100644 index 56ce69ec..00000000 --- a/src/adapters/store_migration/filesystem_migration_restart_root_tests.rs +++ /dev/null @@ -1,46 +0,0 @@ -//! Restart-stable root identity remains binding even when every byte agrees. - -use super::filesystem_migration_restart_test_fixture::{prefix, refusal}; -use super::{StoreMigrationPhase, StoreMigrationRecoveryAmbiguity, StoreMigrationRecoveryError}; -use crate::adapters::filesystem_test_sandbox::TestDirectory; -use std::{error::Error, fs, path::Path}; - -// Size: medium. Oracle: persisted migration intent binds the original root inode; -// copying the entire store must not authorize continuation under a new root. -// Delete only if root binding is explicitly removed or stronger restart evidence subsumes it. -#[test] -fn a_byte_equal_store_copy_refuses_the_persisted_root_identity() -> Result<(), Box> { - let original = prefix( - "restart-original-root", - StoreMigrationPhase::RemoveIntentStage, - )?; - let copied = TestDirectory::create("restart-substituted-root")?; - copy_directory(original.path(), copied.path())?; - let error = refusal(copied.path())?; - assert!( - matches!( - error.downcast_ref::(), - Some(StoreMigrationRecoveryError::Ambiguity { - source: StoreMigrationRecoveryAmbiguity::IntentDiffers - }) - ), - "copied root must not inherit migration authority: {error:?}" - ); - copied.remove()?; - original.remove()?; - Ok(()) -} - -fn copy_directory(source: &Path, destination: &Path) -> Result<(), Box> { - for entry in fs::read_dir(source)? { - let entry = entry?; - let target = destination.join(entry.file_name()); - if entry.file_type()?.is_dir() { - fs::create_dir(&target)?; - copy_directory(&entry.path(), &target)?; - } else { - fs::copy(entry.path(), target)?; - } - } - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_restart_test_fixture.rs b/src/adapters/store_migration/filesystem_migration_restart_test_fixture.rs deleted file mode 100644 index 7bbd259c..00000000 --- a/src/adapters/store_migration/filesystem_migration_restart_test_fixture.rs +++ /dev/null @@ -1,89 +0,0 @@ -//! Filesystem restart fixtures and complete evidence witnesses for migration refusal. - -use std::collections::BTreeMap; -use std::error::Error; -use std::fs; -use std::os::unix::fs::MetadataExt; -use std::path::{Path, PathBuf}; - -use super::filesystem_migration_test_fixture::{maximum_policy, open_authority}; -use super::migration_resumption::{MigrationRecords, execute_phase}; -use super::{FilesystemStoreMigrationAuthority, StoreMigrationPhase, recover_store_migration}; -use crate::adapters::filesystem_test_sandbox::TestDirectory; - -pub(super) fn prefix( - name: &str, - last: StoreMigrationPhase, -) -> Result> { - let (sandbox, mut authority) = open_authority(name)?; - let intent = authority.observe_intent()?; - authority.verify_current(&intent)?; - let records = MigrationRecords::for_intent(&intent); - for phase in StoreMigrationPhase::ALL { - execute_phase(&mut authority, phase, &records)?; - if phase == last { - return Ok(sandbox); - } - } - Err("requested migration prefix was not executed".into()) -} - -pub(super) fn refusal(root: &Path) -> Result, Box> { - let before = witness(root)?; - let error = restart(root) - .err() - .ok_or("ambiguous migration restarted successfully")?; - assert_eq!( - witness(root)?, - before, - "recovery refusal must preserve every name, inode and byte" - ); - Ok(error) -} - -fn restart(root: &Path) -> Result<(), Box> { - let mut authority = FilesystemStoreMigrationAuthority::reopen_for_recovery_unchecked_for_tests( - root, - maximum_policy(), - )?; - let expected = authority.observe_intent()?; - let _receipt = recover_store_migration(&mut authority, &expected)?; - Ok(()) -} - -#[derive(Debug, Eq, PartialEq)] -pub(super) enum Contents { - Directory, - File(Vec), - Link(PathBuf), -} - -type Witness = BTreeMap; - -fn witness(root: &Path) -> Result> { - let mut entries = BTreeMap::new(); - visit(root, Path::new(""), &mut entries)?; - Ok(entries) -} - -fn visit(root: &Path, relative: &Path, entries: &mut Witness) -> Result<(), Box> { - let path = root.join(relative); - let metadata = fs::symlink_metadata(&path)?; - let contents = if metadata.is_dir() { - for entry in fs::read_dir(&path)? { - visit(root, &relative.join(entry?.file_name()), entries)?; - } - Contents::Directory - } else if metadata.is_file() { - Contents::File(fs::read(&path)?) - } else if metadata.is_symlink() { - Contents::Link(fs::read_link(&path)?) - } else { - return Err("unexpected evidence kind in owned fixture".into()); - }; - entries.insert( - relative.to_path_buf(), - (metadata.dev(), metadata.ino(), contents), - ); - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_storage.rs b/src/adapters/store_migration/filesystem_migration_storage.rs deleted file mode 100644 index b7ff6e4d..00000000 --- a/src/adapters/store_migration/filesystem_migration_storage.rs +++ /dev/null @@ -1,242 +0,0 @@ -//! This module binds filesystem migration authority to the storage port. - -use std::io; - -use super::filesystem_migration_fixed_artifact::{ - FilesystemMigrationFixedArtifact as FixedArtifact, FilesystemMigrationFixedStage, -}; -use super::{ - CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, - FilesystemStoreMigrationAuthority, StoreMigrationStorage, filesystem_migration_namespace, -}; -use crate::adapters::filesystem_catalog_artifact; - -impl StoreMigrationStorage for FilesystemStoreMigrationAuthority { - fn verify_current(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - verify_authority(self, intent) - } - - fn write_intent_stage(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - write_stage(self, FixedArtifact::Intent, intent.encoded()) - } - - fn synchronize_intent_stage(&mut self) -> io::Result<()> { - synchronize_stage(self, FixedArtifact::Intent) - } - - fn link_intent(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - link_stage(self, FixedArtifact::Intent, intent.encoded()) - } - - fn synchronize_root_after_intent(&mut self) -> io::Result<()> { - synchronize_root(self)?; - active_stage(self, FixedArtifact::Intent)?.verify_linked(self.root()) - } - - fn remove_intent_stage(&mut self) -> io::Result<()> { - remove_stage(self, FixedArtifact::Intent) - } - - fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()> { - synchronize_root(self)?; - verify_published(self, FixedArtifact::Intent)?; - filesystem_migration_namespace::verify_intent_root(self.root()) - } - - fn admit_reader_fence(&mut self) -> io::Result<()> { - verify_published(self, FixedArtifact::Intent)?; - filesystem_migration_namespace::admit_reader_fence(self.root())?; - verify_published(self, FixedArtifact::Intent) - } - - fn admit_namespace_prefix(&mut self) -> io::Result<()> { - verify_published(self, FixedArtifact::Intent)?; - filesystem_migration_namespace::admit_namespace_prefix(self.root())?; - verify_published(self, FixedArtifact::Intent) - } - - fn synchronize_root_after_namespace(&mut self) -> io::Result<()> { - synchronize_root(self)?; - verify_published(self, FixedArtifact::Intent)?; - filesystem_migration_namespace::verify_namespace_prefix(self.root()) - } - - fn write_marker_stage(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()> { - verify_namespace_view(self)?; - write_stage(self, FixedArtifact::Marker, marker.encoded()) - } - - fn synchronize_marker_stage(&mut self) -> io::Result<()> { - synchronize_stage(self, FixedArtifact::Marker) - } - - fn link_marker(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()> { - link_stage(self, FixedArtifact::Marker, marker.encoded()) - } - - fn synchronize_root_after_marker(&mut self) -> io::Result<()> { - synchronize_root(self)?; - verify_published(self, FixedArtifact::Intent)?; - filesystem_migration_namespace::verify_namespace_contents(self.root())?; - active_stage(self, FixedArtifact::Marker)?.verify_linked(self.root()) - } - - fn remove_marker_stage(&mut self) -> io::Result<()> { - remove_stage(self, FixedArtifact::Marker) - } - - fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()> { - synchronize_root(self)?; - verify_marker_view(self) - } - - fn write_receipt_stage(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { - verify_marker_view(self)?; - write_stage(self, FixedArtifact::Receipt, receipt.encoded()) - } - - fn synchronize_receipt_stage(&mut self) -> io::Result<()> { - synchronize_stage(self, FixedArtifact::Receipt) - } - - fn link_receipt(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { - link_stage(self, FixedArtifact::Receipt, receipt.encoded()) - } - - fn synchronize_root_after_receipt(&mut self) -> io::Result<()> { - synchronize_root(self)?; - verify_published(self, FixedArtifact::Intent)?; - verify_published(self, FixedArtifact::Marker)?; - filesystem_migration_namespace::verify_marker_contents(self.root())?; - active_stage(self, FixedArtifact::Receipt)?.verify_linked(self.root()) - } - - fn remove_receipt_stage(&mut self) -> io::Result<()> { - remove_stage(self, FixedArtifact::Receipt) - } - - fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()> { - synchronize_root(self)?; - verify_receipt_view(self) - } -} - -fn verify_authority( - authority: &FilesystemStoreMigrationAuthority, - intent: &CanonicalStoreMigrationIntent, -) -> io::Result<()> { - authority.verify_current(intent).map_err(io::Error::other) -} - -fn write_stage( - authority: &mut FilesystemStoreMigrationAuthority, - artifact: FixedArtifact, - expected: &[u8], -) -> io::Result<()> { - if authority.fixed_stage.is_some() { - return Err(stage_state("migration fixed stage was already active")); - } - let stage = FilesystemMigrationFixedStage::create(authority.root(), artifact, expected)?; - authority.fixed_stage = Some(stage); - Ok(()) -} - -fn synchronize_stage( - authority: &FilesystemStoreMigrationAuthority, - artifact: FixedArtifact, -) -> io::Result<()> { - active_stage(authority, artifact)?.synchronize(authority.root()) -} - -fn link_stage( - authority: &FilesystemStoreMigrationAuthority, - artifact: FixedArtifact, - expected: &[u8], -) -> io::Result<()> { - active_stage(authority, artifact)?.link(authority.root(), artifact, expected) -} - -fn remove_stage( - authority: &mut FilesystemStoreMigrationAuthority, - artifact: FixedArtifact, -) -> io::Result<()> { - if published_stage(authority, artifact).is_some() { - return Err(stage_state("migration fixed record was already published")); - } - let stage = authority - .fixed_stage - .take() - .ok_or_else(|| stage_state("migration fixed stage was not active"))?; - if stage.artifact() != artifact { - authority.fixed_stage = Some(stage); - return Err(stage_state("a different migration fixed stage was active")); - } - let published = stage.remove(authority.root())?; - match artifact { - FixedArtifact::Intent => authority.published_intent = Some(published), - FixedArtifact::Marker => authority.published_marker = Some(published), - FixedArtifact::Receipt => authority.published_receipt = Some(published), - } - Ok(()) -} - -fn active_stage( - authority: &FilesystemStoreMigrationAuthority, - artifact: FixedArtifact, -) -> io::Result<&FilesystemMigrationFixedStage> { - let stage = authority - .fixed_stage - .as_ref() - .ok_or_else(|| stage_state("migration fixed stage was not active"))?; - if stage.artifact() == artifact { - Ok(stage) - } else { - Err(stage_state("a different migration fixed stage was active")) - } -} - -fn verify_published( - authority: &FilesystemStoreMigrationAuthority, - artifact: FixedArtifact, -) -> io::Result<()> { - published_stage(authority, artifact) - .ok_or_else(|| stage_state("migration fixed record was not published"))? - .verify_canonical(authority.root()) -} - -const fn published_stage( - authority: &FilesystemStoreMigrationAuthority, - artifact: FixedArtifact, -) -> Option<&FilesystemMigrationFixedStage> { - match artifact { - FixedArtifact::Intent => authority.published_intent.as_ref(), - FixedArtifact::Marker => authority.published_marker.as_ref(), - FixedArtifact::Receipt => authority.published_receipt.as_ref(), - } -} - -fn verify_namespace_view(authority: &FilesystemStoreMigrationAuthority) -> io::Result<()> { - verify_published(authority, FixedArtifact::Intent)?; - filesystem_migration_namespace::verify_namespace_prefix(authority.root()) -} - -fn verify_marker_view(authority: &FilesystemStoreMigrationAuthority) -> io::Result<()> { - verify_published(authority, FixedArtifact::Intent)?; - verify_published(authority, FixedArtifact::Marker)?; - filesystem_migration_namespace::verify_marker_view(authority.root()) -} - -fn verify_receipt_view(authority: &FilesystemStoreMigrationAuthority) -> io::Result<()> { - verify_published(authority, FixedArtifact::Intent)?; - verify_published(authority, FixedArtifact::Marker)?; - verify_published(authority, FixedArtifact::Receipt)?; - filesystem_migration_namespace::verify_receipt_view(authority.root()) -} - -fn synchronize_root(authority: &FilesystemStoreMigrationAuthority) -> io::Result<()> { - filesystem_catalog_artifact::synchronize_directory(authority.root()) -} - -fn stage_state(message: &'static str) -> io::Error { - io::Error::new(io::ErrorKind::InvalidData, message) -} diff --git a/src/adapters/store_migration/filesystem_migration_storage_tests.rs b/src/adapters/store_migration/filesystem_migration_storage_tests.rs deleted file mode 100644 index b0cd6560..00000000 --- a/src/adapters/store_migration/filesystem_migration_storage_tests.rs +++ /dev/null @@ -1,228 +0,0 @@ -//! Filesystem migration storage laws. - -use std::collections::BTreeSet; -use std::error::Error; -use std::ffi::OsString; -use std::fs; -use std::io; -use std::path::{Path, PathBuf}; - -use super::filesystem_migration_test_fixture::open_authority; -use super::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, - CanonicalStoreFormatMarker, StoreMigrationStorage, execute_store_migration, -}; - -#[test] -fn complete_migration_preserves_v1_bytes_and_publishes_exact_v2_prefix() --> Result<(), Box> { - let (sandbox, mut authority) = open_authority("filesystem-migration-complete")?; - let before = version_one_witness(sandbox.path())?; - let intent = authority.observe_intent()?; - - let receipt = execute_store_migration(&mut authority, &intent)?; - - assert_eq!(version_one_witness(sandbox.path())?, before); - admit_published_records(sandbox.path(), &intent, receipt.encoded())?; - assert_complete_namespace(sandbox.path())?; - drop(authority); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn existing_intent_stage_is_never_truncated() -> Result<(), Box> { - let (sandbox, mut authority) = open_authority("filesystem-migration-exclusive-stage")?; - let intent = authority.observe_intent()?; - StoreMigrationStorage::verify_current(&mut authority, &intent)?; - let stage = sandbox.path().join("migration.intent.next"); - fs::write(&stage, b"retained partial evidence")?; - - let error = StoreMigrationStorage::write_intent_stage(&mut authority, &intent) - .err() - .ok_or("existing intent stage was unexpectedly replaced")?; - - assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); - assert_eq!(fs::read(&stage)?, b"retained partial evidence"); - drop(authority); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn byte_equal_substituted_canonical_intent_is_refused() -> Result<(), Box> { - let (sandbox, mut authority) = open_authority("filesystem-migration-substituted-target")?; - let intent = authority.observe_intent()?; - StoreMigrationStorage::verify_current(&mut authority, &intent)?; - StoreMigrationStorage::write_intent_stage(&mut authority, &intent)?; - StoreMigrationStorage::synchronize_intent_stage(&mut authority)?; - fs::write(sandbox.path().join("migration.intent"), intent.encoded())?; - - let error = StoreMigrationStorage::link_intent(&mut authority, &intent) - .err() - .ok_or("substituted canonical intent was unexpectedly admitted")?; - - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - drop(authority); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn out_of_order_namespace_refuses_before_creating_its_predecessor() -> Result<(), Box> { - let (sandbox, mut authority) = open_authority("filesystem-migration-prefix-order")?; - let intent = authority.observe_intent()?; - publish_intent_and_reader(&mut authority, &intent)?; - fs::create_dir(sandbox.path().join("gc"))?; - - let error = StoreMigrationStorage::admit_namespace_prefix(&mut authority) - .err() - .ok_or("out-of-order gc namespace was unexpectedly admitted")?; - - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - assert!(!sandbox.path().join("retention").exists()); - drop(authority); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn canonical_intent_drift_refuses_before_marker_stage_creation() -> Result<(), Box> { - let (sandbox, mut authority) = open_authority("filesystem-migration-intent-drift")?; - let intent = authority.observe_intent()?; - publish_intent_and_reader(&mut authority, &intent)?; - StoreMigrationStorage::admit_namespace_prefix(&mut authority)?; - StoreMigrationStorage::synchronize_root_after_namespace(&mut authority)?; - fs::write( - sandbox.path().join("migration.intent"), - vec![0_u8; intent.encoded().len()], - )?; - let marker = CanonicalStoreFormatMarker::version_two(); - - let error = StoreMigrationStorage::write_marker_stage(&mut authority, &marker) - .err() - .ok_or("changed canonical intent unexpectedly authorized marker publication")?; - - assert_eq!(error.kind(), io::ErrorKind::InvalidData); - assert!(!sandbox.path().join("FORMAT.next").exists()); - drop(authority); - sandbox.remove()?; - Ok(()) -} - -fn publish_intent_and_reader( - authority: &mut super::FilesystemStoreMigrationAuthority, - intent: &super::CanonicalStoreMigrationIntent, -) -> io::Result<()> { - StoreMigrationStorage::verify_current(authority, intent)?; - StoreMigrationStorage::write_intent_stage(authority, intent)?; - StoreMigrationStorage::synchronize_intent_stage(authority)?; - StoreMigrationStorage::link_intent(authority, intent)?; - StoreMigrationStorage::synchronize_root_after_intent(authority)?; - StoreMigrationStorage::remove_intent_stage(authority)?; - StoreMigrationStorage::synchronize_root_after_intent_cleanup(authority)?; - StoreMigrationStorage::admit_reader_fence(authority) -} - -fn admit_published_records( - root: &Path, - expected_intent: &super::CanonicalStoreMigrationIntent, - expected_receipt: &[u8], -) -> Result<(), Box> { - let intent_bytes = fs::read(root.join("migration.intent"))?; - let marker_bytes = fs::read(root.join("FORMAT"))?; - let receipt_bytes = fs::read(root.join("migration.receipt"))?; - let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; - let marker = AdmittedStoreFormatMarker::decode(&marker_bytes)?; - let receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &intent, &marker)?; - assert_eq!(intent.encoded(), expected_intent.encoded()); - assert_eq!(receipt.encoded(), expected_receipt); - Ok(()) -} - -fn assert_complete_namespace(root: &Path) -> Result<(), Box> { - let expected = BTreeSet::from([ - OsString::from("FORMAT"), - OsString::from("HEAD"), - OsString::from("catalogs"), - OsString::from("gc"), - OsString::from("migration.intent"), - OsString::from("migration.receipt"), - OsString::from("reader.lock"), - OsString::from("recovery"), - OsString::from("retention"), - OsString::from("segments"), - OsString::from("staging"), - OsString::from("writer.lock"), - ]); - assert_eq!(directory_names(root)?, expected); - assert_eq!(fs::metadata(root.join("reader.lock"))?.len(), 0); - assert_eq!( - directory_names(&root.join("retention"))?, - BTreeSet::from([OsString::from("manifests"), OsString::from("roots")]) - ); - assert_eq!( - directory_names(&root.join("recovery"))?, - BTreeSet::from([OsString::from("dispositions")]) - ); - for relative in [ - "gc", - "retention/roots", - "retention/manifests", - "recovery/dispositions", - ] { - assert!(directory_names(&root.join(relative))?.is_empty()); - } - for stage in [ - "migration.intent.next", - "FORMAT.next", - "migration.receipt.next", - ] { - assert!(!root.join(stage).exists()); - } - Ok(()) -} - -fn version_one_witness(root: &Path) -> io::Result)>> { - let mut witness = Vec::new(); - witness.push((PathBuf::from("HEAD"), fs::read(root.join("HEAD"))?)); - for pool in ["segments", "catalogs"] { - for entry in fs::read_dir(root.join(pool))? { - let path = PathBuf::from(pool).join(entry?.file_name()); - witness.push((path.clone(), fs::read(root.join(path))?)); - } - } - witness.sort_by(|left, right| left.0.cmp(&right.0)); - Ok(witness) -} - -fn directory_names(path: &Path) -> io::Result> { - fs::read_dir(path)? - .map(|entry| entry.map(|entry| entry.file_name())) - .collect() -} - -#[test] -fn a_retained_version_one_stage_refuses_migration_before_any_intent() -> Result<(), Box> -{ - let (sandbox, authority) = open_authority("filesystem-migration-retained-stage")?; - fs::write( - sandbox.path().join("staging").join("current.seg"), - b"interrupted version-one publication", - )?; - let before = version_one_witness(sandbox.path())?; - - let error = authority - .observe_intent() - .err() - .ok_or("a store with a retained version-one stage was admitted for migration")?; - - assert!(matches!( - error, - crate::adapters::FilesystemMigrationAuthorityError::Namespace { .. } - )); - assert_eq!(version_one_witness(sandbox.path())?, before); - assert!(!sandbox.path().join("migration.intent").exists()); - assert!(!sandbox.path().join("migration.intent.next").exists()); - Ok(()) -} diff --git a/src/adapters/store_migration/filesystem_migration_test_fixture.rs b/src/adapters/store_migration/filesystem_migration_test_fixture.rs deleted file mode 100644 index 3463c6f2..00000000 --- a/src/adapters/store_migration/filesystem_migration_test_fixture.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! This test module owns one exact published version-1 migration fixture. - -use std::error::Error; -use std::fs; - -use super::filesystem_migration_authority::FilesystemStoreMigrationAuthority; -use crate::adapters::FilesystemPlatformAdmission; -use crate::adapters::filesystem_test_sandbox::TestDirectory; -use crate::adapters::test_support::decode_hex; - -const SEGMENT_HEX: &str = - include_str!("../../../conformance/segment-store/v1/one-zero-segment.hex"); -const CATALOG_HEX: &str = - include_str!("../../../conformance/segment-store/v1/one-zero-catalog.hex"); -const HEAD_HEX: &str = include_str!("../../../conformance/segment-store/v1/one-zero-head.hex"); -const CATALOG_NAME: &str = - "0000000000000001-04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320.cat"; -const SEGMENT_NAME: &str = "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc.seg"; - -pub(super) fn open_authority( - name: &str, -) -> Result<(TestDirectory, FilesystemStoreMigrationAuthority), Box> { - let sandbox = TestDirectory::create(name)?; - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - fs::write( - sandbox.path().join("segments").join(SEGMENT_NAME), - decode_hex(SEGMENT_HEX.trim())?, - )?; - fs::write( - sandbox.path().join("catalogs").join(CATALOG_NAME), - decode_hex(CATALOG_HEX.trim())?, - )?; - fs::write(sandbox.path().join("HEAD"), decode_hex(HEAD_HEX.trim())?)?; - let authority = FilesystemStoreMigrationAuthority::open(admission, maximum_policy())?; - Ok((sandbox, authority)) -} - -pub(super) const fn maximum_policy() -> crate::adapters::SegmentReadPolicy { - super::filesystem_inventory_catalogs_test_fixture::maximum_policy() -} diff --git a/src/adapters/store_migration/immutable_pool_inventory_digest.rs b/src/adapters/store_migration/immutable_pool_inventory_digest.rs deleted file mode 100644 index 188c0aa1..00000000 --- a/src/adapters/store_migration/immutable_pool_inventory_digest.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! This module owns immutable-pool inventory identity. - -/// Digest coordinate naming one canonical complete immutable-pool inventory. -/// -/// Intent admission does not prove that a current inventory has this digest. -#[must_use] -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct ImmutablePoolInventoryDigest([u8; 32]); - -impl ImmutablePoolInventoryDigest { - /// Returns the exact digest bytes. - #[must_use] - pub const fn as_bytes(&self) -> &[u8; 32] { - &self.0 - } - - pub(super) const fn from_admitted(bytes: [u8; 32]) -> Self { - Self(bytes) - } -} diff --git a/src/adapters/store_migration/initial_gc_state_digest.rs b/src/adapters/store_migration/initial_gc_state_digest.rs deleted file mode 100644 index 394c875f..00000000 --- a/src/adapters/store_migration/initial_gc_state_digest.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! This module owns the registered initial garbage-collection-state identity. - -/// Registered identity of empty version-2 garbage-collection state. -#[must_use] -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct InitialGcStateDigest([u8; 32]); - -impl InitialGcStateDigest { - /// Returns the exact digest bytes. - #[must_use] - pub const fn as_bytes(&self) -> &[u8; 32] { - &self.0 - } - - pub(super) const fn from_hash(hash: [u8; 32]) -> Self { - Self(hash) - } -} diff --git a/src/adapters/store_migration/initial_retention_state_digest.rs b/src/adapters/store_migration/initial_retention_state_digest.rs deleted file mode 100644 index 2dc9cd29..00000000 --- a/src/adapters/store_migration/initial_retention_state_digest.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! This module owns the registered initial retention-state identity. - -/// Registered identity of empty version-2 retention state. -#[must_use] -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct InitialRetentionStateDigest([u8; 32]); - -impl InitialRetentionStateDigest { - /// Returns the exact digest bytes. - #[must_use] - pub const fn as_bytes(&self) -> &[u8; 32] { - &self.0 - } - - pub(super) const fn from_hash(hash: [u8; 32]) -> Self { - Self(hash) - } -} diff --git a/src/adapters/store_migration/migration_catalog_admission.rs b/src/adapters/store_migration/migration_catalog_admission.rs deleted file mode 100644 index bcb97b22..00000000 --- a/src/adapters/store_migration/migration_catalog_admission.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! This module owns bounded catalog admission for migration pool inventory. - -use crate::adapters::{ - AdmittedSegment, CatalogAdmissionError, ChecksummedCatalog, SegmentDigest, SegmentReadPolicy, -}; -use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; - -use super::{migration_catalog_plan, migration_catalog_records}; - -pub(super) struct AdmittedMigrationCatalog<'a> { - catalog: ChecksummedCatalog<'a>, -} - -impl AdmittedMigrationCatalog<'_> { - pub(super) const fn generation(&self) -> CatalogGeneration { - self.catalog.generation() - } - - pub(super) const fn length(&self) -> CatalogLength { - self.catalog.length() - } - - pub(super) const fn digest(&self) -> CatalogDigest { - self.catalog.digest() - } -} - -pub(super) enum MigrationCatalogAdmissionError { - Catalog(Box), - SegmentSource { - digest: SegmentDigest, - source: E, - }, - SegmentCoordinate { - expected: SegmentDigest, - observed: SegmentDigest, - }, -} - -pub(super) enum MigrationSegmentLoadError { - Missing, - Source(E), -} - -pub(super) fn admit( - catalog: ChecksummedCatalog<'_>, - policy: SegmentReadPolicy, - mut load: F, -) -> Result, MigrationCatalogAdmissionError> -where - F: FnMut(SegmentDigest) -> Result, MigrationSegmentLoadError>, -{ - let mut plan = migration_catalog_plan::plan(catalog)?; - plan.sort_unstable_by_key(|entry| entry.physical_order()); - for entries in - plan.chunk_by(|first, second| first.entry.segment_digest() == second.entry.segment_digest()) - { - admit_segment(entries, policy, &mut load)?; - } - Ok(AdmittedMigrationCatalog { catalog }) -} - -fn admit_segment( - entries: &[migration_catalog_plan::PlannedEntry], - policy: SegmentReadPolicy, - load: &mut F, -) -> Result<(), MigrationCatalogAdmissionError> -where - F: FnMut(SegmentDigest) -> Result, MigrationSegmentLoadError>, -{ - let Some(first) = entries.first() else { - return Ok(()); - }; - let expected = first.entry.segment_digest(); - let encoded = match load(expected) { - Ok(encoded) => encoded, - Err(MigrationSegmentLoadError::Missing) => { - return Err(catalog_error(CatalogAdmissionError::MissingSegment { - digest: expected, - })); - } - Err(MigrationSegmentLoadError::Source(source)) => { - return Err(MigrationCatalogAdmissionError::SegmentSource { - digest: expected, - source, - }); - } - }; - let segment = AdmittedSegment::decode(&encoded, policy).map_err(|source| { - catalog_error(CatalogAdmissionError::Segment { - digest: expected, - source: Box::new(source), - }) - })?; - if segment.digest() != expected { - return Err(MigrationCatalogAdmissionError::SegmentCoordinate { - expected, - observed: segment.digest(), - }); - } - migration_catalog_records::validate(entries, &segment).map_err(catalog_error) -} - -pub(super) fn catalog_error(source: CatalogAdmissionError) -> MigrationCatalogAdmissionError { - MigrationCatalogAdmissionError::Catalog(Box::new(source)) -} diff --git a/src/adapters/store_migration/migration_catalog_coordinates.rs b/src/adapters/store_migration/migration_catalog_coordinates.rs deleted file mode 100644 index d3baaaf3..00000000 --- a/src/adapters/store_migration/migration_catalog_coordinates.rs +++ /dev/null @@ -1,52 +0,0 @@ -//! This module owns admitted catalog coordinates for intent encoding. - -use crate::{CatalogDigest, CatalogGeneration, CatalogLength, CatalogSnapshot}; - -#[derive(Clone, Copy)] -pub(super) struct MigrationCatalogCoordinates { - generation: CatalogGeneration, - length: CatalogLength, - digest: CatalogDigest, - predecessor: Option, -} - -impl MigrationCatalogCoordinates { - pub(super) const fn new( - generation: CatalogGeneration, - length: CatalogLength, - digest: CatalogDigest, - predecessor: Option, - ) -> Self { - Self { - generation, - length, - digest, - predecessor, - } - } - - pub(super) const fn from_snapshot(snapshot: &CatalogSnapshot<'_, '_, '_>) -> Self { - Self::new( - snapshot.generation(), - snapshot.catalog_length(), - snapshot.catalog_digest(), - snapshot.previous_catalog_digest(), - ) - } - - pub(super) const fn generation(self) -> CatalogGeneration { - self.generation - } - - pub(super) const fn length(self) -> CatalogLength { - self.length - } - - pub(super) const fn digest(self) -> CatalogDigest { - self.digest - } - - pub(super) const fn predecessor(self) -> Option { - self.predecessor - } -} diff --git a/src/adapters/store_migration/migration_catalog_plan.rs b/src/adapters/store_migration/migration_catalog_plan.rs deleted file mode 100644 index deefce8f..00000000 --- a/src/adapters/store_migration/migration_catalog_plan.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! This module owns bounded physical catalog-entry planning for migration. - -use crate::adapters::{ - CatalogAdmissionError, CatalogAllocationPhase, ChecksummedCatalog, DecodedCatalogEntry, - SegmentDigest, -}; - -use super::migration_catalog_admission::{MigrationCatalogAdmissionError, catalog_error}; - -#[derive(Clone, Copy)] -pub(super) struct PlannedEntry { - pub(super) ordinal: usize, - pub(super) entry: DecodedCatalogEntry, -} - -impl PlannedEntry { - pub(super) const fn physical_order(self) -> (SegmentDigest, u64, usize) { - ( - self.entry.segment_digest(), - self.entry.record_offset(), - self.ordinal, - ) - } -} - -pub(super) fn plan( - catalog: ChecksummedCatalog<'_>, -) -> Result, MigrationCatalogAdmissionError> { - let requested = usize::try_from(catalog.entry_count()).map_err(|_source| { - catalog_error(CatalogAdmissionError::EntryCountHostWidth { - observed: catalog.entry_count(), - }) - })?; - let mut plan = Vec::new(); - plan.try_reserve_exact(requested).map_err(|source| { - catalog_error(CatalogAdmissionError::Allocation { - phase: CatalogAllocationPhase::EntryPlan, - requested, - source, - }) - })?; - let entries = catalog - .entries() - .map_err(|source| catalog_error(CatalogAdmissionError::Catalog { source }))?; - for (ordinal, entry) in entries.enumerate() { - let entry = - entry.map_err(|source| catalog_error(CatalogAdmissionError::Catalog { source }))?; - plan.push(PlannedEntry { ordinal, entry }); - } - Ok(plan) -} diff --git a/src/adapters/store_migration/migration_catalog_records.rs b/src/adapters/store_migration/migration_catalog_records.rs deleted file mode 100644 index 0156c876..00000000 --- a/src/adapters/store_migration/migration_catalog_records.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! This module owns exact migration catalog-to-segment record binding. - -use crate::adapters::{ - AdmittedSegment, AdmittedSegmentRecord, CatalogAdmissionError, DecodedCatalogEntry, -}; - -use super::migration_catalog_plan::PlannedEntry; - -pub(super) fn validate( - entries: &[PlannedEntry], - segment: &AdmittedSegment<'_>, -) -> Result<(), CatalogAdmissionError> { - let digest = segment.digest(); - let mut pending = entries.iter().peekable(); - let mut cursor = segment.record_cursor(); - while let Some(located) = - cursor - .next_record() - .map_err(|source| CatalogAdmissionError::Segment { - digest, - source: Box::new(source), - })? - { - refuse_preceding(&mut pending, located.offset)?; - validate_at_offset(&mut pending, located.offset, located.record)?; - } - cursor - .finish() - .map_err(|source| CatalogAdmissionError::Segment { - digest, - source: Box::new(source), - })?; - pending - .next() - .map_or(Ok(()), |entry| Err(location_error(entry.entry))) -} - -fn refuse_preceding( - pending: &mut std::iter::Peekable>, - record_offset: u64, -) -> Result<(), CatalogAdmissionError> { - match pending.peek() { - Some(entry) if entry.entry.record_offset() < record_offset => { - Err(location_error(entry.entry)) - } - Some(_) | None => Ok(()), - } -} - -fn validate_at_offset( - pending: &mut std::iter::Peekable>, - record_offset: u64, - record: AdmittedSegmentRecord<'_>, -) -> Result<(), CatalogAdmissionError> { - while matches!(pending.peek(), Some(entry) if entry.entry.record_offset() == record_offset) { - let Some(entry) = pending.next() else { - break; - }; - validate_record(entry.entry, record)?; - } - Ok(()) -} - -fn validate_record( - entry: DecodedCatalogEntry, - record: AdmittedSegmentRecord<'_>, -) -> Result<(), CatalogAdmissionError> { - if record.header().record_length() != entry.record_length() { - return Err(location_error(entry)); - } - if record.identity() != entry.identity() { - return Err(CatalogAdmissionError::RecordIdentityMismatch { - expected: entry.identity(), - observed: record.identity(), - }); - } - if record.checksum() != entry.checksum() { - return Err(CatalogAdmissionError::RecordChecksumMismatch { - expected: entry.checksum(), - observed: record.checksum(), - }); - } - Ok(()) -} - -const fn location_error(entry: DecodedCatalogEntry) -> CatalogAdmissionError { - CatalogAdmissionError::LocationNotTopLevel { - identity: entry.identity(), - segment_digest: entry.segment_digest(), - record_offset: entry.record_offset(), - record_length: entry.record_length().get(), - } -} diff --git a/src/adapters/store_migration/migration_error.rs b/src/adapters/store_migration/migration_error.rs deleted file mode 100644 index 9fc39e2f..00000000 --- a/src/adapters/store_migration/migration_error.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! This boundary module owns ordered store-migration execution failures. - -use std::error::Error; -use std::fmt; -use std::io; - -use super::StoreMigrationPhase; - -/// Failure before or during ordered version-2 store migration. -#[derive(Debug)] -pub enum StoreMigrationError { - /// Current version-1 authority could not be revalidated before mutation. - CurrentVerification { - /// Preserved storage refusal. - source: io::Error, - }, - /// One exact durability phase failed. - Storage { - /// Phase attempted when storage refused. - phase: StoreMigrationPhase, - /// Preserved storage refusal. - source: io::Error, - }, -} - -impl fmt::Display for StoreMigrationError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::CurrentVerification { .. } => { - formatter.write_str("store-migration authority verification failed") - } - Self::Storage { phase, .. } => { - write!(formatter, "store-migration phase {phase} failed") - } - } - } -} - -impl Error for StoreMigrationError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::CurrentVerification { source } | Self::Storage { source, .. } => Some(source), - } - } -} diff --git a/src/adapters/store_migration/migration_execution.rs b/src/adapters/store_migration/migration_execution.rs deleted file mode 100644 index 065fdd32..00000000 --- a/src/adapters/store_migration/migration_execution.rs +++ /dev/null @@ -1,136 +0,0 @@ -//! This boundary module owns ordered version-2 store-migration execution. - -use std::io; - -use super::{ - CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, - StoreMigrationError, StoreMigrationPhase, StoreMigrationStorage, -}; - -/// Executes one version-2 migration under revalidated version-1 authority. -/// -/// The returned receipt exists only after all canonical artifacts are visible, -/// all retained stages are removed, and final store-root cleanup is synchronized. -/// -/// # Errors -/// -/// Returns [`StoreMigrationError`] for current-state revalidation or the exact -/// failed durability phase. Failure returns no receipt. -pub fn execute_store_migration( - storage: &mut impl StoreMigrationStorage, - intent: &CanonicalStoreMigrationIntent, -) -> Result { - storage - .verify_current(intent) - .map_err(|source| StoreMigrationError::CurrentVerification { source })?; - let marker = CanonicalStoreFormatMarker::version_two(); - let receipt = CanonicalStoreMigrationReceipt::from_canonical(intent, &marker); - execute_intent(storage, intent)?; - execute_namespace(storage)?; - execute_marker(storage, &marker)?; - execute_receipt(storage, &receipt)?; - Ok(receipt) -} - -fn execute_intent( - storage: &mut impl StoreMigrationStorage, - intent: &CanonicalStoreMigrationIntent, -) -> Result<(), StoreMigrationError> { - require( - storage.write_intent_stage(intent), - StoreMigrationPhase::WriteIntentStage, - )?; - require( - storage.synchronize_intent_stage(), - StoreMigrationPhase::SynchronizeIntentStage, - )?; - require(storage.link_intent(intent), StoreMigrationPhase::LinkIntent)?; - require( - storage.synchronize_root_after_intent(), - StoreMigrationPhase::SynchronizeRootAfterIntent, - )?; - require( - storage.remove_intent_stage(), - StoreMigrationPhase::RemoveIntentStage, - )?; - require( - storage.synchronize_root_after_intent_cleanup(), - StoreMigrationPhase::SynchronizeRootAfterIntentCleanup, - ) -} - -fn execute_namespace(storage: &mut impl StoreMigrationStorage) -> Result<(), StoreMigrationError> { - require( - storage.admit_reader_fence(), - StoreMigrationPhase::AdmitReaderFence, - )?; - require( - storage.admit_namespace_prefix(), - StoreMigrationPhase::AdmitNamespacePrefix, - )?; - require( - storage.synchronize_root_after_namespace(), - StoreMigrationPhase::SynchronizeRootAfterNamespace, - ) -} - -fn execute_marker( - storage: &mut impl StoreMigrationStorage, - marker: &CanonicalStoreFormatMarker, -) -> Result<(), StoreMigrationError> { - require( - storage.write_marker_stage(marker), - StoreMigrationPhase::WriteMarkerStage, - )?; - require( - storage.synchronize_marker_stage(), - StoreMigrationPhase::SynchronizeMarkerStage, - )?; - require(storage.link_marker(marker), StoreMigrationPhase::LinkMarker)?; - require( - storage.synchronize_root_after_marker(), - StoreMigrationPhase::SynchronizeRootAfterMarker, - )?; - require( - storage.remove_marker_stage(), - StoreMigrationPhase::RemoveMarkerStage, - )?; - require( - storage.synchronize_root_after_marker_cleanup(), - StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup, - ) -} - -fn execute_receipt( - storage: &mut impl StoreMigrationStorage, - receipt: &CanonicalStoreMigrationReceipt, -) -> Result<(), StoreMigrationError> { - require( - storage.write_receipt_stage(receipt), - StoreMigrationPhase::WriteReceiptStage, - )?; - require( - storage.synchronize_receipt_stage(), - StoreMigrationPhase::SynchronizeReceiptStage, - )?; - require( - storage.link_receipt(receipt), - StoreMigrationPhase::LinkReceipt, - )?; - require( - storage.synchronize_root_after_receipt(), - StoreMigrationPhase::SynchronizeRootAfterReceipt, - )?; - require( - storage.remove_receipt_stage(), - StoreMigrationPhase::RemoveReceiptStage, - )?; - require( - storage.synchronize_root_after_receipt_cleanup(), - StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup, - ) -} - -fn require(result: io::Result, phase: StoreMigrationPhase) -> Result { - result.map_err(|source| StoreMigrationError::Storage { phase, source }) -} diff --git a/src/adapters/store_migration/migration_intent_decode_error.rs b/src/adapters/store_migration/migration_intent_decode_error.rs deleted file mode 100644 index 5d6c1e3e..00000000 --- a/src/adapters/store_migration/migration_intent_decode_error.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! This boundary module owns store-migration intent decoding failures. - -use crate::{CatalogGenerationError, CatalogLengthError}; - -/// Failure to decode and admit one version-2 migration intent. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StoreMigrationIntentDecodeError { - /// The input was not exactly one complete intent. - WrongLength { - /// Required fixed width. - expected: usize, - /// Observed input width. - observed: usize, - }, - /// The fixed record magic was not canonical. - InvalidMagic { - /// Observed 16 magic bytes. - observed: [u8; 16], - }, - /// The format version is unsupported. - UnsupportedVersion { - /// Supported version. - expected: u16, - /// Observed version. - observed: u16, - }, - /// The record-length field was noncanonical. - InvalidRecordLength { - /// Required record length. - expected: u16, - /// Observed record length. - observed: u16, - }, - /// The intent carried unsupported flags. - UnsupportedFlags { - /// Observed flag bits. - observed: u32, - }, - /// The checksum did not match the exact prefix. - ChecksumMismatch { - /// Computed canonical checksum. - expected: [u8; 32], - /// Checksum stored in the record. - observed: [u8; 32], - }, - /// The catalog generation was not positive. - InvalidCatalogGeneration { - /// Observed generation. - observed: u64, - /// Precise generation refusal. - source: CatalogGenerationError, - }, - /// The catalog length was outside the canonical version-1 grammar. - InvalidCatalogLength { - /// Observed length. - observed: u64, - /// Precise catalog-length refusal. - source: CatalogLengthError, - }, - /// Generation 1 carried a forbidden predecessor. - NonZeroInitialPredecessor { - /// Observed nonzero predecessor digest. - observed: [u8; 32], - }, - /// A later generation omitted its required predecessor. - MissingSuccessorPredecessor { - /// Observed later generation. - generation: u64, - }, - /// The target definition was not the registered version-2 definition. - DefinitionDigestMismatch { - /// Registered version-2 definition digest. - expected: [u8; 32], - /// Observed target definition digest. - observed: [u8; 32], - }, - /// The stored identifier did not match the deterministic derivation. - StoreIdentifierMismatch { - /// Identifier derived from the admitted semantic fields. - expected: [u8; 32], - /// Identifier stored in the record. - observed: [u8; 32], - }, -} diff --git a/src/adapters/store_migration/migration_intent_decode_error_display.rs b/src/adapters/store_migration/migration_intent_decode_error_display.rs deleted file mode 100644 index daeebd27..00000000 --- a/src/adapters/store_migration/migration_intent_decode_error_display.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! This boundary module owns migration-intent error formatting and sources. - -use std::error::Error; -use std::fmt; - -use super::StoreMigrationIntentDecodeError; - -impl fmt::Display for StoreMigrationIntentDecodeError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::WrongLength { expected, observed } => write!( - formatter, - "migration intent requires {expected} bytes, observed {observed}" - ), - Self::InvalidMagic { .. } => formatter.write_str("invalid migration-intent magic"), - Self::UnsupportedVersion { expected, observed } => write!( - formatter, - "unsupported migration-intent version {observed}; expected {expected}" - ), - Self::InvalidRecordLength { expected, observed } => write!( - formatter, - "migration-intent record length {observed}; expected {expected}" - ), - Self::UnsupportedFlags { observed } => { - write!( - formatter, - "unsupported migration-intent flags {observed:#010x}" - ) - } - Self::ChecksumMismatch { .. } => { - formatter.write_str("migration-intent checksum mismatch") - } - Self::InvalidCatalogGeneration { observed, .. } => { - write!(formatter, "invalid migration catalog generation {observed}") - } - Self::InvalidCatalogLength { observed, .. } => { - write!(formatter, "invalid migration catalog length {observed}") - } - Self::NonZeroInitialPredecessor { .. } => { - formatter.write_str("initial migration catalog forbids a predecessor") - } - Self::MissingSuccessorPredecessor { generation } => write!( - formatter, - "migration catalog generation {generation} requires a predecessor" - ), - Self::DefinitionDigestMismatch { .. } => { - formatter.write_str("migration target definition digest mismatch") - } - Self::StoreIdentifierMismatch { .. } => { - formatter.write_str("migration store identifier mismatch") - } - } - } -} - -impl Error for StoreMigrationIntentDecodeError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::InvalidCatalogGeneration { source, .. } => Some(source), - Self::InvalidCatalogLength { source, .. } => Some(source), - _ => None, - } - } -} diff --git a/src/adapters/store_migration/migration_intent_decoder.rs b/src/adapters/store_migration/migration_intent_decoder.rs deleted file mode 100644 index 7eeb34ed..00000000 --- a/src/adapters/store_migration/migration_intent_decoder.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! This boundary module owns store-migration intent decoding order. - -use super::admitted_migration_intent::StoreMigrationIntentFields; -use super::migration_intent_format::{self as format, StoreIdentifierFields}; -use super::migration_record_bytes::{ - read_array, read_u16, read_u32, read_u64, require_length, wrong_length, -}; -use super::{ - AdmittedStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, - StoreIdentifier, StoreMigrationIntentDecodeError, StoreRootDeviceIdentity, - StoreRootFileIdentity, StoreRootMountIdentity, -}; -use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; - -pub(super) fn decode( - encoded: &[u8], -) -> Result, StoreMigrationIntentDecodeError> { - require_length(encoded)?; - validate_fixed_fields(encoded)?; - verify_checksum(encoded)?; - let catalog_generation = read_catalog_generation(encoded)?; - let catalog_length = read_catalog_length(encoded)?; - let catalog_digest = CatalogDigest::from_validated(read_array(encoded, 40)?); - let predecessor_catalog_digest = - read_predecessor(catalog_generation, read_array(encoded, 72)?)?; - let fields = StoreMigrationIntentFields { - catalog_generation, - catalog_length, - catalog_digest, - predecessor_catalog_digest, - inventory_digest: ImmutablePoolInventoryDigest::from_admitted(read_array(encoded, 104)?), - root_device_identity: StoreRootDeviceIdentity::from_admitted(read_u64(encoded, 136)?), - root_mount_identity: StoreRootMountIdentity::from_admitted(read_u64(encoded, 144)?), - root_file_identity: StoreRootFileIdentity::from_admitted(read_u64(encoded, 152)?), - target_definition_digest: read_definition_digest(encoded)?, - store_identifier: StoreIdentifier::from_hash(read_array(encoded, 192)?), - }; - verify_store_identifier(&fields)?; - Ok(AdmittedStoreMigrationIntent::admitted( - encoded, - fields, - format::digest(encoded), - )) -} - -fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { - let magic = read_array(encoded, 0)?; - if magic != format::MAGIC { - return Err(StoreMigrationIntentDecodeError::InvalidMagic { observed: magic }); - } - let version = read_u16(encoded, 16)?; - if version != format::VERSION { - return Err(StoreMigrationIntentDecodeError::UnsupportedVersion { - expected: format::VERSION, - observed: version, - }); - } - let record_length = read_u16(encoded, 18)?; - if record_length != format::RECORD_LENGTH { - return Err(StoreMigrationIntentDecodeError::InvalidRecordLength { - expected: format::RECORD_LENGTH, - observed: record_length, - }); - } - let flags = read_u32(encoded, 20)?; - if flags != 0 { - return Err(StoreMigrationIntentDecodeError::UnsupportedFlags { observed: flags }); - } - Ok(()) -} - -fn verify_checksum(encoded: &[u8]) -> Result<(), StoreMigrationIntentDecodeError> { - let preimage = encoded - .get(..format::CHECKSUM_OFFSET) - .ok_or_else(|| wrong_length(encoded))?; - let observed = read_array(encoded, format::CHECKSUM_OFFSET)?; - let expected = format::checksum(preimage); - if observed == expected { - Ok(()) - } else { - Err(StoreMigrationIntentDecodeError::ChecksumMismatch { expected, observed }) - } -} - -fn read_catalog_generation( - encoded: &[u8], -) -> Result { - let observed = read_u64(encoded, 24)?; - CatalogGeneration::new(observed).map_err(|source| { - StoreMigrationIntentDecodeError::InvalidCatalogGeneration { observed, source } - }) -} - -fn read_catalog_length(encoded: &[u8]) -> Result { - let observed = read_u64(encoded, 32)?; - CatalogLength::new(observed).map_err(|source| { - StoreMigrationIntentDecodeError::InvalidCatalogLength { observed, source } - }) -} - -fn read_predecessor( - generation: CatalogGeneration, - observed: [u8; 32], -) -> Result, StoreMigrationIntentDecodeError> { - if generation.get() == 1 { - return if observed == format::ZERO_DIGEST { - Ok(None) - } else { - Err(StoreMigrationIntentDecodeError::NonZeroInitialPredecessor { observed }) - }; - } - if observed == format::ZERO_DIGEST { - return Err( - StoreMigrationIntentDecodeError::MissingSuccessorPredecessor { - generation: generation.get(), - }, - ); - } - Ok(Some(CatalogDigest::from_validated(observed))) -} - -fn read_definition_digest( - encoded: &[u8], -) -> Result { - let observed = read_array(encoded, 160)?; - if observed == *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes() { - Ok(StoreFormatDefinitionDigest::VERSION_TWO) - } else { - Err(StoreMigrationIntentDecodeError::DefinitionDigestMismatch { - expected: *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes(), - observed, - }) - } -} - -fn verify_store_identifier( - fields: &StoreMigrationIntentFields, -) -> Result<(), StoreMigrationIntentDecodeError> { - let expected = format::store_identifier(&StoreIdentifierFields { - catalog_generation: fields.catalog_generation, - catalog_length: fields.catalog_length, - catalog_digest: fields.catalog_digest, - predecessor_catalog_digest: fields.predecessor_catalog_digest, - inventory_digest: fields.inventory_digest, - target_definition_digest: fields.target_definition_digest, - }); - let observed = *fields.store_identifier.as_bytes(); - if observed == *expected.as_bytes() { - Ok(()) - } else { - Err(StoreMigrationIntentDecodeError::StoreIdentifierMismatch { - expected: *expected.as_bytes(), - observed, - }) - } -} diff --git a/src/adapters/store_migration/migration_intent_digest.rs b/src/adapters/store_migration/migration_intent_digest.rs deleted file mode 100644 index 8cce8fc2..00000000 --- a/src/adapters/store_migration/migration_intent_digest.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! This module owns canonical migration-intent identity. - -/// Domain-separated digest of one complete canonical migration intent. -#[must_use] -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct StoreMigrationIntentDigest([u8; 32]); - -impl StoreMigrationIntentDigest { - /// Returns the exact digest bytes. - #[must_use] - pub const fn as_bytes(&self) -> &[u8; 32] { - &self.0 - } - - pub(super) const fn from_hash(hash: [u8; 32]) -> Self { - Self(hash) - } -} diff --git a/src/adapters/store_migration/migration_intent_encoder.rs b/src/adapters/store_migration/migration_intent_encoder.rs deleted file mode 100644 index 1aebbf1a..00000000 --- a/src/adapters/store_migration/migration_intent_encoder.rs +++ /dev/null @@ -1,86 +0,0 @@ -//! This boundary module owns canonical migration-intent encoding. - -use super::admitted_migration_intent::StoreMigrationIntentFields; -use super::migration_catalog_coordinates::MigrationCatalogCoordinates; -use super::migration_intent_format::StoreIdentifierFields; -use super::store_root_identity::StoreRootIdentities; -use super::{ - CanonicalStoreMigrationIntent, ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, - StoreIdentifier, migration_intent_format as format, -}; - -pub(super) fn encode( - catalog: MigrationCatalogCoordinates, - inventory_digest: ImmutablePoolInventoryDigest, - roots: StoreRootIdentities, -) -> CanonicalStoreMigrationIntent { - let fields = StoreIdentifierFields { - catalog_generation: catalog.generation(), - catalog_length: catalog.length(), - catalog_digest: catalog.digest(), - predecessor_catalog_digest: catalog.predecessor(), - inventory_digest, - target_definition_digest: StoreFormatDefinitionDigest::VERSION_TWO, - }; - let store_identifier = format::store_identifier(&fields); - let mut encoded = [0_u8; format::ENCODED_LENGTH]; - let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); - write_preimage(preimage, &fields, roots, store_identifier); - checksum_slot.copy_from_slice(&format::checksum(preimage)); - let digest = format::digest(&encoded); - CanonicalStoreMigrationIntent::admitted( - encoded, - StoreMigrationIntentFields { - catalog_generation: fields.catalog_generation, - catalog_length: fields.catalog_length, - catalog_digest: fields.catalog_digest, - predecessor_catalog_digest: fields.predecessor_catalog_digest, - inventory_digest: fields.inventory_digest, - root_device_identity: roots.device(), - root_mount_identity: roots.mount(), - root_file_identity: roots.file(), - target_definition_digest: fields.target_definition_digest, - store_identifier, - }, - digest, - ) -} - -fn write_preimage( - output: &mut [u8], - fields: &StoreIdentifierFields, - roots: StoreRootIdentities, - store_identifier: StoreIdentifier, -) { - let (magic, output) = output.split_at_mut(16); - magic.copy_from_slice(&format::MAGIC); - let (version, output) = output.split_at_mut(2); - version.copy_from_slice(&format::VERSION.to_be_bytes()); - let (record_length, output) = output.split_at_mut(2); - record_length.copy_from_slice(&format::RECORD_LENGTH.to_be_bytes()); - let (flags, output) = output.split_at_mut(4); - flags.copy_from_slice(&0_u32.to_be_bytes()); - let (generation, output) = output.split_at_mut(8); - generation.copy_from_slice(&fields.catalog_generation.get().to_be_bytes()); - let (catalog_length, output) = output.split_at_mut(8); - catalog_length.copy_from_slice(&fields.catalog_length.get().to_be_bytes()); - let (catalog_digest, output) = output.split_at_mut(32); - catalog_digest.copy_from_slice(fields.catalog_digest.as_bytes()); - let predecessor = fields - .predecessor_catalog_digest - .as_ref() - .map_or(&format::ZERO_DIGEST, crate::CatalogDigest::as_bytes); - let (predecessor_digest, output) = output.split_at_mut(32); - predecessor_digest.copy_from_slice(predecessor); - let (inventory_digest, output) = output.split_at_mut(32); - inventory_digest.copy_from_slice(fields.inventory_digest.as_bytes()); - let (device_identity, output) = output.split_at_mut(8); - device_identity.copy_from_slice(&roots.device().get().to_be_bytes()); - let (mount_identity, output) = output.split_at_mut(8); - mount_identity.copy_from_slice(&roots.mount().get().to_be_bytes()); - let (file_identity, output) = output.split_at_mut(8); - file_identity.copy_from_slice(&roots.file().get().to_be_bytes()); - let (definition_digest, store_identifier_slot) = output.split_at_mut(32); - definition_digest.copy_from_slice(fields.target_definition_digest.as_bytes()); - store_identifier_slot.copy_from_slice(store_identifier.as_bytes()); -} diff --git a/src/adapters/store_migration/migration_intent_format.rs b/src/adapters/store_migration/migration_intent_format.rs deleted file mode 100644 index 9f0de80e..00000000 --- a/src/adapters/store_migration/migration_intent_format.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! This boundary module owns shared migration-intent format identity. - -use super::{ - ImmutablePoolInventoryDigest, StoreFormatDefinitionDigest, StoreIdentifier, - StoreMigrationIntentDigest, -}; -use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; - -pub(super) const CHECKSUM_OFFSET: usize = 224; -pub(in crate::adapters) const ENCODED_LENGTH: usize = 256; -pub(super) const MAGIC: [u8; 16] = *b"KEEP:MIG:INT2\0\0\0"; -pub(super) const RECORD_LENGTH: u16 = 256; -pub(super) const VERSION: u16 = 2; -const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-intent-checksum/v2\0"; -const DIGEST_DOMAIN: &[u8] = b"keep.store-migration-intent/v2\0"; -const STORE_IDENTIFIER_DOMAIN: &[u8] = b"keep.store-identifier/v2\0"; -pub(super) const ZERO_DIGEST: [u8; 32] = [0; 32]; - -pub(super) struct StoreIdentifierFields { - pub(super) catalog_generation: CatalogGeneration, - pub(super) catalog_length: CatalogLength, - pub(super) catalog_digest: CatalogDigest, - pub(super) predecessor_catalog_digest: Option, - pub(super) inventory_digest: ImmutablePoolInventoryDigest, - pub(super) target_definition_digest: StoreFormatDefinitionDigest, -} - -pub(super) fn checksum(preimage: &[u8]) -> [u8; 32] { - hash(CHECKSUM_DOMAIN, &[preimage]) -} - -pub(super) fn digest(encoded: &[u8]) -> StoreMigrationIntentDigest { - StoreMigrationIntentDigest::from_hash(hash(DIGEST_DOMAIN, &[encoded])) -} - -pub(super) fn store_identifier(fields: &StoreIdentifierFields) -> StoreIdentifier { - let predecessor = fields - .predecessor_catalog_digest - .as_ref() - .map_or(&ZERO_DIGEST, CatalogDigest::as_bytes); - StoreIdentifier::from_hash(hash( - STORE_IDENTIFIER_DOMAIN, - &[ - &fields.catalog_generation.get().to_be_bytes(), - &fields.catalog_length.get().to_be_bytes(), - fields.catalog_digest.as_bytes(), - predecessor, - fields.inventory_digest.as_bytes(), - fields.target_definition_digest.as_bytes(), - ], - )) -} - -fn hash(domain: &[u8], fields: &[&[u8]]) -> [u8; 32] { - let mut hasher = blake3::Hasher::new(); - hasher.update(domain); - for field in fields { - hasher.update(field); - } - *hasher.finalize().as_bytes() -} diff --git a/src/adapters/store_migration/migration_inventory_entry.rs b/src/adapters/store_migration/migration_inventory_entry.rs deleted file mode 100644 index e04b89fc..00000000 --- a/src/adapters/store_migration/migration_inventory_entry.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! This boundary module owns canonical migration inventory entries. - -use crate::{AdmittedCatalog, AdmittedSegment}; - -use super::migration_catalog_admission::AdmittedMigrationCatalog; - -const SEGMENT_KIND: u8 = 1; -const CATALOG_KIND: u8 = 2; -const ENCODED_LENGTH: usize = 56; - -/// Canonical physical coordinate for one admitted version-1 pool artifact. -#[must_use] -#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub struct StoreMigrationInventoryEntry([u8; ENCODED_LENGTH]); - -impl StoreMigrationInventoryEntry { - /// Constructs the canonical entry for one completely admitted segment. - pub const fn from_segment(segment: &AdmittedSegment<'_>) -> Self { - Self(encode( - SEGMENT_KIND, - 0, - segment.segment_length(), - segment.digest().as_bytes(), - )) - } - - /// Constructs the canonical entry for one completely admitted catalog. - pub const fn from_catalog(catalog: &AdmittedCatalog<'_, '_>) -> Self { - Self(encode( - CATALOG_KIND, - catalog.generation().get(), - catalog.length().get(), - catalog.digest().as_bytes(), - )) - } - - pub(super) const fn from_migration_catalog(catalog: &AdmittedMigrationCatalog<'_>) -> Self { - Self(encode( - CATALOG_KIND, - catalog.generation().get(), - catalog.length().get(), - catalog.digest().as_bytes(), - )) - } - - /// Returns the exact 56 canonical bytes. - pub const fn encoded(&self) -> &[u8; ENCODED_LENGTH] { - &self.0 - } -} - -const fn encode(kind: u8, generation: u64, length: u64, digest: &[u8; 32]) -> [u8; ENCODED_LENGTH] { - let mut encoded = [0_u8; ENCODED_LENGTH]; - let (kind_and_reserved, remainder) = encoded.split_at_mut(8); - kind_and_reserved.copy_from_slice(&[kind, 0, 0, 0, 0, 0, 0, 0]); - let (generation_bytes, remainder) = remainder.split_at_mut(8); - generation_bytes.copy_from_slice(&generation.to_be_bytes()); - let (length_bytes, digest_bytes) = remainder.split_at_mut(8); - length_bytes.copy_from_slice(&length.to_be_bytes()); - digest_bytes.copy_from_slice(digest); - encoded -} diff --git a/src/adapters/store_migration/migration_inventory_entry_count.rs b/src/adapters/store_migration/migration_inventory_entry_count.rs deleted file mode 100644 index e0bc3c75..00000000 --- a/src/adapters/store_migration/migration_inventory_entry_count.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! This module owns the bounded migration inventory entry count. - -use super::StoreMigrationInventoryEntryCountError; - -/// Exact number of canonical entries expected in one migration inventory. -#[must_use] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct StoreMigrationInventoryEntryCount(u32); - -impl StoreMigrationInventoryEntryCount { - /// Largest number of immutable-pool entries admitted by version 2. - pub const MAXIMUM: u32 = 2_097_152; - - /// Admits one exact entry count, including an empty inventory. - /// - /// # Errors - /// - /// Returns [`StoreMigrationInventoryEntryCountError`] above - /// [`Self::MAXIMUM`]. - pub const fn new(value: u32) -> Result { - if value <= Self::MAXIMUM { - Ok(Self(value)) - } else { - Err(StoreMigrationInventoryEntryCountError::AboveMaximum { - observed: value, - maximum: Self::MAXIMUM, - }) - } - } - - /// Returns the exact admitted count. - pub const fn get(self) -> u32 { - self.0 - } -} diff --git a/src/adapters/store_migration/migration_inventory_entry_count_error.rs b/src/adapters/store_migration/migration_inventory_entry_count_error.rs deleted file mode 100644 index 6f44336a..00000000 --- a/src/adapters/store_migration/migration_inventory_entry_count_error.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! This boundary module owns migration inventory entry-count refusals. - -use std::error::Error; -use std::fmt; - -/// Failure to admit a bounded migration inventory entry count. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StoreMigrationInventoryEntryCountError { - /// The requested count exceeds the immutable protocol maximum. - AboveMaximum { - /// Count supplied by the caller. - observed: u32, - /// Largest count admitted by the protocol. - maximum: u32, - }, -} - -impl fmt::Display for StoreMigrationInventoryEntryCountError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::AboveMaximum { observed, maximum } => write!( - formatter, - "migration inventory entry count {observed} exceeds maximum {maximum}" - ), - } - } -} - -impl Error for StoreMigrationInventoryEntryCountError {} diff --git a/src/adapters/store_migration/migration_inventory_error.rs b/src/adapters/store_migration/migration_inventory_error.rs deleted file mode 100644 index e1d35b71..00000000 --- a/src/adapters/store_migration/migration_inventory_error.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! This boundary module owns streamed migration inventory refusals. - -use std::error::Error; -use std::fmt; - -use super::{StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount}; - -/// Failure to stream one bounded canonical migration inventory. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StoreMigrationInventoryError { - /// An entry would exceed the declared inventory count. - EntryCountExceeded { - /// Exact count declared before hashing. - expected: StoreMigrationInventoryEntryCount, - /// Count that the attempted entry would produce. - observed: u32, - }, - /// The same canonical entry appeared more than once. - Duplicate { - /// Repeated canonical entry. - entry: StoreMigrationInventoryEntry, - }, - /// Canonical entry order moved backward. - OutOfOrder { - /// Last entry admitted before the refusal. - previous: StoreMigrationInventoryEntry, - /// Entry observed after `previous`. - observed: StoreMigrationInventoryEntry, - }, - /// Finalization observed fewer entries than declared. - Incomplete { - /// Exact count declared before hashing. - expected: StoreMigrationInventoryEntryCount, - /// Exact number of entries admitted. - observed: u32, - }, - /// Checked observed-count arithmetic overflowed. - EntryCountOverflow, -} - -impl fmt::Display for StoreMigrationInventoryError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::EntryCountExceeded { expected, observed } => write!( - formatter, - "migration inventory expected {} entries but observed at least {observed}", - expected.get() - ), - Self::Duplicate { .. } => formatter.write_str("duplicate migration inventory entry"), - Self::OutOfOrder { .. } => { - formatter.write_str("migration inventory entries are out of canonical order") - } - Self::Incomplete { expected, observed } => write!( - formatter, - "migration inventory expected {} entries but observed {observed}", - expected.get() - ), - Self::EntryCountOverflow => { - formatter.write_str("migration inventory entry count overflow") - } - } - } -} - -impl Error for StoreMigrationInventoryError {} diff --git a/src/adapters/store_migration/migration_inventory_hasher.rs b/src/adapters/store_migration/migration_inventory_hasher.rs deleted file mode 100644 index f802e354..00000000 --- a/src/adapters/store_migration/migration_inventory_hasher.rs +++ /dev/null @@ -1,91 +0,0 @@ -//! This boundary module owns streamed canonical migration inventory identity. - -use super::{ - ImmutablePoolInventoryDigest, StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, - StoreMigrationInventoryError, -}; - -const DOMAIN: &[u8] = b"keep.store-v1-pool-inventory/v2\0"; - -/// In-progress bounded digest over one declared canonical pool inventory. -/// -/// Entries must be supplied in complete canonical byte order. The hasher -/// retains only the preceding entry and never materializes the complete -/// encoded inventory. -#[must_use] -pub struct StoreMigrationInventoryHasher { - expected: StoreMigrationInventoryEntryCount, - observed: u32, - previous: Option, - hasher: blake3::Hasher, -} - -impl StoreMigrationInventoryHasher { - /// Begins one inventory whose exact count is known before entry streaming. - pub fn new(expected: StoreMigrationInventoryEntryCount) -> Self { - let mut hasher = blake3::Hasher::new(); - hasher.update(DOMAIN); - hasher.update(&expected.get().to_be_bytes()); - Self { - expected, - observed: 0, - previous: None, - hasher, - } - } - - /// Admits and hashes the next exact canonical entry. - /// - /// # Errors - /// - /// Returns [`StoreMigrationInventoryError`] for count excess, a duplicate, - /// noncanonical order, or checked count overflow. - pub fn push( - &mut self, - entry: StoreMigrationInventoryEntry, - ) -> Result<(), StoreMigrationInventoryError> { - let observed = self - .observed - .checked_add(1) - .ok_or(StoreMigrationInventoryError::EntryCountOverflow)?; - if observed > self.expected.get() { - return Err(StoreMigrationInventoryError::EntryCountExceeded { - expected: self.expected, - observed, - }); - } - if let Some(previous) = self.previous { - if entry == previous { - return Err(StoreMigrationInventoryError::Duplicate { entry }); - } - if entry < previous { - return Err(StoreMigrationInventoryError::OutOfOrder { - previous, - observed: entry, - }); - } - } - self.hasher.update(entry.encoded()); - self.previous = Some(entry); - self.observed = observed; - Ok(()) - } - - /// Finalizes only after the declared number of entries was admitted. - /// - /// # Errors - /// - /// Returns [`StoreMigrationInventoryError::Incomplete`] when fewer entries - /// were supplied than declared. - pub fn finish(self) -> Result { - if self.observed != self.expected.get() { - return Err(StoreMigrationInventoryError::Incomplete { - expected: self.expected, - observed: self.observed, - }); - } - Ok(ImmutablePoolInventoryDigest::from_admitted( - *self.hasher.finalize().as_bytes(), - )) - } -} diff --git a/src/adapters/store_migration/migration_namespace_prefix.rs b/src/adapters/store_migration/migration_namespace_prefix.rs deleted file mode 100644 index 0ccdf880..00000000 --- a/src/adapters/store_migration/migration_namespace_prefix.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! This module owns a validated observation of the migration namespace prefix. - -use super::{MIGRATION_NAMESPACE_PREFIX, StoreMigrationRecoveryAmbiguity, StoreMigrationResidue}; - -/// The exact contiguous namespace prefix observed before recovery writes. -/// -/// This is observation evidence, not a durability claim or storage identity. -/// Only recovery constructs it after validating the residue's ordering. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct StoreMigrationNamespacePrefix { - extent: usize, -} - -impl StoreMigrationNamespacePrefix { - pub(super) fn observe( - residue: &StoreMigrationResidue, - ) -> Result { - residue - .namespace_extent() - .map(|extent| Self { extent }) - .map_err( - |(absent, present)| StoreMigrationRecoveryAmbiguity::NamespaceOutOfOrder { - absent, - present, - }, - ) - } - - /// Returns the number of observed names, including `reader.lock`. - pub const fn len(self) -> usize { - self.extent - } - - /// Returns whether no namespace effect was observed. - pub const fn is_empty(self) -> bool { - self.extent == 0 - } - - /// Iterates the exact observed protocol names in admission order. - /// - /// Iteration allocates nothing and performs no I/O. - pub fn names(self) -> impl Iterator { - std::iter::once("reader.lock") - .chain(MIGRATION_NAMESPACE_PREFIX) - .take(self.extent) - } -} diff --git a/src/adapters/store_migration/migration_phase.rs b/src/adapters/store_migration/migration_phase.rs deleted file mode 100644 index cc14888d..00000000 --- a/src/adapters/store_migration/migration_phase.rs +++ /dev/null @@ -1,114 +0,0 @@ -//! This boundary module owns exact store-migration durability phases. - -use std::fmt; - -/// Storage transition attempted by version-2 store migration. -/// -/// [`Self::ALL`] corresponds in order to `KEEP-CRASH-053` through -/// `KEEP-CRASH-073`. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StoreMigrationPhase { - /// Write the complete canonical `migration.intent.next`. - WriteIntentStage, - /// Synchronize `migration.intent.next`. - SynchronizeIntentStage, - /// Link the synchronized intent stage to `migration.intent`. - LinkIntent, - /// Synchronize the store root after the intent link. - SynchronizeRootAfterIntent, - /// Remove the retained `migration.intent.next`. - RemoveIntentStage, - /// Synchronize the store root after intent-stage cleanup. - SynchronizeRootAfterIntentCleanup, - /// Create or exactly admit the persistent reader fence. - AdmitReaderFence, - /// Create or exactly admit the canonical version-2 directory prefix. - AdmitNamespacePrefix, - /// Synchronize the store root after namespace admission. - SynchronizeRootAfterNamespace, - /// Write the complete canonical `FORMAT.next`. - WriteMarkerStage, - /// Synchronize `FORMAT.next`. - SynchronizeMarkerStage, - /// Link the synchronized marker stage to `FORMAT`. - LinkMarker, - /// Synchronize the store root after the marker link. - SynchronizeRootAfterMarker, - /// Remove the retained `FORMAT.next`. - RemoveMarkerStage, - /// Synchronize the store root after marker-stage cleanup. - SynchronizeRootAfterMarkerCleanup, - /// Write the complete canonical `migration.receipt.next`. - WriteReceiptStage, - /// Synchronize `migration.receipt.next`. - SynchronizeReceiptStage, - /// Link the synchronized receipt stage to `migration.receipt`. - LinkReceipt, - /// Synchronize the store root after the receipt link. - SynchronizeRootAfterReceipt, - /// Remove the retained `migration.receipt.next`. - RemoveReceiptStage, - /// Synchronize the store root after receipt-stage cleanup. - SynchronizeRootAfterReceiptCleanup, -} - -impl StoreMigrationPhase { - /// Every migration phase in normative crash-boundary order. - pub const ALL: [Self; 21] = [ - Self::WriteIntentStage, - Self::SynchronizeIntentStage, - Self::LinkIntent, - Self::SynchronizeRootAfterIntent, - Self::RemoveIntentStage, - Self::SynchronizeRootAfterIntentCleanup, - Self::AdmitReaderFence, - Self::AdmitNamespacePrefix, - Self::SynchronizeRootAfterNamespace, - Self::WriteMarkerStage, - Self::SynchronizeMarkerStage, - Self::LinkMarker, - Self::SynchronizeRootAfterMarker, - Self::RemoveMarkerStage, - Self::SynchronizeRootAfterMarkerCleanup, - Self::WriteReceiptStage, - Self::SynchronizeReceiptStage, - Self::LinkReceipt, - Self::SynchronizeRootAfterReceipt, - Self::RemoveReceiptStage, - Self::SynchronizeRootAfterReceiptCleanup, - ]; -} - -impl fmt::Display for StoreMigrationPhase { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(match self { - Self::WriteIntentStage => "migration-intent stage write", - Self::SynchronizeIntentStage => "migration-intent stage synchronization", - Self::LinkIntent => "migration-intent canonical link", - Self::SynchronizeRootAfterIntent => "store-root synchronization after intent link", - Self::RemoveIntentStage => "migration-intent stage removal", - Self::SynchronizeRootAfterIntentCleanup => { - "store-root synchronization after intent cleanup" - } - Self::AdmitReaderFence => "persistent reader-fence admission", - Self::AdmitNamespacePrefix => "canonical namespace-prefix admission", - Self::SynchronizeRootAfterNamespace => { - "store-root synchronization after namespace admission" - } - Self::WriteMarkerStage => "format-marker stage write", - Self::SynchronizeMarkerStage => "format-marker stage synchronization", - Self::LinkMarker => "format-marker canonical link", - Self::SynchronizeRootAfterMarker => "store-root synchronization after marker link", - Self::RemoveMarkerStage => "format-marker stage removal", - Self::SynchronizeRootAfterMarkerCleanup => { - "store-root synchronization after marker cleanup" - } - Self::WriteReceiptStage => "migration-receipt stage write", - Self::SynchronizeReceiptStage => "migration-receipt stage synchronization", - Self::LinkReceipt => "migration-receipt canonical link", - Self::SynchronizeRootAfterReceipt => "store-root synchronization after receipt link", - Self::RemoveReceiptStage => "migration-receipt stage removal", - Self::SynchronizeRootAfterReceiptCleanup => "final store-root synchronization", - }) - } -} diff --git a/src/adapters/store_migration/migration_receipt_decode_error.rs b/src/adapters/store_migration/migration_receipt_decode_error.rs deleted file mode 100644 index 609a38c1..00000000 --- a/src/adapters/store_migration/migration_receipt_decode_error.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! This boundary module owns store-migration receipt decoding failures. - -/// Failure to decode and admit one version-2 migration receipt. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StoreMigrationReceiptDecodeError { - /// The input was not exactly one complete receipt. - WrongLength { - /// Required fixed width. - expected: usize, - /// Observed input width. - observed: usize, - }, - /// The fixed record magic was not canonical. - InvalidMagic { - /// Observed 16 magic bytes. - observed: [u8; 16], - }, - /// The format version is unsupported. - UnsupportedVersion { - /// Supported version. - expected: u16, - /// Observed version. - observed: u16, - }, - /// The record-length field was noncanonical. - InvalidRecordLength { - /// Required record length. - expected: u16, - /// Observed record length. - observed: u16, - }, - /// The receipt carried unsupported flags. - UnsupportedFlags { - /// Observed flag bits. - observed: u32, - }, - /// The checksum did not match the exact prefix. - ChecksumMismatch { - /// Computed canonical checksum. - expected: [u8; 32], - /// Checksum stored in the record. - observed: [u8; 32], - }, - /// The receipt did not bind the supplied admitted intent. - IntentDigestMismatch { - /// Supplied intent digest. - expected: [u8; 32], - /// Receipt intent digest. - observed: [u8; 32], - }, - /// The receipt did not bind the intent's store identifier. - StoreIdentifierMismatch { - /// Supplied intent store identifier. - expected: [u8; 32], - /// Receipt store identifier. - observed: [u8; 32], - }, - /// The receipt did not bind the supplied admitted marker. - FormatMarkerDigestMismatch { - /// Supplied marker digest. - expected: [u8; 32], - /// Receipt marker digest. - observed: [u8; 32], - }, - /// The initial retention-state digest was not registered. - InitialRetentionStateDigestMismatch { - /// Registered empty-state digest. - expected: [u8; 32], - /// Receipt digest. - observed: [u8; 32], - }, - /// The initial garbage-collection-state digest was not registered. - InitialGcStateDigestMismatch { - /// Registered empty-state digest. - expected: [u8; 32], - /// Receipt digest. - observed: [u8; 32], - }, - /// The empty recovery-disposition-set digest was not registered. - EmptyDispositionSetDigestMismatch { - /// Registered empty-set digest. - expected: [u8; 32], - /// Receipt digest. - observed: [u8; 32], - }, - /// The synchronization mask carried unknown bits. - UnsupportedSynchronizationBits { - /// Complete supported bit set. - supported: u64, - /// Observed mask. - observed: u64, - }, - /// The synchronization mask omitted one or more mandatory bits. - IncompleteSynchronizationMask { - /// Complete required bit set. - required: u64, - /// Observed mask. - observed: u64, - }, -} diff --git a/src/adapters/store_migration/migration_receipt_decode_error_display.rs b/src/adapters/store_migration/migration_receipt_decode_error_display.rs deleted file mode 100644 index 16cc473e..00000000 --- a/src/adapters/store_migration/migration_receipt_decode_error_display.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! This boundary module owns migration-receipt error formatting. - -use std::error::Error; -use std::fmt; - -use super::StoreMigrationReceiptDecodeError; - -impl fmt::Display for StoreMigrationReceiptDecodeError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::WrongLength { expected, observed } => write!( - formatter, - "migration receipt requires {expected} bytes, observed {observed}" - ), - Self::InvalidMagic { .. } => formatter.write_str("invalid migration-receipt magic"), - Self::UnsupportedVersion { expected, observed } => write!( - formatter, - "unsupported migration-receipt version {observed}; expected {expected}" - ), - Self::InvalidRecordLength { expected, observed } => write!( - formatter, - "migration-receipt record length {observed}; expected {expected}" - ), - Self::UnsupportedFlags { observed } => { - write!( - formatter, - "unsupported migration-receipt flags {observed:#010x}" - ) - } - Self::ChecksumMismatch { .. } => { - formatter.write_str("migration-receipt checksum mismatch") - } - Self::IntentDigestMismatch { .. } => { - formatter.write_str("migration-receipt intent digest mismatch") - } - Self::StoreIdentifierMismatch { .. } => { - formatter.write_str("migration-receipt store identifier mismatch") - } - Self::FormatMarkerDigestMismatch { .. } => { - formatter.write_str("migration-receipt format-marker digest mismatch") - } - Self::InitialRetentionStateDigestMismatch { .. } => { - formatter.write_str("migration-receipt initial retention-state digest mismatch") - } - Self::InitialGcStateDigestMismatch { .. } => { - formatter.write_str("migration-receipt initial GC-state digest mismatch") - } - Self::EmptyDispositionSetDigestMismatch { .. } => { - formatter.write_str("migration-receipt empty disposition-set digest mismatch") - } - Self::UnsupportedSynchronizationBits { observed, .. } => write!( - formatter, - "migration-receipt synchronization mask has unknown bits: {observed:#018x}" - ), - Self::IncompleteSynchronizationMask { observed, .. } => write!( - formatter, - "migration-receipt synchronization mask is incomplete: {observed:#018x}" - ), - } - } -} - -impl Error for StoreMigrationReceiptDecodeError {} diff --git a/src/adapters/store_migration/migration_receipt_decoder.rs b/src/adapters/store_migration/migration_receipt_decoder.rs deleted file mode 100644 index cd92d710..00000000 --- a/src/adapters/store_migration/migration_receipt_decoder.rs +++ /dev/null @@ -1,145 +0,0 @@ -//! This boundary module owns store-migration receipt decoding order. - -use super::admitted_migration_receipt::StoreMigrationReceiptFields; -use super::migration_receipt_initial_state::{ - read_empty_disposition_digest, read_initial_gc_digest, read_initial_retention_digest, -}; -use super::migration_record_bytes::{ - read_array, read_u16, read_u32, read_u64, require_length, wrong_length, -}; -use super::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, - MigrationSynchronizationMask, StoreMigrationReceiptDecodeError, - migration_receipt_format as format, -}; - -pub(super) fn decode<'encoded>( - encoded: &'encoded [u8], - intent: &AdmittedStoreMigrationIntent<'_>, - marker: &AdmittedStoreFormatMarker<'_>, -) -> Result, StoreMigrationReceiptDecodeError> { - require_length(encoded)?; - validate_fixed_fields(encoded)?; - verify_checksum(encoded)?; - let intent_digest = bind_intent_digest(encoded, intent)?; - let store_identifier = bind_store_identifier(encoded, intent)?; - let format_marker_digest = bind_marker_digest(encoded, marker)?; - let initial_retention_state_digest = read_initial_retention_digest(encoded)?; - let initial_gc_state_digest = read_initial_gc_digest(encoded)?; - let empty_disposition_set_digest = read_empty_disposition_digest(encoded)?; - let synchronization_mask = read_synchronization_mask(encoded)?; - Ok(AdmittedStoreMigrationReceipt::admitted( - encoded, - StoreMigrationReceiptFields { - intent_digest, - store_identifier, - format_marker_digest, - initial_retention_state_digest, - initial_gc_state_digest, - empty_disposition_set_digest, - synchronization_mask, - }, - )) -} - -fn validate_fixed_fields(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDecodeError> { - let magic = read_array(encoded, 0)?; - if magic != format::MAGIC { - return Err(StoreMigrationReceiptDecodeError::InvalidMagic { observed: magic }); - } - let version = read_u16(encoded, 16)?; - if version != format::VERSION { - return Err(StoreMigrationReceiptDecodeError::UnsupportedVersion { - expected: format::VERSION, - observed: version, - }); - } - let record_length = read_u16(encoded, 18)?; - if record_length != format::RECORD_LENGTH { - return Err(StoreMigrationReceiptDecodeError::InvalidRecordLength { - expected: format::RECORD_LENGTH, - observed: record_length, - }); - } - let flags = read_u32(encoded, 20)?; - if flags != 0 { - return Err(StoreMigrationReceiptDecodeError::UnsupportedFlags { observed: flags }); - } - Ok(()) -} - -fn verify_checksum(encoded: &[u8]) -> Result<(), StoreMigrationReceiptDecodeError> { - let preimage = encoded - .get(..format::CHECKSUM_OFFSET) - .ok_or_else(|| wrong_length(encoded))?; - let observed = read_array(encoded, format::CHECKSUM_OFFSET)?; - let expected = format::checksum(preimage); - if observed == expected { - Ok(()) - } else { - Err(StoreMigrationReceiptDecodeError::ChecksumMismatch { expected, observed }) - } -} - -fn bind_intent_digest( - encoded: &[u8], - intent: &AdmittedStoreMigrationIntent<'_>, -) -> Result { - let expected = *intent.digest().as_bytes(); - let observed = read_array(encoded, 24)?; - if observed == expected { - Ok(intent.digest()) - } else { - Err(StoreMigrationReceiptDecodeError::IntentDigestMismatch { expected, observed }) - } -} - -fn bind_store_identifier( - encoded: &[u8], - intent: &AdmittedStoreMigrationIntent<'_>, -) -> Result { - let expected = *intent.store_identifier().as_bytes(); - let observed = read_array(encoded, 56)?; - if observed == expected { - Ok(intent.store_identifier()) - } else { - Err(StoreMigrationReceiptDecodeError::StoreIdentifierMismatch { expected, observed }) - } -} - -fn bind_marker_digest( - encoded: &[u8], - marker: &AdmittedStoreFormatMarker<'_>, -) -> Result { - let expected = *marker.digest().as_bytes(); - let observed = read_array(encoded, 88)?; - if observed == expected { - Ok(marker.digest()) - } else { - Err(StoreMigrationReceiptDecodeError::FormatMarkerDigestMismatch { expected, observed }) - } -} - -fn read_synchronization_mask( - encoded: &[u8], -) -> Result { - let observed = read_u64(encoded, 216)?; - let supported = MigrationSynchronizationMask::COMPLETE_BITS; - if observed & !supported != 0 { - return Err( - StoreMigrationReceiptDecodeError::UnsupportedSynchronizationBits { - supported, - observed, - }, - ); - } - if observed != supported { - return Err( - StoreMigrationReceiptDecodeError::IncompleteSynchronizationMask { - required: supported, - observed, - }, - ); - } - Ok(MigrationSynchronizationMask::complete()) -} diff --git a/src/adapters/store_migration/migration_receipt_encoder.rs b/src/adapters/store_migration/migration_receipt_encoder.rs deleted file mode 100644 index 9020298e..00000000 --- a/src/adapters/store_migration/migration_receipt_encoder.rs +++ /dev/null @@ -1,52 +0,0 @@ -//! This boundary module owns canonical migration-receipt encoding. - -use super::migration_receipt_initial_state::{ - empty_disposition_digest, initial_gc_digest, initial_retention_digest, -}; -use super::{ - CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, - MigrationSynchronizationMask, migration_receipt_format as format, -}; - -pub(super) fn encode( - intent: &CanonicalStoreMigrationIntent, - marker: &CanonicalStoreFormatMarker, -) -> CanonicalStoreMigrationReceipt { - let mut encoded = [0_u8; format::ENCODED_LENGTH]; - let (preimage, checksum_slot) = encoded.split_at_mut(format::CHECKSUM_OFFSET); - write_preimage(preimage, intent, marker); - checksum_slot.copy_from_slice(&format::checksum(preimage)); - CanonicalStoreMigrationReceipt::admitted(encoded) -} - -fn write_preimage( - output: &mut [u8], - intent: &CanonicalStoreMigrationIntent, - marker: &CanonicalStoreFormatMarker, -) { - let (magic, output) = output.split_at_mut(16); - magic.copy_from_slice(&format::MAGIC); - let (version, output) = output.split_at_mut(2); - version.copy_from_slice(&format::VERSION.to_be_bytes()); - let (record_length, output) = output.split_at_mut(2); - record_length.copy_from_slice(&format::RECORD_LENGTH.to_be_bytes()); - let (flags, output) = output.split_at_mut(4); - flags.copy_from_slice(&0_u32.to_be_bytes()); - let (intent_digest, output) = output.split_at_mut(32); - intent_digest.copy_from_slice(intent.digest().as_bytes()); - let (store_identifier, output) = output.split_at_mut(32); - store_identifier.copy_from_slice(intent.store_identifier().as_bytes()); - let (marker_digest, output) = output.split_at_mut(32); - marker_digest.copy_from_slice(marker.digest().as_bytes()); - let (retention_digest, output) = output.split_at_mut(32); - retention_digest.copy_from_slice(initial_retention_digest().as_bytes()); - let (gc_digest, output) = output.split_at_mut(32); - gc_digest.copy_from_slice(initial_gc_digest().as_bytes()); - let (disposition_digest, synchronization_mask) = output.split_at_mut(32); - disposition_digest.copy_from_slice(empty_disposition_digest().as_bytes()); - synchronization_mask.copy_from_slice( - &MigrationSynchronizationMask::complete() - .bits() - .to_be_bytes(), - ); -} diff --git a/src/adapters/store_migration/migration_receipt_format.rs b/src/adapters/store_migration/migration_receipt_format.rs deleted file mode 100644 index 72841b74..00000000 --- a/src/adapters/store_migration/migration_receipt_format.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! This boundary module owns shared migration-receipt framing and integrity. - -pub(super) const CHECKSUM_OFFSET: usize = 224; -pub(in crate::adapters) const ENCODED_LENGTH: usize = 256; -pub(super) const MAGIC: [u8; 16] = *b"KEEP:MIG:REC2\0\0\0"; -pub(super) const RECORD_LENGTH: u16 = 256; -pub(super) const VERSION: u16 = 2; -const CHECKSUM_DOMAIN: &[u8] = b"keep.store-migration-receipt-checksum/v2\0"; - -pub(super) fn checksum(preimage: &[u8]) -> [u8; 32] { - let mut hasher = blake3::Hasher::new(); - hasher.update(CHECKSUM_DOMAIN); - hasher.update(preimage); - *hasher.finalize().as_bytes() -} diff --git a/src/adapters/store_migration/migration_receipt_initial_state.rs b/src/adapters/store_migration/migration_receipt_initial_state.rs deleted file mode 100644 index d89af3e9..00000000 --- a/src/adapters/store_migration/migration_receipt_initial_state.rs +++ /dev/null @@ -1,76 +0,0 @@ -//! This boundary module owns registered empty-state receipt admission. - -use super::migration_record_bytes::read_array; -use super::{ - EmptyDispositionSetDigest, InitialGcStateDigest, InitialRetentionStateDigest, - StoreMigrationReceiptDecodeError, -}; - -const INITIAL_RETENTION_DOMAIN: &[u8] = b"keep.initial-retention-state/v2\0"; -const INITIAL_GC_DOMAIN: &[u8] = b"keep.initial-gc-state/v2\0"; -const EMPTY_DISPOSITION_DOMAIN: &[u8] = b"keep.empty-disposition-set/v2\0"; - -pub(super) fn read_initial_retention_digest( - encoded: &[u8], -) -> Result { - let admitted = initial_retention_digest(); - let expected = *admitted.as_bytes(); - let observed = read_array(encoded, 120)?; - if observed == expected { - Ok(admitted) - } else { - Err( - StoreMigrationReceiptDecodeError::InitialRetentionStateDigestMismatch { - expected, - observed, - }, - ) - } -} - -pub(super) fn read_initial_gc_digest( - encoded: &[u8], -) -> Result { - let admitted = initial_gc_digest(); - let expected = *admitted.as_bytes(); - let observed = read_array(encoded, 152)?; - if observed == expected { - Ok(admitted) - } else { - Err(StoreMigrationReceiptDecodeError::InitialGcStateDigestMismatch { expected, observed }) - } -} - -pub(super) fn read_empty_disposition_digest( - encoded: &[u8], -) -> Result { - let admitted = empty_disposition_digest(); - let expected = *admitted.as_bytes(); - let observed = read_array(encoded, 184)?; - if observed == expected { - Ok(admitted) - } else { - Err( - StoreMigrationReceiptDecodeError::EmptyDispositionSetDigestMismatch { - expected, - observed, - }, - ) - } -} - -pub(super) fn initial_retention_digest() -> InitialRetentionStateDigest { - InitialRetentionStateDigest::from_hash(digest(INITIAL_RETENTION_DOMAIN)) -} - -pub(super) fn initial_gc_digest() -> InitialGcStateDigest { - InitialGcStateDigest::from_hash(digest(INITIAL_GC_DOMAIN)) -} - -pub(super) fn empty_disposition_digest() -> EmptyDispositionSetDigest { - EmptyDispositionSetDigest::from_hash(digest(EMPTY_DISPOSITION_DOMAIN)) -} - -fn digest(domain: &[u8]) -> [u8; 32] { - *blake3::hash(domain).as_bytes() -} diff --git a/src/adapters/store_migration/migration_record_bytes.rs b/src/adapters/store_migration/migration_record_bytes.rs deleted file mode 100644 index 774469d5..00000000 --- a/src/adapters/store_migration/migration_record_bytes.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! This boundary module owns fixed-width store-migration record field access. - -use super::{StoreMigrationIntentDecodeError, StoreMigrationReceiptDecodeError}; - -const ENCODED_LENGTH: usize = 256; - -pub(super) trait MigrationRecordDecodeError: Sized { - fn wrong_length(expected: usize, observed: usize) -> Self; -} - -impl MigrationRecordDecodeError for StoreMigrationIntentDecodeError { - fn wrong_length(expected: usize, observed: usize) -> Self { - Self::WrongLength { expected, observed } - } -} - -impl MigrationRecordDecodeError for StoreMigrationReceiptDecodeError { - fn wrong_length(expected: usize, observed: usize) -> Self { - Self::WrongLength { expected, observed } - } -} - -pub(super) fn require_length( - encoded: &[u8], -) -> Result<(), Error> { - if encoded.len() == ENCODED_LENGTH { - Ok(()) - } else { - Err(wrong_length(encoded)) - } -} - -pub(super) fn wrong_length(encoded: &[u8]) -> Error { - Error::wrong_length(ENCODED_LENGTH, encoded.len()) -} - -pub(super) fn read_u16( - encoded: &[u8], - offset: usize, -) -> Result { - read_array(encoded, offset).map(u16::from_be_bytes) -} - -pub(super) fn read_u32( - encoded: &[u8], - offset: usize, -) -> Result { - read_array(encoded, offset).map(u32::from_be_bytes) -} - -pub(super) fn read_u64( - encoded: &[u8], - offset: usize, -) -> Result { - read_array(encoded, offset).map(u64::from_be_bytes) -} - -pub(super) fn read_array( - encoded: &[u8], - offset: usize, -) -> Result<[u8; WIDTH], Error> { - let Some(end) = offset.checked_add(WIDTH) else { - return Err(wrong_length(encoded)); - }; - let bytes = encoded - .get(offset..end) - .ok_or_else(|| wrong_length(encoded))?; - <[u8; WIDTH]>::try_from(bytes).map_err(|_| wrong_length(encoded)) -} diff --git a/src/adapters/store_migration/migration_recovery_ambiguity.rs b/src/adapters/store_migration/migration_recovery_ambiguity.rs deleted file mode 100644 index 015fa64c..00000000 --- a/src/adapters/store_migration/migration_recovery_ambiguity.rs +++ /dev/null @@ -1,86 +0,0 @@ -//! This boundary module owns migration residue that no lawful plan resolves. - -use super::{ - StoreFormatMarkerDecodeError, StoreMigrationFixedStage, StoreMigrationIntentDecodeError, - StoreMigrationReceiptDecodeError, -}; - -/// A migration effect that can exist only after a durable intent. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StoreMigrationEffect { - /// `reader.lock` or a prefix directory. - Namespace, - /// `FORMAT` or `FORMAT.next`. - Marker, - /// `migration.receipt` or `migration.receipt.next`. - Receipt, -} - -/// Residue that recovery refuses to interpret. -/// -/// Recovery never guesses which step produced conflicting evidence; every -/// variant names the exact conflict so a human can resolve it. -#[derive(Debug)] -pub enum StoreMigrationRecoveryAmbiguity { - /// A later effect exists although no durable intent does. - EffectBeforeIntent { - /// The earliest effect found. - effect: StoreMigrationEffect, - }, - /// A stage survived an effect that requires its prior removal. - StageAfterEffect { - /// The surviving stage. - stage: StoreMigrationFixedStage, - /// The later effect. - effect: StoreMigrationEffect, - }, - /// A complete-length stage does not decode as a canonical record. - StageUndecodable { - /// Preserved stage decoder failure. - source: super::StoreMigrationStageDecodeError, - /// The stage. - stage: StoreMigrationFixedStage, - }, - /// A stage is longer than its canonical record. - StageOverlong { - /// The stage. - stage: StoreMigrationFixedStage, - /// Observed byte length. - observed: usize, - }, - /// A complete stage and its canonical target, or a complete pre-effect - /// stage and the expected record, carry different bytes. - StageDiffers { - /// The stage. - stage: StoreMigrationFixedStage, - }, - /// `migration.intent` does not decode. - IntentUndecodable { - /// Preserved decode failure. - source: StoreMigrationIntentDecodeError, - }, - /// `migration.intent` binds a different version-1 store or root - /// (any coordinate but the mount identity). - IntentDiffers, - /// `reader.lock` and the directory prefix are not one contiguous prefix. - NamespaceOutOfOrder { - /// Index of the first absent slot (0 is `reader.lock`). - absent: usize, - /// Index of a later present slot. - present: usize, - }, - /// A marker artifact exists before the namespace prefix is complete. - MarkerBeforeNamespace, - /// `FORMAT` does not decode as the registered version-2 marker. - MarkerUndecodable { - /// Preserved decode failure. - source: StoreFormatMarkerDecodeError, - }, - /// A receipt artifact exists before `FORMAT`. - ReceiptBeforeMarker, - /// `migration.receipt` does not bind the observed intent and marker. - ReceiptUndecodable { - /// Preserved decode failure. - source: StoreMigrationReceiptDecodeError, - }, -} diff --git a/src/adapters/store_migration/migration_recovery_ambiguity_display.rs b/src/adapters/store_migration/migration_recovery_ambiguity_display.rs deleted file mode 100644 index f29a23f2..00000000 --- a/src/adapters/store_migration/migration_recovery_ambiguity_display.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! This boundary module owns migration-recovery ambiguity formatting. - -use std::error::Error; -use std::fmt; - -use super::StoreMigrationRecoveryAmbiguity; - -impl fmt::Display for StoreMigrationRecoveryAmbiguity { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::EffectBeforeIntent { effect } => write!( - formatter, - "migration {effect:?} effect exists without a durable intent" - ), - Self::StageAfterEffect { stage, effect } => write!( - formatter, - "migration {stage:?} stage survived later {effect:?} effect" - ), - Self::StageUndecodable { stage, .. } => { - write!( - formatter, - "complete migration {stage:?} stage does not decode" - ) - } - Self::StageOverlong { stage, observed } => write!( - formatter, - "migration {stage:?} stage has {observed} bytes, more than its record" - ), - Self::StageDiffers { stage } => { - write!( - formatter, - "migration {stage:?} stage bytes differ from their record" - ) - } - Self::IntentUndecodable { .. } => { - formatter.write_str("migration intent does not decode") - } - Self::IntentDiffers => { - formatter.write_str("migration intent binds a different store or root") - } - Self::NamespaceOutOfOrder { absent, present } => write!( - formatter, - "migration namespace slot {present} exists before slot {absent}" - ), - Self::MarkerBeforeNamespace => { - formatter.write_str("format marker exists before the namespace prefix") - } - Self::MarkerUndecodable { .. } => formatter.write_str("format marker does not decode"), - Self::ReceiptBeforeMarker => { - formatter.write_str("migration receipt exists before the format marker") - } - Self::ReceiptUndecodable { .. } => { - formatter.write_str("migration receipt does not bind the observed records") - } - } - } -} - -impl Error for StoreMigrationRecoveryAmbiguity { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::StageUndecodable { source, .. } => Some(source), - Self::IntentUndecodable { source } => Some(source), - Self::MarkerUndecodable { source } => Some(source), - Self::ReceiptUndecodable { source } => Some(source), - _ => None, - } - } -} diff --git a/src/adapters/store_migration/migration_recovery_execution.rs b/src/adapters/store_migration/migration_recovery_execution.rs deleted file mode 100644 index fc3d5044..00000000 --- a/src/adapters/store_migration/migration_recovery_execution.rs +++ /dev/null @@ -1,244 +0,0 @@ -//! This boundary module owns ordered migration recovery: observe, plan, -//! adopt, discard, resume. - -use std::error::Error; -use std::fmt; -use std::io; - -use super::migration_resumption::resume_store_migration; -use super::{ - AdmittedStoreMigrationIntent, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, - StoreMigrationError, StoreMigrationFixedStage, StoreMigrationRecoveryAmbiguity, - StoreMigrationRecoveryPlan, StoreMigrationRecoveryStorage, StoreMigrationResidue, - plan_store_migration_recovery, -}; - -/// What one recovery found and did. -#[must_use] -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct StoreMigrationRecoveryReceipt { - plan: StoreMigrationRecoveryPlan, - observed_prefix: super::StoreMigrationNamespacePrefix, - intent_digest: Option, - published: Option, -} - -impl StoreMigrationRecoveryReceipt { - /// Returns the plan the residue admitted. - pub const fn plan(&self) -> StoreMigrationRecoveryPlan { - self.plan - } - - /// Returns the exact namespace prefix observed before any recovery writes. - pub const fn observed_namespace_prefix(&self) -> super::StoreMigrationNamespacePrefix { - self.observed_prefix - } - - /// Returns the intent bound by this recovery, including complete observations. - /// - /// An untouched version-one observation has no migration intent. Resumed - /// recovery binds persisted intent bytes, or the newly published intent - /// after discarding an incomplete pre-effect stage. - pub const fn intent_digest(&self) -> Option { - self.intent_digest - } - - /// Returns the migration receipt this recovery published, when it ran - /// the forward protocol to completion. - pub const fn published(&self) -> Option<&CanonicalStoreMigrationReceipt> { - self.published.as_ref() - } - - /// Lists every forward phase this successful recovery executed, in order. - /// - /// A version-one admission or already-complete observation executed no - /// forward phases. Any pre-effect discard is recorded separately in - /// [`Self::plan`]. No allocation or I/O occurs during iteration. - pub fn executed_phases(&self) -> impl Iterator { - let first = match self.plan { - StoreMigrationRecoveryPlan::Resume { resume } - | StoreMigrationRecoveryPlan::DiscardStage { resume, .. } => Some(resume), - StoreMigrationRecoveryPlan::VersionOne | StoreMigrationRecoveryPlan::Complete => None, - }; - super::StoreMigrationPhase::ALL - .into_iter() - .skip_while(move |phase| Some(*phase) != first) - } -} - -/// Failure to recover one interrupted migration. -#[derive(Debug)] -pub enum StoreMigrationRecoveryError { - /// Current writer authority no longer reproduces the caller's expected intent. - CurrentVerification { - /// Preserved current-state refusal or operational failure. - source: io::Error, - }, - /// The residue or completed namespace could not be admitted without effects. - Observation { - /// Preserved storage failure. - source: io::Error, - }, - /// The residue matches no lawful recovery row. - Ambiguity { - /// The exact conflict. - source: StoreMigrationRecoveryAmbiguity, - }, - /// The exact stage and canonical handles could not be adopted. - Adoption { - /// Preserved storage failure. - source: io::Error, - }, - /// The incomplete pre-effect stage could not be removed. - Discard { - /// The stage. - stage: StoreMigrationFixedStage, - /// Preserved storage failure. - source: io::Error, - }, - /// A resumed forward phase refused. - Resumption { - /// Preserved phase failure. - source: StoreMigrationError, - }, -} - -/// Recovers one interrupted migration under writer authority. -/// -/// Current writer authority first revalidates `expected`. The residue is then -/// observed once and planned against `expected` (the intent the -/// version-1 store derives today, compared on every restart-stable -/// coordinate), and either admitted as version 1, reported complete, or -/// driven through the remaining forward phases with the persisted intent, -/// never the freshly derived one. Nothing is truncated, replaced, or -/// repaired; an incomplete pre-effect stage is the only artifact removed. -/// -/// # Errors -/// -/// Returns [`StoreMigrationRecoveryError`] at the exact boundary that refused. -/// -/// A complete plan verifies the completed version-two namespace before returning, -/// without requiring the retention pools to remain empty. A completion refusal -/// retains its source under [`StoreMigrationRecoveryError::Observation`]. -/// -/// Resumption is internal: external callers cannot bypass recovery admission. -/// -/// ```compile_fail -/// use keep::resume_store_migration; -/// ``` -pub fn recover_store_migration( - storage: &mut impl StoreMigrationRecoveryStorage, - expected: &CanonicalStoreMigrationIntent, -) -> Result { - storage - .verify_current(expected) - .map_err(|source| StoreMigrationRecoveryError::CurrentVerification { source })?; - let residue = storage - .observe_residue() - .map_err(|source| StoreMigrationRecoveryError::Observation { source })?; - let expected_admitted = - AdmittedStoreMigrationIntent::decode(expected.encoded()).map_err(|source| { - StoreMigrationRecoveryError::Observation { - source: io::Error::new(io::ErrorKind::InvalidData, source), - } - })?; - let plan = plan_store_migration_recovery(&expected_admitted, &residue) - .map_err(|source| StoreMigrationRecoveryError::Ambiguity { source })?; - if plan == StoreMigrationRecoveryPlan::Complete { - storage - .verify_complete() - .map_err(|source| StoreMigrationRecoveryError::Observation { source })?; - } - let observed_prefix = super::StoreMigrationNamespacePrefix::observe(&residue) - .map_err(|source| StoreMigrationRecoveryError::Ambiguity { source })?; - let persisted = persisted_intent(&residue, expected)?; - let intent_digest = - (plan != StoreMigrationRecoveryPlan::VersionOne).then(|| persisted.digest()); - let resume = match plan { - StoreMigrationRecoveryPlan::VersionOne | StoreMigrationRecoveryPlan::Complete => { - return Ok(StoreMigrationRecoveryReceipt { - plan, - observed_prefix, - intent_digest, - published: None, - }); - } - StoreMigrationRecoveryPlan::DiscardStage { stage, resume } => { - storage - .adopt_residue(&residue, &persisted) - .map_err(|source| StoreMigrationRecoveryError::Adoption { source })?; - storage - .discard_stage(stage) - .map_err(|source| StoreMigrationRecoveryError::Discard { stage, source })?; - resume - } - StoreMigrationRecoveryPlan::Resume { resume } => { - storage - .adopt_residue(&residue, &persisted) - .map_err(|source| StoreMigrationRecoveryError::Adoption { source })?; - resume - } - }; - let published = resume_store_migration(storage, &persisted, resume) - .map_err(|source| StoreMigrationRecoveryError::Resumption { source })?; - Ok(StoreMigrationRecoveryReceipt { - plan, - observed_prefix, - intent_digest, - published: Some(published), - }) -} - -/// The intent the resumed phases must publish: the durable one when it -/// exists, else the exact staged one, else the freshly derived one. -fn persisted_intent( - residue: &StoreMigrationResidue, - expected: &CanonicalStoreMigrationIntent, -) -> Result { - let staged = residue - .intent - .as_deref() - .or(residue.intent_stage.as_deref()); - let Some(bytes) = staged else { - return Ok(expected.clone()); - }; - match AdmittedStoreMigrationIntent::decode(bytes) { - Ok(admitted) => Ok(CanonicalStoreMigrationIntent::from_admitted(&admitted)), - Err(_) if residue.intent.is_none() => Ok(expected.clone()), - Err(source) => Err(StoreMigrationRecoveryError::Observation { - source: io::Error::new(io::ErrorKind::InvalidData, source), - }), - } -} - -impl fmt::Display for StoreMigrationRecoveryError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::CurrentVerification { .. } => { - formatter.write_str("current migration authority verification failed") - } - Self::Observation { .. } => formatter.write_str("migration residue observation failed"), - Self::Ambiguity { source } => { - write!(formatter, "migration residue is ambiguous: {source}") - } - Self::Adoption { .. } => formatter.write_str("migration residue adoption failed"), - Self::Discard { stage, .. } => { - write!(formatter, "migration {stage:?} stage discard failed") - } - Self::Resumption { source } => write!(formatter, "resumed migration failed: {source}"), - } - } -} - -impl Error for StoreMigrationRecoveryError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::CurrentVerification { source } - | Self::Observation { source } - | Self::Adoption { source } - | Self::Discard { source, .. } => Some(source), - Self::Ambiguity { source } => Some(source), - Self::Resumption { source } => Some(source), - } - } -} diff --git a/src/adapters/store_migration/migration_recovery_plan.rs b/src/adapters/store_migration/migration_recovery_plan.rs deleted file mode 100644 index 92da9c15..00000000 --- a/src/adapters/store_migration/migration_recovery_plan.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! This boundary module owns the lawful responses to migration residue. - -use super::StoreMigrationPhase; - -/// One fixed-name migration stage. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StoreMigrationFixedStage { - /// `migration.intent.next`. - Intent, - /// `FORMAT.next`. - Marker, - /// `migration.receipt.next`. - Receipt, -} - -/// The one lawful response to an observed migration residue. -/// -/// A plan is a statement about the residue; executing it is the recovery -/// storage's job. Every resume point is the earliest phase whose effect the -/// residue cannot prove, so re-running from it is idempotent. -#[must_use] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StoreMigrationRecoveryPlan { - /// No migration artifact exists: the exact version-1 store admits. - VersionOne, - /// Remove one incomplete pre-effect stage, then resume at `resume`. - DiscardStage { - /// The incomplete stage to remove. - stage: StoreMigrationFixedStage, - /// The forward phase to resume at after removal. - resume: StoreMigrationPhase, - }, - /// Resume the forward protocol at `resume`. - Resume { - /// The earliest phase the residue cannot prove complete. - resume: StoreMigrationPhase, - }, - /// The exact receipt is canonical and no stage remains. - Complete, -} diff --git a/src/adapters/store_migration/migration_recovery_planner.rs b/src/adapters/store_migration/migration_recovery_planner.rs deleted file mode 100644 index 521671c2..00000000 --- a/src/adapters/store_migration/migration_recovery_planner.rs +++ /dev/null @@ -1,272 +0,0 @@ -//! This boundary module owns storage-independent migration recovery planning. -//! -//! The planner turns one observed residue into the one lawful response the -//! recovery table in `migration-recovery.md` prescribes, or into the exact -//! ambiguity that refuses it. It reads no storage and mutates nothing. - -use super::StoreMigrationStageDecodeError as DecodeError; -use super::migration_recovery_ambiguity::StoreMigrationEffect as Effect; -use super::migration_recovery_residue::NAMESPACE_NAME_COUNT; -use super::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, - FORMAT_MARKER_LENGTH, MIGRATION_INTENT_LENGTH, MIGRATION_RECEIPT_LENGTH, - StoreMigrationFixedStage as Stage, StoreMigrationPhase as Phase, - StoreMigrationRecoveryAmbiguity as Ambiguity, StoreMigrationRecoveryPlan as Plan, - StoreMigrationResidue, -}; - -/// Plans the one lawful recovery of `residue` against the intent the -/// version-1 store derives today. -/// -/// The expected intent is compared on every coordinate but the mount -/// identity, which is same-process evidence (see `recovery.md`, "Root -/// identity across restart"). -/// -/// # Errors -/// -/// Returns [`StoreMigrationRecoveryAmbiguity`](super::StoreMigrationRecoveryAmbiguity) -/// when the residue matches no table row. -pub fn plan_store_migration_recovery( - expected: &AdmittedStoreMigrationIntent<'_>, - residue: &StoreMigrationResidue, -) -> Result { - let Some(intent_bytes) = residue.intent.as_deref() else { - return plan_before_durable_intent(expected, residue); - }; - let intent = AdmittedStoreMigrationIntent::decode(intent_bytes) - .map_err(|source| Ambiguity::IntentUndecodable { source })?; - if !restart_stable_match(expected, &intent) { - return Err(Ambiguity::IntentDiffers); - } - if let Some(stage) = residue.intent_stage.as_deref() { - let later = if residue.has_namespace_effect() { - Some(Effect::Namespace) - } else if residue.marker_stage.is_some() || residue.marker.is_some() { - Some(Effect::Marker) - } else if residue.has_receipt_effect() { - Some(Effect::Receipt) - } else { - None - }; - if let Some(effect) = later { - return Err(Ambiguity::StageAfterEffect { - stage: Stage::Intent, - effect, - }); - } - return if stage == intent_bytes { - Ok(Plan::Resume { - resume: Phase::SynchronizeRootAfterIntent, - }) - } else { - Err(Ambiguity::StageDiffers { - stage: Stage::Intent, - }) - }; - } - plan_namespace(&intent, residue) -} - -/// Rows one and two: nothing, or an intent stage alone. -fn plan_before_durable_intent( - expected: &AdmittedStoreMigrationIntent<'_>, - residue: &StoreMigrationResidue, -) -> Result { - if residue.has_namespace_effect() { - return Err(Ambiguity::EffectBeforeIntent { - effect: Effect::Namespace, - }); - } - if residue.marker_stage.is_some() || residue.marker.is_some() { - return Err(Ambiguity::EffectBeforeIntent { - effect: Effect::Marker, - }); - } - if residue.has_receipt_effect() { - return Err(Ambiguity::EffectBeforeIntent { - effect: Effect::Receipt, - }); - } - let Some(stage) = residue.intent_stage.as_deref() else { - return Ok(Plan::VersionOne); - }; - match classify_stage(Stage::Intent, stage, MIGRATION_INTENT_LENGTH)? { - StageShape::Incomplete => Ok(Plan::DiscardStage { - stage: Stage::Intent, - resume: Phase::WriteIntentStage, - }), - StageShape::Complete => { - let staged = AdmittedStoreMigrationIntent::decode(stage).map_err(|source| { - Ambiguity::StageUndecodable { - stage: Stage::Intent, - source: DecodeError::Intent { source }, - } - })?; - if restart_stable_match(expected, &staged) { - Ok(Plan::Resume { - resume: Phase::SynchronizeIntentStage, - }) - } else { - Err(Ambiguity::StageDiffers { - stage: Stage::Intent, - }) - } - } - } -} - -/// Rows three and four: a durable intent, then `reader.lock` and the prefix. -fn plan_namespace( - intent: &AdmittedStoreMigrationIntent<'_>, - residue: &StoreMigrationResidue, -) -> Result { - let extent = residue - .namespace_extent() - .map_err(|(absent, present)| Ambiguity::NamespaceOutOfOrder { absent, present })?; - if extent < NAMESPACE_NAME_COUNT { - if residue.marker_stage.is_some() || residue.marker.is_some() { - return Err(Ambiguity::MarkerBeforeNamespace); - } - if residue.has_receipt_effect() { - return Err(Ambiguity::ReceiptBeforeMarker); - } - return Ok(Plan::Resume { - resume: if extent == 0 { - Phase::SynchronizeRootAfterIntentCleanup - } else { - Phase::AdmitNamespacePrefix - }, - }); - } - plan_marker(intent, residue) -} - -/// Rows five and six: the marker stage and the canonical marker. -fn plan_marker( - intent: &AdmittedStoreMigrationIntent<'_>, - residue: &StoreMigrationResidue, -) -> Result { - let Some(marker_bytes) = residue.marker.as_deref() else { - if residue.has_receipt_effect() { - return Err(Ambiguity::ReceiptBeforeMarker); - } - let Some(stage) = residue.marker_stage.as_deref() else { - return Ok(Plan::Resume { - resume: Phase::SynchronizeRootAfterNamespace, - }); - }; - return match classify_stage(Stage::Marker, stage, FORMAT_MARKER_LENGTH)? { - StageShape::Incomplete => Ok(Plan::DiscardStage { - stage: Stage::Marker, - resume: Phase::WriteMarkerStage, - }), - StageShape::Complete => { - AdmittedStoreFormatMarker::decode(stage) - .map(|_| ()) - .map_err(|source| Ambiguity::StageUndecodable { - source: DecodeError::Marker { source }, - stage: Stage::Marker, - })?; - Ok(Plan::Resume { - resume: Phase::SynchronizeMarkerStage, - }) - } - }; - }; - let marker = AdmittedStoreFormatMarker::decode(marker_bytes) - .map_err(|source| Ambiguity::MarkerUndecodable { source })?; - if let Some(stage) = residue.marker_stage.as_deref() { - if residue.has_receipt_effect() { - return Err(Ambiguity::StageAfterEffect { - stage: Stage::Marker, - effect: Effect::Receipt, - }); - } - return if stage == marker_bytes { - Ok(Plan::Resume { - resume: Phase::SynchronizeRootAfterMarker, - }) - } else { - Err(Ambiguity::StageDiffers { - stage: Stage::Marker, - }) - }; - } - plan_receipt(intent, &marker, residue) -} - -/// Row seven: the receipt stage and the canonical receipt. -fn plan_receipt( - intent: &AdmittedStoreMigrationIntent<'_>, - marker: &AdmittedStoreFormatMarker<'_>, - residue: &StoreMigrationResidue, -) -> Result { - let Some(receipt_bytes) = residue.receipt.as_deref() else { - let Some(stage) = residue.receipt_stage.as_deref() else { - return Ok(Plan::Resume { - resume: Phase::SynchronizeRootAfterMarkerCleanup, - }); - }; - return match classify_stage(Stage::Receipt, stage, MIGRATION_RECEIPT_LENGTH)? { - StageShape::Incomplete => Ok(Plan::DiscardStage { - stage: Stage::Receipt, - resume: Phase::WriteReceiptStage, - }), - StageShape::Complete => { - AdmittedStoreMigrationReceipt::decode(stage, intent, marker) - .map(|_| ()) - .map_err(|source| Ambiguity::StageUndecodable { - source: DecodeError::Receipt { source }, - stage: Stage::Receipt, - })?; - Ok(Plan::Resume { - resume: Phase::SynchronizeReceiptStage, - }) - } - }; - }; - AdmittedStoreMigrationReceipt::decode(receipt_bytes, intent, marker) - .map(|_| ()) - .map_err(|source| Ambiguity::ReceiptUndecodable { source })?; - match residue.receipt_stage.as_deref() { - None => Ok(Plan::Complete), - Some(stage) if stage == receipt_bytes => Ok(Plan::Resume { - resume: Phase::SynchronizeRootAfterReceipt, - }), - Some(_) => Err(Ambiguity::StageDiffers { - stage: Stage::Receipt, - }), - } -} - -enum StageShape { - Incomplete, - Complete, -} - -fn classify_stage(stage: Stage, bytes: &[u8], length: usize) -> Result { - match bytes.len().cmp(&length) { - std::cmp::Ordering::Less => Ok(StageShape::Incomplete), - std::cmp::Ordering::Equal => Ok(StageShape::Complete), - std::cmp::Ordering::Greater => Err(Ambiguity::StageOverlong { - stage, - observed: bytes.len(), - }), - } -} - -/// Every intent coordinate but the mount identity. -fn restart_stable_match( - expected: &AdmittedStoreMigrationIntent<'_>, - observed: &AdmittedStoreMigrationIntent<'_>, -) -> bool { - expected.catalog_generation() == observed.catalog_generation() - && expected.catalog_length() == observed.catalog_length() - && expected.catalog_digest() == observed.catalog_digest() - && expected.predecessor_catalog_digest() == observed.predecessor_catalog_digest() - && expected.inventory_digest() == observed.inventory_digest() - && expected.root_device_identity() == observed.root_device_identity() - && expected.root_file_identity() == observed.root_file_identity() - && expected.target_definition_digest() == observed.target_definition_digest() - && expected.store_identifier() == observed.store_identifier() -} diff --git a/src/adapters/store_migration/migration_recovery_residue.rs b/src/adapters/store_migration/migration_recovery_residue.rs deleted file mode 100644 index 90d0daf7..00000000 --- a/src/adapters/store_migration/migration_recovery_residue.rs +++ /dev/null @@ -1,98 +0,0 @@ -//! This boundary module owns the observed residue of one interrupted -//! migration. - -/// Names of the version-2 directory prefix in creation order, after -/// `reader.lock`. -pub const MIGRATION_NAMESPACE_PREFIX: [&str; 6] = [ - "retention", - "retention/roots", - "retention/manifests", - "gc", - "recovery", - "recovery/dispositions", -]; - -pub(super) const NAMESPACE_NAME_COUNT: usize = MIGRATION_NAMESPACE_PREFIX.len() + 1; - -/// Everything a migration may have left behind, as one observation. -/// -/// An observer reads each fixed name without following links and refuses a -/// wrong file kind, a link, or an unknown entry before producing this value, -/// so the planner sees only bytes and presence. Byte fields carry exactly the -/// bytes observed, bounded by the observer to one byte more than the record's -/// canonical length so overlong records stay distinguishable. -#[must_use] -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct StoreMigrationResidue { - /// Bytes of `migration.intent.next`, when present. - pub intent_stage: Option>, - /// Bytes of `migration.intent`, when present. - pub intent: Option>, - /// Whether the persistent `reader.lock` exists. - pub reader_fence: bool, - /// Presence of each directory in [`MIGRATION_NAMESPACE_PREFIX`] order. - pub namespace_prefix: [bool; MIGRATION_NAMESPACE_PREFIX.len()], - /// Bytes of `FORMAT.next`, when present. - pub marker_stage: Option>, - /// Bytes of `FORMAT`, when present. - pub marker: Option>, - /// Bytes of `migration.receipt.next`, when present. - pub receipt_stage: Option>, - /// Bytes of `migration.receipt`, when present. - pub receipt: Option>, -} - -impl StoreMigrationResidue { - /// The observation of a store with no migration artifact at all. - pub const VERSION_ONE: Self = Self { - intent_stage: None, - intent: None, - reader_fence: false, - namespace_prefix: [false; MIGRATION_NAMESPACE_PREFIX.len()], - marker_stage: None, - marker: None, - receipt_stage: None, - receipt: None, - }; - - /// Whether `reader.lock` or any prefix directory exists. - #[must_use] - pub fn has_namespace_effect(&self) -> bool { - self.reader_fence || self.namespace_prefix.iter().any(|present| *present) - } - - /// Whether any marker or receipt artifact or stage exists. - #[must_use] - pub const fn has_marker_or_receipt_effect(&self) -> bool { - self.marker_stage.is_some() - || self.marker.is_some() - || self.receipt_stage.is_some() - || self.receipt.is_some() - } - - /// Whether any receipt artifact or stage exists. - #[must_use] - pub const fn has_receipt_effect(&self) -> bool { - self.receipt_stage.is_some() || self.receipt.is_some() - } - - /// Length of the contiguous present prefix of `reader.lock` followed by - /// the six directories. - /// - /// # Errors - /// - /// Returns the index of the first absent slot and the index of a later - /// present slot when the prefix is not contiguous. - pub fn namespace_extent(&self) -> Result { - let slots = std::iter::once(self.reader_fence).chain(self.namespace_prefix); - let mut first_absent = None; - for (index, present) in slots.enumerate() { - match (first_absent, present) { - (None, false) => first_absent = Some(index), - (Some(absent), true) => return Err((absent, index)), - _ => {} - } - } - Ok(first_absent.unwrap_or(NAMESPACE_NAME_COUNT)) - } -} diff --git a/src/adapters/store_migration/migration_recovery_storage.rs b/src/adapters/store_migration/migration_recovery_storage.rs deleted file mode 100644 index 4b356243..00000000 --- a/src/adapters/store_migration/migration_recovery_storage.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! This boundary module owns the blocking capabilities migration recovery -//! needs beyond the forward protocol. - -use std::io; - -use super::{ - CanonicalStoreMigrationIntent, StoreMigrationFixedStage, StoreMigrationResidue, - StoreMigrationStorage, -}; - -/// Blocking storage capabilities for recovering one interrupted migration. -/// -/// An implementation holds exclusive writer authority over a root that may -/// carry any lawful migration residue. It observes without mutation, adopts -/// the exact stage and canonical handles a resume point needs by reopening -/// them by device and inode identity, and removes only an incomplete -/// pre-effect stage the planner named. -pub trait StoreMigrationRecoveryStorage: StoreMigrationStorage { - /// Observes every fixed migration name without mutation. - /// - /// # Errors - /// - /// Returns the exact I/O failure, or a typed refusal for a wrong file - /// kind, a link, or an unknown entry. - fn observe_residue(&mut self) -> io::Result; - - /// Verifies completed migration namespace admission without effects. - /// - /// Owned post-migration retention state remains permitted. This does not - /// certify retention content or perform retention recovery. - /// - /// # Errors - /// - /// Returns the original namespace refusal or filesystem failure. - fn verify_complete(&mut self) -> io::Result<()>; - - /// Reopens, verifies, and retains every exact stage and canonical record - /// the residue holds, so later phases find the handles the forward - /// protocol would have retained. - /// - /// # Errors - /// - /// Returns the exact reopen, identity, or byte verification failure. - fn adopt_residue( - &mut self, - residue: &StoreMigrationResidue, - intent: &CanonicalStoreMigrationIntent, - ) -> io::Result<()>; - - /// Removes one incomplete pre-effect stage and synchronizes the root. - /// - /// # Errors - /// - /// Returns the exact removal or synchronization failure, or a typed - /// refusal when the stage's canonical target already exists. - fn discard_stage(&mut self, stage: StoreMigrationFixedStage) -> io::Result<()>; -} diff --git a/src/adapters/store_migration/migration_resumption.rs b/src/adapters/store_migration/migration_resumption.rs deleted file mode 100644 index 3f37b89a..00000000 --- a/src/adapters/store_migration/migration_resumption.rs +++ /dev/null @@ -1,162 +0,0 @@ -//! This boundary module owns resuming an interrupted migration at one exact -//! phase. - -use std::io; - -use super::{ - CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, - StoreMigrationError, StoreMigrationPhase, StoreMigrationStorage, -}; - -/// The three canonical records one migration publishes. -pub(super) struct MigrationRecords<'a> { - pub(super) intent: &'a CanonicalStoreMigrationIntent, - pub(super) marker: CanonicalStoreFormatMarker, - pub(super) receipt: CanonicalStoreMigrationReceipt, -} - -impl<'a> MigrationRecords<'a> { - pub(super) fn for_intent(intent: &'a CanonicalStoreMigrationIntent) -> Self { - let marker = CanonicalStoreFormatMarker::version_two(); - let receipt = CanonicalStoreMigrationReceipt::from_canonical(intent, &marker); - Self { - intent, - marker, - receipt, - } - } -} - -/// Resumes one migration at `from` and runs every later phase in order. -/// -/// The storage must already hold the handles the phases before `from` -/// established; a recovery storage adopts them from the observed residue. -/// No current-state verification runs here, because recovery verified the -/// persisted intent before planning. The returned receipt exists only after -/// the final store-root synchronization. -/// -/// # Errors -/// -/// Returns [`StoreMigrationError::Storage`] naming the exact phase that -/// refused. Failure returns no receipt. -pub(super) fn resume_store_migration( - storage: &mut impl StoreMigrationStorage, - intent: &CanonicalStoreMigrationIntent, - from: StoreMigrationPhase, -) -> Result { - let records = MigrationRecords::for_intent(intent); - let start = StoreMigrationPhase::ALL - .iter() - .position(|phase| *phase == from) - .unwrap_or(StoreMigrationPhase::ALL.len()); - for phase in StoreMigrationPhase::ALL.iter().skip(start) { - execute_phase(storage, *phase, &records)?; - } - Ok(records.receipt) -} - -/// Runs exactly one phase against the storage. -pub(super) fn execute_phase( - storage: &mut impl StoreMigrationStorage, - phase: StoreMigrationPhase, - records: &MigrationRecords<'_>, -) -> Result<(), StoreMigrationError> { - let result = match phase { - StoreMigrationPhase::WriteIntentStage - | StoreMigrationPhase::SynchronizeIntentStage - | StoreMigrationPhase::LinkIntent - | StoreMigrationPhase::SynchronizeRootAfterIntent - | StoreMigrationPhase::RemoveIntentStage - | StoreMigrationPhase::SynchronizeRootAfterIntentCleanup - | StoreMigrationPhase::AdmitReaderFence - | StoreMigrationPhase::AdmitNamespacePrefix - | StoreMigrationPhase::SynchronizeRootAfterNamespace => { - intent_and_namespace_phase(storage, phase, records.intent) - } - StoreMigrationPhase::WriteMarkerStage - | StoreMigrationPhase::SynchronizeMarkerStage - | StoreMigrationPhase::LinkMarker - | StoreMigrationPhase::SynchronizeRootAfterMarker - | StoreMigrationPhase::RemoveMarkerStage - | StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup => { - marker_phase(storage, phase, &records.marker) - } - StoreMigrationPhase::WriteReceiptStage - | StoreMigrationPhase::SynchronizeReceiptStage - | StoreMigrationPhase::LinkReceipt - | StoreMigrationPhase::SynchronizeRootAfterReceipt - | StoreMigrationPhase::RemoveReceiptStage - | StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup => { - receipt_phase(storage, phase, &records.receipt) - } - }; - result.map_err(|source| StoreMigrationError::Storage { phase, source }) -} - -fn intent_and_namespace_phase( - storage: &mut impl StoreMigrationStorage, - phase: StoreMigrationPhase, - intent: &CanonicalStoreMigrationIntent, -) -> io::Result<()> { - match phase { - StoreMigrationPhase::WriteIntentStage => storage.write_intent_stage(intent), - StoreMigrationPhase::SynchronizeIntentStage => storage.synchronize_intent_stage(), - StoreMigrationPhase::LinkIntent => storage.link_intent(intent), - StoreMigrationPhase::SynchronizeRootAfterIntent => storage.synchronize_root_after_intent(), - StoreMigrationPhase::RemoveIntentStage => storage.remove_intent_stage(), - StoreMigrationPhase::SynchronizeRootAfterIntentCleanup => { - storage.synchronize_root_after_intent_cleanup() - } - StoreMigrationPhase::AdmitReaderFence => storage.admit_reader_fence(), - StoreMigrationPhase::AdmitNamespacePrefix => storage.admit_namespace_prefix(), - StoreMigrationPhase::SynchronizeRootAfterNamespace => { - storage.synchronize_root_after_namespace() - } - _ => Err(wrong_section(phase)), - } -} - -fn marker_phase( - storage: &mut impl StoreMigrationStorage, - phase: StoreMigrationPhase, - marker: &CanonicalStoreFormatMarker, -) -> io::Result<()> { - match phase { - StoreMigrationPhase::WriteMarkerStage => storage.write_marker_stage(marker), - StoreMigrationPhase::SynchronizeMarkerStage => storage.synchronize_marker_stage(), - StoreMigrationPhase::LinkMarker => storage.link_marker(marker), - StoreMigrationPhase::SynchronizeRootAfterMarker => storage.synchronize_root_after_marker(), - StoreMigrationPhase::RemoveMarkerStage => storage.remove_marker_stage(), - StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup => { - storage.synchronize_root_after_marker_cleanup() - } - _ => Err(wrong_section(phase)), - } -} - -fn receipt_phase( - storage: &mut impl StoreMigrationStorage, - phase: StoreMigrationPhase, - receipt: &CanonicalStoreMigrationReceipt, -) -> io::Result<()> { - match phase { - StoreMigrationPhase::WriteReceiptStage => storage.write_receipt_stage(receipt), - StoreMigrationPhase::SynchronizeReceiptStage => storage.synchronize_receipt_stage(), - StoreMigrationPhase::LinkReceipt => storage.link_receipt(receipt), - StoreMigrationPhase::SynchronizeRootAfterReceipt => { - storage.synchronize_root_after_receipt() - } - StoreMigrationPhase::RemoveReceiptStage => storage.remove_receipt_stage(), - StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup => { - storage.synchronize_root_after_receipt_cleanup() - } - _ => Err(wrong_section(phase)), - } -} - -fn wrong_section(phase: StoreMigrationPhase) -> io::Error { - io::Error::new( - io::ErrorKind::InvalidInput, - format!("migration phase {phase} dispatched to the wrong section"), - ) -} diff --git a/src/adapters/store_migration/migration_stage_decode_error.rs b/src/adapters/store_migration/migration_stage_decode_error.rs deleted file mode 100644 index 1b4454b5..00000000 --- a/src/adapters/store_migration/migration_stage_decode_error.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! This module owns the original decoder failure of a complete migration stage. - -use std::error::Error; -use std::fmt; - -use super::{ - StoreFormatMarkerDecodeError, StoreMigrationIntentDecodeError, StoreMigrationReceiptDecodeError, -}; - -/// The exact typed decoder failure of a complete fixed stage. -#[derive(Debug)] -pub enum StoreMigrationStageDecodeError { - /// An intent stage failed canonical admission. - Intent { - /// Original decoder failure. - source: StoreMigrationIntentDecodeError, - }, - /// A marker stage failed canonical admission. - Marker { - /// Original decoder failure. - source: StoreFormatMarkerDecodeError, - }, - /// A receipt stage failed canonical admission or record binding. - Receipt { - /// Original decoder failure. - source: StoreMigrationReceiptDecodeError, - }, -} - -impl fmt::Display for StoreMigrationStageDecodeError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Intent { source } => write!(formatter, "intent stage: {source}"), - Self::Marker { source } => write!(formatter, "marker stage: {source}"), - Self::Receipt { source } => write!(formatter, "receipt stage: {source}"), - } - } -} - -impl Error for StoreMigrationStageDecodeError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Intent { source } => Some(source), - Self::Marker { source } => Some(source), - Self::Receipt { source } => Some(source), - } - } -} diff --git a/src/adapters/store_migration/migration_storage.rs b/src/adapters/store_migration/migration_storage.rs deleted file mode 100644 index fed5ec8b..00000000 --- a/src/adapters/store_migration/migration_storage.rs +++ /dev/null @@ -1,173 +0,0 @@ -//! This boundary module owns blocking store-migration durability capabilities. - -use std::io; - -use super::{ - CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, -}; - -/// Blocking storage capabilities for one writer-locked version-2 migration. -/// -/// An implementation must retain exclusive writer authority and one pinned -/// store root for the complete operation. After `verify_current`, each method -/// corresponds to one [`StoreMigrationPhase`](super::StoreMigrationPhase) and -/// must not report success before its durability and verification obligations -/// are complete. -pub trait StoreMigrationStorage { - /// Revalidates the exact version-1 authority bound by `intent`. - /// - /// This must verify the catalog coordinates, inventory, physical root, - /// version-1 format, and absence of migration or version-2 artifacts. - /// - /// # Errors - /// - /// Returns the exact current-state or recovery-required refusal. - fn verify_current(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()>; - - /// Exclusively creates and completely writes `migration.intent.next`. - /// - /// # Errors - /// - /// Returns the exact creation, write, or flush failure. - fn write_intent_stage(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()>; - - /// Synchronizes the complete intent stage. - /// - /// # Errors - /// - /// Returns the exact file-synchronization failure. - fn synchronize_intent_stage(&mut self) -> io::Result<()>; - - /// Links and exactly verifies canonical `migration.intent`. - /// - /// # Errors - /// - /// Returns the exact link, reopen, or verification failure. - fn link_intent(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()>; - - /// Synchronizes the store root after the intent link. - /// - /// # Errors - /// - /// Returns the exact directory-synchronization failure. - fn synchronize_root_after_intent(&mut self) -> io::Result<()>; - - /// Removes only the retained intent stage. - /// - /// # Errors - /// - /// Returns the exact removal failure. - fn remove_intent_stage(&mut self) -> io::Result<()>; - - /// Synchronizes the store root after intent-stage cleanup. - /// - /// # Errors - /// - /// Returns the exact directory-synchronization failure. - fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()>; - - /// Creates or exactly admits persistent `reader.lock`. - /// - /// # Errors - /// - /// Returns the exact creation, open, or verification failure. - fn admit_reader_fence(&mut self) -> io::Result<()>; - - /// Creates or exactly admits the complete version-2 directory prefix. - /// - /// # Errors - /// - /// Returns the exact namespace creation or admission failure. - fn admit_namespace_prefix(&mut self) -> io::Result<()>; - - /// Synchronizes created namespaces and the store root. - /// - /// # Errors - /// - /// Returns the exact directory-synchronization failure. - fn synchronize_root_after_namespace(&mut self) -> io::Result<()>; - - /// Exclusively creates and completely writes `FORMAT.next`. - /// - /// # Errors - /// - /// Returns the exact creation, write, or flush failure. - fn write_marker_stage(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()>; - - /// Synchronizes the complete marker stage. - /// - /// # Errors - /// - /// Returns the exact file-synchronization failure. - fn synchronize_marker_stage(&mut self) -> io::Result<()>; - - /// Links and exactly verifies canonical `FORMAT`. - /// - /// # Errors - /// - /// Returns the exact link, reopen, or verification failure. - fn link_marker(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()>; - - /// Synchronizes the store root after the marker link. - /// - /// # Errors - /// - /// Returns the exact directory-synchronization failure. - fn synchronize_root_after_marker(&mut self) -> io::Result<()>; - - /// Removes only the retained marker stage. - /// - /// # Errors - /// - /// Returns the exact removal failure. - fn remove_marker_stage(&mut self) -> io::Result<()>; - - /// Synchronizes the store root after marker-stage cleanup. - /// - /// # Errors - /// - /// Returns the exact directory-synchronization failure. - fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()>; - - /// Exclusively creates and completely writes `migration.receipt.next`. - /// - /// # Errors - /// - /// Returns the exact creation, write, or flush failure. - fn write_receipt_stage(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()>; - - /// Synchronizes the complete receipt stage. - /// - /// # Errors - /// - /// Returns the exact file-synchronization failure. - fn synchronize_receipt_stage(&mut self) -> io::Result<()>; - - /// Links and exactly verifies canonical `migration.receipt`. - /// - /// # Errors - /// - /// Returns the exact link, reopen, or verification failure. - fn link_receipt(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()>; - - /// Synchronizes the store root after the receipt link. - /// - /// # Errors - /// - /// Returns the exact directory-synchronization failure. - fn synchronize_root_after_receipt(&mut self) -> io::Result<()>; - - /// Removes only the retained receipt stage. - /// - /// # Errors - /// - /// Returns the exact removal failure. - fn remove_receipt_stage(&mut self) -> io::Result<()>; - - /// Synchronizes the store root after receipt-stage cleanup. - /// - /// # Errors - /// - /// Returns the exact directory-synchronization failure. - fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()>; -} diff --git a/src/adapters/store_migration/migration_synchronization_mask.rs b/src/adapters/store_migration/migration_synchronization_mask.rs deleted file mode 100644 index 47c46cdb..00000000 --- a/src/adapters/store_migration/migration_synchronization_mask.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! This module owns completed migration synchronization evidence. - -/// Closed set of mandatory migration synchronization transitions. -#[must_use] -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct MigrationSynchronizationMask(u64); - -impl MigrationSynchronizationMask { - pub(super) const COMPLETE_BITS: u64 = 0x03ff; - - /// Returns the complete synchronization bit set. - #[must_use] - pub const fn bits(self) -> u64 { - self.0 - } - - pub(super) const fn complete() -> Self { - Self(Self::COMPLETE_BITS) - } -} diff --git a/src/adapters/store_migration/rationale.md b/src/adapters/store_migration/rationale.md deleted file mode 100644 index 2aaeba51..00000000 --- a/src/adapters/store_migration/rationale.md +++ /dev/null @@ -1,34 +0,0 @@ -# Migration recovery admission - -Recovery must return the exact named bytes or refuse. A residue snapshot is -untrusted evidence, not writer authority. Planning precedes record adoption, -truncated-stage removal, and all forward writes. Nested directory membership -is checked before adoption so a planner-visible prefix cannot hide foreign -entries that would otherwise fail only after creating the next stage. - -The phase-resumption helper is private to recovery. Exposing it would let -callers manufacture a completion receipt without current-state verification, -residue planning, or adoption. The public recovery entry point owns that order. - -A retained intent stage and a later namespace effect cannot arise from the -ordered protocol: stage removal precedes namespace admission. The analogous -marker-stage/receipt combination also refuses. Treating either as a harmless -cleanup would erase evidence of an ambiguous write history. - -Current authority first verifies the caller's freshly derived expected intent, -including its live mount identity, against the presently pinned root and -version-1 evidence. Planning compares a persisted intent with that expected -intent on restart-stable coordinates; a changed device or inode returns -`IntentDiffers`, while a new mount coordinate alone does not. - -Resume the earliest unproven synchronization with the surviving exact intent's -persisted bytes. If only an incomplete pre-effect intent stage exists, discard -uses the freshly verified expected intent because no complete intent survived. -Mount identity remains a fence for one live authority, not a persisted restart -identity. No format bytes change. Sealed version-1 objects remain untouched. -A completed observation performs no migration writes and does not grant -retention publication authority. - -The 68-case subprocess matrix tests process death, not physical power loss. -Exhaustive strict-stage truncations and forward-prefix laws supplement it; -broader hostile restart combinations remain independently tracked in #111. diff --git a/src/adapters/store_migration/store_identifier.rs b/src/adapters/store_migration/store_identifier.rs deleted file mode 100644 index 27f61977..00000000 --- a/src/adapters/store_migration/store_identifier.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! This module owns deterministic logical store identity. - -/// Logical identity derived from admitted version-1 state and the v2 format. -#[must_use] -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] -pub struct StoreIdentifier([u8; 32]); - -impl StoreIdentifier { - /// Returns the exact identifier bytes. - #[must_use] - pub const fn as_bytes(&self) -> &[u8; 32] { - &self.0 - } - - pub(super) const fn from_hash(hash: [u8; 32]) -> Self { - Self(hash) - } -} diff --git a/src/adapters/store_migration/store_root_identity.rs b/src/adapters/store_migration/store_root_identity.rs deleted file mode 100644 index 3ddce7b9..00000000 --- a/src/adapters/store_migration/store_root_identity.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! This module owns physical store-root recovery coordinates. - -use crate::adapters::filesystem_root_identity::FilesystemRootIdentity; - -macro_rules! root_identity { - ($name:ident, $documentation:literal) => { - #[doc = $documentation] - #[must_use] - #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - pub struct $name(u64); - - impl $name { - /// Returns the exact serialized platform coordinate. - /// - /// This value remains a comparison coordinate until a platform - /// adapter revalidates it against the opened store root. - #[must_use] - pub const fn get(self) -> u64 { - self.0 - } - - pub(super) const fn from_admitted(value: u64) -> Self { - Self(value) - } - } - }; -} - -root_identity!( - StoreRootDeviceIdentity, - "Platform device identity bound into a migration intent." -); -root_identity!( - StoreRootMountIdentity, - "Platform mount identity bound into a migration intent." -); -root_identity!( - StoreRootFileIdentity, - "Platform file identity bound into a migration intent." -); - -#[derive(Clone, Copy)] -pub(super) struct StoreRootIdentities { - device: StoreRootDeviceIdentity, - mount: StoreRootMountIdentity, - file: StoreRootFileIdentity, -} - -impl StoreRootIdentities { - pub(super) const fn new( - device: StoreRootDeviceIdentity, - mount: StoreRootMountIdentity, - file: StoreRootFileIdentity, - ) -> Self { - Self { - device, - mount, - file, - } - } - - pub(super) const fn from_filesystem(identity: FilesystemRootIdentity) -> Self { - Self::new( - StoreRootDeviceIdentity::from_admitted(identity.device()), - StoreRootMountIdentity::from_admitted(identity.mount()), - StoreRootFileIdentity::from_admitted(identity.file()), - ) - } - - pub(super) const fn device(self) -> StoreRootDeviceIdentity { - self.device - } - - pub(super) const fn mount(self) -> StoreRootMountIdentity { - self.mount - } - - pub(super) const fn file(self) -> StoreRootFileIdentity { - self.file - } -} diff --git a/src/adapters/version_two_initialization.rs b/src/adapters/version_two_initialization.rs new file mode 100644 index 00000000..f45067e2 --- /dev/null +++ b/src/adapters/version_two_initialization.rs @@ -0,0 +1,56 @@ +//! This module owns direct initialization of a version-two store under writer authority. +//! +//! Initialization is the only way a store comes into existence. It is a +//! recoverable state machine: every interrupted state is a classified prefix +//! that a later call resumes, every other nonempty root is refused before any +//! mutation, and the store admits only after one named durable commit point. +//! The protocol, its states, and its crash matrix are specified in +//! `docs/formats/segment-store-v2/initialization.md`. + +mod initialization_entries; +mod initialization_error; +mod initialization_execution; +mod initialization_namespace; +mod initialization_observation; +mod initialization_phase; +mod initialization_prefix; +#[cfg(test)] +mod initialization_prefix_tests; +mod initialization_reader_fence; +mod initialization_receipt; +mod initialization_refusal; +#[cfg(test)] +mod initialization_resume_tests; +#[cfg(test)] +mod initialization_test_fixture; + +pub use initialization_error::VersionTwoInitializationError; +pub use initialization_phase::{VersionTwoInitializationDirectory, VersionTwoInitializationPhase}; +pub use initialization_prefix::VersionTwoInitializationPrefix; +pub use initialization_receipt::VersionTwoInitialization; +pub use initialization_refusal::VersionTwoInitializationRefusal; + +pub(super) use initialization_execution::{PhaseObserver, initialize}; + +/// Where an initialization observer runs relative to one phase. +#[doc(hidden)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VersionTwoInitializationBoundary { + /// Before the phase begins. + Before, + /// After the phase returned successfully. + After, +} + +/// Repository-task observer of initialization phases. +/// +/// Returning an error fails the phase at that boundary exactly as a storage +/// failure would; the crash matrix blocks inside it for process death. +#[cfg(any(test, feature = "repository-tasks"))] +#[doc(hidden)] +pub type VersionTwoInitializationObserver = Box< + dyn FnMut( + VersionTwoInitializationPhase, + VersionTwoInitializationBoundary, + ) -> std::io::Result<()>, +>; diff --git a/src/adapters/version_two_initialization/initialization_entries.rs b/src/adapters/version_two_initialization/initialization_entries.rs new file mode 100644 index 00000000..cb70c399 --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_entries.rs @@ -0,0 +1,138 @@ +//! This module owns the idempotent creation of every entry initialization makes. +//! +//! Each operation either creates its entry or admits the exact entry a +//! previous attempt created, then synchronizes the parent directory. No +//! operation replaces, truncates, or follows a link. + +use std::io::{self, Write}; + +use cap_fs_ext::DirExt; +use cap_std::fs::{Dir, File}; + +use super::VersionTwoInitializationDirectory as Directory; +use super::initialization_namespace::{MARKER, MARKER_STAGE}; +use crate::adapters::filesystem_exact_record::{ + self as exact_record, EntryIdentity, ExactRecordError, +}; +use crate::adapters::filesystem_namespace_refusal::{ + self as namespace, FilesystemNamespaceRefusal, +}; +use crate::adapters::{ + CanonicalStoreFormatMarker, filesystem_catalog_artifact, sync_capable_directory, +}; + +/// The open `FORMAT.next` stage, bound to the inode it was created or admitted as. +pub(super) struct MarkerStage { + marker: CanonicalStoreFormatMarker, + identity: EntryIdentity, + file: File, +} + +impl MarkerStage { + /// Exclusively creates `FORMAT.next` and writes the canonical marker without synchronizing. + pub(super) fn create(root: &Dir) -> io::Result { + let marker = CanonicalStoreFormatMarker::version_two(); + let mut file = filesystem_catalog_artifact::create_exclusive(root, MARKER_STAGE)?; + let identity = EntryIdentity::of_file(&file)?; + file.write_all(marker.encoded())?; + file.flush()?; + Ok(Self { + marker, + identity, + file, + }) + } + + /// Reopens an existing `FORMAT.next` that holds exactly the canonical marker. + pub(super) fn reopen(root: &Dir) -> io::Result { + let marker = CanonicalStoreFormatMarker::version_two(); + let file = exact_record::open_regular(root, MARKER_STAGE)?; + let identity = EntryIdentity::of_file(&file)?; + let stage = Self { + marker, + identity, + file, + }; + stage.verify(root, MARKER_STAGE)?; + Ok(stage) + } + + /// Synchronizes the stage's bytes and reverifies its name, bytes, and inode. + pub(super) fn synchronize(&self, root: &Dir) -> io::Result<()> { + self.file.sync_all()?; + self.verify(root, MARKER_STAGE) + } + + /// Links the stage to `FORMAT` without replacement and verifies both names. + /// + /// An existing `FORMAT` is never replaced: it is admitted only if it is + /// already this stage's inode with the canonical bytes. + pub(super) fn link(&self, root: &Dir) -> io::Result<()> { + self.verify(root, MARKER_STAGE)?; + exact_record::link_without_replacement(root, MARKER_STAGE, root, MARKER)?; + self.verify_linked(root) + } + + /// Requires `FORMAT` and `FORMAT.next` to both name this stage's inode. + pub(super) fn verify_linked(&self, root: &Dir) -> io::Result<()> { + self.verify(root, MARKER_STAGE)?; + self.verify(root, MARKER) + } + + /// Removes `FORMAT.next`, leaving `FORMAT` as the only name of the marker. + pub(super) fn remove(self, root: &Dir) -> io::Result<()> { + self.verify_linked(root)?; + root.remove_file(MARKER_STAGE)?; + exact_record::require_absent(root, MARKER_STAGE).map_err(ExactRecordError::into_io)?; + self.verify(root, MARKER) + } + + fn verify(&self, root: &Dir, name: &str) -> io::Result<()> { + exact_record::verify_named(root, name, self.marker.encoded(), self.identity) + .map_err(ExactRecordError::into_io) + } +} + +/// Removes an incomplete `FORMAT.next` and synchronizes the root. +/// +/// The caller classified the stage under writer authority as a strict prefix +/// of the canonical marker with nothing else created after it. +pub(super) fn discard_incomplete_stage(root: &Dir) -> io::Result<()> { + root.remove_file(MARKER_STAGE)?; + exact_record::require_absent(root, MARKER_STAGE).map_err(ExactRecordError::into_io)?; + filesystem_catalog_artifact::synchronize_directory(root) +} + +/// Creates or admits one protocol directory, then synchronizes its parent. +/// +/// An existing directory is admitted only if it holds nothing but its +/// canonical children, so a resumed attempt never adopts foreign content. +pub(super) fn admit_directory(root: &Dir, directory: Directory) -> io::Result<()> { + let parent = match directory.parent() { + None => root.try_clone()?, + Some(parent) => sync_capable_directory::open(root, parent.path())?, + }; + match parent.create_dir(directory.name()) { + Ok(()) => {} + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} + Err(source) => return Err(source), + } + namespace::require_directory(&parent, directory.name())?; + let created = parent.open_dir_nofollow(directory.name())?; + for entry in created.entries()? { + let name = entry?.file_name(); + if !directory + .children() + .iter() + .any(|child| name == child.name()) + { + return Err(FilesystemNamespaceRefusal::UnexpectedEntry.into_io()); + } + } + filesystem_catalog_artifact::synchronize_directory(&parent) +} + +/// Synchronizes the store root. +pub(super) fn synchronize_root(root: &Dir) -> io::Result<()> { + filesystem_catalog_artifact::synchronize_directory(root) +} diff --git a/src/adapters/version_two_initialization/initialization_error.rs b/src/adapters/version_two_initialization/initialization_error.rs new file mode 100644 index 00000000..a81bf63d --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_error.rs @@ -0,0 +1,84 @@ +//! This module owns typed direct-initialization failures. + +use std::error::Error; +use std::fmt; +use std::io; + +use super::{VersionTwoInitializationPhase, VersionTwoInitializationRefusal}; +use crate::adapters::{FilesystemPlatformAdmissionError, WriterLockAcquireError}; + +/// Failure of direct version-two initialization. +/// +/// A failure leaves the root in a state that the protocol classifies: either +/// unchanged, or a recoverable prefix that a later call resumes. No variant +/// implies cleanup was attempted. +#[derive(Debug)] +#[non_exhaustive] +pub enum VersionTwoInitializationError { + /// The root could not be opened or does not satisfy the platform profile. + Platform { + /// Preserved platform-admission failure. + source: io::Error, + }, + /// Exclusive writer authority could not be acquired. + WriterLock { + /// Preserved writer-lock failure, including `Busy` for a concurrent attempt. + source: WriterLockAcquireError, + }, + /// The root is not a recoverable initialization prefix. + Refused { + /// The exact refusal and its evidence. + refusal: VersionTwoInitializationRefusal, + }, + /// One phase failed with an I/O error or a concurrent change it detected. + Io { + /// The exact failed phase. + phase: VersionTwoInitializationPhase, + /// Preserved storage failure. + source: io::Error, + }, + /// The committed store failed version-two admission. + Admission { + /// Preserved admission failure. + source: FilesystemPlatformAdmissionError, + }, +} + +impl VersionTwoInitializationError { + pub(super) const fn refused(refusal: VersionTwoInitializationRefusal) -> Self { + Self::Refused { refusal } + } + + pub(super) const fn io(phase: VersionTwoInitializationPhase, source: io::Error) -> Self { + Self::Io { phase, source } + } +} + +impl fmt::Display for VersionTwoInitializationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Platform { .. } => { + formatter.write_str("initialization platform admission failed") + } + Self::WriterLock { .. } => { + formatter.write_str("initialization writer-lock acquisition failed") + } + Self::Refused { refusal } => write!(formatter, "initialization refused: {refusal}"), + Self::Io { phase, .. } => write!(formatter, "initialization failed during {phase}"), + Self::Admission { .. } => { + formatter.write_str("initialized store failed version-two admission") + } + } + } +} + +impl Error for VersionTwoInitializationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Platform { source } | Self::Io { source, .. } => Some(source), + Self::WriterLock { source } => Some(source), + Self::Refused { refusal } => Some(refusal), + Self::Admission { source } => Some(source), + } + } +} diff --git a/src/adapters/version_two_initialization/initialization_execution.rs b/src/adapters/version_two_initialization/initialization_execution.rs new file mode 100644 index 00000000..b1ad79e2 --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_execution.rs @@ -0,0 +1,142 @@ +//! This module owns the ordered execution of direct initialization from one observed prefix. +//! +//! Forward protocol, crash states, and recovery are specified in +//! `docs/formats/segment-store-v2/initialization.md`. Every phase runs +//! between two observer notifications, so the crash matrix kills the process +//! at an exact boundary of this code rather than of a reimplementation. + +use std::io; + +use cap_std::fs::Dir; + +use super::initialization_entries::{self as entries, MarkerStage}; +use super::initialization_reader_fence::admit_reader_fence; +use super::{ + VersionTwoInitializationBoundary as Boundary, VersionTwoInitializationDirectory as Directory, + VersionTwoInitializationError as Error, VersionTwoInitializationPhase as Phase, + VersionTwoInitializationPrefix as Prefix, initialization_observation, +}; +use crate::adapters::FilesystemWriterLock; + +/// Observer notified before and after every phase; an error fails that phase. +pub(in crate::adapters) type PhaseObserver<'observer> = + &'observer mut dyn FnMut(Phase, Boundary) -> io::Result<()>; + +/// Classifies, locks, reclassifies, and completes initialization of `root`. +/// +/// Returns the retained writer lock and the state the call found. The first +/// classification only prevents mutation of a refused root; the second, under +/// writer authority, selects the resumed phases. +/// +/// # Errors +/// +/// Returns the first refusal, lock failure, or phase failure. Effects already +/// made remain a recoverable prefix. +pub(in crate::adapters) fn initialize( + root: &Dir, + observer: PhaseObserver<'_>, +) -> Result<(FilesystemWriterLock, Prefix), Error> { + let unlocked = initialization_observation::observe(root)?; + let mut execution = Execution { root, observer }; + let lock = execution.lock()?; + let locked = initialization_observation::observe(root)?; + execution.resume(locked)?; + let found = if unlocked == Prefix::Empty && locked == Prefix::Locked { + Prefix::Empty + } else { + locked + }; + Ok((lock, found)) +} + +struct Execution<'root, 'observer> { + root: &'root Dir, + observer: PhaseObserver<'observer>, +} + +impl Execution<'_, '_> { + fn phase( + &mut self, + phase: Phase, + operation: impl FnOnce(&Dir) -> io::Result, + ) -> Result { + (self.observer)(phase, Boundary::Before).map_err(|source| Error::io(phase, source))?; + let value = operation(self.root).map_err(|source| Error::io(phase, source))?; + (self.observer)(phase, Boundary::After).map_err(|source| Error::io(phase, source))?; + Ok(value) + } + + fn lock(&mut self) -> Result { + let phase = Phase::OpenAndLockWriterFile; + (self.observer)(phase, Boundary::Before).map_err(|source| Error::io(phase, source))?; + let directory = self + .root + .try_clone() + .map_err(|source| Error::io(phase, source))?; + let lock = FilesystemWriterLock::initialize_in(directory) + .map_err(|source| Error::WriterLock { source })?; + entries::synchronize_root(self.root).map_err(|source| Error::io(phase, source))?; + (self.observer)(phase, Boundary::After).map_err(|source| Error::io(phase, source))?; + Ok(lock) + } + + fn resume(&mut self, prefix: Prefix) -> Result<(), Error> { + let stage = match prefix { + Prefix::Empty | Prefix::Locked => { + self.phase(Phase::WriteMarkerStage, MarkerStage::create)? + } + Prefix::IncompleteMarkerStage { .. } => { + self.phase( + Phase::DiscardIncompleteMarkerStage, + entries::discard_incomplete_stage, + )?; + self.phase(Phase::WriteMarkerStage, MarkerStage::create)? + } + Prefix::MarkerStaged | Prefix::ReaderFenced | Prefix::Directories { .. } => { + reopen(self.root)? + } + Prefix::MarkerLinked => { + let stage = reopen(self.root)?; + stage + .verify_linked(self.root) + .map_err(|source| Error::io(Phase::ObservePrefix, source))?; + return self.commit(stage); + } + }; + self.build(&stage)?; + self.phase(Phase::LinkMarker, |root| stage.link(root))?; + self.commit(stage) + } + + /// Makes the stage durable, then creates the fence and every directory. + fn build(&mut self, stage: &MarkerStage) -> Result<(), Error> { + self.phase(Phase::SynchronizeMarkerStage, |root| { + stage.synchronize(root) + })?; + self.phase( + Phase::SynchronizeRootAfterMarkerStage, + entries::synchronize_root, + )?; + self.phase(Phase::AdmitReaderFence, admit_reader_fence)?; + for directory in Directory::ALL { + self.phase(Phase::AdmitDirectory(directory), |root| { + entries::admit_directory(root, directory) + })?; + } + Ok(()) + } + + /// Commits with the root synchronization after the link, then removes the stage. + fn commit(&mut self, stage: MarkerStage) -> Result<(), Error> { + self.phase(Phase::SynchronizeRootAfterMarker, entries::synchronize_root)?; + self.phase(Phase::RemoveMarkerStage, |root| stage.remove(root))?; + self.phase( + Phase::SynchronizeRootAfterCleanup, + entries::synchronize_root, + ) + } +} + +fn reopen(root: &Dir) -> Result { + MarkerStage::reopen(root).map_err(|source| Error::io(Phase::ObservePrefix, source)) +} diff --git a/src/adapters/version_two_initialization/initialization_namespace.rs b/src/adapters/version_two_initialization/initialization_namespace.rs new file mode 100644 index 00000000..0c9caf09 --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_namespace.rs @@ -0,0 +1,40 @@ +//! This module owns the root entry names that direct initialization creates or admits. + +use std::ffi::OsStr; +use std::io; + +use cap_std::fs::Dir; + +use crate::adapters::filesystem_initialization_namespace; + +pub(super) const WRITER_LOCK: &str = "writer.lock"; +pub(super) const READER_LOCK: &str = "reader.lock"; +pub(super) const MARKER: &str = "FORMAT"; +pub(super) const MARKER_STAGE: &str = "FORMAT.next"; +pub(super) const HEAD: &str = "HEAD"; +pub(super) const NEXT_HEAD: &str = "head.next"; + +/// Every root name an initialization prefix or a committed store may hold. +const ROOT_NAMES: [&str; 12] = [ + WRITER_LOCK, + READER_LOCK, + MARKER, + MARKER_STAGE, + HEAD, + NEXT_HEAD, + "staging", + "segments", + "catalogs", + "retention", + "gc", + "recovery", +]; + +pub(super) fn is_root_name(name: &OsStr) -> bool { + ROOT_NAMES.iter().any(|candidate| name == *candidate) +} + +/// Admits the committed namespace that may still hold `FORMAT.next`. +pub(super) fn admit_linked_namespace(root: &Dir) -> io::Result<()> { + filesystem_initialization_namespace::admit_version_two_marker_linked(root) +} diff --git a/src/adapters/version_two_initialization/initialization_observation.rs b/src/adapters/version_two_initialization/initialization_observation.rs new file mode 100644 index 00000000..e1ee607a --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_observation.rs @@ -0,0 +1,253 @@ +//! This module owns the mutation-free classification of an initialization root. +//! +//! The classifier decides which recoverable state a root is in, or refuses it. +//! It opens nothing for writing and follows no links. Its result authorizes +//! effects only when it runs under writer authority. + +use std::ffi::OsString; +use std::io::{self, Read}; +use std::path::PathBuf; + +use cap_fs_ext::DirExt; +use cap_std::fs::Dir; + +use super::{ + VersionTwoInitializationDirectory as Directory, VersionTwoInitializationError as Error, + VersionTwoInitializationPhase as Phase, VersionTwoInitializationPrefix as Prefix, + VersionTwoInitializationRefusal as Refusal, initialization_namespace as names, +}; +use crate::adapters::filesystem_exact_record::{self as exact_record, EntryIdentity}; +use crate::adapters::filesystem_namespace_refusal::FilesystemEntryKind; +use crate::adapters::{CanonicalStoreFormatMarker, FORMAT_MARKER_LENGTH}; + +/// Classifies `root` as exactly one lawful prefix, or refuses it. +/// +/// # Errors +/// +/// Returns [`Error::Refused`] for every root that is not a recoverable +/// prefix, including a committed one, and [`Error::Io`] at `ObservePrefix` +/// for a failed observation. +pub(super) fn observe(root: &Dir) -> Result { + let present = root_names(root)?; + let has = |name: &str| present.iter().any(|entry| entry == name); + if has(names::MARKER) { + return if has(names::MARKER_STAGE) { + observe_linked(root) + } else { + Err(Error::refused(Refusal::AlreadyInitialized)) + }; + } + for committed_only in [names::HEAD, names::NEXT_HEAD] { + if has(committed_only) { + return not_a_prefix(committed_only); + } + } + if !has(names::WRITER_LOCK) { + return refuse_other(present.first().cloned(), Prefix::Empty); + } + require_empty_lock(root, names::WRITER_LOCK)?; + observe_after_lock(root, &present) +} + +fn observe_after_lock(root: &Dir, present: &[OsString]) -> Result { + let has = |name: &str| present.iter().any(|entry| entry == name); + let others = |allowed: &[&str]| { + present + .iter() + .find(|entry| !allowed.iter().any(|name| *entry == *name)) + .cloned() + }; + if !has(names::MARKER_STAGE) { + return refuse_other(others(&[names::WRITER_LOCK]), Prefix::Locked); + } + let length = observe_marker_stage(root)?; + if length < FORMAT_MARKER_LENGTH { + let allowed = [names::WRITER_LOCK, names::MARKER_STAGE]; + return refuse_other(others(&allowed), Prefix::IncompleteMarkerStage { length }); + } + if !has(names::READER_LOCK) { + let allowed = [names::WRITER_LOCK, names::MARKER_STAGE]; + return refuse_other(others(&allowed), Prefix::MarkerStaged); + } + require_empty_lock(root, names::READER_LOCK)?; + observe_directories(root) +} + +fn refuse_other(other: Option, prefix: Prefix) -> Result { + other.map_or(Ok(prefix), |entry| { + Err(Error::refused(Refusal::NotAPrefix { + entry: PathBuf::from(entry), + })) + }) +} + +/// Requires the present directories to be exactly a creation-order prefix. +fn observe_directories(root: &Dir) -> Result { + let mut last = None; + for directory in Directory::ALL { + let present = directory_present(root, directory)?; + match (present, last) { + (true, previous) if previous == predecessor(directory) => { + require_membership(root, directory)?; + last = Some(directory); + } + (true, _) => return not_a_prefix(directory.path()), + (false, _) => {} + } + } + Ok(last.map_or(Prefix::ReaderFenced, |last| Prefix::Directories { last })) +} + +fn predecessor(directory: Directory) -> Option { + let index = Directory::ALL + .iter() + .position(|candidate| *candidate == directory)?; + index + .checked_sub(1) + .and_then(|previous| Directory::ALL.get(previous).copied()) +} + +fn directory_present(root: &Dir, directory: Directory) -> Result { + if let Some(parent) = directory.parent() + && !directory_present(root, parent)? + { + return Ok(false); + } + match root.symlink_metadata(directory.path()) { + Ok(metadata) => { + let observed = FilesystemEntryKind::observed(metadata.file_type()); + if observed == FilesystemEntryKind::Directory { + Ok(true) + } else { + Err(Error::refused(Refusal::WrongKind { + entry: directory.path(), + expected: FilesystemEntryKind::Directory, + observed, + })) + } + } + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(false), + Err(source) => Err(observation(source)), + } +} + +/// Requires a present directory to hold only its canonical children. +fn require_membership(root: &Dir, directory: Directory) -> Result<(), Error> { + let opened = root + .open_dir_nofollow(directory.path()) + .map_err(observation)?; + for entry in opened.entries().map_err(observation)? { + let name = entry.map_err(observation)?.file_name(); + let canonical = directory + .children() + .iter() + .any(|child| name == child.name()); + if !canonical { + let mut path = PathBuf::from(directory.path()); + path.push(name); + return Err(Error::refused(Refusal::NotAPrefix { entry: path })); + } + } + Ok(()) +} + +/// Returns the stage length after proving it is a canonical byte prefix. +fn observe_marker_stage(root: &Dir) -> Result { + let entry = root + .symlink_metadata(names::MARKER_STAGE) + .map_err(observation)?; + require_kind(names::MARKER_STAGE, &entry, FilesystemEntryKind::File)?; + let file = exact_record::open_regular(root, names::MARKER_STAGE).map_err(observation)?; + let metadata = file.metadata().map_err(observation)?; + require_kind(names::MARKER_STAGE, &metadata, FilesystemEntryKind::File)?; + let mismatch = || { + Error::refused(Refusal::MarkerStageMismatch { + expected_length: FORMAT_MARKER_LENGTH, + observed_length: metadata.len(), + }) + }; + let length = usize::try_from(metadata.len()).map_err(|_| mismatch())?; + if length > FORMAT_MARKER_LENGTH { + return Err(mismatch()); + } + let bound = u64::try_from(FORMAT_MARKER_LENGTH).map_err(|_| mismatch())?; + let mut bytes = Vec::with_capacity(FORMAT_MARKER_LENGTH); + file.take(bound.saturating_add(1)) + .read_to_end(&mut bytes) + .map_err(observation)?; + let canonical = CanonicalStoreFormatMarker::version_two(); + if bytes.len() == length && canonical.encoded().starts_with(&bytes) { + Ok(length) + } else { + Err(mismatch()) + } +} + +/// Classifies `FORMAT` beside `FORMAT.next`: one canonical inode, committed namespace. +fn observe_linked(root: &Dir) -> Result { + let canonical = CanonicalStoreFormatMarker::version_two(); + let stage = exact_record::open_regular(root, names::MARKER_STAGE).map_err(observation)?; + let identity = EntryIdentity::of_file(&stage).map_err(observation)?; + for name in [names::MARKER_STAGE, names::MARKER] { + exact_record::verify_named(root, name, canonical.encoded(), identity) + .map_err(|_| Error::refused(Refusal::MarkerLinkMismatch))?; + } + names::admit_linked_namespace(root).map_err(observation)?; + Ok(Prefix::MarkerLinked) +} + +fn require_empty_lock(root: &Dir, name: &'static str) -> Result<(), Error> { + let metadata = root.symlink_metadata(name).map_err(observation)?; + require_kind(name, &metadata, FilesystemEntryKind::File)?; + if metadata.len() == 0 { + Ok(()) + } else { + Err(Error::refused(Refusal::NonEmptyLockFile { + entry: name, + observed_length: metadata.len(), + })) + } +} + +fn require_kind( + entry: &'static str, + metadata: &cap_std::fs::Metadata, + expected: FilesystemEntryKind, +) -> Result<(), Error> { + let observed = FilesystemEntryKind::observed(metadata.file_type()); + if observed == expected { + Ok(()) + } else { + Err(Error::refused(Refusal::WrongKind { + entry, + expected, + observed, + })) + } +} + +/// Lists root entry names in sorted order, refusing any unknown name. +fn root_names(root: &Dir) -> Result, Error> { + let mut present = Vec::new(); + for entry in root.entries().map_err(observation)? { + let name = entry.map_err(observation)?.file_name(); + if !names::is_root_name(&name) { + return Err(Error::refused(Refusal::NotAPrefix { + entry: PathBuf::from(name), + })); + } + present.push(name); + } + present.sort(); + Ok(present) +} + +fn not_a_prefix(entry: &str) -> Result { + Err(Error::refused(Refusal::NotAPrefix { + entry: PathBuf::from(entry), + })) +} + +const fn observation(source: io::Error) -> Error { + Error::io(Phase::ObservePrefix, source) +} diff --git a/src/adapters/version_two_initialization/initialization_phase.rs b/src/adapters/version_two_initialization/initialization_phase.rs new file mode 100644 index 00000000..cc31c0bf --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_phase.rs @@ -0,0 +1,153 @@ +//! This module owns the ordered phases of direct version-two initialization. + +use std::fmt; + +/// One protocol directory created by initialization, in creation order. +/// +/// A nested directory is created only after its parent, and every directory +/// is created only after the one before it in [`Self::ALL`]. +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum VersionTwoInitializationDirectory { + /// `staging`. + Staging, + /// `segments`. + Segments, + /// `catalogs`. + Catalogs, + /// `retention`. + Retention, + /// `retention/roots`. + RetentionRoots, + /// `retention/manifests`. + RetentionManifests, + /// `gc`. + Gc, + /// `recovery`. + Recovery, + /// `recovery/dispositions`. + RecoveryDispositions, +} + +impl VersionTwoInitializationDirectory { + /// Every directory in creation order. + pub const ALL: [Self; 9] = [ + Self::Staging, + Self::Segments, + Self::Catalogs, + Self::Retention, + Self::RetentionRoots, + Self::RetentionManifests, + Self::Gc, + Self::Recovery, + Self::RecoveryDispositions, + ]; + + /// Returns the directory's path relative to the store root. + #[must_use] + pub const fn path(self) -> &'static str { + match self { + Self::Staging => "staging", + Self::Segments => "segments", + Self::Catalogs => "catalogs", + Self::Retention => "retention", + Self::RetentionRoots => "retention/roots", + Self::RetentionManifests => "retention/manifests", + Self::Gc => "gc", + Self::Recovery => "recovery", + Self::RecoveryDispositions => "recovery/dispositions", + } + } + + /// Returns the directory's own name inside its parent. + #[must_use] + pub const fn name(self) -> &'static str { + match self { + Self::RetentionRoots => "roots", + Self::RetentionManifests => "manifests", + Self::RecoveryDispositions => "dispositions", + other => other.path(), + } + } + + /// Returns the parent directory, or `None` for a root entry. + #[must_use] + pub const fn parent(self) -> Option { + match self { + Self::RetentionRoots | Self::RetentionManifests => Some(Self::Retention), + Self::RecoveryDispositions => Some(Self::Recovery), + _ => None, + } + } + + /// Returns the canonical children created inside this directory. + pub(super) const fn children(self) -> &'static [Self] { + match self { + Self::Retention => &[Self::RetentionRoots, Self::RetentionManifests], + Self::Recovery => &[Self::RecoveryDispositions], + _ => &[], + } + } +} + +/// One exact operation of direct version-two initialization. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VersionTwoInitializationPhase { + /// Open the root without following links and admit the platform profile. + AdmitPlatform, + /// Classify the root without mutation. + ObservePrefix, + /// Create or open `writer.lock`, lock it, and synchronize it and the root. + OpenAndLockWriterFile, + /// Remove an incomplete `FORMAT.next` left by process death and + /// synchronize the root. + DiscardIncompleteMarkerStage, + /// Exclusively create `FORMAT.next` and write the canonical marker. + WriteMarkerStage, + /// Synchronize `FORMAT.next` and reverify its bytes and identity. + SynchronizeMarkerStage, + /// Synchronize the root so the stage name is durable. + SynchronizeRootAfterMarkerStage, + /// Create or admit zero-length `reader.lock` and synchronize the root. + AdmitReaderFence, + /// Create or admit one protocol directory and synchronize its parent. + AdmitDirectory(VersionTwoInitializationDirectory), + /// Hard-link `FORMAT.next` to `FORMAT` without replacement. + LinkMarker, + /// Synchronize the root after the link: the commit point. + SynchronizeRootAfterMarker, + /// Remove `FORMAT.next` after the commit is durable. + RemoveMarkerStage, + /// Synchronize the root after stage removal. + SynchronizeRootAfterCleanup, +} + +impl fmt::Display for VersionTwoInitializationPhase { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AdmitDirectory(directory) => { + write!(formatter, "directory admission of {}", directory.path()) + } + other => formatter.write_str(other.label()), + } + } +} + +impl VersionTwoInitializationPhase { + const fn label(self) -> &'static str { + match self { + Self::AdmitPlatform => "platform admission", + Self::ObservePrefix => "initialization prefix observation", + Self::OpenAndLockWriterFile => "writer-lock admission", + Self::DiscardIncompleteMarkerStage => "incomplete marker-stage discard", + Self::WriteMarkerStage => "marker-stage write", + Self::SynchronizeMarkerStage => "marker-stage synchronization", + Self::SynchronizeRootAfterMarkerStage => "root synchronization after the marker stage", + Self::AdmitReaderFence => "reader-fence admission", + Self::AdmitDirectory(_) => "directory admission", + Self::LinkMarker => "marker link", + Self::SynchronizeRootAfterMarker => "commit synchronization", + Self::RemoveMarkerStage => "marker-stage removal", + Self::SynchronizeRootAfterCleanup => "root synchronization after cleanup", + } + } +} diff --git a/src/adapters/version_two_initialization/initialization_prefix.rs b/src/adapters/version_two_initialization/initialization_prefix.rs new file mode 100644 index 00000000..a12ea3e4 --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_prefix.rs @@ -0,0 +1,33 @@ +//! This module owns the lawful states a root can hold before initialization commits. + +use super::VersionTwoInitializationDirectory; + +/// One recoverable initialization state, observed under writer authority. +/// +/// Each variant is exactly one forward state of the protocol in +/// `docs/formats/segment-store-v2/initialization.md`. A root in any other +/// state is refused, and a committed root is refused as already initialized, +/// so no variant describes a committed store. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VersionTwoInitializationPrefix { + /// The root was empty. + Empty, + /// The root held only a zero-length `writer.lock`. + Locked, + /// `FORMAT.next` held a strict prefix of the canonical marker. + IncompleteMarkerStage { + /// Observed stage length, always below the marker length. + length: usize, + }, + /// `FORMAT.next` held the canonical marker; nothing else was created. + MarkerStaged, + /// The persistent reader fence existed; no directory was created. + ReaderFenced, + /// Every directory up to and including `last` existed, in order. + Directories { + /// The last directory present in creation order. + last: VersionTwoInitializationDirectory, + }, + /// `FORMAT` named the same inode as the canonical `FORMAT.next`. + MarkerLinked, +} diff --git a/src/adapters/version_two_initialization/initialization_prefix_tests.rs b/src/adapters/version_two_initialization/initialization_prefix_tests.rs new file mode 100644 index 00000000..c1f609a2 --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_prefix_tests.rs @@ -0,0 +1,183 @@ +//! Laws: every root that is not a recoverable initialization prefix refuses, unchanged. + +use std::fs; +use std::path::PathBuf; + +use super::initialization_test_fixture::{ + TestResult, initialize, initialize_until, require_refusal, witness, +}; +use super::{ + VersionTwoInitializationDirectory as Directory, VersionTwoInitializationRefusal as Refusal, +}; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::{CanonicalStoreFormatMarker, FORMAT_MARKER_LENGTH, FilesystemEntryKind}; + +/// Number of boundary notifications before `AdmitDirectory(Catalogs)` begins: +/// lock (2), marker write (2), stage sync (2), root sync (2), fence (2), and +/// the staging and segments directories (4). +const BEFORE_CATALOGS: usize = 14; + +fn refuses_unchanged( + name: &str, + prepare: impl FnOnce(&std::path::Path) -> TestResult<()>, + expected: &Refusal, +) -> TestResult<()> { + let sandbox = TestDirectory::create(name)?; + prepare(sandbox.path())?; + let before = witness(sandbox.path())?; + + require_refusal(initialize(sandbox.path()), expected)?; + + assert_eq!( + witness(sandbox.path())?, + before, + "{name}: refusal mutated the root" + ); + Ok(()) +} + +#[test] +fn a_foreign_file_is_refused_before_writer_lock_creation() -> TestResult<()> { + refuses_unchanged( + "initialization-foreign-file", + |root| Ok(fs::write(root.join("notes.txt"), b"mine")?), + &Refusal::NotAPrefix { + entry: PathBuf::from("notes.txt"), + }, + ) +} + +#[test] +fn protocol_directories_without_a_marker_stage_are_not_adopted() -> TestResult<()> { + refuses_unchanged( + "initialization-directories-without-stage", + |root| { + fs::write(root.join("writer.lock"), b"")?; + Ok(fs::create_dir(root.join("staging"))?) + }, + &Refusal::NotAPrefix { + entry: PathBuf::from("staging"), + }, + ) +} + +#[test] +fn a_directory_out_of_creation_order_is_refused() -> TestResult<()> { + refuses_unchanged( + "initialization-out-of-order", + |root| { + let interrupted = initialize_until(root, BEFORE_CATALOGS); + assert!( + interrupted.is_err(), + "the interruption must stop initialization" + ); + fs::remove_dir(root.join(Directory::Segments.path()))?; + Ok(fs::create_dir(root.join(Directory::Catalogs.path()))?) + }, + &Refusal::NotAPrefix { + entry: PathBuf::from("catalogs"), + }, + ) +} + +#[test] +fn retention_content_inside_a_prefix_contradicts_empty_retention() -> TestResult<()> { + refuses_unchanged( + "initialization-stray-retention-root", + |root| { + let interrupted = initialize_until(root, BEFORE_CATALOGS); + assert!( + interrupted.is_err(), + "the interruption must stop initialization" + ); + for directory in [ + Directory::Catalogs, + Directory::Retention, + Directory::RetentionRoots, + ] { + fs::create_dir(root.join(directory.path()))?; + } + Ok(fs::write(root.join("retention/roots/stray"), b"root")?) + }, + &Refusal::NotAPrefix { + entry: PathBuf::from("retention/roots/stray"), + }, + ) +} + +#[test] +fn a_marker_stage_that_is_not_a_canonical_prefix_is_refused() -> TestResult<()> { + refuses_unchanged( + "initialization-foreign-stage", + |root| { + fs::write(root.join("writer.lock"), b"")?; + Ok(fs::write(root.join("FORMAT.next"), b"KEEP:NOT:A:MARKER")?) + }, + &Refusal::MarkerStageMismatch { + expected_length: FORMAT_MARKER_LENGTH, + observed_length: 17, + }, + ) +} + +#[test] +fn a_nonempty_writer_lock_is_refused() -> TestResult<()> { + refuses_unchanged( + "initialization-nonempty-lock", + |root| Ok(fs::write(root.join("writer.lock"), b"x")?), + &Refusal::NonEmptyLockFile { + entry: "writer.lock", + observed_length: 1, + }, + ) +} + +#[cfg(unix)] +#[test] +fn a_linked_protocol_directory_is_refused_by_kind() -> TestResult<()> { + refuses_unchanged( + "initialization-linked-directory", + |root| { + let interrupted = initialize_until(root, BEFORE_CATALOGS); + assert!( + interrupted.is_err(), + "the interruption must stop initialization" + ); + fs::remove_dir(root.join("segments"))?; + Ok(std::os::unix::fs::symlink( + "staging", + root.join("segments"), + )?) + }, + &Refusal::WrongKind { + entry: "segments", + expected: FilesystemEntryKind::Directory, + observed: FilesystemEntryKind::Symlink, + }, + ) +} + +#[test] +fn a_committed_store_refuses_a_second_initialization() -> TestResult<()> { + refuses_unchanged( + "initialization-committed", + |root| { + drop(initialize(root)?); + Ok(()) + }, + &Refusal::AlreadyInitialized, + ) +} + +#[test] +fn a_marker_beside_its_stage_must_be_the_same_inode() -> TestResult<()> { + refuses_unchanged( + "initialization-marker-copy", + |root| { + drop(initialize(root)?); + let marker = CanonicalStoreFormatMarker::version_two(); + Ok(fs::write(root.join("FORMAT.next"), marker.encoded())?) + }, + &Refusal::MarkerLinkMismatch, + ) +} diff --git a/src/adapters/version_two_initialization/initialization_reader_fence.rs b/src/adapters/version_two_initialization/initialization_reader_fence.rs new file mode 100644 index 00000000..a2a127e7 --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_reader_fence.rs @@ -0,0 +1,39 @@ +//! This module owns creation of the persistent reader fence during initialization. + +use std::io; + +use cap_fs_ext::{FollowSymlinks, OpenOptionsFollowExt, OpenOptionsSyncExt}; +use cap_std::fs::{Dir, OpenOptions}; + +use super::initialization_namespace::READER_LOCK; +use crate::adapters::filesystem_catalog_artifact; +use crate::adapters::filesystem_namespace_refusal::{ + self as namespace, FilesystemEntryKind, FilesystemNamespaceRefusal, +}; + +/// Creates or admits zero-length `reader.lock`, then synchronizes the root. +/// +/// The open is exclusive, follows no link, and does not block, so a FIFO or +/// device planted at the name refuses by kind instead of hanging. An existing +/// entry is admitted only as a zero-length regular file. +pub(super) fn admit_reader_fence(root: &Dir) -> io::Result<()> { + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .create_new(true) + .follow(FollowSymlinks::No) + .nonblock(true); + match root.open_with(READER_LOCK, &options) { + Ok(created) => drop(created), + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} + Err(source) => return Err(source), + } + let metadata = root.symlink_metadata(READER_LOCK)?; + namespace::require_kind(metadata.file_type(), FilesystemEntryKind::File)?; + if metadata.len() == 0 { + filesystem_catalog_artifact::synchronize_directory(root) + } else { + Err(FilesystemNamespaceRefusal::UnexpectedEntry.into_io()) + } +} diff --git a/src/adapters/version_two_initialization/initialization_receipt.rs b/src/adapters/version_two_initialization/initialization_receipt.rs new file mode 100644 index 00000000..7bd30f56 --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_receipt.rs @@ -0,0 +1,38 @@ +//! This module owns the result of one completed direct initialization. + +use super::VersionTwoInitializationPrefix; +use crate::adapters::FilesystemVersionTwoAdmission; + +/// Writer authority over a store whose initialization has durably committed. +/// +/// Returned only after the commit synchronization, the removal of +/// `FORMAT.next`, the final root synchronization, and complete version-two +/// admission of the committed root. The store holds no catalog `HEAD` until +/// the first publication. +#[must_use] +pub struct VersionTwoInitialization { + admission: FilesystemVersionTwoAdmission, + found: VersionTwoInitializationPrefix, +} + +impl VersionTwoInitialization { + pub(in crate::adapters) const fn new( + admission: FilesystemVersionTwoAdmission, + found: VersionTwoInitializationPrefix, + ) -> Self { + Self { admission, found } + } + + /// The state the call found under writer authority. + /// + /// A fresh root reports [`VersionTwoInitializationPrefix::Empty`]; any + /// other value names the interrupted state the call resumed. + pub const fn found(&self) -> VersionTwoInitializationPrefix { + self.found + } + + /// Releases the retained writer authority for forward publication. + pub fn into_admission(self) -> FilesystemVersionTwoAdmission { + self.admission + } +} diff --git a/src/adapters/version_two_initialization/initialization_refusal.rs b/src/adapters/version_two_initialization/initialization_refusal.rs new file mode 100644 index 00000000..22be977b --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_refusal.rs @@ -0,0 +1,91 @@ +//! This module owns the typed reasons initialization refuses a root. + +use std::error::Error; +use std::fmt; +use std::path::PathBuf; + +use crate::adapters::FilesystemEntryKind; + +/// Why a root is not a recoverable initialization prefix. +/// +/// Every refusal is returned before the refused root is mutated: the first +/// classification runs before `writer.lock` is created, and the second runs +/// under writer authority before any later phase. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum VersionTwoInitializationRefusal { + /// The root holds `FORMAT` without `FORMAT.next`: initialization has + /// already committed. Reopen the store instead. + AlreadyInitialized, + /// An entry that no initialization prefix contains at this point: an + /// unknown name, a protocol entry without its predecessors, or a + /// directory out of creation order. + NotAPrefix { + /// The entry's path relative to the store root. + entry: PathBuf, + }, + /// A canonical entry has the wrong kind. + WrongKind { + /// The entry's path relative to the store root. + entry: &'static str, + /// The required kind. + expected: FilesystemEntryKind, + /// The kind observed without following links. + observed: FilesystemEntryKind, + }, + /// `writer.lock` or `reader.lock` is not zero-length. + NonEmptyLockFile { + /// The lock file's name. + entry: &'static str, + /// The observed length. + observed_length: u64, + }, + /// `FORMAT.next` is longer than the marker or is not a byte prefix of it. + MarkerStageMismatch { + /// The canonical marker length. + expected_length: usize, + /// The observed stage length. + observed_length: u64, + }, + /// `FORMAT` beside `FORMAT.next` is not the same inode with the same + /// canonical bytes. + MarkerLinkMismatch, +} + +impl fmt::Display for VersionTwoInitializationRefusal { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AlreadyInitialized => formatter.write_str("the store is already initialized"), + Self::NotAPrefix { entry } => write!( + formatter, + "{} is not part of any initialization prefix", + entry.display() + ), + Self::WrongKind { + entry, + expected, + observed, + } => write!(formatter, "{entry} is {observed:?}, expected {expected:?}"), + Self::NonEmptyLockFile { + entry, + observed_length, + } => write!( + formatter, + "{entry} holds {observed_length} bytes, expected 0" + ), + Self::MarkerStageMismatch { + expected_length, + observed_length, + } => write!( + formatter, + "FORMAT.next holds {observed_length} bytes that are not a prefix of the \ + {expected_length}-byte canonical marker" + ), + Self::MarkerLinkMismatch => { + formatter.write_str("FORMAT and FORMAT.next are not one canonical marker inode") + } + } + } +} + +impl Error for VersionTwoInitializationRefusal {} diff --git a/src/adapters/version_two_initialization/initialization_resume_tests.rs b/src/adapters/version_two_initialization/initialization_resume_tests.rs new file mode 100644 index 00000000..6485a2bb --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_resume_tests.rs @@ -0,0 +1,200 @@ +//! Laws: every interrupted initialization state classifies exactly and resumes to one store. + +use std::fs; +use std::sync::mpsc; +use std::thread; + +use super::initialization_test_fixture::{ + COMMITTED, TestResult, initialize, initialize_until, inventory, recorded_boundaries, + require_refusal, +}; +use super::{ + VersionTwoInitializationBoundary as Boundary, VersionTwoInitializationError as Error, + VersionTwoInitializationObserver, VersionTwoInitializationPhase as Phase, + VersionTwoInitializationPrefix as Prefix, VersionTwoInitializationRefusal as Refusal, + initialization_observation, +}; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::{ + CanonicalStoreFormatMarker, FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, + WriterLockAcquireError, +}; + +/// The state a root holds after exactly `completed` phases, by the protocol table. +fn model(completed: &[Phase]) -> Option { + completed.iter().try_fold(Prefix::Empty, |state, phase| { + Some(match phase { + Phase::OpenAndLockWriterFile => Prefix::Locked, + Phase::WriteMarkerStage => Prefix::MarkerStaged, + Phase::AdmitReaderFence => Prefix::ReaderFenced, + Phase::AdmitDirectory(last) => Prefix::Directories { last: *last }, + Phase::LinkMarker => Prefix::MarkerLinked, + Phase::RemoveMarkerStage => return None, + _ => state, + }) + }) +} + +#[test] +fn a_fresh_root_commits_to_exactly_the_initialized_namespace() -> TestResult<()> { + let sandbox = TestDirectory::create("initialization-fresh")?; + + let initialized = initialize(sandbox.path())?; + + assert_eq!(initialized.found(), Prefix::Empty); + let expected = COMMITTED.iter().map(|path| (*path).to_owned()).collect(); + assert_eq!(inventory(sandbox.path())?, expected); + let marker = CanonicalStoreFormatMarker::version_two(); + assert_eq!(fs::read(sandbox.path().join("FORMAT"))?, marker.encoded()); + for lock in ["writer.lock", "reader.lock"] { + assert_eq!(fs::metadata(sandbox.path().join(lock))?.len(), 0, "{lock}"); + } + Ok(()) +} + +/// Every boundary of the forward protocol leaves a state the classifier names +/// exactly as the protocol table does, and resuming from it commits the same +/// store; past the cleanup boundary the store is committed and refuses. +#[test] +fn every_interrupted_boundary_classifies_and_resumes_to_one_store() -> TestResult<()> { + let reference = TestDirectory::create("initialization-boundaries")?; + let boundaries = recorded_boundaries(reference.path())?; + let committed = inventory(reference.path())?; + for (stop, (phase, boundary)) in boundaries.iter().enumerate() { + let sandbox = TestDirectory::create("initialization-interrupted")?; + let interrupted = initialize_until(sandbox.path(), stop); + assert!( + matches!(interrupted, Err(Error::Io { phase: failed, .. }) if failed == *phase), + "boundary {stop} ({phase:?} {boundary:?}) did not fail at its phase" + ); + let completed: Vec = boundaries + .iter() + .take(stop.saturating_add(1)) + .filter(|(_, seen)| *seen == Boundary::After) + .map(|(seen, _)| *seen) + .collect(); + let expected = model(&completed); + let resumed = initialize(sandbox.path()); + match expected { + Some(state) => { + let found = resumed.map_err(|error| format!("{phase:?} {boundary:?}: {error}"))?; + let found = found.found(); + assert_eq!(found, state, "{phase:?} {boundary:?}"); + } + None => require_refusal(resumed, &Refusal::AlreadyInitialized)?, + } + assert_eq!( + inventory(sandbox.path())?, + committed, + "{phase:?} {boundary:?}" + ); + } + Ok(()) +} + +#[test] +fn an_incomplete_marker_stage_is_discarded_and_rewritten() -> TestResult<()> { + let sandbox = TestDirectory::create("initialization-incomplete-stage")?; + fs::write(sandbox.path().join("writer.lock"), b"")?; + let marker = CanonicalStoreFormatMarker::version_two(); + let prefix = marker + .encoded() + .get(..48) + .ok_or("marker shorter than 48 bytes")?; + fs::write(sandbox.path().join("FORMAT.next"), prefix)?; + + let initialized = initialize(sandbox.path())?; + + assert_eq!( + initialized.found(), + Prefix::IncompleteMarkerStage { length: 48 } + ); + assert_eq!(fs::read(sandbox.path().join("FORMAT"))?, marker.encoded()); + Ok(()) +} + +#[test] +fn a_failed_commit_synchronization_leaves_an_unadmitted_resumable_store() -> TestResult<()> { + let sandbox = TestDirectory::create("initialization-commit-sync-failure")?; + let observer: VersionTwoInitializationObserver = Box::new(|phase, boundary| { + if phase == Phase::SynchronizeRootAfterMarker && boundary == Boundary::Before { + Err(std::io::Error::other("injected synchronization failure")) + } else { + Ok(()) + } + }); + + let failed = FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks( + sandbox.path(), + observer, + ); + + assert!(matches!( + failed, + Err(Error::Io { + phase: Phase::SynchronizeRootAfterMarker, + .. + }) + )); + let refused = + FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path()); + assert!(matches!( + refused, + Err(FilesystemPlatformAdmissionError::Namespace { .. }) + )); + assert_eq!(initialize(sandbox.path())?.found(), Prefix::MarkerLinked); + drop(FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?); + Ok(()) +} + +/// A second attempt during the first is refused by writer authority, and once +/// the first commits, a retry is refused as already initialized. +#[test] +fn concurrent_attempts_have_exactly_one_initializer() -> TestResult<()> { + let sandbox = TestDirectory::create("initialization-concurrent")?; + let root = sandbox.path().to_path_buf(); + let (reached, paused) = mpsc::sync_channel(0); + let (resume, resumed) = mpsc::sync_channel::<()>(0); + let first = thread::spawn(move || { + let observer: VersionTwoInitializationObserver = Box::new(move |phase, boundary| { + if phase == Phase::AdmitReaderFence && boundary == Boundary::Before { + reached.send(()).map_err(std::io::Error::other)?; + resumed.recv().map_err(std::io::Error::other)?; + } + Ok(()) + }); + FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks(&root, observer) + .map(|initialized| initialized.found()) + }); + paused.recv()?; + + let contended = initialize(sandbox.path()); + + resume.send(())?; + let first = first.join().map_err(|_| "first initializer panicked")??; + assert_eq!(first, Prefix::Empty); + assert!(matches!( + contended, + Err(Error::WriterLock { + source: WriterLockAcquireError::Busy + }) + )); + require_refusal(initialize(sandbox.path()), &Refusal::AlreadyInitialized) +} + +#[test] +fn classification_of_a_committed_root_is_a_refusal_not_a_prefix() -> TestResult<()> { + let sandbox = TestDirectory::create("initialization-classify-committed")?; + drop(initialize(sandbox.path())?); + let root = cap_std::fs::Dir::open_ambient_dir(sandbox.path(), cap_std::ambient_authority())?; + + let observed = initialization_observation::observe(&root); + + assert!(matches!( + observed, + Err(Error::Refused { + refusal: Refusal::AlreadyInitialized + }) + )); + Ok(()) +} diff --git a/src/adapters/version_two_initialization/initialization_test_fixture.rs b/src/adapters/version_two_initialization/initialization_test_fixture.rs new file mode 100644 index 00000000..9938c580 --- /dev/null +++ b/src/adapters/version_two_initialization/initialization_test_fixture.rs @@ -0,0 +1,126 @@ +//! This test module owns sandboxes, witnesses, and interrupting observers for initialization laws. + +use std::collections::BTreeSet; +use std::error::Error; +use std::fs; +use std::io; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use super::{ + VersionTwoInitialization, VersionTwoInitializationBoundary as Boundary, + VersionTwoInitializationError, VersionTwoInitializationObserver, + VersionTwoInitializationPhase as Phase, VersionTwoInitializationRefusal, +}; +use crate::adapters::FilesystemVersionTwoAdmission; + +pub(super) type TestResult = Result>; + +/// Every path of a freshly committed store, relative to its root. +pub(super) const COMMITTED: [&str; 12] = [ + "FORMAT", + "catalogs", + "gc", + "reader.lock", + "recovery", + "recovery/dispositions", + "retention", + "retention/manifests", + "retention/roots", + "segments", + "staging", + "writer.lock", +]; + +pub(super) fn initialize( + root: &Path, +) -> Result { + FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(root) +} + +/// Initializes with an observer that fails the `stop`-th boundary notification. +/// +/// The failed notification stops the protocol exactly where process death at +/// that boundary would, with every earlier effect in place. +pub(super) fn initialize_until( + root: &Path, + stop: usize, +) -> Result { + let mut seen = 0_usize; + let observer: VersionTwoInitializationObserver = Box::new(move |_phase, _boundary| { + let index = seen; + seen = seen.saturating_add(1); + if index == stop { + Err(io::Error::other("interrupted by the initialization law")) + } else { + Ok(()) + } + }); + FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks(root, observer) +} + +/// Records every boundary notification of one complete initialization. +pub(super) fn recorded_boundaries(root: &Path) -> TestResult> { + let record = Arc::new(Mutex::new(Vec::new())); + let sink = Arc::clone(&record); + let observer: VersionTwoInitializationObserver = Box::new(move |phase, boundary| { + sink.lock() + .map_err(|_| io::Error::other("boundary record poisoned"))? + .push((phase, boundary)); + Ok(()) + }); + drop(FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks(root, observer)?); + let boundaries = record + .lock() + .map_err(|_| "boundary record poisoned")? + .clone(); + Ok(boundaries) +} + +/// Every path below `root`, relative and sorted. +pub(super) fn inventory(root: &Path) -> TestResult> { + let mut paths = BTreeSet::new(); + let mut pending = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + for entry in fs::read_dir(&directory)? { + let path = entry?.path(); + let relative = path.strip_prefix(root)?.to_str().ok_or("non-UTF-8 path")?; + paths.insert(relative.to_owned()); + if fs::symlink_metadata(&path)?.is_dir() { + pending.push(path); + } + } + } + Ok(paths) +} + +/// One path and, for a regular file, its bytes. +pub(super) type WitnessEntry = (String, Option>); + +/// Every path of the root with its bytes, for refusals that must not mutate. +pub(super) fn witness(root: &Path) -> TestResult> { + inventory(root)? + .into_iter() + .map(|relative| { + let path = root.join(&relative); + let bytes = if fs::symlink_metadata(&path)?.is_file() { + Some(fs::read(&path)?) + } else { + None + }; + Ok((relative, bytes)) + }) + .collect() +} + +/// Requires `result` to be exactly `expected` as an initialization refusal. +pub(super) fn require_refusal( + result: Result, + expected: &VersionTwoInitializationRefusal, +) -> TestResult<()> { + match result { + Err(VersionTwoInitializationError::Refused { refusal }) if &refusal == expected => Ok(()), + Err(other) => Err(format!("expected {expected:?}, got {other:?}").into()), + Ok(_) => Err(format!("expected {expected:?}, but initialization admitted").into()), + } +} diff --git a/src/lib.rs b/src/lib.rs index 03c07e71..1b80d3a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,15 +31,10 @@ //! Ordered publication revalidates authority, executes all durability phases, //! and returns a complete receipt. The exact version-2 store-format marker has //! canonical encoding, registered-definition admission, checksum verification, -//! and domain-separated identity. Migration-intent admission validates its -//! framing, checksum, catalog and predecessor grammar, registered definition, -//! deterministic store identity, and typed recovery coordinates. Completion -//! receipts bind an admitted intent and marker, registered empty-state digests, -//! and the complete synchronization mask. Writer-locked filesystem authority -//! now executes one fresh forward migration through exact fixed-record and -//! namespace transitions while retaining version-1 immutable bytes. -//! Partial-prefix recovery now plans and resumes lawful migration residue, -//! returning typed refusals and an ordered execution receipt. Filesystem +//! and domain-separated identity. [`FilesystemVersionTwoAdmission::initialize`] +//! creates a store directly under writer authority through a recoverable +//! state machine with one durable commit point; an interrupted call resumes +//! and every other nonempty root refuses before mutation. Filesystem //! retention publication and complete-stage restart recovery are available. //! //! Incomplete retention stages require explicit disposition before recovery @@ -56,9 +51,9 @@ //! evidence; unsupported depths refuse without downgrade or a partial report. //! //! Production durable ingestion, garbage collection and compaction remain -//! unimplemented here. General version-2 catalog publication still needs a -//! retained-closure gate; retention publication's live closure verification -//! does not establish that an arbitrary successor catalog preserves every root. +//! unimplemented here. Version-2 catalog publication runs only through the +//! retained-closure gate, which verifies every retained root against the +//! candidate catalog before any catalog mutation. #[cfg(test)] extern crate self as keep; @@ -80,52 +75,48 @@ pub use adapters::{ DurableSnapshot, DurableStore, DurableStoreError, DurableView, }; +#[cfg(any(test, feature = "repository-tasks"))] +#[doc(hidden)] +pub use adapters::VersionTwoInitializationObserver; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, - BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, CanonicalLayoutRecord, - CanonicalPublicationHead, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, - CanonicalStoreMigrationReceipt, CatalogAdmissionError, CatalogAllocationPhase, - CatalogDecodeError, CatalogEncodeError, CatalogEntryDecodeError, CatalogPublicationError, - CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogPublicationPhase, - CatalogPublicationReadiness, CatalogPublicationReceipt, CatalogPublicationStorage, - CatalogRestartArtifact, CatalogRestartByteLimit, CatalogRestartByteLimitError, - CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, - CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, - ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, EmptyDispositionSetDigest, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemMigrationAuthorityArtifact, FilesystemMigrationAuthorityError, - FilesystemMigrationInventoryError, FilesystemMigrationInventoryOperation, - FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, - FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, - FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, - FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, - FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, - FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemRecoveryStageError, FilesystemSegmentStage, FilesystemStoreMigrationAuthority, - FilesystemStoreMigrationInventoryReader, FilesystemVersionTwoAdmission, - FilesystemVersionTwoCatalogPublisher, FilesystemVersionTwoCatalogStages, - FilesystemVersionTwoRecoveryAdmission, FilesystemWriterLock, ImmutablePoolInventoryDigest, - InitialGcStateDigest, InitialRetentionStateDigest, LayoutDecodeError, LayoutDecodePolicy, - LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, - MigrationInventoryNamespace, MigrationInventoryPool, MigrationSynchronizationMask, - OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, - RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, RecoveryEntryRole, - RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryLimitError, RecoveryInventoryOperation, RecoveryInventoryStorage, - RecoveryNameClassificationError, RecoveryNameManifest, RecoveryNamedEntry, RecoveryNamespace, - RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationOutcome, - RecoveryNextHeadFinalizationPlanError, RecoveryNextHeadFinalizationReadiness, - RecoveryNextHeadFinalizationReceipt, RecoveryNextHeadFinalizationRequest, - RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, - RecoveryNextHeadFinalizationTarget, RecoveryNextHeadStage, RecoveryNextHeadStageError, - RecoveryPoolNameError, RecoveryRequiredEntry, RecoverySegmentResumeError, - RecoverySegmentResumePlanError, RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, - RecoverySegmentResumeStorageError, RecoverySegmentStage, RecoverySegmentStageError, - RecoverySegmentTruncation, RecoveryStage, RecoveryStageAssessment, - RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, RecoveryStageCompletionError, - RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, RecoveryStageCompletionReceipt, - RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, + AdmittedStoreFormatMarker, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, + CanonicalLayoutRecord, CanonicalPublicationHead, CanonicalStoreFormatMarker, + CatalogAdmissionError, CatalogAllocationPhase, CatalogDecodeError, CatalogEncodeError, + CatalogEntryDecodeError, CatalogPublicationError, CatalogPublicationExpectation, + CatalogPublicationOutcome, CatalogPublicationPhase, CatalogPublicationReadiness, + CatalogPublicationReceipt, CatalogPublicationStorage, CatalogRestartArtifact, + CatalogRestartByteLimit, CatalogRestartByteLimitError, CatalogRestartError, + CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, + CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, + ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, + FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, + FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, + FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, + FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, + FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, + FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, + FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, + FilesystemVersionTwoCatalogStages, FilesystemVersionTwoRecoveryAdmission, FilesystemWriterLock, + LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, + LayoutIdTextParseError, OpenedReusableSegment, PublicationHeadDecodeError, + RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, + RecoveryEntryRole, RecoveryInventory, RecoveryInventoryEntry, RecoveryInventoryError, + RecoveryInventoryLimit, RecoveryInventoryLimitError, RecoveryInventoryOperation, + RecoveryInventoryStorage, RecoveryNameClassificationError, RecoveryNameManifest, + RecoveryNamedEntry, RecoveryNamespace, RecoveryNextHeadFinalizationError, + RecoveryNextHeadFinalizationOutcome, RecoveryNextHeadFinalizationPlanError, + RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationReceipt, + RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, + RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, + RecoveryNextHeadStage, RecoveryNextHeadStageError, RecoveryPoolNameError, + RecoveryRequiredEntry, RecoverySegmentResumeError, RecoverySegmentResumePlanError, + RecoverySegmentResumeRequest, RecoverySegmentResumeStorage, RecoverySegmentResumeStorageError, + RecoverySegmentStage, RecoverySegmentStageError, RecoverySegmentTruncation, RecoveryStage, + RecoveryStageAssessment, RecoveryStageAssessmentError, RecoveryStageByteAdmissionError, + RecoveryStageCompletionError, RecoveryStageCompletionPlanError, RecoveryStageCompletionPool, + RecoveryStageCompletionReceipt, RecoveryStageCompletionRequest, RecoveryStageCompletionStorage, RecoveryStageCompletionStorageError, RecoveryStageCompletionTarget, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardPlanError, RecoveryStageDiscardReason, RecoveryStageDiscardReceipt, RecoveryStageDiscardRequest, RecoveryStageDiscardStorage, @@ -141,20 +132,17 @@ pub use adapters::{ SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, - StoreIdentifier, StoreInitializationError, StoreInitializationPhase, - StoreInitializationReceipt, StoreInitializationStorage, StoreMigrationError, - StoreMigrationIntentDecodeError, StoreMigrationIntentDigest, StoreMigrationInventoryEntry, - StoreMigrationInventoryEntryCount, StoreMigrationInventoryEntryCountError, - StoreMigrationInventoryError, StoreMigrationInventoryHasher, StoreMigrationPhase, - StoreMigrationReceiptDecodeError, StoreMigrationStorage, StoreRootDeviceIdentity, - StoreRootFileIdentity, StoreRootIdentityCoordinate, StoreRootMountIdentity, + StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, + StoreInitializationStorage, VersionTwoInitialization, VersionTwoInitializationBoundary, + VersionTwoInitializationDirectory, VersionTwoInitializationError, + VersionTwoInitializationPhase, VersionTwoInitializationPrefix, VersionTwoInitializationRefusal, VersionTwoRecordRefusal, WriterLockAcquireError, WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, classify_version_two_recovery_names, execute_recovery_next_head_finalization, execute_recovery_segment_resume, execute_recovery_stage_completion, - execute_recovery_stage_discard, execute_store_migration, fingerprint_recovery_stage, - initialize_store, plan_recovery_next_head_finalization, plan_recovery_segment_resume, + execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, + plan_recovery_next_head_finalization, plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, }; @@ -189,13 +177,6 @@ pub use adapters::{ #[cfg(any(test, feature = "repository-tasks"))] #[doc(hidden)] pub use adapters::{CatalogPublicationBoundary, CatalogPublicationObserver}; -pub use adapters::{ - FilesystemMigrationRecoveryRefusal, FilesystemMigrationResidueKind, MIGRATION_NAMESPACE_PREFIX, - StoreMigrationEffect, StoreMigrationFixedStage, StoreMigrationNamespacePrefix, - StoreMigrationRecoveryAmbiguity, StoreMigrationRecoveryError, StoreMigrationRecoveryPlan, - StoreMigrationRecoveryReceipt, StoreMigrationRecoveryStorage, StoreMigrationResidue, - StoreMigrationStageDecodeError, plan_store_migration_recovery, recover_store_migration, -}; #[cfg(feature = "repository-tasks")] #[doc(hidden)] pub use adapters::{ diff --git a/tests/adapters_layout_contract.rs b/tests/adapters_layout_contract.rs index 824e7b79..f16f9247 100644 --- a/tests/adapters_layout_contract.rs +++ b/tests/adapters_layout_contract.rs @@ -66,12 +66,12 @@ const RECORD_READERS: [(&str, &str); 4] = [ include_str!("../src/adapters/retention/filesystem_retention_catalog.rs"), ), ( - "src/adapters/filesystem_version_two_records.rs", - include_str!("../src/adapters/filesystem_version_two_records.rs"), + "src/adapters/filesystem_format_marker.rs", + include_str!("../src/adapters/filesystem_format_marker.rs"), ), ( - "src/adapters/store_migration/filesystem_migration_namespace.rs", - include_str!("../src/adapters/store_migration/filesystem_migration_namespace.rs"), + "src/adapters/version_two_initialization/initialization_observation.rs", + include_str!("../src/adapters/version_two_initialization/initialization_observation.rs"), ), ]; @@ -125,10 +125,10 @@ fn no_source_module_spawns_a_process() -> Result<(), Box> Ok(()) } -const EXACT_RECORD_CONSUMERS: [(&str, &str); 4] = [ +const EXACT_RECORD_CONSUMERS: [(&str, &str); 5] = [ ( - "src/adapters/filesystem_version_two_records.rs", - include_str!("../src/adapters/filesystem_version_two_records.rs"), + "src/adapters/filesystem_format_marker.rs", + include_str!("../src/adapters/filesystem_format_marker.rs"), ), ( "src/adapters/retention/filesystem_retention_current.rs", @@ -139,8 +139,12 @@ const EXACT_RECORD_CONSUMERS: [(&str, &str); 4] = [ include_str!("../src/adapters/retention/filesystem_retention_stage.rs"), ), ( - "src/adapters/store_migration/filesystem_migration_fixed_artifact.rs", - include_str!("../src/adapters/store_migration/filesystem_migration_fixed_artifact.rs"), + "src/adapters/version_two_initialization/initialization_entries.rs", + include_str!("../src/adapters/version_two_initialization/initialization_entries.rs"), + ), + ( + "src/adapters/version_two_initialization/initialization_observation.rs", + include_str!("../src/adapters/version_two_initialization/initialization_observation.rs"), ), ]; diff --git a/tests/golden_file_worldline/durable_refusal_laws.rs b/tests/golden_file_worldline/durable_refusal_laws.rs index 98518b0c..3ea12d13 100644 --- a/tests/golden_file_worldline/durable_refusal_laws.rs +++ b/tests/golden_file_worldline/durable_refusal_laws.rs @@ -1,6 +1,6 @@ //! Evidenced logical absence through a completely admitted durable catalog. //! -//! Size: medium; production publication and migration in an owned ext4 namespace. +//! Size: medium; production initialization and publication in an owned ext4 namespace. //! Oracle: a published layout names the exact missing chunk; no output is permitted. //! Delete when durable exact-layout reads are removed or stronger refusal evidence subsumes these laws. diff --git a/tests/migration_descriptor_exhaustion.rs b/tests/migration_descriptor_exhaustion.rs deleted file mode 100644 index fb1af637..00000000 --- a/tests/migration_descriptor_exhaustion.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! This module owns migration admission diagnostics under descriptor exhaustion. - -#![cfg(all(feature = "repository-tasks", target_os = "linux"))] - -#[path = "segment_filesystem_stage/sandbox.rs"] -pub mod sandbox; - -use std::error::Error; -use std::fs::File; -use std::process::Command; - -use keep::{ - FilesystemMigrationAuthorityError as AuthorityError, FilesystemStoreMigrationAuthority, - RepositoryInitializationStorage, SegmentReadPolicy, initialize_store, -}; - -const CHILD: &str = "KEEP_MIGRATION_DESCRIPTOR_CHILD"; -const TEST: &str = "a_root_clone_failure_reports_the_namespace_boundary"; - -// Size: medium. Oracle: capability duplication failure is Namespace with its original EMFILE. -// The isolated child owns a 64-descriptor ceiling and a 20-second execution ceiling. -// Delete when repository-task migration admission is removed or stronger kernel-fault coverage subsumes it. -#[test] -fn a_root_clone_failure_reports_the_namespace_boundary() -> Result<(), Box> { - if std::env::var_os(CHILD).is_some() { - return exhaust_descriptors(); - } - let output = Command::new("timeout") - .args([ - "20s", - "/bin/sh", - "-c", - "ulimit -n 64; exec \"$@\"", - "migration-descriptor-test", - ]) - .arg(std::env::current_exe()?) - .args(["--exact", TEST, "--nocapture", "--test-threads=1"]) - .env(CHILD, "1") - .output()?; - assert!( - output.status.success(), - "isolated migration clone-failure law failed: {:?}\n{}\n{}", - output.status.code(), - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - Ok(()) -} - -fn exhaust_descriptors() -> Result<(), Box> { - let sandbox = sandbox::TestDirectory::create("migration-clone-descriptor-exhaustion")?; - let mut storage = RepositoryInitializationStorage::admit_unchecked(sandbox.path())?; - let _initialized = initialize_store(&mut storage)?; - let lock = storage.into_writer_lock()?; - let descriptors = fill_descriptors()?; - let result = FilesystemStoreMigrationAuthority::open_unchecked_for_repository_tasks( - lock, - SegmentReadPolicy::MAXIMUM, - ); - drop(descriptors); - let error = result - .err() - .ok_or("descriptor exhaustion unexpectedly admitted migration")?; - assert!( - matches!(error, AuthorityError::Namespace { ref source } - if source.raw_os_error() == Some(rustix::io::Errno::MFILE.raw_os_error())), - "root clone failure must retain Namespace and EMFILE: {error:?}" - ); - sandbox.remove()?; - Ok(()) -} - -fn fill_descriptors() -> Result, Box> { - let mut descriptors = Vec::with_capacity(64); - for _ in 0..128 { - match File::open("/dev/null") { - Ok(file) => descriptors.push(file), - Err(error) if error.raw_os_error() == Some(rustix::io::Errno::MFILE.raw_os_error()) => { - return Ok(descriptors); - } - Err(error) => return Err(error.into()), - } - } - Err("the child descriptor ceiling was not enforced".into()) -} diff --git a/tests/reader_fence_process.rs b/tests/reader_fence_process.rs index 2e85ca3b..6b0d70d5 100644 --- a/tests/reader_fence_process.rs +++ b/tests/reader_fence_process.rs @@ -3,6 +3,12 @@ #[path = "reader_fence_process/fixture.rs"] pub mod fixture; +#[path = "support/populated_store.rs"] +#[allow( + dead_code, + reason = "each consumer uses a subset of the shared builder; none reads every receipt" +)] +pub mod populated_store; #[path = "reader_fence_process/reader.rs"] pub mod reader; #[path = "segment_filesystem_stage/sandbox.rs"] @@ -24,7 +30,7 @@ fn reader_death_releases_collection_without_replacing_the_fence() -> Result<(), if reader::is_child() { return reader::serve(); } - let (store, writer) = fixture::migrated("reader-death")?; + let (store, writer) = fixture::initialized("reader-death")?; assert!( matches!( FilesystemWriterLock::try_acquire(store.path()), @@ -67,7 +73,7 @@ fn collection_excludes_a_new_snapshot_until_release() -> Result<(), Box Result<(TestDirectory, FilesystemStoreMigrationAuthority), Box> { +) -> Result<(TestDirectory, FilesystemVersionTwoCatalogPublisher), Box> { let sandbox = TestDirectory::create(name)?; - let mut storage = RepositoryInitializationStorage::admit_unchecked(sandbox.path())?; - let _initialized = initialize_store(&mut storage)?; - for (name, hex) in [ - ( - "segments/221f6745cd8a5221c9a87c3707593608479282b54a4a74d0e753fd76f70e8db2.seg", - include_str!("../../conformance/segment-store/v1/one-zero-bundle-segment.hex"), - ), - ( - "catalogs/0000000000000001-0b7cad1b6de663d34beacbc214db7497f2e36ab6b08dfbd5febbc8d06a418811.cat", - include_str!("../../conformance/segment-store/v1/one-zero-bundle-catalog.hex"), - ), - ( - "HEAD", - include_str!("../../conformance/segment-store/v1/one-zero-bundle-head.hex"), - ), - ] { - fs::write(sandbox.path().join(name), decode(hex)?)?; - } - let mut migration = FilesystemStoreMigrationAuthority::open_unchecked_for_repository_tasks( - storage.into_writer_lock()?, - SegmentReadPolicy::MAXIMUM, + let (_created, writer) = populated_store::create_holding_authority( + sandbox.path(), + UNCHECKED, + &[&[0]], + FirstCatalogRecords::Complete, + )?; + populated_store::require_first_catalog( + sandbox.path(), + &decode(include_str!( + "../../conformance/segment-store/v1/one-zero-bundle-segment.hex" + ))?, + &decode(include_str!( + "../../conformance/segment-store/v1/one-zero-bundle-catalog.hex" + ))?, + &decode(include_str!( + "../../conformance/segment-store/v1/one-zero-bundle-head.hex" + ))?, )?; - let intent = migration.observe_intent()?; - let _receipt = execute_store_migration(&mut migration, &intent)?; - // Keep writer authority continuous while the caller arranges collection. - // A concurrent spawn may inherit lock descriptions until exec, so dropping - // and immediately reacquiring cannot assume all holders have disappeared. - Ok((sandbox, migration)) + Ok((sandbox, writer)) } fn decode(hex: &str) -> Result, Box> { diff --git a/tests/reader_fence_process/reader.rs b/tests/reader_fence_process/reader.rs index 60d1dd2b..ab1dc7ef 100644 --- a/tests/reader_fence_process/reader.rs +++ b/tests/reader_fence_process/reader.rs @@ -35,7 +35,7 @@ pub(super) fn serve() -> Result<(), Box> { assert_eq!( snapshot.catalog().generation().get(), 1, - "child must load the golden migrated catalog" + "child must load the golden first catalog" ); channel.write_all(b"R")?; let mut stop = [0]; diff --git a/tests/store_format_marker.rs b/tests/store_format_marker.rs index bc23f609..7d75b32d 100644 --- a/tests/store_format_marker.rs +++ b/tests/store_format_marker.rs @@ -11,8 +11,8 @@ use keep::{ const FORMAT_MARKER: &str = include_str!("../conformance/segment-store/v2/format-marker.hex"); const MARKER_DIGEST: [u8; 32] = [ - 0x4b, 0x06, 0x3c, 0x32, 0x90, 0x85, 0xab, 0xde, 0xbe, 0x86, 0xb2, 0x56, 0xd5, 0x31, 0xb1, 0x12, - 0xc7, 0xea, 0x33, 0xcb, 0x2f, 0x54, 0x5c, 0xaa, 0x40, 0xa7, 0xa8, 0x69, 0xff, 0x33, 0x37, 0xce, + 0xec, 0x11, 0x5c, 0xb4, 0x47, 0x00, 0xd1, 0xf1, 0x2a, 0xd8, 0x82, 0xf1, 0xea, 0x07, 0xd0, 0x92, + 0x1c, 0x65, 0xc2, 0x1c, 0x4a, 0xd7, 0xdc, 0x80, 0xe4, 0x44, 0x8a, 0xd4, 0x59, 0xd2, 0x0c, 0xe6, ]; #[test] diff --git a/tests/store_migration_compatibility.rs b/tests/store_migration_compatibility.rs deleted file mode 100644 index 54332c3d..00000000 --- a/tests/store_migration_compatibility.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! Public migration record admission preserves precise version and mandatory-flag refusals. - -#[allow( - dead_code, - reason = "shared immutable record fixtures include vectors used by other codec laws" -)] -#[path = "store_migration_receipt/fixture.rs"] -mod fixture; -mod support; - -use keep::{ - AdmittedStoreFormatMarker as Marker, AdmittedStoreMigrationIntent as Intent, - AdmittedStoreMigrationReceipt as Receipt, StoreFormatMarkerDecodeError as MarkerError, - StoreMigrationIntentDecodeError as IntentError, - StoreMigrationReceiptDecodeError as ReceiptError, -}; -use std::error::Error; - -// Size: small. Oracle: all migration records use version 2 at bytes 16..18 (v2 format specification). -// Delete only if version 2 is retired or stronger public codec laws subsume it. -#[test] -fn unsupported_migration_versions_refuse_with_exact_coordinates() -> Result<(), Box> { - let marker = fixture::marker_bytes()?; - let intent = fixture::intent_bytes()?; - let admitted_marker = Marker::decode(&marker)?; - let admitted_intent = Intent::decode(&intent)?; - for version in [0_u16, 1, 3, u16::MAX] { - let field = version.to_be_bytes(); - assert_eq!( - Marker::decode(&changed(marker.clone(), 16, &field)?).err(), - Some(MarkerError::UnsupportedVersion { - expected: 2, - observed: version - }) - ); - assert_eq!( - Intent::decode(&changed(intent.clone(), 16, &field)?).err(), - Some(IntentError::UnsupportedVersion { - expected: 2, - observed: version - }) - ); - assert_eq!( - Receipt::decode( - &changed(fixture::receipt_bytes()?, 16, &field)?, - &admitted_intent, - &admitted_marker - ) - .err(), - Some(ReceiptError::UnsupportedVersion { - expected: 2, - observed: version - }) - ); - } - Ok(()) -} - -// Size: small. Oracle: every bit of the v2 flags word at bytes 20..24 is mandatory and unsupported. -// Delete only when specific flag bits gain documented compatible semantics. -#[test] -fn every_unknown_mandatory_flag_refuses_without_downgrade() -> Result<(), Box> { - let marker = fixture::marker_bytes()?; - let intent = fixture::intent_bytes()?; - let admitted_marker = Marker::decode(&marker)?; - let admitted_intent = Intent::decode(&intent)?; - for bit in 0..32 { - let flags = 1_u32.checked_shl(bit).ok_or("flag shift overflow")?; - let field = flags.to_be_bytes(); - assert_eq!( - Marker::decode(&changed(marker.clone(), 20, &field)?).err(), - Some(MarkerError::UnsupportedFlags { observed: flags }) - ); - assert_eq!( - Intent::decode(&changed(intent.clone(), 20, &field)?).err(), - Some(IntentError::UnsupportedFlags { observed: flags }) - ); - assert_eq!( - Receipt::decode( - &changed(fixture::receipt_bytes()?, 20, &field)?, - &admitted_intent, - &admitted_marker - ) - .err(), - Some(ReceiptError::UnsupportedFlags { observed: flags }) - ); - } - Ok(()) -} - -fn changed(mut bytes: Vec, start: usize, field: &[u8]) -> Result, Box> { - let end = start - .checked_add(field.len()) - .ok_or("field range overflow")?; - bytes - .get_mut(start..end) - .ok_or("field outside fixture")? - .copy_from_slice(field); - Ok(bytes) -} diff --git a/tests/store_migration_execution.rs b/tests/store_migration_execution.rs deleted file mode 100644 index 1c7f0159..00000000 --- a/tests/store_migration_execution.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! Version-2 store-migration execution laws. - -#[path = "store_migration_storage/recording_storage.rs"] -pub mod recording_storage; -mod support; - -use std::error::Error; -use std::io; - -use keep::{ - AdmittedStoreMigrationIntent, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, - CanonicalStoreMigrationReceipt, StoreMigrationError, StoreMigrationPhase, - execute_store_migration, -}; -use recording_storage::RecordingStorage; - -const INTENT: &str = include_str!("../conformance/segment-store/v2/migration-intent.hex"); - -#[test] -fn migration_executes_every_phase_before_returning_its_receipt() -> Result<(), Box> { - let (intent, marker) = artifacts()?; - let expected = CanonicalStoreMigrationReceipt::from_canonical(&intent, &marker); - let mut storage = RecordingStorage::default(); - - let receipt = execute_store_migration(&mut storage, &intent)?; - - assert_eq!(receipt, expected); - assert_eq!(storage.verification_count(), 1); - assert_eq!(storage.observed(), StoreMigrationPhase::ALL); - Ok(()) -} - -#[test] -fn current_verification_refuses_before_every_migration_phase() -> Result<(), Box> { - let (intent, _marker) = artifacts()?; - let mut storage = RecordingStorage::verification_failure(); - - let Err(error) = execute_store_migration(&mut storage, &intent) else { - return Err("current-state refusal unexpectedly admitted migration".into()); - }; - - match error { - StoreMigrationError::CurrentVerification { source } => { - assert_eq!(source.kind(), io::ErrorKind::PermissionDenied); - } - other @ StoreMigrationError::Storage { .. } => { - return Err(format!("unexpected migration error: {other}").into()); - } - } - assert_eq!(storage.verification_count(), 1); - assert!(storage.observed().is_empty()); - Ok(()) -} - -#[test] -fn every_phase_failure_stops_before_all_later_mutation() -> Result<(), Box> { - let (intent, _marker) = artifacts()?; - for (index, phase) in StoreMigrationPhase::ALL.into_iter().enumerate() { - let mut storage = RecordingStorage::failing_at(phase); - let Err(error) = execute_store_migration(&mut storage, &intent) else { - return Err("injected refusal unexpectedly admitted migration".into()); - }; - assert_storage_error(error, phase)?; - assert_eq!(storage.verification_count(), 1); - let expected = StoreMigrationPhase::ALL - .get(..=index) - .ok_or("migration phase prefix is out of bounds")?; - assert_eq!(storage.observed(), expected); - } - Ok(()) -} - -fn assert_storage_error( - error: StoreMigrationError, - expected_phase: StoreMigrationPhase, -) -> Result<(), Box> { - match error { - StoreMigrationError::Storage { phase, source } => { - assert_eq!(phase, expected_phase); - assert_eq!(source.kind(), io::ErrorKind::Other); - Ok(()) - } - other @ StoreMigrationError::CurrentVerification { .. } => { - Err(format!("unexpected migration error: {other}").into()) - } - } -} - -fn artifacts() -> Result<(CanonicalStoreMigrationIntent, CanonicalStoreFormatMarker), Box> -{ - let intent_bytes = support::decode_hex(INTENT.trim_end())?; - let admitted = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; - Ok(( - CanonicalStoreMigrationIntent::from_admitted(&admitted), - CanonicalStoreFormatMarker::version_two(), - )) -} diff --git a/tests/store_migration_intent.rs b/tests/store_migration_intent.rs deleted file mode 100644 index 2040286e..00000000 --- a/tests/store_migration_intent.rs +++ /dev/null @@ -1,200 +0,0 @@ -//! Canonical version-2 store-migration intent laws. - -#[path = "store_migration_intent/fixture.rs"] -mod fixture; -mod support; - -use std::io; - -use fixture::{CATALOG_DIGEST, INTENT_DIGEST, INVENTORY_DIGEST, STORE_IDENTIFIER, fixture_bytes}; -use keep::{ - AdmittedStoreMigrationIntent, CatalogGeneration, CatalogGenerationError, CatalogLength, - CatalogLengthError, StoreFormatDefinitionDigest, StoreMigrationIntentDecodeError, -}; - -#[test] -fn intent_admits_every_frozen_coordinate() -> Result<(), Box> { - let bytes = fixture_bytes()?; - let intent = AdmittedStoreMigrationIntent::decode(&bytes)?; - - assert_eq!(intent.encoded(), bytes); - assert_eq!(intent.catalog_generation(), CatalogGeneration::new(1)?); - assert_eq!(intent.catalog_length(), CatalogLength::new(352)?); - assert_eq!(intent.catalog_digest().as_bytes(), &CATALOG_DIGEST); - assert_eq!(intent.predecessor_catalog_digest(), None); - assert_eq!(intent.inventory_digest().as_bytes(), &INVENTORY_DIGEST); - assert_eq!(intent.root_device_identity().get(), 1); - assert_eq!(intent.root_mount_identity().get(), 2); - assert_eq!(intent.root_file_identity().get(), 3); - assert_eq!( - intent.target_definition_digest(), - StoreFormatDefinitionDigest::VERSION_TWO - ); - assert_eq!(intent.store_identifier().as_bytes(), &STORE_IDENTIFIER); - assert_eq!(intent.digest().as_bytes(), &INTENT_DIGEST); - Ok(()) -} - -#[test] -fn intent_framing_has_exact_first_refusals() -> Result<(), Box> { - let bytes = fixture_bytes()?; - let mut truncated = bytes.clone(); - assert!(truncated.pop().is_some()); - assert_eq!( - AdmittedStoreMigrationIntent::decode(&truncated), - Err(StoreMigrationIntentDecodeError::WrongLength { - expected: 256, - observed: 255, - }) - ); - let mut extended = bytes.clone(); - extended.push(0); - assert_eq!( - AdmittedStoreMigrationIntent::decode(&extended), - Err(StoreMigrationIntentDecodeError::WrongLength { - expected: 256, - observed: 257, - }) - ); - assert_fixed_refusal( - 0, - StoreMigrationIntentDecodeError::InvalidMagic { - observed: mutated_array(&bytes, 0, 0)?, - }, - )?; - assert_fixed_refusal( - 17, - StoreMigrationIntentDecodeError::UnsupportedVersion { - expected: 2, - observed: 3, - }, - )?; - assert_fixed_refusal( - 19, - StoreMigrationIntentDecodeError::InvalidRecordLength { - expected: 256, - observed: 257, - }, - )?; - assert_fixed_refusal( - 23, - StoreMigrationIntentDecodeError::UnsupportedFlags { observed: 1 }, - )?; - Ok(()) -} - -#[test] -fn checksum_and_semantic_laws_have_exact_precedence() -> Result<(), Box> { - let mut bytes = fixture_bytes()?; - flip_byte(&mut bytes, 31)?; - assert!(matches!( - AdmittedStoreMigrationIntent::decode(&bytes), - Err(StoreMigrationIntentDecodeError::ChecksumMismatch { .. }) - )); - refresh_checksum(&mut bytes)?; - assert_eq!( - AdmittedStoreMigrationIntent::decode(&bytes), - Err(StoreMigrationIntentDecodeError::InvalidCatalogGeneration { - observed: 0, - source: CatalogGenerationError::Zero, - }) - ); - - assert_semantic_refusal( - 39, - StoreMigrationIntentDecodeError::InvalidCatalogLength { - observed: 353, - source: CatalogLengthError::NotCongruent { observed: 353 }, - }, - )?; - assert_semantic_refusal( - 103, - StoreMigrationIntentDecodeError::NonZeroInitialPredecessor { - observed: mutated_array(&fixture_bytes()?, 72, 31)?, - }, - )?; - - let mut successor = fixture_bytes()?; - let generation = successor - .get_mut(31) - .ok_or_else(|| io::Error::other("intent lacks generation field"))?; - *generation = 2; - refresh_checksum(&mut successor)?; - assert_eq!( - AdmittedStoreMigrationIntent::decode(&successor), - Err(StoreMigrationIntentDecodeError::MissingSuccessorPredecessor { generation: 2 }) - ); - - assert_semantic_refusal( - 160, - StoreMigrationIntentDecodeError::DefinitionDigestMismatch { - expected: *StoreFormatDefinitionDigest::VERSION_TWO.as_bytes(), - observed: mutated_array(&fixture_bytes()?, 160, 0)?, - }, - )?; - assert_semantic_refusal( - 223, - StoreMigrationIntentDecodeError::StoreIdentifierMismatch { - expected: STORE_IDENTIFIER, - observed: mutated_array(&fixture_bytes()?, 192, 31)?, - }, - )?; - Ok(()) -} - -fn assert_fixed_refusal( - offset: usize, - expected: StoreMigrationIntentDecodeError, -) -> Result<(), Box> { - let mut bytes = fixture_bytes()?; - flip_byte(&mut bytes, offset)?; - assert_eq!(AdmittedStoreMigrationIntent::decode(&bytes), Err(expected)); - Ok(()) -} - -fn assert_semantic_refusal( - offset: usize, - expected: StoreMigrationIntentDecodeError, -) -> Result<(), Box> { - let mut bytes = fixture_bytes()?; - flip_byte(&mut bytes, offset)?; - refresh_checksum(&mut bytes)?; - assert_eq!(AdmittedStoreMigrationIntent::decode(&bytes), Err(expected)); - Ok(()) -} - -fn mutated_array( - bytes: &[u8], - offset: usize, - relative: usize, -) -> Result<[u8; WIDTH], io::Error> { - let end = offset - .checked_add(WIDTH) - .ok_or_else(|| io::Error::other("intent field offset overflow"))?; - let field = bytes - .get(offset..end) - .ok_or_else(|| io::Error::other("intent lacks fixed field"))?; - let mut observed = <[u8; WIDTH]>::try_from(field) - .map_err(|_| io::Error::other("intent field width mismatch"))?; - flip_byte(&mut observed, relative)?; - Ok(observed) -} - -fn flip_byte(bytes: &mut [u8], offset: usize) -> Result<(), io::Error> { - let byte = bytes - .get_mut(offset) - .ok_or_else(|| io::Error::other("intent mutation offset is out of bounds"))?; - *byte ^= 1; - Ok(()) -} - -fn refresh_checksum(bytes: &mut [u8]) -> Result<(), io::Error> { - let (preimage, checksum) = bytes - .split_at_mut_checked(224) - .ok_or_else(|| io::Error::other("intent lacks checksum boundary"))?; - let mut hasher = blake3::Hasher::new(); - hasher.update(b"keep.store-migration-intent-checksum/v2\0"); - hasher.update(preimage); - checksum.copy_from_slice(hasher.finalize().as_bytes()); - Ok(()) -} diff --git a/tests/store_migration_intent/fixture.rs b/tests/store_migration_intent/fixture.rs deleted file mode 100644 index 83da59d8..00000000 --- a/tests/store_migration_intent/fixture.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! This module owns frozen migration-intent fixture values. -#![allow( - clippy::redundant_pub_crate, - reason = "the parent integration-test module consumes this private fixture" -)] - -use std::io; - -use super::support; - -const MIGRATION_INTENT: &str = - include_str!("../../conformance/segment-store/v2/migration-intent.hex"); - -pub(super) const CATALOG_DIGEST: [u8; 32] = [ - 0x04, 0xb8, 0x25, 0x19, 0xb0, 0x39, 0x9b, 0xae, 0xfd, 0x0b, 0x9c, 0x0f, 0x32, 0xa8, 0x71, 0x05, - 0x2e, 0x4c, 0x47, 0xe3, 0xa0, 0x02, 0x26, 0xab, 0x03, 0xb2, 0x16, 0x61, 0x47, 0x0f, 0x73, 0x20, -]; -pub(super) const INVENTORY_DIGEST: [u8; 32] = [ - 0x40, 0xbf, 0x5d, 0x49, 0xc3, 0x48, 0x47, 0xac, 0x9c, 0xf4, 0x6a, 0x25, 0x6f, 0x34, 0x3c, 0xee, - 0x80, 0xcd, 0x98, 0x0d, 0x14, 0x05, 0xd2, 0xdd, 0x02, 0xce, 0xff, 0x8f, 0x58, 0xd6, 0x74, 0xf9, -]; -pub(super) const STORE_IDENTIFIER: [u8; 32] = [ - 0x0c, 0xd9, 0xd3, 0xdf, 0xbe, 0xc9, 0xb3, 0x49, 0xfe, 0x42, 0xd2, 0x14, 0x75, 0x27, 0x1b, 0x0e, - 0x8d, 0xe2, 0x3c, 0x04, 0x34, 0x40, 0xd6, 0x42, 0x7a, 0x1c, 0x37, 0x89, 0x8a, 0xd1, 0xdd, 0x79, -]; -pub(super) const INTENT_DIGEST: [u8; 32] = [ - 0xa1, 0x5a, 0x00, 0x00, 0x02, 0x19, 0xdf, 0x20, 0x97, 0x9d, 0xa3, 0x64, 0x19, 0x04, 0x6e, 0xae, - 0x9a, 0x0b, 0xa9, 0x98, 0x64, 0x5f, 0xbf, 0xe3, 0x08, 0xea, 0x43, 0x35, 0xa8, 0x32, 0x6b, 0x44, -]; - -pub(super) fn fixture_bytes() -> Result, io::Error> { - support::decode_hex(MIGRATION_INTENT.trim_end()) -} diff --git a/tests/store_migration_intent_encoding.rs b/tests/store_migration_intent_encoding.rs deleted file mode 100644 index 34a85a93..00000000 --- a/tests/store_migration_intent_encoding.rs +++ /dev/null @@ -1,129 +0,0 @@ -//! Canonical version-2 store-migration intent encoding laws. - -#[path = "store_migration_intent/fixture.rs"] -mod fixture; -mod support; - -use std::error::Error; - -use keep::{ - AdmittedSegment, AdmittedStoreMigrationIntent, CanonicalStoreMigrationIntent, - ChecksummedCatalog, ChecksummedPublicationHead, LayoutEntryLimit, SegmentReadPolicy, - SegmentRecordLimit, -}; -use support::decode_hex; - -const SEGMENT: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); -const CATALOG: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); -const HEAD: &str = include_str!("../conformance/segment-store/v1/one-zero-head.hex"); -const CATALOG_TWO: &str = - include_str!("../conformance/segment-store/v1/one-zero-catalog-generation-two.hex"); -const HEAD_TWO: &str = - include_str!("../conformance/segment-store/v1/one-zero-head-generation-two.hex"); -const PREDECESSOR_OFFSET: usize = 72; -const PREDECESSOR_END: usize = 104; - -#[test] -fn admitted_coordinates_reproduce_the_frozen_intent() -> Result<(), Box> { - let expected = fixture::fixture_bytes()?; - let admitted = AdmittedStoreMigrationIntent::decode(&expected)?; - assert_eq!( - admitted.inventory_digest().as_bytes(), - &fixture::INVENTORY_DIGEST - ); - let canonical = canonical_intent(CATALOG, HEAD, &admitted)?; - - assert_eq!(canonical.encoded(), expected); - assert_eq!(canonical.digest(), admitted.digest()); - assert_canonical_coordinates(&canonical, &admitted); - assert_eq!(canonical.digest().as_bytes(), &fixture::INTENT_DIGEST); - assert_eq!( - canonical.store_identifier().as_bytes(), - &fixture::STORE_IDENTIFIER - ); - Ok(()) -} - -fn assert_canonical_coordinates( - canonical: &CanonicalStoreMigrationIntent, - admitted: &AdmittedStoreMigrationIntent<'_>, -) { - assert_eq!( - canonical.catalog_generation(), - admitted.catalog_generation() - ); - assert_eq!(canonical.catalog_length(), admitted.catalog_length()); - assert_eq!(canonical.catalog_digest(), admitted.catalog_digest()); - assert_eq!( - canonical.predecessor_catalog_digest(), - admitted.predecessor_catalog_digest() - ); - assert_eq!(canonical.inventory_digest(), admitted.inventory_digest()); - assert_eq!( - canonical.root_device_identity(), - admitted.root_device_identity() - ); - assert_eq!( - canonical.root_mount_identity(), - admitted.root_mount_identity() - ); - assert_eq!( - canonical.root_file_identity(), - admitted.root_file_identity() - ); - assert_eq!( - canonical.target_definition_digest(), - admitted.target_definition_digest() - ); - assert_eq!(canonical.store_identifier(), admitted.store_identifier()); -} - -#[test] -fn successor_intent_encodes_the_exact_predecessor() -> Result<(), Box> { - let source_bytes = fixture::fixture_bytes()?; - let source = AdmittedStoreMigrationIntent::decode(&source_bytes)?; - let canonical = canonical_intent(CATALOG_TWO, HEAD_TWO, &source)?; - let admitted = AdmittedStoreMigrationIntent::decode(canonical.encoded())?; - - assert_eq!(admitted.catalog_generation().get(), 2); - assert_eq!( - canonical.encoded().get(PREDECESSOR_OFFSET..PREDECESSOR_END), - Some(fixture::CATALOG_DIGEST.as_slice()) - ); - assert_eq!( - admitted - .predecessor_catalog_digest() - .ok_or("successor intent omitted its predecessor")? - .as_bytes(), - &fixture::CATALOG_DIGEST - ); - Ok(()) -} - -fn canonical_intent( - catalog_hex: &str, - head_hex: &str, - source: &AdmittedStoreMigrationIntent<'_>, -) -> Result> { - let segment_bytes = protocol_fixture(SEGMENT)?; - let catalog_bytes = protocol_fixture(catalog_hex)?; - let head_bytes = protocol_fixture(head_hex)?; - let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; - let catalog = ChecksummedCatalog::decode(&catalog_bytes)?.admit(&[segment])?; - let snapshot = ChecksummedPublicationHead::decode(&head_bytes)?.admit(catalog)?; - Ok(CanonicalStoreMigrationIntent::from_snapshot( - &snapshot, - source.inventory_digest(), - source.root_device_identity(), - source.root_mount_identity(), - source.root_file_identity(), - )) -} - -const fn maximum_policy() -> SegmentReadPolicy { - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) -} - -fn protocol_fixture(hex: &str) -> Result, Box> { - decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) -} diff --git a/tests/store_migration_inventory.rs b/tests/store_migration_inventory.rs deleted file mode 100644 index ffe49fc7..00000000 --- a/tests/store_migration_inventory.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! Canonical version-1 immutable-pool migration inventory laws. - -mod support; - -use std::error::Error; - -use keep::{ - AdmittedSegment, ChecksummedCatalog, LayoutEntryLimit, SegmentReadPolicy, SegmentRecordLimit, - StoreMigrationInventoryEntry, StoreMigrationInventoryEntryCount, - StoreMigrationInventoryEntryCountError, StoreMigrationInventoryError, - StoreMigrationInventoryHasher, -}; -use support::decode_hex; - -const SEGMENT: &str = include_str!("../conformance/segment-store/v1/one-zero-segment.hex"); -const CATALOG: &str = include_str!("../conformance/segment-store/v1/one-zero-catalog.hex"); -const SEGMENT_ENTRY: &str = concat!( - "0100000000000000", - "0000000000000000", - "0000000000000151", - "b7542dced2ab770894a14d1d04b066e3a899942602c5986d35ba6df6c1a35cfc", -); -const CATALOG_ENTRY: &str = concat!( - "0200000000000000", - "0000000000000001", - "0000000000000160", - "04b82519b0399baefd0b9c0f32a871052e4c47e3a00226ab03b21661470f7320", -); -const INVENTORY_DIGEST: &str = "40bf5d49c34847ac9cf46a256f343cee80cd980d1405d2dd02ceff8f58d674f9"; - -#[test] -fn frozen_inventory_entries_and_digest_are_exact() -> Result<(), Box> { - let (segment, catalog) = frozen_entries()?; - assert_eq!(segment.encoded().as_slice(), decode_hex(SEGMENT_ENTRY)?); - assert_eq!(catalog.encoded().as_slice(), decode_hex(CATALOG_ENTRY)?); - - let count = StoreMigrationInventoryEntryCount::new(2)?; - let mut inventory = StoreMigrationInventoryHasher::new(count); - inventory.push(segment)?; - inventory.push(catalog)?; - let digest = inventory.finish()?; - assert_eq!(digest.as_bytes().as_slice(), decode_hex(INVENTORY_DIGEST)?); - Ok(()) -} - -#[test] -fn inventory_refuses_duplicate_and_out_of_order_entries() -> Result<(), Box> { - let (segment, catalog) = frozen_entries()?; - let count = StoreMigrationInventoryEntryCount::new(2)?; - - let mut duplicate = StoreMigrationInventoryHasher::new(count); - duplicate.push(segment)?; - assert_eq!( - duplicate.push(segment), - Err(StoreMigrationInventoryError::Duplicate { entry: segment }) - ); - - let mut out_of_order = StoreMigrationInventoryHasher::new(count); - out_of_order.push(catalog)?; - assert_eq!( - out_of_order.push(segment), - Err(StoreMigrationInventoryError::OutOfOrder { - previous: catalog, - observed: segment, - }) - ); - Ok(()) -} - -#[test] -fn inventory_refuses_count_overrun_and_incomplete_finalization() -> Result<(), Box> { - let (segment, catalog) = frozen_entries()?; - let one = StoreMigrationInventoryEntryCount::new(1)?; - let two = StoreMigrationInventoryEntryCount::new(2)?; - - let mut overrun = StoreMigrationInventoryHasher::new(one); - overrun.push(segment)?; - assert_eq!( - overrun.push(catalog), - Err(StoreMigrationInventoryError::EntryCountExceeded { - expected: one, - observed: 2, - }) - ); - - let mut incomplete = StoreMigrationInventoryHasher::new(two); - incomplete.push(segment)?; - assert_eq!( - incomplete.finish(), - Err(StoreMigrationInventoryError::Incomplete { - expected: two, - observed: 1, - }) - ); - Ok(()) -} - -#[test] -fn inventory_count_has_the_exact_protocol_bound() { - assert_eq!( - StoreMigrationInventoryEntryCount::new(0).map(StoreMigrationInventoryEntryCount::get), - Ok(0) - ); - assert_eq!( - StoreMigrationInventoryEntryCount::new(StoreMigrationInventoryEntryCount::MAXIMUM) - .map(StoreMigrationInventoryEntryCount::get), - Ok(StoreMigrationInventoryEntryCount::MAXIMUM) - ); - assert_eq!( - StoreMigrationInventoryEntryCount::new(2_097_153), - Err(StoreMigrationInventoryEntryCountError::AboveMaximum { - observed: 2_097_153, - maximum: 2_097_152, - }) - ); -} - -fn frozen_entries() --> Result<(StoreMigrationInventoryEntry, StoreMigrationInventoryEntry), Box> { - let segment_bytes = fixture(SEGMENT)?; - let catalog_bytes = fixture(CATALOG)?; - let segment = AdmittedSegment::decode(&segment_bytes, maximum_policy())?; - let segment_entry = StoreMigrationInventoryEntry::from_segment(&segment); - let catalog = ChecksummedCatalog::decode(&catalog_bytes)?.admit(&[segment])?; - let catalog_entry = StoreMigrationInventoryEntry::from_catalog(&catalog); - Ok((segment_entry, catalog_entry)) -} - -const fn maximum_policy() -> SegmentReadPolicy { - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) -} - -fn fixture(hex: &str) -> Result, Box> { - decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) -} diff --git a/tests/store_migration_phase.rs b/tests/store_migration_phase.rs deleted file mode 100644 index 978cad1a..00000000 --- a/tests/store_migration_phase.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! Ordered version-2 store-migration durability phase laws. - -use keep::StoreMigrationPhase; - -const EXPECTED: [(StoreMigrationPhase, &str); 21] = [ - ( - StoreMigrationPhase::WriteIntentStage, - "migration-intent stage write", - ), - ( - StoreMigrationPhase::SynchronizeIntentStage, - "migration-intent stage synchronization", - ), - ( - StoreMigrationPhase::LinkIntent, - "migration-intent canonical link", - ), - ( - StoreMigrationPhase::SynchronizeRootAfterIntent, - "store-root synchronization after intent link", - ), - ( - StoreMigrationPhase::RemoveIntentStage, - "migration-intent stage removal", - ), - ( - StoreMigrationPhase::SynchronizeRootAfterIntentCleanup, - "store-root synchronization after intent cleanup", - ), - ( - StoreMigrationPhase::AdmitReaderFence, - "persistent reader-fence admission", - ), - ( - StoreMigrationPhase::AdmitNamespacePrefix, - "canonical namespace-prefix admission", - ), - ( - StoreMigrationPhase::SynchronizeRootAfterNamespace, - "store-root synchronization after namespace admission", - ), - ( - StoreMigrationPhase::WriteMarkerStage, - "format-marker stage write", - ), - ( - StoreMigrationPhase::SynchronizeMarkerStage, - "format-marker stage synchronization", - ), - ( - StoreMigrationPhase::LinkMarker, - "format-marker canonical link", - ), - ( - StoreMigrationPhase::SynchronizeRootAfterMarker, - "store-root synchronization after marker link", - ), - ( - StoreMigrationPhase::RemoveMarkerStage, - "format-marker stage removal", - ), - ( - StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup, - "store-root synchronization after marker cleanup", - ), - ( - StoreMigrationPhase::WriteReceiptStage, - "migration-receipt stage write", - ), - ( - StoreMigrationPhase::SynchronizeReceiptStage, - "migration-receipt stage synchronization", - ), - ( - StoreMigrationPhase::LinkReceipt, - "migration-receipt canonical link", - ), - ( - StoreMigrationPhase::SynchronizeRootAfterReceipt, - "store-root synchronization after receipt link", - ), - ( - StoreMigrationPhase::RemoveReceiptStage, - "migration-receipt stage removal", - ), - ( - StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup, - "final store-root synchronization", - ), -]; - -#[test] -fn migration_phases_are_complete_ordered_and_stably_named() { - assert_eq!( - StoreMigrationPhase::ALL, - EXPECTED.map(|(phase, _name)| phase) - ); - for (phase, name) in EXPECTED { - assert_eq!(phase.to_string(), name); - } -} diff --git a/tests/store_migration_receipt.rs b/tests/store_migration_receipt.rs deleted file mode 100644 index bbacfd2a..00000000 --- a/tests/store_migration_receipt.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! Canonical version-2 store-migration receipt laws. - -#[path = "store_migration_receipt/binding_laws.rs"] -mod binding_laws; -#[path = "store_migration_receipt/fixture.rs"] -mod fixture; -#[path = "store_migration_receipt/harness.rs"] -mod harness; -mod support; - -use fixture::{ - DISPOSITION_DIGEST, INITIAL_GC_DIGEST, INITIAL_RETENTION_DIGEST, intent_bytes, marker_bytes, - receipt_bytes, -}; -use harness::{assert_fixed_refusal, assert_receipt_refusal, mutated_array}; -use keep::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, - StoreMigrationReceiptDecodeError, -}; - -#[test] -fn receipt_admits_every_frozen_completion_coordinate() -> Result<(), Box> { - let intent_bytes = intent_bytes()?; - let marker_bytes = marker_bytes()?; - let receipt_bytes = receipt_bytes()?; - let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; - let marker = AdmittedStoreFormatMarker::decode(&marker_bytes)?; - let receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &intent, &marker)?; - - assert_eq!(receipt.encoded(), receipt_bytes); - assert_eq!(receipt.intent_digest(), intent.digest()); - assert_eq!(receipt.store_identifier(), intent.store_identifier()); - assert_eq!(receipt.format_marker_digest(), marker.digest()); - assert_eq!( - receipt.initial_retention_state_digest().as_bytes(), - &INITIAL_RETENTION_DIGEST - ); - assert_eq!( - receipt.initial_gc_state_digest().as_bytes(), - &INITIAL_GC_DIGEST - ); - assert_eq!( - receipt.empty_disposition_set_digest().as_bytes(), - &DISPOSITION_DIGEST - ); - assert_eq!(receipt.synchronization_mask().bits(), 0x03ff); - Ok(()) -} - -#[test] -fn receipt_framing_has_exact_first_refusals() -> Result<(), Box> { - let bytes = receipt_bytes()?; - let mut truncated = bytes.clone(); - assert!(truncated.pop().is_some()); - assert_receipt_refusal( - &truncated, - StoreMigrationReceiptDecodeError::WrongLength { - expected: 256, - observed: 255, - }, - )?; - let mut extended = bytes.clone(); - extended.push(0); - assert_receipt_refusal( - &extended, - StoreMigrationReceiptDecodeError::WrongLength { - expected: 256, - observed: 257, - }, - )?; - assert_fixed_refusal( - 0, - StoreMigrationReceiptDecodeError::InvalidMagic { - observed: mutated_array(&bytes, 0, 0)?, - }, - )?; - assert_fixed_refusal( - 17, - StoreMigrationReceiptDecodeError::UnsupportedVersion { - expected: 2, - observed: 3, - }, - )?; - assert_fixed_refusal( - 19, - StoreMigrationReceiptDecodeError::InvalidRecordLength { - expected: 256, - observed: 257, - }, - )?; - assert_fixed_refusal( - 23, - StoreMigrationReceiptDecodeError::UnsupportedFlags { observed: 1 }, - )?; - Ok(()) -} diff --git a/tests/store_migration_receipt/binding_laws.rs b/tests/store_migration_receipt/binding_laws.rs deleted file mode 100644 index c7a8cc03..00000000 --- a/tests/store_migration_receipt/binding_laws.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! This module owns migration-receipt integrity and binding laws. - -use keep::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, - StoreMigrationReceiptDecodeError, -}; - -use super::fixture::{ - DISPOSITION_DIGEST, INITIAL_GC_DIGEST, INITIAL_RETENTION_DIGEST, intent_bytes, marker_bytes, - receipt_bytes, -}; -use super::harness::{ - assert_semantic_refusal, decode_receipt, digest_intent, flip_byte, mutated_array, - refresh_checksum, -}; - -#[test] -fn receipt_integrity_and_binding_have_exact_precedence() -> Result<(), Box> { - let mut checksum = receipt_bytes()?; - flip_byte(&mut checksum, 24)?; - assert!(matches!( - decode_receipt(&checksum)?, - Err(StoreMigrationReceiptDecodeError::ChecksumMismatch { .. }) - )); - - let intent = intent_bytes()?; - let marker = marker_bytes()?; - assert_semantic_refusal( - 24, - StoreMigrationReceiptDecodeError::IntentDigestMismatch { - expected: digest_intent(&intent), - observed: mutated_array(&receipt_bytes()?, 24, 0)?, - }, - )?; - assert_semantic_refusal( - 56, - StoreMigrationReceiptDecodeError::StoreIdentifierMismatch { - expected: *AdmittedStoreMigrationIntent::decode(&intent)? - .store_identifier() - .as_bytes(), - observed: mutated_array(&receipt_bytes()?, 56, 0)?, - }, - )?; - assert_semantic_refusal( - 88, - StoreMigrationReceiptDecodeError::FormatMarkerDigestMismatch { - expected: *AdmittedStoreFormatMarker::decode(&marker)? - .digest() - .as_bytes(), - observed: mutated_array(&receipt_bytes()?, 88, 0)?, - }, - )?; - assert_semantic_refusal( - 120, - StoreMigrationReceiptDecodeError::InitialRetentionStateDigestMismatch { - expected: INITIAL_RETENTION_DIGEST, - observed: mutated_array(&receipt_bytes()?, 120, 0)?, - }, - )?; - assert_semantic_refusal( - 152, - StoreMigrationReceiptDecodeError::InitialGcStateDigestMismatch { - expected: INITIAL_GC_DIGEST, - observed: mutated_array(&receipt_bytes()?, 152, 0)?, - }, - )?; - assert_semantic_refusal( - 184, - StoreMigrationReceiptDecodeError::EmptyDispositionSetDigestMismatch { - expected: DISPOSITION_DIGEST, - observed: mutated_array(&receipt_bytes()?, 184, 0)?, - }, - )?; - assert_semantic_refusal( - 221, - StoreMigrationReceiptDecodeError::UnsupportedSynchronizationBits { - supported: 0x03ff, - observed: 0x0001_03ff, - }, - )?; - assert_semantic_refusal( - 222, - StoreMigrationReceiptDecodeError::IncompleteSynchronizationMask { - required: 0x03ff, - observed: 0x02ff, - }, - )?; - - let mut alternative_intent = intent; - flip_byte(&mut alternative_intent, 159)?; - refresh_checksum( - &mut alternative_intent, - 224, - b"keep.store-migration-intent-checksum/v2\0", - )?; - let alternative = AdmittedStoreMigrationIntent::decode(&alternative_intent)?; - let receipt = receipt_bytes()?; - let marker = AdmittedStoreFormatMarker::decode(&marker)?; - assert!(matches!( - AdmittedStoreMigrationReceipt::decode(&receipt, &alternative, &marker), - Err(StoreMigrationReceiptDecodeError::IntentDigestMismatch { .. }) - )); - Ok(()) -} diff --git a/tests/store_migration_receipt/fixture.rs b/tests/store_migration_receipt/fixture.rs deleted file mode 100644 index 10a6ff27..00000000 --- a/tests/store_migration_receipt/fixture.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! This module owns frozen migration-receipt fixture values. -#![allow( - clippy::redundant_pub_crate, - reason = "the parent integration-test module consumes this private fixture" -)] - -use std::io; - -use super::support; - -const RECEIPT: &str = include_str!("../../conformance/segment-store/v2/migration-receipt.hex"); -const INTENT: &str = include_str!("../../conformance/segment-store/v2/migration-intent.hex"); -const MARKER: &str = include_str!("../../conformance/segment-store/v2/format-marker.hex"); - -pub(super) const INITIAL_RETENTION_DIGEST: [u8; 32] = [ - 0xd5, 0x2f, 0x1f, 0x02, 0x2e, 0xdb, 0x1d, 0xe7, 0xb8, 0x40, 0xc5, 0xbf, 0x8f, 0xb5, 0x5d, 0xe7, - 0x93, 0x2c, 0xa6, 0x93, 0x70, 0xae, 0x85, 0xe2, 0xbe, 0xe4, 0x17, 0x91, 0x43, 0x79, 0x2b, 0xc3, -]; -pub(super) const INITIAL_GC_DIGEST: [u8; 32] = [ - 0xba, 0x0e, 0xa2, 0x00, 0xa5, 0xb0, 0x67, 0x41, 0x56, 0x4c, 0x43, 0xa7, 0x9a, 0x91, 0x94, 0x5b, - 0xef, 0x0b, 0x0f, 0xac, 0x51, 0xc9, 0x60, 0xea, 0x4f, 0x82, 0x07, 0x09, 0x4f, 0x3e, 0x1e, 0x31, -]; -pub(super) const DISPOSITION_DIGEST: [u8; 32] = [ - 0xa8, 0x02, 0x59, 0xfc, 0xd1, 0x23, 0x72, 0x03, 0xea, 0x6c, 0x6c, 0xc5, 0x06, 0x55, 0x14, 0xab, - 0xde, 0xb0, 0x1d, 0xa6, 0x03, 0xc3, 0x19, 0x4b, 0x09, 0x6a, 0x04, 0x5c, 0xf6, 0x94, 0xc9, 0x5a, -]; - -pub(super) fn receipt_bytes() -> Result, io::Error> { - support::decode_hex(RECEIPT.trim_end()) -} - -pub(super) fn intent_bytes() -> Result, io::Error> { - support::decode_hex(INTENT.trim_end()) -} - -pub(super) fn marker_bytes() -> Result, io::Error> { - support::decode_hex(MARKER.trim_end()) -} diff --git a/tests/store_migration_receipt/harness.rs b/tests/store_migration_receipt/harness.rs deleted file mode 100644 index 296b8631..00000000 --- a/tests/store_migration_receipt/harness.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! This module owns migration-receipt mutation and admission test mechanics. -#![allow( - clippy::redundant_pub_crate, - reason = "sibling private test modules consume this harness" -)] - -use std::io; - -use keep::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, - StoreMigrationReceiptDecodeError, -}; - -use super::fixture::{intent_bytes, marker_bytes, receipt_bytes}; - -pub(super) fn assert_fixed_refusal( - offset: usize, - expected: StoreMigrationReceiptDecodeError, -) -> Result<(), Box> { - let mut bytes = receipt_bytes()?; - flip_byte(&mut bytes, offset)?; - assert_receipt_refusal(&bytes, expected) -} - -pub(super) fn assert_semantic_refusal( - offset: usize, - expected: StoreMigrationReceiptDecodeError, -) -> Result<(), Box> { - let mut bytes = receipt_bytes()?; - flip_byte(&mut bytes, offset)?; - refresh_checksum( - &mut bytes, - 224, - b"keep.store-migration-receipt-checksum/v2\0", - )?; - assert_receipt_refusal(&bytes, expected) -} - -pub(super) fn assert_receipt_refusal( - bytes: &[u8], - expected: StoreMigrationReceiptDecodeError, -) -> Result<(), Box> { - assert_eq!(decode_receipt(bytes)?, Err(expected)); - Ok(()) -} - -pub(super) fn decode_receipt( - bytes: &[u8], -) -> Result< - Result, StoreMigrationReceiptDecodeError>, - Box, -> { - let intent_bytes = intent_bytes()?; - let marker_bytes = marker_bytes()?; - let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; - let marker = AdmittedStoreFormatMarker::decode(&marker_bytes)?; - Ok(AdmittedStoreMigrationReceipt::decode( - bytes, &intent, &marker, - )) -} - -pub(super) fn mutated_array( - bytes: &[u8], - offset: usize, - relative: usize, -) -> Result<[u8; WIDTH], io::Error> { - let end = offset - .checked_add(WIDTH) - .ok_or_else(|| io::Error::other("receipt field offset overflow"))?; - let field = bytes - .get(offset..end) - .ok_or_else(|| io::Error::other("receipt lacks fixed field"))?; - let mut observed = <[u8; WIDTH]>::try_from(field) - .map_err(|_| io::Error::other("receipt field width mismatch"))?; - flip_byte(&mut observed, relative)?; - Ok(observed) -} - -pub(super) fn flip_byte(bytes: &mut [u8], offset: usize) -> Result<(), io::Error> { - let byte = bytes - .get_mut(offset) - .ok_or_else(|| io::Error::other("receipt mutation offset is out of bounds"))?; - *byte ^= 1; - Ok(()) -} - -pub(super) fn refresh_checksum( - bytes: &mut [u8], - offset: usize, - domain: &[u8], -) -> Result<(), io::Error> { - let (preimage, checksum) = bytes - .split_at_mut_checked(offset) - .ok_or_else(|| io::Error::other("record lacks checksum boundary"))?; - let mut hasher = blake3::Hasher::new(); - hasher.update(domain); - hasher.update(preimage); - checksum.copy_from_slice(hasher.finalize().as_bytes()); - Ok(()) -} - -pub(super) fn digest_intent(bytes: &[u8]) -> [u8; 32] { - let mut hasher = blake3::Hasher::new(); - hasher.update(b"keep.store-migration-intent/v2\0"); - hasher.update(bytes); - *hasher.finalize().as_bytes() -} diff --git a/tests/store_migration_receipt_encoding.rs b/tests/store_migration_receipt_encoding.rs deleted file mode 100644 index c5f87935..00000000 --- a/tests/store_migration_receipt_encoding.rs +++ /dev/null @@ -1,46 +0,0 @@ -//! Canonical version-2 store-migration receipt encoding laws. - -#[path = "store_migration_receipt/fixture.rs"] -mod fixture; -mod support; - -use std::error::Error; - -use keep::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, - CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, -}; - -#[test] -fn canonical_completion_receipt_reproduces_every_frozen_field() -> Result<(), Box> { - let expected = fixture::receipt_bytes()?; - let intent_bytes = fixture::intent_bytes()?; - let admitted_intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; - let intent = CanonicalStoreMigrationIntent::from_admitted(&admitted_intent); - let marker = CanonicalStoreFormatMarker::version_two(); - assert_eq!(marker.encoded(), fixture::marker_bytes()?); - - let canonical = CanonicalStoreMigrationReceipt::from_canonical(&intent, &marker); - assert_eq!(canonical.encoded(), expected); - - let admitted_marker = AdmittedStoreFormatMarker::decode(marker.encoded())?; - let admitted = AdmittedStoreMigrationReceipt::decode( - canonical.encoded(), - &admitted_intent, - &admitted_marker, - )?; - assert_eq!( - admitted.initial_retention_state_digest().as_bytes(), - &fixture::INITIAL_RETENTION_DIGEST - ); - assert_eq!( - admitted.initial_gc_state_digest().as_bytes(), - &fixture::INITIAL_GC_DIGEST - ); - assert_eq!( - admitted.empty_disposition_set_digest().as_bytes(), - &fixture::DISPOSITION_DIGEST - ); - assert_eq!(admitted.synchronization_mask().bits(), 0x03ff); - Ok(()) -} diff --git a/tests/store_migration_recovery.rs b/tests/store_migration_recovery.rs deleted file mode 100644 index ae94cde8..00000000 --- a/tests/store_migration_recovery.rs +++ /dev/null @@ -1,424 +0,0 @@ -//! Storage-independent migration recovery planning laws. -//! -//! One law per row of the recovery table in `migration-recovery.md`, plus -//! one per ambiguity rule, each over the frozen version-2 records. - -mod support; - -use std::error::Error; - -use keep::{ - AdmittedStoreMigrationIntent, StoreMigrationEffect, StoreMigrationFixedStage as Stage, - StoreMigrationPhase as Phase, StoreMigrationRecoveryAmbiguity as Ambiguity, - StoreMigrationRecoveryPlan as Plan, StoreMigrationResidue, plan_store_migration_recovery, -}; - -use crate::support::{domain_hash, patch}; - -const INTENT: &str = include_str!("../conformance/segment-store/v2/migration-intent.hex"); -const MARKER: &str = include_str!("../conformance/segment-store/v2/format-marker.hex"); -const RECEIPT: &str = include_str!("../conformance/segment-store/v2/migration-receipt.hex"); -const INTENT_CHECKSUM_OFFSET: usize = 224; -const ROOT_DEVICE_OFFSET: usize = 136; -const ROOT_MOUNT_OFFSET: usize = 144; - -struct Records { - intent: Vec, - marker: Vec, - receipt: Vec, -} - -#[test] -fn no_artifact_admits_version_one() -> Result<(), Box> { - let records = records()?; - let expected = AdmittedStoreMigrationIntent::decode(&records.intent)?; - assert_eq!( - plan_store_migration_recovery(&expected, &StoreMigrationResidue::VERSION_ONE)?, - Plan::VersionOne - ); - Ok(()) -} - -#[test] -fn an_intent_stage_alone_resumes_when_exact_and_is_discarded_when_incomplete() --> Result<(), Box> { - let records = records()?; - let expected = AdmittedStoreMigrationIntent::decode(&records.intent)?; - - let exact = StoreMigrationResidue { - intent_stage: Some(records.intent.clone()), - ..StoreMigrationResidue::VERSION_ONE - }; - assert_eq!( - plan_store_migration_recovery(&expected, &exact)?, - Plan::Resume { - resume: Phase::SynchronizeIntentStage - } - ); - - let mut truncated = records.intent.clone(); - truncated.truncate(100); - let incomplete = StoreMigrationResidue { - intent_stage: Some(truncated), - ..StoreMigrationResidue::VERSION_ONE - }; - assert_eq!( - plan_store_migration_recovery(&expected, &incomplete)?, - Plan::DiscardStage { - stage: Stage::Intent, - resume: Phase::WriteIntentStage, - } - ); - - let mut overlong = records.intent.clone(); - overlong.push(0); - let overlong = StoreMigrationResidue { - intent_stage: Some(overlong), - ..StoreMigrationResidue::VERSION_ONE - }; - assert!(matches!( - plan_store_migration_recovery(&expected, &overlong), - Err(Ambiguity::StageOverlong { - stage: Stage::Intent, - observed: 257, - }) - )); - - let mut corrupt = records.intent.clone(); - let last = corrupt.last_mut().ok_or("intent is empty")?; - *last ^= 1; - let corrupt = StoreMigrationResidue { - intent_stage: Some(corrupt), - ..StoreMigrationResidue::VERSION_ONE - }; - let error = plan_store_migration_recovery(&expected, &corrupt) - .err() - .ok_or("corrupt stage admitted")?; - assert!(error.source().and_then(|source| source.source()).is_some()); - assert!(matches!( - error, - Ambiguity::StageUndecodable { - stage: Stage::Intent, - source: keep::StoreMigrationStageDecodeError::Intent { - source: keep::StoreMigrationIntentDecodeError::ChecksumMismatch { .. } - } - } - )); - Ok(()) -} - -#[test] -fn a_durable_intent_resumes_after_its_cleanup_and_binds_restart_stable_coordinates() --> Result<(), Box> { - let records = records()?; - let expected = AdmittedStoreMigrationIntent::decode(&records.intent)?; - - let durable = StoreMigrationResidue { - intent: Some(records.intent.clone()), - ..StoreMigrationResidue::VERSION_ONE - }; - assert_eq!( - plan_store_migration_recovery(&expected, &durable)?, - Plan::Resume { - resume: Phase::SynchronizeRootAfterIntentCleanup - } - ); - - let with_stage = StoreMigrationResidue { - intent_stage: Some(records.intent.clone()), - ..durable.clone() - }; - assert_eq!( - plan_store_migration_recovery(&expected, &with_stage)?, - Plan::Resume { - resume: Phase::SynchronizeRootAfterIntent - } - ); - - let mut remounted = records.intent.clone(); - patch(&mut remounted, ROOT_MOUNT_OFFSET, &9_u64.to_be_bytes())?; - reseal_intent(&mut remounted)?; - let remounted = AdmittedStoreMigrationIntent::decode(&remounted)?; - assert_eq!( - plan_store_migration_recovery(&remounted, &durable)?, - Plan::Resume { - resume: Phase::SynchronizeRootAfterIntentCleanup - }, - "a rebooted root's new mount id must not reject the store's own intent" - ); - - let mut moved = records.intent.clone(); - patch(&mut moved, ROOT_DEVICE_OFFSET, &9_u64.to_be_bytes())?; - reseal_intent(&mut moved)?; - let moved = AdmittedStoreMigrationIntent::decode(&moved)?; - assert!(matches!( - plan_store_migration_recovery(&moved, &durable), - Err(Ambiguity::IntentDiffers) - )); - - let mut corrupt = durable; - if let Some(bytes) = corrupt.intent.as_mut() - && let Some(last) = bytes.last_mut() - { - *last ^= 1; - } - assert!(matches!( - plan_store_migration_recovery(&expected, &corrupt), - Err(Ambiguity::IntentUndecodable { - source: keep::StoreMigrationIntentDecodeError::ChecksumMismatch { .. } - }) - )); - Ok(()) -} - -#[test] -fn every_effect_needs_a_durable_intent_first() -> Result<(), Box> { - let records = records()?; - let expected = AdmittedStoreMigrationIntent::decode(&records.intent)?; - for (residue, effect) in [ - ( - StoreMigrationResidue { - reader_fence: true, - ..StoreMigrationResidue::VERSION_ONE - }, - StoreMigrationEffect::Namespace, - ), - ( - StoreMigrationResidue { - marker: Some(records.marker.clone()), - ..StoreMigrationResidue::VERSION_ONE - }, - StoreMigrationEffect::Marker, - ), - ( - StoreMigrationResidue { - receipt_stage: Some(records.receipt.clone()), - ..StoreMigrationResidue::VERSION_ONE - }, - StoreMigrationEffect::Receipt, - ), - ] { - assert!(matches!( - plan_store_migration_recovery(&expected, &residue), - Err(Ambiguity::EffectBeforeIntent { effect: observed }) if observed == effect - )); - } - Ok(()) -} - -#[test] -fn the_namespace_prefix_resumes_in_order_and_refuses_gaps() -> Result<(), Box> { - let records = records()?; - let expected = AdmittedStoreMigrationIntent::decode(&records.intent)?; - let durable = StoreMigrationResidue { - intent: Some(records.intent.clone()), - ..StoreMigrationResidue::VERSION_ONE - }; - - let partial = StoreMigrationResidue { - reader_fence: true, - namespace_prefix: [true, true, false, false, false, false], - ..durable.clone() - }; - assert_eq!( - plan_store_migration_recovery(&expected, &partial)?, - Plan::Resume { - resume: Phase::AdmitNamespacePrefix - } - ); - - let complete = StoreMigrationResidue { - reader_fence: true, - namespace_prefix: [true; 6], - ..durable.clone() - }; - assert_eq!( - plan_store_migration_recovery(&expected, &complete)?, - Plan::Resume { - resume: Phase::SynchronizeRootAfterNamespace - } - ); - - let gap = StoreMigrationResidue { - reader_fence: true, - namespace_prefix: [true, false, true, false, false, false], - ..durable.clone() - }; - assert!(matches!( - plan_store_migration_recovery(&expected, &gap), - Err(Ambiguity::NamespaceOutOfOrder { - absent: 2, - present: 3, - }) - )); - - let no_fence = StoreMigrationResidue { - namespace_prefix: [true, false, false, false, false, false], - ..durable - }; - assert!(matches!( - plan_store_migration_recovery(&expected, &no_fence), - Err(Ambiguity::NamespaceOutOfOrder { - absent: 0, - present: 1, - }) - )); - - let early_marker = StoreMigrationResidue { - marker_stage: Some(records.marker.clone()), - ..partial - }; - assert!(matches!( - plan_store_migration_recovery(&expected, &early_marker), - Err(Ambiguity::MarkerBeforeNamespace) - )); - Ok(()) -} - -#[test] -fn the_marker_stage_and_marker_resume_at_their_first_unproven_phase() -> Result<(), Box> -{ - let records = records()?; - let expected = AdmittedStoreMigrationIntent::decode(&records.intent)?; - let namespace = StoreMigrationResidue { - intent: Some(records.intent.clone()), - reader_fence: true, - namespace_prefix: [true; 6], - ..StoreMigrationResidue::VERSION_ONE - }; - - let staged = StoreMigrationResidue { - marker_stage: Some(records.marker.clone()), - ..namespace.clone() - }; - assert_eq!( - plan_store_migration_recovery(&expected, &staged)?, - Plan::Resume { - resume: Phase::SynchronizeMarkerStage - } - ); - - let mut short = records.marker.clone(); - short.truncate(10); - let incomplete = StoreMigrationResidue { - marker_stage: Some(short), - ..namespace.clone() - }; - assert_eq!( - plan_store_migration_recovery(&expected, &incomplete)?, - Plan::DiscardStage { - stage: Stage::Marker, - resume: Phase::WriteMarkerStage, - } - ); - - let linked = StoreMigrationResidue { - marker: Some(records.marker.clone()), - marker_stage: Some(records.marker.clone()), - ..namespace.clone() - }; - assert_eq!( - plan_store_migration_recovery(&expected, &linked)?, - Plan::Resume { - resume: Phase::SynchronizeRootAfterMarker - } - ); - - let published = StoreMigrationResidue { - marker: Some(records.marker.clone()), - ..namespace.clone() - }; - assert_eq!( - plan_store_migration_recovery(&expected, &published)?, - Plan::Resume { - resume: Phase::SynchronizeRootAfterMarkerCleanup - } - ); - - let early_receipt = StoreMigrationResidue { - receipt: Some(records.receipt.clone()), - ..namespace - }; - assert!(matches!( - plan_store_migration_recovery(&expected, &early_receipt), - Err(Ambiguity::ReceiptBeforeMarker) - )); - Ok(()) -} - -#[test] -fn the_receipt_completes_the_migration_only_when_exact_and_alone() -> Result<(), Box> { - let records = records()?; - let expected = AdmittedStoreMigrationIntent::decode(&records.intent)?; - let marked = StoreMigrationResidue { - intent: Some(records.intent.clone()), - reader_fence: true, - namespace_prefix: [true; 6], - marker: Some(records.marker.clone()), - ..StoreMigrationResidue::VERSION_ONE - }; - - let staged = StoreMigrationResidue { - receipt_stage: Some(records.receipt.clone()), - ..marked.clone() - }; - assert_eq!( - plan_store_migration_recovery(&expected, &staged)?, - Plan::Resume { - resume: Phase::SynchronizeReceiptStage - } - ); - - let linked = StoreMigrationResidue { - receipt: Some(records.receipt.clone()), - receipt_stage: Some(records.receipt.clone()), - ..marked.clone() - }; - assert_eq!( - plan_store_migration_recovery(&expected, &linked)?, - Plan::Resume { - resume: Phase::SynchronizeRootAfterReceipt - } - ); - - let complete = StoreMigrationResidue { - receipt: Some(records.receipt.clone()), - ..marked.clone() - }; - assert_eq!( - plan_store_migration_recovery(&expected, &complete)?, - Plan::Complete - ); - - let mut conflicting = records.receipt.clone(); - let last = conflicting.last_mut().ok_or("receipt is empty")?; - *last ^= 1; - let conflicting = StoreMigrationResidue { - receipt: Some(conflicting), - ..marked - }; - assert!(matches!( - plan_store_migration_recovery(&expected, &conflicting), - Err(Ambiguity::ReceiptUndecodable { - source: keep::StoreMigrationReceiptDecodeError::ChecksumMismatch { .. } - }) - )); - Ok(()) -} - -fn records() -> Result> { - Ok(Records { - intent: support::decode_hex(INTENT.trim_end())?, - marker: support::decode_hex(MARKER.trim_end())?, - receipt: support::decode_hex(RECEIPT.trim_end())?, - }) -} - -fn reseal_intent(bytes: &mut [u8]) -> Result<(), Box> { - let preimage = bytes - .get(..INTENT_CHECKSUM_OFFSET) - .ok_or("intent lacks its checksum preimage")?; - let checksum = domain_hash(b"keep.store-migration-intent-checksum/v2\0", preimage); - patch(bytes, INTENT_CHECKSUM_OFFSET, &checksum)?; - Ok(()) -} diff --git a/tests/store_migration_recovery_order.rs b/tests/store_migration_recovery_order.rs deleted file mode 100644 index 43d0ba37..00000000 --- a/tests/store_migration_recovery_order.rs +++ /dev/null @@ -1,121 +0,0 @@ -//! This module owns refusal of migration stages surviving later effects. - -mod support; - -use keep::{ - AdmittedStoreMigrationIntent, MIGRATION_NAMESPACE_PREFIX, StoreMigrationEffect, - StoreMigrationFixedStage, StoreMigrationRecoveryAmbiguity, StoreMigrationResidue, - plan_store_migration_recovery, -}; -use std::error::Error; - -#[test] -fn an_intent_stage_cannot_survive_reader_fence_creation() -> Result<(), Box> { - let intent = support::decode_hex( - include_str!("../conformance/segment-store/v2/migration-intent.hex").trim(), - )?; - let expected = AdmittedStoreMigrationIntent::decode(&intent)?; - let residue = StoreMigrationResidue { - intent: Some(intent.clone()), - intent_stage: Some(intent.clone()), - reader_fence: true, - ..StoreMigrationResidue::VERSION_ONE - }; - assert!(matches!( - plan_store_migration_recovery(&expected, &residue), - Err(StoreMigrationRecoveryAmbiguity::StageAfterEffect { - stage: StoreMigrationFixedStage::Intent, - effect: StoreMigrationEffect::Namespace - }) - )); - Ok(()) -} - -#[test] -fn a_marker_stage_cannot_survive_receipt_publication() -> Result<(), Box> { - let intent = support::decode_hex( - include_str!("../conformance/segment-store/v2/migration-intent.hex").trim(), - )?; - let marker = support::decode_hex( - include_str!("../conformance/segment-store/v2/format-marker.hex").trim(), - )?; - let receipt = support::decode_hex( - include_str!("../conformance/segment-store/v2/migration-receipt.hex").trim(), - )?; - let expected = AdmittedStoreMigrationIntent::decode(&intent)?; - let residue = StoreMigrationResidue { - intent: Some(intent.clone()), - reader_fence: true, - namespace_prefix: [true; 6], - marker: Some(marker.clone()), - marker_stage: Some(marker), - receipt: Some(receipt), - ..StoreMigrationResidue::VERSION_ONE - }; - assert!(matches!( - plan_store_migration_recovery(&expected, &residue), - Err(StoreMigrationRecoveryAmbiguity::StageAfterEffect { - stage: StoreMigrationFixedStage::Marker, - effect: StoreMigrationEffect::Receipt - }) - )); - Ok(()) -} - -#[test] -fn a_receipt_only_later_effect_is_reported_as_receipt() -> Result<(), Box> { - let intent = support::decode_hex( - include_str!("../conformance/segment-store/v2/migration-intent.hex").trim(), - )?; - let receipt = support::decode_hex( - include_str!("../conformance/segment-store/v2/migration-receipt.hex").trim(), - )?; - let expected = AdmittedStoreMigrationIntent::decode(&intent)?; - let residue = StoreMigrationResidue { - intent: Some(intent.clone()), - intent_stage: Some(intent.clone()), - receipt: Some(receipt), - ..StoreMigrationResidue::VERSION_ONE - }; - assert!(matches!( - plan_store_migration_recovery(&expected, &residue), - Err(StoreMigrationRecoveryAmbiguity::StageAfterEffect { - stage: StoreMigrationFixedStage::Intent, - effect: StoreMigrationEffect::Receipt, - }) - )); - Ok(()) -} - -#[test] -fn a_receipt_before_the_marker_reports_the_missing_marker_at_every_partial_namespace() --> Result<(), Box> { - let intent = support::decode_hex( - include_str!("../conformance/segment-store/v2/migration-intent.hex").trim(), - )?; - let receipt = support::decode_hex( - include_str!("../conformance/segment-store/v2/migration-receipt.hex").trim(), - )?; - let expected = AdmittedStoreMigrationIntent::decode(&intent)?; - for extent in 0..=MIGRATION_NAMESPACE_PREFIX.len() { - for (receipt_stage, canonical) in - [(Some(receipt.clone()), None), (None, Some(receipt.clone()))] - { - let residue = StoreMigrationResidue { - intent: Some(intent.clone()), - reader_fence: extent > 0, - namespace_prefix: std::array::from_fn(|position: usize| { - position.checked_add(1).is_some_and(|next| next < extent) - }), - receipt_stage, - receipt: canonical, - ..StoreMigrationResidue::VERSION_ONE - }; - assert!(matches!( - plan_store_migration_recovery(&expected, &residue), - Err(StoreMigrationRecoveryAmbiguity::ReceiptBeforeMarker) - )); - } - } - Ok(()) -} diff --git a/tests/store_migration_storage.rs b/tests/store_migration_storage.rs deleted file mode 100644 index a1b31429..00000000 --- a/tests/store_migration_storage.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! Version-2 store-migration storage capability laws. - -#[path = "store_migration_storage/recording_storage.rs"] -pub mod recording_storage; -mod support; - -use std::error::Error; -use std::io; - -use keep::{ - AdmittedStoreMigrationIntent, CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, - CanonicalStoreMigrationReceipt, StoreMigrationPhase, StoreMigrationStorage, -}; -use recording_storage::RecordingStorage; - -const INTENT: &str = include_str!("../conformance/segment-store/v2/migration-intent.hex"); - -#[test] -fn storage_port_names_every_migration_phase() -> Result<(), Box> { - let intent_bytes = support::decode_hex(INTENT.trim_end())?; - let admitted = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; - let intent = CanonicalStoreMigrationIntent::from_admitted(&admitted); - let marker = CanonicalStoreFormatMarker::version_two(); - let receipt = CanonicalStoreMigrationReceipt::from_canonical(&intent, &marker); - let mut storage = RecordingStorage::default(); - - exercise_storage(&mut storage, &intent, &marker, &receipt)?; - - assert_eq!(storage.verification_count(), 1); - assert_eq!(storage.observed(), StoreMigrationPhase::ALL); - Ok(()) -} - -fn exercise_storage( - storage: &mut impl StoreMigrationStorage, - intent: &CanonicalStoreMigrationIntent, - marker: &CanonicalStoreFormatMarker, - receipt: &CanonicalStoreMigrationReceipt, -) -> io::Result<()> { - storage.verify_current(intent)?; - exercise_intent(storage, intent)?; - exercise_namespace(storage)?; - exercise_marker(storage, marker)?; - exercise_receipt(storage, receipt) -} - -fn exercise_intent( - storage: &mut impl StoreMigrationStorage, - intent: &CanonicalStoreMigrationIntent, -) -> io::Result<()> { - storage.write_intent_stage(intent)?; - storage.synchronize_intent_stage()?; - storage.link_intent(intent)?; - storage.synchronize_root_after_intent()?; - storage.remove_intent_stage()?; - storage.synchronize_root_after_intent_cleanup() -} - -fn exercise_namespace(storage: &mut impl StoreMigrationStorage) -> io::Result<()> { - storage.admit_reader_fence()?; - storage.admit_namespace_prefix()?; - storage.synchronize_root_after_namespace() -} - -fn exercise_marker( - storage: &mut impl StoreMigrationStorage, - marker: &CanonicalStoreFormatMarker, -) -> io::Result<()> { - storage.write_marker_stage(marker)?; - storage.synchronize_marker_stage()?; - storage.link_marker(marker)?; - storage.synchronize_root_after_marker()?; - storage.remove_marker_stage()?; - storage.synchronize_root_after_marker_cleanup() -} - -fn exercise_receipt( - storage: &mut impl StoreMigrationStorage, - receipt: &CanonicalStoreMigrationReceipt, -) -> io::Result<()> { - storage.write_receipt_stage(receipt)?; - storage.synchronize_receipt_stage()?; - storage.link_receipt(receipt)?; - storage.synchronize_root_after_receipt()?; - storage.remove_receipt_stage()?; - storage.synchronize_root_after_receipt_cleanup() -} diff --git a/tests/store_migration_storage/recording_storage.rs b/tests/store_migration_storage/recording_storage.rs deleted file mode 100644 index d3dbf25a..00000000 --- a/tests/store_migration_storage/recording_storage.rs +++ /dev/null @@ -1,149 +0,0 @@ -//! This module owns the store-migration storage test double. - -use std::io; - -use keep::{ - CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, - StoreMigrationPhase, StoreMigrationStorage, -}; - -#[derive(Default)] -/// Storage port that records every attempted migration phase. -pub struct RecordingStorage { - observed: Vec, - verification_count: usize, - fail_at: Option, - verification_failure: Option, -} - -impl RecordingStorage { - /// Creates storage that refuses at one exact migration phase. - pub fn failing_at(phase: StoreMigrationPhase) -> Self { - Self { - fail_at: Some(phase), - ..Self::default() - } - } - - /// Creates storage that refuses current-state verification. - pub fn verification_failure() -> Self { - Self { - verification_failure: Some(io::ErrorKind::PermissionDenied), - ..Self::default() - } - } - - /// Returns attempted migration phases in call order. - pub fn observed(&self) -> &[StoreMigrationPhase] { - &self.observed - } - - /// Returns the number of current-state verification attempts. - pub const fn verification_count(&self) -> usize { - self.verification_count - } - - fn record(&mut self, phase: StoreMigrationPhase) -> io::Result<()> { - self.observed.push(phase); - if self.fail_at == Some(phase) { - Err(io::Error::other("injected store-migration failure")) - } else { - Ok(()) - } - } -} - -impl StoreMigrationStorage for RecordingStorage { - fn verify_current(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - self.verification_count = self - .verification_count - .checked_add(1) - .ok_or_else(|| io::Error::other("verification count overflow"))?; - self.verification_failure - .map_or(Ok(()), |kind| Err(kind.into())) - } - - fn write_intent_stage(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - self.record(StoreMigrationPhase::WriteIntentStage) - } - - fn synchronize_intent_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeIntentStage) - } - - fn link_intent(&mut self, _intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - self.record(StoreMigrationPhase::LinkIntent) - } - - fn synchronize_root_after_intent(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterIntent) - } - - fn remove_intent_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::RemoveIntentStage) - } - - fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterIntentCleanup) - } - - fn admit_reader_fence(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::AdmitReaderFence) - } - - fn admit_namespace_prefix(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::AdmitNamespacePrefix) - } - - fn synchronize_root_after_namespace(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterNamespace) - } - - fn write_marker_stage(&mut self, _marker: &CanonicalStoreFormatMarker) -> io::Result<()> { - self.record(StoreMigrationPhase::WriteMarkerStage) - } - - fn synchronize_marker_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeMarkerStage) - } - - fn link_marker(&mut self, _marker: &CanonicalStoreFormatMarker) -> io::Result<()> { - self.record(StoreMigrationPhase::LinkMarker) - } - - fn synchronize_root_after_marker(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterMarker) - } - - fn remove_marker_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::RemoveMarkerStage) - } - - fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup) - } - - fn write_receipt_stage(&mut self, _receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { - self.record(StoreMigrationPhase::WriteReceiptStage) - } - - fn synchronize_receipt_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeReceiptStage) - } - - fn link_receipt(&mut self, _receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { - self.record(StoreMigrationPhase::LinkReceipt) - } - - fn synchronize_root_after_receipt(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterReceipt) - } - - fn remove_receipt_stage(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::RemoveReceiptStage) - } - - fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()> { - self.record(StoreMigrationPhase::SynchronizeRootAfterReceiptCleanup) - } -} diff --git a/tests/support/populated_store.rs b/tests/support/populated_store.rs index 5098513a..298316b1 100644 --- a/tests/support/populated_store.rs +++ b/tests/support/populated_store.rs @@ -3,10 +3,9 @@ //! Every populated version-two test store is created by [`create`] and then //! extended by [`retain`] and by the version-two writer in `successor_catalog.rs`. //! Only public production entry points run, so test code is not a second -//! implementation of the store format. Today [`create`] initializes version -//! one, publishes the first catalog through `FilesystemCatalogPublisher`, and -//! migrates. When direct initialization replaces migration (#198), only that -//! creation step changes. +//! implementation of the store format. [`create`] initializes the store +//! directly and publishes the first catalog through +//! `FilesystemVersionTwoCatalogPublisher` and the retained-closure gate. //! //! Consumers include this file by path beside their laws, so it names the //! crate as `keep` and uses `super`-relative visibility. @@ -19,16 +18,16 @@ use std::path::Path; use keep::{ AdmittedLayout, AdmittedRetentionManifest, AdmittedRetentionRoot, AdmittedSegment, AdmittedSegmentRecord, BlobId, CanonicalCatalog, CanonicalLayoutRecord, CanonicalRetentionRoot, - CanonicalStoreMigrationReceipt, CatalogGeneration, CatalogPublicationExpectation, - CatalogPublicationReceipt, CatalogRestartByteLimit, CatalogRestartPolicy, ChunkId, ChunkSpan, - FastCdc, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, - FilesystemRetentionPublicationAuthority, FilesystemStoreMigrationAuthority, - FilesystemVersionTwoAdmission, LayoutEntryLimit, RegisteredRetentionProfile, - RegisteredStorageProfile, RetentionAnchor, RetentionClosureLimits, - RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, + CatalogGeneration, CatalogPublicationExpectation, CatalogRestartByteLimit, + CatalogRestartPolicy, ChunkId, ChunkSpan, FastCdc, FilesystemCatalogSnapshot, + FilesystemRetentionPublicationAuthority, FilesystemVersionTwoAdmission, + FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, RegisteredRetentionProfile, + RegisteredStorageProfile, RetainedCatalogPublicationReceipt, RetentionAnchor, + RetentionClosureLimits, RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionPublicationReceipt, RetentionRoot, RootGeneration, SegmentReadPolicy, - SegmentRecordLimit, StagedSegment, execute_retention_publication, execute_store_migration, - preflight_retention_transition, prepare_retention_publication, publish_catalog_generation, + SegmentRecordLimit, StagedSegment, execute_retention_publication, + preflight_retention_transition, prepare_retention_publication, + publish_retained_catalog_generation, }; pub(super) type TestResult = Result>; @@ -40,19 +39,16 @@ pub(super) type TestResult = Result>; /// after admission is identical. #[derive(Clone, Copy)] pub(super) struct StoreAdmission { - /// Initializes an empty root. - pub(super) initialize: fn(&Path) -> TestResult, - /// Reacquires authority over the published pre-migration root. - pub(super) reopen_version_one: fn(&Path) -> TestResult, - /// Reacquires authority over the created version-two root. + /// Initializes an empty root and returns its writer authority. + pub(super) initialize: fn(&Path) -> TestResult, + /// Reacquires authority over the created root. pub(super) reopen_version_two: fn(&Path) -> TestResult, } impl StoreAdmission { /// The production platform profile. pub(super) const PLATFORM: Self = Self { - initialize: |root| Ok(FilesystemPlatformAdmission::initialize(root)?), - reopen_version_one: |root| Ok(FilesystemPlatformAdmission::reopen(root)?), + initialize: |root| Ok(FilesystemVersionTwoAdmission::initialize(root)?.into_admission()), reopen_version_two: |root| Ok(FilesystemVersionTwoAdmission::reopen(root)?), }; } @@ -81,13 +77,11 @@ impl FirstCatalogRecords { } } -/// The receipts of one completed creation step. +/// The receipt of one completed creation step. #[must_use] pub(super) struct CreatedStore { - /// The first catalog publication. - pub(super) catalog: CatalogPublicationReceipt, - /// The step that made the root version two. - pub(super) migration: CanonicalStoreMigrationReceipt, + /// The first catalog publication, through the retained-closure gate. + pub(super) catalog: RetainedCatalogPublicationReceipt, } /// The canonical identity of one blob under the registered storage profile. @@ -130,9 +124,11 @@ pub(super) const fn segment_policy() -> SegmentReadPolicy { SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) } -/// Creates a version-two store at the empty directory `root`. +/// Initializes a version-two store at the empty directory `root`. /// -/// The first catalog holds one segment with the `selection` of `contents`' +/// The store has no catalog `HEAD` and no retention head, so the first +/// publication runs with no predecessor and proves both absences through +/// the gate. The first catalog holds one segment with the `selection` of `contents`' /// records. Writer authority is released before this returns. pub(super) fn create( root: &Path, @@ -140,27 +136,31 @@ pub(super) fn create( contents: &[&[u8]], selection: FirstCatalogRecords, ) -> TestResult { + let (created, publisher) = create_holding_authority(root, admission, contents, selection)?; + drop(publisher); + Ok(created) +} + +/// Creates the store exactly as [`create`] does and keeps its writer authority. +/// +/// The returned publisher still holds the writer lock, for laws that must +/// exclude every competing writer from creation onward. +pub(super) fn create_holding_authority( + root: &Path, + admission: StoreAdmission, + contents: &[&[u8]], + selection: FirstCatalogRecords, +) -> TestResult<(CreatedStore, FilesystemVersionTwoCatalogPublisher)> { let identified = contents .iter() .map(|bytes| identify(bytes)) .collect::>>()?; let blobs = contents.iter().copied().zip(&identified); let records = select_records(blobs, selection)?; - let catalog = publish_first_catalog(root, (admission.initialize)(root)?, records)?; - let migration = migrate(root, admission)?; - Ok(CreatedStore { catalog, migration }) -} - -/// Makes the published root version two and releases writer authority. -fn migrate(root: &Path, admission: StoreAdmission) -> TestResult { - let mut authority = FilesystemStoreMigrationAuthority::open( - (admission.reopen_version_one)(root)?, - segment_policy(), - )?; - let intent = authority.observe_intent()?; - let receipt = execute_store_migration(&mut authority, &intent)?; - drop(authority); - Ok(receipt) + let mut publisher = + FilesystemVersionTwoCatalogPublisher::open((admission.initialize)(root)?, policy()?)?; + let catalog = publish_first_catalog(root, &mut publisher, records)?; + Ok((CreatedStore { catalog }, publisher)) } /// Every selected chunk record, deduplicated across blobs, then each new layout. @@ -189,10 +189,9 @@ pub(super) fn select_records<'content>( fn publish_first_catalog( root: &Path, - admission: FilesystemPlatformAdmission, + publisher: &mut FilesystemVersionTwoCatalogPublisher, records: Vec>, -) -> TestResult { - let mut publisher = FilesystemCatalogPublisher::open(admission, policy()?)?; +) -> TestResult { let mut stage = StagedSegment::begin( publisher.create_segment_stage()?, SegmentRecordLimit::MAXIMUM, @@ -206,8 +205,8 @@ fn publish_first_catalog( let segment = segments.first().ok_or("segment absent")?; let selected = publisher.select_segment(sealed, segment)?; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - Ok(publish_catalog_generation( - &mut publisher, + Ok(publish_retained_catalog_generation( + publisher, CatalogPublicationExpectation::uninitialized(), selected, &catalog, diff --git a/tests/version_two_admission_contract.rs b/tests/version_two_admission_contract.rs index b6ad2df4..29014609 100644 --- a/tests/version_two_admission_contract.rs +++ b/tests/version_two_admission_contract.rs @@ -6,8 +6,6 @@ const CATALOG_PUBLISHER: &str = include_str!("../src/adapters/filesystem_catalog const STORE_INITIALIZER: &str = include_str!("../src/adapters/filesystem_store_initializer.rs"); const VERSION_TWO_ADMISSION: &str = include_str!("../src/adapters/filesystem_version_two_admission.rs"); -const MIGRATION_AUTHORITY: &str = - include_str!("../src/adapters/store_migration/filesystem_migration_authority.rs"); const ADMISSION_ERROR: &str = include_str!("../src/adapters/filesystem_platform_admission_error.rs"); @@ -25,22 +23,21 @@ fn retention_publication_consumes_only_version_two_authority() { fn version_one_publishers_consume_only_version_one_authority() { assert!(CATALOG_PUBLISHER.contains("FilesystemPlatformAdmission")); assert!(!CATALOG_PUBLISHER.contains("FilesystemVersionTwoAdmission")); - assert!(MIGRATION_AUTHORITY.contains("FilesystemPlatformAdmission")); - assert!(!MIGRATION_AUTHORITY.contains("FilesystemVersionTwoAdmission")); } #[test] fn version_two_reopen_produces_only_version_two_authority() { assert!(!STORE_INITIALIZER.contains("fn reopen_version_two")); + assert!(!STORE_INITIALIZER.contains("FilesystemVersionTwoAdmission")); assert!(VERSION_TWO_ADMISSION.contains("pub struct FilesystemVersionTwoAdmission")); assert!(VERSION_TWO_ADMISSION.contains("pub fn reopen(store_root: &Path)")); - assert!(VERSION_TWO_ADMISSION.contains("filesystem_version_two_records::admit")); + assert!(VERSION_TWO_ADMISSION.contains("pub fn initialize(")); + assert!(VERSION_TWO_ADMISSION.contains("filesystem_format_marker::admit")); assert!(VERSION_TWO_ADMISSION.contains("filesystem_platform_profile::open_version_two")); } -/// Admission refusals grow with the platform surface (`MigrationRecord` and -/// `RootIdentityChanged` arrived after the first release candidate), so the -/// public error is non-exhaustive and a new refusal is not a breaking change. +/// Admission refusals grow with the platform surface, so the public error is +/// non-exhaustive and a new refusal is not a breaking change. #[test] fn admission_error_is_non_exhaustive() { assert!( diff --git a/tests/version_two_catalog_writer.rs b/tests/version_two_catalog_writer.rs index dbe14956..ea77fd6a 100644 --- a/tests/version_two_catalog_writer.rs +++ b/tests/version_two_catalog_writer.rs @@ -1,7 +1,7 @@ -//! Version-two catalog writer and stage-recovery laws on migrated stores. +//! Version-two catalog writer and stage-recovery laws on initialized stores. //! -//! Every law builds a real version-one store, migrates it, publishes a -//! retention root, and then drives the version-two writer and recovery tools +//! Every law initializes a real store, publishes its first catalog and a +//! retention root through the public writer, and then drives the version-two writer and recovery tools //! through their public production entry points. #![cfg(target_os = "linux")] diff --git a/tests/version_two_catalog_writer/version_laws.rs b/tests/version_two_catalog_writer/version_laws.rs index 6e9135d4..1e9f3bb0 100644 --- a/tests/version_two_catalog_writer/version_laws.rs +++ b/tests/version_two_catalog_writer/version_laws.rs @@ -1,9 +1,7 @@ //! This module owns typed refusal of cross-version writer and recovery entry. -use std::io; - use keep::{ - AdmittedSegment, AdmittedSegmentRecord, FilesystemCatalogPublisher, + AdmittedSegment, AdmittedSegmentRecord, FilesystemCatalogPublisher, FilesystemNamespaceRefusal, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageDiscardOpenError, @@ -60,8 +58,6 @@ fn version_one_entry_points_refuse_a_version_two_root() -> TestResult<()> { for role in [ RecoveryEntryRole::ReaderFence, RecoveryEntryRole::FormatMarker, - RecoveryEntryRole::MigrationIntent, - RecoveryEntryRole::MigrationReceipt, RecoveryEntryRole::RetentionDirectory, RecoveryEntryRole::GcDirectory, RecoveryEntryRole::RecoveryDirectory, @@ -84,7 +80,8 @@ fn version_two_entry_points_refuse_a_version_one_root() -> TestResult<()> { matches!( error, FilesystemPlatformAdmissionError::Namespace { source } - if source.kind() == io::ErrorKind::NotFound + if source.get_ref().and_then(|cause| cause.downcast_ref::()) + == Some(&FilesystemNamespaceRefusal::MissingFormatMarker) ) }; @@ -114,7 +111,7 @@ fn version_two_entry_points_refuse_a_version_one_root() -> TestResult<()> { assert!(matches!( classify_version_two_recovery_names(inventory), Err(RecoveryNameClassificationError::Missing { - required: RecoveryRequiredEntry::CurrentHead + required: RecoveryRequiredEntry::ReaderFence }) )); Ok(()) diff --git a/tests/version_two_catalog_writer/writer_fixture.rs b/tests/version_two_catalog_writer/writer_fixture.rs index 62673f2b..e64cd43c 100644 --- a/tests/version_two_catalog_writer/writer_fixture.rs +++ b/tests/version_two_catalog_writer/writer_fixture.rs @@ -27,7 +27,7 @@ pub(super) fn first_blob() -> Vec { /// The content each law ingests through the version-two writer. pub(super) fn second_blob() -> Vec { - b"KEEP V2 WRITER -- the second blob, ingested after migration | ".repeat(2_500) + b"KEEP V2 WRITER -- the second blob, ingested after the first catalog | ".repeat(2_500) } /// Opens the version-two writer under the production platform profile. diff --git a/xtask/src/durability_crash_case.rs b/xtask/src/durability_crash_case.rs index 71a1b10c..4a5cd2e2 100644 --- a/xtask/src/durability_crash_case.rs +++ b/xtask/src/durability_crash_case.rs @@ -21,8 +21,8 @@ impl DurabilityCrashCase { /// Returns [`DurabilityCrashCaseError::MissingOccurrence`] when a repeated /// transition lacks an occurrence, or /// [`DurabilityCrashCaseError::UnexpectedOccurrence`] when a non-repeated - /// transition receives one. Namespace admission also returns - /// [`DurabilityCrashCaseError::OccurrenceOutOfRange`] outside its six + /// transition receives one. Initialization directory admission also returns + /// [`DurabilityCrashCaseError::OccurrenceOutOfRange`] outside its nine /// `during` occurrences or sole `before` and `after` occurrences. pub const fn new( point: DurabilityCrashPoint, @@ -119,7 +119,7 @@ const fn validate_fixed_range( position: DurabilityCrashPosition, observed: DurabilityCrashOccurrence, ) -> Result<(), DurabilityCrashCaseError> { - if !matches!(point, DurabilityCrashPoint::MigrationAdmitNamespacePrefix) { + if !matches!(point, DurabilityCrashPoint::InitializeAdmitDirectory) { return Ok(()); } let exclusive_limit = if matches!(position, DurabilityCrashPosition::During) { diff --git a/xtask/src/durability_crash_matrix/error.rs b/xtask/src/durability_crash_matrix/error.rs index 4c76aaac..986beb50 100644 --- a/xtask/src/durability_crash_matrix/error.rs +++ b/xtask/src/durability_crash_matrix/error.rs @@ -7,7 +7,7 @@ use std::error::Error; use std::io; use std::time::Duration; -use keep::StoreMigrationRecoveryPlan; +use keep::VersionTwoInitializationPrefix; use xtask::protocol_admission::HexError; use xtask::{ DurabilityCrashCase, DurabilityCrashCaseError, DurabilityCrashOccurrence, DurabilityCrashPoint, @@ -83,9 +83,9 @@ pub(crate) enum DurabilityCrashMatrixError { PointSequenceMismatch { point: DurabilityCrashPoint, }, - RecoveryPlanMismatch { - expected: StoreMigrationRecoveryPlan, - observed: StoreMigrationRecoveryPlan, + InitializationPrefixMismatch { + expected: Option, + observed: Option, }, RepeatedInventoryPath { path: String, @@ -169,7 +169,7 @@ impl Error for DurabilityCrashMatrixError { | Self::MissingVisibleRecord { .. } | Self::NonUnicodeStatePath | Self::PointSequenceMismatch { .. } - | Self::RecoveryPlanMismatch { .. } + | Self::InitializationPrefixMismatch { .. } | Self::RepeatedInventoryPath { .. } | Self::SnapshotGenerationMismatch { .. } | Self::Timeout { .. } diff --git a/xtask/src/durability_crash_matrix/error/display.rs b/xtask/src/durability_crash_matrix/error/display.rs index f269fa17..00b18d0c 100644 --- a/xtask/src/durability_crash_matrix/error/display.rs +++ b/xtask/src/durability_crash_matrix/error/display.rs @@ -18,7 +18,7 @@ impl fmt::Display for DurabilityCrashMatrixError { | Self::HardLinkIdentityMismatch { .. } | Self::InventoryMismatch { .. } | Self::MissingVisibleRecord { .. } - | Self::RecoveryPlanMismatch { .. } + | Self::InitializationPrefixMismatch { .. } | Self::RepeatedInventoryPath { .. } | Self::SnapshotGenerationMismatch { .. } | Self::UnexpectedArtifactKind { .. } => format_state(self, formatter), @@ -87,9 +87,9 @@ fn format_state( "post-crash snapshot lacks visible record `{record}`" ) } - DurabilityCrashMatrixError::RecoveryPlanMismatch { expected, observed } => write!( + DurabilityCrashMatrixError::InitializationPrefixMismatch { expected, observed } => write!( formatter, - "post-crash migration recovery planned {observed:?}, expected {expected:?}" + "post-crash initialization resumed from {observed:?}, expected {expected:?}" ), DurabilityCrashMatrixError::RepeatedInventoryPath { path } => { write!(formatter, "post-crash inventory repeated path `{path}`") @@ -235,7 +235,7 @@ fn format_command( DurabilityCrashMatrixError::Usage => formatter.write_str( "usage: cargo xtask durability-crash-matrix \ [--case [] \ - | --sequence ]", + | --sequence ]", ), _ => Err(fmt::Error), } diff --git a/xtask/src/durability_crash_matrix/production_protocol.rs b/xtask/src/durability_crash_matrix/production_protocol.rs index d80f24ec..c68dbb0d 100644 --- a/xtask/src/durability_crash_matrix/production_protocol.rs +++ b/xtask/src/durability_crash_matrix/production_protocol.rs @@ -5,9 +5,7 @@ pub(super) mod crash_blob; pub(super) mod fixture; pub(super) mod initialization; mod initialization_storage; -mod migration; -mod migration_storage; -mod populated_store; +pub(super) mod populated_store; mod publication; mod publication_storage; mod recovery; @@ -16,6 +14,7 @@ pub(super) mod retention; mod retention_storage; mod segment_stage; pub(super) mod version_two_catalog; +pub(super) mod version_two_initialization; use std::error::Error; use std::fs; @@ -51,8 +50,8 @@ pub(super) fn run( DurabilityCrashSequence::Retention => { retention::run(&store_root, &mut control)?; } - DurabilityCrashSequence::Migration => { - migration::run(&store_root, &mut control)?; + DurabilityCrashSequence::VersionTwoInitialization => { + version_two_initialization::run(&store_root, control)?; } DurabilityCrashSequence::VersionTwoCatalog => { version_two_catalog::run(&store_root, control)?; diff --git a/xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs b/xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs index 601d5d23..4ca9eab1 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs @@ -13,7 +13,7 @@ use super::verification; /// The exact bytes the version-two writer ingests before process death. pub(in crate::durability_crash_matrix) const VERSION_TWO_CONTENT: &[u8] = - b"keep version-two crash campaign: one blob ingested after migration"; + b"keep version-two crash campaign: one blob ingested after the first catalog"; /// The synthetic source byte of the frozen one-zero bundle corpus. const BUNDLE_CONTENT: &[u8] = &[0]; diff --git a/xtask/src/durability_crash_matrix/production_protocol/migration.rs b/xtask/src/durability_crash_matrix/production_protocol/migration.rs deleted file mode 100644 index 6b932783..00000000 --- a/xtask/src/durability_crash_matrix/production_protocol/migration.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! This module owns execution of the production store-migration protocol. - -use std::path::Path; - -use keep::{FilesystemStoreMigrationAuthority, FilesystemWriterLock, execute_store_migration}; - -use super::control::CrashControl; -use super::initialization; -use super::migration_storage::CrashMigrationStorage; -use super::publication; -use super::{DurabilityCrashMatrixError, verification}; - -/// Publishes the Golden File Worldline version-1 store, then migrates it -/// through the production 21-phase protocol with the selected boundary gated -/// for process death. -/// -/// No version-1 gate fires for a migration case, so the publication -/// precondition runs to completion and releases its writer lock before the -/// migration authority reacquires it. -pub(super) fn run( - store_root: &Path, - control: &mut CrashControl, -) -> Result<(), DurabilityCrashMatrixError> { - publication::run(store_root, control)?; - let lock = FilesystemWriterLock::try_acquire(store_root) - .map_err(|source| verification("reacquire writer lock for migration", source))?; - let authority = FilesystemStoreMigrationAuthority::open_unchecked_for_repository_tasks( - lock, - initialization::segment_policy(), - ) - .map_err(|source| verification("open production migration authority", source))?; - let intent = authority - .observe_intent() - .map_err(|source| verification("observe production migration intent", source))?; - let mut storage = CrashMigrationStorage::new(authority, control); - execute_store_migration(&mut storage, &intent) - .map(|_receipt| ()) - .map_err(|source| verification("execute production store migration", source)) -} diff --git a/xtask/src/durability_crash_matrix/production_protocol/migration_stage_prefix_laws.rs b/xtask/src/durability_crash_matrix/production_protocol/migration_stage_prefix_laws.rs deleted file mode 100644 index 1bde7755..00000000 --- a/xtask/src/durability_crash_matrix/production_protocol/migration_stage_prefix_laws.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! This module owns nonempty strict migration interruption-prefix laws. - -use std::error::Error; - -use keep::{ - AdmittedStoreFormatMarker, AdmittedStoreMigrationIntent, AdmittedStoreMigrationReceipt, -}; -use xtask::protocol_admission::{EmptyHex, decode_lower_hex}; - -use super::{INTENT_INTERRUPTION, MARKER_INTERRUPTION, RECEIPT_INTERRUPTION}; - -const INTENT: &str = include_str!("../../../../conformance/segment-store/v2/migration-intent.hex"); -const MARKER: &str = include_str!("../../../../conformance/segment-store/v2/format-marker.hex"); -const RECEIPT: &str = - include_str!("../../../../conformance/segment-store/v2/migration-receipt.hex"); - -#[test] -fn stage_interruptions_are_nonempty_strict_prefixes_of_admitted_records() --> Result<(), Box> { - let intent_bytes = decode(INTENT)?; - let marker_bytes = decode(MARKER)?; - let receipt_bytes = decode(RECEIPT)?; - let intent = AdmittedStoreMigrationIntent::decode(&intent_bytes)?; - let marker = AdmittedStoreFormatMarker::decode(&marker_bytes)?; - let receipt = AdmittedStoreMigrationReceipt::decode(&receipt_bytes, &intent, &marker)?; - for (bytes, prefix) in [ - (intent.encoded(), INTENT_INTERRUPTION), - (marker.encoded(), MARKER_INTERRUPTION), - (receipt.encoded(), RECEIPT_INTERRUPTION), - ] { - assert!(prefix > 0, "an interruption must leave a nonempty prefix"); - assert!( - prefix < bytes.len(), - "an interruption must not complete the record" - ); - } - Ok(()) -} - -fn decode(text: &str) -> Result, Box> { - let hex = text - .strip_suffix('\n') - .ok_or("fixture has no final newline")?; - Ok(decode_lower_hex(hex, 256, EmptyHex::Refuse)?) -} diff --git a/xtask/src/durability_crash_matrix/production_protocol/migration_storage.rs b/xtask/src/durability_crash_matrix/production_protocol/migration_storage.rs deleted file mode 100644 index ececb482..00000000 --- a/xtask/src/durability_crash_matrix/production_protocol/migration_storage.rs +++ /dev/null @@ -1,315 +0,0 @@ -//! This module owns crash injection around production store migration. - -use std::io; - -use keep::{ - CanonicalStoreFormatMarker, CanonicalStoreMigrationIntent, CanonicalStoreMigrationReceipt, - FilesystemStoreMigrationAuthority, StoreMigrationFixedStage, StoreMigrationStorage, -}; -use xtask::{DurabilityCrashPoint, DurabilityCrashPosition}; - -use super::control::{CrashControl, DuringTiming}; - -#[cfg(test)] -#[path = "migration_stage_prefix_laws.rs"] -mod stage_prefix_laws; - -// The fixture law below admits all three records and requires these fixed -// interruption offsets to remain nonempty strict prefixes of their encodings. -const INTENT_INTERRUPTION: usize = 128; -const MARKER_INTERRUPTION: usize = 48; -const RECEIPT_INTERRUPTION: usize = 128; - -pub(super) struct CrashMigrationStorage<'control> { - inner: FilesystemStoreMigrationAuthority, - control: &'control mut CrashControl, -} - -impl<'control> CrashMigrationStorage<'control> { - pub(super) const fn new( - inner: FilesystemStoreMigrationAuthority, - control: &'control mut CrashControl, - ) -> Self { - Self { inner, control } - } -} - -impl StoreMigrationStorage for CrashMigrationStorage<'_> { - fn verify_current(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - StoreMigrationStorage::verify_current(&mut self.inner, intent) - } - - fn write_intent_stage(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - execute_write( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationWriteIntentStage, - |inner| inner.write_intent_stage(intent), - |inner| { - inner.write_fixed_stage_prefix_for_repository_tasks( - StoreMigrationFixedStage::Intent, - intent.encoded(), - INTENT_INTERRUPTION, - ) - }, - ) - } - - fn synchronize_intent_stage(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationSynchronizeIntentStage, - DuringTiming::Before, - FilesystemStoreMigrationAuthority::synchronize_intent_stage, - ) - } - - fn link_intent(&mut self, intent: &CanonicalStoreMigrationIntent) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationLinkIntent, - DuringTiming::After, - |inner| inner.link_intent(intent), - ) - } - - fn synchronize_root_after_intent(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationSynchronizeRootAfterIntent, - DuringTiming::Before, - FilesystemStoreMigrationAuthority::synchronize_root_after_intent, - ) - } - - fn remove_intent_stage(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationRemoveIntentStage, - DuringTiming::After, - FilesystemStoreMigrationAuthority::remove_intent_stage, - ) - } - - fn synchronize_root_after_intent_cleanup(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationSynchronizeRootAfterIntentCleanup, - DuringTiming::Before, - FilesystemStoreMigrationAuthority::synchronize_root_after_intent_cleanup, - ) - } - - fn admit_reader_fence(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationAdmitReaderFence, - DuringTiming::After, - FilesystemStoreMigrationAuthority::admit_reader_fence, - ) - } - - fn admit_namespace_prefix(&mut self) -> io::Result<()> { - let point = DurabilityCrashPoint::MigrationAdmitNamespacePrefix; - match self.control.position(point) { - None => self.inner.admit_namespace_prefix(), - Some(DurabilityCrashPosition::Before) => self.control.await_process_death(), - Some(DurabilityCrashPosition::During) => { - let reached = self - .control - .occurrence() - .map_or(1, |occurrence| occurrence.get().saturating_add(1)); - let reached = usize::try_from(reached).map_err(io::Error::other)?; - self.inner - .admit_namespace_prefix_for_repository_tasks(reached)?; - self.control.await_process_death() - } - Some(DurabilityCrashPosition::After) => { - self.inner.admit_namespace_prefix()?; - self.control.await_process_death() - } - } - } - - fn synchronize_root_after_namespace(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationSynchronizeRootAfterNamespace, - DuringTiming::Before, - FilesystemStoreMigrationAuthority::synchronize_root_after_namespace, - ) - } - - fn write_marker_stage(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()> { - execute_write( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationWriteMarkerStage, - |inner| inner.write_marker_stage(marker), - |inner| { - inner.write_fixed_stage_prefix_for_repository_tasks( - StoreMigrationFixedStage::Marker, - marker.encoded(), - MARKER_INTERRUPTION, - ) - }, - ) - } - - fn synchronize_marker_stage(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationSynchronizeMarkerStage, - DuringTiming::Before, - FilesystemStoreMigrationAuthority::synchronize_marker_stage, - ) - } - - fn link_marker(&mut self, marker: &CanonicalStoreFormatMarker) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationLinkMarker, - DuringTiming::After, - |inner| inner.link_marker(marker), - ) - } - - fn synchronize_root_after_marker(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationSynchronizeRootAfterMarker, - DuringTiming::Before, - FilesystemStoreMigrationAuthority::synchronize_root_after_marker, - ) - } - - fn remove_marker_stage(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationRemoveMarkerStage, - DuringTiming::After, - FilesystemStoreMigrationAuthority::remove_marker_stage, - ) - } - - fn synchronize_root_after_marker_cleanup(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationSynchronizeRootAfterMarkerCleanup, - DuringTiming::Before, - FilesystemStoreMigrationAuthority::synchronize_root_after_marker_cleanup, - ) - } - - fn write_receipt_stage(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { - execute_write( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationWriteReceiptStage, - |inner| inner.write_receipt_stage(receipt), - |inner| { - inner.write_fixed_stage_prefix_for_repository_tasks( - StoreMigrationFixedStage::Receipt, - receipt.encoded(), - RECEIPT_INTERRUPTION, - ) - }, - ) - } - - fn synchronize_receipt_stage(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationSynchronizeReceiptStage, - DuringTiming::Before, - FilesystemStoreMigrationAuthority::synchronize_receipt_stage, - ) - } - - fn link_receipt(&mut self, receipt: &CanonicalStoreMigrationReceipt) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationLinkReceipt, - DuringTiming::After, - |inner| inner.link_receipt(receipt), - ) - } - - fn synchronize_root_after_receipt(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationSynchronizeRootAfterReceipt, - DuringTiming::Before, - FilesystemStoreMigrationAuthority::synchronize_root_after_receipt, - ) - } - - fn remove_receipt_stage(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationRemoveReceiptStage, - DuringTiming::After, - FilesystemStoreMigrationAuthority::remove_receipt_stage, - ) - } - - fn synchronize_root_after_receipt_cleanup(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::MigrationSynchronizeRootAfterReceiptCleanup, - DuringTiming::Before, - FilesystemStoreMigrationAuthority::synchronize_root_after_receipt_cleanup, - ) - } -} - -fn execute( - inner: &mut FilesystemStoreMigrationAuthority, - control: &mut CrashControl, - point: DurabilityCrashPoint, - during: DuringTiming, - operation: impl FnOnce(&mut FilesystemStoreMigrationAuthority) -> io::Result, -) -> io::Result { - control.before(point, during)?; - let result = operation(inner)?; - control.after(point, during)?; - Ok(result) -} - -fn execute_write( - inner: &mut FilesystemStoreMigrationAuthority, - control: &mut CrashControl, - point: DurabilityCrashPoint, - complete: impl FnOnce(&mut FilesystemStoreMigrationAuthority) -> io::Result<()>, - interrupted: impl FnOnce(&mut FilesystemStoreMigrationAuthority) -> io::Result<()>, -) -> io::Result<()> { - match control.position(point) { - None => complete(inner), - Some(DurabilityCrashPosition::Before) => control.await_process_death(), - Some(DurabilityCrashPosition::During) => { - interrupted(inner)?; - control.await_process_death() - } - Some(DurabilityCrashPosition::After) => { - complete(inner)?; - control.await_process_death() - } - } -} diff --git a/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs b/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs index 045b2832..fb43cfb2 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs @@ -2,24 +2,22 @@ //! //! It is the crash matrix's counterpart of `tests/support/populated_store.rs`: //! the same public entry points, under the matrix's typed error, because a -//! repository task cannot include integration-test support. Today creation is -//! version-one initialization, the first catalog through -//! `FilesystemCatalogPublisher`, and migration. When direct initialization -//! replaces migration (#198), only [`create_bundle_store`] changes. +//! repository task cannot include integration-test support. Creation is direct +//! initialization, then the first catalog through +//! `FilesystemVersionTwoCatalogPublisher` and the retained-closure gate. use std::fs; use std::path::Path; use keep::{ AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationExpectation, - FilesystemCatalogPublisher, FilesystemPlatformAdmission, FilesystemSegmentStage, - FilesystemStoreMigrationAuthority, SealedSegment, SegmentRecordLimit, StagedSegment, - execute_store_migration, publish_catalog_generation, + FilesystemSegmentStage, FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, + SealedSegment, SegmentRecordLimit, StagedSegment, publish_retained_catalog_generation, }; use super::crash_blob::CrashBlob; use super::fixture::{BUNDLE_CATALOG_NAME, BUNDLE_SEGMENT_NAME, GoldenFixture}; -use super::{DurabilityCrashMatrixError, initialization, verification}; +use super::{DurabilityCrashMatrixError, initialization, verification, version_two_catalog}; /// Creates the one-zero bundle store at the empty directory `store_root`. /// @@ -27,13 +25,22 @@ use super::{DurabilityCrashMatrixError, initialization, verification}; /// against the frozen bundle snapshot and retention root, so the pools must /// hold exactly the frozen bundle bytes; any other bytes refuse here. pub(super) fn create_bundle_store(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - publish_bundle_catalog(store_root)?; - migrate(store_root)?; + let admission = + FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("initialize crash bundle store", source))? + .into_admission(); + publish_bundle_catalog(store_root, admission)?; require_frozen_bundle(store_root) } -fn publish_bundle_catalog(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let mut publisher = initialization::publisher(store_root)?; +/// Publishes the bundle blob as the first catalog of an initialized store. +pub(in crate::durability_crash_matrix) fn publish_bundle_catalog( + store_root: &Path, + admission: FilesystemVersionTwoAdmission, +) -> Result<(), DurabilityCrashMatrixError> { + let mut publisher = + FilesystemVersionTwoCatalogPublisher::open(admission, version_two_catalog::policy()?) + .map_err(|source| verification("open crash bundle publisher", source))?; let blob = CrashBlob::bundle()?; let sealed = stage_bundle(&publisher, &blob)?; let bytes = fs::read(store_root.join("staging/current.seg")) @@ -49,7 +56,7 @@ fn publish_bundle_catalog(store_root: &Path) -> Result<(), DurabilityCrashMatrix .map_err(|source| verification("construct bundle catalog generation", source))?; let catalog = CanonicalCatalog::from_segments(generation, None, &segments) .map_err(|source| verification("encode bundle catalog", source))?; - publish_catalog_generation( + publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), selection, @@ -61,7 +68,7 @@ fn publish_bundle_catalog(store_root: &Path) -> Result<(), DurabilityCrashMatrix } fn stage_bundle<'publisher>( - publisher: &'publisher FilesystemCatalogPublisher, + publisher: &'publisher FilesystemVersionTwoCatalogPublisher, blob: &CrashBlob, ) -> Result>, DurabilityCrashMatrixError> { let stage = publisher @@ -79,21 +86,9 @@ fn stage_bundle<'publisher>( .map_err(|source| verification("seal bundle segment", source)) } -fn migrate(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let admission = FilesystemPlatformAdmission::reopen(store_root) - .map_err(|source| verification("reopen bundle store for migration", source))?; - let mut migration = - FilesystemStoreMigrationAuthority::open(admission, initialization::segment_policy()) - .map_err(|source| verification("open crash migration authority", source))?; - let intent = migration - .observe_intent() - .map_err(|source| verification("observe crash migration intent", source))?; - execute_store_migration(&mut migration, &intent) - .map(|_receipt| ()) - .map_err(|source| verification("execute crash store migration", source)) -} - -fn require_frozen_bundle(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { +pub(in crate::durability_crash_matrix) fn require_frozen_bundle( + store_root: &Path, +) -> Result<(), DurabilityCrashMatrixError> { for (artifact, relative, expected) in [ ( "bundle segment", diff --git a/xtask/src/durability_crash_matrix/production_protocol/retention.rs b/xtask/src/durability_crash_matrix/production_protocol/retention.rs index 1cd5663a..2ff9bd99 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/retention.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/retention.rs @@ -16,13 +16,13 @@ use super::populated_store; use super::retention_storage::CrashRetentionStorage; use super::{DurabilityCrashMatrixError, verification}; -/// Migrates a fresh bundle store and publishes retention generation one, +/// Initializes a fresh bundle store and publishes retention generation one, /// dying at the selected coordinate. pub(super) fn run( store_root: &Path, control: &mut CrashControl, ) -> Result<(), DurabilityCrashMatrixError> { - let authority = migrated_authority(store_root)?; + let authority = bundle_authority(store_root)?; let root = GoldenFixture::retention_root()?; let preparation = preparation(root.bytes())?; let mut storage = CrashRetentionStorage::new(authority, control, store_root); @@ -31,16 +31,16 @@ pub(super) fn run( .map_err(|source| verification("execute production retention publication", source)) } -/// Creates the bundle store through the public writer, then reopens it as -/// version two and returns retention authority over it. -pub(super) fn migrated_authority( +/// Creates the bundle store through the public writer, then reopens it and +/// returns retention authority over it. +pub(super) fn bundle_authority( store_root: &Path, ) -> Result { populated_store::create_bundle_store(store_root)?; reopened_authority(store_root) } -/// Reopens the migrated store and returns retention authority over it. +/// Reopens the bundle store and returns retention authority over it. pub(in crate::durability_crash_matrix) fn reopened_authority( store_root: &Path, ) -> Result { diff --git a/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs b/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs index cb87fa55..bc1e0ee3 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs @@ -1,6 +1,6 @@ //! This module owns execution of the production version-two catalog writer. //! -//! The child migrates the bundle store, publishes retention generation one, +//! The child initializes the bundle store, publishes retention generation one, //! then ingests one new blob through the version-two writer. An observer //! installed on the writer's own stage type blocks at the selected boundary //! for process death; no transition is reimplemented here. @@ -77,11 +77,11 @@ pub(super) fn run( .map_err(|source| verification("execute version-two catalog publication", source)) } -/// Migrates the bundle store and publishes its golden retention root. +/// Creates the bundle store and publishes its golden retention root. pub(in crate::durability_crash_matrix) fn retained_bundle_store( store_root: &Path, ) -> Result<(), DurabilityCrashMatrixError> { - let mut authority = retention::migrated_authority(store_root)?; + let mut authority = retention::bundle_authority(store_root)?; let root = GoldenFixture::retention_root()?; let preparation = retention::preparation(root.bytes())?; execute_retention_publication(&mut authority, &preparation) diff --git a/xtask/src/durability_crash_matrix/production_protocol/version_two_initialization.rs b/xtask/src/durability_crash_matrix/production_protocol/version_two_initialization.rs new file mode 100644 index 00000000..db773d7a --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/version_two_initialization.rs @@ -0,0 +1,178 @@ +//! This module owns execution of the production version-two initializer. +//! +//! An observer installed on the initializer's own phases blocks at the +//! selected boundary for process death; no phase is reimplemented here. The +//! recovery row first stops one initialization after its writer lock, with an +//! incomplete `FORMAT.next` in place, then dies inside the resuming call. + +use std::fs::OpenOptions; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +use keep::{ + CanonicalStoreFormatMarker, FilesystemVersionTwoAdmission, VersionTwoInitializationBoundary, + VersionTwoInitializationDirectory, VersionTwoInitializationObserver, + VersionTwoInitializationPhase, +}; +use xtask::{DurabilityCrashOccurrence, DurabilityCrashPoint, DurabilityCrashPosition}; + +use super::control::{CrashControl, DuringTiming}; +use super::{DurabilityCrashMatrixError, verification}; + +/// Length of the incomplete marker stage a torn write leaves; a nonempty +/// strict prefix of the 96-byte marker. +pub(in crate::durability_crash_matrix) const INCOMPLETE_STAGE_LENGTH: usize = 48; + +/// Initializes the store root, dying at the selected coordinate. +pub(super) fn run( + store_root: &Path, + control: CrashControl, +) -> Result<(), DurabilityCrashMatrixError> { + if control + .position(DurabilityCrashPoint::InitializeDiscardIncompleteMarkerStage) + .is_some() + { + leave_incomplete_stage(store_root)?; + } + let observer = observer(control, store_root.to_path_buf()); + FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks(store_root, observer) + .map(drop) + .map_err(|source| verification("execute production initialization", source)) +} + +/// Stops a real initialization at its marker write with a torn stage in place. +fn leave_incomplete_stage(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let root = store_root.to_path_buf(); + let observer: VersionTwoInitializationObserver = Box::new(move |phase, boundary| { + if phase == VersionTwoInitializationPhase::WriteMarkerStage + && boundary == VersionTwoInitializationBoundary::Before + { + write_incomplete_stage(&root)?; + return Err(io::Error::other("stopped with an incomplete marker stage")); + } + Ok(()) + }); + match FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks( + store_root, observer, + ) { + Err(_) => Ok(()), + Ok(_) => Err(verification( + "prepare an incomplete marker stage", + io::Error::other("initialization completed past the marker write"), + )), + } +} + +/// Writes the strict canonical prefix a torn write could leave, unsynchronized. +fn write_incomplete_stage(root: &Path) -> io::Result<()> { + let marker = CanonicalStoreFormatMarker::version_two(); + let prefix = marker + .encoded() + .get(..INCOMPLETE_STAGE_LENGTH) + .ok_or_else(|| io::Error::other("marker shorter than the incomplete stage"))?; + let mut stage = OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join("FORMAT.next"))?; + stage.write_all(prefix)?; + stage.flush() +} + +/// Maps the initializer's own phases onto the eleven initialization crash points. +fn observer(mut control: CrashControl, root: PathBuf) -> VersionTwoInitializationObserver { + Box::new(move |phase, boundary| { + if phase == VersionTwoInitializationPhase::WriteMarkerStage + && boundary == VersionTwoInitializationBoundary::Before + && control.position(DurabilityCrashPoint::InitializeWriteMarkerStage) + == Some(DurabilityCrashPosition::During) + { + write_incomplete_stage(&root)?; + return control.await_process_death(); + } + if let VersionTwoInitializationPhase::AdmitDirectory(directory) = phase { + return directory_boundary(&mut control, directory, boundary); + } + let Some((point, timing)) = crash_point(phase) else { + return Ok(()); + }; + match boundary { + VersionTwoInitializationBoundary::Before => control.before(point, timing), + VersionTwoInitializationBoundary::After => control.after(point, timing), + } + }) +} + +/// Directory admission is one occurrence-counted point: `before` precedes the +/// first directory, `after` follows the last, and `during` occurrence *k* +/// follows directory *k* + 1. +fn directory_boundary( + control: &mut CrashControl, + directory: VersionTwoInitializationDirectory, + boundary: VersionTwoInitializationBoundary, +) -> io::Result<()> { + let point = DurabilityCrashPoint::InitializeAdmitDirectory; + let ordinal = VersionTwoInitializationDirectory::ALL + .iter() + .position(|candidate| *candidate == directory) + .ok_or_else(|| io::Error::other("directory outside the creation order"))?; + let last = VersionTwoInitializationDirectory::ALL + .len() + .saturating_sub(1); + let selected = match (control.position(point), boundary) { + (Some(DurabilityCrashPosition::Before), VersionTwoInitializationBoundary::Before) => { + ordinal == 0 + } + (Some(DurabilityCrashPosition::After), VersionTwoInitializationBoundary::After) => { + ordinal == last + } + (Some(DurabilityCrashPosition::During), VersionTwoInitializationBoundary::After) => { + let occurrence = control + .occurrence() + .map_or(0, DurabilityCrashOccurrence::get); + usize::try_from(occurrence).map_err(io::Error::other)? == ordinal + } + _ => false, + }; + if selected { + control.await_process_death() + } else { + Ok(()) + } +} + +const fn crash_point( + phase: VersionTwoInitializationPhase, +) -> Option<(DurabilityCrashPoint, DuringTiming)> { + use DurabilityCrashPoint as Point; + use VersionTwoInitializationPhase as Phase; + Some(match phase { + Phase::OpenAndLockWriterFile => { + (Point::InitializeOpenAndLockWriterFile, DuringTiming::After) + } + Phase::WriteMarkerStage => (Point::InitializeWriteMarkerStage, DuringTiming::After), + Phase::SynchronizeMarkerStage => ( + Point::InitializeSynchronizeMarkerStage, + DuringTiming::Before, + ), + Phase::SynchronizeRootAfterMarkerStage => ( + Point::InitializeSynchronizeRootAfterMarkerStage, + DuringTiming::Before, + ), + Phase::AdmitReaderFence => (Point::InitializeAdmitReaderFence, DuringTiming::After), + Phase::LinkMarker => (Point::InitializeLinkMarker, DuringTiming::After), + Phase::SynchronizeRootAfterMarker => ( + Point::InitializeSynchronizeRootAfterMarker, + DuringTiming::Before, + ), + Phase::RemoveMarkerStage => (Point::InitializeRemoveMarkerStage, DuringTiming::After), + Phase::SynchronizeRootAfterCleanup => ( + Point::InitializeSynchronizeRootAfterCleanup, + DuringTiming::Before, + ), + Phase::DiscardIncompleteMarkerStage => ( + Point::InitializeDiscardIncompleteMarkerStage, + DuringTiming::After, + ), + _ => return None, + }) +} diff --git a/xtask/src/durability_crash_matrix/restart.rs b/xtask/src/durability_crash_matrix/restart.rs index eda5a0db..a5020f3e 100644 --- a/xtask/src/durability_crash_matrix/restart.rs +++ b/xtask/src/durability_crash_matrix/restart.rs @@ -1,13 +1,12 @@ //! This module owns independent post-process-death store verification. mod expectation; -mod migration; -mod migration_expectation; mod retention; mod retention_incomplete; mod retention_snapshot; mod semantic; mod version_two_catalog; +mod version_two_initialization; mod version_two_recovery; use std::collections::BTreeSet; @@ -27,8 +26,8 @@ pub(super) fn verify( if case.point().sequence() == xtask::DurabilityCrashSequence::Retention { return retention::verify(store_root, case); } - if case.point().sequence() == DurabilityCrashSequence::Migration { - return migration::verify(store_root, case); + if case.point().sequence() == DurabilityCrashSequence::VersionTwoInitialization { + return version_two_initialization::verify(store_root, case); } if case.point().sequence() == DurabilityCrashSequence::VersionTwoCatalog { return version_two_catalog::verify(store_root, case); diff --git a/xtask/src/durability_crash_matrix/restart/expectation.rs b/xtask/src/durability_crash_matrix/restart/expectation.rs index 32a52923..a5d1c2fd 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation.rs @@ -65,8 +65,8 @@ impl ExpectedStoreState { DurabilityCrashSequence::RecoveryDiscard => sequence::recovery(case), DurabilityCrashSequence::Initialization => sequence::initialization(case), DurabilityCrashSequence::Retention - | DurabilityCrashSequence::Migration - | DurabilityCrashSequence::VersionTwoCatalog => { + | DurabilityCrashSequence::VersionTwoCatalog + | DurabilityCrashSequence::VersionTwoInitialization => { Err(DurabilityCrashMatrixError::PointSequenceMismatch { point: case.point(), }) diff --git a/xtask/src/durability_crash_matrix/restart/migration.rs b/xtask/src/durability_crash_matrix/restart/migration.rs deleted file mode 100644 index ce219195..00000000 --- a/xtask/src/durability_crash_matrix/restart/migration.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! This module owns independent post-process-death migration verification: -//! the exact residue, the predicted recovery plan, the production recovery, -//! and the complete migration it must leave behind. - -use std::fs; -use std::path::Path; - -use keep::{ - FilesystemStoreMigrationAuthority, FilesystemWriterLock, StoreMigrationRecoveryPlan as Plan, - execute_store_migration, recover_store_migration, -}; -use xtask::DurabilityCrashCase; - -use super::migration_expectation::{MigrationExpectation, complete_paths}; -use crate::durability_crash_matrix::DurabilityCrashMatrixError; -use crate::durability_crash_matrix::production_protocol::fixture::{ - CATALOG_POOL_PATH, GoldenFixture, SEGMENT_POOL_PATH, -}; -use crate::durability_crash_matrix::production_protocol::initialization::segment_policy; -use crate::durability_crash_matrix::production_protocol::verification; - -pub(super) fn verify( - store_root: &Path, - case: DurabilityCrashCase, -) -> Result<(), DurabilityCrashMatrixError> { - let expected = MigrationExpectation::for_case(case)?; - require_inventory(store_root, expected.paths())?; - verify_version_one_bytes(store_root)?; - - let plan = recover(store_root)?; - if plan != expected.plan() { - return Err(DurabilityCrashMatrixError::RecoveryPlanMismatch { - expected: expected.plan(), - observed: plan, - }); - } - if plan == Plan::VersionOne { - migrate_forward(store_root)?; - } - - require_inventory(store_root, &complete_paths())?; - verify_version_one_bytes(store_root)?; - let settled = recover(store_root)?; - if settled == Plan::Complete { - Ok(()) - } else { - Err(DurabilityCrashMatrixError::RecoveryPlanMismatch { - expected: Plan::Complete, - observed: settled, - }) - } -} - -fn require_inventory( - store_root: &Path, - expected: &std::collections::BTreeSet, -) -> Result<(), DurabilityCrashMatrixError> { - let observed = super::inventory(store_root)?; - if &observed == expected { - Ok(()) - } else { - Err(DurabilityCrashMatrixError::InventoryMismatch { - expected: expected.clone(), - observed, - }) - } -} - -/// Reacquires writer authority the way a restarted process would and runs -/// the production recovery once, reporting the plan the residue admitted. -fn recover(store_root: &Path) -> Result { - let mut authority = - FilesystemStoreMigrationAuthority::reopen_for_recovery_unchecked_for_repository_tasks( - store_root, - segment_policy(), - ) - .map_err(|source| verification("reopen migration for recovery", source))?; - let expected = authority - .observe_intent() - .map_err(|source| verification("observe recovery migration intent", source))?; - let receipt = recover_store_migration(&mut authority, &expected) - .map_err(|source| verification("recover production migration", source))?; - Ok(receipt.plan()) -} - -/// The forward retry after an untouched version-1 store admits. -fn migrate_forward(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let lock = FilesystemWriterLock::try_acquire(store_root) - .map_err(|source| verification("reacquire writer lock for forward retry", source))?; - let mut authority = FilesystemStoreMigrationAuthority::open_unchecked_for_repository_tasks( - lock, - segment_policy(), - ) - .map_err(|source| verification("open forward-retry migration authority", source))?; - let intent = authority - .observe_intent() - .map_err(|source| verification("observe forward-retry migration intent", source))?; - execute_store_migration(&mut authority, &intent) - .map(|_receipt| ()) - .map_err(|source| verification("execute forward-retry migration", source)) -} - -/// Migration never rewrites a version-1 byte: the pools and `HEAD` remain -/// the Golden File Worldline fixtures before and after recovery. -fn verify_version_one_bytes(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - for (relative, fixture) in [ - (SEGMENT_POOL_PATH, GoldenFixture::segment()?), - (CATALOG_POOL_PATH, GoldenFixture::catalog()?), - ("HEAD", GoldenFixture::head()?), - ] { - let observed = fs::read(store_root.join(relative)) - .map_err(|source| DurabilityCrashMatrixError::io("read version-1 artifact", source))?; - if observed != fixture.bytes() { - return Err(DurabilityCrashMatrixError::artifact_bytes( - relative, - fixture.bytes(), - &observed, - )); - } - } - Ok(()) -} diff --git a/xtask/src/durability_crash_matrix/restart/migration_expectation.rs b/xtask/src/durability_crash_matrix/restart/migration_expectation.rs deleted file mode 100644 index 6a54ed9f..00000000 --- a/xtask/src/durability_crash_matrix/restart/migration_expectation.rs +++ /dev/null @@ -1,246 +0,0 @@ -//! This module owns the independent expected state after migration process -//! death: the exact root inventory and the one lawful recovery plan. -//! -//! The rules here restate the recovery table in -//! `docs/formats/segment-store-v2/migration-recovery.md` without reading any -//! production classifier, so a planner defect cannot hide behind itself. - -use std::collections::BTreeSet; - -use keep::{StoreMigrationFixedStage, StoreMigrationPhase, StoreMigrationRecoveryPlan as Plan}; -use xtask::{ - DurabilityCrashCase, DurabilityCrashCaseError, DurabilityCrashPoint, DurabilityCrashPosition, -}; - -use crate::durability_crash_matrix::DurabilityCrashMatrixError; -use crate::durability_crash_matrix::production_protocol::fixture::{ - CATALOG_POOL_PATH, SEGMENT_POOL_PATH, -}; - -pub(super) const INTENT_STAGE: &str = "migration.intent.next"; -pub(super) const INTENT: &str = "migration.intent"; -pub(super) const READER_LOCK: &str = "reader.lock"; -pub(super) const MARKER_STAGE: &str = "FORMAT.next"; -pub(super) const MARKER: &str = "FORMAT"; -pub(super) const RECEIPT_STAGE: &str = "migration.receipt.next"; -pub(super) const RECEIPT: &str = "migration.receipt"; - -/// The namespace prefix in admission order; each entry is one `during` -/// occurrence of `KEEP-CRASH-060`. -const PREFIX_DIRECTORIES: [&str; 6] = [ - "retention", - "retention/roots", - "retention/manifests", - "gc", - "recovery", - "recovery/dispositions", -]; - -pub(super) struct MigrationExpectation { - paths: BTreeSet, - plan: Plan, -} - -impl MigrationExpectation { - pub(super) fn for_case(case: DurabilityCrashCase) -> Result { - let step = migration_step(case.point())?; - let partial = partial_stage(case); - let done = if case.position() == DurabilityCrashPosition::After - || (case.position() == DurabilityCrashPosition::During && atomic(case.point())) - { - step.saturating_add(1) - } else { - step - }; - let prefix_reached = prefix_reached(case)?; - let mut paths = version_one_paths(); - if (1..5).contains(&done) { - paths.insert(INTENT_STAGE.into()); - } - if done >= 3 { - paths.insert(INTENT.into()); - } - if done >= 7 { - paths.insert(READER_LOCK.into()); - } - let directories = if done >= 8 { - PREFIX_DIRECTORIES.len() - } else { - prefix_reached.unwrap_or(0) - }; - paths.extend( - PREFIX_DIRECTORIES - .iter() - .take(directories) - .map(|path| (*path).into()), - ); - if (10..14).contains(&done) { - paths.insert(MARKER_STAGE.into()); - } - if done >= 12 { - paths.insert(MARKER.into()); - } - if (16..20).contains(&done) { - paths.insert(RECEIPT_STAGE.into()); - } - if done >= 18 { - paths.insert(RECEIPT.into()); - } - if let Some(stage) = partial { - paths.insert(stage_name(stage).into()); - } - let plan = partial.map_or_else( - || plan_after(done, prefix_reached), - |stage| Plan::DiscardStage { - stage, - resume: write_phase(stage), - }, - ); - Ok(Self { paths, plan }) - } - - pub(super) const fn paths(&self) -> &BTreeSet { - &self.paths - } - - pub(super) const fn plan(&self) -> Plan { - self.plan - } -} - -/// The exact version-1 store the migration starts from. -pub(super) fn version_one_paths() -> BTreeSet { - [ - "writer.lock", - "staging", - "segments", - "catalogs", - "HEAD", - SEGMENT_POOL_PATH, - CATALOG_POOL_PATH, - ] - .into_iter() - .map(Into::into) - .collect() -} - -/// The exact root after one complete migration: no stage remains. -pub(super) fn complete_paths() -> BTreeSet { - let mut paths = version_one_paths(); - paths.extend( - [INTENT, READER_LOCK, MARKER, RECEIPT] - .into_iter() - .chain(PREFIX_DIRECTORIES) - .map(Into::into), - ); - paths -} - -fn migration_step(point: DurabilityCrashPoint) -> Result { - DurabilityCrashPoint::MIGRATION - .into_iter() - .position(|candidate| candidate == point) - .ok_or(DurabilityCrashMatrixError::PointSequenceMismatch { point }) -} - -/// Boundaries whose effect is one link, unlink, or exclusive creation, so -/// `during` cannot leave a partial effect. -const fn atomic(point: DurabilityCrashPoint) -> bool { - matches!( - point, - DurabilityCrashPoint::MigrationLinkIntent - | DurabilityCrashPoint::MigrationRemoveIntentStage - | DurabilityCrashPoint::MigrationAdmitReaderFence - | DurabilityCrashPoint::MigrationLinkMarker - | DurabilityCrashPoint::MigrationRemoveMarkerStage - | DurabilityCrashPoint::MigrationLinkReceipt - | DurabilityCrashPoint::MigrationRemoveReceiptStage - ) -} - -/// Process death during a stage write leaves an incomplete pre-effect stage. -fn partial_stage(case: DurabilityCrashCase) -> Option { - if case.position() != DurabilityCrashPosition::During { - return None; - } - match case.point() { - DurabilityCrashPoint::MigrationWriteIntentStage => Some(StoreMigrationFixedStage::Intent), - DurabilityCrashPoint::MigrationWriteMarkerStage => Some(StoreMigrationFixedStage::Marker), - DurabilityCrashPoint::MigrationWriteReceiptStage => Some(StoreMigrationFixedStage::Receipt), - _ => None, - } -} - -/// Process death during namespace admission leaves the first `occurrence + 1` -/// prefix directories, each with its parent synchronized. -fn prefix_reached(case: DurabilityCrashCase) -> Result, DurabilityCrashMatrixError> { - if case.point() != DurabilityCrashPoint::MigrationAdmitNamespacePrefix - || case.position() != DurabilityCrashPosition::During - { - return Ok(None); - } - let point = case.point(); - let observed = case - .occurrence() - .ok_or(DurabilityCrashMatrixError::InvalidCase( - DurabilityCrashCaseError::MissingOccurrence { point }, - ))?; - let reached = observed - .get() - .checked_add(1) - .ok_or(DurabilityCrashMatrixError::InvalidCase( - DurabilityCrashCaseError::OccurrenceOutOfRange { - point, - observed, - exclusive_limit: point.during_occurrences(), - }, - ))?; - usize::try_from(reached) - .map(Some) - .map_err(|source| DurabilityCrashMatrixError::Verification { - phase: "represent reached migration namespace prefix", - source: Box::new(source), - }) -} - -const fn stage_name(stage: StoreMigrationFixedStage) -> &'static str { - match stage { - StoreMigrationFixedStage::Intent => INTENT_STAGE, - StoreMigrationFixedStage::Marker => MARKER_STAGE, - StoreMigrationFixedStage::Receipt => RECEIPT_STAGE, - } -} - -const fn write_phase(stage: StoreMigrationFixedStage) -> StoreMigrationPhase { - match stage { - StoreMigrationFixedStage::Intent => StoreMigrationPhase::WriteIntentStage, - StoreMigrationFixedStage::Marker => StoreMigrationPhase::WriteMarkerStage, - StoreMigrationFixedStage::Receipt => StoreMigrationPhase::WriteReceiptStage, - } -} - -/// The recovery-table row for a residue in which the first `done` phases -/// completed and nothing else happened: resume at the earliest phase the -/// residue cannot prove. -const fn plan_after(done: usize, prefix_reached: Option) -> Plan { - let resume = match done { - 0 => return Plan::VersionOne, - 1 | 2 => StoreMigrationPhase::SynchronizeIntentStage, - 3 | 4 => StoreMigrationPhase::SynchronizeRootAfterIntent, - 5 | 6 => StoreMigrationPhase::SynchronizeRootAfterIntentCleanup, - 7 => match prefix_reached { - Some(reached) if reached == PREFIX_DIRECTORIES.len() => { - StoreMigrationPhase::SynchronizeRootAfterNamespace - } - _ => StoreMigrationPhase::AdmitNamespacePrefix, - }, - 8 | 9 => StoreMigrationPhase::SynchronizeRootAfterNamespace, - 10 | 11 => StoreMigrationPhase::SynchronizeMarkerStage, - 12 | 13 => StoreMigrationPhase::SynchronizeRootAfterMarker, - 14 | 15 => StoreMigrationPhase::SynchronizeRootAfterMarkerCleanup, - 16 | 17 => StoreMigrationPhase::SynchronizeReceiptStage, - 18 | 19 => StoreMigrationPhase::SynchronizeRootAfterReceipt, - _ => return Plan::Complete, - }; - Plan::Resume { resume } -} diff --git a/xtask/src/durability_crash_matrix/restart/retention.rs b/xtask/src/durability_crash_matrix/restart/retention.rs index 9538059b..0876fbb1 100644 --- a/xtask/src/durability_crash_matrix/restart/retention.rs +++ b/xtask/src/durability_crash_matrix/restart/retention.rs @@ -1,6 +1,6 @@ //! This module owns post-process-death verification of retention publication. //! -//! After the child dies at its coordinate, restart reopens the migrated store +//! After the child dies at its coordinate, restart reopens the initialized store //! through the same admission a production caller would use, runs retention //! recovery, and requires the documented steps and outcome for that exact //! prefix; then it independently reads the persistent snapshot before requiring diff --git a/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs b/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs index 74961c9f..1a089fb8 100644 --- a/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs +++ b/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs @@ -3,7 +3,7 @@ //! After the child dies at its coordinate, restart requires the exact stage //! residue that coordinate leaves, recovers it with the public version-two //! recovery tools, republishes when the candidate never reached `HEAD`, and -//! then reads both the migrated retained blob and the ingested blob through +//! then reads both the retained bundle blob and the ingested blob through //! the production durable reader. use std::fs; @@ -205,7 +205,7 @@ fn retain_blob(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { .map_err(|source| verification("publish crash blob retention", source)) } -/// Reads the migrated retained blob and the ingested blob through the durable reader. +/// Reads the retained bundle blob and the ingested blob through the durable reader. fn read_back(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { let store = DurableStore::open(store_root, policy()?, ReaderAttemptLimit::DEFAULT) .map_err(|source| verification("open recovered durable store", source))?; diff --git a/xtask/src/durability_crash_matrix/restart/version_two_initialization.rs b/xtask/src/durability_crash_matrix/restart/version_two_initialization.rs new file mode 100644 index 00000000..d29efaa6 --- /dev/null +++ b/xtask/src/durability_crash_matrix/restart/version_two_initialization.rs @@ -0,0 +1,303 @@ +//! This module owns post-process-death verification of direct initialization. +//! +//! After the child dies at its coordinate, restart requires the exact entry +//! inventory and marker bytes that coordinate leaves and the exact admission +//! refusal or success, then runs the public initializer again as recovery and +//! requires the state it reports. It then reopens the store, publishes the +//! bundle as the first catalog through the version-two writer, retains it, and +//! reads it back through the production durable reader. + +use std::collections::BTreeSet; +use std::fs; +use std::io; +use std::os::unix::fs::MetadataExt; +use std::path::Path; + +use keep::{ + BlobId, CanonicalStoreFormatMarker, DurableStore, FilesystemNamespaceRefusal, + FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, ReaderAttemptLimit, + VersionTwoInitializationDirectory as Directory, VersionTwoInitializationError, + VersionTwoInitializationPrefix as Prefix, VersionTwoInitializationRefusal, + execute_retention_publication, +}; +use xtask::{ + DurabilityCrashCase, DurabilityCrashOccurrence, DurabilityCrashPoint as Point, + DurabilityCrashPosition, +}; + +use super::super::DurabilityCrashMatrixError; +use super::super::production_protocol::crash_blob::CrashBlob; +use super::super::production_protocol::fixture::GoldenFixture; +use super::super::production_protocol::populated_store; +use super::super::production_protocol::retention::{preparation, reopened_authority}; +use super::super::production_protocol::verification; +use super::super::production_protocol::version_two_catalog::policy; +use super::super::production_protocol::version_two_initialization::INCOMPLETE_STAGE_LENGTH; +use super::inventory; + +/// The protocol state one coordinate leaves behind. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum State { + Empty, + Locked, + IncompleteStage, + Staged, + Fenced, + Directories(usize), + Linked, + Committed, +} + +pub(super) fn verify( + store_root: &Path, + case: DurabilityCrashCase, +) -> Result<(), DurabilityCrashMatrixError> { + let state = state(case)?; + require_inventory(store_root, state)?; + require_marker_bytes(store_root, state)?; + require_reopen(store_root, state)?; + let resumed = match FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( + store_root, + ) { + Ok(initialized) => Some(initialized.found()), + Err(VersionTwoInitializationError::Refused { + refusal: VersionTwoInitializationRefusal::AlreadyInitialized, + }) => None, + Err(source) => return Err(verification("recover interrupted initialization", source)), + }; + let expected = resumes_from(state)?; + if resumed != expected { + return Err(DurabilityCrashMatrixError::InitializationPrefixMismatch { + expected, + observed: resumed, + }); + } + require_inventory(store_root, State::Committed)?; + publish_and_read(store_root) +} + +fn state(case: DurabilityCrashCase) -> Result { + use DurabilityCrashPosition::{After, Before, During}; + let position = case.position(); + Ok(match case.point() { + Point::InitializeOpenAndLockWriterFile => match position { + Before => State::Empty, + During | After => State::Locked, + }, + Point::InitializeWriteMarkerStage => match position { + Before => State::Locked, + During => State::IncompleteStage, + After => State::Staged, + }, + Point::InitializeSynchronizeMarkerStage + | Point::InitializeSynchronizeRootAfterMarkerStage => State::Staged, + Point::InitializeAdmitReaderFence => match position { + Before => State::Staged, + During | After => State::Fenced, + }, + Point::InitializeAdmitDirectory => directory_state(case)?, + Point::InitializeLinkMarker => match position { + Before => State::Directories(Directory::ALL.len()), + During | After => State::Linked, + }, + Point::InitializeSynchronizeRootAfterMarker => State::Linked, + Point::InitializeRemoveMarkerStage => match position { + Before => State::Linked, + During | After => State::Committed, + }, + Point::InitializeSynchronizeRootAfterCleanup => State::Committed, + Point::InitializeDiscardIncompleteMarkerStage => match position { + Before => State::IncompleteStage, + During | After => State::Locked, + }, + point => return Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), + }) +} + +fn directory_state(case: DurabilityCrashCase) -> Result { + Ok(match case.position() { + DurabilityCrashPosition::Before => State::Fenced, + DurabilityCrashPosition::After => State::Directories(Directory::ALL.len()), + DurabilityCrashPosition::During => { + let occurrence = case.occurrence().map_or(0, DurabilityCrashOccurrence::get); + let reached = usize::try_from(occurrence) + .ok() + .and_then(|ordinal| ordinal.checked_add(1)) + .ok_or(DurabilityCrashMatrixError::FixtureRange)?; + State::Directories(reached) + } + }) +} + +fn resumes_from(state: State) -> Result, DurabilityCrashMatrixError> { + Ok(Some(match state { + State::Empty => Prefix::Empty, + State::Locked => Prefix::Locked, + State::IncompleteStage => Prefix::IncompleteMarkerStage { + length: INCOMPLETE_STAGE_LENGTH, + }, + State::Staged => Prefix::MarkerStaged, + State::Fenced => Prefix::ReaderFenced, + State::Directories(count) => Prefix::Directories { + last: *count + .checked_sub(1) + .and_then(|index| Directory::ALL.get(index)) + .ok_or(DurabilityCrashMatrixError::FixtureRange)?, + }, + State::Linked => Prefix::MarkerLinked, + State::Committed => return Ok(None), + })) +} + +fn expected_paths(state: State) -> BTreeSet { + let directories = match state { + State::Directories(count) => count, + State::Linked | State::Committed => Directory::ALL.len(), + _ => 0, + }; + let mut paths: BTreeSet = Directory::ALL + .iter() + .take(directories) + .map(|directory| directory.path().to_owned()) + .collect(); + let files: &[&str] = match state { + State::Empty => &[], + State::Locked => &["writer.lock"], + State::IncompleteStage | State::Staged => &["writer.lock", "FORMAT.next"], + State::Fenced | State::Directories(_) => &["writer.lock", "FORMAT.next", "reader.lock"], + State::Linked => &["writer.lock", "FORMAT.next", "reader.lock", "FORMAT"], + State::Committed => &["writer.lock", "reader.lock", "FORMAT"], + }; + paths.extend(files.iter().map(|file| (*file).to_owned())); + paths +} + +fn require_inventory(store_root: &Path, state: State) -> Result<(), DurabilityCrashMatrixError> { + let observed = inventory(store_root)?; + let expected = expected_paths(state); + if observed == expected { + Ok(()) + } else { + Err(DurabilityCrashMatrixError::InventoryMismatch { expected, observed }) + } +} + +/// Requires the stage and marker bytes, and one inode when both names exist. +fn require_marker_bytes(store_root: &Path, state: State) -> Result<(), DurabilityCrashMatrixError> { + let marker = CanonicalStoreFormatMarker::version_two(); + let canonical = marker.encoded(); + let expected_stage = match state { + State::IncompleteStage => canonical.get(..INCOMPLETE_STAGE_LENGTH), + State::Staged | State::Fenced | State::Directories(_) | State::Linked => Some(canonical), + _ => None, + }; + if let Some(expected) = expected_stage { + require_bytes(store_root, "FORMAT.next", expected)?; + } + if matches!(state, State::Linked | State::Committed) { + require_bytes(store_root, "FORMAT", canonical)?; + } + if state == State::Linked { + let stage_entry = metadata(store_root, "FORMAT.next")?; + let marker_entry = metadata(store_root, "FORMAT")?; + if (stage_entry.dev(), stage_entry.ino()) != (marker_entry.dev(), marker_entry.ino()) { + return Err(mismatch("FORMAT and FORMAT.next are different inodes")); + } + } + Ok(()) +} + +fn require_bytes( + store_root: &Path, + name: &'static str, + expected: &[u8], +) -> Result<(), DurabilityCrashMatrixError> { + let observed = fs::read(store_root.join(name)) + .map_err(|source| DurabilityCrashMatrixError::io("read initialization record", source))?; + if observed == expected { + Ok(()) + } else { + Err(DurabilityCrashMatrixError::artifact_bytes( + name, expected, &observed, + )) + } +} + +fn metadata(store_root: &Path, name: &str) -> Result { + fs::symlink_metadata(store_root.join(name)) + .map_err(|source| DurabilityCrashMatrixError::io("inspect initialization record", source)) +} + +/// Requires the exact admission outcome: refusal before commit, success after. +fn require_reopen(store_root: &Path, state: State) -> Result<(), DurabilityCrashMatrixError> { + let reopened = FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store_root); + let expected = match state { + State::Empty => { + return match reopened { + Err(FilesystemPlatformAdmissionError::WriterLock { .. }) => Ok(()), + Ok(_) => Err(mismatch("an empty root reopened")), + Err(other) => Err(mismatch(format!("empty root reopened as {other:?}"))), + }; + } + State::Committed => { + return reopened + .map(drop) + .map_err(|source| verification("reopen committed store", source)); + } + State::Locked => FilesystemNamespaceRefusal::MissingFormatMarker, + _ => FilesystemNamespaceRefusal::InitializationIncomplete, + }; + let observed = match &reopened { + Err(FilesystemPlatformAdmissionError::Namespace { source }) => source + .get_ref() + .and_then(|cause| cause.downcast_ref::()) + .copied(), + _ => None, + }; + if observed == Some(expected) { + Ok(()) + } else { + Err(mismatch(format!( + "uncommitted root reopened as {:?}, expected {expected:?}", + reopened.err() + ))) + } +} + +/// Publishes and retains the bundle, then reconstructs its exact bytes. +fn publish_and_read(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let admission = + FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("reopen recovered store", source))?; + populated_store::publish_bundle_catalog(store_root, admission)?; + populated_store::require_frozen_bundle(store_root)?; + let mut authority = reopened_authority(store_root)?; + let root = GoldenFixture::retention_root()?; + let prepared = preparation(root.bytes())?; + execute_retention_publication(&mut authority, &prepared) + .map(drop) + .map_err(|source| verification("retain recovered bundle", source))?; + drop(authority); + let store = DurableStore::open(store_root, policy()?, ReaderAttemptLimit::DEFAULT) + .map_err(|source| verification("open recovered durable store", source))?; + let bundle = CrashBlob::bundle()?; + let mut output = Vec::new(); + store + .reconstruct(bundle.target(), &mut output) + .map(drop) + .map_err(|source| verification("reconstruct recovered bundle", source))?; + let observed = BlobId::hash_bytes(&output) + .map_err(|source| verification("identify recovered bundle", source))?; + if observed == bundle.target() && output == [0] { + Ok(()) + } else { + Err(mismatch("recovered bundle differs from its content")) + } +} + +fn mismatch(message: impl Into) -> DurabilityCrashMatrixError { + verification( + "verify initialization crash recovery", + io::Error::other(message.into()), + ) +} diff --git a/xtask/src/durability_crash_point.rs b/xtask/src/durability_crash_point.rs index bbcff405..1ba8acfa 100644 --- a/xtask/src/durability_crash_point.rs +++ b/xtask/src/durability_crash_point.rs @@ -15,8 +15,9 @@ pub enum DurabilityCrashSequence { Initialization, /// Version-two retention publication, `KEEP-CRASH-036` through `052`. Retention, - /// One-way version-1 to version-2 store migration. - Migration, + /// Direct version-two initialization and its recovery, `KEEP-CRASH-081` + /// through `091`. + VersionTwoInitialization, /// Version-two catalog publication and its stage recovery, `KEEP-CRASH-074` /// through `077`. VersionTwoCatalog, @@ -31,8 +32,8 @@ impl DurabilityCrashSequence { Self::RecoveryDiscard, Self::Initialization, Self::Retention, - Self::Migration, Self::VersionTwoCatalog, + Self::VersionTwoInitialization, ]; /// Returns the stable identifier used by the crash-matrix command line. @@ -45,8 +46,8 @@ impl DurabilityCrashSequence { Self::RecoveryDiscard => "recovery-discard", Self::Initialization => "initialization", Self::Retention => "retention", - Self::Migration => "migration", Self::VersionTwoCatalog => "version-two-catalog", + Self::VersionTwoInitialization => "version-two-initialization", } } @@ -166,49 +167,6 @@ pub enum DurabilityCrashPoint { RemoveManifestStage, /// Retention cleanup synchronization. SynchronizeRetentionCleanup, - /// Write the complete canonical `migration.intent.next`. - MigrationWriteIntentStage, - /// Synchronize `migration.intent.next`. - MigrationSynchronizeIntentStage, - /// Link the synchronized intent stage to `migration.intent`. - MigrationLinkIntent, - /// Synchronize the store root after the intent link. - MigrationSynchronizeRootAfterIntent, - /// Remove the retained `migration.intent.next`. - MigrationRemoveIntentStage, - /// Synchronize the store root after intent-stage cleanup. - MigrationSynchronizeRootAfterIntentCleanup, - /// Create or exactly admit the persistent reader fence. - MigrationAdmitReaderFence, - /// Create or exactly admit the canonical version-2 directory prefix; the - /// occurrence names the directory-prefix length reached. - MigrationAdmitNamespacePrefix, - /// Synchronize the store root after namespace admission. - MigrationSynchronizeRootAfterNamespace, - /// Write the complete canonical `FORMAT.next`. - MigrationWriteMarkerStage, - /// Synchronize `FORMAT.next`. - MigrationSynchronizeMarkerStage, - /// Link the synchronized marker stage to `FORMAT`. - MigrationLinkMarker, - /// Synchronize the store root after the marker link. - MigrationSynchronizeRootAfterMarker, - /// Remove the retained `FORMAT.next`. - MigrationRemoveMarkerStage, - /// Synchronize the store root after marker-stage cleanup. - MigrationSynchronizeRootAfterMarkerCleanup, - /// Write the complete canonical `migration.receipt.next`. - MigrationWriteReceiptStage, - /// Synchronize `migration.receipt.next`. - MigrationSynchronizeReceiptStage, - /// Link the synchronized receipt stage to `migration.receipt`. - MigrationLinkReceipt, - /// Synchronize the store root after the receipt link. - MigrationSynchronizeRootAfterReceipt, - /// Remove the retained `migration.receipt.next`. - MigrationRemoveReceiptStage, - /// Synchronize the store root after receipt-stage cleanup. - MigrationSynchronizeRootAfterReceiptCleanup, /// Link the version-two writer's sealed segment into the immutable pool. VersionTwoLinkSegment, /// Exclusively create the version-two writer's `staging/current.cat`. @@ -217,11 +175,36 @@ pub enum DurabilityCrashPoint { VersionTwoSynchronizeHead, /// Replace `HEAD` after the version-two retained-closure revalidation. VersionTwoReplaceHead, + /// Create or open `writer.lock`, lock it, and synchronize it and the root. + InitializeOpenAndLockWriterFile, + /// Exclusively create `FORMAT.next` and write the canonical marker; the + /// `during` case leaves a strict prefix. + InitializeWriteMarkerStage, + /// Synchronize `FORMAT.next`. + InitializeSynchronizeMarkerStage, + /// Synchronize the root after the marker stage. + InitializeSynchronizeRootAfterMarkerStage, + /// Create or admit `reader.lock` and synchronize the root. + InitializeAdmitReaderFence, + /// Create or admit one protocol directory and synchronize its parent; the + /// occurrence names the directory reached. + InitializeAdmitDirectory, + /// Link `FORMAT.next` to `FORMAT` without replacement. + InitializeLinkMarker, + /// Synchronize the root after the link: the commit point. + InitializeSynchronizeRootAfterMarker, + /// Remove `FORMAT.next` after the commit. + InitializeRemoveMarkerStage, + /// Synchronize the root after stage removal. + InitializeSynchronizeRootAfterCleanup, + /// Discard an incomplete `FORMAT.next` while recovering an interrupted + /// initialization. + InitializeDiscardIncompleteMarkerStage, } impl DurabilityCrashPoint { /// Every crash boundary in stable protocol order. - pub const ALL: [Self; 77] = [ + pub const ALL: [Self; 67] = [ Self::CreateSegmentStage, Self::WriteSegmentHeader, Self::AppendSegmentRecord, @@ -274,62 +257,26 @@ impl DurabilityCrashPoint { Self::RemoveRootStage, Self::RemoveManifestStage, Self::SynchronizeRetentionCleanup, - Self::MigrationWriteIntentStage, - Self::MigrationSynchronizeIntentStage, - Self::MigrationLinkIntent, - Self::MigrationSynchronizeRootAfterIntent, - Self::MigrationRemoveIntentStage, - Self::MigrationSynchronizeRootAfterIntentCleanup, - Self::MigrationAdmitReaderFence, - Self::MigrationAdmitNamespacePrefix, - Self::MigrationSynchronizeRootAfterNamespace, - Self::MigrationWriteMarkerStage, - Self::MigrationSynchronizeMarkerStage, - Self::MigrationLinkMarker, - Self::MigrationSynchronizeRootAfterMarker, - Self::MigrationRemoveMarkerStage, - Self::MigrationSynchronizeRootAfterMarkerCleanup, - Self::MigrationWriteReceiptStage, - Self::MigrationSynchronizeReceiptStage, - Self::MigrationLinkReceipt, - Self::MigrationSynchronizeRootAfterReceipt, - Self::MigrationRemoveReceiptStage, - Self::MigrationSynchronizeRootAfterReceiptCleanup, Self::VersionTwoLinkSegment, Self::VersionTwoCreateCatalogStage, Self::VersionTwoSynchronizeHead, Self::VersionTwoReplaceHead, + Self::InitializeOpenAndLockWriterFile, + Self::InitializeWriteMarkerStage, + Self::InitializeSynchronizeMarkerStage, + Self::InitializeSynchronizeRootAfterMarkerStage, + Self::InitializeAdmitReaderFence, + Self::InitializeAdmitDirectory, + Self::InitializeLinkMarker, + Self::InitializeSynchronizeRootAfterMarker, + Self::InitializeRemoveMarkerStage, + Self::InitializeSynchronizeRootAfterCleanup, + Self::InitializeDiscardIncompleteMarkerStage, ]; - /// The migration boundaries in `StoreMigrationPhase::ALL` order. - pub const MIGRATION: [Self; 21] = [ - Self::MigrationWriteIntentStage, - Self::MigrationSynchronizeIntentStage, - Self::MigrationLinkIntent, - Self::MigrationSynchronizeRootAfterIntent, - Self::MigrationRemoveIntentStage, - Self::MigrationSynchronizeRootAfterIntentCleanup, - Self::MigrationAdmitReaderFence, - Self::MigrationAdmitNamespacePrefix, - Self::MigrationSynchronizeRootAfterNamespace, - Self::MigrationWriteMarkerStage, - Self::MigrationSynchronizeMarkerStage, - Self::MigrationLinkMarker, - Self::MigrationSynchronizeRootAfterMarker, - Self::MigrationRemoveMarkerStage, - Self::MigrationSynchronizeRootAfterMarkerCleanup, - Self::MigrationWriteReceiptStage, - Self::MigrationSynchronizeReceiptStage, - Self::MigrationLinkReceipt, - Self::MigrationSynchronizeRootAfterReceipt, - Self::MigrationRemoveReceiptStage, - Self::MigrationSynchronizeRootAfterReceiptCleanup, - ]; - - /// The number of directories the migration namespace prefix admits; each - /// is one `during` occurrence of - /// [`Self::MigrationAdmitNamespacePrefix`]. - pub const NAMESPACE_PREFIX_DIRECTORIES: u32 = 6; + /// The number of protocol directories initialization creates; each is one + /// `during` occurrence of [`Self::InitializeAdmitDirectory`]. + pub const INITIALIZATION_DIRECTORIES: u32 = 9; /// Parses one exact stable crash identifier. #[must_use] @@ -395,28 +342,18 @@ impl DurabilityCrashPoint { | Self::RemoveRootStage | Self::RemoveManifestStage | Self::SynchronizeRetentionCleanup => DurabilityCrashSequence::Retention, - Self::MigrationWriteIntentStage - | Self::MigrationSynchronizeIntentStage - | Self::MigrationLinkIntent - | Self::MigrationSynchronizeRootAfterIntent - | Self::MigrationRemoveIntentStage - | Self::MigrationSynchronizeRootAfterIntentCleanup - | Self::MigrationAdmitReaderFence - | Self::MigrationAdmitNamespacePrefix - | Self::MigrationSynchronizeRootAfterNamespace - | Self::MigrationWriteMarkerStage - | Self::MigrationSynchronizeMarkerStage - | Self::MigrationLinkMarker - | Self::MigrationSynchronizeRootAfterMarker - | Self::MigrationRemoveMarkerStage - | Self::MigrationSynchronizeRootAfterMarkerCleanup - | Self::MigrationWriteReceiptStage - | Self::MigrationSynchronizeReceiptStage - | Self::MigrationLinkReceipt - | Self::MigrationSynchronizeRootAfterReceipt - | Self::MigrationRemoveReceiptStage - | Self::MigrationSynchronizeRootAfterReceiptCleanup => { - DurabilityCrashSequence::Migration + Self::InitializeOpenAndLockWriterFile + | Self::InitializeWriteMarkerStage + | Self::InitializeSynchronizeMarkerStage + | Self::InitializeSynchronizeRootAfterMarkerStage + | Self::InitializeAdmitReaderFence + | Self::InitializeAdmitDirectory + | Self::InitializeLinkMarker + | Self::InitializeSynchronizeRootAfterMarker + | Self::InitializeRemoveMarkerStage + | Self::InitializeSynchronizeRootAfterCleanup + | Self::InitializeDiscardIncompleteMarkerStage => { + DurabilityCrashSequence::VersionTwoInitialization } Self::VersionTwoLinkSegment | Self::VersionTwoCreateCatalogStage @@ -430,17 +367,17 @@ impl DurabilityCrashPoint { pub const fn occurrence_counted(self) -> bool { matches!( self, - Self::AppendSegmentRecord | Self::MigrationAdmitNamespacePrefix + Self::AppendSegmentRecord | Self::InitializeAdmitDirectory ) } /// Returns how many distinct `during` occurrences the canonical matrix - /// runs for this boundary: one directory-prefix length per occurrence - /// for the migration namespace prefix, otherwise exactly one. + /// runs for this boundary: one directory per occurrence for initialization + /// directory admission, otherwise exactly one. #[must_use] pub const fn during_occurrences(self) -> u32 { match self { - Self::MigrationAdmitNamespacePrefix => Self::NAMESPACE_PREFIX_DIRECTORIES, + Self::InitializeAdmitDirectory => Self::INITIALIZATION_DIRECTORIES, _ => 1, } } diff --git a/xtask/src/durability_crash_point_identity.rs b/xtask/src/durability_crash_point_identity.rs index 0316962e..89bf2bd7 100644 --- a/xtask/src/durability_crash_point_identity.rs +++ b/xtask/src/durability_crash_point_identity.rs @@ -59,31 +59,21 @@ impl DurabilityCrashPoint { Self::RemoveRootStage => "KEEP-CRASH-050", Self::RemoveManifestStage => "KEEP-CRASH-051", Self::SynchronizeRetentionCleanup => "KEEP-CRASH-052", - Self::MigrationWriteIntentStage => "KEEP-CRASH-053", - Self::MigrationSynchronizeIntentStage => "KEEP-CRASH-054", - Self::MigrationLinkIntent => "KEEP-CRASH-055", - Self::MigrationSynchronizeRootAfterIntent => "KEEP-CRASH-056", - Self::MigrationRemoveIntentStage => "KEEP-CRASH-057", - Self::MigrationSynchronizeRootAfterIntentCleanup => "KEEP-CRASH-058", - Self::MigrationAdmitReaderFence => "KEEP-CRASH-059", - Self::MigrationAdmitNamespacePrefix => "KEEP-CRASH-060", - Self::MigrationSynchronizeRootAfterNamespace => "KEEP-CRASH-061", - Self::MigrationWriteMarkerStage => "KEEP-CRASH-062", - Self::MigrationSynchronizeMarkerStage => "KEEP-CRASH-063", - Self::MigrationLinkMarker => "KEEP-CRASH-064", - Self::MigrationSynchronizeRootAfterMarker => "KEEP-CRASH-065", - Self::MigrationRemoveMarkerStage => "KEEP-CRASH-066", - Self::MigrationSynchronizeRootAfterMarkerCleanup => "KEEP-CRASH-067", - Self::MigrationWriteReceiptStage => "KEEP-CRASH-068", - Self::MigrationSynchronizeReceiptStage => "KEEP-CRASH-069", - Self::MigrationLinkReceipt => "KEEP-CRASH-070", - Self::MigrationSynchronizeRootAfterReceipt => "KEEP-CRASH-071", - Self::MigrationRemoveReceiptStage => "KEEP-CRASH-072", - Self::MigrationSynchronizeRootAfterReceiptCleanup => "KEEP-CRASH-073", Self::VersionTwoLinkSegment => "KEEP-CRASH-074", Self::VersionTwoCreateCatalogStage => "KEEP-CRASH-075", Self::VersionTwoSynchronizeHead => "KEEP-CRASH-076", Self::VersionTwoReplaceHead => "KEEP-CRASH-077", + Self::InitializeOpenAndLockWriterFile => "KEEP-CRASH-081", + Self::InitializeWriteMarkerStage => "KEEP-CRASH-082", + Self::InitializeSynchronizeMarkerStage => "KEEP-CRASH-083", + Self::InitializeSynchronizeRootAfterMarkerStage => "KEEP-CRASH-084", + Self::InitializeAdmitReaderFence => "KEEP-CRASH-085", + Self::InitializeAdmitDirectory => "KEEP-CRASH-086", + Self::InitializeLinkMarker => "KEEP-CRASH-087", + Self::InitializeSynchronizeRootAfterMarker => "KEEP-CRASH-088", + Self::InitializeRemoveMarkerStage => "KEEP-CRASH-089", + Self::InitializeSynchronizeRootAfterCleanup => "KEEP-CRASH-090", + Self::InitializeDiscardIncompleteMarkerStage => "KEEP-CRASH-091", } } } diff --git a/xtask/src/fuzz_campaign/target/tests.rs b/xtask/src/fuzz_campaign/target/tests.rs index 65303aff..2a60ae2c 100644 --- a/xtask/src/fuzz_campaign/target/tests.rs +++ b/xtask/src/fuzz_campaign/target/tests.rs @@ -29,9 +29,9 @@ fn checked_in_harness_set_is_exact_and_sorted() -> Result<(), Box> { "blob_id_text", "catalog_format", "fast_cdc", + "format_marker", "golden_protocol", "layout_record", - "migration_format", "repository_json", "retention_format", "segment_format", diff --git a/xtask/src/fuzz_seed_corpus.rs b/xtask/src/fuzz_seed_corpus.rs index 48dd7d20..7f416ac1 100644 --- a/xtask/src/fuzz_seed_corpus.rs +++ b/xtask/src/fuzz_seed_corpus.rs @@ -4,10 +4,9 @@ mod benchmark_report_seeds; mod catalog_seeds; mod cdc_seeds; mod filesystem; +mod format_marker_seeds; mod identity_seeds; mod layout_seeds; -mod migration_recovery_seeds; -mod migration_seeds; mod retention_seeds; mod segment_seeds; mod segment_store_v2_fixture; @@ -75,7 +74,7 @@ pub(super) fn prepare(repository_root: &Path) -> Result<(), FuzzSeedError> { seeds.extend(cdc_seeds::seeds()?); seeds.extend(golden_protocol_seeds_from(&files)?); seeds.extend(layout_seeds::seeds(&files)?); - seeds.extend(migration_seeds::seeds(&files)?); + seeds.extend(format_marker_seeds::seeds(&files)?); seeds.extend(retention_seeds::seeds(&files)?); seeds.extend(segment_seeds::seeds(&files)?); files.write_seeds(&seeds) diff --git a/xtask/src/fuzz_seed_corpus/format_marker_seeds.rs b/xtask/src/fuzz_seed_corpus/format_marker_seeds.rs new file mode 100644 index 00000000..43d1115e --- /dev/null +++ b/xtask/src/fuzz_seed_corpus/format_marker_seeds.rs @@ -0,0 +1,32 @@ +//! This module owns `FORMAT` marker fuzz seeds. + +use super::filesystem::RepositoryFiles; +use super::segment_store_v2_fixture; +use super::{FuzzSeedError, Seed}; + +const FORMAT_MARKER_FIXTURE: &str = "format-marker.hex"; + +/// Every version-two fixture the seeds read, with the selector the copy law keys on. +pub(super) const FIXTURES: [(u8, &str); 1] = [(0, FORMAT_MARKER_FIXTURE)]; + +pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> { + let [(_selector, fixture)] = FIXTURES; + let marker = segment_store_v2_fixture::read_hex(files, fixture)?; + let mut unsupported_version = marker.clone(); + *unsupported_version + .get_mut(17) + .ok_or_else(|| FuzzSeedError::violation("marker lacks version"))? = 3; + let mut corrupt_checksum = marker.clone(); + *corrupt_checksum + .last_mut() + .ok_or_else(|| FuzzSeedError::violation("marker lacks checksum"))? ^= 1; + Ok(vec![ + Seed::new("format_marker", "format-marker", marker)?, + Seed::new( + "format_marker", + "unsupported-marker-version", + unsupported_version, + )?, + Seed::new("format_marker", "corrupt-marker-checksum", corrupt_checksum)?, + ]) +} diff --git a/xtask/src/fuzz_seed_corpus/migration_recovery_seeds.rs b/xtask/src/fuzz_seed_corpus/migration_recovery_seeds.rs deleted file mode 100644 index 9f5e9f0d..00000000 --- a/xtask/src/fuzz_seed_corpus/migration_recovery_seeds.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! Migration recovery seeds encode valid prefixes and contradictory transition evidence. - -use super::{FuzzSeedError, Seed, prefixed}; - -pub(super) fn seeds( - marker: &[u8], - intent: &[u8], - receipt: &[u8], -) -> Result, FuzzSeedError> { - let records = [intent, intent, marker, marker, receipt, receipt]; - let mut seeds = Vec::new(); - for (name, presence, namespace) in [ - ("recovery-version-one", 0, 0), - ("recovery-intent-stage", 1, 0), - ("recovery-durable-intent", 2, 0), - ("recovery-partial-namespace", 2, 7), - ("recovery-full-namespace", 2, 127), - ("recovery-marker-stage", 6, 127), - ("recovery-marker", 10, 127), - ("recovery-receipt-stage", 26, 127), - ("recovery-complete", 42, 127), - ("recovery-effect-before-intent", 0, 1), - ("recovery-stage-after-effect", 3, 1), - ("recovery-namespace-hole", 2, 5), - ("recovery-receipt-before-marker", 18, 127), - ] { - seeds.push(seed(name, presence, namespace, intent, records)?); - } - let mut corrupt = intent.to_vec(); - let checksum = corrupt - .last_mut() - .ok_or_else(|| FuzzSeedError::violation("empty intent fixture"))?; - *checksum ^= 1; - seeds.push(seed( - "recovery-corrupt-intent", - 2, - 0, - intent, - [intent, &corrupt, marker, marker, receipt, receipt], - )?); - Ok(seeds) -} - -fn seed( - name: &'static str, - presence: u8, - namespace: u8, - expected: &[u8], - records: [&[u8]; 6], -) -> Result { - if expected.len() != 256 || records.iter().any(|record| record.len() > 513) { - return Err(FuzzSeedError::violation( - "recovery seed exceeds its record framing bounds", - )); - } - let mut payload = expected.to_vec(); - payload.extend_from_slice(&[presence, namespace]); - for record in records { - let length = u16::try_from(record.len()) - .map_err(|_| FuzzSeedError::violation("recovery seed record exceeds framing bound"))?; - payload.extend_from_slice(&length.to_le_bytes()); - payload.extend_from_slice(record); - } - Seed::new("migration_format", name, prefixed(3, &payload)?) -} diff --git a/xtask/src/fuzz_seed_corpus/migration_seeds.rs b/xtask/src/fuzz_seed_corpus/migration_seeds.rs deleted file mode 100644 index f2d2dc94..00000000 --- a/xtask/src/fuzz_seed_corpus/migration_seeds.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! This module owns canonical store-migration record fuzz seeds. - -use super::filesystem::RepositoryFiles; -use super::segment_store_v2_fixture; -use super::{FuzzSeedError, MAX_SEED_BYTES, Seed, prefixed}; - -const FORMAT_MARKER_FIXTURE: &str = "format-marker.hex"; -const MIGRATION_INTENT_FIXTURE: &str = "migration-intent.hex"; -const MIGRATION_RECEIPT_FIXTURE: &str = "migration-receipt.hex"; - -pub(super) const FIXTURES: [(u8, &str); 3] = [ - (0, FORMAT_MARKER_FIXTURE), - (1, MIGRATION_INTENT_FIXTURE), - (2, MIGRATION_RECEIPT_FIXTURE), -]; - -pub(super) fn seeds(files: &RepositoryFiles) -> Result, FuzzSeedError> { - let [ - (marker_selector, marker_fixture), - (intent_selector, intent_fixture), - (receipt_selector, receipt_fixture), - ] = FIXTURES; - let marker = segment_store_v2_fixture::read_hex(files, marker_fixture)?; - let intent = segment_store_v2_fixture::read_hex(files, intent_fixture)?; - let receipt = segment_store_v2_fixture::read_hex(files, receipt_fixture)?; - let mut seeds = vec![ - Seed::new( - "migration_format", - "format-marker", - prefixed(marker_selector, &marker)?, - )?, - Seed::new( - "migration_format", - "migration-intent", - prefixed(intent_selector, &intent)?, - )?, - Seed::new( - "migration_format", - "migration-receipt", - receipt_seed(receipt_selector, &marker, &intent, &receipt)?, - )?, - ]; - seeds.extend(malformed_seeds(&marker, &intent, &receipt)?); - seeds.extend(super::migration_recovery_seeds::seeds( - &marker, &intent, &receipt, - )?); - Ok(seeds) -} - -fn receipt_seed( - selector: u8, - marker: &[u8], - intent: &[u8], - receipt: &[u8], -) -> Result, FuzzSeedError> { - let payload_bytes = marker - .len() - .checked_add(intent.len()) - .and_then(|length| length.checked_add(receipt.len())) - .ok_or_else(|| FuzzSeedError::violation("migration receipt seed length overflow"))?; - let framed_bytes = payload_bytes - .checked_add(1) - .ok_or_else(|| FuzzSeedError::violation("migration receipt seed length overflow"))?; - if framed_bytes > MAX_SEED_BYTES { - return Err(FuzzSeedError::violation( - "migration receipt seed exceeds the input bound", - )); - } - let mut payload = Vec::with_capacity(payload_bytes); - payload.extend_from_slice(marker); - payload.extend_from_slice(intent); - payload.extend_from_slice(receipt); - prefixed(selector, &payload) -} - -fn malformed_seeds( - marker: &[u8], - intent: &[u8], - receipt: &[u8], -) -> Result, FuzzSeedError> { - let mut bad_marker = marker.to_vec(); - *bad_marker - .get_mut(17) - .ok_or_else(|| FuzzSeedError::violation("marker lacks version"))? = 3; - let mut bad_intent = intent.to_vec(); - *bad_intent - .get_mut(23) - .ok_or_else(|| FuzzSeedError::violation("intent lacks flags"))? = 1; - let mut bad_receipt = receipt.to_vec(); - *bad_receipt - .last_mut() - .ok_or_else(|| FuzzSeedError::violation("receipt lacks checksum"))? ^= 1; - Ok(vec![ - Seed::new( - "migration_format", - "unsupported-marker-version", - prefixed(0, &bad_marker)?, - )?, - Seed::new( - "migration_format", - "unsupported-intent-flag", - prefixed(1, &bad_intent)?, - )?, - Seed::new( - "migration_format", - "corrupt-receipt-checksum", - receipt_seed(2, marker, intent, &bad_receipt)?, - )?, - ]) -} - -#[cfg(test)] -mod tests { - use super::{FuzzSeedError, MAX_SEED_BYTES, receipt_seed}; - - #[test] - fn receipt_seed_frames_dependencies_before_the_receipt() -> Result<(), FuzzSeedError> { - let seed = receipt_seed(2, b"marker", b"intent", b"receipt")?; - assert_eq!(seed, b"\x02markerintentreceipt"); - Ok(()) - } - - #[test] - fn receipt_seed_refuses_before_allocating_above_the_seed_bound() -> Result<(), FuzzSeedError> { - let oversized_marker = vec![0; MAX_SEED_BYTES]; - let Err(FuzzSeedError::Violation(message)) = receipt_seed(2, &oversized_marker, &[], &[]) - else { - return Err(FuzzSeedError::violation( - "oversized migration receipt seed was admitted", - )); - }; - assert_eq!(message, "migration receipt seed exceeds the input bound"); - Ok(()) - } -} diff --git a/xtask/src/fuzz_seed_corpus/tests/materialization.rs b/xtask/src/fuzz_seed_corpus/tests/materialization.rs index ee58a55d..5a53168d 100644 --- a/xtask/src/fuzz_seed_corpus/tests/materialization.rs +++ b/xtask/src/fuzz_seed_corpus/tests/materialization.rs @@ -4,7 +4,7 @@ use std::collections::BTreeMap; use std::path::Path; use super::super::{ - FuzzSeedError, catalog_seeds, layout_seeds, migration_seeds, prepare, retention_seeds, + FuzzSeedError, catalog_seeds, format_marker_seeds, layout_seeds, prepare, retention_seeds, segment_seeds, }; use crate::test_directory::TestDirectory; @@ -127,7 +127,7 @@ fn copy_version_two_fixtures(source_root: &Path, root: &Path) -> Result<(), Fuzz })?; let fixtures = retention_seeds::FIXTURES .into_iter() - .chain(migration_seeds::FIXTURES); + .chain(format_marker_seeds::FIXTURES); for (_selector, fixture) in fixtures { let source_path = source_root .join("conformance/segment-store/v2") diff --git a/xtask/tests/durability_crash_case_contract.rs b/xtask/tests/durability_crash_case_contract.rs index c062a5eb..c1fd036b 100644 --- a/xtask/tests/durability_crash_case_contract.rs +++ b/xtask/tests/durability_crash_case_contract.rs @@ -6,7 +6,7 @@ use std::error::Error; use xtask::{ DurabilityCrashCase, DurabilityCrashCaseError, DurabilityCrashOccurrence, DurabilityCrashPoint, - DurabilityCrashPosition, + DurabilityCrashPosition, DurabilityCrashSequence, }; #[test] @@ -70,12 +70,12 @@ fn identifiers_and_positions_round_trip_without_aliases() { } #[test] -fn namespace_occurrences_are_admitted_only_within_the_protocol_fixed_range() +fn directory_occurrences_are_admitted_only_within_the_protocol_fixed_range() -> Result<(), Box> { - let point = DurabilityCrashPoint::MigrationAdmitNamespacePrefix; + let point = DurabilityCrashPoint::InitializeAdmitDirectory; for position in DurabilityCrashPosition::ALL { let exclusive_limit = if position == DurabilityCrashPosition::During { - DurabilityCrashPoint::NAMESPACE_PREFIX_DIRECTORIES + DurabilityCrashPoint::INITIALIZATION_DIRECTORIES } else { 1 }; @@ -98,3 +98,19 @@ fn namespace_occurrences_are_admitted_only_within_the_protocol_fixed_range() } Ok(()) } + +/// The documented 41 initialization cases: eleven boundaries at three +/// positions, plus one extra `during` case for each of the nine directories +/// beyond the first. +#[test] +fn the_initialization_sequence_runs_its_documented_forty_one_cases() { + let cases: Vec<_> = + DurabilityCrashCase::in_sequence(DurabilityCrashSequence::VersionTwoInitialization) + .collect(); + assert_eq!(cases.len(), 41); + let directory_cases = cases + .iter() + .filter(|case| case.point() == DurabilityCrashPoint::InitializeAdmitDirectory) + .count(); + assert_eq!(directory_cases, 11); +} diff --git a/xtask/tests/durability_crash_documentation.rs b/xtask/tests/durability_crash_documentation.rs index 78193699..e55ca70f 100644 --- a/xtask/tests/durability_crash_documentation.rs +++ b/xtask/tests/durability_crash_documentation.rs @@ -7,8 +7,7 @@ const RECOVERY: &str = include_str!("../../docs/formats/segment-store-v1/recover const REQUIREMENTS: &str = include_str!("../../docs/formats/segment-store-v1/requirements.md"); const CORPUS_README: &str = include_str!("../../conformance/segment-store/v1/README.md"); const V2_CORPUS_README: &str = include_str!("../../conformance/segment-store/v2/README.md"); -const MIGRATION_CRASH: &str = - include_str!("../../docs/formats/segment-store-v2/migration-crash.md"); +const INITIALIZATION: &str = include_str!("../../docs/formats/segment-store-v2/initialization.md"); const V2_REQUIREMENTS: &str = include_str!("../../docs/formats/segment-store-v2/requirements.md"); #[test] @@ -18,12 +17,15 @@ fn living_documentation_routes_the_complete_crash_matrix_and_its_limits() { (RECOVERY, "## Process-death crash matrix"), (REQUIREMENTS, "`KEEP-RECOVERY-021`"), (CORPUS_README, "105 canonical process-death cases"), - (V2_CORPUS_README, "68 canonical process-death cases"), ( - MIGRATION_CRASH, - "cargo xtask durability-crash-matrix --sequence migration", + V2_CORPUS_README, + "41 canonical initialization process-death cases", ), - (V2_REQUIREMENTS, "`KEEP-MIGRATION-007`"), + ( + INITIALIZATION, + "cargo xtask durability-crash-matrix --sequence version-two-initialization", + ), + (V2_REQUIREMENTS, "`KEEP-INITIALIZATION-004`"), ] { assert!( document.contains(claim), @@ -34,9 +36,5 @@ fn living_documentation_routes_the_complete_crash_matrix_and_its_limits() { "Process-death injection, retention, compaction, and garbage collection remain planned." )); assert!(RECOVERY.contains("does not simulate host power loss")); - assert!( - !MIGRATION_CRASH - .contains("this page claims in-process recovery, not process-death recovery") - ); - assert!(!ROOT_README.contains("Process-death evidence for migration recovery")); + assert!(INITIALIZATION.contains("It is process-death evidence.")); } diff --git a/xtask/tests/durability_crash_point_contract.rs b/xtask/tests/durability_crash_point_contract.rs index f7b417a0..db315598 100644 --- a/xtask/tests/durability_crash_point_contract.rs +++ b/xtask/tests/durability_crash_point_contract.rs @@ -5,8 +5,8 @@ use xtask::{DurabilityCrashPoint, DurabilityCrashSequence}; use DurabilityCrashSequence::{ - Catalog, Head, Initialization, Migration, RecoveryDiscard, Retention, Segment, - VersionTwoCatalog, + Catalog, Head, Initialization, RecoveryDiscard, Retention, Segment, VersionTwoCatalog, + VersionTwoInitialization, }; const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ @@ -247,129 +247,79 @@ const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ Retention, ), ( - DurabilityCrashPoint::MigrationWriteIntentStage, - "KEEP-CRASH-053", - Migration, - ), - ( - DurabilityCrashPoint::MigrationSynchronizeIntentStage, - "KEEP-CRASH-054", - Migration, - ), - ( - DurabilityCrashPoint::MigrationLinkIntent, - "KEEP-CRASH-055", - Migration, - ), - ( - DurabilityCrashPoint::MigrationSynchronizeRootAfterIntent, - "KEEP-CRASH-056", - Migration, - ), - ( - DurabilityCrashPoint::MigrationRemoveIntentStage, - "KEEP-CRASH-057", - Migration, - ), - ( - DurabilityCrashPoint::MigrationSynchronizeRootAfterIntentCleanup, - "KEEP-CRASH-058", - Migration, - ), - ( - DurabilityCrashPoint::MigrationAdmitReaderFence, - "KEEP-CRASH-059", - Migration, - ), - ( - DurabilityCrashPoint::MigrationAdmitNamespacePrefix, - "KEEP-CRASH-060", - Migration, - ), - ( - DurabilityCrashPoint::MigrationSynchronizeRootAfterNamespace, - "KEEP-CRASH-061", - Migration, - ), - ( - DurabilityCrashPoint::MigrationWriteMarkerStage, - "KEEP-CRASH-062", - Migration, - ), - ( - DurabilityCrashPoint::MigrationSynchronizeMarkerStage, - "KEEP-CRASH-063", - Migration, + DurabilityCrashPoint::VersionTwoLinkSegment, + "KEEP-CRASH-074", + VersionTwoCatalog, ), ( - DurabilityCrashPoint::MigrationLinkMarker, - "KEEP-CRASH-064", - Migration, + DurabilityCrashPoint::VersionTwoCreateCatalogStage, + "KEEP-CRASH-075", + VersionTwoCatalog, ), ( - DurabilityCrashPoint::MigrationSynchronizeRootAfterMarker, - "KEEP-CRASH-065", - Migration, + DurabilityCrashPoint::VersionTwoSynchronizeHead, + "KEEP-CRASH-076", + VersionTwoCatalog, ), ( - DurabilityCrashPoint::MigrationRemoveMarkerStage, - "KEEP-CRASH-066", - Migration, + DurabilityCrashPoint::VersionTwoReplaceHead, + "KEEP-CRASH-077", + VersionTwoCatalog, ), ( - DurabilityCrashPoint::MigrationSynchronizeRootAfterMarkerCleanup, - "KEEP-CRASH-067", - Migration, + DurabilityCrashPoint::InitializeOpenAndLockWriterFile, + "KEEP-CRASH-081", + VersionTwoInitialization, ), ( - DurabilityCrashPoint::MigrationWriteReceiptStage, - "KEEP-CRASH-068", - Migration, + DurabilityCrashPoint::InitializeWriteMarkerStage, + "KEEP-CRASH-082", + VersionTwoInitialization, ), ( - DurabilityCrashPoint::MigrationSynchronizeReceiptStage, - "KEEP-CRASH-069", - Migration, + DurabilityCrashPoint::InitializeSynchronizeMarkerStage, + "KEEP-CRASH-083", + VersionTwoInitialization, ), ( - DurabilityCrashPoint::MigrationLinkReceipt, - "KEEP-CRASH-070", - Migration, + DurabilityCrashPoint::InitializeSynchronizeRootAfterMarkerStage, + "KEEP-CRASH-084", + VersionTwoInitialization, ), ( - DurabilityCrashPoint::MigrationSynchronizeRootAfterReceipt, - "KEEP-CRASH-071", - Migration, + DurabilityCrashPoint::InitializeAdmitReaderFence, + "KEEP-CRASH-085", + VersionTwoInitialization, ), ( - DurabilityCrashPoint::MigrationRemoveReceiptStage, - "KEEP-CRASH-072", - Migration, + DurabilityCrashPoint::InitializeAdmitDirectory, + "KEEP-CRASH-086", + VersionTwoInitialization, ), ( - DurabilityCrashPoint::MigrationSynchronizeRootAfterReceiptCleanup, - "KEEP-CRASH-073", - Migration, + DurabilityCrashPoint::InitializeLinkMarker, + "KEEP-CRASH-087", + VersionTwoInitialization, ), ( - DurabilityCrashPoint::VersionTwoLinkSegment, - "KEEP-CRASH-074", - VersionTwoCatalog, + DurabilityCrashPoint::InitializeSynchronizeRootAfterMarker, + "KEEP-CRASH-088", + VersionTwoInitialization, ), ( - DurabilityCrashPoint::VersionTwoCreateCatalogStage, - "KEEP-CRASH-075", - VersionTwoCatalog, + DurabilityCrashPoint::InitializeRemoveMarkerStage, + "KEEP-CRASH-089", + VersionTwoInitialization, ), ( - DurabilityCrashPoint::VersionTwoSynchronizeHead, - "KEEP-CRASH-076", - VersionTwoCatalog, + DurabilityCrashPoint::InitializeSynchronizeRootAfterCleanup, + "KEEP-CRASH-090", + VersionTwoInitialization, ), ( - DurabilityCrashPoint::VersionTwoReplaceHead, - "KEEP-CRASH-077", - VersionTwoCatalog, + DurabilityCrashPoint::InitializeDiscardIncompleteMarkerStage, + "KEEP-CRASH-091", + VersionTwoInitialization, ), ]; @@ -382,7 +332,7 @@ fn crash_boundaries_have_one_contiguous_stable_vocabulary() { } #[test] -fn only_record_append_and_namespace_prefix_select_an_occurrence() { +fn only_record_append_and_initialization_directories_select_an_occurrence() { let occurrence_counted: Vec<_> = DurabilityCrashPoint::ALL .into_iter() .filter(|point| point.occurrence_counted()) @@ -392,16 +342,16 @@ fn only_record_append_and_namespace_prefix_select_an_occurrence() { occurrence_counted, [ DurabilityCrashPoint::AppendSegmentRecord, - DurabilityCrashPoint::MigrationAdmitNamespacePrefix + DurabilityCrashPoint::InitializeAdmitDirectory ] ); } #[test] -fn the_namespace_prefix_runs_one_during_case_per_directory() { +fn initialization_directory_admission_runs_one_during_case_per_directory() { for point in DurabilityCrashPoint::ALL { - let expected = if point == DurabilityCrashPoint::MigrationAdmitNamespacePrefix { - DurabilityCrashPoint::NAMESPACE_PREFIX_DIRECTORIES + let expected = if point == DurabilityCrashPoint::InitializeAdmitDirectory { + DurabilityCrashPoint::INITIALIZATION_DIRECTORIES } else { 1 }; @@ -412,27 +362,35 @@ fn the_namespace_prefix_runs_one_during_case_per_directory() { point.identifier() ); } - assert_eq!(DurabilityCrashPoint::NAMESPACE_PREFIX_DIRECTORIES, 6); + assert_eq!( + DurabilityCrashPoint::INITIALIZATION_DIRECTORIES, + u32::try_from(keep::VersionTwoInitializationDirectory::ALL.len()).unwrap_or(u32::MAX) + ); +} + +#[test] +fn retired_crash_identifiers_are_never_reassigned() { + for ordinal in (53..=73).chain(78..=80) { + let identifier = format!("KEEP-CRASH-{ordinal:03}"); + assert_eq!( + DurabilityCrashPoint::from_identifier(&identifier), + None, + "{identifier} is retired or reserved and must not name a boundary" + ); + } } #[test] -fn migration_boundaries_follow_the_twenty_one_phases_in_order() { - let migration: Vec<_> = DurabilityCrashPoint::ALL +fn initialization_boundaries_take_consecutive_new_identifiers() { + let initialization: Vec<_> = DurabilityCrashPoint::ALL .into_iter() - .filter(|point| point.sequence() == Migration) + .filter(|point| point.sequence() == VersionTwoInitialization) + .map(DurabilityCrashPoint::identifier) .collect(); - - assert_eq!(migration, DurabilityCrashPoint::MIGRATION); - assert_eq!(migration.len(), keep::StoreMigrationPhase::ALL.len()); - assert_eq!( - DurabilityCrashPoint::MIGRATION.map(DurabilityCrashPoint::identifier), - std::array::from_fn::<_, 21, _>(|index| { - let ordinal = 53 + index; - let identifier = format!("KEEP-CRASH-{ordinal:03}"); - DurabilityCrashPoint::from_identifier(&identifier) - .map_or("missing", DurabilityCrashPoint::identifier) - }) - ); + let expected: Vec<_> = (81..=91) + .map(|ordinal| format!("KEEP-CRASH-{ordinal:03}")) + .collect(); + assert_eq!(initialization, expected); } // Subject: repository-tool runtime, not Keep recovery acceptance evidence. @@ -447,8 +405,8 @@ fn crash_matrix_cli_admits_its_stable_sequence_names() { ("recovery-discard", RecoveryDiscard), ("initialization", Initialization), ("retention", Retention), - ("migration", Migration), ("version-two-catalog", VersionTwoCatalog), + ("version-two-initialization", VersionTwoInitialization), ] { assert_eq!( DurabilityCrashSequence::from_identifier(name), diff --git a/xtask/tests/durability_crash_production_contract.rs b/xtask/tests/durability_crash_production_contract.rs index 3d1c2277..488db717 100644 --- a/xtask/tests/durability_crash_production_contract.rs +++ b/xtask/tests/durability_crash_production_contract.rs @@ -15,7 +15,7 @@ fn crash_children_execute_every_claimed_production_protocol() -> Result<(), Box< "durability_crash_matrix/production_protocol/initialization.rs", "durability_crash_matrix/production_protocol/publication.rs", "durability_crash_matrix/production_protocol/recovery.rs", - "durability_crash_matrix/production_protocol/migration.rs", + "durability_crash_matrix/production_protocol/version_two_initialization.rs", "durability_crash_matrix/production_protocol/version_two_catalog.rs", ] .into_iter() @@ -29,7 +29,7 @@ fn crash_children_execute_every_claimed_production_protocol() -> Result<(), Box< "publish_catalog_generation(", "initialize_store(", "execute_recovery_stage_discard(", - "execute_store_migration(", + "initialize_observed_for_repository_tasks(", "publish_retained_catalog_generation(", ] { assert!( diff --git a/xtask/tests/retention_store_v2_conformance_contract.rs b/xtask/tests/retention_store_v2_conformance_contract.rs index 9c947f34..902452b0 100644 --- a/xtask/tests/retention_store_v2_conformance_contract.rs +++ b/xtask/tests/retention_store_v2_conformance_contract.rs @@ -14,13 +14,9 @@ const REQUIRED_PATHS: &[&str] = &[ "ORIGIN.md", "definition.tsv", "retention-profile.tsv", - "inventory.tsv", - "migration-source.tsv", "artifacts.tsv", "transitions.tsv", "format-marker.hex", - "migration-intent.hex", - "migration-receipt.hex", "one-anchor-root.hex", "one-root-manifest.hex", "one-root-head.hex", diff --git a/xtask/tests/retention_store_v2_format_oracle.rs b/xtask/tests/retention_store_v2_format_oracle.rs index 86081ecc..ea446cd5 100644 --- a/xtask/tests/retention_store_v2_format_oracle.rs +++ b/xtask/tests/retention_store_v2_format_oracle.rs @@ -7,9 +7,6 @@ const PROFILE_DEFINITION: &str = include_str!("../../conformance/segment-store/v2/retention-profile.tsv"); const FORMAT_DEFINITION: &str = include_str!("../../conformance/segment-store/v2/definition.tsv"); const LAYOUTS: &str = include_str!("../../conformance/layout/v1/layouts.tsv"); -const V1_SEGMENT: &str = include_str!("../../conformance/segment-store/v1/one-zero-segment.hex"); -const V1_CATALOG: &str = include_str!("../../conformance/segment-store/v1/one-zero-catalog.hex"); -const V1_HEAD: &str = include_str!("../../conformance/segment-store/v1/one-zero-head.hex"); struct Artifact { case_name: &'static str, @@ -25,38 +22,9 @@ struct Artifact { struct Corpus { profile_digest: [u8; 32], definition_digest: [u8; 32], - inventory: Inventory, - migration: MigrationSource, artifacts: Vec, } -struct Inventory { - rows: Vec, - digest: [u8; 32], -} - -struct InventoryRow { - kind: &'static str, - generation: u64, - byte_length: u64, - artifact_digest: [u8; 32], - source_fixture: &'static str, - bytes: [u8; 56], -} - -struct MigrationSource { - catalog_generation: u64, - catalog_length: u64, - catalog_digest: [u8; 32], - predecessor_digest: [u8; 32], - inventory_digest: [u8; 32], - definition_digest: [u8; 32], - store_id: [u8; 32], - root_device: u64, - root_mount: u64, - root_file: u64, -} - struct RootArtifact { bytes: Vec, digest: [u8; 32], diff --git a/xtask/tests/retention_store_v2_format_oracle/artifacts.rs b/xtask/tests/retention_store_v2_format_oracle/artifacts.rs index 06b2f504..80b39ba1 100644 --- a/xtask/tests/retention_store_v2_format_oracle/artifacts.rs +++ b/xtask/tests/retention_store_v2_format_oracle/artifacts.rs @@ -2,24 +2,16 @@ const PROFILE_DOMAIN: &[u8] = b"keep.retention-realization-profile/v1\0"; const DEFINITION_DOMAIN: &[u8] = b"keep.segment-store-definition/v2\0"; -const INVENTORY_DOMAIN: &[u8] = b"keep.store-v1-pool-inventory/v2\0"; -const STORE_ID_DOMAIN: &[u8] = b"keep.store-identifier/v2\0"; fn build_corpus() -> Result { let profile_digest = hash(PROFILE_DOMAIN, &[PROFILE_DEFINITION.as_bytes()]); let definition_digest = hash(DEFINITION_DOMAIN, &[FORMAT_DEFINITION.as_bytes()]); - let inventory = build_inventory()?; - let migration = migration_source(definition_digest, inventory.digest)?; let format = build_format_marker(definition_digest)?; - let intent = build_migration_intent(&migration)?; - let receipt = build_migration_receipt(&migration, &format, &intent)?; let root = build_retention_root(profile_digest)?; let manifest = build_retention_manifest(&root)?; let head = build_retention_head(&manifest)?; let artifacts = vec![ format, - intent, - receipt, Artifact { case_name: "one-anchor-root", kind: "retention-root", @@ -45,101 +37,9 @@ fn build_corpus() -> Result { Ok(Corpus { profile_digest, definition_digest, - inventory, - migration, artifacts, }) } -fn build_inventory() -> Result { - let segment = decode_hex(V1_SEGMENT)?; - let catalog = decode_hex(V1_CATALOG)?; - require_length(&segment, 337, "version-1 source segment")?; - require_length(&catalog, 352, "version-1 source catalog")?; - let segment_digest = array_32(&segment, 273)?; - let catalog_digest = array_32(&catalog, 320)?; - let mut rows = vec![ - inventory_row(1, 0, &segment, segment_digest, "one-zero-segment.hex")?, - inventory_row(2, 1, &catalog, catalog_digest, "one-zero-catalog.hex")?, - ]; - rows.sort_by_key(|row| row.bytes); - let entry_count = - u32::try_from(rows.len()).map_err(|_| "inventory entry count overflow".to_owned())?; - let mut count = entry_count.to_be_bytes().to_vec(); - for row in &rows { - count.extend_from_slice(&row.bytes); - } - let digest = hash(INVENTORY_DOMAIN, &[&count]); - Ok(Inventory { rows, digest }) -} - -fn inventory_row( - kind: u8, - generation: u64, - artifact: &[u8], - digest: [u8; 32], - source_fixture: &'static str, -) -> Result { - let byte_length = - u64::try_from(artifact.len()).map_err(|_| "inventory length overflow".to_owned())?; - let mut bytes = Vec::with_capacity(56); - bytes.push(kind); - bytes.extend_from_slice(&[0; 7]); - push_u64(&mut bytes, generation); - push_u64(&mut bytes, byte_length); - bytes.extend_from_slice(&digest); - require_length(&bytes, 56, "migration inventory entry")?; - let kind_name = match kind { - 1 => "segment", - 2 => "catalog", - _ => return Err("unregistered inventory artifact kind".to_owned()), - }; - Ok(InventoryRow { - kind: kind_name, - generation, - byte_length, - artifact_digest: digest, - source_fixture, - bytes: <[u8; 56]>::try_from(bytes) - .map_err(|_| "inventory entry conversion failed".to_owned())?, - }) -} - -fn migration_source( - definition_digest: [u8; 32], - inventory_digest: [u8; 32], -) -> Result { - let head = decode_hex(V1_HEAD)?; - let catalog = decode_hex(V1_CATALOG)?; - require_length(&head, 128, "version-1 source head")?; - let catalog_generation = u64_at(&head, 24)?; - let catalog_length = u64_at(&head, 32)?; - let catalog_digest = array_32(&head, 40)?; - let predecessor_digest = array_32(&catalog, 32)?; - let store_id = hash( - STORE_ID_DOMAIN, - &[ - &catalog_generation.to_be_bytes(), - &catalog_length.to_be_bytes(), - &catalog_digest, - &predecessor_digest, - &inventory_digest, - &definition_digest, - ], - ); - Ok(MigrationSource { - catalog_generation, - catalog_length, - catalog_digest, - predecessor_digest, - inventory_digest, - definition_digest, - store_id, - root_device: 1, - root_mount: 2, - root_file: 3, - }) -} - -include!("artifacts/migration.rs"); +include!("artifacts/format_marker.rs"); include!("artifacts/retention.rs"); diff --git a/xtask/tests/retention_store_v2_format_oracle/artifacts/format_marker.rs b/xtask/tests/retention_store_v2_format_oracle/artifacts/format_marker.rs new file mode 100644 index 00000000..e068f2de --- /dev/null +++ b/xtask/tests/retention_store_v2_format_oracle/artifacts/format_marker.rs @@ -0,0 +1,28 @@ +// This included source owns construction of the format marker. + +fn build_format_marker(definition_digest: [u8; 32]) -> Result { + let mut bytes = Vec::with_capacity(96); + bytes.extend_from_slice(b"KEEP:STORE:V2\0\0\0"); + push_u16(&mut bytes, 2); + push_u16(&mut bytes, 96); + push_u32(&mut bytes, 0); + bytes.extend_from_slice(&definition_digest); + push_u32(&mut bytes, 4_096); + push_u32(&mut bytes, 0); + require_length(&bytes, 64, "format-marker checksum preimage")?; + let checksum = hash(b"keep.segment-store-marker-checksum/v2\0", &[&bytes]); + bytes.extend_from_slice(&checksum); + require_length(&bytes, 96, "format marker")?; + let marker_digest = hash(b"keep.store-format-marker/v2\0", &[&bytes]); + Ok(Artifact { + case_name: "format-marker", + kind: "format-marker", + generation: "-", + entry_count: "-", + bound_digest: marker_digest, + final_checksum: checksum, + fixture: "format-marker.hex", + bytes, + }) +} + diff --git a/xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs b/xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs deleted file mode 100644 index 41202583..00000000 --- a/xtask/tests/retention_store_v2_format_oracle/artifacts/migration.rs +++ /dev/null @@ -1,96 +0,0 @@ -// This included source owns construction of the format marker and migration records. - -fn build_format_marker(definition_digest: [u8; 32]) -> Result { - let mut bytes = Vec::with_capacity(96); - bytes.extend_from_slice(b"KEEP:STORE:V2\0\0\0"); - push_u16(&mut bytes, 2); - push_u16(&mut bytes, 96); - push_u32(&mut bytes, 0); - bytes.extend_from_slice(&definition_digest); - push_u32(&mut bytes, 4_096); - push_u32(&mut bytes, 0); - require_length(&bytes, 64, "format-marker checksum preimage")?; - let checksum = hash(b"keep.segment-store-marker-checksum/v2\0", &[&bytes]); - bytes.extend_from_slice(&checksum); - require_length(&bytes, 96, "format marker")?; - let marker_digest = hash(b"keep.store-format-marker/v2\0", &[&bytes]); - Ok(Artifact { - case_name: "format-marker", - kind: "format-marker", - generation: "-", - entry_count: "-", - bound_digest: marker_digest, - final_checksum: checksum, - fixture: "format-marker.hex", - bytes, - }) -} - -fn build_migration_intent(source: &MigrationSource) -> Result { - let mut bytes = Vec::with_capacity(256); - bytes.extend_from_slice(b"KEEP:MIG:INT2\0\0\0"); - push_u16(&mut bytes, 2); - push_u16(&mut bytes, 256); - push_u32(&mut bytes, 0); - push_u64(&mut bytes, source.catalog_generation); - push_u64(&mut bytes, source.catalog_length); - bytes.extend_from_slice(&source.catalog_digest); - bytes.extend_from_slice(&source.predecessor_digest); - bytes.extend_from_slice(&source.inventory_digest); - push_u64(&mut bytes, source.root_device); - push_u64(&mut bytes, source.root_mount); - push_u64(&mut bytes, source.root_file); - bytes.extend_from_slice(&source.definition_digest); - bytes.extend_from_slice(&source.store_id); - require_length(&bytes, 224, "migration-intent checksum preimage")?; - let checksum = hash(b"keep.store-migration-intent-checksum/v2\0", &[&bytes]); - bytes.extend_from_slice(&checksum); - require_length(&bytes, 256, "migration intent")?; - let intent_digest = hash(b"keep.store-migration-intent/v2\0", &[&bytes]); - Ok(Artifact { - case_name: "migration-intent", - kind: "migration-intent", - generation: "1", - entry_count: "2", - bound_digest: intent_digest, - final_checksum: checksum, - fixture: "migration-intent.hex", - bytes, - }) -} - -fn build_migration_receipt( - source: &MigrationSource, - format: &Artifact, - intent: &Artifact, -) -> Result { - let initial_retention = hash(b"keep.initial-retention-state/v2\0", &[]); - let initial_gc = hash(b"keep.initial-gc-state/v2\0", &[]); - let empty_dispositions = hash(b"keep.empty-disposition-set/v2\0", &[]); - let mut bytes = Vec::with_capacity(256); - bytes.extend_from_slice(b"KEEP:MIG:REC2\0\0\0"); - push_u16(&mut bytes, 2); - push_u16(&mut bytes, 256); - push_u32(&mut bytes, 0); - bytes.extend_from_slice(&intent.bound_digest); - bytes.extend_from_slice(&source.store_id); - bytes.extend_from_slice(&format.bound_digest); - bytes.extend_from_slice(&initial_retention); - bytes.extend_from_slice(&initial_gc); - bytes.extend_from_slice(&empty_dispositions); - push_u64(&mut bytes, 0x03ff); - require_length(&bytes, 224, "migration-receipt checksum preimage")?; - let checksum = hash(b"keep.store-migration-receipt-checksum/v2\0", &[&bytes]); - bytes.extend_from_slice(&checksum); - require_length(&bytes, 256, "migration receipt")?; - Ok(Artifact { - case_name: "migration-receipt", - kind: "migration-receipt", - generation: "1", - entry_count: "2", - bound_digest: intent.bound_digest, - final_checksum: checksum, - fixture: "migration-receipt.hex", - bytes, - }) -} diff --git a/xtask/tests/retention_store_v2_format_oracle/encoding.rs b/xtask/tests/retention_store_v2_format_oracle/encoding.rs index 2275b17b..85d6c692 100644 --- a/xtask/tests/retention_store_v2_format_oracle/encoding.rs +++ b/xtask/tests/retention_store_v2_format_oracle/encoding.rs @@ -26,16 +26,6 @@ fn push_u64(bytes: &mut Vec, value: u64) { bytes.extend_from_slice(&value.to_be_bytes()); } -fn array_32(bytes: &[u8], offset: usize) -> Result<[u8; 32], String> { - let end = offset - .checked_add(32) - .ok_or_else(|| "32-byte field offset overflow".to_owned())?; - let field = bytes - .get(offset..end) - .ok_or_else(|| format!("missing 32-byte field at offset {offset}"))?; - <[u8; 32]>::try_from(field).map_err(|_| "32-byte field conversion failed".to_owned()) -} - fn u64_at(bytes: &[u8], offset: usize) -> Result { let end = offset .checked_add(8) diff --git a/xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs b/xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs index 88b3570d..e5f91af5 100644 --- a/xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs +++ b/xtask/tests/retention_store_v2_format_oracle/fixture_assertion.rs @@ -22,7 +22,7 @@ fn golden_artifacts_match_the_independent_oracle() -> Result<(), String> { } #[test] -fn definition_profile_and_migration_sources_match_the_oracle() -> Result<(), String> { +fn definition_and_profile_digests_match_the_oracle() -> Result<(), String> { let corpus = build_corpus()?; let profile_hex = encode_digest(&corpus.profile_digest)?; let definition_hex = encode_digest(&corpus.definition_digest)?; @@ -30,18 +30,6 @@ fn definition_profile_and_migration_sources_match_the_oracle() -> Result<(), Str FORMAT_DEFINITION.contains(&format!("retention.profile.digest\t{profile_hex}")), "format definition does not bind the exact profile digest" ); - assert_eq!( - corpus.definition_digest, corpus.migration.definition_digest, - "migration source does not bind the exact format definition" - ); - assert_eq!( - read_corpus_file("inventory.tsv").map_err(|error| error.to_string())?, - inventory_table(&corpus.inventory)? - ); - assert_eq!( - read_corpus_file("migration-source.tsv").map_err(|error| error.to_string())?, - migration_source_table(&corpus.migration)? - ); for documentation in ["README.md", "ORIGIN.md"] { assert!( read_corpus_file(documentation) @@ -148,50 +136,6 @@ fn artifacts_table(artifacts: &[Artifact]) -> Result { Ok(table) } -fn inventory_table(inventory: &Inventory) -> Result { - let mut table = String::from( - "keep.segment-store-v2.inventory/v1\n\ - kind\tgeneration\tbyte_length\tartifact_digest_hex\tsource_fixture\n", - ); - for row in &inventory.rows { - writeln!( - table, - "{}\t{}\t{}\t{}\t{}", - row.kind, - row.generation, - row.byte_length, - encode_digest(&row.artifact_digest)?, - row.source_fixture - ) - .map_err(|_| "inventory table formatting failed".to_owned())?; - } - Ok(table) -} - -fn migration_source_table(source: &MigrationSource) -> Result { - let mut table = String::from( - "keep.segment-store-v2.migration-source/v1\n\ - case\tcatalog_generation\tcatalog_length\tcatalog_digest_hex\t\ - predecessor_digest_hex\tinventory_digest_hex\tdefinition_digest_hex\t\ - store_id_hex\troot_device\troot_mount\troot_file\n", - ); - writeln!( - table, - "one-zero\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}", - source.catalog_generation, - source.catalog_length, - encode_digest(&source.catalog_digest)?, - encode_digest(&source.predecessor_digest)?, - encode_digest(&source.inventory_digest)?, - encode_digest(&source.definition_digest)?, - encode_digest(&source.store_id)?, - source.root_device, - source.root_mount, - source.root_file - ) - .map_err(|_| "migration-source table formatting failed".to_owned())?; - Ok(table) -} fn encode_digest(digest: &[u8; 32]) -> Result { encode_hex(digest).map(|encoded| encoded.trim_end().to_owned()) diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index 7926925a..d923da6b 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -4,8 +4,8 @@ #[path = "retention_store_v2_protocol_contract/closure_contract_laws.rs"] mod closure_contract_laws; -#[path = "retention_store_v2_protocol_contract/migration_contract_laws.rs"] -mod migration_contract_laws; +#[path = "retention_store_v2_protocol_contract/initialization_contract_laws.rs"] +mod initialization_contract_laws; #[path = "retention_store_v2_protocol_contract/parser_fuzz_laws.rs"] mod parser_fuzz_laws; #[path = "retention_store_v2_protocol_contract/transition_laws.rs"] @@ -56,10 +56,8 @@ fn version_two_is_one_routed_protocol() -> Result<(), Box "[Closure verification](closure.md)", "[Closure corruption boundary](closure-corruption.md)", "[GC and disposition records](gc.md)", - "[Migration and recovery](recovery.md)", - "[Migration protocol and recovery](migration-recovery.md)", - "[Migration crash points](migration-crash.md)", - "[Migration inventory](migration-inventory.md)", + "[Format, namespace, and portability](recovery.md)", + "[Direct initialization](initialization.md)", "[Requirements and evidence](requirements.md)", "[Format rationale](rationale.md)", ] { @@ -145,10 +143,9 @@ fn version_two_pages_stay_within_the_review_threshold() -> Result<(), Box Result<(), Box> { + // recovery.md owns the namespace, marker, fence, and portability laws; + // initialization.md owns the ordered protocol, its states, and its + // process-death rows. Retention restart has its own page. These are + // documentation guards, not executable evidence for the storage claims + // written in the pages. + let recovery = format!( + "{}\n{}\n{}", + normalized(&read(&format!("{FORMAT_ROOT}/recovery.md"))?), + normalized(&read(&format!("{FORMAT_ROOT}/initialization.md"))?), + normalized(&read(&format!("{FORMAT_ROOT}/retention-recovery.md"))?) + ); + + for required in [ + "direct initialization", + "`FORMAT.next`", + "commit point", + "`AlreadyInitialized`", + "`InitializationIncomplete`", + "`MissingFormatMarker`", + "`definition.tsv`", + "keep.store-format-marker/v2\\0", + "never means \"nothing is retained\"", + "root.next` is durable before a new namespace directory", + "`KEEP-CRASH-036`", + "`KEEP-CRASH-081`", + "`KEEP-CRASH-091`", + "Version-1 admission refuses", + "`reader.lock`", + "`GcRetirementIntent`", + "`GcRetirementReceipt`", + "`RecoveryDispositionReceipt`", + "unknown entry", + "unrecoverable ambiguity", + "idempotent", + "process death", + "records no device, inode, mount, or path", + ] { + assert!( + recovery.contains(required), + "segment-store v2 initialization and recovery pages omit `{required}`" + ); + } + Ok(()) +} + +#[test] +fn initialization_never_writes_the_marker_in_place() -> Result<(), Box> { + let initialization = normalized(&read(&format!("{FORMAT_ROOT}/initialization.md"))?); + + for required in [ + "never writes `FORMAT` in place", + "`FORMAT.next`", + "without replacement", + "strict prefix", + "`KEEP-CRASH-081`", + "`KEEP-CRASH-091`", + "before, during, and after", + ] { + assert!( + initialization.contains(required), + "segment-store v2 initialization protocol omits `{required}`" + ); + } + Ok(()) +} diff --git a/xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs deleted file mode 100644 index 728702c7..00000000 --- a/xtask/tests/retention_store_v2_protocol_contract/migration_contract_laws.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! Migration and recovery written-contract laws. - -use super::{FORMAT_ROOT, normalized, read}; - -#[test] -fn migration_and_recovery_define_every_authority_boundary() -> Result<(), Box> -{ - // recovery.md owns the namespace, marker, fence, and record grammars; - // migration-recovery.md owns the ordered protocol and partial recovery. The - // Retention restart has its own page; these are documentation guards, - // not executable evidence for the storage claims written in the pages. - let recovery = format!( - "{}\n{}\n{}", - normalized(&read(&format!("{FORMAT_ROOT}/recovery.md"))?), - normalized(&read(&format!("{FORMAT_ROOT}/migration-recovery.md"))?), - normalized(&read(&format!("{FORMAT_ROOT}/retention-recovery.md"))?) - ); - - for required in [ - "one-way explicit migration", - "`migration.intent`", - "`migration.intent.next`", - "`migration.receipt`", - "`migration.receipt.next`", - "`FORMAT.next`", - "`migration.intent` is exactly 256 bytes", - "`migration.receipt` is exactly 256 bytes", - "catalog generation, length, and digest", - "`definition.tsv`", - "migration inventory entry is exactly 56 bytes", - "2,097,152", - "keep.store-migration-intent/v2\\0", - "keep.store-format-marker/v2\\0", - "deterministically derived store identifier", - "absence of `retention/HEAD` is the canonical empty retention state", - "keep.initial-retention-state/v2\\0", - "keep.initial-gc-state/v2\\0", - "keep.empty-disposition-set/v2\\0", - "root.next` is durable before a new namespace directory", - "`KEEP-CRASH-036`", - "`KEEP-CRASH-073`", - "partial migration", - "Version-1 admission refuses", - "`reader.lock`", - "`GcRetirementIntent`", - "`GcRetirementReceipt`", - "`RecoveryDispositionReceipt`", - "unknown entry", - "unrecoverable ambiguity", - "idempotent", - "process death", - ] { - assert!( - recovery.contains(required), - "segment-store v2 migration and recovery pages omit `{required}`" - ); - } - Ok(()) -} - -#[test] -fn migration_never_writes_canonical_fixed_names_in_place() -> Result<(), Box> -{ - let migration = normalized(&read(&format!("{FORMAT_ROOT}/migration-crash.md"))?); - - for required in [ - "never writes canonical fixed names in place", - "`migration.intent.next`", - "`FORMAT.next`", - "`migration.receipt.next`", - "linked without replacement", - "pre-effect incomplete stage", - "`KEEP-CRASH-053`", - "`KEEP-CRASH-073`", - "`0x00000000000003ff`", - "before, during, and after process-death evidence", - ] { - assert!( - migration.contains(required), - "segment-store v2 migration crash protocol omits `{required}`" - ); - } - Ok(()) -} diff --git a/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs index 763d4db0..22ca52fd 100644 --- a/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs +++ b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs @@ -31,24 +31,24 @@ fn retention_decoders_have_registered_seeded_fuzz_evidence() -> Result<(), Box Result<(), Box> { +fn format_marker_decoder_has_registered_seeded_fuzz_evidence() -> Result<(), Box> { let repository_root = Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .ok_or("xtask manifest must have a repository parent")?; assert!( repository_root - .join("fuzz/fuzz_targets/migration_format.rs") + .join("fuzz/fuzz_targets/format_marker.rs") .is_file() ); assert!( repository_root - .join("xtask/src/fuzz_seed_corpus/migration_seeds.rs") + .join("xtask/src/fuzz_seed_corpus/format_marker_seeds.rs") .is_file() ); - assert!(FUZZ_MANIFEST.contains("name = \"migration_format\"")); - assert!(FUZZ_MANIFEST.contains("path = \"fuzz_targets/migration_format.rs\"")); - assert!(FUZZ_GUIDE.contains("The `migration_format` seeds")); - assert!(REQUIREMENTS.contains("`migration_format`")); + assert!(FUZZ_MANIFEST.contains("name = \"format_marker\"")); + assert!(FUZZ_MANIFEST.contains("path = \"fuzz_targets/format_marker.rs\"")); + assert!(FUZZ_GUIDE.contains("The `format_marker` seeds")); + assert!(REQUIREMENTS.contains("`format_marker`")); Ok(()) } diff --git a/xtask/tests/retention_store_v2_protocol_contract/transition_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/transition_laws.rs index d29f0934..89a3b718 100644 --- a/xtask/tests/retention_store_v2_protocol_contract/transition_laws.rs +++ b/xtask/tests/retention_store_v2_protocol_contract/transition_laws.rs @@ -1,36 +1,27 @@ -//! This module owns canonical migration transition-ledger admission laws. +//! This module owns canonical initialization transition-ledger admission laws. -use keep::StoreMigrationPhase; +use xtask::{DurabilityCrashPoint, DurabilityCrashSequence}; const TRANSITIONS: &str = include_str!("../../../conformance/segment-store/v2/transitions.tsv"); const HEADER: &str = "keep.segment-store.transitions/v2\n\ crash_id\tphase\toperation\tpre_state\tinterrupted_class\t\ post_state\trecovery_posture\n"; -/// The operation column in `StoreMigrationPhase::ALL` order. -const OPERATIONS: [&str; 21] = [ - "write-intent-stage", - "sync-intent-stage", - "link-intent", - "sync-root-after-intent", - "remove-intent-stage", - "sync-root-after-intent-cleanup", - "admit-reader-fence", - "admit-namespace-prefix", - "sync-root-after-namespace", +/// The operation column in initialization crash-point order, `KEEP-CRASH-081` onward. +const OPERATIONS: [&str; 11] = [ + "open-and-lock-writer-file", "write-marker-stage", "sync-marker-stage", + "sync-root-after-marker-stage", + "admit-reader-fence", + "admit-directory", "link-marker", "sync-root-after-marker", "remove-marker-stage", - "sync-root-after-marker-cleanup", - "write-receipt-stage", - "sync-receipt-stage", - "link-receipt", - "sync-root-after-receipt", - "remove-receipt-stage", - "sync-root-after-receipt-cleanup", + "sync-root-after-cleanup", + "discard-incomplete-marker-stage", ]; +const FIRST_ORDINAL: usize = 81; #[derive(Debug, Eq, PartialEq)] enum LedgerRefusal { @@ -40,12 +31,16 @@ enum LedgerRefusal { Fields { row: usize }, Coordinate { row: usize }, Posture { row: usize }, - NamespaceExtent, + DirectoryExtent, } #[test] -fn migration_transition_coordinates_are_complete_and_ordered() { - assert_eq!(OPERATIONS.len(), StoreMigrationPhase::ALL.len()); +fn initialization_transition_coordinates_are_complete_and_ordered() { + let points = DurabilityCrashPoint::ALL + .into_iter() + .filter(|point| point.sequence() == DurabilityCrashSequence::VersionTwoInitialization) + .count(); + assert_eq!(OPERATIONS.len(), points); assert_eq!(admit(TRANSITIONS), Ok(())); } @@ -63,30 +58,30 @@ fn transition_encoding_rejects_crlf_and_missing_final_newline() -> Result<(), St #[test] fn discard_posture_cannot_be_hidden_in_another_column() { let misplaced = TRANSITIONS.replacen( - "admitted-version-one-store\tabsent-or-incomplete-intent-stage\t\ - complete-intent-stage\tdiscard-incomplete-stage-or-resume-stage-sync", - "discard-incomplete-stage\tabsent-or-incomplete-intent-stage\t\ - complete-intent-stage\tresume-stage-sync", + "locked-root\tabsent-or-incomplete-marker-stage\tcomplete-marker-stage\t\ + discard-incomplete-stage-or-resume-stage-sync", + "discard-incomplete-stage\tabsent-or-incomplete-marker-stage\t\ + complete-marker-stage\tresume-stage-sync", 1, ); assert_ne!(misplaced, TRANSITIONS); - assert_eq!(admit(&misplaced), Err(LedgerRefusal::Posture { row: 0 })); + assert_eq!(admit(&misplaced), Err(LedgerRefusal::Posture { row: 1 })); } #[test] -fn completion_posture_requires_its_exact_field_value() { +fn committed_posture_requires_its_exact_field_value() { let altered = TRANSITIONS.replacen( - "\tadmit-complete-migration\n", - "\tunknown-admit-complete-migration\n", + "\tadmit-committed-store\n", + "\tunknown-admit-committed-store\n", 1, ); - assert_eq!(admit(&altered), Err(LedgerRefusal::Posture { row: 19 })); + assert_eq!(admit(&altered), Err(LedgerRefusal::Posture { row: 9 })); } #[test] fn extra_transition_rows_refuse_before_row_admission() { let extra = format!("{TRANSITIONS}foreign\n"); - assert_eq!(admit(&extra), Err(LedgerRefusal::RowCount { observed: 22 })); + assert_eq!(admit(&extra), Err(LedgerRefusal::RowCount { observed: 12 })); } fn admit(document: &str) -> Result<(), LedgerRefusal> { @@ -120,10 +115,12 @@ fn admit_row(offset: usize, row: &str) -> Result<(), LedgerRefusal> { return Err(LedgerRefusal::Fields { row: offset }); } let ordinal = offset - .checked_add(53) + .checked_add(FIRST_ORDINAL) .ok_or(LedgerRefusal::Coordinate { row: offset })?; if id != format!("KEEP-CRASH-{ordinal:03}") - || phase != "migration" + || DurabilityCrashPoint::from_identifier(id).map(DurabilityCrashPoint::sequence) + != Some(DurabilityCrashSequence::VersionTwoInitialization) + || phase != "initialization" || Some(&operation) != OPERATIONS.get(offset) { return Err(LedgerRefusal::Coordinate { row: offset }); @@ -131,36 +128,25 @@ fn admit_row(offset: usize, row: &str) -> Result<(), LedgerRefusal> { if Some(posture) != expected_posture(operation) { return Err(LedgerRefusal::Posture { row: offset }); } - if id == "KEEP-CRASH-060" && interrupted != "directory-prefix-length-zero-to-six" { - return Err(LedgerRefusal::NamespaceExtent); + if id == "KEEP-CRASH-086" && interrupted != "directory-prefix-length-zero-to-nine" { + return Err(LedgerRefusal::DirectoryExtent); } Ok(()) } fn expected_posture(operation: &str) -> Option<&'static str> { match operation { - "write-intent-stage" | "write-marker-stage" | "write-receipt-stage" => { - Some("discard-incomplete-stage-or-resume-stage-sync") - } - "sync-intent-stage" | "sync-marker-stage" | "sync-receipt-stage" => { - Some("resume-stage-sync") - } - "link-intent" | "link-marker" | "link-receipt" => { - Some("verify-no-clobber-link-and-resume-root-sync") - } - "sync-root-after-intent" - | "sync-root-after-marker" - | "sync-root-after-receipt" - | "sync-root-after-namespace" => Some("resume-root-sync"), - "remove-intent-stage" - | "remove-marker-stage" - | "sync-root-after-intent-cleanup" - | "sync-root-after-marker-cleanup" => Some("resume-cleanup-sync"), - "admit-reader-fence" => Some("resume-namespace-prefix"), - "admit-namespace-prefix" => Some("resume-namespace-prefix-or-root-sync"), - "remove-receipt-stage" | "sync-root-after-receipt-cleanup" => { - Some("admit-complete-migration") - } + "open-and-lock-writer-file" => Some("resume-marker-stage-write"), + "write-marker-stage" => Some("discard-incomplete-stage-or-resume-stage-sync"), + "sync-marker-stage" + | "sync-root-after-marker-stage" + | "admit-reader-fence" + | "admit-directory" => Some("resume-stage-sync"), + "link-marker" => Some("verify-no-clobber-link-and-resume-commit-sync"), + "sync-root-after-marker" => Some("resume-commit-sync"), + "remove-marker-stage" => Some("resume-commit-sync-or-admit-committed-store"), + "sync-root-after-cleanup" => Some("admit-committed-store"), + "discard-incomplete-marker-stage" => Some("discard-incomplete-stage-or-resume-stage-write"), _ => None, } } diff --git a/xtask/tests/segment_store_implementation_documentation.rs b/xtask/tests/segment_store_implementation_documentation.rs index 89deaac5..7698434d 100644 --- a/xtask/tests/segment_store_implementation_documentation.rs +++ b/xtask/tests/segment_store_implementation_documentation.rs @@ -12,7 +12,7 @@ const CORPUS_README: &str = include_str!("../../conformance/segment-store/v1/REA fn version_one_crash_evidence_is_distinguished_from_the_complete_command() { assert!( RECOVERY.contains("105 version-one cases are a subset"), - "the unqualified crash command also executes version-two migration cases" + "the unqualified crash command also executes version-two cases" ); } From 185c903351fe81cd1a92870e5275bf5a8e01d12d Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 09:45:55 -0700 Subject: [PATCH 15/36] Test: public single-format laws for initialization, portability, and acceptance (#198) tests/single_format.rs drives the production entry points under the admitted ext4 profile. Initialization laws: an initialized store reopens but refuses readers as Unpublished, version-1 admission refuses it, a committed store refuses re-initialization unchanged, an uncommitted root refuses writers and readers, and a first publication refuses a stray retention root without a head, leaving the store unchanged. The five ADR-0010 portability laws copy a quiescent store byte for byte into a new directory: the copy opens and reads; corrupt and incomplete copies refuse exactly as the same damage to the original; a replaced writer lock and retention directory still refuse; a copy and its original take independent locks, diverge, and keep the shared segment bytes; and a backup reopens with the original gone, then recovers an interrupted write. The acceptance sequence (ADR-0010 step 4) initializes, ingests, reopens, reconstructs exact bytes, retains, publishes a second catalog, interrupts a third publication before HEAD replacement, recovers through gated finalization, reads both retained blobs, and refuses a lossy candidate with every byte intact. Requirement rows KEEP-INITIALIZATION-006 and KEEP-PORTABILITY-001..005 record the evidence. --- docs/formats/segment-store-v2/requirements.md | 19 +- tests/single_format.rs | 11 ++ tests/single_format/acceptance_laws.rs | 183 ++++++++++++++++++ tests/single_format/initialization_laws.rs | 153 +++++++++++++++ tests/single_format/portability_laws.rs | 183 ++++++++++++++++++ tests/single_format/store_copy.rs | 31 +++ tests/single_format/suite.rs | 45 +++++ tests/support/populated_store.rs | 19 ++ 8 files changed, 642 insertions(+), 2 deletions(-) create mode 100644 tests/single_format.rs create mode 100644 tests/single_format/acceptance_laws.rs create mode 100644 tests/single_format/initialization_laws.rs create mode 100644 tests/single_format/portability_laws.rs create mode 100644 tests/single_format/store_copy.rs create mode 100644 tests/single_format/suite.rs diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 3ea000ad..377eee71 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -43,11 +43,26 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | -| `KEEP-INITIALIZATION-001` | Direct initialization creates exactly the committed namespace under exclusive writer authority, links `FORMAT` from a durable `FORMAT.next` without replacement, and returns authority only after the commit and cleanup synchronizations and complete admission | `a_fresh_root_commits_to_exactly_the_initialized_namespace` in `initialization_resume_tests` | Implemented (#198) | +| `KEEP-INITIALIZATION-001` | Direct initialization creates exactly the committed namespace under exclusive writer authority, links `FORMAT` from a durable `FORMAT.next` without replacement, and returns authority only after the commit and cleanup synchronizations and complete admission | `a_fresh_root_commits_to_exactly_the_initialized_namespace` in `initialization_resume_tests`; production-profile laws in `tests/single_format.rs` | Implemented (#198) | | `KEEP-INITIALIZATION-002` | Every interrupted state classifies as exactly one prefix and resumes idempotently; every other nonempty root, including a committed one, refuses with typed evidence before any mutation, writer-lock creation included | `every_interrupted_boundary_classifies_and_resumes_to_one_store` (every boundary of a recorded run, against a model of the protocol table) and the incomplete-stage and sync-failure laws in `initialization_resume_tests`; foreign, out-of-order, stray-retention, foreign-stage, nonempty-lock, linked-directory, committed, and copied-marker refusals with unchanged witnesses in `initialization_prefix_tests` | Implemented (#198) | -| `KEEP-INITIALIZATION-003` | Admission requires a committed marker: writers and readers refuse `FORMAT.next` as `InitializationIncomplete` and a missing `FORMAT` as `MissingFormatMarker`; a committed store without catalog `HEAD` admits writers, refuses readers as `Unpublished`, and its first publication proves both absences instead of assuming them | `filesystem_version_two_admission_tests` | Implemented (#198) | +| `KEEP-INITIALIZATION-003` | Admission requires a committed marker: writers and readers refuse `FORMAT.next` as `InitializationIncomplete` and a missing `FORMAT` as `MissingFormatMarker`; a committed store without catalog `HEAD` admits writers, refuses readers as `Unpublished`, and its first publication proves both absences instead of assuming them | `filesystem_version_two_admission_tests`; `tests/single_format.rs` initialization laws, including a first publication refused by a stray retention root with an unchanged store | Implemented (#198) | | `KEEP-INITIALIZATION-004` | Process death before, during, and after every initialization boundary reaches a documented state that `initialize` recovers | `cargo xtask durability-crash-matrix --sequence version-two-initialization` runs 41 cases at `KEEP-CRASH-081..=091`, checks the exact inventory, marker bytes and inode, admission result, and resumed state, then publishes, retains, and reconstructs the first blob | Implemented (#198); process death only, not power loss | | `KEEP-INITIALIZATION-005` | Concurrent initialization attempts have exactly one initializer: the other refuses on writer authority and, after commit, as already initialized | `concurrent_attempts_have_exactly_one_initializer` in `initialization_resume_tests` | Implemented (#198) | +| `KEEP-INITIALIZATION-006` | One public-API sequence initializes, ingests, reopens, reconstructs exact bytes, retains, publishes a second catalog, interrupts publication, recovers, and reads both retained contents; a candidate that would lose retained content refuses with its evidence intact | `the_single_format_serves_every_operation_from_initialization_to_recovery` in `tests/single_format.rs` | Implemented (#198) | + + + +## Portability + + + +| ID | Requirement | Evidence | Status | +| --- | --- | --- | --- | +| `KEEP-PORTABILITY-001` | A valid quiescent copy opens and reads the same content; nothing persisted binds a store to its directory | `a_valid_quiescent_copy_opens_and_reads_the_same_content` in `tests/single_format.rs` | Implemented (#198) | +| `KEEP-PORTABILITY-002` | A corrupt or incomplete copy refuses exactly as the original with the same damage would | `a_corrupt_copy_refuses_exactly_as_the_corrupt_original_would` and `an_incomplete_copy_refuses_exactly_as_the_incomplete_original_would` | Implemented (#198) | +| `KEEP-PORTABILITY-003` | A lock file or protocol directory replaced during an operation still refuses | `a_replaced_lock_or_protocol_directory_during_an_operation_still_refuses` | Implemented (#198) | +| `KEEP-PORTABILITY-004` | A copy and its original take independent locks and diverge independently while unchanged content keeps its identity and bytes | `a_copy_and_its_original_diverge_independently_and_keep_shared_identity` | Implemented (#198) | +| `KEEP-PORTABILITY-005` | Backup round trip: close, copy, make the original unavailable, reopen and verify the copy, interrupt a new write on the copy, and recover it | `a_backup_copy_reopens_verifies_and_recovers_an_interrupted_write` | Implemented (#198) | diff --git a/tests/single_format.rs b/tests/single_format.rs new file mode 100644 index 00000000..e863133e --- /dev/null +++ b/tests/single_format.rs @@ -0,0 +1,11 @@ +//! Public laws of the single store format: direct initialization, portable +//! stores, and the integrated acceptance sequence that replaced migration. +//! +//! Every law creates its store through `FilesystemVersionTwoAdmission::initialize` +//! and drives the version-two writer, recovery tools, and durable reader +//! through their public production entry points. + +#![cfg(target_os = "linux")] + +#[path = "single_format/suite.rs"] +mod suite; diff --git a/tests/single_format/acceptance_laws.rs b/tests/single_format/acceptance_laws.rs new file mode 100644 index 00000000..8604a77f --- /dev/null +++ b/tests/single_format/acceptance_laws.rs @@ -0,0 +1,183 @@ +//! This module owns the integrated acceptance sequence and the backup round trip. +//! +//! Both interrupt a real publication through the writer's repository-task +//! observer: `HEAD` replacement fails after `head.next` is synchronized, +//! which is exactly the residue process death there leaves. Recovery then +//! runs through the public, gated next-head finalizer. + +use std::fs; +use std::io; +use std::path::Path; + +use keep::{ + CatalogPublicationBoundary, CatalogPublicationPhase, DurableStore, FilesystemCatalogSnapshot, + FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, FilesystemVersionTwoAdmission, + FilesystemVersionTwoCatalogPublisher, ReaderAttemptLimit, RecoveryEntryRole, + RecoveryNextHeadFinalizationOutcome, RetainedCatalogPublicationError, + VersionTwoInitializationPrefix, +}; + +use super::durable_fixture; +use super::durable_sandbox::TestDirectory; +use super::populated_store; +use super::recovery_fixture::{finalize, stage_roles}; +use super::store_copy::copy_store; +use super::writer_fixture::{ + Predecessor, TestResult, assert_reads, first_blob, open_publisher, pool_segments, publish, + retain, second_blob, witness, +}; + +/// Publishes `content` as the next generation and stops before `HEAD` replacement. +/// +/// Returns the exact pooled segments the staged candidate references. +fn interrupted_publication(root: &Path, content: &[u8]) -> TestResult>> { + let mut candidate = pool_segments(root)?; + let mut publisher = open_publisher(root)?; + publisher.observe_catalog_phases_for_repository_tasks(Box::new(|phase, boundary| { + if phase == CatalogPublicationPhase::ReplaceHead + && boundary == CatalogPublicationBoundary::Before + { + Err(io::Error::other("interrupted before HEAD replacement")) + } else { + Ok(()) + } + })); + let outcome = publish(&mut publisher, root, content, Predecessor::Kept)?; + drop(publisher); + if !matches!( + outcome, + Err(RetainedCatalogPublicationError::Catalog { .. }) + ) { + return Err(format!("the interruption did not stop publication: {outcome:?}").into()); + } + assert_eq!(stage_roles(root)?, [RecoveryEntryRole::NextHeadStage]); + let mut pooled = pool_segments(root)?; + pooled.retain(|segment| !candidate.contains(segment)); + let [staged] = <[Vec; 1]>::try_from(pooled) + .map_err(|_| "the interrupted publication pooled other than one segment")?; + candidate.push(staged); + Ok(candidate) +} + +/// Finalizes the staged head through the gate and requires the expected generation. +fn recover(root: &Path, candidate: &[Vec], generation: u64) -> TestResult<()> { + let receipt = finalize(root, candidate)??; + assert_eq!( + receipt.outcome(), + RecoveryNextHeadFinalizationOutcome::Finalized + ); + assert_eq!(receipt.target().generation().get(), generation); + assert!(stage_roles(root)?.is_empty()); + Ok(()) +} + +fn current_generation(root: &Path) -> TestResult { + Ok( + FilesystemCatalogSnapshot::load(root, populated_store::policy()?)? + .generation() + .get(), + ) +} + +/// ADR-0010 step 4: initialize, ingest, reopen, reconstruct exact bytes, +/// retain, publish a second catalog, interrupt publication, recover, and read +/// both retained contents; a candidate that would lose retained content +/// refuses with its evidence intact. +#[test] +fn the_single_format_serves_every_operation_from_initialization_to_recovery() -> TestResult<()> { + let sandbox = TestDirectory::create("single-format-acceptance")?; + let root = sandbox.path(); + let (first, second) = (first_blob(), second_blob()); + let third = b"keep acceptance: the publication interrupted before HEAD. ".repeat(1_500); + + let initialized = FilesystemVersionTwoAdmission::initialize(root)?; + assert_eq!(initialized.found(), VersionTwoInitializationPrefix::Empty); + let mut writer = FilesystemVersionTwoCatalogPublisher::open( + initialized.into_admission(), + populated_store::policy()?, + )?; + let ingested = populated_store::publish_first(root, &mut writer, &[&first])?; + assert_eq!(ingested.generation().get(), 1); + drop(writer); + + drop(FilesystemVersionTwoAdmission::reopen(root)?); + let layout = populated_store::identify(&first)?.record.id(); + let store = DurableStore::open( + root, + populated_store::policy()?, + ReaderAttemptLimit::DEFAULT, + )?; + let mut output = Vec::new(); + let _receipt = store.reconstruct_layout(layout, &mut output)?; + assert_eq!(output, first, "the first blob must reconstruct exactly"); + + retain(root, b"acceptance-first", &first)?; + let mut writer = open_publisher(root)?; + let _second = publish(&mut writer, root, &second, Predecessor::Kept)??; + drop(writer); + retain(root, b"acceptance-second", &second)?; + assert_eq!(current_generation(root)?, 2); + + let candidate = interrupted_publication(root, &third)?; + assert_eq!( + current_generation(root)?, + 2, + "HEAD must not move before recovery" + ); + recover(root, &candidate, 3)?; + assert_reads(root, &first)?; + assert_reads(root, &second)?; + + let before = witness(root)?; + let mut writer = open_publisher(root)?; + let outcome = publish( + &mut writer, + root, + b"keep acceptance: a lossy candidate", + Predecessor::Dropped, + )?; + drop(writer); + let Err(RetainedCatalogPublicationError::Closure { .. }) = outcome else { + return Err(format!("a candidate losing retained content published: {outcome:?}").into()); + }; + let mut after = witness(root)?; + after.retain(|(path, _)| path != "staging/current.seg"); + assert_eq!( + after, before, + "the refused candidate must leave every byte in place" + ); + assert_reads(root, &first)?; + assert_reads(root, &second) +} + +/// ADR-0010 portability: close, copy, make the original unavailable, reopen +/// and verify the copy, then interrupt a new write on the copy and recover it. +#[test] +fn a_backup_copy_reopens_verifies_and_recovers_an_interrupted_write() -> TestResult<()> { + let first = first_blob(); + let original = durable_fixture::build("backup-original", &[&first])?; + let backup = TestDirectory::create("backup-copy")?; + copy_store(original.path(), backup.path())?; + let moved = original.path().with_extension("unavailable"); + fs::rename(original.path(), &moved)?; + let unavailable = FilesystemRetentionSnapshot::load( + original.path(), + populated_store::policy()?, + ReaderAttemptLimit::DEFAULT, + ); + assert!(matches!( + unavailable, + Err(FilesystemRetentionSnapshotError::Admission { .. }) + )); + + let root = backup.path(); + drop(FilesystemVersionTwoAdmission::reopen(root)?); + assert_reads(root, &first)?; + let candidate = interrupted_publication(root, &second_blob())?; + recover(root, &candidate, 2)?; + retain(root, b"backup-second", &second_blob())?; + assert_reads(root, &first)?; + assert_reads(root, &second_blob())?; + fs::remove_dir_all(moved)?; + Ok(()) +} diff --git a/tests/single_format/initialization_laws.rs b/tests/single_format/initialization_laws.rs new file mode 100644 index 00000000..4cf2d7ee --- /dev/null +++ b/tests/single_format/initialization_laws.rs @@ -0,0 +1,153 @@ +//! This module owns public direct-initialization laws under the production profile. + +use std::fs; + +use keep::{ + FilesystemNamespaceRefusal, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, + FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, FilesystemVersionTwoAdmission, + FilesystemVersionTwoCatalogPublisher, ReaderAttemptLimit, RetainedCatalogPublicationError, + RetentionCurrentStateRefusal, RetentionEvidencePhase, VersionTwoInitializationError, + VersionTwoInitializationPrefix, VersionTwoInitializationRefusal, +}; + +use super::durable_sandbox::TestDirectory; +use super::populated_store; +use super::writer_fixture::{TestResult, first_blob, witness}; + +#[test] +fn an_initialized_store_has_no_catalog_and_admits_writers_but_not_readers() -> TestResult<()> { + let sandbox = TestDirectory::create("single-format-unpublished")?; + let root = sandbox.path(); + + let initialized = FilesystemVersionTwoAdmission::initialize(root)?; + + assert_eq!(initialized.found(), VersionTwoInitializationPrefix::Empty); + drop(initialized); + assert!(!root.join("HEAD").exists()); + drop(FilesystemVersionTwoAdmission::reopen(root)?); + let reader = FilesystemRetentionSnapshot::load( + root, + populated_store::policy()?, + ReaderAttemptLimit::DEFAULT, + ); + assert!(matches!( + reader, + Err(FilesystemRetentionSnapshotError::Unpublished) + )); + Ok(()) +} + +#[test] +fn version_one_admission_refuses_an_initialized_store() -> TestResult<()> { + let sandbox = TestDirectory::create("single-format-v1-refuses")?; + drop(FilesystemVersionTwoAdmission::initialize(sandbox.path())?); + + let refused = FilesystemPlatformAdmission::reopen(sandbox.path()); + + assert!(matches!( + refused, + Err(FilesystemPlatformAdmissionError::Namespace { .. }) + )); + Ok(()) +} + +#[test] +fn a_second_initialization_of_a_committed_store_refuses_unchanged() -> TestResult<()> { + let sandbox = TestDirectory::create("single-format-reinitialize")?; + let root = sandbox.path(); + let (_created, writer) = populated_store::create_holding_authority( + root, + populated_store::StoreAdmission::PLATFORM, + &[&first_blob()], + populated_store::FirstCatalogRecords::Complete, + )?; + drop(writer); + let before = witness(root)?; + + let refused = FilesystemVersionTwoAdmission::initialize(root); + + assert!(matches!( + refused, + Err(VersionTwoInitializationError::Refused { + refusal: VersionTwoInitializationRefusal::AlreadyInitialized + }) + )); + assert_eq!(witness(root)?, before); + Ok(()) +} + +#[test] +fn an_uncommitted_root_is_refused_by_writers_and_readers() -> TestResult<()> { + let sandbox = TestDirectory::create("single-format-uncommitted")?; + let root = sandbox.path(); + drop(FilesystemVersionTwoAdmission::initialize(root)?); + fs::hard_link(root.join("FORMAT"), root.join("FORMAT.next"))?; + + let Err(FilesystemPlatformAdmissionError::Namespace { source }) = + FilesystemVersionTwoAdmission::reopen(root) + else { + return Err("an uncommitted root admitted a writer".into()); + }; + + assert_eq!( + source + .get_ref() + .and_then(|cause| cause.downcast_ref::()), + Some(&FilesystemNamespaceRefusal::InitializationIncomplete) + ); + assert!(matches!( + FilesystemRetentionSnapshot::load( + root, + populated_store::policy()?, + ReaderAttemptLimit::DEFAULT + ), + Err(FilesystemRetentionSnapshotError::Admission { .. }) + )); + assert_eq!( + FilesystemVersionTwoAdmission::initialize(root)?.found(), + VersionTwoInitializationPrefix::MarkerLinked + ); + Ok(()) +} + +/// The first publication has no predecessor, so it must prove that no +/// retention state exists instead of assuming it: a retained root without a +/// retention head contradicts the absence and refuses before any effect. +#[test] +fn the_first_publication_refuses_retention_artifacts_without_a_head() -> TestResult<()> { + let sandbox = TestDirectory::create("single-format-first-contradicted")?; + let root = sandbox.path(); + let admission = FilesystemVersionTwoAdmission::initialize(root)?.into_admission(); + let namespace = root.join("retention/roots").join("a".repeat(64)); + fs::create_dir(&namespace)?; + let before = witness(root)?; + let mut publisher = + FilesystemVersionTwoCatalogPublisher::open(admission, populated_store::policy()?)?; + + let refused = populated_store::publish_first(root, &mut publisher, &[&first_blob()]); + drop(publisher); + + let error = refused + .err() + .ok_or("a contradicted first publication published")?; + let Some(RetainedCatalogPublicationError::RetentionStorage { + phase: RetentionEvidencePhase::AdmitHead, + source, + }) = error.downcast_ref::() + else { + return Err(format!("unexpected first-publication refusal: {error:?}").into()); + }; + assert!(matches!( + source + .get_ref() + .and_then(|cause| cause.downcast_ref::()), + Some(RetentionCurrentStateRefusal::HeadAbsentWithArtifacts) + )); + let mut after = witness(root)?; + after.retain(|(path, _)| path != "staging/current.seg"); + assert_eq!( + after, before, + "a refused first publication mutated the store" + ); + Ok(()) +} diff --git a/tests/single_format/portability_laws.rs b/tests/single_format/portability_laws.rs new file mode 100644 index 00000000..2841c668 --- /dev/null +++ b/tests/single_format/portability_laws.rs @@ -0,0 +1,183 @@ +//! This module owns the ADR-0010 portability laws: a store is not bound to its directory. +//! +//! Each law copies a quiescent store byte for byte into a new directory, so +//! every inode differs, including the root's. Admission must treat the copy +//! as the same store, refuse a damaged copy exactly as it refuses the +//! damaged original, and keep every live check. + +use std::fs; +use std::io; +use std::path::Path; + +use keep::{ + FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, + FilesystemVersionTwoCatalogPublisher, RetainedCatalogPublicationError, + RetentionCurrentStateRefusal, VersionTwoRecordRefusal, WriterLockAcquireError, +}; + +use super::durable_fixture; +use super::durable_sandbox::TestDirectory; +use super::populated_store; +use super::store_copy::copy_store; +use super::writer_fixture::{ + Predecessor, TestResult, assert_reads, first_blob, open_publisher, pool_segments, publish, + retain, second_blob, +}; + +/// A quiescent copy of a retained store under a new directory. +fn copied(name: &str, original: &Path) -> TestResult { + let copy = TestDirectory::create(name)?; + copy_store(original, copy.path())?; + Ok(copy) +} + +#[test] +fn a_valid_quiescent_copy_opens_and_reads_the_same_content() -> TestResult<()> { + let first = first_blob(); + let original = durable_fixture::build("portable-original", &[&first])?; + let copy = copied("portable-copy", original.path())?; + + drop(FilesystemVersionTwoAdmission::reopen(copy.path())?); + + assert_reads(copy.path(), &first) +} + +/// The same damage refuses with the same typed cause in the copy and the original. +#[test] +fn a_corrupt_copy_refuses_exactly_as_the_corrupt_original_would() -> TestResult<()> { + let original = durable_fixture::build("portable-corrupt-original", &[&first_blob()])?; + let copy = copied("portable-corrupt-copy", original.path())?; + for root in [original.path(), copy.path()] { + let marker = root.join("FORMAT"); + let mut bytes = fs::read(&marker)?; + *bytes.get_mut(40).ok_or("marker shorter than 41 bytes")? ^= 1; + fs::write(&marker, &bytes)?; + } + + let original_refusal = marker_refusal(original.path())?; + let copy_refusal = marker_refusal(copy.path())?; + + assert_eq!(copy_refusal, original_refusal); + Ok(()) +} + +#[test] +fn an_incomplete_copy_refuses_exactly_as_the_incomplete_original_would() -> TestResult<()> { + let original = durable_fixture::build("portable-incomplete-original", &[&first_blob()])?; + let copy = copied("portable-incomplete-copy", original.path())?; + for root in [original.path(), copy.path()] { + fs::remove_dir(root.join("gc"))?; + } + + let original_refusal = namespace_refusal_kind(original.path())?; + let copy_refusal = namespace_refusal_kind(copy.path())?; + + assert_eq!(copy_refusal, original_refusal); + assert_eq!(copy_refusal, io::ErrorKind::NotFound); + Ok(()) +} + +/// Live authority still binds the copy's own lock file and directories: a +/// writer lock replaced while a writer holds the store refuses a second +/// writer, and a retention directory replaced after admission refuses +/// publication. +#[test] +fn a_replaced_lock_or_protocol_directory_during_an_operation_still_refuses() -> TestResult<()> { + let original = durable_fixture::build("portable-replaced-original", &[&first_blob()])?; + let copy = copied("portable-replaced-copy", original.path())?; + let root = copy.path(); + let held = FilesystemVersionTwoAdmission::reopen(root)?; + fs::remove_file(root.join("writer.lock"))?; + fs::write(root.join("writer.lock"), b"")?; + + let contended = FilesystemVersionTwoAdmission::reopen(root); + + assert!(matches!( + contended, + Err(FilesystemPlatformAdmissionError::WriterLock { + source: WriterLockAcquireError::Busy + }) + )); + fs::rename(root.join("retention"), root.join("retention.moved"))?; + fs::create_dir_all(root.join("retention/roots"))?; + fs::create_dir(root.join("retention/manifests"))?; + let mut publisher = + FilesystemVersionTwoCatalogPublisher::open(held, populated_store::policy()?)?; + let outcome = publish(&mut publisher, root, &second_blob(), Predecessor::Kept)?; + let Err(RetainedCatalogPublicationError::RetentionStorage { source, .. }) = outcome else { + return Err(format!("publication used a replaced retention directory: {outcome:?}").into()); + }; + assert!(matches!( + source + .get_ref() + .and_then(|cause| cause.downcast_ref::()), + Some(RetentionCurrentStateRefusal::ProtocolDirectoryReplaced) + )); + Ok(()) +} + +/// A copy and its original are two stores: both take writer authority at +/// once, each publishes its own successor, and the content they still share +/// keeps its exact identity and bytes in both. +#[test] +fn a_copy_and_its_original_diverge_independently_and_keep_shared_identity() -> TestResult<()> { + let first = first_blob(); + let original = durable_fixture::build("portable-diverge-original", &[&first])?; + let copy = copied("portable-diverge-copy", original.path())?; + let second = second_blob(); + let other = b"keep portability: content only the copy publishes. ".repeat(2_000); + let shared = pool_segments(original.path())?; + let mut original_writer = open_publisher(original.path())?; + let mut copy_writer = open_publisher(copy.path())?; + + let _original = publish( + &mut original_writer, + original.path(), + &second, + Predecessor::Kept, + )??; + let _copy = publish(&mut copy_writer, copy.path(), &other, Predecessor::Kept)??; + drop((original_writer, copy_writer)); + retain(original.path(), b"original-only", &second)?; + retain(copy.path(), b"copy-only", &other)?; + + for root in [original.path(), copy.path()] { + assert_reads(root, &first)?; + } + assert_reads(original.path(), &second)?; + assert_reads(copy.path(), &other)?; + for root in [original.path(), copy.path()] { + let pool = pool_segments(root)?; + assert!( + shared.iter().all(|segment| pool.contains(segment)), + "unchanged content must keep its exact segment bytes in both stores" + ); + } + assert_ne!( + fs::read(original.path().join("HEAD"))?, + fs::read(copy.path().join("HEAD"))?, + "independent publications must produce different catalogs" + ); + Ok(()) +} + +fn marker_refusal(root: &Path) -> TestResult { + let Err(FilesystemPlatformAdmissionError::FormatMarker { source }) = + FilesystemVersionTwoAdmission::reopen(root) + else { + return Err("a corrupt marker was not refused at the marker boundary".into()); + }; + let refusal = source + .get_ref() + .and_then(|cause| cause.downcast_ref::()) + .ok_or("the marker refusal lost its typed cause")?; + Ok(format!("{refusal:?}")) +} + +fn namespace_refusal_kind(root: &Path) -> TestResult { + match FilesystemVersionTwoAdmission::reopen(root) { + Err(FilesystemPlatformAdmissionError::Namespace { source }) => Ok(source.kind()), + Err(other) => Err(format!("unexpected incomplete-store refusal: {other:?}").into()), + Ok(_) => Err("an incomplete store was admitted".into()), + } +} diff --git a/tests/single_format/store_copy.rs b/tests/single_format/store_copy.rs new file mode 100644 index 00000000..c97d4a1d --- /dev/null +++ b/tests/single_format/store_copy.rs @@ -0,0 +1,31 @@ +//! This module owns byte-exact copies of a quiescent store for portability laws. +//! +//! A copy is what a backup or a move to another disk produces: every +//! directory and regular file recreated with the same bytes under a new root, +//! so every inode, and the root's own device-and-inode pair, differs. + +use std::fs; +use std::path::Path; + +use super::writer_fixture::TestResult; + +/// Copies the complete store at `source` into the empty directory `destination`. +pub(super) fn copy_store(source: &Path, destination: &Path) -> TestResult<()> { + let mut pending = vec![(source.to_path_buf(), destination.to_path_buf())]; + while let Some((from, to)) = pending.pop() { + for entry in fs::read_dir(&from)? { + let entry = entry?; + let kind = entry.file_type()?; + let target = to.join(entry.file_name()); + if kind.is_dir() { + fs::create_dir(&target)?; + pending.push((entry.path(), target)); + } else if kind.is_file() { + fs::copy(entry.path(), &target)?; + } else { + return Err(format!("{} is not a store entry", entry.path().display()).into()); + } + } + } + Ok(()) +} diff --git a/tests/single_format/suite.rs b/tests/single_format/suite.rs new file mode 100644 index 00000000..52b37ace --- /dev/null +++ b/tests/single_format/suite.rs @@ -0,0 +1,45 @@ +//! This module owns the single-format suite's fixture imports. + +#[cfg(feature = "repository-tasks")] +#[path = "acceptance_laws.rs"] +mod acceptance_laws; +#[path = "../golden_file_worldline/durable_fixture.rs"] +#[allow( + dead_code, + reason = "these laws use complete stores; golden laws cover partial-record construction" +)] +mod durable_fixture; +#[path = "../segment_filesystem_stage/sandbox.rs"] +#[allow( + dead_code, + reason = "these laws use Drop cleanup; other filesystem laws check explicit removal" +)] +mod durable_sandbox; +#[path = "initialization_laws.rs"] +mod initialization_laws; +#[path = "../support/populated_store.rs"] +#[allow( + dead_code, + reason = "each consumer uses a subset of the shared builder; none reads every receipt" +)] +mod populated_store; +#[path = "portability_laws.rs"] +mod portability_laws; +#[path = "../version_two_catalog_writer/recovery_fixture.rs"] +#[allow(dead_code, reason = "these laws use finalization only")] +mod recovery_fixture; +#[path = "store_copy.rs"] +mod store_copy; +#[path = "../support/successor_catalog.rs"] +#[allow( + dead_code, + reason = "these laws publish successors only through `publish`" +)] +mod successor_catalog; +#[path = "../version_two_catalog_writer/writer_fixture.rs"] +#[allow( + dead_code, + unused_imports, + reason = "these laws use a subset of the writer fixture and its re-exports" +)] +mod writer_fixture; diff --git a/tests/support/populated_store.rs b/tests/support/populated_store.rs index 298316b1..e2688d66 100644 --- a/tests/support/populated_store.rs +++ b/tests/support/populated_store.rs @@ -163,6 +163,25 @@ pub(super) fn create_holding_authority( Ok((CreatedStore { catalog }, publisher)) } +/// Publishes every record of `contents` as the first catalog of an initialized store. +/// +/// The publication has no predecessor: it runs with the uninitialized +/// expectation and proves through the gate that neither catalog `HEAD` nor +/// `retention/HEAD` exists uncontradicted. +pub(super) fn publish_first( + root: &Path, + publisher: &mut FilesystemVersionTwoCatalogPublisher, + contents: &[&[u8]], +) -> TestResult { + let identified = contents + .iter() + .map(|bytes| identify(bytes)) + .collect::>>()?; + let blobs = contents.iter().copied().zip(&identified); + let records = select_records(blobs, FirstCatalogRecords::Complete)?; + publish_first_catalog(root, publisher, records) +} + /// Every selected chunk record, deduplicated across blobs, then each new layout. pub(super) fn select_records<'content>( blobs: impl IntoIterator, From 43c9ed319089090d48bd3325fd3de7b41018b259 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 11:04:16 -0700 Subject: [PATCH 16/36] Style: drop the trailing blank line in the marker oracle (#198) --- .../retention_store_v2_format_oracle/artifacts/format_marker.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/xtask/tests/retention_store_v2_format_oracle/artifacts/format_marker.rs b/xtask/tests/retention_store_v2_format_oracle/artifacts/format_marker.rs index e068f2de..b2ccab3a 100644 --- a/xtask/tests/retention_store_v2_format_oracle/artifacts/format_marker.rs +++ b/xtask/tests/retention_store_v2_format_oracle/artifacts/format_marker.rs @@ -25,4 +25,3 @@ fn build_format_marker(definition_digest: [u8; 32]) -> Result bytes, }) } - From 7d3e146c7b4c9d121b7b68a2e141748d266abb72 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 10:00:31 -0700 Subject: [PATCH 17/36] Test: run the catalog-protocol crash rows on single-format stores (#198) KEEP-CRASH-001..030 name the segment, catalog, head, and recovery-discard boundaries of the shared catalog stages, which the single-format writer executes unchanged. Their crash children now create the store by direct initialization and publish generation one through FilesystemVersionTwoCatalogPublisher and the retained-closure gate instead of the version-1 publisher. The driver observes the writer's own segment-stage port and catalog transitions; the version-1 CatalogPublicationStorage decorator is deleted. A `during` catalog or head write writes the same strict prefix into the stage the writer created and dies before the writer's own write. The recovery-discard rows open the discarder through version-2 recovery admission. Their head precondition is now a writer publication interrupted at the head write, so both pools hold the published segment and catalog. Restart inspection expects the committed single-format inventory, including the corpus FORMAT bytes, and admits every residue through version-2 recovery admission in place of a bare writer-lock reacquire. The rows keep their identifiers because each boundary is the same code. --- docs/formats/segment-store-v1/recovery.md | 21 +- .../production_protocol.rs | 3 +- .../production_protocol/fixture.rs | 7 + .../production_protocol/initialization.rs | 14 +- .../production_protocol/publication.rs | 243 +++++++++++++--- .../publication_storage.rs | 270 ------------------ .../production_protocol/recovery.rs | 107 +++++-- xtask/src/durability_crash_matrix/restart.rs | 9 +- .../restart/expectation.rs | 59 +++- .../restart/expectation/sequence.rs | 13 + .../restart/semantic.rs | 30 +- .../durability_crash_production_contract.rs | 1 - 12 files changed, 407 insertions(+), 370 deletions(-) delete mode 100644 xtask/src/durability_crash_matrix/production_protocol/publication_storage.rs diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 91f3ce5c..14435a02 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -142,14 +142,19 @@ deadline bounds failure handling; successful synchronization does not depend on elapsed time, sleeps, or test ordering. The production protocol driver and expected-state model are separate -implementations. The driver delegates every target mutation to the ordinary -filesystem adapters. Repository-only partial-write methods share the same -catalog and head stage writers and stop at deterministic proper prefixes; -they do not hand-construct a replacement namespace. Restart inspection proves -the complete path set and exact Golden File Worldline bytes. It additionally -verifies hard-link identity at link transitions, reacquires `writer.lock` -after process death, classifies `current.seg`, `current.cat`, and `head.next` -through the production recovery classifiers, admits immutable segment and +implementations. The segment, catalog, head, and recovery-discard rows run on +a store created by direct initialization and published through the +single-format writer and the retained-closure gate; the driver observes the +writer's own segment-stage port and catalog transitions and delegates every +target mutation to them. A `during` catalog or head write is the crash itself: +the driver writes a deterministic proper prefix into the stage file the writer +created and dies before the writer's own write. It does not hand-construct a +namespace. Restart inspection proves the complete path set, including the +canonical `FORMAT` marker, and exact Golden File Worldline bytes. It +additionally verifies hard-link identity at link transitions, admits every +catalog-protocol residue through single-format recovery admission, classifies +`current.seg`, `current.cat`, and `head.next` through the production recovery +classifiers, admits immutable segment and catalog bytes through the production decoders, and loads the exact generation-1 snapshot when `HEAD` is present. That snapshot must expose the one-zero chunk with the exact payload `00`. diff --git a/xtask/src/durability_crash_matrix/production_protocol.rs b/xtask/src/durability_crash_matrix/production_protocol.rs index c68dbb0d..f2030d06 100644 --- a/xtask/src/durability_crash_matrix/production_protocol.rs +++ b/xtask/src/durability_crash_matrix/production_protocol.rs @@ -7,7 +7,6 @@ pub(super) mod initialization; mod initialization_storage; pub(super) mod populated_store; mod publication; -mod publication_storage; mod recovery; mod recovery_storage; pub(super) mod retention; @@ -42,7 +41,7 @@ pub(super) fn run( DurabilityCrashSequence::Segment | DurabilityCrashSequence::Catalog | DurabilityCrashSequence::Head => { - publication::run(&store_root, &mut control)?; + publication::run(&store_root, control)?; } DurabilityCrashSequence::RecoveryDiscard => { recovery::run(&store_root, &mut control)?; diff --git a/xtask/src/durability_crash_matrix/production_protocol/fixture.rs b/xtask/src/durability_crash_matrix/production_protocol/fixture.rs index 4f6f1b08..96cadc37 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/fixture.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/fixture.rs @@ -19,6 +19,8 @@ const BUNDLE_HEAD_HEX: &str = include_str!("../../../../conformance/segment-store/v1/one-zero-bundle-head.hex"); const RETENTION_ROOT_HEX: &str = include_str!("../../../../conformance/segment-store/v2/one-anchor-root.hex"); +const FORMAT_MARKER_HEX: &str = + include_str!("../../../../conformance/segment-store/v2/format-marker.hex"); /// Pool name of the bundle segment the retention root's closure references. pub(in crate::durability_crash_matrix) const BUNDLE_SEGMENT_NAME: &str = "221f6745cd8a5221c9a87c3707593608479282b54a4a74d0e753fd76f70e8db2.seg"; @@ -69,6 +71,11 @@ impl GoldenFixture { Self::decode("head", HEAD_HEX, 128) } + pub(in crate::durability_crash_matrix) fn format_marker() + -> Result { + Self::decode("format marker", FORMAT_MARKER_HEX, 96) + } + pub(in crate::durability_crash_matrix) fn bytes(&self) -> &[u8] { &self.bytes } diff --git a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs index 6ee9cea9..ac7f4c82 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs @@ -3,8 +3,8 @@ use std::path::Path; use keep::{ - CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, - FilesystemPlatformAdmission, LayoutEntryLimit, RepositoryInitializationStorage, + CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemVersionTwoAdmission, + FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, RepositoryInitializationStorage, SegmentReadPolicy, SegmentRecordLimit, initialize_store, }; @@ -26,12 +26,14 @@ pub(super) fn run( .map_err(|source| verification("execute production store initialization", source)) } +/// Initializes the single-format store and opens its writer, as production does. pub(super) fn publisher( store_root: &Path, -) -> Result { - let admission = FilesystemPlatformAdmission::initialize(store_root) - .map_err(|source| verification("admit crash catalog publisher platform", source))?; - FilesystemCatalogPublisher::open(admission, restart_policy()?) +) -> Result { + let admission = FilesystemVersionTwoAdmission::initialize(store_root) + .map_err(|source| verification("initialize crash catalog store", source))? + .into_admission(); + FilesystemVersionTwoCatalogPublisher::open(admission, restart_policy()?) .map_err(|source| DurabilityCrashMatrixError::io("open crash catalog publisher", source)) } diff --git a/xtask/src/durability_crash_matrix/production_protocol/publication.rs b/xtask/src/durability_crash_matrix/production_protocol/publication.rs index b692ab68..14fbd269 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/publication.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/publication.rs @@ -1,26 +1,92 @@ //! This module owns execution of the production publication protocols. +//! +//! The child initializes a store directly and publishes its first catalog +//! through `FilesystemVersionTwoCatalogPublisher` and the retained-closure +//! gate. Segment staging boundaries are observed through the segment-stage +//! port and every pool, catalog, and head boundary through the writer's own +//! catalog observer, so no transition is reimplemented here. use std::fs; -use std::path::Path; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; use keep::{ - AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, - CatalogPublicationExpectation, ObservedSegmentStage, SegmentRecordLimit, StagedSegment, - publish_catalog_generation, + AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CanonicalPublicationHead, + CatalogGeneration, CatalogPublicationBoundary, CatalogPublicationExpectation, + CatalogPublicationObserver, CatalogPublicationPhase, FilesystemSegmentStage, + FilesystemVersionTwoCatalogPublisher, ObservedSegmentStage, SealedSegment, SegmentRecordLimit, + StagedSegment, publish_retained_catalog_generation, }; -use xtask::DurabilityCrashPoint; +use xtask::{DurabilityCrashPoint, DurabilityCrashPosition}; use super::control::{CrashControl, DuringTiming}; use super::initialization; -use super::publication_storage::CrashPublicationStorage; use super::segment_stage::CrashSegmentObserver; use super::{DurabilityCrashMatrixError, verification}; +const CATALOG_INTERRUPTION: usize = 176; +const HEAD_INTERRUPTION: usize = 64; +const CATALOG_STAGE: &str = "staging/current.cat"; +const HEAD_STAGE: &str = "head.next"; + pub(super) fn run( store_root: &Path, - control: &mut CrashControl, + mut control: CrashControl, ) -> Result<(), DurabilityCrashMatrixError> { - let publisher = initialization::publisher(store_root)?; + let mut publisher = initialization::publisher(store_root)?; + let sealed = stage_observed(&publisher, &mut control)?; + let segment_bytes = fs::read(store_root.join("staging/current.seg")) + .map_err(|source| DurabilityCrashMatrixError::io("read sealed crash segment", source))?; + let segments = [ + AdmittedSegment::decode(&segment_bytes, initialization::segment_policy()) + .map_err(|source| verification("admit sealed crash segment", source))?, + ]; + let catalog = first_catalog(&segments)?; + let selection = publisher + .select_segment(sealed, &segments[0]) + .map_err(|source| verification("select production segment stage", source))?; + let torn = TornWrites::new(store_root, &catalog)?; + publisher.observe_catalog_phases_for_repository_tasks(observer(control, torn)); + publish_retained_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + selection, + &catalog, + &segments, + ) + .map(|_receipt| ()) + .map_err(|source| verification("execute production catalog publication", source)) +} + +/// Encodes generation one over `segments`, the catalog every crash row publishes. +pub(super) fn first_catalog( + segments: &[AdmittedSegment<'_>], +) -> Result { + let generation = CatalogGeneration::new(1) + .map_err(|source| verification("construct crash catalog generation", source))?; + CanonicalCatalog::from_segments(generation, None, segments) + .map_err(|source| verification("encode crash catalog", source)) +} + +/// Stages and seals the one-zero segment with no crash observation. +pub(super) fn stage_one_zero( + publisher: &FilesystemVersionTwoCatalogPublisher, +) -> Result>, DurabilityCrashMatrixError> { + let stage = publisher + .create_segment_stage() + .map_err(|source| verification("create production segment stage", source))?; + StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM) + .map_err(|source| verification("write production segment header", source))? + .append(one_zero_record()?) + .map_err(|source| verification("append production segment record", source))? + .seal() + .map_err(|source| verification("seal production segment", source)) +} + +fn stage_observed<'publisher>( + publisher: &'publisher FilesystemVersionTwoCatalogPublisher, + control: &mut CrashControl, +) -> Result>, DurabilityCrashMatrixError> { let point = DurabilityCrashPoint::CreateSegmentStage; control .before(point, DuringTiming::After) @@ -31,42 +97,145 @@ pub(super) fn run( control .after(point, DuringTiming::After) .map_err(crash_gate)?; - let stage = ObservedSegmentStage::new(stage, CrashSegmentObserver::new(control)); - let record = AdmittedSegmentRecord::for_chunk(&[0]) - .map_err(|source| verification("admit crash segment record", source))?; - let sealed = StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM) + Ok(StagedSegment::begin(stage, SegmentRecordLimit::MAXIMUM) .map_err(|source| verification("write production segment header", source))? - .append(record) + .append(one_zero_record()?) .map_err(|source| verification("append production segment record", source))? .seal() .map_err(|source| verification("seal production segment", source))? - .without_observer(); + .without_observer()) +} - let segment_bytes = fs::read(store_root.join("staging/current.seg")) - .map_err(|source| DurabilityCrashMatrixError::io("read sealed crash segment", source))?; - let segment = AdmittedSegment::decode(&segment_bytes, initialization::segment_policy()) - .map_err(|source| verification("admit sealed crash segment", source))?; - let segments = [segment]; - let generation = CatalogGeneration::new(1) - .map_err(|source| verification("construct crash catalog generation", source))?; - let catalog = CanonicalCatalog::from_segments(generation, None, &segments) - .map_err(|source| verification("encode crash catalog", source))?; - let selection = publisher - .select_segment(sealed, &segments[0]) - .map_err(|source| verification("select production segment stage", source))?; - let mut storage = CrashPublicationStorage::new(publisher, control); - publish_catalog_generation( - &mut storage, - CatalogPublicationExpectation::uninitialized(), - selection, - &catalog, - &segments, - ) - .map(|_receipt| ()) - .map_err(|source| verification("execute production catalog publication", source)) +fn one_zero_record() -> Result, DurabilityCrashMatrixError> { + AdmittedSegmentRecord::for_chunk(&[0]) + .map_err(|source| verification("admit crash segment record", source)) +} + +/// The strict prefixes a `during` write leaves in each fixed stage. +struct TornWrites { + catalog_stage: PathBuf, + catalog: Vec, + head_stage: PathBuf, + head: Vec, +} + +impl TornWrites { + fn new( + store_root: &Path, + catalog: &CanonicalCatalog, + ) -> Result { + let head = CanonicalPublicationHead::for_catalog(catalog.checksummed()); + Ok(Self { + catalog_stage: store_root.join(CATALOG_STAGE), + catalog: prefix(catalog.encoded(), CATALOG_INTERRUPTION)?, + head_stage: store_root.join(HEAD_STAGE), + head: prefix(head.encoded(), HEAD_INTERRUPTION)?, + }) + } + + fn select( + &self, + phase: CatalogPublicationPhase, + ) -> Option<(DurabilityCrashPoint, &Path, &[u8])> { + match phase { + CatalogPublicationPhase::WriteCatalog => Some(( + DurabilityCrashPoint::WriteCatalog, + &self.catalog_stage, + &self.catalog, + )), + CatalogPublicationPhase::WriteHead => Some(( + DurabilityCrashPoint::WriteHead, + &self.head_stage, + &self.head, + )), + _ => None, + } + } +} + +fn prefix(encoded: &[u8], length: usize) -> Result, DurabilityCrashMatrixError> { + encoded + .get(..length) + .map(<[u8]>::to_vec) + .ok_or(DurabilityCrashMatrixError::FixtureRange) +} + +/// Maps the writer's own transitions onto `KEEP-CRASH-009` through `-026`. +fn observer(mut control: CrashControl, torn: TornWrites) -> CatalogPublicationObserver { + Box::new(move |phase, boundary| { + if let Some((point, stage, bytes)) = torn.select(phase) { + return interrupt_write(&mut control, point, boundary, stage, bytes); + } + let Some((point, timing)) = crash_point(phase) else { + return Ok(()); + }; + match boundary { + CatalogPublicationBoundary::Before => control.before(point, timing), + CatalogPublicationBoundary::After => control.after(point, timing), + } + }) +} + +/// Dies before, inside, or after one stage write. +/// +/// The `during` case is the crash itself: the strict prefix reaches the stage +/// file the writer created and the process dies before the writer's own +/// write runs. +fn interrupt_write( + control: &mut CrashControl, + point: DurabilityCrashPoint, + boundary: CatalogPublicationBoundary, + stage: &Path, + bytes: &[u8], +) -> io::Result<()> { + match (boundary, control.position(point)) { + (CatalogPublicationBoundary::Before, Some(DurabilityCrashPosition::Before)) + | (CatalogPublicationBoundary::After, Some(DurabilityCrashPosition::After)) => { + control.await_process_death() + } + (CatalogPublicationBoundary::Before, Some(DurabilityCrashPosition::During)) => { + fs::OpenOptions::new() + .write(true) + .open(stage)? + .write_all(bytes)?; + control.await_process_death() + } + _ => Ok(()), + } +} + +const fn crash_point( + phase: CatalogPublicationPhase, +) -> Option<(DurabilityCrashPoint, DuringTiming)> { + use CatalogPublicationPhase as Phase; + use DurabilityCrashPoint as Point; + let mapped = match phase { + Phase::LinkSegment => (Point::LinkSegment, DuringTiming::After), + Phase::SynchronizeSegments => (Point::SynchronizeSegmentPool, DuringTiming::Before), + Phase::RemoveSegmentStage => (Point::RemoveSegmentStage, DuringTiming::After), + Phase::SynchronizeStagingAfterSegment => { + (Point::SynchronizeStagingAfterSegment, DuringTiming::Before) + } + Phase::CreateCatalogStage => (Point::CreateCatalogStage, DuringTiming::After), + Phase::FlushCatalog => (Point::FlushCatalog, DuringTiming::Before), + Phase::SynchronizeCatalog => (Point::SynchronizeCatalog, DuringTiming::Before), + Phase::LinkCatalog => (Point::LinkCatalog, DuringTiming::After), + Phase::SynchronizeCatalogs => (Point::SynchronizeCatalogPool, DuringTiming::Before), + Phase::RemoveCatalogStage => (Point::RemoveCatalogStage, DuringTiming::After), + Phase::SynchronizeStagingAfterCatalog => { + (Point::SynchronizeStagingAfterCatalog, DuringTiming::Before) + } + Phase::CreateHeadStage => (Point::CreateHeadStage, DuringTiming::After), + Phase::FlushHead => (Point::FlushHead, DuringTiming::Before), + Phase::SynchronizeHead => (Point::SynchronizeHead, DuringTiming::Before), + Phase::ReplaceHead => (Point::ReplaceHead, DuringTiming::After), + Phase::SynchronizeRoot => (Point::SynchronizeRootAfterHead, DuringTiming::Before), + _ => return None, + }; + Some(mapped) } -const fn crash_gate(source: std::io::Error) -> DurabilityCrashMatrixError { +const fn crash_gate(source: io::Error) -> DurabilityCrashMatrixError { DurabilityCrashMatrixError::io("await production crash boundary", source) } diff --git a/xtask/src/durability_crash_matrix/production_protocol/publication_storage.rs b/xtask/src/durability_crash_matrix/production_protocol/publication_storage.rs deleted file mode 100644 index 55206443..00000000 --- a/xtask/src/durability_crash_matrix/production_protocol/publication_storage.rs +++ /dev/null @@ -1,270 +0,0 @@ -//! This module owns crash injection around production catalog publication. - -use std::io; - -use keep::{ - AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, - CatalogPublicationReadiness, CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, - FilesystemCatalogPublisher, SegmentPublication, -}; -use xtask::{DurabilityCrashPoint, DurabilityCrashPosition}; - -use super::control::{CrashControl, DuringTiming}; - -const CATALOG_INTERRUPTION: usize = 176; -const HEAD_INTERRUPTION: usize = 64; - -pub(super) struct CrashPublicationStorage<'control> { - inner: FilesystemCatalogPublisher, - control: &'control mut CrashControl, -} - -impl<'control> CrashPublicationStorage<'control> { - pub(super) const fn new( - inner: FilesystemCatalogPublisher, - control: &'control mut CrashControl, - ) -> Self { - Self { inner, control } - } -} - -impl CatalogPublicationStorage for CrashPublicationStorage<'_> { - fn verify_current( - &mut self, - expected: CatalogPublicationExpectation, - candidate: &CatalogSnapshot<'_, '_, '_>, - segment: &SegmentPublication<'_, '_>, - ) -> io::Result { - self.inner.verify_current(expected, candidate, segment) - } - - fn link_segment(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::LinkSegment, - DuringTiming::After, - |inner| inner.link_segment(segment), - ) - } - - fn verify_segment_pool(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { - self.inner.verify_segment_pool(segment) - } - - fn synchronize_segments(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::SynchronizeSegmentPool, - DuringTiming::Before, - FilesystemCatalogPublisher::synchronize_segments, - ) - } - - fn remove_segment_stage(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::RemoveSegmentStage, - DuringTiming::After, - FilesystemCatalogPublisher::remove_segment_stage, - ) - } - - fn synchronize_staging_after_segment(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::SynchronizeStagingAfterSegment, - DuringTiming::Before, - FilesystemCatalogPublisher::synchronize_staging_after_segment, - ) - } - - fn create_catalog_stage(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::CreateCatalogStage, - DuringTiming::After, - FilesystemCatalogPublisher::create_catalog_stage, - ) - } - - fn write_catalog(&mut self, catalog: &CanonicalCatalog) -> io::Result<()> { - execute_write( - &mut self.inner, - self.control, - DurabilityCrashPoint::WriteCatalog, - |inner| inner.write_catalog(catalog), - |inner| inner.write_catalog_prefix_for_repository_tasks(catalog, CATALOG_INTERRUPTION), - ) - } - - fn flush_catalog(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::FlushCatalog, - DuringTiming::Before, - FilesystemCatalogPublisher::flush_catalog, - ) - } - - fn synchronize_catalog(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::SynchronizeCatalog, - DuringTiming::Before, - FilesystemCatalogPublisher::synchronize_catalog, - ) - } - - fn link_catalog(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::LinkCatalog, - DuringTiming::After, - |inner| inner.link_catalog(catalog), - ) - } - - fn verify_catalog_pool(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { - self.inner.verify_catalog_pool(catalog) - } - - fn synchronize_catalogs(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::SynchronizeCatalogPool, - DuringTiming::Before, - FilesystemCatalogPublisher::synchronize_catalogs, - ) - } - - fn remove_catalog_stage(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::RemoveCatalogStage, - DuringTiming::After, - FilesystemCatalogPublisher::remove_catalog_stage, - ) - } - - fn synchronize_staging_after_catalog(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::SynchronizeStagingAfterCatalog, - DuringTiming::Before, - FilesystemCatalogPublisher::synchronize_staging_after_catalog, - ) - } - - fn create_head_stage(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::CreateHeadStage, - DuringTiming::After, - FilesystemCatalogPublisher::create_head_stage, - ) - } - - fn write_head(&mut self, head: &CanonicalPublicationHead) -> io::Result<()> { - execute_write( - &mut self.inner, - self.control, - DurabilityCrashPoint::WriteHead, - |inner| inner.write_head(head), - |inner| inner.write_head_prefix_for_repository_tasks(head, HEAD_INTERRUPTION), - ) - } - - fn flush_head(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::FlushHead, - DuringTiming::Before, - FilesystemCatalogPublisher::flush_head, - ) - } - - fn synchronize_head(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::SynchronizeHead, - DuringTiming::Before, - FilesystemCatalogPublisher::synchronize_head, - ) - } - - fn verify_head_view( - &mut self, - head: &CanonicalPublicationHead, - snapshot: &CatalogSnapshot<'_, '_, '_>, - ) -> io::Result<()> { - self.inner.verify_head_view(head, snapshot) - } - - fn replace_head(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::ReplaceHead, - DuringTiming::After, - FilesystemCatalogPublisher::replace_head, - ) - } - - fn synchronize_root(&mut self) -> io::Result<()> { - execute( - &mut self.inner, - self.control, - DurabilityCrashPoint::SynchronizeRootAfterHead, - DuringTiming::Before, - FilesystemCatalogPublisher::synchronize_root, - ) - } -} - -fn execute( - inner: &mut FilesystemCatalogPublisher, - control: &mut CrashControl, - point: DurabilityCrashPoint, - during: DuringTiming, - operation: impl FnOnce(&mut FilesystemCatalogPublisher) -> io::Result, -) -> io::Result { - control.before(point, during)?; - let result = operation(inner)?; - control.after(point, during)?; - Ok(result) -} - -fn execute_write( - inner: &mut FilesystemCatalogPublisher, - control: &mut CrashControl, - point: DurabilityCrashPoint, - complete: impl FnOnce(&mut FilesystemCatalogPublisher) -> io::Result<()>, - interrupted: impl FnOnce(&mut FilesystemCatalogPublisher) -> io::Result<()>, -) -> io::Result<()> { - match control.position(point) { - None => complete(inner), - Some(DurabilityCrashPosition::Before) => control.await_process_death(), - Some(DurabilityCrashPosition::During) => { - interrupted(inner)?; - control.await_process_death() - } - Some(DurabilityCrashPosition::After) => { - complete(inner)?; - control.await_process_death() - } - } -} diff --git a/xtask/src/durability_crash_matrix/production_protocol/recovery.rs b/xtask/src/durability_crash_matrix/production_protocol/recovery.rs index c87ba5de..21df9d91 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/recovery.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/recovery.rs @@ -1,23 +1,24 @@ //! This module owns execution of the production recovery-discard protocol. use std::fs; -use std::io::Write; +use std::io::{self, Write}; use std::path::Path; use keep::{ - AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogGeneration, - CatalogPublicationStorage, FilesystemRecoveryStageDiscarder, RecoveryStage, - RecoveryStageDiscardRequest, RecoveryStageMetadata, admit_recovery_stage_bytes, - assess_recovery_stage, execute_recovery_stage_discard, fingerprint_recovery_stage, - plan_recovery_stage_discard, + AdmittedSegment, CanonicalPublicationHead, CatalogPublicationBoundary, CatalogPublicationError, + CatalogPublicationExpectation, CatalogPublicationObserver, CatalogPublicationPhase, + FilesystemRecoveryStageDiscarder, FilesystemVersionTwoRecoveryAdmission, RecoveryStage, + RecoveryStageDiscardRequest, RecoveryStageMetadata, RetainedCatalogPublicationError, + admit_recovery_stage_bytes, assess_recovery_stage, execute_recovery_stage_discard, + fingerprint_recovery_stage, plan_recovery_stage_discard, publish_retained_catalog_generation, }; use xtask::DurabilityCrashPoint; use super::control::CrashControl; use super::fixture::GoldenFixture; -use super::initialization; use super::recovery_storage::CrashRecoveryStorage; use super::{DurabilityCrashMatrixError, verification}; +use super::{initialization, publication}; const SEGMENT_INTERRUPTION: usize = 32; const HEAD_INTERRUPTION: usize = 64; @@ -32,7 +33,10 @@ pub(super) fn run( prepare_head(store_root)? }; let request = discard_request(store_root, stage)?; - let storage = FilesystemRecoveryStageDiscarder::open_unchecked_for_repository_tasks(store_root) + let admission = + FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("admit production recovery store", source))?; + let storage = FilesystemRecoveryStageDiscarder::open_version_two(admission) .map_err(|source| verification("open production recovery discarder", source))?; let mut storage = CrashRecoveryStorage::new(storage, control); execute_recovery_stage_discard(&mut storage, request) @@ -65,28 +69,77 @@ fn prepare_segment(store_root: &Path) -> Result Result { let mut publisher = initialization::publisher(store_root)?; - let segment_fixture = GoldenFixture::segment()?; - let segment = - AdmittedSegment::decode(segment_fixture.bytes(), initialization::segment_policy()) - .map_err(|source| verification("admit recovery head segment", source))?; - let segments = [segment]; - let generation = CatalogGeneration::new(1) - .map_err(|source| verification("construct recovery head generation", source))?; - let catalog = CanonicalCatalog::from_segments(generation, None, &segments) - .map_err(|source| verification("encode recovery head catalog", source))?; + let sealed = publication::stage_one_zero(&publisher)?; + let segment_bytes = fs::read(store_root.join("staging/current.seg")) + .map_err(|source| DurabilityCrashMatrixError::io("read recovery head segment", source))?; + let segments = [ + AdmittedSegment::decode(&segment_bytes, initialization::segment_policy()) + .map_err(|source| verification("admit recovery head segment", source))?, + ]; + let catalog = publication::first_catalog(&segments)?; + let selection = publisher + .select_segment(sealed, &segments[0]) + .map_err(|source| verification("select recovery head segment", source))?; let head = CanonicalPublicationHead::for_catalog(catalog.checksummed()); - publisher.create_head_stage().map_err(|source| { - DurabilityCrashMatrixError::io("create recovery head precondition", source) - })?; - publisher - .write_head_prefix_for_repository_tasks(&head, HEAD_INTERRUPTION) - .map_err(|source| { - DurabilityCrashMatrixError::io("write recovery head precondition", source) - })?; - drop(publisher); - Ok(RecoveryStage::NextHead) + let torn = head + .encoded() + .get(..HEAD_INTERRUPTION) + .ok_or(DurabilityCrashMatrixError::FixtureRange)? + .to_vec(); + publisher.observe_catalog_phases_for_repository_tasks(interrupt_head(store_root, torn)); + require_interrupted_head(publish_retained_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + selection, + &catalog, + &segments, + )) +} + +/// Requires the precondition publication to have failed exactly at the head write. +fn require_interrupted_head( + interrupted: Result, +) -> Result { + match interrupted { + Err(RetainedCatalogPublicationError::Catalog { + source: + CatalogPublicationError::Storage { + phase: CatalogPublicationPhase::WriteHead, + .. + }, + }) => Ok(RecoveryStage::NextHead), + Err(source) => Err(verification("interrupt recovery head precondition", source)), + Ok(_receipt) => Err(DurabilityCrashMatrixError::io( + "interrupt recovery head precondition", + io::Error::other("the precondition publication completed"), + )), + } +} + +/// Writes `torn` into the created `head.next`, then fails the head write. +fn interrupt_head(store_root: &Path, torn: Vec) -> CatalogPublicationObserver { + let stage = store_root.join("head.next"); + Box::new(move |phase, boundary| { + if phase != CatalogPublicationPhase::WriteHead + || boundary != CatalogPublicationBoundary::Before + { + return Ok(()); + } + fs::OpenOptions::new() + .write(true) + .open(&stage)? + .write_all(&torn)?; + Err(io::Error::other( + "recovery precondition interrupts the head write", + )) + }) } fn discard_request( diff --git a/xtask/src/durability_crash_matrix/restart.rs b/xtask/src/durability_crash_matrix/restart.rs index a5020f3e..5964c4c7 100644 --- a/xtask/src/durability_crash_matrix/restart.rs +++ b/xtask/src/durability_crash_matrix/restart.rs @@ -15,8 +15,7 @@ use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; use super::DurabilityCrashMatrixError; -use super::production_protocol::fixture::GoldenFixture; -use expectation::ExpectedStoreState; +use expectation::{ArtifactFixtures, ExpectedStoreState}; use xtask::{DurabilityCrashCase, DurabilityCrashSequence}; pub(super) fn verify( @@ -40,13 +39,11 @@ pub(super) fn verify( observed: observed_paths, }); } - let segment = GoldenFixture::segment()?; - let catalog = GoldenFixture::catalog()?; - let head = GoldenFixture::head()?; + let fixtures = ArtifactFixtures::load()?; for (relative, bytes) in expected.artifacts() { let observed = fs::read(store_root.join(relative)) .map_err(|source| DurabilityCrashMatrixError::io("read crash artifact", source))?; - let expected_bytes = bytes.resolve(&segment, &catalog, &head)?; + let expected_bytes = bytes.resolve(&fixtures)?; if observed != expected_bytes { return Err(DurabilityCrashMatrixError::artifact_bytes( relative, diff --git a/xtask/src/durability_crash_matrix/restart/expectation.rs b/xtask/src/durability_crash_matrix/restart/expectation.rs index a5d1c2fd..48f746f4 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation.rs @@ -17,9 +17,26 @@ pub(super) const SEGMENT_STAGE: &str = "staging/current.seg"; pub(super) const CATALOG_STAGE: &str = "staging/current.cat"; pub(super) const NEXT_HEAD: &str = "head.next"; pub(super) const HEAD: &str = "HEAD"; +pub(super) const READER_LOCK: &str = "reader.lock"; +pub(super) const FORMAT: &str = "FORMAT"; + +/// Every protocol directory of a committed single-format store, parents first. +const PROTOCOL_DIRECTORIES: [&str; 9] = [ + STAGING, + SEGMENTS, + CATALOGS, + "retention", + "retention/roots", + "retention/manifests", + "gc", + "recovery", + "recovery/dispositions", +]; pub(super) enum ArtifactBytes { Empty, + /// The canonical `FORMAT` marker of the corpus. + Marker, Segment(usize), Catalog(usize), Head(usize), @@ -29,6 +46,7 @@ impl ArtifactBytes { pub(super) const fn kind(&self) -> &'static str { match self { Self::Empty => "empty", + Self::Marker => "marker", Self::Segment(_) => "segment", Self::Catalog(_) => "catalog", Self::Head(_) => "head", @@ -37,12 +55,17 @@ impl ArtifactBytes { pub(super) fn resolve<'a>( &self, - segment: &'a GoldenFixture, - catalog: &'a GoldenFixture, - head: &'a GoldenFixture, + fixtures: &'a ArtifactFixtures, ) -> Result<&'a [u8], DurabilityCrashMatrixError> { + let ArtifactFixtures { + marker, + segment, + catalog, + head, + } = fixtures; match self { Self::Empty => Ok(&[]), + Self::Marker => Ok(marker.bytes()), Self::Segment(end) => segment.prefix(*end), Self::Catalog(end) => catalog.prefix(*end), Self::Head(end) => head.prefix(*end), @@ -50,6 +73,25 @@ impl ArtifactBytes { } } +/// The decoded corpus artifacts expected bytes are prefixes of. +pub(super) struct ArtifactFixtures { + pub(super) marker: GoldenFixture, + pub(super) segment: GoldenFixture, + pub(super) catalog: GoldenFixture, + pub(super) head: GoldenFixture, +} + +impl ArtifactFixtures { + pub(super) fn load() -> Result { + Ok(Self { + marker: GoldenFixture::format_marker()?, + segment: GoldenFixture::segment()?, + catalog: GoldenFixture::catalog()?, + head: GoldenFixture::head()?, + }) + } +} + pub(super) struct ExpectedStoreState { directories: BTreeSet<&'static str>, artifacts: BTreeMap<&'static str, ArtifactBytes>, @@ -95,9 +137,16 @@ impl ExpectedStoreState { self.hard_link } + /// The committed single-format store that direct initialization leaves. fn initialized() -> Self { - let directories = [STAGING, SEGMENTS, CATALOGS].into_iter().collect(); - let artifacts = std::iter::once((WRITER_LOCK, ArtifactBytes::Empty)).collect(); + let directories = PROTOCOL_DIRECTORIES.into_iter().collect(); + let artifacts = [ + (WRITER_LOCK, ArtifactBytes::Empty), + (READER_LOCK, ArtifactBytes::Empty), + (FORMAT, ArtifactBytes::Marker), + ] + .into_iter() + .collect(); Self { directories, artifacts, diff --git a/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs b/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs index 1679b4f6..37a74f54 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs @@ -98,6 +98,19 @@ pub(super) fn recovery( case: DurabilityCrashCase, ) -> Result { let mut state = ExpectedStoreState::initialized(); + if matches!( + case.point(), + DurabilityCrashPoint::RemoveRecoveryHead + | DurabilityCrashPoint::SynchronizeRootAfterRecovery + ) { + // The interrupted first publication linked both pools before `head.next`. + state + .artifacts + .insert(SEGMENT_POOL_PATH, ArtifactBytes::Segment(337)); + state + .artifacts + .insert(CATALOG_POOL_PATH, ArtifactBytes::Catalog(352)); + } match case.point() { DurabilityCrashPoint::RemoveRecoveryStage if case.position() == DurabilityCrashPosition::Before => diff --git a/xtask/src/durability_crash_matrix/restart/semantic.rs b/xtask/src/durability_crash_matrix/restart/semantic.rs index 8d5238f6..687c2133 100644 --- a/xtask/src/durability_crash_matrix/restart/semantic.rs +++ b/xtask/src/durability_crash_matrix/restart/semantic.rs @@ -6,14 +6,15 @@ use std::path::Path; use keep::{ AdmittedSegment, CatalogRestartByteLimit, CatalogRestartPolicy, ChecksummedCatalog, ChunkId, - FilesystemCatalogSnapshot, FilesystemWriterLock, LayoutEntryLimit, RecoveryCatalogStage, - RecoveryNextHeadStage, RecoverySegmentStage, SegmentReadPolicy, SegmentRecordIdentity, - SegmentRecordLimit, classify_recovery_catalog_stage, classify_recovery_next_head_stage, - classify_recovery_segment_stage, + FilesystemCatalogSnapshot, FilesystemVersionTwoRecoveryAdmission, FilesystemWriterLock, + LayoutEntryLimit, RecoveryCatalogStage, RecoveryNextHeadStage, RecoverySegmentStage, + SegmentReadPolicy, SegmentRecordIdentity, SegmentRecordLimit, classify_recovery_catalog_stage, + classify_recovery_next_head_stage, classify_recovery_segment_stage, }; use super::expectation::{ - ArtifactBytes, CATALOG_STAGE, ExpectedStoreState, HEAD, NEXT_HEAD, SEGMENT_STAGE, WRITER_LOCK, + ArtifactBytes, CATALOG_STAGE, ExpectedStoreState, FORMAT, HEAD, NEXT_HEAD, SEGMENT_STAGE, + WRITER_LOCK, }; use crate::durability_crash_matrix::DurabilityCrashMatrixError; use crate::durability_crash_matrix::production_protocol::fixture::{ @@ -34,10 +35,23 @@ pub(super) fn verify( verify_published_snapshot(store_root, expected) } +/// Requires the dead writer's authority to be reacquirable. +/// +/// A committed store must also pass single-format recovery admission: every +/// residue a catalog-protocol crash leaves is a lawful state of the format. fn verify_writer_release( store_root: &Path, expected: &ExpectedStoreState, ) -> Result<(), DurabilityCrashMatrixError> { + if expected.artifact(FORMAT).is_some() { + let admission = + FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks( + store_root, + ) + .map_err(|source| verification("admit the crash residue for recovery", source))?; + drop(admission); + return Ok(()); + } if expected.artifact(WRITER_LOCK).is_none() { return Ok(()); } @@ -67,7 +81,7 @@ fn verify_segment_stage( ArtifactBytes::Segment(64 | 209) => "reusable", ArtifactBytes::Segment(337) => "complete", ArtifactBytes::Empty | ArtifactBytes::Segment(_) => "truncated", - ArtifactBytes::Catalog(_) | ArtifactBytes::Head(_) => { + ArtifactBytes::Marker | ArtifactBytes::Catalog(_) | ArtifactBytes::Head(_) => { return Err(DurabilityCrashMatrixError::UnexpectedArtifactKind { artifact: SEGMENT_STAGE, expected: "segment", @@ -97,7 +111,7 @@ fn verify_catalog_stage( let expected_class = match bytes { ArtifactBytes::Catalog(352) => "complete", ArtifactBytes::Empty | ArtifactBytes::Catalog(_) => "truncated", - ArtifactBytes::Segment(_) | ArtifactBytes::Head(_) => { + ArtifactBytes::Marker | ArtifactBytes::Segment(_) | ArtifactBytes::Head(_) => { return Err(DurabilityCrashMatrixError::UnexpectedArtifactKind { artifact: CATALOG_STAGE, expected: "catalog", @@ -126,7 +140,7 @@ fn verify_next_head( let expected_class = match bytes { ArtifactBytes::Head(128) => "complete", ArtifactBytes::Empty | ArtifactBytes::Head(_) => "truncated", - ArtifactBytes::Segment(_) | ArtifactBytes::Catalog(_) => { + ArtifactBytes::Marker | ArtifactBytes::Segment(_) | ArtifactBytes::Catalog(_) => { return Err(DurabilityCrashMatrixError::UnexpectedArtifactKind { artifact: NEXT_HEAD, expected: "head", diff --git a/xtask/tests/durability_crash_production_contract.rs b/xtask/tests/durability_crash_production_contract.rs index 488db717..cc4f9051 100644 --- a/xtask/tests/durability_crash_production_contract.rs +++ b/xtask/tests/durability_crash_production_contract.rs @@ -26,7 +26,6 @@ fn crash_children_execute_every_claimed_production_protocol() -> Result<(), Box< assert!(!child.contains("state::prepare(")); for required in [ "StagedSegment::begin(", - "publish_catalog_generation(", "initialize_store(", "execute_recovery_stage_discard(", "initialize_observed_for_repository_tasks(", From fca1a2999b18be0f2eb4d2a8b9f63d4dea5b74cb Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 10:13:02 -0700 Subject: [PATCH 18/36] Test: move stage, publication, and recovery laws onto single-format stores (#198) Every law below constrains code the single format runs unchanged, so it moves rather than retires. Each fixture now creates its store through direct initialization instead of hand-writing writer.lock, staging, segments, and catalogs: - the four recovery fixtures (stage completion, stage discard, segment continuation, next-head finalization) open their tools through FilesystemVersionTwoRecoveryAdmission; the planted crash residue (stages, pool entries, head.next) is unchanged. Writer exclusion is now observed where single-format recovery takes the lock: a second recovery admission refuses with WriterLock Busy; - the recovery-inventory and stage-fingerprint laws open the reader through version-2 admission; the symlinked-staging law now refuses at admission with WrongKind { Directory, Symlink }; - catalog_filesystem_publication runs through FilesystemVersionTwoCatalogPublisher and the retained-closure gate. A leftover head.next and a non-directory or linked protocol directory refuse at writer admission with exact namespace refusals; two new laws keep the stages' own no-follow checks for a namespace replaced after admission; - segment-stage, observed-stage, and catalog-restart laws use the single-format writer; catalog_restart's one-zero store is now published by the writer and required to equal the frozen corpus bytes; - name-classification laws classify the single-format root. Laws that plant a link target in the store root plant it after admission, which refuses unknown root entries, so each tool's own no-follow open remains the boundary under test. Test-only version-1 openers left without callers are removed. --- src/adapters/filesystem_catalog_publisher.rs | 11 -- .../filesystem_catalog_publisher_tests.rs | 73 +++++++---- src/adapters/filesystem_platform_admission.rs | 5 - .../filesystem_recovery_inventory_tests.rs | 78 ++++++++---- ...m_recovery_next_head_finalization_tests.rs | 8 +- .../fixture.rs | 31 +++-- .../refusal_laws.rs | 4 +- ...filesystem_recovery_next_head_finalizer.rs | 10 -- ...ilesystem_recovery_segment_resume_tests.rs | 6 +- .../fixture.rs | 36 ++++-- .../refusal_laws.rs | 3 +- .../filesystem_recovery_segment_resumer.rs | 15 +-- .../filesystem_recovery_stage_completer.rs | 9 -- ...esystem_recovery_stage_completion_tests.rs | 6 +- .../fixture.rs | 26 ++-- .../refusal_laws.rs | 4 +- ...filesystem_recovery_stage_discard_tests.rs | 8 +- .../fixture.rs | 26 ++-- .../filesystem_recovery_stage_tests.rs | 22 ++-- .../filesystem_segment_stage_tests.rs | 21 ++-- .../authority_laws.rs | 23 ++-- .../directory_laws.rs | 85 ++++++++++--- .../initialization_laws.rs | 14 +-- .../refusal_laws.rs | 114 +++++++----------- tests/catalog_restart.rs | 72 +++++++++-- tests/observed_segment_stage.rs | 14 +-- tests/recovery_inventory.rs | 2 +- tests/recovery_name_classification.rs | 21 +++- .../grammar_laws.rs | 4 +- .../refusal_laws.rs | 18 +-- tests/recovery_name_classification_memory.rs | 19 ++- tests/segment_filesystem_stage.rs | 16 ++- tests/support/admission_refusal.rs | 20 +++ tests/support/mod.rs | 2 + 34 files changed, 502 insertions(+), 324 deletions(-) create mode 100644 tests/support/admission_refusal.rs diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs index c0b7ef21..56f817bc 100644 --- a/src/adapters/filesystem_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -82,17 +82,6 @@ impl FilesystemCatalogPublisher { ) } - #[cfg(test)] - pub(super) fn open_unchecked_for_tests( - lock: FilesystemWriterLock, - policy: CatalogRestartPolicy, - ) -> io::Result { - Self::open( - FilesystemPlatformAdmission::unchecked_for_tests(lock), - policy, - ) - } - /// Exclusively creates `staging/current.seg` under this writer authority. /// /// The returned stage borrows this publisher until the stage is dropped or diff --git a/src/adapters/filesystem_catalog_publisher_tests.rs b/src/adapters/filesystem_catalog_publisher_tests.rs index 87d70297..b84e1f05 100644 --- a/src/adapters/filesystem_catalog_publisher_tests.rs +++ b/src/adapters/filesystem_catalog_publisher_tests.rs @@ -17,11 +17,12 @@ use super::filesystem_test_sandbox::TestDirectory; use super::test_support::decode_hex; use crate::{ AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, - CatalogPublicationExpectation, CatalogPublicationOutcome, CatalogRestartByteLimit, - CatalogRestartPolicy, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemPlatformAdmission, FilesystemSegmentStage, FilesystemWriterLock, LayoutEntryLimit, + CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, + CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogSnapshot, + FilesystemPlatformAdmissionError, FilesystemSegmentStage, FilesystemVersionTwoAdmission, + FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, RetainedCatalogPublicationError, SealedSegment, SegmentPublication, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, - publish_catalog_generation, + publish_retained_catalog_generation, }; const SEGMENT_HEX: &str = include_str!("../../conformance/segment-store/v1/one-zero-segment.hex"); @@ -37,9 +38,7 @@ type StagedFixture<'publisher> = (SealedSegment Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-success")?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; assert_eq!(segment_bytes, fixture(SEGMENT_HEX)?); let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; @@ -47,7 +46,7 @@ fn successful_publication_materializes_only_the_exact_durable_view() -> Result<( let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let selection = publisher.select_segment(sealed, &segments[0])?; - let receipt = publish_catalog_generation( + let receipt = publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), selection, @@ -72,15 +71,13 @@ fn successful_publication_materializes_only_the_exact_durable_view() -> Result<( #[test] fn durable_publication_retry_returns_the_same_synchronized_receipt() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-retry")?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let selection = publisher.select_segment(sealed, &segments[0])?; - let first = publish_catalog_generation( + let first = publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), selection, @@ -89,9 +86,8 @@ fn durable_publication_retry_returns_the_same_synchronized_receipt() -> Result<( )?; drop(publisher); - let admission = FilesystemPlatformAdmission::reopen_unchecked_for_tests(store.path())?; - let mut publisher = FilesystemCatalogPublisher::open(admission, restart_policy()?)?; - let retry = publish_catalog_generation( + let mut publisher = open_publisher(&store)?; + let retry = publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), SegmentPublication::none(), @@ -115,13 +111,13 @@ struct StoreFixture { impl StoreFixture { fn create(name: &str) -> Result> { let sandbox = TestDirectory::create(name)?; - fs::write(sandbox.path().join("writer.lock"), [])?; - let staging = sandbox.path().join("staging"); + drop( + FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( + sandbox.path(), + )?, + ); let segments = sandbox.path().join("segments"); let catalogs = sandbox.path().join("catalogs"); - fs::create_dir(&staging)?; - fs::create_dir(&segments)?; - fs::create_dir(&catalogs)?; Ok(Self { catalog_path: catalogs.join(format!("0000000000000001-{CATALOG_DIGEST}.cat")), segment_path: segments.join(format!("{SEGMENT_DIGEST}.seg")), @@ -150,6 +146,33 @@ impl StoreFixture { } } +/// Reacquires single-format writer authority over the initialized store. +fn admit_writer( + store: &StoreFixture, +) -> Result { + FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store.path()) +} + +/// Reopens the initialized store and pins its single-format writer. +fn open_publisher( + store: &StoreFixture, +) -> Result> { + Ok(FilesystemVersionTwoCatalogPublisher::open( + admit_writer(store)?, + restart_policy()?, + )?) +} + +/// The catalog-protocol refusal inside one gated publication failure. +fn catalog_refusal( + error: RetainedCatalogPublicationError, +) -> Result> { + match error { + RetainedCatalogPublicationError::Catalog { source } => Ok(source), + other => Err(format!("refusal outside the catalog protocol: {other}").into()), + } +} + fn restart_policy() -> Result> { Ok(CatalogRestartPolicy::new( maximum_segment_policy(), @@ -158,7 +181,7 @@ fn restart_policy() -> Result> { } fn stage_one_zero<'publisher>( - publisher: &'publisher FilesystemCatalogPublisher, + publisher: &'publisher FilesystemVersionTwoCatalogPublisher, store: &StoreFixture, ) -> Result, Box> { let stage = publisher.create_segment_stage()?; @@ -181,12 +204,12 @@ fn fixture(hex: &str) -> Result, Box> { #[test] fn publisher_drop_closes_writable_stages_before_releasing_writer_authority() -> Result<(), Box> { - let source = include_str!("filesystem_catalog_publisher.rs"); + let source = include_str!("filesystem_version_two_catalog_publisher.rs"); let stages = source - .find("pub(super) stages: FilesystemCatalogStages,") - .ok_or("publisher must retain the shared catalog stages")?; + .find("pub(super) stages: FilesystemVersionTwoCatalogStages,") + .ok_or("publisher must retain the gated catalog stages")?; let writer_lock = source - .find("pub(super) _lock:") + .find(" _lock: FilesystemWriterLock,") .ok_or("publisher must retain writer authority")?; let shared = include_str!("filesystem_catalog_stages.rs"); let owner = shared diff --git a/src/adapters/filesystem_platform_admission.rs b/src/adapters/filesystem_platform_admission.rs index dc296494..097265df 100644 --- a/src/adapters/filesystem_platform_admission.rs +++ b/src/adapters/filesystem_platform_admission.rs @@ -16,11 +16,6 @@ impl FilesystemPlatformAdmission { Self { lock } } - #[cfg(test)] - pub(super) const fn unchecked_for_tests(lock: FilesystemWriterLock) -> Self { - Self::initialized(lock) - } - #[cfg(feature = "repository-tasks")] pub(super) fn from_repository_writer_lock(lock: FilesystemWriterLock) -> std::io::Result { // Writer capabilities may be O_PATH; profile ioctls require a readable diff --git a/src/adapters/filesystem_recovery_inventory_tests.rs b/src/adapters/filesystem_recovery_inventory_tests.rs index e19c204d..dd6c9bbe 100644 --- a/src/adapters/filesystem_recovery_inventory_tests.rs +++ b/src/adapters/filesystem_recovery_inventory_tests.rs @@ -9,14 +9,13 @@ use std::os::unix::ffi::OsStringExt; use super::filesystem_test_sandbox::TestDirectory; use super::{ - FilesystemRecoveryInventoryReader, RecoveryInventoryError, RecoveryInventoryLimit, - RecoveryInventoryOperation, RecoveryNamespace, + FilesystemEntryKind, FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError, + FilesystemRecoveryInventoryReader, FilesystemVersionTwoAdmission, RecoveryInventoryError, + RecoveryInventoryLimit, RecoveryInventoryOperation, RecoveryNamespace, }; -const LOCK_NAME: &str = "writer.lock"; const STAGING_NAME: &str = "staging"; const SEGMENTS_NAME: &str = "segments"; -const CATALOGS_NAME: &str = "catalogs"; #[test] fn filesystem_inventory_returns_deterministic_name_order() -> Result<(), Box> { @@ -26,7 +25,7 @@ fn filesystem_inventory_returns_deterministic_name_order() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { let sandbox = initialized_namespace("recovery-inventory-limit")?; fs::write(sandbox.path().join(SEGMENTS_NAME).join("one"), [])?; - let mut reader = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path())?; + let mut reader = open_reader(&sandbox)?; let Err(error) = reader.read(RecoveryInventoryLimit::new(4)?) else { return Err("filesystem inventory exceeded the global entry ceiling".into()); }; @@ -98,24 +97,33 @@ fn filesystem_inventory_never_follows_protocol_directory_links() -> Result<(), B use std::os::unix::fs::symlink; let sandbox = TestDirectory::create("recovery-inventory-symlink")?; - fs::write(sandbox.path().join(LOCK_NAME), [])?; + let store = sandbox.path().join("store"); + fs::create_dir(&store)?; + drop(FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(&store)?); fs::create_dir(sandbox.path().join("target"))?; - symlink("target", sandbox.path().join(STAGING_NAME))?; - fs::create_dir(sandbox.path().join(SEGMENTS_NAME))?; - fs::create_dir(sandbox.path().join(CATALOGS_NAME))?; + fs::remove_dir(store.join(STAGING_NAME))?; + symlink("../target", store.join(STAGING_NAME))?; - let Err(error) = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path()) + let Err(error) = + FilesystemRecoveryInventoryReader::open_version_two_unchecked_for_repository_tasks(&store) else { return Err("filesystem inventory followed a staging-directory link".into()); }; - assert!(matches!( - error, - RecoveryInventoryError::Io { - namespace: RecoveryNamespace::Staging, - operation: RecoveryInventoryOperation::OpenNamespace, - .. - } - )); + let RecoveryInventoryError::Io { + namespace: RecoveryNamespace::Root, + operation: RecoveryInventoryOperation::OpenNamespace, + source, + } = error + else { + return Err(format!("unexpected inventory refusal {error:?}").into()); + }; + assert_eq!( + namespace_refusal(&source), + Some(FilesystemNamespaceRefusal::WrongKind { + expected: FilesystemEntryKind::Directory, + observed: FilesystemEntryKind::Symlink, + }) + ); sandbox.remove()?; Ok(()) } @@ -123,7 +131,7 @@ fn filesystem_inventory_never_follows_protocol_directory_links() -> Result<(), B #[test] fn replaced_protocol_directory_refuses_the_pinned_inventory() -> Result<(), Box> { let sandbox = initialized_namespace("recovery-inventory-replacement")?; - let mut reader = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path())?; + let mut reader = open_reader(&sandbox)?; fs::rename( sandbox.path().join(STAGING_NAME), sandbox.path().join("displaced-staging"), @@ -146,11 +154,31 @@ fn replaced_protocol_directory_refuses_the_pinned_inventory() -> Result<(), Box< Ok(()) } +/// A store committed by direct initialization. fn initialized_namespace(name: &str) -> Result> { let sandbox = TestDirectory::create(name)?; - fs::write(sandbox.path().join(LOCK_NAME), [])?; - fs::create_dir(sandbox.path().join(STAGING_NAME))?; - fs::create_dir(sandbox.path().join(SEGMENTS_NAME))?; - fs::create_dir(sandbox.path().join(CATALOGS_NAME))?; + drop(FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(sandbox.path())?); Ok(sandbox) } + +fn open_reader( + sandbox: &TestDirectory, +) -> Result { + FilesystemRecoveryInventoryReader::open_version_two_unchecked_for_repository_tasks( + sandbox.path(), + ) +} + +/// The namespace refusal one root admission failure carries, if any. +fn namespace_refusal(source: &std::io::Error) -> Option { + let admission = source + .get_ref()? + .downcast_ref::()?; + let FilesystemPlatformAdmissionError::Namespace { source } = admission else { + return None; + }; + source + .get_ref()? + .downcast_ref::() + .copied() +} diff --git a/src/adapters/filesystem_recovery_next_head_finalization_tests.rs b/src/adapters/filesystem_recovery_next_head_finalization_tests.rs index 00dfe2b2..f6d439a0 100644 --- a/src/adapters/filesystem_recovery_next_head_finalization_tests.rs +++ b/src/adapters/filesystem_recovery_next_head_finalization_tests.rs @@ -51,13 +51,15 @@ fn one_writer_excludes_a_second_finalizer() -> Result<(), Box> { let first = fixture.finalizer()?; let error = fixture - .finalizer() + .recovery_admission() .err() .ok_or("second next-head finalizer acquired writer authority")?; assert!(matches!( - error.downcast_ref::(), - Some(super::FilesystemRecoveryNextHeadFinalizationOpenError::WriterLock { .. }) + error, + super::FilesystemPlatformAdmissionError::WriterLock { + source: super::WriterLockAcquireError::Busy, + } )); drop(first); fixture.remove()?; diff --git a/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs b/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs index 328d7b40..07fabd8d 100644 --- a/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs +++ b/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs @@ -9,10 +9,12 @@ use crate::LayoutEntryLimit; use super::super::{ AdmittedSegment, CatalogPublicationExpectation, CatalogRestartByteLimit, CatalogRestartPolicy, CatalogSnapshot, ChecksummedCatalog, ChecksummedPublicationHead, - FilesystemRecoveryNextHeadFinalizer, RecoveryNextHeadFinalizationRequest, RecoveryStage, - RecoveryStageMetadata, SegmentReadPolicy, SegmentRecordLimit, admit_recovery_stage_bytes, - assess_recovery_stage, filesystem_test_sandbox::TestDirectory, fingerprint_recovery_stage, - physical_pool_name, plan_recovery_next_head_finalization, test_support::decode_hex, + FilesystemPlatformAdmissionError, FilesystemRecoveryNextHeadFinalizer, + FilesystemVersionTwoAdmission, FilesystemVersionTwoRecoveryAdmission, + RecoveryNextHeadFinalizationRequest, RecoveryStage, RecoveryStageMetadata, SegmentReadPolicy, + SegmentRecordLimit, admit_recovery_stage_bytes, assess_recovery_stage, + filesystem_test_sandbox::TestDirectory, fingerprint_recovery_stage, physical_pool_name, + plan_recovery_next_head_finalization, test_support::decode_hex, }; const SEGMENT_HEX: &str = @@ -33,10 +35,11 @@ pub(super) struct FinalizationFixture { impl FinalizationFixture { pub(super) fn new(name: &str) -> Result> { let directory = TestDirectory::create(name)?; - fs::write(directory.path().join("writer.lock"), [])?; - for name in ["staging", "segments", "catalogs"] { - fs::create_dir(directory.path().join(name))?; - } + drop( + FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( + directory.path(), + )?, + ); Ok(Self { directory }) } @@ -101,8 +104,18 @@ impl FinalizationFixture { ) } + /// Acquires single-format recovery admission, the writer authority finalization runs under. + pub(super) fn recovery_admission( + &self, + ) -> Result { + FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) + } + pub(super) fn finalizer(&self) -> Result> { - Ok(FilesystemRecoveryNextHeadFinalizer::open_unchecked_for_tests(self.root(), policy()?)?) + Ok(FilesystemRecoveryNextHeadFinalizer::open_version_two( + self.recovery_admission()?, + policy()?, + )?) } pub(super) fn remove(self) -> std::io::Result<()> { diff --git a/src/adapters/filesystem_recovery_next_head_finalization_tests/refusal_laws.rs b/src/adapters/filesystem_recovery_next_head_finalization_tests/refusal_laws.rs index 194c8e2a..5a184422 100644 --- a/src/adapters/filesystem_recovery_next_head_finalization_tests/refusal_laws.rs +++ b/src/adapters/filesystem_recovery_next_head_finalization_tests/refusal_laws.rs @@ -16,10 +16,12 @@ fn symbolic_candidate_is_never_followed_or_published() -> Result<(), Box Result { - FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(store_root) - .map(|discarder| Self::version_one(discarder, policy)) - .map_err(Into::into) - } - const fn version_one( discarder: FilesystemRecoveryStageDiscarder, policy: CatalogRestartPolicy, diff --git a/src/adapters/filesystem_recovery_segment_resume_tests.rs b/src/adapters/filesystem_recovery_segment_resume_tests.rs index 229dc039..8aff8679 100644 --- a/src/adapters/filesystem_recovery_segment_resume_tests.rs +++ b/src/adapters/filesystem_recovery_segment_resume_tests.rs @@ -4,7 +4,7 @@ use std::error::Error; use std::fs; use super::{ - AdmittedSegment, AdmittedSegmentRecord, FilesystemRecoverySegmentResumeOpenError, + AdmittedSegment, AdmittedSegmentRecord, FilesystemPlatformAdmissionError, WriterLockAcquireError, execute_recovery_segment_resume, }; @@ -41,7 +41,7 @@ fn exact_prefix_resumes_seals_and_retains_writer_authority() -> Result<(), Box Result<(), Box Result> { let directory = TestDirectory::create(name)?; - fs::write(directory.path().join("writer.lock"), [])?; - for name in ["staging", "segments", "catalogs"] { - fs::create_dir(directory.path().join(name))?; - } + drop( + FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( + directory.path(), + )?, + ); Ok(Self { directory }) } @@ -76,22 +77,31 @@ impl ResumeFixture { self.root().join("staging/current.seg") } - pub(super) fn resumer( + /// Acquires single-format recovery admission, the writer authority continuation runs under. + pub(super) fn recovery_admission( &self, - ) -> Result { - FilesystemRecoverySegmentResumer::open_unchecked_for_tests(self.root()) + ) -> Result { + FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) + } + + pub(super) fn resumer(&self) -> Result> { + Ok(FilesystemRecoverySegmentResumer::open_version_two( + self.recovery_admission()?, + )?) } pub(super) fn resumer_before_handoff( &self, before_handoff: F, - ) -> Result + ) -> Result> where F: FnOnce() + 'static, { - FilesystemRecoverySegmentResumer::open_unchecked_for_tests_before_handoff( - self.root(), - before_handoff, + Ok( + FilesystemRecoverySegmentResumer::open_before_handoff_for_tests( + self.recovery_admission()?, + before_handoff, + )?, ) } diff --git a/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs b/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs index 467c99f4..96c19f48 100644 --- a/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs +++ b/src/adapters/filesystem_recovery_segment_resume_tests/refusal_laws.rs @@ -58,11 +58,12 @@ fn changed_stage_is_refused_before_a_write() -> Result<(), Box> { fn symbolic_stage_is_never_followed() -> Result<(), Box> { let fixture = ResumeFixture::new("filesystem-segment-resume-link")?; let prefix = reusable_prefix()?; + let resumer = fixture.resumer()?; let target = fixture.root().join("target.seg"); fs::write(&target, &prefix)?; symlink(&target, fixture.stage_path())?; - let error = execute_recovery_segment_resume(fixture.resumer()?, resume_request(&prefix)?) + let error = execute_recovery_segment_resume(resumer, resume_request(&prefix)?) .err() .ok_or("symbolic stage unexpectedly resumed")?; diff --git a/src/adapters/filesystem_recovery_segment_resumer.rs b/src/adapters/filesystem_recovery_segment_resumer.rs index 98b5d153..97062216 100644 --- a/src/adapters/filesystem_recovery_segment_resumer.rs +++ b/src/adapters/filesystem_recovery_segment_resumer.rs @@ -60,23 +60,14 @@ impl FilesystemRecoverySegmentResumer { } #[cfg(test)] - pub(super) fn open_unchecked_for_tests( - store_root: &Path, - ) -> Result { - FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(store_root) - .map(Self::from_discarder) - .map_err(Into::into) - } - - #[cfg(test)] - pub(super) fn open_unchecked_for_tests_before_handoff( - store_root: &Path, + pub(super) fn open_before_handoff_for_tests( + admission: FilesystemVersionTwoRecoveryAdmission, before_handoff: F, ) -> Result where F: FnOnce() + 'static, { - let mut resumer = Self::open_unchecked_for_tests(store_root)?; + let mut resumer = Self::open_version_two(admission)?; resumer.before_handoff = Some(Box::new(before_handoff)); Ok(resumer) } diff --git a/src/adapters/filesystem_recovery_stage_completer.rs b/src/adapters/filesystem_recovery_stage_completer.rs index db5f6860..527d18f3 100644 --- a/src/adapters/filesystem_recovery_stage_completer.rs +++ b/src/adapters/filesystem_recovery_stage_completer.rs @@ -49,13 +49,4 @@ impl FilesystemRecoveryStageCompleter { .map(|discarder| Self { discarder }) .map_err(Into::into) } - - #[cfg(test)] - pub(super) fn open_unchecked_for_tests( - store_root: &Path, - ) -> Result { - FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(store_root) - .map(|discarder| Self { discarder }) - .map_err(Into::into) - } } diff --git a/src/adapters/filesystem_recovery_stage_completion_tests.rs b/src/adapters/filesystem_recovery_stage_completion_tests.rs index 5d43ddc9..7414810c 100644 --- a/src/adapters/filesystem_recovery_stage_completion_tests.rs +++ b/src/adapters/filesystem_recovery_stage_completion_tests.rs @@ -4,7 +4,7 @@ use std::error::Error; use std::fs; use super::{ - FilesystemRecoveryStageCompletionOpenError, RecoveryStage, RecoveryStageDiscardOutcome, + FilesystemPlatformAdmissionError, RecoveryStage, RecoveryStageDiscardOutcome, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, WriterLockAcquireError, execute_recovery_stage_completion, }; @@ -83,13 +83,13 @@ fn retained_completer_authority_excludes_a_second_writer() -> Result<(), Box Result> { let directory = TestDirectory::create(name)?; - fs::write(directory.path().join("writer.lock"), [])?; - for name in ["staging", "segments", "catalogs"] { - fs::create_dir(directory.path().join(name))?; - } + drop( + FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( + directory.path(), + )?, + ); Ok(Self { directory }) } @@ -91,10 +92,17 @@ impl CompletionFixture { } } - pub(super) fn completer( + /// Acquires single-format recovery admission, the writer authority completion runs under. + pub(super) fn recovery_admission( &self, - ) -> Result { - FilesystemRecoveryStageCompleter::open_unchecked_for_tests(self.root()) + ) -> Result { + FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) + } + + pub(super) fn completer(&self) -> Result> { + Ok(FilesystemRecoveryStageCompleter::open_version_two( + self.recovery_admission()?, + )?) } pub(super) fn remove(self) -> std::io::Result<()> { diff --git a/src/adapters/filesystem_recovery_stage_completion_tests/refusal_laws.rs b/src/adapters/filesystem_recovery_stage_completion_tests/refusal_laws.rs index 7530e84c..b50fe307 100644 --- a/src/adapters/filesystem_recovery_stage_completion_tests/refusal_laws.rs +++ b/src/adapters/filesystem_recovery_stage_completion_tests/refusal_laws.rs @@ -42,10 +42,10 @@ fn symbolic_stage_is_never_followed_or_linked() -> Result<(), Box> { let fixture = CompletionFixture::new("filesystem-stage-completion-stage-link")?; let bytes = segment_bytes()?; let request = request(RecoveryStage::Segment, &bytes)?; + let mut completer = fixture.completer()?; let target = fixture.root().join("outside-stage"); fs::write(&target, &bytes)?; symlink(&target, fixture.stage_path(RecoveryStage::Segment))?; - let mut completer = fixture.completer()?; let error = execute_recovery_stage_completion(&mut completer, request) .err() @@ -73,11 +73,11 @@ fn symbolic_pool_is_never_followed_or_admitted() -> Result<(), Box> { let fixture = CompletionFixture::new("filesystem-stage-completion-pool-link")?; let bytes = catalog_bytes()?; let request = request(RecoveryStage::Catalog, &bytes)?; + let mut completer = fixture.completer()?; let target = fixture.root().join("outside-pool"); fs::write(fixture.stage_path(RecoveryStage::Catalog), &bytes)?; fs::write(&target, &bytes)?; symlink(&target, fixture.pool_path(request))?; - let mut completer = fixture.completer()?; let error = execute_recovery_stage_completion(&mut completer, request) .err() diff --git a/src/adapters/filesystem_recovery_stage_discard_tests.rs b/src/adapters/filesystem_recovery_stage_discard_tests.rs index f5eda4d4..3627ffe5 100644 --- a/src/adapters/filesystem_recovery_stage_discard_tests.rs +++ b/src/adapters/filesystem_recovery_stage_discard_tests.rs @@ -4,7 +4,7 @@ use std::error::Error; use std::fs; use super::{ - FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageError, RecoveryStage, + FilesystemPlatformAdmissionError, FilesystemRecoveryStageError, RecoveryStage, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardStorageError, WriterLockAcquireError, execute_recovery_stage_discard, }; @@ -81,10 +81,10 @@ fn symbolic_stage_is_never_followed_or_removed() -> Result<(), Box> { let fixture = DiscardFixture::new("filesystem-stage-discard-symlink")?; let bytes = truncated_bytes(RecoveryStage::Segment, 1)?; let request = request(RecoveryStage::Segment, &bytes)?; + let mut discarder = fixture.discarder()?; let target = fixture.root().join("outside"); fs::write(&target, &bytes)?; symlink(&target, fixture.stage_path(RecoveryStage::Segment))?; - let mut discarder = fixture.discarder()?; let error = execute_recovery_stage_discard(&mut discarder, request) .err() @@ -190,13 +190,13 @@ fn retained_discarder_authority_excludes_a_second_writer() -> Result<(), Box Result> { let directory = TestDirectory::create(name)?; - fs::write(directory.path().join("writer.lock"), [])?; - for name in ["staging", "segments", "catalogs"] { - fs::create_dir(directory.path().join(name))?; - } + drop( + FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( + directory.path(), + )?, + ); Ok(Self { directory }) } @@ -79,10 +80,17 @@ impl DiscardFixture { } } - pub(super) fn discarder( + /// Acquires single-format recovery admission, the writer authority discard runs under. + pub(super) fn recovery_admission( &self, - ) -> Result { - FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(self.root()) + ) -> Result { + FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) + } + + pub(super) fn discarder(&self) -> Result> { + Ok(FilesystemRecoveryStageDiscarder::open_version_two( + self.recovery_admission()?, + )?) } pub(super) fn remove(self) -> std::io::Result<()> { diff --git a/src/adapters/filesystem_recovery_stage_tests.rs b/src/adapters/filesystem_recovery_stage_tests.rs index f12bf548..3ea83e4f 100644 --- a/src/adapters/filesystem_recovery_stage_tests.rs +++ b/src/adapters/filesystem_recovery_stage_tests.rs @@ -4,8 +4,8 @@ use std::error::Error; use std::fs; use super::{ - FilesystemRecoveryInventoryReader, FilesystemRecoveryStageError, RecoveryStage, - RecoveryStageMetadataError, filesystem_test_sandbox::TestDirectory, + FilesystemRecoveryInventoryReader, FilesystemRecoveryStageError, FilesystemVersionTwoAdmission, + RecoveryStage, RecoveryStageMetadataError, filesystem_test_sandbox::TestDirectory, }; #[test] @@ -37,10 +37,10 @@ fn fixed_stage_symbolic_link_is_never_followed() -> Result<(), Box> { use std::os::unix::fs::symlink; let fixture = StageFixture::new("recovery-stage-symlink")?; + let reader = fixture.reader()?; let target = fixture.root().join("outside"); fs::write(&target, b"outside bytes")?; symlink(&target, fixture.segment_path())?; - let reader = fixture.reader()?; let Err(error) = reader.fingerprint_stage(RecoveryStage::Segment) else { return Err("symbolic stage was followed".into()); @@ -174,9 +174,11 @@ struct StageFixture { impl StageFixture { fn new(name: &str) -> Result> { let directory = TestDirectory::create(name)?; - for name in ["staging", "segments", "catalogs"] { - fs::create_dir(directory.path().join(name))?; - } + drop( + FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( + directory.path(), + )?, + ); Ok(Self { directory }) } @@ -197,9 +199,11 @@ impl StageFixture { } fn reader(&self) -> Result> { - Ok(FilesystemRecoveryInventoryReader::open_unchecked_for_tests( - self.root(), - )?) + Ok( + FilesystemRecoveryInventoryReader::open_version_two_unchecked_for_repository_tasks( + self.root(), + )?, + ) } fn remove(self) -> std::io::Result<()> { diff --git a/src/adapters/filesystem_segment_stage_tests.rs b/src/adapters/filesystem_segment_stage_tests.rs index 08369fc5..59348220 100644 --- a/src/adapters/filesystem_segment_stage_tests.rs +++ b/src/adapters/filesystem_segment_stage_tests.rs @@ -8,8 +8,8 @@ use super::filesystem_test_sandbox::TestDirectory; use super::test_support::decode_hex; use crate::{ AdmittedSegmentRecord, CatalogRestartByteLimit, CatalogRestartPolicy, - FilesystemCatalogPublisher, FilesystemWriterLock, LayoutEntryLimit, SegmentHeader, - SegmentReadPolicy, SegmentRecordLimit, SegmentStageCreateError, StagedSegment, + FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, + SegmentHeader, SegmentReadPolicy, SegmentRecordLimit, SegmentStageCreateError, StagedSegment, }; const ONE_ZERO_SEGMENT_HEX: &str = @@ -108,17 +108,18 @@ fn dropping_an_unsealed_stage_preserves_the_reusable_prefix() -> Result<(), Box< Ok(()) } -fn open_publisher(sandbox: &TestDirectory) -> Result> { - fs::write(sandbox.path().join("writer.lock"), [])?; - fs::create_dir(sandbox.path().join("staging"))?; - fs::create_dir(sandbox.path().join("segments"))?; - fs::create_dir(sandbox.path().join("catalogs"))?; - let lock = FilesystemWriterLock::try_acquire(sandbox.path())?; +/// Initializes a single-format store in `sandbox` and opens its writer. +fn open_publisher( + sandbox: &TestDirectory, +) -> Result> { + let admission = + FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(sandbox.path())? + .into_admission(); let policy = CatalogRestartPolicy::new( SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), CatalogRestartByteLimit::new(1_048_576)?, ); - Ok(FilesystemCatalogPublisher::open_unchecked_for_tests( - lock, policy, + Ok(FilesystemVersionTwoCatalogPublisher::open( + admission, policy, )?) } diff --git a/tests/catalog_filesystem_publication/authority_laws.rs b/tests/catalog_filesystem_publication/authority_laws.rs index c3c07d71..d49826d7 100644 --- a/tests/catalog_filesystem_publication/authority_laws.rs +++ b/tests/catalog_filesystem_publication/authority_laws.rs @@ -5,12 +5,13 @@ use std::io::{self, Write}; use crate::{ AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationPhase, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, FilesystemWriterLock, SegmentPublication, SegmentRecordLimit, - SegmentStage, StagedSegment, publish_catalog_generation, + SegmentPublication, SegmentRecordLimit, SegmentStage, StagedSegment, + publish_retained_catalog_generation, }; use super::{ - SEGMENT_HEX, StoreFixture, fixture, maximum_segment_policy, restart_policy, stage_one_zero, + SEGMENT_HEX, StoreFixture, catalog_refusal, fixture, maximum_segment_policy, open_publisher, + stage_one_zero, }; #[test] @@ -26,11 +27,9 @@ fn metadata_equivalent_external_stage_cannot_authorize_publication() -> Result<( let segments = [segment]; let selection = SegmentPublication::one(staged.seal()?.close(), &segments[0])?; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; - let Err(error) = publish_catalog_generation( + let Err(error) = publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), selection, @@ -39,7 +38,7 @@ fn metadata_equivalent_external_stage_cannot_authorize_publication() -> Result<( ) else { return Err("external closed-stage metadata authorized publication".into()); }; - let CatalogPublicationError::Storage { phase, source } = error else { + let CatalogPublicationError::Storage { phase, source } = catalog_refusal(error)? else { return Err("external stage reached the wrong refusal boundary".into()); }; @@ -59,12 +58,8 @@ fn metadata_equivalent_external_stage_cannot_authorize_publication() -> Result<( fn one_publisher_cannot_select_another_publishers_stage() -> Result<(), Box> { let first_store = StoreFixture::create("catalog-filesystem-stage-owner")?; let second_store = StoreFixture::create("catalog-filesystem-stage-substitute")?; - let first_lock = FilesystemWriterLock::try_acquire(first_store.path())?; - let first_publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(first_lock, restart_policy()?)?; - let second_lock = FilesystemWriterLock::try_acquire(second_store.path())?; - let second_publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(second_lock, restart_policy()?)?; + let first_publisher = open_publisher(&first_store)?; + let second_publisher = open_publisher(&second_store)?; let (sealed, segment_bytes) = stage_one_zero(&first_publisher, &first_store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; diff --git a/tests/catalog_filesystem_publication/directory_laws.rs b/tests/catalog_filesystem_publication/directory_laws.rs index 1d7302d3..5e5308ab 100644 --- a/tests/catalog_filesystem_publication/directory_laws.rs +++ b/tests/catalog_filesystem_publication/directory_laws.rs @@ -2,20 +2,78 @@ use std::error::Error; use std::fs; use std::io; -use crate::{FilesystemCatalogPublisher, FilesystemWriterLock}; +use cap_std::ambient_authority; +use cap_std::fs::Dir; -use super::{StoreFixture, restart_policy}; +use crate::{FilesystemEntryKind, FilesystemNamespaceRefusal}; + +use super::super::filesystem_catalog_stages::FilesystemCatalogStages; +use super::super::test_support::namespace_refusal; +use super::{StoreFixture, admit_writer, restart_policy}; #[test] fn publisher_refuses_a_non_directory_protocol_namespace() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-nondirectory")?; fs::remove_dir(store.staging())?; fs::write(store.staging(), [])?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let Err(error) = FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?) - else { - return Err("publisher admitted a non-directory namespace".into()); + let Err(error) = admit_writer(&store) else { + return Err("writer admission accepted a non-directory namespace".into()); + }; + + assert_eq!( + namespace_refusal(&error), + Some(FilesystemNamespaceRefusal::WrongKind { + expected: FilesystemEntryKind::Directory, + observed: FilesystemEntryKind::File, + }) + ); + store.remove() +} + +#[cfg(unix)] +#[test] +fn publisher_never_follows_a_symbolic_protocol_namespace() -> Result<(), Box> { + use std::os::unix::fs::symlink; + + let store = StoreFixture::create("catalog-filesystem-directory-link")?; + fs::remove_dir(store.staging())?; + let target = store.path().join("segments"); + symlink(&target, store.staging())?; + + let Err(error) = admit_writer(&store) else { + return Err("writer admission followed a symbolic namespace".into()); + }; + + assert_eq!( + namespace_refusal(&error), + Some(FilesystemNamespaceRefusal::WrongKind { + expected: FilesystemEntryKind::Directory, + observed: FilesystemEntryKind::Symlink, + }) + ); + assert!(store.staging().is_symlink()); + store.remove() +} + +/// Opens the shared catalog stages directly, past writer admission. +/// +/// The stages repeat the no-follow directory check that admission performs, +/// so a namespace replaced after admission still refuses here. +fn open_stages_past_admission(store: &StoreFixture) -> Result, Box> { + let root = Dir::open_ambient_dir(store.path(), ambient_authority())?; + Ok(FilesystemCatalogStages::open(&root, restart_policy()?).map(drop)) +} + +#[test] +fn stages_refuse_a_non_directory_namespace_replaced_after_admission() -> Result<(), Box> +{ + let store = StoreFixture::create("catalog-stages-nondirectory")?; + fs::remove_dir(store.staging())?; + fs::write(store.staging(), [])?; + + let Err(error) = open_stages_past_admission(&store)? else { + return Err("catalog stages admitted a non-directory namespace".into()); }; assert_eq!(error.kind(), io::ErrorKind::NotADirectory); @@ -24,7 +82,8 @@ fn publisher_refuses_a_non_directory_protocol_namespace() -> Result<(), Box Result<(), Box> { +fn stages_never_follow_a_symbolic_namespace_replaced_after_admission() -> Result<(), Box> +{ use std::os::unix::fs::symlink; #[cfg(target_os = "linux")] @@ -32,16 +91,12 @@ fn publisher_never_follows_a_symbolic_protocol_namespace() -> Result<(), Box Result<(), Box> { @@ -27,11 +27,9 @@ fn require_empty_durable_pools(pool: DurablePool) -> Result<(), Box> let artifact = pool.write(&store)?; let segments = []; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; - let Err(error) = publish_catalog_generation( + let Err(error) = publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), SegmentPublication::none(), @@ -40,7 +38,7 @@ fn require_empty_durable_pools(pool: DurablePool) -> Result<(), Box> ) else { return Err(format!("absent HEAD admitted retained {} bytes", pool.name()).into()); }; - let CatalogPublicationError::Storage { phase, source } = error else { + let CatalogPublicationError::Storage { phase, source } = catalog_refusal(error)? else { return Err("retained pool artifact reached the wrong refusal boundary".into()); }; diff --git a/tests/catalog_filesystem_publication/refusal_laws.rs b/tests/catalog_filesystem_publication/refusal_laws.rs index 4aaf5f87..7ab258db 100644 --- a/tests/catalog_filesystem_publication/refusal_laws.rs +++ b/tests/catalog_filesystem_publication/refusal_laws.rs @@ -6,26 +6,24 @@ use std::fs; use crate::{ AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationPhase, CatalogRestartError, - FilesystemCatalogPublicationError, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, - FilesystemWriterLock, SegmentPublication, publish_catalog_generation, + FilesystemCatalogPublicationError, FilesystemCatalogSnapshot, FilesystemNamespaceRefusal, + SegmentPublication, publish_retained_catalog_generation, }; -use super::super::test_support::require_error; +use super::super::test_support::{namespace_refusal, require_error}; use super::{ - EMPTY_SEGMENT_HEX, StoreFixture, fixture, maximum_segment_policy, restart_policy, - stage_one_zero, + EMPTY_SEGMENT_HEX, StoreFixture, admit_writer, catalog_refusal, fixture, + maximum_segment_policy, open_publisher, restart_policy, stage_one_zero, }; #[test] fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-conflict")?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; let initial_segments = []; let initial_catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &initial_segments)?; - let _initial_receipt = publish_catalog_generation( + let _initial_receipt = publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), SegmentPublication::none(), @@ -53,10 +51,16 @@ fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box let selection = publisher.select_segment(sealed, &segments[0])?; let error = require_error( - publish_catalog_generation(&mut publisher, expectation, selection, &catalog, &segments), + publish_retained_catalog_generation( + &mut publisher, + expectation, + selection, + &catalog, + &segments, + ), "conflicting immutable segment was published", )?; - let CatalogPublicationError::Storage { phase, source } = error else { + let CatalogPublicationError::Storage { phase, source } = catalog_refusal(error)? else { return Err("segment conflict reached the wrong refusal boundary".into()); }; assert_eq!(phase, CatalogPublicationPhase::VerifySegmentPool); @@ -76,15 +80,13 @@ fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box #[test] fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-stale")?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let selection = publisher.select_segment(sealed, &segments[0])?; - let _receipt = publish_catalog_generation( + let _receipt = publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), selection, @@ -94,11 +96,9 @@ fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box< drop(publisher); let stale_candidate = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &[])?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; let error = require_error( - publish_catalog_generation( + publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), SegmentPublication::none(), @@ -108,7 +108,7 @@ fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box< "stale uninitialized expectation was accepted", )?; - let CatalogPublicationError::Storage { phase, source } = error else { + let CatalogPublicationError::Storage { phase, source } = catalog_refusal(error)? else { return Err("stale expectation returned the wrong error".into()); }; assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); @@ -130,49 +130,29 @@ fn stale_current_head_refuses_before_creating_catalog_state() -> Result<(), Box< #[test] fn leftover_next_head_requires_recovery_before_any_mutation() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-next-head")?; - fs::write(store.path().join("head.next"), [])?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; - let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; - let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; - let segments = [segment]; - let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; - let selection = publisher.select_segment(sealed, &segments[0])?; + fs::write(store.path().join("head.next"), b"recovery evidence")?; - let error = require_error( - publish_catalog_generation( - &mut publisher, - CatalogPublicationExpectation::uninitialized(), - selection, - &catalog, - &segments, - ), - "leftover head.next was silently replaced", - )?; - let CatalogPublicationError::Storage { phase, source } = error else { - return Err("head recovery evidence returned the wrong error".into()); + let Err(error) = admit_writer(&store) else { + return Err("writer admission accepted a leftover head.next".into()); }; - assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); - assert!(matches!( - source - .get_ref() - .and_then(|error| error.downcast_ref::()), - Some(FilesystemCatalogPublicationError::HeadRecoveryRequired) - )); - drop(publisher); - assert!(store.path().join("head.next").exists()); - assert!(store.staging().join("current.seg").exists()); + + assert_eq!( + namespace_refusal(&error), + Some(FilesystemNamespaceRefusal::UnexpectedEntry) + ); + assert_eq!( + fs::read(store.path().join("head.next"))?, + b"recovery evidence" + ); assert!(!store.path().join("HEAD").exists()); + assert!(fs::read_dir(store.staging())?.next().is_none()); store.remove() } #[test] fn leftover_catalog_stage_refuses_before_segment_pool_mutation() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-catalog-recovery")?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; fs::write(store.staging().join("current.cat"), b"recovery evidence")?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; @@ -180,7 +160,7 @@ fn leftover_catalog_stage_refuses_before_segment_pool_mutation() -> Result<(), B let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let selection = publisher.select_segment(sealed, &segments[0])?; let error = require_error( - publish_catalog_generation( + publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), selection, @@ -190,7 +170,7 @@ fn leftover_catalog_stage_refuses_before_segment_pool_mutation() -> Result<(), B "leftover catalog stage was discovered after segment publication", )?; - let CatalogPublicationError::Storage { phase, source } = error else { + let CatalogPublicationError::Storage { phase, source } = catalog_refusal(error)? else { return Err("catalog-stage recovery returned the wrong error".into()); }; assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); @@ -216,11 +196,9 @@ fn catalog_only_publication_refuses_a_leftover_segment_stage() -> Result<(), Box let stage = store.staging().join("current.seg"); fs::write(&stage, b"recovery evidence")?; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &[])?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; let error = require_error( - publish_catalog_generation( + publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), SegmentPublication::none(), @@ -230,7 +208,7 @@ fn catalog_only_publication_refuses_a_leftover_segment_stage() -> Result<(), Box "catalog-only publication ignored a leftover segment stage", )?; - let CatalogPublicationError::Storage { phase, source } = error else { + let CatalogPublicationError::Storage { phase, source } = catalog_refusal(error)? else { return Err("segment-stage recovery returned the wrong error".into()); }; assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); @@ -249,15 +227,13 @@ fn catalog_only_publication_refuses_a_leftover_segment_stage() -> Result<(), Box #[test] fn already_published_retry_refuses_a_recreated_segment_stage() -> Result<(), Box> { let store = StoreFixture::create("catalog-filesystem-retry-stage")?; - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; let segment = AdmittedSegment::decode(&segment_bytes, maximum_segment_policy())?; let segments = [segment]; let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; let selection = publisher.select_segment(sealed, &segments[0])?; - let _receipt = publish_catalog_generation( + let _receipt = publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), selection, @@ -266,15 +242,13 @@ fn already_published_retry_refuses_a_recreated_segment_stage() -> Result<(), Box )?; drop(publisher); - let lock = FilesystemWriterLock::try_acquire(store.path())?; - let mut publisher = - FilesystemCatalogPublisher::open_unchecked_for_tests(lock, restart_policy()?)?; + let mut publisher = open_publisher(&store)?; let (sealed, retry_bytes) = stage_one_zero(&publisher, &store)?; let retry_segment = AdmittedSegment::decode(&retry_bytes, maximum_segment_policy())?; let retry_segments = [retry_segment]; let selection = publisher.select_segment(sealed, &retry_segments[0])?; let error = require_error( - publish_catalog_generation( + publish_retained_catalog_generation( &mut publisher, CatalogPublicationExpectation::uninitialized(), selection, @@ -284,7 +258,7 @@ fn already_published_retry_refuses_a_recreated_segment_stage() -> Result<(), Box "already-published retry ignored a recreated segment stage", )?; - let CatalogPublicationError::Storage { phase, source } = error else { + let CatalogPublicationError::Storage { phase, source } = catalog_refusal(error)? else { return Err("already-published stage returned the wrong error".into()); }; assert_eq!(phase, CatalogPublicationPhase::VerifyCurrent); diff --git a/tests/catalog_restart.rs b/tests/catalog_restart.rs index 23607571..ea3afd2a 100644 --- a/tests/catalog_restart.rs +++ b/tests/catalog_restart.rs @@ -1,4 +1,6 @@ -//! Published catalog restart-loading laws. +//! Published catalog restart-loading laws over a writer-published single-format store. + +#![cfg(target_os = "linux")] #[path = "catalog_restart/refusal_laws.rs"] mod refusal_laws; @@ -13,8 +15,11 @@ use std::fs; use std::path::{Path, PathBuf}; use keep::{ - CatalogRestartByteLimit, CatalogRestartPolicy, ChunkId, FilesystemCatalogSnapshot, - LayoutEntryLimit, SegmentReadPolicy, SegmentRecordIdentity, SegmentRecordLimit, + AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, + CatalogPublicationExpectation, CatalogRestartByteLimit, CatalogRestartPolicy, ChunkId, + FilesystemCatalogSnapshot, FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, + LayoutEntryLimit, SegmentReadPolicy, SegmentRecordIdentity, SegmentRecordLimit, StagedSegment, + publish_retained_catalog_generation, }; use sandbox::TestDirectory; use support::decode_hex; @@ -52,17 +57,30 @@ struct StoreFixture { } impl StoreFixture { + /// Initializes a store and publishes the one-zero chunk through the writer. + /// + /// The published HEAD, catalog, and segment must equal the frozen one-zero + /// corpus bytes, so every law below reads exactly those artifacts. fn create(name: &str) -> Result> { let sandbox = TestDirectory::create(name)?; - let catalogs = sandbox.path().join("catalogs"); - let segments = sandbox.path().join("segments"); - fs::create_dir(&catalogs)?; - fs::create_dir(&segments)?; - fs::write(sandbox.path().join("HEAD"), fixture(HEAD_HEX)?)?; - let catalog_path = catalogs.join(format!("0000000000000001-{CATALOG_DIGEST}.cat")); - let segment_path = segments.join(format!("{SEGMENT_DIGEST}.seg")); - fs::write(&catalog_path, fixture(CATALOG_HEX)?)?; - fs::write(&segment_path, fixture(SEGMENT_HEX)?)?; + publish_one_zero(sandbox.path())?; + let catalog_path = sandbox + .path() + .join(format!("catalogs/0000000000000001-{CATALOG_DIGEST}.cat")); + let segment_path = sandbox + .path() + .join(format!("segments/{SEGMENT_DIGEST}.seg")); + for (path, hex) in [ + (sandbox.path().join("HEAD"), HEAD_HEX), + (catalog_path.clone(), CATALOG_HEX), + (segment_path.clone(), SEGMENT_HEX), + ] { + if fs::read(&path)? != fixture(hex)? { + return Err( + format!("writer output {} differs from the corpus", path.display()).into(), + ); + } + } Ok(Self { sandbox, catalog_path, @@ -81,7 +99,7 @@ impl StoreFixture { fn restart_policy() -> Result> { Ok(CatalogRestartPolicy::new( - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), + segment_policy(), CatalogRestartByteLimit::new(RETAINED_SEGMENT_LIMIT)?, )) } @@ -89,3 +107,31 @@ fn restart_policy() -> Result> { fn fixture(hex: &str) -> Result, Box> { decode_hex(hex.strip_suffix('\n').ok_or("fixture must end in one LF")?).map_err(Into::into) } + +/// Publishes generation one holding only the one-zero chunk record. +fn publish_one_zero(root: &Path) -> Result<(), Box> { + let admission = FilesystemVersionTwoAdmission::initialize(root)?.into_admission(); + let mut publisher = FilesystemVersionTwoCatalogPublisher::open(admission, restart_policy()?)?; + let sealed = StagedSegment::begin( + publisher.create_segment_stage()?, + SegmentRecordLimit::MAXIMUM, + )? + .append(AdmittedSegmentRecord::for_chunk(&[0])?)? + .seal()?; + let bytes = fs::read(root.join("staging/current.seg"))?; + let segments = [AdmittedSegment::decode(&bytes, segment_policy())?]; + let selection = publisher.select_segment(sealed, &segments[0])?; + let catalog = CanonicalCatalog::from_segments(CatalogGeneration::new(1)?, None, &segments)?; + let _receipt = publish_retained_catalog_generation( + &mut publisher, + CatalogPublicationExpectation::uninitialized(), + selection, + &catalog, + &segments, + )?; + Ok(()) +} + +const fn segment_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/tests/observed_segment_stage.rs b/tests/observed_segment_stage.rs index 66f0eaef..faea7e18 100644 --- a/tests/observed_segment_stage.rs +++ b/tests/observed_segment_stage.rs @@ -16,10 +16,10 @@ use std::fs; use std::io; use keep::{ - AdmittedSegment, CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, - FilesystemPlatformAdmission, LayoutEntryLimit, ObservedSegmentStage, SegmentReadPolicy, - SegmentRecordLimit, SegmentStageDurabilityEvent, SegmentStageObserver, SegmentWriteError, - SegmentWritePhase, StagedSegment, + AdmittedSegment, CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemVersionTwoAdmission, + FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, ObservedSegmentStage, + SegmentReadPolicy, SegmentRecordLimit, SegmentStageDurabilityEvent, SegmentStageObserver, + SegmentWriteError, SegmentWritePhase, StagedSegment, }; #[test] @@ -134,9 +134,9 @@ impl SegmentStageObserver for Observer { fn publisher( directory: &sandbox::TestDirectory, -) -> Result> { - let admission = FilesystemPlatformAdmission::initialize(directory.path())?; - Ok(FilesystemCatalogPublisher::open( +) -> Result> { + let admission = FilesystemVersionTwoAdmission::initialize(directory.path())?.into_admission(); + Ok(FilesystemVersionTwoCatalogPublisher::open( admission, CatalogRestartPolicy::new(read_policy(), CatalogRestartByteLimit::new(1_048_576)?), )?) diff --git a/tests/recovery_inventory.rs b/tests/recovery_inventory.rs index e6aa6844..1efaa704 100644 --- a/tests/recovery_inventory.rs +++ b/tests/recovery_inventory.rs @@ -66,7 +66,7 @@ fn every_count_precedes_name_retention_in_fixed_namespace_order() -> Result<(), fn unsupported_platform_refuses_before_filesystem_inventory_mutation() -> Result<(), Box> { let sandbox = TestDirectory::create("recovery-inventory-unsupported")?; - let Err(error) = FilesystemRecoveryInventoryReader::open(sandbox.path()) else { + let Err(error) = FilesystemRecoveryInventoryReader::open_version_two(sandbox.path()) else { return Err("filesystem inventory admitted an unsupported platform".into()); }; diff --git a/tests/recovery_name_classification.rs b/tests/recovery_name_classification.rs index 9d616e87..e37b2876 100644 --- a/tests/recovery_name_classification.rs +++ b/tests/recovery_name_classification.rs @@ -12,7 +12,7 @@ use std::error::Error; use inventory_double::InventoryDouble; use keep::{ RecoveryEntryName, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryLimit, - classify_recovery_names, read_recovery_inventory, + classify_version_two_recovery_names, read_recovery_inventory, }; #[test] @@ -20,13 +20,13 @@ fn canonical_names_produce_exact_typed_roles() -> Result<(), Box> { let segment_name = format!("{}.seg", "00".repeat(32)); let catalog_name = format!("{:016x}-{}.cat", 1_u64, "11".repeat(32)); let inventory = inventory([ - names(&["writer.lock", "staging", "segments", "catalogs", "HEAD"])?, + names(&[INITIALIZED_ROOT.as_slice(), &["HEAD"]].concat())?, names(&["current.seg"])?, vec![name(segment_name.as_bytes())?], vec![name(catalog_name.as_bytes())?], ])?; - let manifest = classify_recovery_names(inventory)?; + let manifest = classify_version_two_recovery_names(inventory)?; assert!( manifest @@ -67,8 +67,21 @@ pub(crate) fn inventory( Ok(inventory) } +/// Every root entry an initialized single-format store holds before its first `HEAD`. +pub(crate) const INITIALIZED_ROOT: [&str; 9] = [ + "writer.lock", + "staging", + "segments", + "catalogs", + "reader.lock", + "FORMAT", + "retention", + "gc", + "recovery", +]; + pub(crate) fn initialized_root() -> Result, Box> { - names(&["writer.lock", "staging", "segments", "catalogs"]) + names(&INITIALIZED_ROOT) } pub(crate) fn names(values: &[&str]) -> Result, Box> { diff --git a/tests/recovery_name_classification/grammar_laws.rs b/tests/recovery_name_classification/grammar_laws.rs index 82877f25..c4877f33 100644 --- a/tests/recovery_name_classification/grammar_laws.rs +++ b/tests/recovery_name_classification/grammar_laws.rs @@ -4,7 +4,7 @@ use std::error::Error; use keep::{ RecoveryNameClassificationError, RecoveryNamespace, RecoveryPoolNameError, - classify_recovery_names, + classify_version_two_recovery_names, }; use super::{initialized_root, inventory, name}; @@ -97,7 +97,7 @@ fn pool_error( return Err("pool grammar test requires an immutable namespace".into()); } }; - let Err(error) = classify_recovery_names(inventory(entries)?) else { + let Err(error) = classify_version_two_recovery_names(inventory(entries)?) else { return Err("name classification admitted a noncanonical pool name".into()); }; match error { diff --git a/tests/recovery_name_classification/refusal_laws.rs b/tests/recovery_name_classification/refusal_laws.rs index 53a2fef9..31823b00 100644 --- a/tests/recovery_name_classification/refusal_laws.rs +++ b/tests/recovery_name_classification/refusal_laws.rs @@ -4,14 +4,14 @@ use std::error::Error; use keep::{ RecoveryEntryRole, RecoveryNameClassificationError, RecoveryNamespace, RecoveryPoolNameError, - RecoveryRequiredEntry, classify_recovery_names, + RecoveryRequiredEntry, classify_version_two_recovery_names, }; -use super::{initialized_root, inventory, name, names}; +use super::{INITIALIZED_ROOT, initialized_root, inventory, name, names}; #[test] fn unknown_name_is_refused_with_its_exact_namespace_and_bytes() -> Result<(), Box> { - let Err(error) = classify_recovery_names(inventory([ + let Err(error) = classify_version_two_recovery_names(inventory([ initialized_root()?, Vec::new(), names(&["not-a-segment"])?, @@ -36,8 +36,12 @@ fn unknown_name_is_refused_with_its_exact_namespace_and_bytes() -> Result<(), Bo #[test] fn missing_initialized_root_entry_is_an_exact_refusal() -> Result<(), Box> { - let Err(error) = classify_recovery_names(inventory([ - names(&["staging", "segments", "catalogs"])?, + let Err(error) = classify_version_two_recovery_names(inventory([ + names( + INITIALIZED_ROOT + .get(1..) + .ok_or("initialized root is empty")?, + )?, Vec::new(), Vec::new(), Vec::new(), @@ -56,7 +60,7 @@ fn missing_initialized_root_entry_is_an_exact_refusal() -> Result<(), Box Result<(), Box> { - let Err(error) = classify_recovery_names(inventory([ + let Err(error) = classify_version_two_recovery_names(inventory([ initialized_root()?, Vec::new(), vec![name(format!("{}.seg", "AA".repeat(32)).as_bytes())?], @@ -77,7 +81,7 @@ fn uppercase_pool_digest_is_noncanonical() -> Result<(), Box> { #[test] fn simultaneous_fixed_stages_are_unrecoverable_ambiguity() -> Result<(), Box> { - let Err(error) = classify_recovery_names(inventory([ + let Err(error) = classify_version_two_recovery_names(inventory([ initialized_root()?, names(&["current.seg", "current.cat"])?, Vec::new(), diff --git a/tests/recovery_name_classification_memory.rs b/tests/recovery_name_classification_memory.rs index b622af37..008b8cb1 100644 --- a/tests/recovery_name_classification_memory.rs +++ b/tests/recovery_name_classification_memory.rs @@ -9,18 +9,29 @@ use allocation_counter::measure; use inventory_double::InventoryDouble; use keep::{ RecoveryEntryName, RecoveryInventoryLimit, RecoveryNameClassificationError, - classify_recovery_names, read_recovery_inventory, + classify_version_two_recovery_names, read_recovery_inventory, }; #[test] fn refusal_moves_the_exact_name_without_an_extra_allocation() -> Result<(), Box> { let names = [ - recovery_names(&["writer.lock", "staging", "segments", "catalogs", "unknown"])?, + recovery_names(&[ + "writer.lock", + "staging", + "segments", + "catalogs", + "reader.lock", + "FORMAT", + "retention", + "gc", + "recovery", + "unknown", + ])?, Vec::new(), Vec::new(), Vec::new(), ]; - let mut storage = InventoryDouble::new([5, 0, 0, 0], names); + let mut storage = InventoryDouble::new([10, 0, 0, 0], names); let inventory = read_recovery_inventory(&mut storage, RecoveryInventoryLimit::protocol_maximum())?; assert_eq!(storage.calls().len(), 8); @@ -29,7 +40,7 @@ fn refusal_moves_the_exact_name_without_an_extra_allocation() -> Result<(), Box< let allocations = measure(|| { if let Some(inventory) = input.take() { - result = Some(classify_recovery_names(inventory)); + result = Some(classify_version_two_recovery_names(inventory)); } }); let error = match result.ok_or("name classification did not run")? { diff --git a/tests/segment_filesystem_stage.rs b/tests/segment_filesystem_stage.rs index 8d00d182..48880446 100644 --- a/tests/segment_filesystem_stage.rs +++ b/tests/segment_filesystem_stage.rs @@ -14,9 +14,9 @@ use std::io::ErrorKind; use keep::{ AdmittedSegment, AdmittedSegmentRecord, CatalogRestartByteLimit, CatalogRestartPolicy, - FilesystemCatalogPublisher, FilesystemPlatformAdmission, LayoutEntryLimit, SegmentHeader, - SegmentReadError, SegmentReadPolicy, SegmentRecordLimit, SegmentStageCreateError, - StagedSegment, + FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, + SegmentHeader, SegmentReadError, SegmentReadPolicy, SegmentRecordLimit, + SegmentStageCreateError, StagedSegment, }; use sandbox::TestDirectory; use support::decode_hex; @@ -133,10 +133,14 @@ fn dropping_an_unsealed_stage_preserves_evidence_without_sealing() -> Result<(), Ok(()) } -fn open_publisher(sandbox: &TestDirectory) -> Result> { - let admission = FilesystemPlatformAdmission::initialize(sandbox.path())?; +fn open_publisher( + sandbox: &TestDirectory, +) -> Result> { + let admission = FilesystemVersionTwoAdmission::initialize(sandbox.path())?.into_admission(); let policy = CatalogRestartPolicy::new(read_policy(), CatalogRestartByteLimit::new(1_048_576)?); - Ok(FilesystemCatalogPublisher::open(admission, policy)?) + Ok(FilesystemVersionTwoCatalogPublisher::open( + admission, policy, + )?) } const fn read_policy() -> SegmentReadPolicy { diff --git a/tests/support/admission_refusal.rs b/tests/support/admission_refusal.rs new file mode 100644 index 00000000..0f610b27 --- /dev/null +++ b/tests/support/admission_refusal.rs @@ -0,0 +1,20 @@ +//! This module owns extraction of typed namespace refusals from store admission. + +use keep::{FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError}; + +/// The namespace refusal one store admission failure carries, if any. +/// +/// Admission preserves the refusal as the source of its namespace boundary, +/// so laws assert the exact contradiction instead of an error string. +#[must_use] +pub(crate) fn namespace_refusal( + error: &FilesystemPlatformAdmissionError, +) -> Option { + let FilesystemPlatformAdmissionError::Namespace { source } = error else { + return None; + }; + source + .get_ref()? + .downcast_ref::() + .copied() +} diff --git a/tests/support/mod.rs b/tests/support/mod.rs index 2a72bd69..92227dc1 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -6,6 +6,7 @@ reason = "shared fixtures are crate-visible across binary integration-test modules" )] +pub(crate) mod admission_refusal; pub(crate) mod byte_patches; pub(crate) mod byte_readers; pub(crate) mod byte_writers; @@ -14,6 +15,7 @@ use std::io; use keep::{ChunkSpan, FastCdc}; +pub(crate) use admission_refusal::namespace_refusal; pub(crate) use byte_patches::{domain_hash, patch}; pub(crate) use byte_readers::{FailingReader, LyingReader, PartitionReader}; pub(crate) use byte_writers::{ From 579e64fefaeeb126d6ad629aef7b4d4965ab3b88 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 10:32:18 -0700 Subject: [PATCH 19/36] Feat!: delete version-1 admission and initialization (#198) ADR-0010 step 5. The single format now does everything version 1 did, so version 1 is deleted and every open path refuses a root without the format's FORMAT marker. Deleted: FilesystemPlatformAdmission, the version-1 FilesystemCatalogPublisher and its ungated port impl, the version-1 initializer (initialize_store, StoreInitialization{Error,Phase,Receipt, Storage}, RepositoryInitializationStorage, the filesystem initializer and its storage), the version-1 namespace laws (admit, admit_published, admit_recoverable), the version-1 platform-profile open, the version-1 recovery openers (open(store_root) on the completer, discarder, resumer, finalizer, and inventory reader, and the unchecked variants), classify_recovery_names, and the Platform and WriterLock variants of the four recovery open errors, which no remaining constructor produces. Next-head finalization always runs the retained-closure gate; the version-1 branch that skipped it is gone. FilesystemCatalogSnapshot::load now admits the exact namespace (an in-flight head.next allowed) and the FORMAT marker before opening any catalog artifact, refusing with the new CatalogRestartError::Admission. Tests: the version-1 initializer, platform-admission, and cross-version laws are deleted with their subject. format_marker_laws requires every open path (writer, recovery, inventory, catalog restart, retention and durable readers) to refuse a published, retained store whose FORMAT was removed as MissingFormatMarker, unchanged; a new single-format law refuses initialization on tmpfs before mutation (the obligation of the deleted catalog_platform_admission law); compile_fail doctests now refuse a bare FilesystemWriterLock instead of the deleted type. The replaced-retention portability law displaces the directory outside the root, since catalog restart now refuses unknown root entries. Crash matrix: the version-1 initialization sequence and KEEP-CRASH-031..035 are retired; the retention restart model derives its phase from WriteRootStage's position instead of a hard-coded index. KEEP-STORE-020, KEEP-RECOVERY-002/003, and KEEP-RETENTION-014 are retired in docs/formats/segment-store-v2/retired-identifiers.md with their last evidence and successors; format and corpus pages describe the single format's admission. --- README.md | 2 +- conformance/segment-store/v1/README.md | 17 +- conformance/segment-store/v1/transitions.tsv | 5 - docs/formats/segment-store-v1/publication.md | 105 +++++------- docs/formats/segment-store-v1/rationale.md | 14 +- docs/formats/segment-store-v1/recovery.md | 32 ++-- docs/formats/segment-store-v1/requirements.md | 31 ++-- docs/formats/segment-store-v2/README.md | 2 +- .../segment-store-v2/catalog-publication.md | 30 ++-- .../segment-store-v2/initialization.md | 15 +- docs/formats/segment-store-v2/rationale.md | 7 +- docs/formats/segment-store-v2/recovery.md | 5 +- docs/formats/segment-store-v2/requirements.md | 3 +- .../segment-store-v2/retired-identifiers.md | 75 +++++++++ src/adapters/catalog_restart_error.rs | 11 +- src/adapters/catalog_restart_loader.rs | 15 +- src/adapters/exports.rs | 9 - src/adapters/filesystem_catalog_catalog.rs | 25 +-- src/adapters/filesystem_catalog_current.rs | 12 +- src/adapters/filesystem_catalog_head.rs | 24 +-- src/adapters/filesystem_catalog_publisher.rs | 149 ---------------- .../filesystem_catalog_publisher_storage.rs | 113 ------------- src/adapters/filesystem_catalog_segment.rs | 12 +- src/adapters/filesystem_catalog_snapshot.rs | 12 +- src/adapters/filesystem_catalog_stages.rs | 18 +- src/adapters/filesystem_catalog_storage.rs | 5 +- .../filesystem_initialization_namespace.rs | 63 +------ .../filesystem_initialization_storage.rs | 112 ------------ src/adapters/filesystem_platform_admission.rs | 32 ---- src/adapters/filesystem_platform_profile.rs | 48 +----- .../filesystem_platform_profile_tests.rs | 10 +- .../filesystem_recovery_inventory_reader.rs | 53 +----- ...filesystem_recovery_next_head_candidate.rs | 24 +-- ...overy_next_head_finalization_open_error.rs | 33 +--- ...filesystem_recovery_next_head_finalizer.rs | 62 ++----- ...stem_recovery_segment_resume_open_error.rs | 33 +--- .../filesystem_recovery_segment_resumer.rs | 22 +-- .../filesystem_recovery_stage_completer.rs | 22 +-- ...em_recovery_stage_completion_open_error.rs | 33 +--- ...ystem_recovery_stage_discard_open_error.rs | 24 +-- .../filesystem_recovery_stage_discarder.rs | 90 ++-------- src/adapters/filesystem_segment_stage.rs | 2 +- src/adapters/filesystem_store_initializer.rs | 107 ------------ .../filesystem_store_initializer_tests.rs | 159 ------------------ .../filesystem_version_two_admission.rs | 12 +- ...ilesystem_version_two_catalog_publisher.rs | 13 +- .../filesystem_version_two_catalog_stages.rs | 4 +- ...lesystem_version_two_recovery_admission.rs | 9 +- src/adapters/mod.rs | 14 -- src/adapters/recovery.rs | 4 +- .../recovery/recovery_name_classification.rs | 70 ++------ .../repository_initialization_storage.rs | 67 -------- src/adapters/retention.rs | 2 - .../filesystem_recovery_admission_tests.rs | 44 ----- .../filesystem_retention_authority.rs | 12 +- src/adapters/segment_publication.rs | 2 +- src/adapters/store_initialization.rs | 68 -------- src/adapters/store_initialization_error.rs | 43 ----- src/adapters/store_initialization_phase.rs | 33 ---- src/adapters/store_initialization_receipt.rs | 16 -- src/adapters/store_initialization_storage.rs | 52 ------ src/adapters/verification_failure_class.rs | 19 ++- src/lib.rs | 26 ++- tests/catalog_platform_admission.rs | 74 -------- tests/single_format/initialization_laws.rs | 23 +-- tests/single_format/portability_laws.rs | 5 +- tests/single_format/suite.rs | 2 + .../unsupported_directory.rs | 6 +- tests/store_initialization.rs | 130 -------------- tests/version_two_admission_contract.rs | 29 ++-- .../format_marker_laws.rs | 142 ++++++++++++++++ tests/version_two_catalog_writer/suite.rs | 4 +- .../version_laws.rs | 138 --------------- .../production_protocol.rs | 4 - .../production_protocol/initialization.rs | 19 +-- .../initialization_storage.rs | 63 ------- .../restart/expectation.rs | 1 - .../restart/expectation/sequence.rs | 36 +--- .../restart/expectation/steps.rs | 17 -- .../restart/retention.rs | 14 +- .../restart/semantic.rs | 40 ++--- xtask/src/durability_crash_point.rs | 28 +-- xtask/src/durability_crash_point_identity.rs | 5 - xtask/tests/durability_crash_documentation.rs | 2 +- .../tests/durability_crash_point_contract.rs | 31 +--- .../durability_crash_production_contract.rs | 2 - .../initialization_contract_laws.rs | 2 +- ...ment_store_implementation_documentation.rs | 4 +- .../publication_laws.rs | 17 +- .../transition_laws.rs | 23 +-- 90 files changed, 620 insertions(+), 2428 deletions(-) delete mode 100644 src/adapters/filesystem_catalog_publisher.rs delete mode 100644 src/adapters/filesystem_catalog_publisher_storage.rs delete mode 100644 src/adapters/filesystem_initialization_storage.rs delete mode 100644 src/adapters/filesystem_platform_admission.rs delete mode 100644 src/adapters/filesystem_store_initializer.rs delete mode 100644 src/adapters/filesystem_store_initializer_tests.rs delete mode 100644 src/adapters/repository_initialization_storage.rs delete mode 100644 src/adapters/retention/filesystem_recovery_admission_tests.rs delete mode 100644 src/adapters/store_initialization.rs delete mode 100644 src/adapters/store_initialization_error.rs delete mode 100644 src/adapters/store_initialization_phase.rs delete mode 100644 src/adapters/store_initialization_receipt.rs delete mode 100644 src/adapters/store_initialization_storage.rs delete mode 100644 tests/catalog_platform_admission.rs rename tests/{catalog_platform_admission => single_format}/unsupported_directory.rs (83%) delete mode 100644 tests/store_initialization.rs create mode 100644 tests/version_two_catalog_writer/format_marker_laws.rs delete mode 100644 tests/version_two_catalog_writer/version_laws.rs delete mode 100644 xtask/src/durability_crash_matrix/production_protocol/initialization_storage.rs diff --git a/README.md b/README.md index e2194c82..afa5b222 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Incomplete retention stages are preserved and block publication pending explicit Complete orphans remain recovery-protected until explicit disposition lands with garbage collection (#21). -Version-1 admission still exists and refuses every version-2 store; no conversion exists, and a later unit of [#198](https://github.com/flyingrobots/keep/issues/198) deletes version 1. +Version-1 admission is deleted ([#198](https://github.com/flyingrobots/keep/issues/198)); every open path refuses a root without the `FORMAT` marker, and no conversion from version 1 exists. | Gap | Tracked | | --- | --- | diff --git a/conformance/segment-store/v1/README.md b/conformance/segment-store/v1/README.md index f507da2b..e5bc7a39 100644 --- a/conformance/segment-store/v1/README.md +++ b/conformance/segment-store/v1/README.md @@ -39,7 +39,7 @@ oracles for issues #16 and #17. names the second record at checked offset 209. - `one-zero-bundle-head.hex` publishes the two-entry bundle catalog. - `transitions.tsv` assigns one stable crash-point identifier to every - version-1 durable transition and records its pre-state, interrupted-state + segment, catalog, head, and stage-discard transition and records its pre-state, interrupted-state classification, post-state, and recovery posture. - `ORIGIN.md` records construction and review provenance. @@ -95,9 +95,13 @@ second binds unpublishable `head.next` unlink and root-directory synchronization. Publication unlink identifiers are not reused with false preconditions. -`KEEP-CRASH-031`–`KEEP-CRASH-035` own crash-safe initialization. They bind -writer-lock establishment, each canonical directory creation, and final -root-directory synchronization to explicit partial-state recovery. +`KEEP-CRASH-031`–`KEEP-CRASH-035` are retired: they named the version-1 +initializer, which was deleted when the single store format became the only +format. A store is now created by direct initialization, whose rows are +`KEEP-CRASH-081`–`KEEP-CRASH-091` in the +[version-2 corpus](../v2/transitions.tsv). The +[retirement ledger](../../../docs/formats/segment-store-v2/retired-identifiers.md) +records their last evidence. No row reuses a retired identifier. The `interrupted_class` column names the weakest state that interruption may leave. Recovery must inspect exact bytes and may refine that state only after @@ -105,8 +109,9 @@ complete validation. A `truncated-tail-or-reusable-stage` result, for example, means the observed prefix decides between those two typed outcomes; it does not authorize truncation. -The issue #17 harness executes 105 canonical process-death cases from this -table. It terminates an isolated writer process group, compares +The crash harness executes 90 canonical process-death cases from this table, +each on a store created by direct initialization and written by the +single-format writer. It terminates an isolated writer process group, compares the exact restarted namespace and bytes against an independent expected-state model, and exercises the production recovery classifiers and Golden File Worldline restart loader. The harness proves application process-death diff --git a/conformance/segment-store/v1/transitions.tsv b/conformance/segment-store/v1/transitions.tsv index 4979bfcc..e6b2a032 100644 --- a/conformance/segment-store/v1/transitions.tsv +++ b/conformance/segment-store/v1/transitions.tsv @@ -30,8 +30,3 @@ KEEP-CRASH-027 recovery unlink-truncated-stage named-truncated-stage named-trunc KEEP-CRASH-028 recovery sync-staging-after-discard unlinked-truncated-stage named-truncated-stage-or-discarded-stage discarded-truncated-stage report-explicit-discard KEEP-CRASH-029 recovery unlink-next-head named-unpublishable-next-head named-unpublishable-next-head-or-unlinked-next-head unlinked-next-head verify-requested-next-head-fingerprint KEEP-CRASH-030 recovery sync-root-after-next-head-discard unlinked-next-head named-unpublishable-next-head-or-discarded-next-head discarded-next-head report-explicit-next-head-discard -KEEP-CRASH-031 initialization establish-writer-lock capability-probed-initialization-root recoverable-initialization locked-initialization-root reopen-or-create-and-lock -KEEP-CRASH-032 initialization create-staging-directory locked-initialization-root recoverable-initialization staging-directory-present verify-canonical-initialization-set -KEEP-CRASH-033 initialization create-segment-directory staging-directory-present recoverable-initialization segment-directory-present verify-canonical-initialization-set -KEEP-CRASH-034 initialization create-catalog-directory segment-directory-present recoverable-initialization complete-unsynchronized-initialization-set verify-canonical-initialization-set -KEEP-CRASH-035 initialization sync-root-after-initialization complete-unsynchronized-initialization-set recoverable-initialization-or-uninitialized-store uninitialized-store-admitted report-initialization diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index e2d64879..a76c1f72 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -51,35 +51,20 @@ publication use generation 1 with an all-zero predecessor digest. A missing head in a root containing protocol artifacts requires recovery; it is never silently interpreted as an empty store. -## Crash-safe initialization - -Before mutation, the initializer admits the opened root only when the -capability probe passes and its protocol namespace is either empty or a -partial canonical initialization set containing only `writer.lock`, -`staging`, `segments`, and `catalogs`. Every existing name must have the -required regular-file or directory kind and must pass no-follow admission. -Any artifact, unknown name, wrong kind, or alias is unrecoverable ambiguity. - -The initializer creates `writer.lock` exclusively when absent, or reopens the -existing regular file without following links (`KEEP-CRASH-031`). Its contents -prove nothing. The initializer acquires the exclusive advisory lock before -creating any directory. Under that lock, it creates each absent directory -exclusively in this order: - -1. `staging` (`KEEP-CRASH-032`); -2. `segments` (`KEEP-CRASH-033`); and -3. `catalogs` (`KEEP-CRASH-034`). - -An exact existing directory is idempotent success for its creation step. Each -retry reopens and verifies every existing canonical name. A crash before the -final synchronization leaves a recoverable partial initialization set; the -next initializer reacquires the lock, verifies the observed subset, and -continues without deleting or renaming it. - -After all four canonical names exist, the initializer synchronizes the store -root (`KEEP-CRASH-035`). Only the completed root synchronization admits an -uninitialized store and returns an initialization receipt. No reader treats a -partial set or an unsynchronized complete set as an admitted store. +## Store initialization + +A store is created only by direct initialization of the single store format, +specified in [Initialization](../segment-store-v2/initialization.md) and +crash-tested at `KEEP-CRASH-081`–`KEEP-CRASH-091`. It commits `writer.lock`, +the catalog-protocol directories `staging`, `segments`, and `catalogs`, and the +format's marker, reader fence, and retention, GC, and recovery directories +before any publication. An initialized store has no `HEAD` until its first +publication. + +The version-1 initializer, which created only `writer.lock` and the three +catalog-protocol directories, is retired with its crash rows +`KEEP-CRASH-031`–`KEEP-CRASH-035`; the +[retirement ledger](../segment-store-v2/retired-identifiers.md) records them. ## State and visibility @@ -121,39 +106,31 @@ process-scoped exclusion are unsupported. `FilesystemWriterLock::try_acquire` opens the existing regular `writer.lock` without following symbolic links and acquires its exclusive advisory lock without blocking. That lock alone cannot construct a publisher. -`FilesystemCatalogPublisher::open` consumes `FilesystemPlatformAdmission`, -which owns the lock after initialization and platform checks, then pins the -existing store root plus `staging`, `segments`, and `catalogs`. Both operations -perform blocking filesystem I/O. Neither operation repairs, enumerates, or -removes protocol state. - -`FilesystemPlatformAdmission` has private fields, so only Keep's own -initialization and platform-admission boundary can produce a production value. -A new store obtains one through `FilesystemPlatformAdmission::initialize`, -which runs the ordered initialization protocol (`KEEP-RECOVERY-002`); a -published store reacquires one through `FilesystemPlatformAdmission::reopen`, -which mutates nothing and admits the production platform -(`KEEP-RECOVERY-003`). Both are described in -[Recovery and platform contract](recovery.md). - -The crash matrix opens its catalog publisher through ordinary production -initialization and admission, then wraps that publisher in fault-injecting -decorators. Publication campaigns therefore require admitted Linux ext4 scratch -storage. - -The feature-gated legacy `open_unchecked_for_repository_tasks` entry point now -checks the same production filesystem profile against its retained root -capability before constructing authority; its name no longer denotes a platform -bypass. It reopens that pinned directory with a readable descriptor for profile -ioctls, checks each existing protocol directory, and requires strict root identity. - -The pinned-root route does not re-admit an ambient pathname or claim to establish -the historical spelling used to acquire the lock. Private unit-test construction -remains separate and is not callable by external consumers. - -`publish_catalog_generation` performs complete semantic preflight before the -first storage transition. With `FilesystemCatalogPublisher`, it then executes -the forward segment, catalog, and head protocols below. Every writable catalog +`FilesystemVersionTwoCatalogPublisher::open` consumes +`FilesystemVersionTwoAdmission`, which owns the lock after the production +platform, the exact committed namespace, and the `FORMAT` marker are admitted, +then pins the store root plus `staging`, `segments`, and `catalogs` and the +retention directories. Both operations perform blocking filesystem I/O. +Neither operation repairs, enumerates, or removes protocol state. + +The admission has private fields, so only Keep's own initialization and +admission boundary can produce a value. A new store obtains one through +`FilesystemVersionTwoAdmission::initialize`, which runs the +[initialization protocol](../segment-store-v2/initialization.md); a committed +store reacquires one through `FilesystemVersionTwoAdmission::reopen`, which +mutates nothing. A root without the `FORMAT` marker refuses at every open +path. The version-1 initializer, `FilesystemPlatformAdmission`, and their +rows `KEEP-CRASH-031`–`-035` are retired. + +The crash matrix initializes its store and opens the writer through these +production entry points, then observes the writer's own transitions. +Publication campaigns therefore require admitted Linux ext4 scratch storage. + +The writer publishes only through `publish_retained_catalog_generation`, +which runs the retained-closure gate and then the storage-independent +`publish_catalog_generation` protocol. Complete semantic preflight precedes +the first storage transition. The writer then executes the forward segment, +catalog, and head protocols below. Every writable catalog or head handle is closed before the synchronized stage is reopened read-only. Existing immutable-pool coordinates are never replaced; their bytes are reopened and compared against the preflighted canonical artifact before the @@ -161,7 +138,7 @@ protocol advances. Publisher teardown closes retained writable handles and pinned directory capabilities before releasing the writer lock. Publishing a new filesystem segment additionally requires -`FilesystemCatalogPublisher::select_segment`. The method consumes the +`FilesystemVersionTwoCatalogPublisher::select_segment`. The method consumes the `SealedSegment`, drops Keep's owned writable stage, proves that this publisher created the stage, and binds its record count, byte length, and digest to the exact `AdmittedSegment` bytes. A storage-agnostic @@ -169,7 +146,7 @@ exact `AdmittedSegment` bytes. A storage-agnostic adapters, but its handle-free `ClosedSegment` receipt cannot authorize `staging/current.seg`. Catalog publication cannot select an unrelated or still-open sealed stage. -`FilesystemCatalogPublisher::create_segment_stage` is the only public +`FilesystemVersionTwoCatalogPublisher::create_segment_stage` is the only public filesystem-stage constructor. Its returned lifetime keeps the acquired writer authority borrowed while `current.seg` remains writable. diff --git a/docs/formats/segment-store-v1/rationale.md b/docs/formats/segment-store-v1/rationale.md index d22d7056..27aa0faf 100644 --- a/docs/formats/segment-store-v1/rationale.md +++ b/docs/formats/segment-store-v1/rationale.md @@ -151,17 +151,19 @@ availability. The writer lock proves process-scoped exclusion; it does not prove case-sensitive names, single-host ownership, hard-link and replacement -semantics, or directory durability. `FilesystemCatalogPublisher::open` -therefore consumes an opaque `FilesystemPlatformAdmission` that owns the -acquired lock. The proof is bound to that exact root authority and cannot be +semantics, or directory durability. The writer therefore consumes an opaque +admission value, now `FilesystemVersionTwoAdmission`, that owns the acquired +lock. The proof is bound to that exact root authority and cannot be constructed from metadata or caller assertion. The production proof producer admits one deliberately narrow profile: Linux, writable ext4, a store-root inode without ext4 casefolding, no symbolic link in the selected path, and successful file and directory synchronization calls. -Initialization then refuses any noncanonical root entry before mutation, -acquires and retains `writer.lock`, completes the three canonical directories -in protocol order, and synchronizes the root before returning the proof. +Initialization then refuses any root that is not an initialization prefix +before mutation, acquires and retains `writer.lock`, and returns the proof only +after its commit point; the version-1 initializer that created just the three +catalog-protocol directories is retired +([Initialization](../segment-store-v2/initialization.md)). The profile treats local single-host mounting as an environmental precondition, not as a fact derivable from `statfs`. A shared or multiply mounted ext4 volume diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 14435a02..9c47f0e5 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -5,7 +5,7 @@ the minimum filesystem semantics for `keep.segment-store/v1`. ## Recovery -Opening is read-only. It returns an admitted reader snapshot, an uninitialized +Opening is read-only. It returns an admitted reader snapshot, an unpublished state, a typed recovery inventory and plan, or a refusal. It never changes durable bytes. @@ -29,20 +29,23 @@ the configured and protocol ceilings, count stability, duplicate refusal, and namespace-plus-raw-byte ordering. `FilesystemRecoveryInventoryReader` pins the root and three no-follow child directories, bounds each scan by the remaining global budget, preserves raw Linux name bytes, and verifies child-directory -identity before and after inventory. `classify_recovery_names` requires the -four initialized root entries, types each fixed name and immutable-pool -coordinate, and refuses an unknown or conflicting name without artifact I/O. +identity before and after inventory. `classify_version_two_recovery_names` +requires every root entry of an initialized store, types each fixed name and +immutable-pool coordinate, and refuses an unknown or conflicting name without +artifact I/O. `fingerprint_recovery_stage` then reads a fixed stage through a zero-allocation bounded stream, refuses metadata or observed bytes above the name-selected maximum, and returns its exact observed length and `KEEP:RECOVERY:STAGE\0` fingerprint. New stores obtain writer authority through -`FilesystemPlatformAdmission::initialize`. A stable published store reacquires -authority through `FilesystemPlatformAdmission::reopen`, which performs no +`FilesystemVersionTwoAdmission::initialize`. A committed store reacquires +authority through `FilesystemVersionTwoAdmission::reopen`, and every recovery +tool through `FilesystemVersionTwoRecoveryAdmission::reopen`; each performs no protocol mutation, admits the production platform, acquires the existing -writer lock, and requires exactly `writer.lock`, `staging`, `segments`, -`catalogs`, and a regular `HEAD` in the root. Missing or additional root +writer lock, and requires the exact committed namespace and the `FORMAT` +marker, with `HEAD` optional and, for recovery only, a retained `head.next`. +A root without `FORMAT` refuses at every open path. Missing or additional root evidence remains a typed namespace refusal; content-level head, catalog, and segment verification stays at the publisher and restart boundaries. @@ -127,14 +130,15 @@ Run the repository-owned matrix: cargo xtask durability-crash-matrix ``` -The 105 version-one cases are a subset of the complete command, which also -executes version-two cases. The version-one subset executes the -three ordered positions for each stable `KEEP-CRASH-001`–`KEEP-CRASH-035` -point. Each version-one case owns a +The 90 catalog-protocol cases are a subset of the complete command, which also +executes the retention, writer, and initialization cases. The catalog-protocol +subset executes the three ordered positions for each stable +`KEEP-CRASH-001`–`KEEP-CRASH-030` point. Each case owns a fresh filesystem store and an isolated child process group. The child retains the writer lock and any open staged artifact while it executes the production -initialization, segment-writing, catalog-publication, or recovery-discard -protocol. A fault-injecting port decorator sends one readiness byte at the +segment-writing, catalog-publication, or recovery-discard protocol on a store +it created by direct initialization. An observer installed on the writer's +own segment stage or catalog transitions sends one readiness byte at the selected semantic boundary and waits on the retained connection. The parent keeps the accepted socket open, sends `SIGKILL` to the complete process group, reaps the child, and only then begins restart inspection. A ten-second diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 35f59bae..0adbceac 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -28,7 +28,6 @@ nonclaims, and golden evidence for `keep.segment-store/v1`. | `KEEP-STORE-017` | Catalog locations equal verified top-level segment-record spans | Catalog-entry admission | Specified in #14 | | `KEEP-STORE-018` | Truncated-stage discard has explicit unlink and directory-sync crash boundaries | `KEEP-CRASH-027`–`028` | Specified in #14 | | `KEEP-STORE-019` | Leftover next-head recovery either finalizes an exact successor or discards explicit evidence durably | `KEEP-CRASH-025`–`026`, `029`–`030` | Specified in #14 | -| `KEEP-STORE-020` | Initialization is writer-locked, idempotent, recoverable from every partial canonical namespace set, and admitted only after root synchronization | `KEEP-CRASH-031`–`035` | Specified in #14 | | `KEEP-STORE-021` | Explicit recovery completes a durable fixed-name stage into its immutable pool without publishing a head | `KEEP-CRASH-008`–`012`, `016`–`020` | Specified in #14 | @@ -59,9 +58,9 @@ retention, or garbage collection. ## Catalog implementation evidence Issue #16 implements catalog-generation admission, writer-locked filesystem -publication mechanics, and immutable reader snapshots. Production publisher -construction requires `FilesystemPlatformAdmission`, whose platform-checked -producer is implemented as the initialization slice of issue #17. Explicit +publication mechanics, and immutable reader snapshots. Publisher construction +now requires the single-format `FilesystemVersionTwoAdmission` (#198), which +admits the platform, namespace, and `FORMAT` marker under the writer lock. Explicit recovery is implemented in issue #17; its evidence is the recovery table below. @@ -75,7 +74,7 @@ below. | `KEEP-CATALOG-004` | Every catalog location equals a verified top-level record span in the exact named segment; construction and admission require every supplied segment to be referenced, and admission scans each referenced segment once | Bounded grouped lookup plan and golden artifacts | `tests/catalog_encoding.rs`, `tests/catalog_locations.rs` | Implemented in #16 | | `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | | `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Implemented in #16 | -| `KEEP-CATALOG-007` | Retained kernel locks on the pinned store root and persistent writer file exclude a second cooperative writer even if the directory entry is replaced; neither lock is deleted on release, and lock ownership alone cannot construct a publisher without platform admission | Multi-handle lock model, replacement fixture, and public unsupported-platform refusal | `tests/catalog_writer_lock.rs`, `tests/catalog_platform_admission.rs`; [admission evidence](../../testing-evidence/catalog-platform-admission.md) | Implemented in #16; replacement-hardened in #17; repository-task admission corrected for #150 | +| `KEEP-CATALOG-007` | Retained kernel locks on the pinned store root and persistent writer file exclude a second cooperative writer even if the directory entry is replaced; neither lock is deleted on release, and lock ownership alone cannot construct a publisher without platform admission | Multi-handle lock model, replacement fixture, and public unsupported-platform refusal | `tests/catalog_writer_lock.rs`; the `compile_fail` doctests on `FilesystemVersionTwoCatalogPublisher::open` and `FilesystemRetentionPublicationAuthority::open` refuse a bare `FilesystemWriterLock`; `initialization_refuses_an_unsupported_platform_before_any_mutation` in `tests/single_format.rs`; historical [admission evidence](../../testing-evidence/catalog-platform-admission.md) | Implemented in #16; replacement-hardened in #17; repository-task admission corrected for #150; single-format admission since #198 | | `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented synchronization order; retained fixed-name recovery state refuses before mutation; an absent head requires empty immutable pools; retry of an already-current candidate performs no publication mutation and re-synchronizes the root | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `src/adapters/filesystem_catalog_publisher_tests.rs` | Implemented in #16 | | `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Implemented in #16 | | `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs`, `tests/catalog_model/generated_histories.rs`, `tests/catalog_model/transition_refusals.rs`; bounded oracle and calibration in [model evidence](../../testing-evidence/catalog-model-histories.md) | Implemented in #16; generated model evidence in #166 | @@ -115,13 +114,11 @@ device fault evidence. | ID | Implemented requirement | Oracle | Executable evidence | Status | | --- | --- | --- | --- | --- | -| `KEEP-RECOVERY-001` | Crash identifiers `KEEP-CRASH-001` through `KEEP-CRASH-035` form one contiguous typed vocabulary, map to the exact owning protocol sequence, and admit an occurrence counter only for record append | Ordered identifier-and-sequence ledger | `xtask/tests/durability_crash_point_contract.rs` | Implemented in #17 | -| `KEEP-RECOVERY-002` | Initialization admits the platform before mutation, opens and locks the writer file, admits `staging`, `segments`, and `catalogs` in order, and returns a receipt only after root synchronization; every failed operation retains its exact phase and prevents later transitions | Fault-recording initialization port | `tests/store_initialization.rs` | Implemented in #17 | -| `KEEP-RECOVERY-003` | Production initialization admits only one writable, non-casefolded Linux ext4 store profile; every existing protocol directory must independently satisfy that profile and share the root's device and mount identity; initialization refuses any noncanonical root entry before mutation, completes an empty or partial canonical namespace without replacing evidence, excludes a second initializer, and retains writer authority through the synchronized receipt; published-store reopen performs no mutation, reacquires the writer lock, and requires the complete initialized root plus regular `HEAD` | Capability-relative initialization, published-reopen, child-profile, and exact platform-profile matrix | `src/adapters/filesystem_store_initializer_tests.rs`, `src/adapters/filesystem_catalog_publisher_tests.rs`, `src/adapters/filesystem_platform_profile.rs`, `tests/store_initialization.rs` | Implemented in #17 | +| `KEEP-RECOVERY-001` | Crash identifiers `KEEP-CRASH-001` through `KEEP-CRASH-030` form one contiguous typed vocabulary of catalog-protocol boundaries, map to the exact owning protocol sequence, and admit an occurrence counter only for record append; the retired `KEEP-CRASH-031`–`035` name no boundary | Ordered identifier-and-sequence ledger | `xtask/tests/durability_crash_point_contract.rs` | Implemented in #17 | | `KEEP-RECOVERY-004` | Writer authority is returned only when the locked handle still has the exact device and inode resolved by the canonical `writer.lock` entry after kernel acquisition | Deterministic replacement through the shared authority-producing acquisition boundary; calibrated ignored-refusal mutant | `src/adapters/filesystem_writer_lock_tests.rs`; [acquisition evidence](../../testing-evidence/writer-acquisition-identity.md) | Implemented in #17; acquisition-boundary evidence strengthened in #169 | | `KEEP-RECOVERY-005` | Recovery counts the root and three protocol directories in fixed order before retaining names, refuses at the configured or protocol entry ceiling with the exact observed-at-least count, then returns one duplicate-free inventory sorted by namespace and raw name bytes | Fault-recording inventory port | `tests/recovery_inventory.rs` | Implemented in #17 | | `KEEP-RECOVERY-006` | Filesystem inventory pins the admitted root and protocol directories without following links, verifies child-directory identity before and after scanning, stops each count at the remaining global budget plus one, preserves raw Linux entry-name bytes, and performs no protocol mutation | Capability-relative filesystem fixture | `src/adapters/filesystem_recovery_inventory_tests.rs`, `tests/recovery_inventory.rs` | Implemented in #17 | -| `KEEP-RECOVERY-007` | Name classification requires the four initialized root entries, admits only fixed protocol names and canonical pool coordinates in their owning namespaces, refuses simultaneous fixed recovery stages before artifact reads, and moves a refused raw name without duplicating its allocation | Canonical-name matrix and allocation counter | `tests/recovery_name_classification.rs`, `tests/recovery_name_classification_memory.rs` | Implemented in #17 | +| `KEEP-RECOVERY-007` | Name classification requires every root entry of an initialized single-format store, admits only fixed protocol names and canonical pool coordinates in their owning namespaces, refuses simultaneous fixed recovery stages before artifact reads, and moves a refused raw name without duplicating its allocation | Canonical-name matrix and allocation counter | `tests/recovery_name_classification.rs`, `tests/recovery_name_classification_memory.rs` | Implemented in #17 | | `KEEP-RECOVERY-008` | Stage evidence is fingerprinted through a zero-allocation bounded streaming reader under the named recovery domain; metadata and observed bytes cannot exceed the name-selected protocol maximum, and failures retain exact stage and offset | Independent framing oracle, adversarial reader matrix, and allocation counter | `tests/recovery_stage_fingerprint.rs`, `tests/recovery_stage_fingerprint_memory.rs` | Implemented in #17 | | `KEEP-RECOVERY-009` | Filesystem stage observation uses the pinned inventory capability, never follows a fixed-stage link, admits only regular files, and refuses entry replacement or length drift after bounded fingerprinting | Capability-relative replacement fixtures | `src/adapters/filesystem_recovery_stage_tests.rs` | Implemented in #17 | | `KEEP-RECOVERY-010` | Whole-byte segment-stage classification distinguishes a validated reusable prefix, a complete admitted immutable segment, and exact header, record, or seal truncation only while every available fixed-framing byte remains canonical; proven partial-framing corruption, complete-looking corruption, duplicates, and resource-limit excess remain typed refusals | Exhaustive available-framing-byte, canonical prefix, and corruption matrix | `tests/recovery_segment_classification.rs`, `tests/recovery_segment_classification/*.rs`, `tests/recovery_partial_seal.rs`, `tests/recovery_partial_seal/*.rs` | Implemented in #17; partial-seal framing correction in #171 ([evidence](../../testing-evidence/partial-seal-corruption.md)) | @@ -135,7 +132,7 @@ device fault evidence. | `KEEP-RECOVERY-018` | Filesystem next-head finalization retains root and `writer.lock` authority, pins every protocol directory, revalidates namespace identity and exact stage evidence around bounded complete current and candidate loads, synchronizes and reverifies the exact candidate before atomic replacement, refuses current drift, missing or reappeared candidates, links, corrupt transitive views, and namespace replacement, and returns only after root synchronization | Initial and successor finalization, exact retry, candidate-sync, evidence-drift, missing, link, corruption, namespace-replacement, current-drift, and writer-exclusion matrix | `src/adapters/filesystem_recovery_next_head_finalization_tests.rs`, `src/adapters/filesystem_recovery_next_head_finalization_tests/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-019` | Reusable-segment continuation plans only from an exact reusable `current.seg` assessment within the selected record policy, consumes the storage authority that reopens the stage, re-admits the complete materialized prefix against prior evidence, rebuilds digest and duplicate-identity state, and returns the ordinary append-only stage without rewriting admitted bytes | Reusable-only planning, policy refusal, changed-evidence, storage-failure, duplicate-identity, append, seal, and independent decode matrix | `tests/recovery_segment_resume.rs`, `tests/recovery_segment_resume/*.rs` | Implemented in #17 | | `KEEP-RECOVERY-020` | Filesystem reusable-segment continuation retains root and `writer.lock` authority in the returned stage, pins every protocol directory, opens `current.seg` read-write without following links or truncation, bounds and re-admits its complete bytes, recomputes exact evidence immediately before handoff, revalidates entry identity after reading, positions the handle at the exact validated append boundary, refuses missing, changed, linked, replaced, or namespace-drifted evidence, and preserves the prefix on append or empty seal | Empty and nonempty continuation, writer exclusion, missing, changed evidence, link, namespace replacement, writable-handoff entry or byte replacement, append, seal, and independent decode matrix | `src/adapters/filesystem_recovery_segment_resume_tests.rs`, `src/adapters/filesystem_recovery_segment_resume_tests/*.rs` | Implemented in #17 | -| `KEEP-RECOVERY-021` | Every crash point has exact before, during, and after coordinates; each child executes the production initialization, segment-writing, catalog-publication, or recovery-discard protocol through a fault-injecting port decorator; a deadline-bounded parent receives readiness over a Unix socket, retains that socket, terminates the isolated child process group, and independently verifies the exact Golden File Worldline namespace, bytes, hard-link identity, released writer lock, recovery-stage class, immutable-artifact admission, generation, and visible chunk after restart | Production protocol driver, ordered 105-case model, independent expected-state model, production recovery classifiers, production restart loader, and explicit debug/optimized CI commands | `xtask/tests/durability_crash_production_contract.rs`, `xtask/tests/durability_crash_case_contract.rs`, `xtask/tests/durability_crash_process_contract.rs`, `xtask/src/durability_crash_matrix/`, `.github/workflows/ci.yml` | Implemented in #17 | +| `KEEP-RECOVERY-021` | Every crash point has exact before, during, and after coordinates; each child initializes a single-format store and executes the production segment-writing, gated catalog-publication, or recovery-discard protocol through the writer's own observers; a deadline-bounded parent receives readiness over a Unix socket, retains that socket, terminates the isolated child process group, and independently verifies the exact Golden File Worldline namespace, bytes, hard-link identity, committed `FORMAT` marker, recovery admission of the residue, recovery-stage class, immutable-artifact admission, generation, and visible chunk after restart | Production protocol driver, ordered 90-case model, independent expected-state model, production recovery classifiers, production restart loader, and explicit debug/optimized CI commands | `xtask/tests/durability_crash_production_contract.rs`, `xtask/tests/durability_crash_case_contract.rs`, `xtask/tests/durability_crash_process_contract.rs`, `xtask/src/durability_crash_matrix/`, `.github/workflows/ci.yml` | Implemented in #17 | @@ -183,7 +180,7 @@ contains: - a publication head naming that cross-kind catalog; - exact canonical bytes, checksums, physical digests, lengths, counts, and offsets; and -- the complete `KEEP-CRASH-001`–`KEEP-CRASH-035` transition ledger. +- the complete `KEEP-CRASH-001`–`KEEP-CRASH-030` transition ledger. The test-only Rust oracle reconstructs every artifact directly from these tables and formulas. The issue #15 segment implementation matches the frozen @@ -191,9 +188,15 @@ segment corpus and adds parser fuzzing and corruption evidence. Issue #16 matches the catalog and publication-head corpus, executes the documented publication order through a real filesystem adapter, reconstructs exact immutable restart snapshots, and adds deterministic transition-model and -seeded parser-fuzz evidence. Issue #17 adds initialization, platform -admission, explicit recovery, and the 105-case process-death crash matrix -(`KEEP-RECOVERY-001`–`KEEP-RECOVERY-021`). +seeded parser-fuzz evidence. Issue #17 adds platform admission, explicit +recovery, and the process-death crash matrix +(`KEEP-RECOVERY-001`–`KEEP-RECOVERY-021`). Issue #198 runs those 90 cases on +single-format stores and retires the version-1 initializer. + +`KEEP-STORE-020`, `KEEP-RECOVERY-002`, and `KEEP-RECOVERY-003` named the +version-1 initializer and its platform-checked reopen. They are retired, never +reused, and mapped to their successors in the +[retirement ledger](../segment-store-v2/retired-identifiers.md). The format-local tradeoffs are recorded in the [colocated rationale](rationale.md). diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index fef0360c..56b20499 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -119,4 +119,4 @@ Production durable ingestion ([#82](https://github.com/flyingrobots/keep/issues/ The `KEEP-CRASH-036..052` initial-publication process-death sequence includes independent recovered-reader checks; its [evidence receipt](../../testing-evidence/retention-crash-reader-oracle.md) records the assertions, calibration, and scope. -Version-1 admission still exists and refuses every version-2 store; version-2 admission refuses every version-1 store. No conversion between them exists. Version-1 admission is removed by a later unit of [#198](https://github.com/flyingrobots/keep/issues/198). +Version-1 admission is deleted ([#198](https://github.com/flyingrobots/keep/issues/198)): this is the only format, and every open path refuses a root without its `FORMAT` marker. No conversion from version 1 exists. diff --git a/docs/formats/segment-store-v2/catalog-publication.md b/docs/formats/segment-store-v2/catalog-publication.md index ec3d841b..36c6a347 100644 --- a/docs/formats/segment-store-v2/catalog-publication.md +++ b/docs/formats/segment-store-v2/catalog-publication.md @@ -27,8 +27,8 @@ created by [direct initialization](initialization.md). `FilesystemVersionTwoAdmission::reopen` admits the root and acquires the writer lock. `FilesystemVersionTwoCatalogPublisher::open` consumes that admission. It pins the store root, `staging`, `segments`, `catalogs`, `retention`, -`retention/roots`, and `retention/manifests`. Version-1 writer authority -cannot open it. +`retention/roots`, and `retention/manifests`. A bare writer lock cannot open +it. The publisher implements only `RetainedCatalogPublicationStorage`, so it publishes only through `publish_retained_catalog_generation`. Its stage type, @@ -138,19 +138,19 @@ the gate's exact refusal, and `head.next`, the pools, and `HEAD` stay unchanged. A retained retention stage, or a retention head absent while the pools hold artifacts, refuses in the same way. -## Cross-version entry - -- Version-1 publisher admission, version-1 inventory, and every version-1 - recovery tool keep refusing a version-2 root through - `admit_recoverable` or the exact published namespace. -- Version-2 writer admission, version-2 recovery admission, and the version-2 - inventory refuse a version-1 root because its marker, records, and reader - fence are absent. -- `classify_recovery_names` refuses every version-2 root entry as unexpected. - `classify_version_two_recovery_names` refuses a version-1 inventory as - missing a required entry. -- A segment stage sealed under one publisher's authority, version-1 or - version-2, refuses selection by any other publisher. +## Roots without the marker + +- Writer admission, recovery admission, the recovery inventory, catalog + restart, and the retention and durable readers refuse a root without + `FORMAT` as `MissingFormatMarker`, even when every other entry and record of + a published, retained store is intact. +- `classify_version_two_recovery_names` requires every root entry of an + initialized store and refuses a missing one as a missing required entry. +- A segment stage sealed under one publisher's authority refuses selection by + any other publisher. + +Version-1 admission, the version-1 publisher, and the version-1 recovery tools +are deleted (#198), so no cross-version entry remains. ## Process-death matrix diff --git a/docs/formats/segment-store-v2/initialization.md b/docs/formats/segment-store-v2/initialization.md index 42790f76..fcc0816e 100644 --- a/docs/formats/segment-store-v2/initialization.md +++ b/docs/formats/segment-store-v2/initialization.md @@ -154,13 +154,14 @@ refuses with `WriterLock { source: Busy }`, or, if it acquires them after the first committed, with `AlreadyInitialized`. Both classifications run under the lock, so the second attempt never resumes the first one's phases. -### Version 1 - -A root holding only an empty `writer.lock` is also a version-1 initialization -prefix; it holds nothing of value and either protocol may continue it. Every -later state holds `FORMAT.next` or `FORMAT`, which version-1 admission and -version-1 recovery refuse. Version-1 admission is removed by a later unit of -[#198](https://github.com/flyingrobots/keep/issues/198). +### Roots of the retired version 1 + +Version-1 admission and initialization are deleted +([#198](https://github.com/flyingrobots/keep/issues/198)). A former version-1 +root (`writer.lock`, `staging`, `segments`, `catalogs`, and `HEAD`) is not an +initialization prefix, so `initialize` refuses it unchanged and every open +path refuses it as `MissingFormatMarker`. A root holding only an empty +`writer.lock` is the `Locked` prefix of this protocol and resumes. ## Process-death matrix diff --git a/docs/formats/segment-store-v2/rationale.md b/docs/formats/segment-store-v2/rationale.md index 00fec31e..6c5636ac 100644 --- a/docs/formats/segment-store-v2/rationale.md +++ b/docs/formats/segment-store-v2/rationale.md @@ -187,9 +187,10 @@ transitions as version 1. A copy would let the two formats drift in exactly the steps that decide durability. `FilesystemCatalogStages` therefore holds the pinned catalog directories, the writable stages, and the stage-authority token, and it is the only `CatalogPublicationStorage` implementation of those -transitions. `FilesystemCatalogPublisher` owns one beside its version-1 writer -lock. `FilesystemVersionTwoCatalogStages` wraps one under the version-2 writer -lock and has no public constructor. Each forwards transitions unchanged. +transitions. `FilesystemVersionTwoCatalogStages` wraps one under the writer lock +and has no public constructor; it forwards transitions unchanged. (Until #198 +deleted version 1, the version-1 `FilesystemCatalogPublisher` owned a second +one beside its own writer lock.) The version-2 stage type is reachable only through `catalog_stages`, so only the retained-closure gate can drive it. Repository tasks install an observer diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 5eb45c3b..4955c88f 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -39,9 +39,8 @@ data-dependent but canonically named. Every other root or protocol-directory entry is an unknown entry and unrecoverable ambiguity. Operations are capability-relative and never follow links. -Version-1 admission refuses every root that holds `FORMAT`, `FORMAT.next`, -`reader.lock`, `retention`, `gc`, or `recovery`. Version-1 recovery refuses -the same entries before it pins a pool. +Version-1 admission and recovery, which refused every root holding these +entries, are deleted (#198); every open path now requires `FORMAT`. ## Format marker diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 377eee71..5fb71bc5 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -22,7 +22,6 @@ case is not evidence. | `KEEP-RETENTION-011` | Catalog publication over retention state admits the current retention head, manifest, and every manifest root under writer authority; verifies every retained closure against the candidate before any catalog storage call; and refuses `HEAD` replacement if the retention head changed | port-level laws in `tests/retained_catalog_publication.rs`: dropped and substituted members refuse with the exact missing identity and no catalog stage; every manifest root is verified wherever it sorts; absent, corrupt, substituted, and contradictory head, manifest, and root evidence refuse with typed expected and observed state; a head changed, removed, or published before commit refuses after `head.next` verification without `replace_head`; an exhaustive 24-case candidate-by-root-set model and a repeat-run determinism check | Implemented at the storage port; the filesystem version-2 publisher and version-2 next-head finalization use it (#198, `KEEP-RETENTION-012` and `013`) | | `KEEP-RETENTION-012` | Under version-2 writer authority, one staged segment and a catalog successor publish only through the retained-closure gate, with no-follow bounded retention reads, head absence admitted only when uncontradicted, fixed retention stages refused, and no reader fence taken | `FilesystemVersionTwoCatalogPublisher`; `tests/version_two_catalog_writer.rs`: a preserving successor publishes, reopens, and reads both blobs while the retention root still verifies; a pinned reader view keeps its generation across publication; a candidate that drops retained content, a contradicted head absence, and a fixed retention stage each refuse with the exact typed error and an unchanged store witness; runtime RED for the contradiction and stage checks | Implemented for one staged segment per publication; segment rollover is #121 | | `KEEP-RETENTION-013` | Version-2 segment and catalog stage recovery admits version-2 root entries as known, reuses the version-1 executors, and finalizes a staged `head.next` only after the retained-closure gate admits it | `FilesystemVersionTwoRecoveryAdmission`, `classify_version_two_recovery_names`, and the `open_version_two` constructors; `tests/version_two_catalog_writer.rs` covers classification, completion, discard, reusable-prefix segment resume, and finalization that preserves or would lose retained content (runtime RED with the gate result ignored); `filesystem_version_two_catalog_recovery_tests` recovers injected interruptions after segment link, catalog-stage creation, and `head.next` synchronization | Implemented | -| `KEEP-RETENTION-014` | Version-1 writer and recovery entry points refuse version-2 roots, and version-2 entry points refuse version-1 roots, with typed errors | `tests/version_two_catalog_writer.rs` version laws, including cross-publisher segment selection | Implemented | | `KEEP-RETENTION-015` | Process death in the version-2 writer and its recovery reaches documented recoverable states | `cargo xtask durability-crash-matrix --sequence version-two-catalog` runs `KEEP-CRASH-074..=077`, 12 cases, through recovery and durable reads; `KEEP-CRASH-078..=080` (process death inside version-2 recovery) are planned | In progress: writer rows implemented, recovery rows planned | @@ -45,7 +44,7 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-INITIALIZATION-001` | Direct initialization creates exactly the committed namespace under exclusive writer authority, links `FORMAT` from a durable `FORMAT.next` without replacement, and returns authority only after the commit and cleanup synchronizations and complete admission | `a_fresh_root_commits_to_exactly_the_initialized_namespace` in `initialization_resume_tests`; production-profile laws in `tests/single_format.rs` | Implemented (#198) | | `KEEP-INITIALIZATION-002` | Every interrupted state classifies as exactly one prefix and resumes idempotently; every other nonempty root, including a committed one, refuses with typed evidence before any mutation, writer-lock creation included | `every_interrupted_boundary_classifies_and_resumes_to_one_store` (every boundary of a recorded run, against a model of the protocol table) and the incomplete-stage and sync-failure laws in `initialization_resume_tests`; foreign, out-of-order, stray-retention, foreign-stage, nonempty-lock, linked-directory, committed, and copied-marker refusals with unchanged witnesses in `initialization_prefix_tests` | Implemented (#198) | -| `KEEP-INITIALIZATION-003` | Admission requires a committed marker: writers and readers refuse `FORMAT.next` as `InitializationIncomplete` and a missing `FORMAT` as `MissingFormatMarker`; a committed store without catalog `HEAD` admits writers, refuses readers as `Unpublished`, and its first publication proves both absences instead of assuming them | `filesystem_version_two_admission_tests`; `tests/single_format.rs` initialization laws, including a first publication refused by a stray retention root with an unchanged store | Implemented (#198) | +| `KEEP-INITIALIZATION-003` | Admission requires a committed marker: writers and readers refuse `FORMAT.next` as `InitializationIncomplete`, and every open path (writer, recovery, recovery inventory, catalog restart, retention reader, durable reader) refuses a missing `FORMAT` as `MissingFormatMarker`, so no root without the single format's marker is admitted; a committed store without catalog `HEAD` admits writers, refuses readers as `Unpublished`, and its first publication proves both absences instead of assuming them | `filesystem_version_two_admission_tests`; `every_open_path_refuses_a_root_without_the_format_marker` in `tests/version_two_catalog_writer.rs`; `tests/single_format.rs` initialization laws, including a first publication refused by a stray retention root with an unchanged store | Implemented (#198); every open path since version-1 admission was deleted | | `KEEP-INITIALIZATION-004` | Process death before, during, and after every initialization boundary reaches a documented state that `initialize` recovers | `cargo xtask durability-crash-matrix --sequence version-two-initialization` runs 41 cases at `KEEP-CRASH-081..=091`, checks the exact inventory, marker bytes and inode, admission result, and resumed state, then publishes, retains, and reconstructs the first blob | Implemented (#198); process death only, not power loss | | `KEEP-INITIALIZATION-005` | Concurrent initialization attempts have exactly one initializer: the other refuses on writer authority and, after commit, as already initialized | `concurrent_attempts_have_exactly_one_initializer` in `initialization_resume_tests` | Implemented (#198) | | `KEEP-INITIALIZATION-006` | One public-API sequence initializes, ingests, reopens, reconstructs exact bytes, retains, publishes a second catalog, interrupts publication, recovers, and reads both retained contents; a candidate that would lose retained content refuses with its evidence intact | `the_single_format_serves_every_operation_from_initialization_to_recovery` in `tests/single_format.rs` | Implemented (#198) | diff --git a/docs/formats/segment-store-v2/retired-identifiers.md b/docs/formats/segment-store-v2/retired-identifiers.md index 24bc0502..7ab41257 100644 --- a/docs/formats/segment-store-v2/retired-identifiers.md +++ b/docs/formats/segment-store-v2/retired-identifiers.md @@ -101,3 +101,78 @@ recovery and are not assigned here. PR [#191](https://github.com/flyingrobots/keep/pull/191) proposed typed refusals at the migration storage boundaries; those parts are moot. + +## Version-1 admission (ADR-0010 step 5, #198) + +The version-1 initializer, `FilesystemPlatformAdmission`, the version-1 +`FilesystemCatalogPublisher`, the version-1 recovery openers, and +`classify_recovery_names` were deleted once the single format could do +everything they did. A root without the `FORMAT` marker now refuses at every +open path (`KEEP-INITIALIZATION-003`). + +### Rule for crash identifiers + +A crash identifier names a boundary in code, not a store format. A row whose +boundary is still executed unchanged by the single-format writer keeps its +identifier and moves: `KEEP-CRASH-001` to `-030` name the segment, catalog, +head, and stage-discard transitions of the shared catalog stages, which the +single-format writer runs, so they keep their identifiers and their crash +children now run on stores created by direct initialization. A row whose +boundary no longer exists is retired here and never reused. + +### Requirements + + + +| ID | Obligation | Last evidence on `main` | Disposition | +| --- | --- | --- | --- | +| `KEEP-STORE-020` | Initialization is writer-locked, idempotent, recoverable from every partial canonical namespace set, and admitted only after root synchronization | `docs/formats/segment-store-v1/requirements.md` and rows `KEEP-CRASH-031`–`035` at `e03c2a1` | Moved to `KEEP-INITIALIZATION-001`, `-002`, and `-004` | +| `KEEP-RECOVERY-002` | Initialization admits the platform, locks the writer file, and admits the three directories in order with exact failure phases | `tests/store_initialization.rs` at `c60def9` | Moved to `KEEP-INITIALIZATION-001` and `-002`; every initialization phase keeps its typed failure | +| `KEEP-RECOVERY-003` | Production initialization and published-store reopen admit only the Linux ext4 profile for every protocol directory and the exact version-1 namespace | `src/adapters/filesystem_store_initializer_tests.rs` at `ecf4f7e`; `tests/store_initialization.rs` at `c60def9` | The profile obligation moved to single-format admission: `filesystem_platform_profile` admits every protocol directory, and `initialization_refuses_an_unsupported_platform_before_any_mutation` in `tests/single_format.rs` refuses tmpfs before mutation. The version-1 namespace disappeared | +| `KEEP-RETENTION-014` | Version-1 entry points refuse version-2 roots and version-2 entry points refuse version-1 roots | Never on `main`; introduced and retired within #198 (`tests/version_two_catalog_writer.rs` version laws at `e3930f0`) | The version-1 half disappeared. The version-2 half moved to `KEEP-INITIALIZATION-003`, evidenced by `every_open_path_refuses_a_root_without_the_format_marker` | + + + +`KEEP-MIGRATION-001` and `-008` above deferred their version-2 refusal to +`KEEP-RETENTION-014` until this step; that obligation now lives in +`KEEP-INITIALIZATION-003`. `KEEP-CATALOG-007` keeps its identifier: lock +ownership alone still cannot construct a publisher, now because only +single-format admission produces publisher authority. + +### Crash identifiers + +`KEEP-CRASH-031` to `-035` named the version-1 initializer. Their last +evidence is the initialization crash child at `1fc1ad0` and `f7956c0`, and the +transition ledger at `e03c2a1`. + + + +| ID | Version-1 initialization boundary | Disposition | +| --- | --- | --- | +| `KEEP-CRASH-031` | create or reopen `writer.lock` and lock it | Moved to `KEEP-CRASH-081`, which also synchronizes the lock and the root | +| `KEEP-CRASH-032` | create `staging` | Moved to `KEEP-CRASH-086` (one occurrence per directory) | +| `KEEP-CRASH-033` | create `segments` | Moved to `KEEP-CRASH-086` | +| `KEEP-CRASH-034` | create `catalogs` | Moved to `KEEP-CRASH-086` | +| `KEEP-CRASH-035` | synchronize the root after initialization | Moved to `KEEP-CRASH-088`, the commit point; each directory now also synchronizes its parent inside `KEEP-CRASH-086` | + + + +`KEEP-CRASH-074` to `-077` stay: they are the gated writer's second-generation +publication with a retained root, a scenario distinct from the first +publication that `KEEP-CRASH-009`, `-013`, `-024`, and `-025` cover. + +### Code and evidence removed + +- `src/adapters/filesystem_platform_admission.rs`, + `filesystem_catalog_publisher.rs`, `filesystem_catalog_publisher_storage.rs`, + `filesystem_store_initializer.rs`, `filesystem_initialization_storage.rs`, + `repository_initialization_storage.rs`, and `store_initialization*.rs`; the + version-1 namespace laws; the version-1 recovery openers; and the + `Platform` and `WriterLock` variants of the four recovery open errors, + which no remaining constructor can produce. +- `tests/store_initialization.rs`, `tests/catalog_platform_admission.rs`, the + version-1 initializer unit laws, and the laws whose subject was the boundary + between the two formats (`filesystem_recovery_admission_tests.rs` and the + cross-version laws, replaced by `format_marker_laws.rs`). +- The `initialization` crash sequence, its child and storage decorator, and + transition rows `KEEP-CRASH-031`–`035`. diff --git a/src/adapters/catalog_restart_error.rs b/src/adapters/catalog_restart_error.rs index d61b19a8..3e345d25 100644 --- a/src/adapters/catalog_restart_error.rs +++ b/src/adapters/catalog_restart_error.rs @@ -7,13 +7,20 @@ use std::io; use super::{ CatalogAdmissionError, CatalogDecodeError, CatalogRestartArtifact, CatalogRestartPhase, - CatalogSnapshotError, PublicationHeadDecodeError, SegmentDigest, SegmentReadError, + CatalogSnapshotError, FilesystemPlatformAdmissionError, PublicationHeadDecodeError, + SegmentDigest, SegmentReadError, }; use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; /// Failure to reconstruct one exact published catalog snapshot. #[derive(Debug)] pub enum CatalogRestartError { + /// The root is not an admitted committed store: its namespace or `FORMAT` + /// marker refused before any catalog artifact was opened. + Admission { + /// Preserved namespace or format-marker refusal. + source: Box, + }, /// One capability-relative filesystem operation failed. Io { /// Exact operation that failed. @@ -156,6 +163,7 @@ impl CatalogRestartError { impl fmt::Display for CatalogRestartError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::Admission { .. } => formatter.write_str("catalog restart store admission failed"), Self::Io { phase, .. } | Self::SegmentIo { phase, .. } => { write!(formatter, "catalog restart {phase} failed") } @@ -201,6 +209,7 @@ impl Error for CatalogRestartError { .. } | Self::SegmentIndexAllocation { source, .. } => Some(source), + Self::Admission { source } => Some(source), Self::Head { source } => Some(source), Self::Catalog { source } => Some(source), Self::Segment { source, .. } => Some(source), diff --git a/src/adapters/catalog_restart_loader.rs b/src/adapters/catalog_restart_loader.rs index ce96d44d..a0bf6a3e 100644 --- a/src/adapters/catalog_restart_loader.rs +++ b/src/adapters/catalog_restart_loader.rs @@ -9,7 +9,8 @@ use crate::CatalogLength; use super::{ CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, ChecksummedCatalog, ChecksummedPublicationHead, FilesystemCatalogSnapshot, catalog_restart_io, - catalog_restart_segments, physical_pool_name, + catalog_restart_segments, filesystem_initialization_namespace, + filesystem_version_two_admission, physical_pool_name, }; const HEAD_NAME: &str = "HEAD"; @@ -17,11 +18,23 @@ const CATALOGS_NAME: &str = "catalogs"; const SEGMENTS_NAME: &str = "segments"; const HEAD_LENGTH: u64 = 128; +/// Opens `root`, admits it as a committed store, and loads the `HEAD` snapshot. +/// +/// Admission is the reader law: the exact namespace, which may hold an +/// in-flight `head.next`, and the canonical `FORMAT` marker. A root without the +/// marker refuses before any catalog artifact is opened. pub(super) fn load( root: &Path, policy: CatalogRestartPolicy, ) -> Result { let directory = catalog_restart_io::open_root(root)?; + filesystem_version_two_admission::admit_root( + &directory, + filesystem_initialization_namespace::admit_version_two_with_catalog_stage, + ) + .map_err(|source| CatalogRestartError::Admission { + source: Box::new(source), + })?; load_from_directory(&directory, HEAD_NAME, policy) } diff --git a/src/adapters/exports.rs b/src/adapters/exports.rs index 0635a404..027d4295 100644 --- a/src/adapters/exports.rs +++ b/src/adapters/exports.rs @@ -37,9 +37,7 @@ pub use super::checksummed_publication_head::ChecksummedPublicationHead; pub use super::checksummed_segment_record::ChecksummedSegmentRecord; pub use super::closed_segment::ClosedSegment; pub use super::filesystem_catalog_publication_error::FilesystemCatalogPublicationError; -pub use super::filesystem_catalog_publisher::FilesystemCatalogPublisher; pub use super::filesystem_catalog_snapshot::FilesystemCatalogSnapshot; -pub use super::filesystem_platform_admission::FilesystemPlatformAdmission; pub use super::filesystem_platform_admission_error::FilesystemPlatformAdmissionError; pub use super::filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; pub use super::filesystem_recovery_next_head_finalization_open_error::FilesystemRecoveryNextHeadFinalizationOpenError; @@ -76,8 +74,6 @@ pub use super::observed_segment_stage::ObservedSegmentStage; pub use super::opened_reusable_segment::OpenedReusableSegment; pub use super::publication_head_decode_error::PublicationHeadDecodeError; pub use super::recovery::*; -#[cfg(feature = "repository-tasks")] -pub use super::repository_initialization_storage::RepositoryInitializationStorage; pub use super::retained_catalog_publication::{ PreservedRetentionManifest, RetainedCatalogPublicationError, RetainedCatalogPublicationReceipt, RetainedCatalogPublicationStorage, RetainedClosureAdmission, RetentionEvidencePhase, @@ -112,11 +108,6 @@ pub use super::segment_write_phase::{SegmentDurabilityPhase, SegmentWritePhase}; pub use super::staged_segment::StagedSegment; pub use super::storage_profile_id_text_error::StorageProfileIdParseError; pub use super::store_format::*; -pub use super::store_initialization::initialize_store; -pub use super::store_initialization_error::StoreInitializationError; -pub use super::store_initialization_phase::StoreInitializationPhase; -pub use super::store_initialization_receipt::StoreInitializationReceipt; -pub use super::store_initialization_storage::StoreInitializationStorage; #[cfg(any(test, feature = "repository-tasks"))] pub use super::version_two_initialization::VersionTwoInitializationObserver; pub use super::version_two_initialization::{ diff --git a/src/adapters/filesystem_catalog_catalog.rs b/src/adapters/filesystem_catalog_catalog.rs index 3c419fce..e5621948 100644 --- a/src/adapters/filesystem_catalog_catalog.rs +++ b/src/adapters/filesystem_catalog_catalog.rs @@ -2,11 +2,10 @@ use std::io::{self, Write}; -use super::filesystem_catalog_stages::FilesystemCatalogStages; +use super::filesystem_catalog_stages::{self, FilesystemCatalogStages}; use super::{ CanonicalCatalog, CatalogRestartArtifact, ChecksummedCatalog, - FilesystemCatalogPublicationError, filesystem_catalog_artifact, filesystem_catalog_publisher, - physical_pool_name, + FilesystemCatalogPublicationError, filesystem_catalog_artifact, physical_pool_name, }; pub(super) fn create_stage(stages: &mut FilesystemCatalogStages) -> io::Result<()> { @@ -15,7 +14,7 @@ pub(super) fn create_stage(stages: &mut FilesystemCatalogStages) -> io::Result<( } stages.catalog_stage = Some(filesystem_catalog_artifact::create_exclusive( &stages.staging, - filesystem_catalog_publisher::CURRENT_CATALOG, + filesystem_catalog_stages::CURRENT_CATALOG, )?); Ok(()) } @@ -27,18 +26,6 @@ pub(super) fn write( write_bytes(stages, catalog.encoded()) } -#[cfg(feature = "repository-tasks")] -pub(super) fn write_prefix( - stages: &mut FilesystemCatalogStages, - catalog: &CanonicalCatalog, - prefix: usize, -) -> io::Result<()> { - let bytes = catalog.encoded().get(..prefix).ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidInput, "catalog prefix exceeds bytes") - })?; - write_bytes(stages, bytes) -} - fn write_bytes(stages: &mut FilesystemCatalogStages, bytes: &[u8]) -> io::Result<()> { stage_mut(stages)?.write_all(bytes) } @@ -59,12 +46,12 @@ pub(super) fn link( drop(stage); filesystem_catalog_artifact::verify_catalog( &stages.staging, - filesystem_catalog_publisher::CURRENT_CATALOG, + filesystem_catalog_stages::CURRENT_CATALOG, catalog, )?; super::filesystem_exact_record::link_without_replacement( &stages.staging, - filesystem_catalog_publisher::CURRENT_CATALOG, + filesystem_catalog_stages::CURRENT_CATALOG, &stages.catalogs, &physical_pool_name::catalog(catalog.generation(), catalog.digest()), ) @@ -88,7 +75,7 @@ pub(super) fn synchronize_pool(stages: &FilesystemCatalogStages) -> io::Result<( pub(super) fn remove_stage(stages: &FilesystemCatalogStages) -> io::Result<()> { stages .staging - .remove_file(filesystem_catalog_publisher::CURRENT_CATALOG) + .remove_file(filesystem_catalog_stages::CURRENT_CATALOG) } pub(super) fn synchronize_staging(stages: &FilesystemCatalogStages) -> io::Result<()> { diff --git a/src/adapters/filesystem_catalog_current.rs b/src/adapters/filesystem_catalog_current.rs index b36501db..e139a955 100644 --- a/src/adapters/filesystem_catalog_current.rs +++ b/src/adapters/filesystem_catalog_current.rs @@ -2,11 +2,11 @@ use std::io; -use super::filesystem_catalog_stages::FilesystemCatalogStages; +use super::filesystem_catalog_stages::{self, FilesystemCatalogStages}; use super::{ CatalogPublicationExpectation, CatalogPublicationReadiness, CatalogRestartError, CatalogRestartPhase, CatalogSnapshot, FilesystemCatalogPublicationError, SegmentPublication, - catalog_restart_loader, filesystem_catalog_artifact, filesystem_catalog_publisher, + catalog_restart_loader, filesystem_catalog_artifact, }; pub(super) fn verify( @@ -23,7 +23,7 @@ pub(super) fn verify( } let readiness = match catalog_restart_loader::load_from_directory( &stages.root, - filesystem_catalog_publisher::HEAD, + filesystem_catalog_stages::HEAD, stages.policy, ) { Ok(observed) => { @@ -76,7 +76,7 @@ fn require_segment_authority( fn require_no_next_head(stages: &FilesystemCatalogStages) -> io::Result<()> { match stages .root - .symlink_metadata(filesystem_catalog_publisher::NEXT_HEAD) + .symlink_metadata(filesystem_catalog_stages::NEXT_HEAD) { Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), Err(source) => Err(source), @@ -89,7 +89,7 @@ fn require_no_next_head(stages: &FilesystemCatalogStages) -> io::Result<()> { fn require_no_catalog_stage(stages: &FilesystemCatalogStages) -> io::Result<()> { require_absent( &stages.staging, - filesystem_catalog_publisher::CURRENT_CATALOG, + filesystem_catalog_stages::CURRENT_CATALOG, FilesystemCatalogPublicationError::CatalogRecoveryRequired, ) } @@ -97,7 +97,7 @@ fn require_no_catalog_stage(stages: &FilesystemCatalogStages) -> io::Result<()> fn require_no_segment_stage(stages: &FilesystemCatalogStages) -> io::Result<()> { require_absent( &stages.staging, - filesystem_catalog_publisher::CURRENT_SEGMENT, + filesystem_catalog_stages::CURRENT_SEGMENT, FilesystemCatalogPublicationError::SegmentRecoveryRequired, ) } diff --git a/src/adapters/filesystem_catalog_head.rs b/src/adapters/filesystem_catalog_head.rs index 5d3f3831..80ac3ad8 100644 --- a/src/adapters/filesystem_catalog_head.rs +++ b/src/adapters/filesystem_catalog_head.rs @@ -2,11 +2,10 @@ use std::io::{self, Write}; -use super::filesystem_catalog_stages::FilesystemCatalogStages; +use super::filesystem_catalog_stages::{self, FilesystemCatalogStages}; use super::{ CanonicalPublicationHead, CatalogRestartArtifact, CatalogSnapshot, FilesystemCatalogPublicationError, catalog_restart_loader, filesystem_catalog_artifact, - filesystem_catalog_publisher, }; pub(super) fn create_stage(stages: &mut FilesystemCatalogStages) -> io::Result<()> { @@ -15,7 +14,7 @@ pub(super) fn create_stage(stages: &mut FilesystemCatalogStages) -> io::Result<( } stages.head_stage = Some(filesystem_catalog_artifact::create_exclusive( &stages.root, - filesystem_catalog_publisher::NEXT_HEAD, + filesystem_catalog_stages::NEXT_HEAD, )?); Ok(()) } @@ -27,19 +26,6 @@ pub(super) fn write( write_bytes(stages, head.encoded()) } -#[cfg(feature = "repository-tasks")] -pub(super) fn write_prefix( - stages: &mut FilesystemCatalogStages, - head: &CanonicalPublicationHead, - prefix: usize, -) -> io::Result<()> { - let bytes = head - .encoded() - .get(..prefix) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "head prefix exceeds bytes"))?; - write_bytes(stages, bytes) -} - fn write_bytes(stages: &mut FilesystemCatalogStages, bytes: &[u8]) -> io::Result<()> { stage_mut(stages)?.write_all(bytes) } @@ -61,7 +47,7 @@ pub(super) fn verify_view( drop(stage); let observed = catalog_restart_loader::load_from_directory( &stages.root, - filesystem_catalog_publisher::NEXT_HEAD, + filesystem_catalog_stages::NEXT_HEAD, stages.policy, ) .map_err(filesystem_catalog_artifact::invalid_data)?; @@ -90,9 +76,9 @@ pub(super) fn verify_view( pub(super) fn replace(stages: &FilesystemCatalogStages) -> io::Result<()> { stages.root.rename( - filesystem_catalog_publisher::NEXT_HEAD, + filesystem_catalog_stages::NEXT_HEAD, &stages.root, - filesystem_catalog_publisher::HEAD, + filesystem_catalog_stages::HEAD, ) } diff --git a/src/adapters/filesystem_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs deleted file mode 100644 index 56f817bc..00000000 --- a/src/adapters/filesystem_catalog_publisher.rs +++ /dev/null @@ -1,149 +0,0 @@ -//! This module owns writer-locked filesystem catalog publication state. - -use std::io; - -use super::filesystem_catalog_stages::FilesystemCatalogStages; -use super::{ - AdmittedSegment, CatalogRestartPolicy, FilesystemPlatformAdmission, FilesystemSegmentStage, - FilesystemWriterLock, SealedSegment, SegmentPublication, SegmentPublicationError, - SegmentStageCreateError, -}; -#[cfg(feature = "repository-tasks")] -use super::{CanonicalCatalog, CanonicalPublicationHead, filesystem_catalog_catalog}; - -pub(super) const CURRENT_SEGMENT: &str = "current.seg"; -pub(super) const CURRENT_CATALOG: &str = "current.cat"; -pub(super) const HEAD: &str = "HEAD"; -pub(super) const NEXT_HEAD: &str = "head.next"; - -/// Exclusive filesystem authority for one catalog publication at a time. -/// -/// The publisher owns the writer lock and pinned root, staging, segment-pool, -/// and catalog-pool directory capabilities until it is dropped. Dropping it -/// closes open stages and directory capabilities before releasing the writer -/// lock, but never publishes, removes, truncates, or repairs protocol state. -/// -/// Construction consumes a [`FilesystemPlatformAdmission`] proof created by -/// initializing a new store or reopening a completely published store. -#[must_use] -pub struct FilesystemCatalogPublisher { - pub(super) stages: FilesystemCatalogStages, - // Fields drop in declaration order. Writer authority must outlive every - // directory capability and retained writable stage. - pub(super) _lock: FilesystemWriterLock, -} - -impl FilesystemCatalogPublisher { - /// Pins canonical publication directories under admitted writer authority. - /// - /// # Errors - /// - /// Returns the exact root-clone, no-follow directory-open, or - /// directory-inspection failure. A namespace entry that is not a directory - /// returns [`io::ErrorKind::NotADirectory`]. A failure drops `lock` and - /// therefore releases writer authority. Success allocates one ephemeral - /// authority token that binds later stage selection to this publisher. - pub fn open( - admission: FilesystemPlatformAdmission, - policy: CatalogRestartPolicy, - ) -> io::Result { - let lock = admission.into_lock(); - let pinned_root = lock.clone_directory()?; - let stages = FilesystemCatalogStages::open(&pinned_root, policy)?; - Ok(Self { - stages, - _lock: lock, - }) - } - - /// Admits an already locked root and opens a publisher for repository tasks. - /// - /// This legacy route accepts an already retained writer lock. The catalog - /// crash campaign instead uses ordinary platform initialization and - /// [`Self::open`]. - /// Despite the legacy name, this method now checks the full production - /// platform profile, including protocol-directory mount/device identity. - /// The pinned root is the authority; this does not re-admit an ambient path. - /// Unsupported filesystems cannot obtain publication authority through it. - /// - /// # Errors - /// - /// Returns platform-profile or required root-identity failures before - /// constructing authority, then the same directory failures as [`Self::open`]. - #[cfg(feature = "repository-tasks")] - #[doc(hidden)] - pub fn open_unchecked_for_repository_tasks( - lock: FilesystemWriterLock, - policy: CatalogRestartPolicy, - ) -> io::Result { - Self::open( - FilesystemPlatformAdmission::from_repository_writer_lock(lock)?, - policy, - ) - } - - /// Exclusively creates `staging/current.seg` under this writer authority. - /// - /// The returned stage borrows this publisher until the stage is dropped or - /// consumed by [`crate::StagedSegment`] and explicitly closed after - /// sealing. Creation performs blocking, capability-relative filesystem I/O. - /// - /// # Errors - /// - /// Returns [`SegmentStageCreateError`] without opening or truncating an - /// existing filesystem entry. - pub fn create_segment_stage( - &self, - ) -> Result, SegmentStageCreateError> { - self.stages.create_segment_stage() - } - - /// Closes and selects one synchronized stage created by this publisher. - /// - /// The returned selection remains bound to this exact publisher instance - /// and cannot authorize a metadata-equivalent retained stage owned by - /// another publisher or storage implementation. - /// - /// # Errors - /// - /// Returns [`SegmentPublicationError::PublisherAuthority`] when `sealed` - /// was created by another publisher. Other variants preserve exact - /// closed-stage to admitted-segment disagreements. - pub fn select_segment<'selection, 'records>( - &self, - sealed: SealedSegment>, - admitted: &'selection AdmittedSegment<'records>, - ) -> Result, SegmentPublicationError> { - self.stages.select_segment(sealed, admitted) - } - - /// Writes a strict prefix through the production catalog-stage adapter. - /// - /// # Errors - /// - /// Returns the exact missing-stage, prefix-bound, or write failure. - #[cfg(feature = "repository-tasks")] - #[doc(hidden)] - pub fn write_catalog_prefix_for_repository_tasks( - &mut self, - catalog: &CanonicalCatalog, - prefix: usize, - ) -> io::Result<()> { - filesystem_catalog_catalog::write_prefix(&mut self.stages, catalog, prefix) - } - - /// Writes a strict prefix through the production head-stage adapter. - /// - /// # Errors - /// - /// Returns the exact missing-stage, prefix-bound, or write failure. - #[cfg(feature = "repository-tasks")] - #[doc(hidden)] - pub fn write_head_prefix_for_repository_tasks( - &mut self, - head: &CanonicalPublicationHead, - prefix: usize, - ) -> io::Result<()> { - super::filesystem_catalog_head::write_prefix(&mut self.stages, head, prefix) - } -} diff --git a/src/adapters/filesystem_catalog_publisher_storage.rs b/src/adapters/filesystem_catalog_publisher_storage.rs deleted file mode 100644 index 6d09a5a7..00000000 --- a/src/adapters/filesystem_catalog_publisher_storage.rs +++ /dev/null @@ -1,113 +0,0 @@ -//! This module binds version-one publication authority to the shared stages. -//! -//! Every transition forwards unchanged to -//! [`FilesystemCatalogStages`](super::filesystem_catalog_stages::FilesystemCatalogStages), -//! so the version-one port and the version-two retained-closure port execute -//! one implementation. - -use std::io; - -use super::{ - AdmittedSegment, CanonicalCatalog, CanonicalPublicationHead, CatalogPublicationExpectation, - CatalogPublicationReadiness, CatalogPublicationStorage, CatalogSnapshot, ChecksummedCatalog, - FilesystemCatalogPublisher, SegmentPublication, -}; - -impl CatalogPublicationStorage for FilesystemCatalogPublisher { - fn verify_current( - &mut self, - expected: CatalogPublicationExpectation, - candidate: &CatalogSnapshot<'_, '_, '_>, - segment: &SegmentPublication<'_, '_>, - ) -> io::Result { - self.stages.verify_current(expected, candidate, segment) - } - - fn link_segment(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { - self.stages.link_segment(segment) - } - - fn verify_segment_pool(&mut self, segment: &AdmittedSegment<'_>) -> io::Result<()> { - self.stages.verify_segment_pool(segment) - } - - fn synchronize_segments(&mut self) -> io::Result<()> { - self.stages.synchronize_segments() - } - - fn remove_segment_stage(&mut self) -> io::Result<()> { - self.stages.remove_segment_stage() - } - - fn synchronize_staging_after_segment(&mut self) -> io::Result<()> { - self.stages.synchronize_staging_after_segment() - } - - fn create_catalog_stage(&mut self) -> io::Result<()> { - self.stages.create_catalog_stage() - } - - fn write_catalog(&mut self, catalog: &CanonicalCatalog) -> io::Result<()> { - self.stages.write_catalog(catalog) - } - - fn flush_catalog(&mut self) -> io::Result<()> { - self.stages.flush_catalog() - } - - fn synchronize_catalog(&mut self) -> io::Result<()> { - self.stages.synchronize_catalog() - } - - fn link_catalog(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { - self.stages.link_catalog(catalog) - } - - fn verify_catalog_pool(&mut self, catalog: ChecksummedCatalog<'_>) -> io::Result<()> { - self.stages.verify_catalog_pool(catalog) - } - - fn synchronize_catalogs(&mut self) -> io::Result<()> { - self.stages.synchronize_catalogs() - } - - fn remove_catalog_stage(&mut self) -> io::Result<()> { - self.stages.remove_catalog_stage() - } - - fn synchronize_staging_after_catalog(&mut self) -> io::Result<()> { - self.stages.synchronize_staging_after_catalog() - } - - fn create_head_stage(&mut self) -> io::Result<()> { - self.stages.create_head_stage() - } - - fn write_head(&mut self, head: &CanonicalPublicationHead) -> io::Result<()> { - self.stages.write_head(head) - } - - fn flush_head(&mut self) -> io::Result<()> { - self.stages.flush_head() - } - - fn synchronize_head(&mut self) -> io::Result<()> { - self.stages.synchronize_head() - } - - fn verify_head_view( - &mut self, - head: &CanonicalPublicationHead, - snapshot: &CatalogSnapshot<'_, '_, '_>, - ) -> io::Result<()> { - self.stages.verify_head_view(head, snapshot) - } - - fn replace_head(&mut self) -> io::Result<()> { - self.stages.replace_head() - } - - fn synchronize_root(&mut self) -> io::Result<()> { - self.stages.synchronize_root() - } -} diff --git a/src/adapters/filesystem_catalog_segment.rs b/src/adapters/filesystem_catalog_segment.rs index 3a498d88..7e747510 100644 --- a/src/adapters/filesystem_catalog_segment.rs +++ b/src/adapters/filesystem_catalog_segment.rs @@ -2,10 +2,8 @@ use std::io; -use super::filesystem_catalog_stages::FilesystemCatalogStages; -use super::{ - AdmittedSegment, filesystem_catalog_artifact, filesystem_catalog_publisher, physical_pool_name, -}; +use super::filesystem_catalog_stages::{self, FilesystemCatalogStages}; +use super::{AdmittedSegment, filesystem_catalog_artifact, physical_pool_name}; pub(super) fn link( stages: &FilesystemCatalogStages, @@ -13,13 +11,13 @@ pub(super) fn link( ) -> io::Result<()> { filesystem_catalog_artifact::verify_segment( &stages.staging, - filesystem_catalog_publisher::CURRENT_SEGMENT, + filesystem_catalog_stages::CURRENT_SEGMENT, segment, stages.policy.segment_read(), )?; super::filesystem_exact_record::link_without_replacement( &stages.staging, - filesystem_catalog_publisher::CURRENT_SEGMENT, + filesystem_catalog_stages::CURRENT_SEGMENT, &stages.segments, &physical_pool_name::segment(segment.digest()), ) @@ -44,7 +42,7 @@ pub(super) fn synchronize_pool(stages: &FilesystemCatalogStages) -> io::Result<( pub(super) fn remove_stage(stages: &FilesystemCatalogStages) -> io::Result<()> { stages .staging - .remove_file(filesystem_catalog_publisher::CURRENT_SEGMENT) + .remove_file(filesystem_catalog_stages::CURRENT_SEGMENT) } pub(super) fn synchronize_staging(stages: &FilesystemCatalogStages) -> io::Result<()> { diff --git a/src/adapters/filesystem_catalog_snapshot.rs b/src/adapters/filesystem_catalog_snapshot.rs index d90ef6df..f27fbe30 100644 --- a/src/adapters/filesystem_catalog_snapshot.rs +++ b/src/adapters/filesystem_catalog_snapshot.rs @@ -25,14 +25,18 @@ pub struct FilesystemCatalogSnapshot { } impl FilesystemCatalogSnapshot { - /// Loads and admits the exact snapshot selected by `HEAD`. + /// Admits `root` as a committed store, then loads the exact snapshot selected by `HEAD`. /// - /// The returned owner retains bounded segment bytes so every later logical - /// lookup remains pinned to this immutable generation. + /// Admission requires the exact store namespace, which may hold an in-flight + /// `head.next`, and the canonical `FORMAT` marker; it takes no lock and no + /// reader fence. The returned owner retains bounded segment bytes so every + /// later logical lookup remains pinned to this immutable generation. /// /// # Errors /// - /// Returns [`CatalogRestartError`] for filesystem refusal, malformed or + /// Returns [`CatalogRestartError::Admission`] for a root that is not a + /// committed store, including one without `FORMAT`, and otherwise + /// [`CatalogRestartError`] for filesystem refusal, malformed or /// noncanonical bytes, coordinate disagreement, resource-limit refusal, or /// failed catalog-to-record admission. pub fn load(root: &Path, policy: CatalogRestartPolicy) -> Result { diff --git a/src/adapters/filesystem_catalog_stages.rs b/src/adapters/filesystem_catalog_stages.rs index 1f6d2c70..d6b6c22d 100644 --- a/src/adapters/filesystem_catalog_stages.rs +++ b/src/adapters/filesystem_catalog_stages.rs @@ -1,10 +1,9 @@ -//! This module owns the catalog protocol stages shared by every writer authority. +//! This module owns the catalog protocol stages and their fixed names. //! //! One implementation of the segment, catalog, and head transitions runs under -//! version-one publication authority and under version-two retained-closure -//! authority. Each authority owns its writer lock and decides which protocol -//! entry point may drive these stages; the stages themselves never acquire, -//! release, or widen authority. +//! the writer's retained-closure authority, which owns its writer lock and +//! decides which protocol entry point may drive these stages; the stages +//! themselves never acquire, release, or widen authority. use std::io; @@ -16,6 +15,15 @@ use super::{ SegmentPublication, SegmentPublicationError, SegmentStageCreateError, sync_capable_directory, }; +/// The fixed segment stage inside `staging`. +pub(super) const CURRENT_SEGMENT: &str = "current.seg"; +/// The fixed catalog stage inside `staging`. +pub(super) const CURRENT_CATALOG: &str = "current.cat"; +/// The published catalog head in the store root. +pub(super) const HEAD: &str = "HEAD"; +/// The staged catalog head in the store root. +pub(super) const NEXT_HEAD: &str = "head.next"; + /// Pinned catalog-protocol directories and the stages one writer retains. /// /// The value pins the store root, `staging`, `segments`, and `catalogs` diff --git a/src/adapters/filesystem_catalog_storage.rs b/src/adapters/filesystem_catalog_storage.rs index 4454f656..d88b7a15 100644 --- a/src/adapters/filesystem_catalog_storage.rs +++ b/src/adapters/filesystem_catalog_storage.rs @@ -1,8 +1,7 @@ //! This module binds the shared filesystem catalog stages to the publication port. //! -//! This is the one stage implementation. Each writer authority forwards its own -//! port to it, so version-one and version-two publication execute identical -//! transitions. +//! This is the one stage implementation; the gated writer stages forward every +//! transition to it. use std::io; diff --git a/src/adapters/filesystem_initialization_namespace.rs b/src/adapters/filesystem_initialization_namespace.rs index d216eb1b..d92acd4b 100644 --- a/src/adapters/filesystem_initialization_namespace.rs +++ b/src/adapters/filesystem_initialization_namespace.rs @@ -14,22 +14,6 @@ const STAGING_NAME: &str = "staging"; const SEGMENTS_NAME: &str = "segments"; const CATALOGS_NAME: &str = "catalogs"; const HEAD_NAME: &str = "HEAD"; -const INITIALIZATION_NAMES: [&str; 4] = [LOCK_NAME, STAGING_NAME, SEGMENTS_NAME, CATALOGS_NAME]; -const PUBLISHED_NAMES: [&str; 5] = [ - LOCK_NAME, - STAGING_NAME, - SEGMENTS_NAME, - CATALOGS_NAME, - HEAD_NAME, -]; -const VERSION_TWO_MARKERS: [&str; 6] = [ - "reader.lock", - "FORMAT", - "FORMAT.next", - "retention", - "gc", - "recovery", -]; const READER_LOCK_NAME: &str = "reader.lock"; const MARKER_NAME: &str = "FORMAT"; const MARKER_STAGE_NAME: &str = "FORMAT.next"; @@ -66,42 +50,6 @@ const VERSION_TWO_STAGED_NAMES: [&str; 11] = [ RECOVERY_NAME, ]; -pub(super) fn admit(directory: &Dir) -> io::Result<()> { - admit_optional_file(directory, LOCK_NAME)?; - admit_optional_directory(directory, STAGING_NAME)?; - admit_optional_directory(directory, SEGMENTS_NAME)?; - admit_optional_directory(directory, CATALOGS_NAME)?; - admit_membership(directory, &INITIALIZATION_NAMES) -} - -pub(super) fn admit_published(directory: &Dir) -> io::Result<()> { - admit_required_file(directory, LOCK_NAME)?; - admit_required_directory(directory, STAGING_NAME)?; - admit_required_directory(directory, SEGMENTS_NAME)?; - admit_required_directory(directory, CATALOGS_NAME)?; - admit_required_file(directory, HEAD_NAME)?; - admit_membership(directory, &PUBLISHED_NAMES) -} - -/// Refuses version-two state before version-one recovery touches a pool. -/// -/// Recovery may open a store at any lawful point of its version-one lifecycle, -/// including before first publication and with `head.next` retained, and the -/// recovery inventory already classifies every unknown root entry as -/// unexpected. What that classification cannot express is that a root is a -/// version-two store: a format marker or its stage, a reader fence, or a -/// `retention`, `gc`, or `recovery` directory means version-one recovery must -/// refuse before pinning anything. -pub(super) fn admit_recoverable(directory: &Dir) -> io::Result<()> { - for entry in directory.entries()? { - let name = entry?.file_name(); - if is_canonical(&name, &VERSION_TWO_MARKERS) { - return Err(ambiguous_namespace()); - } - } - Ok(()) -} - /// Admits the exact committed version-2 root namespace. /// /// `writer.lock`, `staging`, `segments`, `catalogs`, the persistent reader @@ -118,10 +66,9 @@ pub(super) fn admit_version_two(directory: &Dir) -> io::Result<()> { /// Admits a committed version-2 root that may retain a catalog `head.next`. /// -/// `head.next` is the version-1 catalog-protocol head stage, which version 2 -/// preserves unchanged. A reader selects only `HEAD`, so a retained or -/// in-flight `head.next` does not change its view, and catalog-stage recovery -/// must be able to open the store that holds one. Writer admission keeps the +/// `head.next` is the catalog-protocol head stage. A reader selects only +/// `HEAD`, so a retained or in-flight `head.next` does not change its view, and +/// catalog-stage recovery must be able to open the store that holds one. Writer admission keeps the /// exact law of [`admit_version_two`] and refuses it. Every other entry and /// every protocol directory is admitted exactly as [`admit_version_two`] does. pub(super) fn admit_version_two_with_catalog_stage(directory: &Dir) -> io::Result<()> { @@ -174,10 +121,6 @@ fn admit_optional_file(directory: &Dir, name: &str) -> io::Result<()> { admit_optional_kind(directory, name, FilesystemEntryKind::File) } -fn admit_optional_directory(directory: &Dir, name: &str) -> io::Result<()> { - admit_optional_kind(directory, name, FilesystemEntryKind::Directory) -} - fn admit_required_file(directory: &Dir, name: &str) -> io::Result<()> { admit_required_kind(directory, name, FilesystemEntryKind::File) } diff --git a/src/adapters/filesystem_initialization_storage.rs b/src/adapters/filesystem_initialization_storage.rs deleted file mode 100644 index 833b47e1..00000000 --- a/src/adapters/filesystem_initialization_storage.rs +++ /dev/null @@ -1,112 +0,0 @@ -//! This module owns concrete filesystem initialization operations. - -use std::io; -use std::path::Path; - -#[cfg(any(test, feature = "repository-tasks"))] -use cap_std::ambient_authority; -use cap_std::fs::Dir; - -use super::filesystem_initialization_namespace; -use super::filesystem_platform_profile; -use super::sync_capable_directory; -use super::{FilesystemWriterLock, StoreInitializationStorage}; - -const STAGING_NAME: &str = "staging"; -const SEGMENTS_NAME: &str = "segments"; -const CATALOGS_NAME: &str = "catalogs"; - -pub(super) struct FilesystemInitializationStorage { - directory: Dir, - lock: Option, -} - -impl FilesystemInitializationStorage { - pub(super) fn admit(store_root: &Path) -> io::Result { - let directory = filesystem_platform_profile::open(store_root)?; - Ok(Self { - directory, - lock: None, - }) - } - - #[cfg(test)] - pub(super) fn admit_unchecked_for_tests(store_root: &Path) -> io::Result { - Self::admit_unchecked(store_root) - } - - #[cfg(feature = "repository-tasks")] - pub(super) fn admit_unchecked_for_repository_tasks(store_root: &Path) -> io::Result { - Self::admit_unchecked(store_root) - } - - #[cfg(any(test, feature = "repository-tasks"))] - fn admit_unchecked(store_root: &Path) -> io::Result { - let directory = Dir::open_ambient_dir(store_root, ambient_authority())?; - Ok(Self { - directory, - lock: None, - }) - } - - pub(super) fn into_lock(self) -> io::Result { - self.lock.ok_or_else(|| { - io::Error::other("initialization completed without retained writer authority") - }) - } - - fn admit_directory(&self, name: &str) -> io::Result<()> { - if self.lock.is_none() { - return Err(io::Error::other( - "initialization directory mutation requires writer authority", - )); - } - match self.directory.create_dir(name) { - Ok(()) => {} - Err(source) if source.kind() == io::ErrorKind::AlreadyExists => {} - Err(source) => return Err(source), - } - let admitted = sync_capable_directory::open(&self.directory, name)?; - drop(admitted); - Ok(()) - } -} - -impl StoreInitializationStorage for FilesystemInitializationStorage { - fn admit_platform(&mut self) -> io::Result<()> { - filesystem_initialization_namespace::admit(&self.directory) - } - - fn open_and_lock_writer_file(&mut self) -> io::Result<()> { - if self.lock.is_some() { - return Err(io::Error::other("writer authority was already acquired")); - } - let lock = FilesystemWriterLock::initialize_in(self.directory.try_clone()?) - .map_err(io::Error::other)?; - self.lock = Some(lock); - filesystem_initialization_namespace::admit(&self.directory) - } - - fn admit_staging_directory(&mut self) -> io::Result<()> { - self.admit_directory(STAGING_NAME) - } - - fn admit_segment_pool_directory(&mut self) -> io::Result<()> { - self.admit_directory(SEGMENTS_NAME) - } - - fn admit_catalog_pool_directory(&mut self) -> io::Result<()> { - self.admit_directory(CATALOGS_NAME) - } - - fn synchronize_root(&mut self) -> io::Result<()> { - if self.lock.is_none() { - return Err(io::Error::other( - "root synchronization requires writer authority", - )); - } - sync_capable_directory::open(&self.directory, ".")? - .into_std_file() - .sync_all() - } -} diff --git a/src/adapters/filesystem_platform_admission.rs b/src/adapters/filesystem_platform_admission.rs deleted file mode 100644 index 097265df..00000000 --- a/src/adapters/filesystem_platform_admission.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! This module owns proof that a filesystem root passed platform admission. - -use super::FilesystemWriterLock; - -/// Exclusive writer authority over a platform-admitted filesystem root. -/// -/// Fields are private so only Keep's initialization and platform-admission -/// boundary can create production values. -#[must_use] -pub struct FilesystemPlatformAdmission { - lock: FilesystemWriterLock, -} - -impl FilesystemPlatformAdmission { - pub(super) const fn initialized(lock: FilesystemWriterLock) -> Self { - Self { lock } - } - - #[cfg(feature = "repository-tasks")] - pub(super) fn from_repository_writer_lock(lock: FilesystemWriterLock) -> std::io::Result { - // Writer capabilities may be O_PATH; profile ioctls require a readable - // descriptor of that same pinned directory, without an ambient reopen. - let pinned = lock.clone_directory()?; - let directory = super::sync_capable_directory::open(&pinned, ".")?; - super::filesystem_platform_profile::admit_locked_root(&directory)?; - Ok(Self::initialized(lock)) - } - - pub(super) fn into_lock(self) -> FilesystemWriterLock { - self.lock - } -} diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index 64aea792..c06f5321 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -5,8 +5,6 @@ use std::path::Path; use cap_std::fs::Dir; -#[cfg(target_os = "linux")] -const PROTOCOL_DIRECTORIES: [&str; 3] = ["staging", "segments", "catalogs"]; /// Every version-two protocol directory, including nested immutable pools. /// /// A version-two root receives writer authority only when each of these shares @@ -36,43 +34,11 @@ struct LinuxDirectoryProperties { mount_id: u64, } -#[cfg(target_os = "linux")] -pub(super) fn open(store_root: &Path) -> io::Result { - use std::fs::File; - - use rustix::fs::{CWD, Mode, OFlags, ResolveFlags, openat2}; - - let descriptor = openat2( - CWD, - store_root, - OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, - Mode::empty(), - ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS, - )?; - let directory = Dir::from_std_file(File::from(descriptor)); - admit_linux_profile(&directory, &PROTOCOL_DIRECTORIES)?; - Ok(directory) -} - -/// Applies the production profile to the root already pinned by writer authority. -#[cfg(all(target_os = "linux", feature = "repository-tasks"))] -pub(super) fn admit_locked_root(directory: &Dir) -> io::Result<()> { - admit_linux_profile(directory, &PROTOCOL_DIRECTORIES) -} - -#[cfg(all(not(target_os = "linux"), feature = "repository-tasks"))] -pub(super) fn admit_locked_root(_directory: &Dir) -> io::Result<()> { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "catalog publication requires the admitted Linux ext4 profile", - )) -} - /// Opens one version-two store root under the admitted Linux profile. /// -/// Identical to [`open`], but every version-two protocol directory that -/// exists must satisfy the same filesystem, mount, and inode-flag laws as the -/// root; absence is left to namespace admission. +/// The root is opened without following links, and every protocol directory +/// that exists must satisfy the same filesystem, mount, and inode-flag laws as +/// the root; absence is left to namespace admission. #[cfg(target_os = "linux")] pub(super) fn open_version_two(store_root: &Path) -> io::Result { use std::fs::File; @@ -91,14 +57,6 @@ pub(super) fn open_version_two(store_root: &Path) -> io::Result { Ok(directory) } -#[cfg(not(target_os = "linux"))] -pub(super) fn open(_store_root: &Path) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "filesystem initialization currently requires the admitted Linux ext4 profile", - )) -} - #[cfg(not(target_os = "linux"))] pub(super) fn open_version_two(_store_root: &Path) -> io::Result { Err(io::Error::new( diff --git a/src/adapters/filesystem_platform_profile_tests.rs b/src/adapters/filesystem_platform_profile_tests.rs index 09082761..81994487 100644 --- a/src/adapters/filesystem_platform_profile_tests.rs +++ b/src/adapters/filesystem_platform_profile_tests.rs @@ -1,8 +1,8 @@ //! Linux filesystem platform-profile laws. use super::{ - LinuxDirectoryProperties, PROTOCOL_DIRECTORIES, VERSION_TWO_PROTOCOL_DIRECTORIES, - admit_linux_child_properties, admit_linux_properties, + LinuxDirectoryProperties, VERSION_TWO_PROTOCOL_DIRECTORIES, admit_linux_child_properties, + admit_linux_properties, }; use rustix::fs::{NFS_SUPER_MAGIC, StatVfsMountFlags}; @@ -31,7 +31,6 @@ fn only_writable_case_sensitive_ext4_is_admitted() { #[test] fn every_protocol_child_must_share_the_root_filesystem_and_mount() { - assert_eq!(PROTOCOL_DIRECTORIES, ["staging", "segments", "catalogs"]); let root = properties(8, 1, 41); let mut casefolded = root; casefolded.inode_flags = EXT4_CASEFOLD_FLAG; @@ -74,7 +73,7 @@ const fn properties( } #[test] -fn version_two_admission_covers_every_protocol_directory() { +fn admission_covers_every_protocol_directory() { assert_eq!( VERSION_TWO_PROTOCOL_DIRECTORIES, [ @@ -89,7 +88,4 @@ fn version_two_admission_covers_every_protocol_directory() { "recovery/dispositions", ] ); - for name in PROTOCOL_DIRECTORIES { - assert!(VERSION_TWO_PROTOCOL_DIRECTORIES.contains(&name)); - } } diff --git a/src/adapters/filesystem_recovery_inventory_reader.rs b/src/adapters/filesystem_recovery_inventory_reader.rs index 99c776a8..91764be8 100644 --- a/src/adapters/filesystem_recovery_inventory_reader.rs +++ b/src/adapters/filesystem_recovery_inventory_reader.rs @@ -25,8 +25,9 @@ const CATALOGS_NAME: &str = "catalogs"; /// A pinned, read-only view of the four recovery inventory namespaces. /// /// Opening performs no protocol mutation. The production constructor admits -/// the same Linux ext4 profile as store initialization and pins all three child -/// directories without following symbolic links. +/// the same Linux ext4 profile, namespace, and `FORMAT` marker as recovery +/// admission and pins all three child directories without following symbolic +/// links. #[must_use] pub struct FilesystemRecoveryInventoryReader { root: Dir, @@ -36,45 +37,10 @@ pub struct FilesystemRecoveryInventoryReader { } impl FilesystemRecoveryInventoryReader { - /// Opens one initialized store for read-only recovery inventory. + /// Opens one committed store for read-only stage inventory. /// - /// The call is synchronous, allocates no content-sized memory, and may - /// block on filesystem I/O. - /// - /// # Errors - /// - /// Returns [`RecoveryInventoryError::Io`] with the exact namespace and - /// open phase for an unsupported platform or missing, replaced, linked, or - /// non-directory protocol namespace. - pub fn open(store_root: &Path) -> Result { - let root = filesystem_platform_profile::open(store_root).map_err(|source| { - RecoveryInventoryError::io( - RecoveryNamespace::Root, - RecoveryInventoryOperation::OpenNamespace, - source, - ) - })?; - Self::from_root(root) - } - - #[cfg(test)] - pub(super) fn open_unchecked_for_tests( - store_root: &Path, - ) -> Result { - let root = Dir::open_ambient_dir(store_root, ambient_authority()).map_err(|source| { - RecoveryInventoryError::io( - RecoveryNamespace::Root, - RecoveryInventoryOperation::OpenNamespace, - source, - ) - })?; - Self::from_root(root) - } - - /// Opens one committed version-two store for read-only stage inventory. - /// - /// Admission is the version-two recovery law: the production platform for - /// every version-two protocol directory, the exact namespace with an + /// Admission is the recovery law: the production platform for every + /// protocol directory, the exact namespace with an /// optional retained catalog `head.next`, and the canonical `FORMAT` marker. /// No writer lock is taken /// and no protocol state is mutated. Classify the resulting inventory with @@ -85,7 +51,7 @@ impl FilesystemRecoveryInventoryReader { /// Returns [`RecoveryInventoryError::Io`] for the store root when platform /// or version-two admission refuses; the source carries the exact /// [`FilesystemPlatformAdmissionError`](crate::FilesystemPlatformAdmissionError). - /// A version-one root refuses because its marker and records are absent. + /// A root without the `FORMAT` marker refuses. pub fn open_version_two(store_root: &Path) -> Result { let root = filesystem_platform_profile::open_version_two(store_root).map_err(root_error)?; admit_root( @@ -118,11 +84,6 @@ impl FilesystemRecoveryInventoryReader { Self::pin(root) } - pub(super) fn from_root(root: Dir) -> Result { - filesystem_initialization_namespace::admit_recoverable(&root).map_err(root_error)?; - Self::pin(root) - } - /// Pins an already admitted, writer-locked version-two root. pub(super) fn from_version_two_root(root: Dir) -> Result { filesystem_initialization_namespace::admit_version_two_with_catalog_stage(&root) diff --git a/src/adapters/filesystem_recovery_next_head_candidate.rs b/src/adapters/filesystem_recovery_next_head_candidate.rs index f75dffda..e4e4ef39 100644 --- a/src/adapters/filesystem_recovery_next_head_candidate.rs +++ b/src/adapters/filesystem_recovery_next_head_candidate.rs @@ -2,7 +2,6 @@ use std::io; -use super::filesystem_recovery_next_head_finalizer::FinalizationRetention; use super::retained_catalog_publication::admit_retained_closures; use super::{ CatalogSnapshot, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageError, @@ -116,7 +115,7 @@ fn stage(source: FilesystemRecoveryStageError) -> RecoveryNextHeadFinalizationSt /// Runs the forward catalog gate's retained-closure admission on the candidate. /// -/// Version-two finalization changes `HEAD` exactly as forward publication +/// Finalization changes `HEAD` exactly as forward publication /// does, so every root named by the current retention head must reconstruct /// its complete closure from the candidate catalog. The check reads retention /// evidence under the finalizer's writer authority and mutates nothing; a @@ -125,17 +124,12 @@ fn require_retained_closures( finalizer: &FilesystemRecoveryNextHeadFinalizer, candidate: &CatalogSnapshot<'_, '_, '_>, ) -> Result<(), RecoveryNextHeadFinalizationStorageError> { - match &finalizer.retention { - FinalizationRetention::VersionOne => Ok(()), - FinalizationRetention::VersionTwo(evidence) => { - let mut source = evidence; - admit_retained_closures(&mut source, candidate) - .map(|_admission| ()) - .map_err( - |source| RecoveryNextHeadFinalizationStorageError::RetainedClosure { - source: Box::new(source), - }, - ) - } - } + let mut source = &finalizer.retention; + admit_retained_closures(&mut source, candidate) + .map(|_admission| ()) + .map_err( + |source| RecoveryNextHeadFinalizationStorageError::RetainedClosure { + source: Box::new(source), + }, + ) } diff --git a/src/adapters/filesystem_recovery_next_head_finalization_open_error.rs b/src/adapters/filesystem_recovery_next_head_finalization_open_error.rs index a6634808..173cac8c 100644 --- a/src/adapters/filesystem_recovery_next_head_finalization_open_error.rs +++ b/src/adapters/filesystem_recovery_next_head_finalization_open_error.rs @@ -4,23 +4,11 @@ use std::error::Error; use std::fmt; use std::io; -use super::{ - FilesystemRecoveryStageDiscardOpenError, RecoveryInventoryError, WriterLockAcquireError, -}; +use super::{FilesystemRecoveryStageDiscardOpenError, RecoveryInventoryError}; /// Why a pinned writer-authorized next-head finalizer could not be opened. #[derive(Debug)] pub enum FilesystemRecoveryNextHeadFinalizationOpenError { - /// The store root did not satisfy the supported platform profile. - Platform { - /// Exact platform-admission failure. - source: io::Error, - }, - /// Exclusive writer authority could not be acquired. - WriterLock { - /// Exact writer-lock acquisition refusal. - source: WriterLockAcquireError, - }, /// The locked root capability could not be cloned for recovery inventory. CloneRoot { /// Exact root-capability clone failure. @@ -38,12 +26,6 @@ impl From { fn from(source: FilesystemRecoveryStageDiscardOpenError) -> Self { match source { - FilesystemRecoveryStageDiscardOpenError::Platform { source } => { - Self::Platform { source } - } - FilesystemRecoveryStageDiscardOpenError::WriterLock { source } => { - Self::WriterLock { source } - } FilesystemRecoveryStageDiscardOpenError::CloneRoot { source } => { Self::CloneRoot { source } } @@ -57,16 +39,6 @@ impl From impl fmt::Display for FilesystemRecoveryNextHeadFinalizationOpenError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Platform { source } => { - write!( - formatter, - "recovery next-head platform was refused: {source}" - ) - } - Self::WriterLock { source } => write!( - formatter, - "recovery next-head writer lock was refused: {source}" - ), Self::CloneRoot { source } => { write!(formatter, "locked recovery root clone failed: {source}") } @@ -81,8 +53,7 @@ impl fmt::Display for FilesystemRecoveryNextHeadFinalizationOpenError { impl Error for FilesystemRecoveryNextHeadFinalizationOpenError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::Platform { source } | Self::CloneRoot { source } => Some(source), - Self::WriterLock { source } => Some(source), + Self::CloneRoot { source } => Some(source), Self::Namespace { source } => Some(source), } } diff --git a/src/adapters/filesystem_recovery_next_head_finalizer.rs b/src/adapters/filesystem_recovery_next_head_finalizer.rs index 92cc0174..64d05a95 100644 --- a/src/adapters/filesystem_recovery_next_head_finalizer.rs +++ b/src/adapters/filesystem_recovery_next_head_finalizer.rs @@ -1,7 +1,5 @@ //! This module owns pinned writer authority for filesystem head finalization. -use std::path::Path; - use cap_std::fs::Dir; use super::retention::FilesystemRetainedEvidence; @@ -13,52 +11,26 @@ use super::{ /// Writer-authorized pinned filesystem adapter for exact next-head finalization. /// -/// Opening proves the supported platform, pins and exclusively locks the store -/// root and `writer.lock`, then pins all three protocol child directories +/// Opening consumes recovery admission, which holds the exclusive writer lock +/// over an admitted root, then pins all three protocol child directories /// without following links. The synchronous adapter may block on bounded /// filesystem I/O and retains writer authority until dropped. /// -/// A finalizer opened over a version-two store also retains the admitted -/// retention capabilities and, immediately before every candidate -/// verification that can precede `HEAD` replacement, verifies every current -/// retained closure against the staged candidate with the same admission the -/// forward catalog gate runs. A candidate that would lose retained content -/// refuses and leaves `head.next` in place. +/// The finalizer also retains the admitted retention capabilities and, +/// immediately before every candidate verification that can precede `HEAD` +/// replacement, verifies every current retained closure against the staged +/// candidate with the same admission the forward catalog gate runs. A +/// candidate that would lose retained content refuses and leaves `head.next` +/// in place. #[must_use] pub struct FilesystemRecoveryNextHeadFinalizer { pub(super) discarder: FilesystemRecoveryStageDiscarder, pub(super) policy: CatalogRestartPolicy, - pub(super) retention: FinalizationRetention, -} - -/// Retention evidence a finalizer must preserve, by store format. -pub(super) enum FinalizationRetention { - /// A version-one store has no retention state to preserve. - VersionOne, - /// A version-two store's retained closures gate every replacement. - VersionTwo(FilesystemRetainedEvidence), + pub(super) retention: FilesystemRetainedEvidence, } impl FilesystemRecoveryNextHeadFinalizer { - /// Opens an initialized supported store for explicit head finalization. - /// - /// The call performs no protocol mutation and does not read `HEAD` or - /// `head.next`. - /// - /// # Errors - /// - /// Returns [`FilesystemRecoveryNextHeadFinalizationOpenError`] on platform, - /// writer-authority, root-clone, or namespace admission failure. - pub fn open( - store_root: &Path, - policy: CatalogRestartPolicy, - ) -> Result { - FilesystemRecoveryStageDiscarder::open(store_root) - .map(|discarder| Self::version_one(discarder, policy)) - .map_err(Into::into) - } - - /// Pins a committed version-two store for gated head finalization. + /// Pins a committed store for gated head finalization. /// /// The call performs no protocol mutation and reads no record. Execution /// verifies every retained closure named by the current retention head @@ -77,25 +49,13 @@ impl FilesystemRecoveryNextHeadFinalizer { .clone_directory() .map_err(|source| FilesystemRecoveryStageDiscardOpenError::CloneRoot { source })?; let discarder = FilesystemRecoveryStageDiscarder::from_version_two_lock(lock)?; - let evidence = FilesystemRetainedEvidence::new(root, retention, roots, manifests); Ok(Self { discarder, policy, - retention: FinalizationRetention::VersionTwo(evidence), + retention: FilesystemRetainedEvidence::new(root, retention, roots, manifests), }) } - const fn version_one( - discarder: FilesystemRecoveryStageDiscarder, - policy: CatalogRestartPolicy, - ) -> Self { - Self { - discarder, - policy, - retention: FinalizationRetention::VersionOne, - } - } - pub(super) const fn root(&self) -> &Dir { self.discarder .inventory diff --git a/src/adapters/filesystem_recovery_segment_resume_open_error.rs b/src/adapters/filesystem_recovery_segment_resume_open_error.rs index ae024c8b..ec9ebda9 100644 --- a/src/adapters/filesystem_recovery_segment_resume_open_error.rs +++ b/src/adapters/filesystem_recovery_segment_resume_open_error.rs @@ -4,23 +4,11 @@ use std::error::Error; use std::fmt; use std::io; -use super::{ - FilesystemRecoveryStageDiscardOpenError, RecoveryInventoryError, WriterLockAcquireError, -}; +use super::{FilesystemRecoveryStageDiscardOpenError, RecoveryInventoryError}; /// Why a pinned writer-authorized segment resumer could not be opened. #[derive(Debug)] pub enum FilesystemRecoverySegmentResumeOpenError { - /// The store root did not satisfy the supported platform profile. - Platform { - /// Exact platform-admission failure. - source: io::Error, - }, - /// Exclusive writer authority could not be acquired. - WriterLock { - /// Exact writer-lock acquisition refusal. - source: WriterLockAcquireError, - }, /// The locked root capability could not be cloned for recovery inventory. CloneRoot { /// Exact root-capability clone failure. @@ -36,12 +24,6 @@ pub enum FilesystemRecoverySegmentResumeOpenError { impl From for FilesystemRecoverySegmentResumeOpenError { fn from(source: FilesystemRecoveryStageDiscardOpenError) -> Self { match source { - FilesystemRecoveryStageDiscardOpenError::Platform { source } => { - Self::Platform { source } - } - FilesystemRecoveryStageDiscardOpenError::WriterLock { source } => { - Self::WriterLock { source } - } FilesystemRecoveryStageDiscardOpenError::CloneRoot { source } => { Self::CloneRoot { source } } @@ -55,16 +37,6 @@ impl From for FilesystemRecoverySegment impl fmt::Display for FilesystemRecoverySegmentResumeOpenError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Platform { source } => { - write!( - formatter, - "segment continuation platform was refused: {source}" - ) - } - Self::WriterLock { source } => write!( - formatter, - "segment continuation writer lock was refused: {source}" - ), Self::CloneRoot { source } => { write!(formatter, "locked recovery root clone failed: {source}") } @@ -79,8 +51,7 @@ impl fmt::Display for FilesystemRecoverySegmentResumeOpenError { impl Error for FilesystemRecoverySegmentResumeOpenError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::Platform { source } | Self::CloneRoot { source } => Some(source), - Self::WriterLock { source } => Some(source), + Self::CloneRoot { source } => Some(source), Self::Namespace { source } => Some(source), } } diff --git a/src/adapters/filesystem_recovery_segment_resumer.rs b/src/adapters/filesystem_recovery_segment_resumer.rs index 97062216..5f67a98d 100644 --- a/src/adapters/filesystem_recovery_segment_resumer.rs +++ b/src/adapters/filesystem_recovery_segment_resumer.rs @@ -1,7 +1,5 @@ //! This module owns pinned writer authority for filesystem segment continuation. -use std::path::Path; - use super::{ FilesystemRecoverySegmentResumeOpenError, FilesystemRecoveryStageDiscarder, FilesystemVersionTwoRecoveryAdmission, @@ -9,8 +7,8 @@ use super::{ /// Writer-authorized pinned filesystem adapter for reusable segment recovery. /// -/// Opening proves the supported platform, pins and exclusively locks the store -/// root and `writer.lock`, then pins all three protocol child directories +/// Opening consumes recovery admission, which holds the exclusive writer lock +/// over an admitted root, then pins all three protocol child directories /// without following links. Execution consumes this value so the returned /// writable stage retains that authority until it is sealed or dropped. #[must_use] @@ -21,21 +19,7 @@ pub struct FilesystemRecoverySegmentResumer { } impl FilesystemRecoverySegmentResumer { - /// Opens an initialized supported store for explicit segment continuation. - /// - /// The call performs no protocol mutation. - /// - /// # Errors - /// - /// Returns [`FilesystemRecoverySegmentResumeOpenError`] on platform, - /// writer-authority, root-clone, or namespace admission failure. - pub fn open(store_root: &Path) -> Result { - FilesystemRecoveryStageDiscarder::open(store_root) - .map(Self::from_discarder) - .map_err(Into::into) - } - - /// Pins a committed version-two store for explicit segment continuation. + /// Pins a committed store for explicit segment continuation. /// /// The call performs no protocol mutation. /// diff --git a/src/adapters/filesystem_recovery_stage_completer.rs b/src/adapters/filesystem_recovery_stage_completer.rs index 527d18f3..83182218 100644 --- a/src/adapters/filesystem_recovery_stage_completer.rs +++ b/src/adapters/filesystem_recovery_stage_completer.rs @@ -1,7 +1,5 @@ //! This module owns pinned writer authority for filesystem stage completion. -use std::path::Path; - use super::{ FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscarder, FilesystemVersionTwoRecoveryAdmission, @@ -9,8 +7,8 @@ use super::{ /// Writer-authorized pinned filesystem adapter for exact stage completion. /// -/// Opening proves the supported platform, pins and exclusively locks the store -/// root and `writer.lock`, then pins all three protocol child directories +/// Opening consumes recovery admission, which holds the exclusive writer lock +/// over an admitted root, then pins all three protocol child directories /// without following links. The synchronous adapter may block on filesystem /// I/O and retains writer authority until dropped. #[must_use] @@ -19,21 +17,7 @@ pub struct FilesystemRecoveryStageCompleter { } impl FilesystemRecoveryStageCompleter { - /// Opens an initialized supported store for explicit stage completion. - /// - /// The call performs no protocol mutation. - /// - /// # Errors - /// - /// Returns [`FilesystemRecoveryStageCompletionOpenError`] on platform, - /// writer-authority, root-clone, or namespace admission failure. - pub fn open(store_root: &Path) -> Result { - FilesystemRecoveryStageDiscarder::open(store_root) - .map(|discarder| Self { discarder }) - .map_err(Into::into) - } - - /// Pins a committed version-two store for explicit stage completion. + /// Pins a committed store for explicit stage completion. /// /// The call performs no protocol mutation. Completion links a stage into /// an immutable pool and never changes `HEAD`. diff --git a/src/adapters/filesystem_recovery_stage_completion_open_error.rs b/src/adapters/filesystem_recovery_stage_completion_open_error.rs index f3e36382..7baa3cbc 100644 --- a/src/adapters/filesystem_recovery_stage_completion_open_error.rs +++ b/src/adapters/filesystem_recovery_stage_completion_open_error.rs @@ -4,23 +4,11 @@ use std::error::Error; use std::fmt; use std::io; -use super::{ - FilesystemRecoveryStageDiscardOpenError, RecoveryInventoryError, WriterLockAcquireError, -}; +use super::{FilesystemRecoveryStageDiscardOpenError, RecoveryInventoryError}; /// Why a pinned writer-authorized stage completer could not be opened. #[derive(Debug)] pub enum FilesystemRecoveryStageCompletionOpenError { - /// The store root did not satisfy the supported platform profile. - Platform { - /// Exact platform-admission failure. - source: io::Error, - }, - /// Exclusive writer authority could not be acquired. - WriterLock { - /// Exact writer-lock acquisition refusal. - source: WriterLockAcquireError, - }, /// The locked root capability could not be cloned for recovery inventory. CloneRoot { /// Exact root-capability clone failure. @@ -36,12 +24,6 @@ pub enum FilesystemRecoveryStageCompletionOpenError { impl From for FilesystemRecoveryStageCompletionOpenError { fn from(source: FilesystemRecoveryStageDiscardOpenError) -> Self { match source { - FilesystemRecoveryStageDiscardOpenError::Platform { source } => { - Self::Platform { source } - } - FilesystemRecoveryStageDiscardOpenError::WriterLock { source } => { - Self::WriterLock { source } - } FilesystemRecoveryStageDiscardOpenError::CloneRoot { source } => { Self::CloneRoot { source } } @@ -55,16 +37,6 @@ impl From for FilesystemRecoveryStageCo impl fmt::Display for FilesystemRecoveryStageCompletionOpenError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Platform { source } => { - write!( - formatter, - "recovery completion platform was refused: {source}" - ) - } - Self::WriterLock { source } => write!( - formatter, - "recovery completion writer lock was refused: {source}" - ), Self::CloneRoot { source } => { write!(formatter, "locked recovery root clone failed: {source}") } @@ -79,8 +51,7 @@ impl fmt::Display for FilesystemRecoveryStageCompletionOpenError { impl Error for FilesystemRecoveryStageCompletionOpenError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::Platform { source } | Self::CloneRoot { source } => Some(source), - Self::WriterLock { source } => Some(source), + Self::CloneRoot { source } => Some(source), Self::Namespace { source } => Some(source), } } diff --git a/src/adapters/filesystem_recovery_stage_discard_open_error.rs b/src/adapters/filesystem_recovery_stage_discard_open_error.rs index 69307c4d..ffffc0ad 100644 --- a/src/adapters/filesystem_recovery_stage_discard_open_error.rs +++ b/src/adapters/filesystem_recovery_stage_discard_open_error.rs @@ -4,21 +4,11 @@ use std::error::Error; use std::fmt; use std::io; -use super::{RecoveryInventoryError, WriterLockAcquireError}; +use super::RecoveryInventoryError; /// Why a pinned writer-authorized stage discarder could not be opened. #[derive(Debug)] pub enum FilesystemRecoveryStageDiscardOpenError { - /// The store root did not satisfy the supported platform profile. - Platform { - /// Exact platform-admission failure. - source: io::Error, - }, - /// Exclusive writer authority could not be acquired. - WriterLock { - /// Exact writer-lock acquisition refusal. - source: WriterLockAcquireError, - }, /// The locked root capability could not be cloned for recovery inventory. CloneRoot { /// Exact root-capability clone failure. @@ -34,15 +24,6 @@ pub enum FilesystemRecoveryStageDiscardOpenError { impl fmt::Display for FilesystemRecoveryStageDiscardOpenError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Platform { source } => { - write!(formatter, "recovery discard platform was refused: {source}") - } - Self::WriterLock { source } => { - write!( - formatter, - "recovery discard writer lock was refused: {source}" - ) - } Self::CloneRoot { source } => { write!(formatter, "locked recovery root clone failed: {source}") } @@ -59,8 +40,7 @@ impl fmt::Display for FilesystemRecoveryStageDiscardOpenError { impl Error for FilesystemRecoveryStageDiscardOpenError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { - Self::Platform { source } | Self::CloneRoot { source } => Some(source), - Self::WriterLock { source } => Some(source), + Self::CloneRoot { source } => Some(source), Self::Namespace { source } => Some(source), } } diff --git a/src/adapters/filesystem_recovery_stage_discarder.rs b/src/adapters/filesystem_recovery_stage_discarder.rs index e50249c9..7366872b 100644 --- a/src/adapters/filesystem_recovery_stage_discarder.rs +++ b/src/adapters/filesystem_recovery_stage_discarder.rs @@ -1,24 +1,17 @@ //! This module owns pinned writer authority for filesystem stage discard. -use std::path::Path; - -#[cfg(any(test, feature = "repository-tasks"))] -use cap_std::ambient_authority; -use cap_std::fs::Dir; - use super::{ FilesystemRecoveryInventoryReader, FilesystemRecoveryStageDiscardOpenError, - FilesystemVersionTwoRecoveryAdmission, FilesystemWriterLock, filesystem_platform_profile, + FilesystemVersionTwoRecoveryAdmission, FilesystemWriterLock, }; -#[cfg(any(test, feature = "repository-tasks"))] -use super::{RecoveryInventoryError, RecoveryInventoryOperation, RecoveryNamespace}; /// Writer-authorized pinned filesystem adapter for exact stage discard. /// -/// Opening proves the supported platform, pins and exclusively locks the store -/// root and `writer.lock`, then pins all three protocol child directories -/// without following links. The synchronous adapter may block on filesystem -/// I/O and retains writer authority until dropped. +/// Opening consumes recovery admission, which proved the supported platform, +/// the exact namespace, and the `FORMAT` marker under the exclusive writer +/// lock; it then pins all three protocol child directories without following +/// links. The synchronous adapter may block on filesystem I/O and retains +/// writer authority until dropped. #[must_use] pub struct FilesystemRecoveryStageDiscarder { pub(super) inventory: FilesystemRecoveryInventoryReader, @@ -26,61 +19,12 @@ pub struct FilesystemRecoveryStageDiscarder { } impl FilesystemRecoveryStageDiscarder { - /// Opens an initialized supported store for explicit stage discard. - /// - /// The call performs no protocol mutation. - /// - /// # Errors - /// - /// Returns [`FilesystemRecoveryStageDiscardOpenError`] on platform, - /// writer-authority, root-clone, or namespace admission failure. - pub fn open(store_root: &Path) -> Result { - let root = filesystem_platform_profile::open(store_root) - .map_err(|source| FilesystemRecoveryStageDiscardOpenError::Platform { source })?; - Self::from_root(root) - } - - #[cfg(test)] - pub(super) fn open_unchecked_for_tests( - store_root: &Path, - ) -> Result { - Self::open_unchecked(store_root) - } - - /// Opens repository crash-test storage without the production platform - /// profile probe. - /// - /// # Errors - /// - /// Returns the exact root, writer-lock, or namespace admission failure. - #[cfg(feature = "repository-tasks")] - #[doc(hidden)] - pub fn open_unchecked_for_repository_tasks( - store_root: &Path, - ) -> Result { - Self::open_unchecked(store_root) - } - - #[cfg(any(test, feature = "repository-tasks"))] - fn open_unchecked(store_root: &Path) -> Result { - let root = Dir::open_ambient_dir(store_root, ambient_authority()).map_err(|source| { - FilesystemRecoveryStageDiscardOpenError::Namespace { - source: RecoveryInventoryError::io( - RecoveryNamespace::Root, - RecoveryInventoryOperation::OpenNamespace, - source, - ), - } - })?; - Self::from_root(root) - } - - /// Pins a committed version-two store for explicit stage discard. + /// Pins a committed store for explicit stage discard. /// /// The admission already holds writer authority over a root that passed - /// version-two recovery admission; this call pins the catalog-protocol - /// directories under it and performs no protocol mutation. The retained - /// retention capabilities are released: discard never changes `HEAD`. + /// recovery admission; this call pins the catalog-protocol directories + /// under it and performs no protocol mutation. The retained retention + /// capabilities are released: discard never changes `HEAD`. /// /// # Errors /// @@ -106,18 +50,4 @@ impl FilesystemRecoveryStageDiscarder { _authority: authority, }) } - - fn from_root(root: Dir) -> Result { - let authority = FilesystemWriterLock::try_acquire_in(root) - .map_err(|source| FilesystemRecoveryStageDiscardOpenError::WriterLock { source })?; - let inventory_root = authority - .clone_directory() - .map_err(|source| FilesystemRecoveryStageDiscardOpenError::CloneRoot { source })?; - let inventory = FilesystemRecoveryInventoryReader::from_root(inventory_root) - .map_err(|source| FilesystemRecoveryStageDiscardOpenError::Namespace { source })?; - Ok(Self { - inventory, - _authority: authority, - }) - } } diff --git a/src/adapters/filesystem_segment_stage.rs b/src/adapters/filesystem_segment_stage.rs index afeea1f0..d55a5c41 100644 --- a/src/adapters/filesystem_segment_stage.rs +++ b/src/adapters/filesystem_segment_stage.rs @@ -5,7 +5,7 @@ use std::io::{self, Write}; use cap_std::fs::File; use super::filesystem_catalog_artifact; -use super::filesystem_catalog_publisher::CURRENT_SEGMENT; +use super::filesystem_catalog_stages::CURRENT_SEGMENT; use super::filesystem_catalog_stages::FilesystemCatalogStages; use super::filesystem_publisher_authority::FilesystemPublisherAuthority; use super::{SegmentStage, SegmentStageCreateError}; diff --git a/src/adapters/filesystem_store_initializer.rs b/src/adapters/filesystem_store_initializer.rs deleted file mode 100644 index e122eeb2..00000000 --- a/src/adapters/filesystem_store_initializer.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! This module owns production filesystem platform admission and initialization. - -use std::path::Path; - -#[cfg(test)] -use cap_std::ambient_authority; -#[cfg(test)] -use cap_std::fs::Dir; - -use super::filesystem_initialization_storage::FilesystemInitializationStorage; -use super::{ - FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, FilesystemWriterLock, - StoreInitializationError, StoreInitializationPhase, filesystem_initialization_namespace, - filesystem_platform_profile, initialize_store, -}; - -impl FilesystemPlatformAdmission { - /// Initializes or resumes one admitted filesystem store root. - /// - /// The production adapter currently admits only a writable, - /// case-sensitive Linux ext4 root opened without symbolic links. It - /// validates an empty or partial canonical namespace before mutation, - /// retains the exclusive writer lock, and returns only after synchronizing - /// the complete root namespace. The call is synchronous, allocates no - /// content-sized memory, and may block on filesystem I/O. - /// - /// # Errors - /// - /// Returns [`StoreInitializationError::Io`] at the exact failed phase. - /// Unsupported, read-only, aliased, unknown, or ambiguous platform state is - /// refused before protocol directory creation. - pub fn initialize(store_root: &Path) -> Result { - let storage = FilesystemInitializationStorage::admit(store_root).map_err(|source| { - StoreInitializationError::io(StoreInitializationPhase::AdmitPlatform, source) - })?; - initialize_storage(storage) - } - - /// Reacquires writer authority over one completely published store. - /// - /// The call mutates no protocol state. It admits the production platform, - /// acquires the existing writer lock, and requires the exact published root - /// namespace: `writer.lock`, `staging`, `segments`, `catalogs`, and a - /// regular `HEAD`. Publication and restart adapters perform content-level - /// validation under the returned authority. The synchronous call may block - /// on filesystem I/O. - /// - /// # Errors - /// - /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, - /// writer-lock, or namespace boundary and preserved source. - pub fn reopen(store_root: &Path) -> Result { - let root = filesystem_platform_profile::open(store_root) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; - reopen_root(root) - } - - #[cfg(test)] - pub(super) fn initialize_unchecked_for_tests( - store_root: &Path, - ) -> Result { - let storage = FilesystemInitializationStorage::admit_unchecked_for_tests(store_root) - .map_err(|source| { - StoreInitializationError::io(StoreInitializationPhase::AdmitPlatform, source) - })?; - initialize_storage(storage) - } - - #[cfg(test)] - pub(super) fn reopen_unchecked_for_tests( - store_root: &Path, - ) -> Result { - let root = Dir::open_ambient_dir(store_root, ambient_authority()) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; - reopen_root(root) - } -} - -fn initialize_storage( - mut storage: FilesystemInitializationStorage, -) -> Result { - let _receipt = initialize_store(&mut storage)?; - let lock = storage.into_lock().map_err(|source| { - StoreInitializationError::io(StoreInitializationPhase::OpenAndLockWriterFile, source) - })?; - Ok(FilesystemPlatformAdmission::initialized(lock)) -} - -fn reopen_root( - root: cap_std::fs::Dir, -) -> Result { - admit_reopened(root, filesystem_initialization_namespace::admit_published) -} - -fn admit_reopened( - root: cap_std::fs::Dir, - admit_namespace: fn(&cap_std::fs::Dir) -> std::io::Result<()>, -) -> Result { - let lock = FilesystemWriterLock::try_acquire_in(root) - .map_err(|source| FilesystemPlatformAdmissionError::WriterLock { source })?; - let directory = lock - .clone_directory() - .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; - admit_namespace(&directory) - .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; - Ok(FilesystemPlatformAdmission::initialized(lock)) -} diff --git a/src/adapters/filesystem_store_initializer_tests.rs b/src/adapters/filesystem_store_initializer_tests.rs deleted file mode 100644 index 96847878..00000000 --- a/src/adapters/filesystem_store_initializer_tests.rs +++ /dev/null @@ -1,159 +0,0 @@ -//! Concrete crash-safe filesystem initialization laws. - -use std::error::Error; -use std::fs; - -use super::filesystem_test_sandbox::TestDirectory; -use super::{ - FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, StoreInitializationError, - StoreInitializationPhase, -}; - -const LOCK_NAME: &str = "writer.lock"; -const STAGING_NAME: &str = "staging"; -const SEGMENTS_NAME: &str = "segments"; -const CATALOGS_NAME: &str = "catalogs"; - -#[test] -fn empty_namespace_is_admitted_only_with_the_complete_root_shape() -> Result<(), Box> { - let sandbox = TestDirectory::create("store-initialization-empty")?; - - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - - assert!(sandbox.path().join(LOCK_NAME).is_file()); - assert!(sandbox.path().join(STAGING_NAME).is_dir()); - assert!(sandbox.path().join(SEGMENTS_NAME).is_dir()); - assert!(sandbox.path().join(CATALOGS_NAME).is_dir()); - drop(admission); - sandbox.remove()?; - Ok(()) -} - -#[cfg(target_os = "linux")] -#[test] -fn linux_initializer_synchronizes_an_opath_root() -> Result<(), Box> { - let sandbox = TestDirectory::create("store-initialization-opath-sync")?; - - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - - drop(admission); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn partial_canonical_namespace_is_completed_without_replacing_evidence() --> Result<(), Box> { - let sandbox = TestDirectory::create("store-initialization-partial")?; - let retained = b"retained lock evidence"; - fs::write(sandbox.path().join(LOCK_NAME), retained)?; - fs::create_dir(sandbox.path().join(STAGING_NAME))?; - - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - - assert_eq!(fs::read(sandbox.path().join(LOCK_NAME))?, retained); - assert!(sandbox.path().join(STAGING_NAME).is_dir()); - assert!(sandbox.path().join(SEGMENTS_NAME).is_dir()); - assert!(sandbox.path().join(CATALOGS_NAME).is_dir()); - drop(admission); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn unknown_namespace_refuses_before_writer_file_creation() -> Result<(), Box> { - let sandbox = TestDirectory::create("store-initialization-unknown")?; - fs::write(sandbox.path().join("unknown"), [])?; - - let Err(error) = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path()) - else { - return Err("initializer admitted an unknown namespace entry".into()); - }; - - assert!(matches!( - error, - StoreInitializationError::Io { - phase: StoreInitializationPhase::AdmitPlatform, - ref source, - } if source.kind() == std::io::ErrorKind::InvalidData - )); - assert!(!sandbox.path().join(LOCK_NAME).exists()); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn retained_initializer_authority_excludes_a_second_initializer() -> Result<(), Box> { - let sandbox = TestDirectory::create("store-initialization-exclusion")?; - let first = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - - let Err(error) = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path()) - else { - return Err("second initializer acquired live writer authority".into()); - }; - - assert!(matches!( - error, - StoreInitializationError::Io { - phase: StoreInitializationPhase::OpenAndLockWriterFile, - ref source, - } if matches!( - source - .get_ref() - .and_then(|nested| nested.downcast_ref::()), - Some(super::WriterLockAcquireError::Busy) - ) - )); - drop(first); - let successor = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - drop(successor); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn published_reopen_requires_a_complete_root_namespace() -> Result<(), Box> { - let sandbox = TestDirectory::create("store-reopen-incomplete")?; - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - drop(admission); - - let error = FilesystemPlatformAdmission::reopen_unchecked_for_tests(sandbox.path()) - .err() - .ok_or("published admission accepted a store without HEAD")?; - - assert!(matches!( - error, - FilesystemPlatformAdmissionError::Namespace { - ref source, - } if source.kind() == std::io::ErrorKind::NotFound - )); - assert!(!sandbox.path().join("HEAD").exists()); - sandbox.remove()?; - Ok(()) -} - -#[test] -fn published_reopen_refuses_unknown_root_evidence() -> Result<(), Box> { - let sandbox = TestDirectory::create("store-reopen-unknown")?; - let admission = FilesystemPlatformAdmission::initialize_unchecked_for_tests(sandbox.path())?; - drop(admission); - fs::write(sandbox.path().join("HEAD"), b"published head")?; - fs::write(sandbox.path().join("unknown"), b"retained evidence")?; - - let error = FilesystemPlatformAdmission::reopen_unchecked_for_tests(sandbox.path()) - .err() - .ok_or("published admission accepted an unknown root entry")?; - - assert!(matches!( - error, - FilesystemPlatformAdmissionError::Namespace { - ref source, - } if source.kind() == std::io::ErrorKind::InvalidData - )); - assert_eq!( - fs::read(sandbox.path().join("unknown"))?, - b"retained evidence" - ); - sandbox.remove()?; - Ok(()) -} diff --git a/src/adapters/filesystem_version_two_admission.rs b/src/adapters/filesystem_version_two_admission.rs index 9579c24a..f59e3dc3 100644 --- a/src/adapters/filesystem_version_two_admission.rs +++ b/src/adapters/filesystem_version_two_admission.rs @@ -16,14 +16,12 @@ use super::{ filesystem_platform_profile, }; -/// Exclusive writer authority over a committed version-two root. +/// Exclusive writer authority over a committed store root. /// -/// This type is deliberately distinct from -/// [`FilesystemPlatformAdmission`](super::FilesystemPlatformAdmission): a -/// version-one publisher cannot consume it, so version-one catalog publication -/// can never run against a version-two root and leave residue no adapter -/// admits. Fields are private so only version-two admission and -/// initialization can create values. The admitted `retention`, `roots`, and +/// A value proves the production platform, the exact committed namespace, and +/// the canonical `FORMAT` marker under the writer lock; every writer consumes +/// one, so no writer runs against a root without the marker. Fields are +/// private so only admission and initialization can create values. The admitted `retention`, `roots`, and /// `manifests` capabilities are retained, so the authority built from this /// value operates on the directories that passed admission and not on whatever /// those names resolve to later. diff --git a/src/adapters/filesystem_version_two_catalog_publisher.rs b/src/adapters/filesystem_version_two_catalog_publisher.rs index 68178b5c..079c833b 100644 --- a/src/adapters/filesystem_version_two_catalog_publisher.rs +++ b/src/adapters/filesystem_version_two_catalog_publisher.rs @@ -52,14 +52,15 @@ pub struct FilesystemVersionTwoCatalogPublisher { } impl FilesystemVersionTwoCatalogPublisher { - /// Pins the catalog and retention directories under version-two writer authority. + /// Pins the catalog and retention directories under admitted writer authority. /// - /// Only [`FilesystemVersionTwoAdmission`] is accepted, so a version-one - /// root or version-one writer authority cannot reach this publisher: + /// Only [`FilesystemVersionTwoAdmission`] is accepted, so a bare writer + /// lock, which proves neither the namespace nor the `FORMAT` marker, cannot + /// reach this publisher: /// /// ```compile_fail - /// fn open(admission: keep::FilesystemPlatformAdmission, policy: keep::CatalogRestartPolicy) { - /// let _ = keep::FilesystemVersionTwoCatalogPublisher::open(admission, policy); + /// fn open(lock: keep::FilesystemWriterLock, policy: keep::CatalogRestartPolicy) { + /// let _ = keep::FilesystemVersionTwoCatalogPublisher::open(lock, policy); /// } /// ``` /// @@ -105,7 +106,7 @@ impl FilesystemVersionTwoCatalogPublisher { /// # Errors /// /// Returns [`SegmentPublicationError::PublisherAuthority`] when `sealed` - /// was created by another publisher, including a version-one publisher. + /// was created by another publisher. /// Other variants preserve exact closed-stage to admitted-segment /// disagreements. pub fn select_segment<'selection, 'records>( diff --git a/src/adapters/filesystem_version_two_catalog_stages.rs b/src/adapters/filesystem_version_two_catalog_stages.rs index c03ac03a..2b908724 100644 --- a/src/adapters/filesystem_version_two_catalog_stages.rs +++ b/src/adapters/filesystem_version_two_catalog_stages.rs @@ -40,8 +40,8 @@ pub type CatalogPublicationObserver = /// Version-two catalog protocol transitions guarded by retained-closure admission. /// -/// Every transition is the shared version-one stage implementation, executed -/// under the version-two publisher's writer authority. Fields are private and +/// Every transition is the shared stage implementation, executed under the +/// publisher's writer authority. Fields are private and /// no constructor is public. #[must_use] pub struct FilesystemVersionTwoCatalogStages { diff --git a/src/adapters/filesystem_version_two_recovery_admission.rs b/src/adapters/filesystem_version_two_recovery_admission.rs index 582c8bef..5641b2da 100644 --- a/src/adapters/filesystem_version_two_recovery_admission.rs +++ b/src/adapters/filesystem_version_two_recovery_admission.rs @@ -12,9 +12,9 @@ use super::{ filesystem_platform_profile, }; -/// Exclusive writer authority over a committed version-two root for stage recovery. +/// Exclusive writer authority over a committed root for stage recovery. /// -/// Admission is the version-two writer admission with one difference: the +/// Admission is the writer admission with one difference: the /// root may retain the catalog `head.next` that an interrupted publication /// left. Every other law is the same: the production platform, the writer /// lock, the exact namespace with its protocol directories, and the canonical @@ -22,9 +22,8 @@ use super::{ /// are retained so recovery finalization verifies retained closures against /// the directories that passed admission. /// -/// Only the version-two recovery constructors consume this value; version-one -/// recovery keeps refusing a version-two root, and this admission refuses a -/// version-one root because the marker is required. +/// Every recovery tool that mutates a stage consumes this value, so recovery +/// never runs against a root without the marker. #[must_use] pub struct FilesystemVersionTwoRecoveryAdmission { parts: AdmittedVersionTwoParts, diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 01d8a46e..4997b37a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -82,8 +82,6 @@ mod filesystem_catalog_catalog; mod filesystem_catalog_current; mod filesystem_catalog_head; mod filesystem_catalog_publication_error; -mod filesystem_catalog_publisher; -mod filesystem_catalog_publisher_storage; #[cfg(test)] mod filesystem_catalog_publisher_tests; mod filesystem_catalog_segment; @@ -93,9 +91,7 @@ mod filesystem_catalog_storage; mod filesystem_exact_record; mod filesystem_format_marker; mod filesystem_initialization_namespace; -mod filesystem_initialization_storage; mod filesystem_namespace_refusal; -mod filesystem_platform_admission; mod filesystem_platform_admission_error; mod filesystem_platform_profile; mod filesystem_publisher_authority; @@ -137,9 +133,6 @@ mod filesystem_recovery_stage_tests; mod filesystem_segment_stage; #[cfg(test)] mod filesystem_segment_stage_tests; -mod filesystem_store_initializer; -#[cfg(test)] -mod filesystem_store_initializer_tests; #[cfg(test)] #[path = "../../tests/segment_filesystem_stage/sandbox.rs"] mod filesystem_test_sandbox; @@ -182,8 +175,6 @@ mod publication_head_decode_error_display; mod publication_head_decoder; mod publication_head_encoder; mod recovery; -#[cfg(feature = "repository-tasks")] -mod repository_initialization_storage; mod retained_catalog_publication; mod retention; mod sealed_segment; @@ -249,11 +240,6 @@ mod staged_segment; mod storage_profile_id_text; mod storage_profile_id_text_error; mod store_format; -mod store_initialization; -mod store_initialization_error; -mod store_initialization_phase; -mod store_initialization_receipt; -mod store_initialization_storage; mod sync_capable_directory; #[cfg(test)] #[path = "../../tests/support/mod.rs"] diff --git a/src/adapters/recovery.rs b/src/adapters/recovery.rs index 5bde5474..1af60de2 100644 --- a/src/adapters/recovery.rs +++ b/src/adapters/recovery.rs @@ -107,9 +107,7 @@ pub use recovery_inventory_error::RecoveryInventoryError; pub use recovery_inventory_limit::{RecoveryInventoryLimit, RecoveryInventoryLimitError}; pub use recovery_inventory_operation::RecoveryInventoryOperation; pub use recovery_inventory_storage::RecoveryInventoryStorage; -pub use recovery_name_classification::{ - classify_recovery_names, classify_version_two_recovery_names, -}; +pub use recovery_name_classification::classify_version_two_recovery_names; pub use recovery_name_classification_error::RecoveryNameClassificationError; pub use recovery_name_manifest::{RecoveryNameManifest, RecoveryNamedEntry}; pub use recovery_namespace::RecoveryNamespace; diff --git a/src/adapters/recovery/recovery_name_classification.rs b/src/adapters/recovery/recovery_name_classification.rs index f1b7c27c..2ab61cd3 100644 --- a/src/adapters/recovery/recovery_name_classification.rs +++ b/src/adapters/recovery/recovery_name_classification.rs @@ -6,28 +6,13 @@ use super::{ recovery_pool_name, }; -/// Classifies one complete version-one inventory without opening artifact bytes. -/// -/// The operation preserves inventory order and allocates at most one manifest -/// entry per inventory entry. It rejects unknown names, missing initialized -/// root entries, noncanonical pool coordinates, and simultaneous fixed stages. -/// Every version-two root entry is unknown here. -/// -/// # Errors -/// -/// Returns [`RecoveryNameClassificationError`] at the first deterministic -/// namespace, grammar, required-entry, or stage-conflict failure. -pub fn classify_recovery_names( - inventory: RecoveryInventory, -) -> Result { - classify_inventory(inventory, StoreLayout::VersionOne) -} - /// Classifies one complete committed version-two inventory without opening bytes. /// -/// The version-one laws apply unchanged, and the version-two root entries -/// `reader.lock`, `FORMAT`, `retention`, `gc`, and `recovery` are classified -/// as known roles, never as unexpected or ignored. Each of them is required; +/// The operation preserves inventory order and allocates at most one manifest +/// entry per inventory entry. It rejects unknown names, missing root entries, +/// noncanonical pool coordinates, and simultaneous fixed stages. The root +/// entries `reader.lock`, `FORMAT`, `retention`, `gc`, and `recovery` are +/// classified as known roles, never as unexpected or ignored. Each is required; /// catalog `HEAD` is not, because the first publication of an initialized /// store can be interrupted before it exists. `FORMAT.next` is unknown: a /// committed root never holds it. @@ -40,27 +25,13 @@ pub fn classify_recovery_names( /// namespace, grammar, required-entry, or stage-conflict failure. pub fn classify_version_two_recovery_names( inventory: RecoveryInventory, -) -> Result { - classify_inventory(inventory, StoreLayout::VersionTwo) -} - -/// The root layout one classification admits. -#[derive(Clone, Copy)] -enum StoreLayout { - VersionOne, - VersionTwo, -} - -fn classify_inventory( - inventory: RecoveryInventory, - layout: StoreLayout, ) -> Result { let mut required = RequiredEntries::default(); let mut first_stage = None; let mut named = Vec::with_capacity(inventory.entries().len()); for entry in inventory.into_entries() { let (namespace, name) = entry.into_parts(); - let role = match classify(namespace, &name, layout) { + let role = match classify(namespace, &name) { Ok(role) => role, Err(NameFailure::Unexpected) => { return Err(RecoveryNameClassificationError::Unexpected { namespace, name }); @@ -85,7 +56,7 @@ fn classify_inventory( } named.push(RecoveryNamedEntry::new(namespace, name, role)); } - if let Some(required) = required.first_missing(layout) { + if let Some(required) = required.first_missing() { return Err(RecoveryNameClassificationError::Missing { required }); } Ok(RecoveryNameManifest::new(named)) @@ -94,10 +65,9 @@ fn classify_inventory( fn classify( namespace: RecoveryNamespace, name: &RecoveryEntryName, - layout: StoreLayout, ) -> Result { match namespace { - RecoveryNamespace::Root => classify_root(name, layout), + RecoveryNamespace::Root => classify_root(name), RecoveryNamespace::Staging => classify_staging(name), RecoveryNamespace::Segments => recovery_pool_name::segment(name) .map(|digest| RecoveryEntryRole::ImmutableSegment { digest }) @@ -108,10 +78,7 @@ fn classify( } } -fn classify_root( - name: &RecoveryEntryName, - layout: StoreLayout, -) -> Result { +fn classify_root(name: &RecoveryEntryName) -> Result { match name.as_bytes() { b"writer.lock" => Ok(RecoveryEntryRole::WriterLock), b"staging" => Ok(RecoveryEntryRole::StagingDirectory), @@ -119,15 +86,6 @@ fn classify_root( b"catalogs" => Ok(RecoveryEntryRole::CatalogPoolDirectory), b"HEAD" => Ok(RecoveryEntryRole::CurrentHead), b"head.next" => Ok(RecoveryEntryRole::NextHeadStage), - other => match layout { - StoreLayout::VersionOne => Err(NameFailure::Unexpected), - StoreLayout::VersionTwo => classify_version_two_root(other), - }, - } -} - -fn classify_version_two_root(name: &[u8]) -> Result { - match name { b"reader.lock" => Ok(RecoveryEntryRole::ReaderFence), b"FORMAT" => Ok(RecoveryEntryRole::FormatMarker), b"retention" => Ok(RecoveryEntryRole::RetentionDirectory), @@ -150,7 +108,7 @@ enum NameFailure { Pool(super::RecoveryPoolNameError), } -/// Required root entries in report order; version one requires the prefix. +/// Required root entries in report order. const REQUIRED: [RecoveryRequiredEntry; 9] = [ RecoveryRequiredEntry::WriterLock, RecoveryRequiredEntry::StagingDirectory, @@ -162,7 +120,6 @@ const REQUIRED: [RecoveryRequiredEntry; 9] = [ RecoveryRequiredEntry::GcDirectory, RecoveryRequiredEntry::RecoveryDirectory, ]; -const VERSION_ONE_REQUIRED: usize = 4; #[derive(Default)] struct RequiredEntries { @@ -181,15 +138,10 @@ impl RequiredEntries { } } - fn first_missing(&self, layout: StoreLayout) -> Option { - let count = match layout { - StoreLayout::VersionOne => VERSION_ONE_REQUIRED, - StoreLayout::VersionTwo => REQUIRED.len(), - }; + fn first_missing(&self) -> Option { REQUIRED .into_iter() .zip(self.observed) - .take(count) .find_map(|(entry, seen)| (!seen).then_some(entry)) } } diff --git a/src/adapters/repository_initialization_storage.rs b/src/adapters/repository_initialization_storage.rs deleted file mode 100644 index fc822533..00000000 --- a/src/adapters/repository_initialization_storage.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! This module owns repository-only access to production initialization storage. - -use std::io; -use std::path::Path; - -use super::filesystem_initialization_storage::FilesystemInitializationStorage; -use super::{FilesystemWriterLock, StoreInitializationStorage}; - -/// Repository-tooling adapter over the production filesystem initialization -/// implementation. -/// -/// This adapter bypasses the Linux ext4 profile probe so the process-death -/// harness can run on development hosts. It does not bypass namespace, -/// synchronization, writer-lock, or initialization protocol operations. -#[doc(hidden)] -pub struct RepositoryInitializationStorage { - inner: FilesystemInitializationStorage, -} - -impl RepositoryInitializationStorage { - /// Opens `store_root` without applying the production platform profile. - /// - /// # Errors - /// - /// Returns the exact ambient root-open failure. - pub fn admit_unchecked(store_root: &Path) -> io::Result { - FilesystemInitializationStorage::admit_unchecked_for_repository_tasks(store_root) - .map(|inner| Self { inner }) - } - - /// Consumes completed initialization storage and returns retained writer - /// authority. - /// - /// # Errors - /// - /// Returns [`io::ErrorKind::Other`] when initialization did not acquire - /// writer authority. - pub fn into_writer_lock(self) -> io::Result { - self.inner.into_lock() - } -} - -impl StoreInitializationStorage for RepositoryInitializationStorage { - fn admit_platform(&mut self) -> io::Result<()> { - self.inner.admit_platform() - } - - fn open_and_lock_writer_file(&mut self) -> io::Result<()> { - self.inner.open_and_lock_writer_file() - } - - fn admit_staging_directory(&mut self) -> io::Result<()> { - self.inner.admit_staging_directory() - } - - fn admit_segment_pool_directory(&mut self) -> io::Result<()> { - self.inner.admit_segment_pool_directory() - } - - fn admit_catalog_pool_directory(&mut self) -> io::Result<()> { - self.inner.admit_catalog_pool_directory() - } - - fn synchronize_root(&mut self) -> io::Result<()> { - self.inner.synchronize_root() - } -} diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index 2f930cb1..b9e76660 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -17,8 +17,6 @@ mod closure_verifier; mod durable_read_law_tests; #[cfg(test)] mod durable_view_law_tests; -#[cfg(test)] -mod filesystem_recovery_admission_tests; mod filesystem_retained_evidence; #[cfg(test)] mod filesystem_retention_anchor_order_prefix_tests; diff --git a/src/adapters/retention/filesystem_recovery_admission_tests.rs b/src/adapters/retention/filesystem_recovery_admission_tests.rs deleted file mode 100644 index 8de6fcec..00000000 --- a/src/adapters/retention/filesystem_recovery_admission_tests.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! Version-one recovery adapters refuse a version-two root. - -use std::error::Error; - -use super::filesystem_retention_test_fixture::initialized_store; -use crate::adapters::{ - FilesystemRecoveryInventoryReader, FilesystemRecoveryStageDiscardOpenError, - FilesystemRecoveryStageDiscarder, RecoveryInventoryError, RecoveryInventoryOperation, - RecoveryNamespace, -}; - -#[test] -fn recovery_inventory_reader_refuses_a_version_two_root() -> Result<(), Box> { - let sandbox = initialized_store("recovery-admission-inventory-version-two")?; - - let error = FilesystemRecoveryInventoryReader::open_unchecked_for_tests(sandbox.path()) - .err() - .ok_or("version-one recovery inventory opened a version-two root")?; - - assert!(matches!( - error, - RecoveryInventoryError::Io { - namespace: RecoveryNamespace::Root, - operation: RecoveryInventoryOperation::OpenNamespace, - .. - } - )); - Ok(()) -} - -#[test] -fn recovery_stage_discarder_refuses_a_version_two_root() -> Result<(), Box> { - let sandbox = initialized_store("recovery-admission-discarder-version-two")?; - - let error = FilesystemRecoveryStageDiscarder::open_unchecked_for_tests(sandbox.path()) - .err() - .ok_or("version-one recovery discarder opened a version-two root")?; - - assert!(matches!( - error, - FilesystemRecoveryStageDiscardOpenError::Namespace { .. } - )); - Ok(()) -} diff --git a/src/adapters/retention/filesystem_retention_authority.rs b/src/adapters/retention/filesystem_retention_authority.rs index 36d61efa..c10fc379 100644 --- a/src/adapters/retention/filesystem_retention_authority.rs +++ b/src/adapters/retention/filesystem_retention_authority.rs @@ -40,15 +40,15 @@ pub struct FilesystemRetentionPublicationAuthority { } impl FilesystemRetentionPublicationAuthority { - /// Pins one admitted version-two root for retention publication. + /// Pins one admitted root for retention publication. /// - /// Only [`FilesystemVersionTwoAdmission`] is accepted, so version-one - /// writer authority can never reach retention publication; the type - /// system refuses it: + /// Only [`FilesystemVersionTwoAdmission`] is accepted, so a bare writer + /// lock, which proves neither the namespace nor the `FORMAT` marker, can + /// never reach retention publication; the type system refuses it: /// /// ```compile_fail - /// fn publish(admission: keep::FilesystemPlatformAdmission) { - /// let _ = keep::FilesystemRetentionPublicationAuthority::open(admission); + /// fn publish(lock: keep::FilesystemWriterLock) { + /// let _ = keep::FilesystemRetentionPublicationAuthority::open(lock); /// } /// ``` /// diff --git a/src/adapters/segment_publication.rs b/src/adapters/segment_publication.rs index 31987541..3a64a9e6 100644 --- a/src/adapters/segment_publication.rs +++ b/src/adapters/segment_publication.rs @@ -29,7 +29,7 @@ impl<'selection, 'records> SegmentPublication<'selection, 'records> { /// Binds one storage-agnostic closed stage to its exact admitted bytes. /// /// This selection carries no filesystem-publisher authority. Use - /// [`crate::FilesystemCatalogPublisher::select_segment`] for filesystem + /// [`crate::FilesystemVersionTwoCatalogPublisher::select_segment`] for filesystem /// publication. /// /// # Errors diff --git a/src/adapters/store_initialization.rs b/src/adapters/store_initialization.rs deleted file mode 100644 index 7a324843..00000000 --- a/src/adapters/store_initialization.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! This module owns ordered segment-store namespace initialization. - -use std::io; - -use super::{ - store_initialization_error::StoreInitializationError, - store_initialization_phase::StoreInitializationPhase, - store_initialization_receipt::StoreInitializationReceipt, - store_initialization_storage::StoreInitializationStorage, -}; - -/// Executes the ordered segment-store initialization protocol. -/// -/// The storage port must admit its platform before any namespace mutation, -/// retain writer authority after opening the lock file, and make every -/// directory operation idempotent by admitting an exact existing directory. -/// The function returns a receipt only after root synchronization succeeds. -/// It is synchronous, performs no internal heap allocation, invokes each port -/// operation at most once, and may block only inside the supplied storage -/// operations. Failure performs no implicit cleanup. -/// -/// # Errors -/// -/// Returns [`StoreInitializationError::Io`] at the first failed phase and does -/// not execute any later phase. -pub fn initialize_store( - storage: &mut impl StoreInitializationStorage, -) -> Result { - phase( - StoreInitializationPhase::AdmitPlatform, - storage.admit_platform(), - )?; - phase( - StoreInitializationPhase::OpenAndLockWriterFile, - storage.open_and_lock_writer_file(), - )?; - admit_directories(storage)?; - phase( - StoreInitializationPhase::SynchronizeRoot, - storage.synchronize_root(), - )?; - Ok(StoreInitializationReceipt::new()) -} - -fn admit_directories( - storage: &mut impl StoreInitializationStorage, -) -> Result<(), StoreInitializationError> { - phase( - StoreInitializationPhase::AdmitStagingDirectory, - storage.admit_staging_directory(), - )?; - phase( - StoreInitializationPhase::AdmitSegmentPoolDirectory, - storage.admit_segment_pool_directory(), - )?; - phase( - StoreInitializationPhase::AdmitCatalogPoolDirectory, - storage.admit_catalog_pool_directory(), - )?; - Ok(()) -} - -fn phase( - phase: StoreInitializationPhase, - result: io::Result<()>, -) -> Result<(), StoreInitializationError> { - result.map_err(|source| StoreInitializationError::Io { phase, source }) -} diff --git a/src/adapters/store_initialization_error.rs b/src/adapters/store_initialization_error.rs deleted file mode 100644 index a2d357db..00000000 --- a/src/adapters/store_initialization_error.rs +++ /dev/null @@ -1,43 +0,0 @@ -//! This module owns typed segment-store initialization failures. - -use std::error::Error; -use std::fmt; -use std::io; - -use super::store_initialization_phase::StoreInitializationPhase; - -/// Failure while executing ordered store initialization. -#[derive(Debug)] -pub enum StoreInitializationError { - /// One initialization phase returned an I/O failure. - Io { - /// Exact failed initialization phase. - phase: StoreInitializationPhase, - /// Underlying storage failure. - source: io::Error, - }, -} - -impl StoreInitializationError { - pub(super) const fn io(phase: StoreInitializationPhase, source: io::Error) -> Self { - Self::Io { phase, source } - } -} - -impl fmt::Display for StoreInitializationError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Io { phase, .. } => { - write!(formatter, "store initialization failed during {phase}") - } - } - } -} - -impl Error for StoreInitializationError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Io { source, .. } => Some(source), - } - } -} diff --git a/src/adapters/store_initialization_phase.rs b/src/adapters/store_initialization_phase.rs deleted file mode 100644 index 90900909..00000000 --- a/src/adapters/store_initialization_phase.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! This module owns exact segment-store initialization phases. - -use std::fmt; - -/// Exact operation attempted during store initialization. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum StoreInitializationPhase { - /// Prove the platform contract before namespace mutation. - AdmitPlatform, - /// Create or reopen `writer.lock`, then acquire writer authority. - OpenAndLockWriterFile, - /// Create or verify the `staging` directory. - AdmitStagingDirectory, - /// Create or verify the `segments` directory. - AdmitSegmentPoolDirectory, - /// Create or verify the `catalogs` directory. - AdmitCatalogPoolDirectory, - /// Synchronize the store root after all canonical names exist. - SynchronizeRoot, -} - -impl fmt::Display for StoreInitializationPhase { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(match self { - Self::AdmitPlatform => "platform admission", - Self::OpenAndLockWriterFile => "writer-lock admission", - Self::AdmitStagingDirectory => "staging-directory admission", - Self::AdmitSegmentPoolDirectory => "segment-pool admission", - Self::AdmitCatalogPoolDirectory => "catalog-pool admission", - Self::SynchronizeRoot => "root synchronization", - }) - } -} diff --git a/src/adapters/store_initialization_receipt.rs b/src/adapters/store_initialization_receipt.rs deleted file mode 100644 index 569847c3..00000000 --- a/src/adapters/store_initialization_receipt.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! This module owns proof of complete store-root initialization. - -/// Proof that ordered store initialization reached root synchronization. -/// -/// Private fields prevent callers from manufacturing a successful receipt. -#[derive(Debug)] -#[must_use] -pub struct StoreInitializationReceipt { - _private: (), -} - -impl StoreInitializationReceipt { - pub(super) const fn new() -> Self { - Self { _private: () } - } -} diff --git a/src/adapters/store_initialization_storage.rs b/src/adapters/store_initialization_storage.rs deleted file mode 100644 index 5ab21b1b..00000000 --- a/src/adapters/store_initialization_storage.rs +++ /dev/null @@ -1,52 +0,0 @@ -//! This module owns the storage port for store initialization. - -use std::io; - -/// Semantic storage operations required by ordered store initialization. -/// -/// Implementations own platform proof, namespace technology, writer -/// authority, and idempotent create-or-admit behavior. The orchestration layer -/// owns only ordering and exact failure attribution. -pub trait StoreInitializationStorage { - /// Proves the complete platform contract without mutating the namespace. - /// - /// # Errors - /// - /// Returns an I/O error when the platform cannot be proved admissible. - fn admit_platform(&mut self) -> io::Result<()>; - - /// Creates or reopens the writer file and retains its exclusive lock. - /// - /// # Errors - /// - /// Returns an I/O error when the writer file or lock cannot be admitted. - fn open_and_lock_writer_file(&mut self) -> io::Result<()>; - - /// Creates or verifies the exact `staging` directory. - /// - /// # Errors - /// - /// Returns an I/O error when the staging directory cannot be admitted. - fn admit_staging_directory(&mut self) -> io::Result<()>; - - /// Creates or verifies the exact `segments` directory. - /// - /// # Errors - /// - /// Returns an I/O error when the segment-pool directory cannot be admitted. - fn admit_segment_pool_directory(&mut self) -> io::Result<()>; - - /// Creates or verifies the exact `catalogs` directory. - /// - /// # Errors - /// - /// Returns an I/O error when the catalog-pool directory cannot be admitted. - fn admit_catalog_pool_directory(&mut self) -> io::Result<()>; - - /// Synchronizes the store root after all canonical names exist. - /// - /// # Errors - /// - /// Returns an I/O error when root synchronization fails. - fn synchronize_root(&mut self) -> io::Result<()>; -} diff --git a/src/adapters/verification_failure_class.rs b/src/adapters/verification_failure_class.rs index 4e27e3be..3ede3a18 100644 --- a/src/adapters/verification_failure_class.rs +++ b/src/adapters/verification_failure_class.rs @@ -1,8 +1,8 @@ //! This module owns resource-versus-content classification at durable ingress. use crate::{ - CatalogAdmissionError, CatalogRestartError, LayoutDecodeError, SegmentReadError, - SegmentRecordAdmissionError, + CatalogAdmissionError, CatalogRestartError, FilesystemPlatformAdmissionError, + LayoutDecodeError, SegmentReadError, SegmentRecordAdmissionError, }; #[derive(Clone, Copy)] @@ -88,6 +88,7 @@ pub(super) fn catalog_class(error: &CatalogRestartError) -> FailureClass { | CatalogRestartError::SegmentIndexAllocation { .. } | CatalogRestartError::RetainedSegmentBytes { .. } | CatalogRestartError::RetainedSegmentByteArithmetic { .. } => FailureClass::Operational, + CatalogRestartError::Admission { source } => store_admission_class(source), CatalogRestartError::Segment { source, .. } => segment_class(source), CatalogRestartError::CatalogAdmission { source } => admission_class(source), CatalogRestartError::NotRegular { .. } @@ -115,3 +116,17 @@ fn admission_class(error: &CatalogAdmissionError) -> FailureClass { | CatalogAdmissionError::RecordChecksumMismatch { .. } => FailureClass::Corrupt, } } + +/// A namespace or marker refusal is demonstrated corruption; anything else +/// that stopped admission is an operational failure. +fn store_admission_class(error: &FilesystemPlatformAdmissionError) -> FailureClass { + match error { + FilesystemPlatformAdmissionError::Namespace { source } + | FilesystemPlatformAdmissionError::FormatMarker { source } + if source.kind() == std::io::ErrorKind::InvalidData => + { + FailureClass::Corrupt + } + _ => FailureClass::Operational, + } +} diff --git a/src/lib.rs b/src/lib.rs index 1b80d3a5..54bf8f02 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -90,8 +90,7 @@ pub use adapters::{ CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, - FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemPlatformAdmission, - FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, + FilesystemCatalogSnapshot, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, @@ -132,16 +131,14 @@ pub use adapters::{ SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, - StoreInitializationError, StoreInitializationPhase, StoreInitializationReceipt, - StoreInitializationStorage, VersionTwoInitialization, VersionTwoInitializationBoundary, - VersionTwoInitializationDirectory, VersionTwoInitializationError, - VersionTwoInitializationPhase, VersionTwoInitializationPrefix, VersionTwoInitializationRefusal, - VersionTwoRecordRefusal, WriterLockAcquireError, WriterLockAcquirePhase, - admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, - classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, - classify_version_two_recovery_names, execute_recovery_next_head_finalization, - execute_recovery_segment_resume, execute_recovery_stage_completion, - execute_recovery_stage_discard, fingerprint_recovery_stage, initialize_store, + VersionTwoInitialization, VersionTwoInitializationBoundary, VersionTwoInitializationDirectory, + VersionTwoInitializationError, VersionTwoInitializationPhase, VersionTwoInitializationPrefix, + VersionTwoInitializationRefusal, VersionTwoRecordRefusal, WriterLockAcquireError, + WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, + classify_recovery_catalog_stage, classify_recovery_next_head_stage, + classify_recovery_segment_stage, classify_version_two_recovery_names, + execute_recovery_next_head_finalization, execute_recovery_segment_resume, + execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, plan_recovery_next_head_finalization, plan_recovery_segment_resume, plan_recovery_stage_completion, plan_recovery_stage_discard, publish_catalog_generation, read_recovery_inventory, @@ -179,10 +176,7 @@ pub use adapters::{ pub use adapters::{CatalogPublicationBoundary, CatalogPublicationObserver}; #[cfg(feature = "repository-tasks")] #[doc(hidden)] -pub use adapters::{ - ObservedSegmentStage, RepositoryInitializationStorage, SegmentStageDurabilityEvent, - SegmentStageObserver, -}; +pub use adapters::{ObservedSegmentStage, SegmentStageDurabilityEvent, SegmentStageObserver}; pub use adapters::{ PreservedRetentionManifest, RetainedCatalogPublicationError, RetainedCatalogPublicationReceipt, RetainedCatalogPublicationStorage, RetainedClosureAdmission, RetentionEvidencePhase, diff --git a/tests/catalog_platform_admission.rs b/tests/catalog_platform_admission.rs deleted file mode 100644 index 3e6b35fd..00000000 --- a/tests/catalog_platform_admission.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Public platform-admission laws; medium Linux filesystem evidence for #150. -//! Oracle: KEEP-CATALOG-007 requires platform admission independently of lock ownership. - -#![cfg(all(target_os = "linux", feature = "repository-tasks"))] - -#[path = "segment_filesystem_stage/sandbox.rs"] -pub mod admitted_directory; -#[path = "catalog_platform_admission/unsupported_directory.rs"] -pub mod unsupported_directory; - -use std::error::Error; -use std::fs; -use std::io::ErrorKind; -use std::io::Write; - -use keep::{ - CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, - FilesystemPlatformAdmission, FilesystemWriterLock, LayoutEntryLimit, SegmentReadPolicy, - SegmentRecordLimit, StoreInitializationError, StoreInitializationPhase, -}; - -#[test] -fn a_writer_lock_cannot_mint_publication_authority_on_a_refused_platform() --> Result<(), Box> { - let directory = unsupported_directory::UnsupportedDirectory::create()?; - let refusal = match FilesystemPlatformAdmission::initialize(directory.path()) { - Ok(_admitted) => return Err("probe filesystem unexpectedly admitted".into()), - Err(error) => error, - }; - assert!( - matches!(refusal, StoreInitializationError::Io { phase, ref source } - if phase == StoreInitializationPhase::AdmitPlatform - && source.kind() == ErrorKind::Unsupported) - ); - fs::write(directory.path().join("writer.lock"), [])?; - for name in ["staging", "segments", "catalogs"] { - fs::create_dir(directory.path().join(name))?; - } - let lock = FilesystemWriterLock::try_acquire(directory.path())?; - let policy = CatalogRestartPolicy::new( - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), - CatalogRestartByteLimit::new(1_048_576)?, - ); - let error = match FilesystemCatalogPublisher::open_unchecked_for_repository_tasks(lock, policy) - { - Ok(_publisher) => return Err("refused platform acquired public publisher authority".into()), - Err(error) => error, - }; - assert_eq!(error.kind(), ErrorKind::Unsupported); - directory.remove()?; - Ok(()) -} - -#[test] -fn repository_publisher_preserves_staging_on_an_admitted_platform() -> Result<(), Box> { - let directory = admitted_directory::TestDirectory::create("catalog-admitted-platform")?; - drop(FilesystemPlatformAdmission::initialize(directory.path())?); - let lock = FilesystemWriterLock::try_acquire(directory.path())?; - let policy = CatalogRestartPolicy::new( - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), - CatalogRestartByteLimit::new(1_048_576)?, - ); - let publisher = FilesystemCatalogPublisher::open_unchecked_for_repository_tasks(lock, policy)?; - let mut stage = publisher.create_segment_stage()?; - stage.write_all(b"retained stage evidence")?; - drop(stage); - drop(publisher); - assert_eq!( - fs::read(directory.path().join("staging/current.seg"))?, - b"retained stage evidence" - ); - directory.remove()?; - Ok(()) -} diff --git a/tests/single_format/initialization_laws.rs b/tests/single_format/initialization_laws.rs index 4cf2d7ee..aaf26711 100644 --- a/tests/single_format/initialization_laws.rs +++ b/tests/single_format/initialization_laws.rs @@ -1,10 +1,11 @@ //! This module owns public direct-initialization laws under the production profile. use std::fs; +use std::io::ErrorKind; use keep::{ - FilesystemNamespaceRefusal, FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, - FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, FilesystemVersionTwoAdmission, + FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError, FilesystemRetentionSnapshot, + FilesystemRetentionSnapshotError, FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, ReaderAttemptLimit, RetainedCatalogPublicationError, RetentionCurrentStateRefusal, RetentionEvidencePhase, VersionTwoInitializationError, VersionTwoInitializationPrefix, VersionTwoInitializationRefusal, @@ -12,6 +13,7 @@ use keep::{ use super::durable_sandbox::TestDirectory; use super::populated_store; +use super::unsupported_directory::UnsupportedDirectory; use super::writer_fixture::{TestResult, first_blob, witness}; #[test] @@ -38,16 +40,17 @@ fn an_initialized_store_has_no_catalog_and_admits_writers_but_not_readers() -> T } #[test] -fn version_one_admission_refuses_an_initialized_store() -> TestResult<()> { - let sandbox = TestDirectory::create("single-format-v1-refuses")?; - drop(FilesystemVersionTwoAdmission::initialize(sandbox.path())?); +fn initialization_refuses_an_unsupported_platform_before_any_mutation() -> TestResult<()> { + let directory = UnsupportedDirectory::create()?; - let refused = FilesystemPlatformAdmission::reopen(sandbox.path()); + let refused = FilesystemVersionTwoAdmission::initialize(directory.path()); - assert!(matches!( - refused, - Err(FilesystemPlatformAdmissionError::Namespace { .. }) - )); + let Err(VersionTwoInitializationError::Platform { source }) = refused else { + return Err("initialization admitted an unsupported platform".into()); + }; + assert_eq!(source.kind(), ErrorKind::Unsupported); + assert_eq!(fs::read_dir(directory.path())?.count(), 0); + directory.remove()?; Ok(()) } diff --git a/tests/single_format/portability_laws.rs b/tests/single_format/portability_laws.rs index 2841c668..ab8cb40f 100644 --- a/tests/single_format/portability_laws.rs +++ b/tests/single_format/portability_laws.rs @@ -98,7 +98,10 @@ fn a_replaced_lock_or_protocol_directory_during_an_operation_still_refuses() -> source: WriterLockAcquireError::Busy }) )); - fs::rename(root.join("retention"), root.join("retention.moved"))?; + // The displaced directory leaves the root, so every path-based reader still + // admits it; only the publisher's pinned capability can notice. + let displaced = TestDirectory::create("portable-replaced-displaced")?; + fs::rename(root.join("retention"), displaced.path().join("retention"))?; fs::create_dir_all(root.join("retention/roots"))?; fs::create_dir(root.join("retention/manifests"))?; let mut publisher = diff --git a/tests/single_format/suite.rs b/tests/single_format/suite.rs index 52b37ace..21aacba5 100644 --- a/tests/single_format/suite.rs +++ b/tests/single_format/suite.rs @@ -36,6 +36,8 @@ mod store_copy; reason = "these laws publish successors only through `publish`" )] mod successor_catalog; +#[path = "unsupported_directory.rs"] +mod unsupported_directory; #[path = "../version_two_catalog_writer/writer_fixture.rs"] #[allow( dead_code, diff --git a/tests/catalog_platform_admission/unsupported_directory.rs b/tests/single_format/unsupported_directory.rs similarity index 83% rename from tests/catalog_platform_admission/unsupported_directory.rs rename to tests/single_format/unsupported_directory.rs index 8f763170..257b3e4b 100644 --- a/tests/catalog_platform_admission/unsupported_directory.rs +++ b/tests/single_format/unsupported_directory.rs @@ -11,8 +11,10 @@ pub(super) struct UnsupportedDirectory { impl UnsupportedDirectory { pub(super) fn create() -> io::Result { - let path = - Path::new("/dev/shm").join(format!("keep-catalog-platform-{}", std::process::id())); + let path = Path::new("/dev/shm").join(format!( + "keep-single-format-platform-{}", + std::process::id() + )); fs::create_dir(&path)?; Ok(Self { path }) } diff --git a/tests/store_initialization.rs b/tests/store_initialization.rs deleted file mode 100644 index e594a954..00000000 --- a/tests/store_initialization.rs +++ /dev/null @@ -1,130 +0,0 @@ -//! Deterministic segment-store initialization laws. - -use std::error::Error; -use std::io; - -#[cfg(not(target_os = "linux"))] -use keep::FilesystemPlatformAdmission; -use keep::{ - StoreInitializationError, StoreInitializationPhase, StoreInitializationStorage, - initialize_store, -}; - -#[cfg(not(target_os = "linux"))] -#[path = "segment_filesystem_stage/sandbox.rs"] -pub mod sandbox; - -const PHASES: [StoreInitializationPhase; 6] = [ - StoreInitializationPhase::AdmitPlatform, - StoreInitializationPhase::OpenAndLockWriterFile, - StoreInitializationPhase::AdmitStagingDirectory, - StoreInitializationPhase::AdmitSegmentPoolDirectory, - StoreInitializationPhase::AdmitCatalogPoolDirectory, - StoreInitializationPhase::SynchronizeRoot, -]; - -#[test] -fn initialization_admits_platform_before_every_namespace_transition() -> Result<(), Box> -{ - let mut storage = RecordingStorage::new(None); - - let _receipt = initialize_store(&mut storage)?; - - assert_eq!(storage.attempted, PHASES); - Ok(()) -} - -#[test] -fn initialization_stops_at_and_preserves_every_exact_failure_phase() -> Result<(), Box> { - let mut expected_prefix = Vec::new(); - for phase in PHASES { - expected_prefix.push(phase); - let mut storage = RecordingStorage::new(Some(phase)); - let Err(error) = initialize_store(&mut storage) else { - return Err(format!("initialization ignored injected failure at {phase}").into()); - }; - - match error { - StoreInitializationError::Io { - phase: observed, - source, - } => { - assert_eq!(observed, phase); - assert_eq!(source.kind(), io::ErrorKind::Other); - } - } - assert_eq!(storage.attempted, expected_prefix); - } - Ok(()) -} - -#[cfg(not(target_os = "linux"))] -#[test] -fn unsupported_production_platform_refuses_before_namespace_mutation() -> Result<(), Box> -{ - let sandbox = sandbox::TestDirectory::create("store-initialization-unsupported")?; - - let Err(error) = FilesystemPlatformAdmission::initialize(sandbox.path()) else { - return Err("unsupported platform produced filesystem authority".into()); - }; - - assert!(matches!( - error, - StoreInitializationError::Io { - phase: StoreInitializationPhase::AdmitPlatform, - ref source, - } if source.kind() == io::ErrorKind::Unsupported - )); - assert!(std::fs::read_dir(sandbox.path())?.next().is_none()); - sandbox.remove()?; - Ok(()) -} - -struct RecordingStorage { - attempted: Vec, - fail_at: Option, -} - -impl RecordingStorage { - const fn new(fail_at: Option) -> Self { - Self { - attempted: Vec::new(), - fail_at, - } - } - - fn attempt(&mut self, phase: StoreInitializationPhase) -> io::Result<()> { - self.attempted.push(phase); - if self.fail_at == Some(phase) { - Err(io::Error::other("injected initialization failure")) - } else { - Ok(()) - } - } -} - -impl StoreInitializationStorage for RecordingStorage { - fn admit_platform(&mut self) -> io::Result<()> { - self.attempt(StoreInitializationPhase::AdmitPlatform) - } - - fn open_and_lock_writer_file(&mut self) -> io::Result<()> { - self.attempt(StoreInitializationPhase::OpenAndLockWriterFile) - } - - fn admit_staging_directory(&mut self) -> io::Result<()> { - self.attempt(StoreInitializationPhase::AdmitStagingDirectory) - } - - fn admit_segment_pool_directory(&mut self) -> io::Result<()> { - self.attempt(StoreInitializationPhase::AdmitSegmentPoolDirectory) - } - - fn admit_catalog_pool_directory(&mut self) -> io::Result<()> { - self.attempt(StoreInitializationPhase::AdmitCatalogPoolDirectory) - } - - fn synchronize_root(&mut self) -> io::Result<()> { - self.attempt(StoreInitializationPhase::SynchronizeRoot) - } -} diff --git a/tests/version_two_admission_contract.rs b/tests/version_two_admission_contract.rs index 29014609..a0dbc41e 100644 --- a/tests/version_two_admission_contract.rs +++ b/tests/version_two_admission_contract.rs @@ -1,34 +1,27 @@ -//! Version-two writer authority is a distinct type that version-one publishers cannot consume. +//! Writer authority over a store comes only from single-format admission. const RETENTION_AUTHORITY: &str = include_str!("../src/adapters/retention/filesystem_retention_authority.rs"); -const CATALOG_PUBLISHER: &str = include_str!("../src/adapters/filesystem_catalog_publisher.rs"); -const STORE_INITIALIZER: &str = include_str!("../src/adapters/filesystem_store_initializer.rs"); +const CATALOG_PUBLISHER: &str = + include_str!("../src/adapters/filesystem_version_two_catalog_publisher.rs"); const VERSION_TWO_ADMISSION: &str = include_str!("../src/adapters/filesystem_version_two_admission.rs"); const ADMISSION_ERROR: &str = include_str!("../src/adapters/filesystem_platform_admission_error.rs"); -/// The type-level proof is the `compile_fail` doctest on -/// `FilesystemRetentionPublicationAuthority::open`, which refuses a -/// `FilesystemPlatformAdmission` argument at compile time. These markers only -/// keep each file on its side of the boundary without pinning a signature. +/// The type-level proofs are the `compile_fail` doctests on +/// `FilesystemRetentionPublicationAuthority::open` and +/// `FilesystemVersionTwoCatalogPublisher::open`, which refuse a bare +/// `FilesystemWriterLock`. These markers only keep each writer on the +/// admission side of the boundary without pinning a signature. #[test] -fn retention_publication_consumes_only_version_two_authority() { +fn every_writer_consumes_admitted_authority() { assert!(RETENTION_AUTHORITY.contains("FilesystemVersionTwoAdmission")); - assert!(!RETENTION_AUTHORITY.contains("admission: FilesystemPlatformAdmission")); + assert!(CATALOG_PUBLISHER.contains("admission: FilesystemVersionTwoAdmission")); } #[test] -fn version_one_publishers_consume_only_version_one_authority() { - assert!(CATALOG_PUBLISHER.contains("FilesystemPlatformAdmission")); - assert!(!CATALOG_PUBLISHER.contains("FilesystemVersionTwoAdmission")); -} - -#[test] -fn version_two_reopen_produces_only_version_two_authority() { - assert!(!STORE_INITIALIZER.contains("fn reopen_version_two")); - assert!(!STORE_INITIALIZER.contains("FilesystemVersionTwoAdmission")); +fn reopen_admits_the_marker_under_the_production_profile() { assert!(VERSION_TWO_ADMISSION.contains("pub struct FilesystemVersionTwoAdmission")); assert!(VERSION_TWO_ADMISSION.contains("pub fn reopen(store_root: &Path)")); assert!(VERSION_TWO_ADMISSION.contains("pub fn initialize(")); diff --git a/tests/version_two_catalog_writer/format_marker_laws.rs b/tests/version_two_catalog_writer/format_marker_laws.rs new file mode 100644 index 00000000..8bfce719 --- /dev/null +++ b/tests/version_two_catalog_writer/format_marker_laws.rs @@ -0,0 +1,142 @@ +//! This module owns refusal of a root without the format marker at every open path. +//! +//! Removing `FORMAT` from an otherwise valid, published, retained store is the +//! sharpest form of a root that is not the single format: every other entry +//! and record still admits, so a refusal can only come from the marker law. + +use std::fs; +use std::path::Path; + +use keep::{ + CatalogRestartError, DurableStore, DurableStoreError, FilesystemCatalogSnapshot, + FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError, + FilesystemRecoveryInventoryReader, FilesystemRetentionSnapshot, + FilesystemRetentionSnapshotError, FilesystemVersionTwoAdmission, + FilesystemVersionTwoRecoveryAdmission, ReaderAttemptLimit, RecoveryEntryRole, + RecoveryInventoryError, RecoveryInventoryLimit, RecoveryNamespace, + VersionTwoInitializationError, classify_version_two_recovery_names, +}; + +use super::durable_fixture; +use super::writer_fixture::{TestResult, first_blob, witness}; + +fn limit() -> TestResult { + Ok(RecoveryInventoryLimit::new(4096)?) +} + +#[test] +fn every_root_entry_of_a_published_store_is_a_known_role() -> TestResult<()> { + let sandbox = durable_fixture::build("format-marker-roles", &[&first_blob()])?; + + let inventory = + FilesystemRecoveryInventoryReader::open_version_two(sandbox.path())?.read(limit()?)?; + let manifest = classify_version_two_recovery_names(inventory)?; + + for role in [ + RecoveryEntryRole::WriterLock, + RecoveryEntryRole::ReaderFence, + RecoveryEntryRole::FormatMarker, + RecoveryEntryRole::RetentionDirectory, + RecoveryEntryRole::GcDirectory, + RecoveryEntryRole::RecoveryDirectory, + RecoveryEntryRole::CurrentHead, + ] { + assert!( + manifest.entries().iter().any(|entry| entry.role() == role), + "root entry {role:?} was not classified as known" + ); + } + Ok(()) +} + +#[test] +fn every_open_path_refuses_a_root_without_the_format_marker() -> TestResult<()> { + let sandbox = durable_fixture::build("format-marker-missing", &[&first_blob()])?; + let root = sandbox.path(); + fs::remove_file(root.join("FORMAT"))?; + let before = witness(root)?; + + require_writer_paths_refuse(root)?; + require_inventory_refuses(root)?; + require_content_readers_refuse(root)?; + + assert_eq!(witness(root)?, before); + Ok(()) +} + +fn require_writer_paths_refuse(root: &Path) -> TestResult<()> { + let Err(writer) = FilesystemVersionTwoAdmission::reopen(root) else { + return Err("writer admission admitted a root without FORMAT".into()); + }; + require_missing_marker(&writer)?; + let Err(recovery) = FilesystemVersionTwoRecoveryAdmission::reopen(root) else { + return Err("recovery admission admitted a root without FORMAT".into()); + }; + require_missing_marker(&recovery)?; + let Err(VersionTwoInitializationError::Refused { .. }) = + FilesystemVersionTwoAdmission::initialize(root) + else { + return Err("initialization adopted a populated root without FORMAT".into()); + }; + Ok(()) +} + +fn require_inventory_refuses(root: &Path) -> TestResult<()> { + let Err(RecoveryInventoryError::Io { + namespace: RecoveryNamespace::Root, + source, + .. + }) = FilesystemRecoveryInventoryReader::open_version_two(root) + else { + return Err("recovery inventory admitted a root without FORMAT".into()); + }; + require_missing_marker( + source + .get_ref() + .and_then(|inner| inner.downcast_ref::()) + .ok_or("inventory refusal lost its admission source")?, + )?; + Ok(()) +} + +fn require_content_readers_refuse(root: &Path) -> TestResult<()> { + let policy = durable_fixture::policy()?; + let Err(CatalogRestartError::Admission { source }) = + FilesystemCatalogSnapshot::load(root, policy) + else { + return Err("catalog restart admitted a root without FORMAT".into()); + }; + require_missing_marker(&source)?; + let Err(FilesystemRetentionSnapshotError::Admission { source }) = + FilesystemRetentionSnapshot::load(root, policy, ReaderAttemptLimit::DEFAULT) + else { + return Err("the retention reader admitted a root without FORMAT".into()); + }; + require_refusal(&source)?; + let Err(DurableStoreError::Snapshot(snapshot)) = + DurableStore::open(root, policy, ReaderAttemptLimit::DEFAULT)?.snapshot() + else { + return Err("the durable reader admitted a root without FORMAT".into()); + }; + let FilesystemRetentionSnapshotError::Admission { source } = snapshot.as_ref() else { + return Err(format!("unexpected durable refusal {snapshot:?}").into()); + }; + require_refusal(source) +} + +fn require_missing_marker(error: &FilesystemPlatformAdmissionError) -> TestResult<()> { + let FilesystemPlatformAdmissionError::Namespace { source } = error else { + return Err(format!("unexpected admission refusal {error:?}").into()); + }; + require_refusal(source) +} + +fn require_refusal(source: &std::io::Error) -> TestResult<()> { + match source + .get_ref() + .and_then(|cause| cause.downcast_ref::()) + { + Some(FilesystemNamespaceRefusal::MissingFormatMarker) => Ok(()), + _ => Err(format!("refusal is not MissingFormatMarker: {source:?}").into()), + } +} diff --git a/tests/version_two_catalog_writer/suite.rs b/tests/version_two_catalog_writer/suite.rs index 94559109..63315315 100644 --- a/tests/version_two_catalog_writer/suite.rs +++ b/tests/version_two_catalog_writer/suite.rs @@ -14,6 +14,8 @@ mod durable_fixture; mod durable_sandbox; #[path = "finalization_laws.rs"] mod finalization_laws; +#[path = "format_marker_laws.rs"] +mod format_marker_laws; #[path = "forward_laws.rs"] mod forward_laws; #[path = "../support/populated_store.rs"] @@ -30,7 +32,5 @@ mod refusal_laws; mod stage_recovery_laws; #[path = "../support/successor_catalog.rs"] mod successor_catalog; -#[path = "version_laws.rs"] -mod version_laws; #[path = "writer_fixture.rs"] mod writer_fixture; diff --git a/tests/version_two_catalog_writer/version_laws.rs b/tests/version_two_catalog_writer/version_laws.rs deleted file mode 100644 index 1e9f3bb0..00000000 --- a/tests/version_two_catalog_writer/version_laws.rs +++ /dev/null @@ -1,138 +0,0 @@ -//! This module owns typed refusal of cross-version writer and recovery entry. - -use keep::{ - AdmittedSegment, AdmittedSegmentRecord, FilesystemCatalogPublisher, FilesystemNamespaceRefusal, - FilesystemPlatformAdmission, FilesystemPlatformAdmissionError, - FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, - FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageDiscardOpenError, - FilesystemRecoveryStageDiscarder, FilesystemVersionTwoAdmission, - FilesystemVersionTwoRecoveryAdmission, RecoveryEntryRole, RecoveryInventoryError, - RecoveryInventoryLimit, RecoveryNameClassificationError, RecoveryNamespace, - RecoveryRequiredEntry, SegmentPublicationError, SegmentRecordLimit, StagedSegment, - classify_recovery_names, classify_version_two_recovery_names, -}; - -use super::durable_fixture; -use super::durable_sandbox::TestDirectory; -use super::writer_fixture::{TestResult, first_blob, open_publisher, segment_policy}; - -fn limit() -> TestResult { - Ok(RecoveryInventoryLimit::new(4096)?) -} - -#[test] -fn version_one_entry_points_refuse_a_version_two_root() -> TestResult<()> { - let sandbox = durable_fixture::build("v2-versions-refuse-v1", &[&first_blob()])?; - let root = sandbox.path(); - let policy = durable_fixture::policy()?; - - assert!(matches!( - FilesystemPlatformAdmission::reopen(root), - Err(FilesystemPlatformAdmissionError::Namespace { .. }) - )); - assert!(matches!( - FilesystemRecoveryInventoryReader::open(root), - Err(RecoveryInventoryError::Io { - namespace: RecoveryNamespace::Root, - .. - }) - )); - assert!(matches!( - FilesystemRecoveryStageDiscarder::open(root), - Err(FilesystemRecoveryStageDiscardOpenError::Namespace { .. }) - )); - assert!(matches!( - FilesystemRecoveryNextHeadFinalizer::open(root, policy), - Err(FilesystemRecoveryNextHeadFinalizationOpenError::Namespace { .. }) - )); - - let inventory = FilesystemRecoveryInventoryReader::open_version_two(root)?.read(limit()?)?; - assert!(matches!( - classify_recovery_names(inventory.clone()), - Err(RecoveryNameClassificationError::Unexpected { - namespace: RecoveryNamespace::Root, - .. - }) - )); - let manifest = classify_version_two_recovery_names(inventory)?; - for role in [ - RecoveryEntryRole::ReaderFence, - RecoveryEntryRole::FormatMarker, - RecoveryEntryRole::RetentionDirectory, - RecoveryEntryRole::GcDirectory, - RecoveryEntryRole::RecoveryDirectory, - RecoveryEntryRole::CurrentHead, - ] { - assert!( - manifest.entries().iter().any(|entry| entry.role() == role), - "version-two root entry {role:?} was not classified as known" - ); - } - Ok(()) -} - -#[test] -fn version_two_entry_points_refuse_a_version_one_root() -> TestResult<()> { - let sandbox = TestDirectory::create("v2-versions-refuse-v2")?; - let root = sandbox.path(); - drop(FilesystemPlatformAdmission::initialize(root)?); - let missing = |error: &FilesystemPlatformAdmissionError| { - matches!( - error, - FilesystemPlatformAdmissionError::Namespace { source } - if source.get_ref().and_then(|cause| cause.downcast_ref::()) - == Some(&FilesystemNamespaceRefusal::MissingFormatMarker) - ) - }; - - let Err(writer) = FilesystemVersionTwoAdmission::reopen(root) else { - return Err("version-two writer admission admitted a version-one root".into()); - }; - assert!(missing(&writer), "unexpected refusal: {writer:?}"); - let Err(recovery) = FilesystemVersionTwoRecoveryAdmission::reopen(root) else { - return Err("version-two recovery admission admitted a version-one root".into()); - }; - assert!(missing(&recovery), "unexpected refusal: {recovery:?}"); - let Err(RecoveryInventoryError::Io { - namespace: RecoveryNamespace::Root, - source, - .. - }) = FilesystemRecoveryInventoryReader::open_version_two(root) - else { - return Err("version-two inventory admitted a version-one root".into()); - }; - let admission = source - .get_ref() - .and_then(|inner| inner.downcast_ref::()) - .ok_or("inventory refusal lost its admission source")?; - assert!(missing(admission), "unexpected refusal: {admission:?}"); - - let inventory = FilesystemRecoveryInventoryReader::open(root)?.read(limit()?)?; - assert!(matches!( - classify_version_two_recovery_names(inventory), - Err(RecoveryNameClassificationError::Missing { - required: RecoveryRequiredEntry::ReaderFence - }) - )); - Ok(()) -} - -#[test] -fn a_version_one_segment_stage_cannot_be_selected_by_the_version_two_writer() -> TestResult<()> { - let version_one = TestDirectory::create("v2-versions-stage-v1")?; - let admission = FilesystemPlatformAdmission::initialize(version_one.path())?; - let legacy = FilesystemCatalogPublisher::open(admission, durable_fixture::policy()?)?; - let sealed = StagedSegment::begin(legacy.create_segment_stage()?, SegmentRecordLimit::MAXIMUM)? - .append(AdmittedSegmentRecord::for_chunk(b"version-one stage")?)? - .seal()?; - let bytes = std::fs::read(version_one.path().join("staging/current.seg"))?; - let segment = AdmittedSegment::decode(&bytes, segment_policy())?; - - let version_two = durable_fixture::build("v2-versions-stage-v2", &[&first_blob()])?; - let publisher = open_publisher(version_two.path())?; - assert!(matches!( - publisher.select_segment(sealed, &segment), - Err(SegmentPublicationError::PublisherAuthority) - )); - Ok(()) -} diff --git a/xtask/src/durability_crash_matrix/production_protocol.rs b/xtask/src/durability_crash_matrix/production_protocol.rs index f2030d06..a6f7b53c 100644 --- a/xtask/src/durability_crash_matrix/production_protocol.rs +++ b/xtask/src/durability_crash_matrix/production_protocol.rs @@ -4,7 +4,6 @@ mod control; pub(super) mod crash_blob; pub(super) mod fixture; pub(super) mod initialization; -mod initialization_storage; pub(super) mod populated_store; mod publication; mod recovery; @@ -35,9 +34,6 @@ pub(super) fn run( let store_root = create_store_root(case_root)?; let mut control = CrashControl::new(case, readiness); match case.point().sequence() { - DurabilityCrashSequence::Initialization => { - initialization::run(&store_root, &mut control)?; - } DurabilityCrashSequence::Segment | DurabilityCrashSequence::Catalog | DurabilityCrashSequence::Head => { diff --git a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs index ac7f4c82..bf30c95e 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs @@ -1,31 +1,16 @@ -//! This module owns execution of the production initialization protocol. +//! This module owns the crash children' initialized writer and read policies. use std::path::Path; use keep::{ CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemVersionTwoAdmission, - FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, RepositoryInitializationStorage, - SegmentReadPolicy, SegmentRecordLimit, initialize_store, + FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, SegmentReadPolicy, SegmentRecordLimit, }; -use super::control::CrashControl; -use super::initialization_storage::CrashInitializationStorage; use super::{DurabilityCrashMatrixError, verification}; const RESTART_BYTE_LIMIT: u64 = 1_048_576; -pub(super) fn run( - store_root: &Path, - control: &mut CrashControl, -) -> Result<(), DurabilityCrashMatrixError> { - let storage = RepositoryInitializationStorage::admit_unchecked(store_root) - .map_err(|source| DurabilityCrashMatrixError::io("open initialization storage", source))?; - let mut storage = CrashInitializationStorage::new(storage, control); - initialize_store(&mut storage) - .map(|_receipt| ()) - .map_err(|source| verification("execute production store initialization", source)) -} - /// Initializes the single-format store and opens its writer, as production does. pub(super) fn publisher( store_root: &Path, diff --git a/xtask/src/durability_crash_matrix/production_protocol/initialization_storage.rs b/xtask/src/durability_crash_matrix/production_protocol/initialization_storage.rs deleted file mode 100644 index 4967fa0a..00000000 --- a/xtask/src/durability_crash_matrix/production_protocol/initialization_storage.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! This module owns crash injection around production store initialization. - -use std::io; - -use keep::{RepositoryInitializationStorage, StoreInitializationStorage}; -use xtask::DurabilityCrashPoint; - -use super::control::{CrashControl, DuringTiming}; - -pub(super) struct CrashInitializationStorage<'control> { - inner: RepositoryInitializationStorage, - control: &'control mut CrashControl, -} - -impl<'control> CrashInitializationStorage<'control> { - pub(super) const fn new( - inner: RepositoryInitializationStorage, - control: &'control mut CrashControl, - ) -> Self { - Self { inner, control } - } -} - -impl StoreInitializationStorage for CrashInitializationStorage<'_> { - fn admit_platform(&mut self) -> io::Result<()> { - self.inner.admit_platform() - } - - fn open_and_lock_writer_file(&mut self) -> io::Result<()> { - let point = DurabilityCrashPoint::OpenAndLockWriterFile; - self.control.before(point, DuringTiming::After)?; - self.inner.open_and_lock_writer_file()?; - self.control.after(point, DuringTiming::After) - } - - fn admit_staging_directory(&mut self) -> io::Result<()> { - let point = DurabilityCrashPoint::CreateStagingDirectory; - self.control.before(point, DuringTiming::After)?; - self.inner.admit_staging_directory()?; - self.control.after(point, DuringTiming::After) - } - - fn admit_segment_pool_directory(&mut self) -> io::Result<()> { - let point = DurabilityCrashPoint::CreateSegmentPoolDirectory; - self.control.before(point, DuringTiming::After)?; - self.inner.admit_segment_pool_directory()?; - self.control.after(point, DuringTiming::After) - } - - fn admit_catalog_pool_directory(&mut self) -> io::Result<()> { - let point = DurabilityCrashPoint::CreateCatalogPoolDirectory; - self.control.before(point, DuringTiming::After)?; - self.inner.admit_catalog_pool_directory()?; - self.control.after(point, DuringTiming::After) - } - - fn synchronize_root(&mut self) -> io::Result<()> { - let point = DurabilityCrashPoint::SynchronizeRootAfterInitialization; - self.control.before(point, DuringTiming::Before)?; - self.inner.synchronize_root()?; - self.control.after(point, DuringTiming::Before) - } -} diff --git a/xtask/src/durability_crash_matrix/restart/expectation.rs b/xtask/src/durability_crash_matrix/restart/expectation.rs index 48f746f4..6ab3a4b7 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation.rs @@ -105,7 +105,6 @@ impl ExpectedStoreState { DurabilityCrashSequence::Catalog => sequence::catalog(case), DurabilityCrashSequence::Head => sequence::head(case), DurabilityCrashSequence::RecoveryDiscard => sequence::recovery(case), - DurabilityCrashSequence::Initialization => sequence::initialization(case), DurabilityCrashSequence::Retention | DurabilityCrashSequence::VersionTwoCatalog | DurabilityCrashSequence::VersionTwoInitialization => { diff --git a/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs b/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs index 37a74f54..f9e7e6e9 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation/sequence.rs @@ -1,15 +1,10 @@ //! This module owns expected states for each durable protocol sequence. -use std::collections::{BTreeMap, BTreeSet}; - use super::steps::{ - catalog_bytes, catalog_step, completed_step, head_bytes, head_step, initialization_step, - interrupted_segment_bytes, segment_bytes, segment_step, -}; -use super::{ - ArtifactBytes, CATALOG_STAGE, CATALOGS, ExpectedStoreState, HEAD, NEXT_HEAD, SEGMENT_STAGE, - SEGMENTS, STAGING, WRITER_LOCK, + catalog_bytes, catalog_step, completed_step, head_bytes, head_step, interrupted_segment_bytes, + segment_bytes, segment_step, }; +use super::{ArtifactBytes, CATALOG_STAGE, ExpectedStoreState, HEAD, NEXT_HEAD, SEGMENT_STAGE}; use crate::durability_crash_matrix::DurabilityCrashMatrixError; use crate::durability_crash_matrix::production_protocol::fixture::{ CATALOG_POOL_PATH, SEGMENT_POOL_PATH, @@ -132,28 +127,3 @@ pub(super) fn recovery( } Ok(state) } - -pub(super) fn initialization( - case: DurabilityCrashCase, -) -> Result { - let step = initialization_step(case.point())?; - let completed = completed_step(step, case.position(), case.point()); - let mut state = ExpectedStoreState { - directories: BTreeSet::new(), - artifacts: BTreeMap::new(), - hard_link: None, - }; - if let Some(completed) = completed { - state.artifacts.insert(WRITER_LOCK, ArtifactBytes::Empty); - if completed >= 1 { - state.directories.insert(STAGING); - } - if completed >= 2 { - state.directories.insert(SEGMENTS); - } - if completed >= 3 { - state.directories.insert(CATALOGS); - } - } - Ok(state) -} diff --git a/xtask/src/durability_crash_matrix/restart/expectation/steps.rs b/xtask/src/durability_crash_matrix/restart/expectation/steps.rs index 19d6a6ec..3674649b 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation/steps.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation/steps.rs @@ -31,10 +31,6 @@ const fn atomic(point: DurabilityCrashPoint) -> bool { | DurabilityCrashPoint::ReplaceHead | DurabilityCrashPoint::RemoveRecoveryStage | DurabilityCrashPoint::RemoveRecoveryHead - | DurabilityCrashPoint::OpenAndLockWriterFile - | DurabilityCrashPoint::CreateStagingDirectory - | DurabilityCrashPoint::CreateSegmentPoolDirectory - | DurabilityCrashPoint::CreateCatalogPoolDirectory ) } @@ -123,19 +119,6 @@ pub(super) fn head_step(point: DurabilityCrashPoint) -> Result Result { - const POINTS: [DurabilityCrashPoint; 5] = [ - DurabilityCrashPoint::OpenAndLockWriterFile, - DurabilityCrashPoint::CreateStagingDirectory, - DurabilityCrashPoint::CreateSegmentPoolDirectory, - DurabilityCrashPoint::CreateCatalogPoolDirectory, - DurabilityCrashPoint::SynchronizeRootAfterInitialization, - ]; - step(POINTS, point) -} - fn step( points: [DurabilityCrashPoint; N], point: DurabilityCrashPoint, diff --git a/xtask/src/durability_crash_matrix/restart/retention.rs b/xtask/src/durability_crash_matrix/restart/retention.rs index 0876fbb1..dfcd0e2f 100644 --- a/xtask/src/durability_crash_matrix/restart/retention.rs +++ b/xtask/src/durability_crash_matrix/restart/retention.rs @@ -95,10 +95,9 @@ fn mismatch(message: String) -> DurabilityCrashMatrixError { /// The number of completed publication phases and any truncated stage the /// coordinate leaves behind. fn prefix(case: DurabilityCrashCase) -> (usize, Option) { - let index = DurabilityCrashPoint::ALL - .iter() - .position(|point| *point == case.point()) - .and_then(|index| index.checked_sub(35)) + let index = ordinal(case.point()) + .zip(ordinal(DurabilityCrashPoint::WriteRootStage)) + .and_then(|(point, first)| point.checked_sub(first)) .unwrap_or(0); // Phase 1 is current-state verification; point n is phase n + 2. let phase = index.saturating_add(2); @@ -118,6 +117,13 @@ fn prefix(case: DurabilityCrashCase) -> (usize, Option) { } } +/// The position of `point` in stable protocol order. +fn ordinal(point: DurabilityCrashPoint) -> Option { + DurabilityCrashPoint::ALL + .iter() + .position(|candidate| *candidate == point) +} + const fn atomic(point: DurabilityCrashPoint) -> bool { matches!( point, diff --git a/xtask/src/durability_crash_matrix/restart/semantic.rs b/xtask/src/durability_crash_matrix/restart/semantic.rs index 687c2133..7903cfe4 100644 --- a/xtask/src/durability_crash_matrix/restart/semantic.rs +++ b/xtask/src/durability_crash_matrix/restart/semantic.rs @@ -6,15 +6,14 @@ use std::path::Path; use keep::{ AdmittedSegment, CatalogRestartByteLimit, CatalogRestartPolicy, ChecksummedCatalog, ChunkId, - FilesystemCatalogSnapshot, FilesystemVersionTwoRecoveryAdmission, FilesystemWriterLock, - LayoutEntryLimit, RecoveryCatalogStage, RecoveryNextHeadStage, RecoverySegmentStage, - SegmentReadPolicy, SegmentRecordIdentity, SegmentRecordLimit, classify_recovery_catalog_stage, + FilesystemCatalogSnapshot, FilesystemVersionTwoRecoveryAdmission, LayoutEntryLimit, + RecoveryCatalogStage, RecoveryNextHeadStage, RecoverySegmentStage, SegmentReadPolicy, + SegmentRecordIdentity, SegmentRecordLimit, classify_recovery_catalog_stage, classify_recovery_next_head_stage, classify_recovery_segment_stage, }; use super::expectation::{ - ArtifactBytes, CATALOG_STAGE, ExpectedStoreState, FORMAT, HEAD, NEXT_HEAD, SEGMENT_STAGE, - WRITER_LOCK, + ArtifactBytes, CATALOG_STAGE, ExpectedStoreState, HEAD, NEXT_HEAD, SEGMENT_STAGE, }; use crate::durability_crash_matrix::DurabilityCrashMatrixError; use crate::durability_crash_matrix::production_protocol::fixture::{ @@ -27,7 +26,7 @@ pub(super) fn verify( store_root: &Path, expected: &ExpectedStoreState, ) -> Result<(), DurabilityCrashMatrixError> { - verify_writer_release(store_root, expected)?; + verify_recovery_admission(store_root)?; verify_segment_stage(store_root, expected)?; verify_catalog_stage(store_root, expected)?; verify_next_head(store_root, expected)?; @@ -35,29 +34,16 @@ pub(super) fn verify( verify_published_snapshot(store_root, expected) } -/// Requires the dead writer's authority to be reacquirable. +/// Requires the dead writer's authority to be reacquirable over a lawful state. /// -/// A committed store must also pass single-format recovery admission: every -/// residue a catalog-protocol crash leaves is a lawful state of the format. -fn verify_writer_release( - store_root: &Path, - expected: &ExpectedStoreState, -) -> Result<(), DurabilityCrashMatrixError> { - if expected.artifact(FORMAT).is_some() { - let admission = - FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks( - store_root, - ) +/// Recovery admission takes the writer lock and admits the namespace and +/// `FORMAT` marker: every residue a catalog-protocol crash leaves is a state +/// of the format that recovery can open. +fn verify_recovery_admission(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let admission = + FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(store_root) .map_err(|source| verification("admit the crash residue for recovery", source))?; - drop(admission); - return Ok(()); - } - if expected.artifact(WRITER_LOCK).is_none() { - return Ok(()); - } - let lock = FilesystemWriterLock::try_acquire(store_root) - .map_err(|source| verification("reacquire writer lock after process death", source))?; - drop(lock); + drop(admission); Ok(()) } diff --git a/xtask/src/durability_crash_point.rs b/xtask/src/durability_crash_point.rs index 1ba8acfa..f6a8f35e 100644 --- a/xtask/src/durability_crash_point.rs +++ b/xtask/src/durability_crash_point.rs @@ -11,8 +11,6 @@ pub enum DurabilityCrashSequence { Head, /// Explicit discard of fingerprint-bound recovery evidence. RecoveryDiscard, - /// Writer-locked store initialization. - Initialization, /// Version-two retention publication, `KEEP-CRASH-036` through `052`. Retention, /// Direct version-two initialization and its recovery, `KEEP-CRASH-081` @@ -25,12 +23,11 @@ pub enum DurabilityCrashSequence { impl DurabilityCrashSequence { /// Every sequence in stable protocol order. - pub const ALL: [Self; 8] = [ + pub const ALL: [Self; 7] = [ Self::Segment, Self::Catalog, Self::Head, Self::RecoveryDiscard, - Self::Initialization, Self::Retention, Self::VersionTwoCatalog, Self::VersionTwoInitialization, @@ -44,7 +41,6 @@ impl DurabilityCrashSequence { Self::Catalog => "catalog", Self::Head => "head", Self::RecoveryDiscard => "recovery-discard", - Self::Initialization => "initialization", Self::Retention => "retention", Self::VersionTwoCatalog => "version-two-catalog", Self::VersionTwoInitialization => "version-two-initialization", @@ -123,16 +119,6 @@ pub enum DurabilityCrashPoint { RemoveRecoveryHead, /// Synchronize the store root after recovery-head removal. SynchronizeRootAfterRecovery, - /// Create or reopen `writer.lock`, then acquire writer authority. - OpenAndLockWriterFile, - /// Create or verify the staging directory. - CreateStagingDirectory, - /// Create or verify the segment-pool directory. - CreateSegmentPoolDirectory, - /// Create or verify the catalog-pool directory. - CreateCatalogPoolDirectory, - /// Synchronize the store root after initialization. - SynchronizeRootAfterInitialization, /// Retention root stage write. WriteRootStage, /// Retention root stage synchronization. @@ -204,7 +190,7 @@ pub enum DurabilityCrashPoint { impl DurabilityCrashPoint { /// Every crash boundary in stable protocol order. - pub const ALL: [Self; 67] = [ + pub const ALL: [Self; 62] = [ Self::CreateSegmentStage, Self::WriteSegmentHeader, Self::AppendSegmentRecord, @@ -235,11 +221,6 @@ impl DurabilityCrashPoint { Self::SynchronizeStagingAfterRecovery, Self::RemoveRecoveryHead, Self::SynchronizeRootAfterRecovery, - Self::OpenAndLockWriterFile, - Self::CreateStagingDirectory, - Self::CreateSegmentPoolDirectory, - Self::CreateCatalogPoolDirectory, - Self::SynchronizeRootAfterInitialization, Self::WriteRootStage, Self::SynchronizeRootStage, Self::AdmitRootNamespace, @@ -320,11 +301,6 @@ impl DurabilityCrashPoint { | Self::SynchronizeStagingAfterRecovery | Self::RemoveRecoveryHead | Self::SynchronizeRootAfterRecovery => DurabilityCrashSequence::RecoveryDiscard, - Self::OpenAndLockWriterFile - | Self::CreateStagingDirectory - | Self::CreateSegmentPoolDirectory - | Self::CreateCatalogPoolDirectory - | Self::SynchronizeRootAfterInitialization => DurabilityCrashSequence::Initialization, Self::WriteRootStage | Self::SynchronizeRootStage | Self::AdmitRootNamespace diff --git a/xtask/src/durability_crash_point_identity.rs b/xtask/src/durability_crash_point_identity.rs index 89bf2bd7..54693b15 100644 --- a/xtask/src/durability_crash_point_identity.rs +++ b/xtask/src/durability_crash_point_identity.rs @@ -37,11 +37,6 @@ impl DurabilityCrashPoint { Self::SynchronizeStagingAfterRecovery => "KEEP-CRASH-028", Self::RemoveRecoveryHead => "KEEP-CRASH-029", Self::SynchronizeRootAfterRecovery => "KEEP-CRASH-030", - Self::OpenAndLockWriterFile => "KEEP-CRASH-031", - Self::CreateStagingDirectory => "KEEP-CRASH-032", - Self::CreateSegmentPoolDirectory => "KEEP-CRASH-033", - Self::CreateCatalogPoolDirectory => "KEEP-CRASH-034", - Self::SynchronizeRootAfterInitialization => "KEEP-CRASH-035", Self::WriteRootStage => "KEEP-CRASH-036", Self::SynchronizeRootStage => "KEEP-CRASH-037", Self::AdmitRootNamespace => "KEEP-CRASH-038", diff --git a/xtask/tests/durability_crash_documentation.rs b/xtask/tests/durability_crash_documentation.rs index e55ca70f..90e25f49 100644 --- a/xtask/tests/durability_crash_documentation.rs +++ b/xtask/tests/durability_crash_documentation.rs @@ -16,7 +16,7 @@ fn living_documentation_routes_the_complete_crash_matrix_and_its_limits() { (ROOT_README, "cargo xtask durability-crash-matrix"), (RECOVERY, "## Process-death crash matrix"), (REQUIREMENTS, "`KEEP-RECOVERY-021`"), - (CORPUS_README, "105 canonical process-death cases"), + (CORPUS_README, "90 canonical process-death cases"), ( V2_CORPUS_README, "41 canonical initialization process-death cases", diff --git a/xtask/tests/durability_crash_point_contract.rs b/xtask/tests/durability_crash_point_contract.rs index db315598..4c0ac3ce 100644 --- a/xtask/tests/durability_crash_point_contract.rs +++ b/xtask/tests/durability_crash_point_contract.rs @@ -5,8 +5,7 @@ use xtask::{DurabilityCrashPoint, DurabilityCrashSequence}; use DurabilityCrashSequence::{ - Catalog, Head, Initialization, RecoveryDiscard, Retention, Segment, VersionTwoCatalog, - VersionTwoInitialization, + Catalog, Head, RecoveryDiscard, Retention, Segment, VersionTwoCatalog, VersionTwoInitialization, }; const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ @@ -140,31 +139,6 @@ const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ "KEEP-CRASH-030", RecoveryDiscard, ), - ( - DurabilityCrashPoint::OpenAndLockWriterFile, - "KEEP-CRASH-031", - Initialization, - ), - ( - DurabilityCrashPoint::CreateStagingDirectory, - "KEEP-CRASH-032", - Initialization, - ), - ( - DurabilityCrashPoint::CreateSegmentPoolDirectory, - "KEEP-CRASH-033", - Initialization, - ), - ( - DurabilityCrashPoint::CreateCatalogPoolDirectory, - "KEEP-CRASH-034", - Initialization, - ), - ( - DurabilityCrashPoint::SynchronizeRootAfterInitialization, - "KEEP-CRASH-035", - Initialization, - ), ( DurabilityCrashPoint::WriteRootStage, "KEEP-CRASH-036", @@ -370,7 +344,7 @@ fn initialization_directory_admission_runs_one_during_case_per_directory() { #[test] fn retired_crash_identifiers_are_never_reassigned() { - for ordinal in (53..=73).chain(78..=80) { + for ordinal in (31..=35).chain(53..=73).chain(78..=80) { let identifier = format!("KEEP-CRASH-{ordinal:03}"); assert_eq!( DurabilityCrashPoint::from_identifier(&identifier), @@ -403,7 +377,6 @@ fn crash_matrix_cli_admits_its_stable_sequence_names() { ("catalog", Catalog), ("head", Head), ("recovery-discard", RecoveryDiscard), - ("initialization", Initialization), ("retention", Retention), ("version-two-catalog", VersionTwoCatalog), ("version-two-initialization", VersionTwoInitialization), diff --git a/xtask/tests/durability_crash_production_contract.rs b/xtask/tests/durability_crash_production_contract.rs index cc4f9051..15fe61e7 100644 --- a/xtask/tests/durability_crash_production_contract.rs +++ b/xtask/tests/durability_crash_production_contract.rs @@ -12,7 +12,6 @@ fn crash_children_execute_every_claimed_production_protocol() -> Result<(), Box< let child = fs::read_to_string(source_root.join("durability_crash_matrix/child.rs"))?; let protocol = [ "durability_crash_matrix/production_protocol.rs", - "durability_crash_matrix/production_protocol/initialization.rs", "durability_crash_matrix/production_protocol/publication.rs", "durability_crash_matrix/production_protocol/recovery.rs", "durability_crash_matrix/production_protocol/version_two_initialization.rs", @@ -26,7 +25,6 @@ fn crash_children_execute_every_claimed_production_protocol() -> Result<(), Box< assert!(!child.contains("state::prepare(")); for required in [ "StagedSegment::begin(", - "initialize_store(", "execute_recovery_stage_discard(", "initialize_observed_for_repository_tasks(", "publish_retained_catalog_generation(", diff --git a/xtask/tests/retention_store_v2_protocol_contract/initialization_contract_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/initialization_contract_laws.rs index 725e9a09..6b247f56 100644 --- a/xtask/tests/retention_store_v2_protocol_contract/initialization_contract_laws.rs +++ b/xtask/tests/retention_store_v2_protocol_contract/initialization_contract_laws.rs @@ -31,7 +31,7 @@ fn initialization_and_recovery_define_every_authority_boundary() "`KEEP-CRASH-036`", "`KEEP-CRASH-081`", "`KEEP-CRASH-091`", - "Version-1 admission refuses", + "every open path now requires `FORMAT`", "`reader.lock`", "`GcRetirementIntent`", "`GcRetirementReceipt`", diff --git a/xtask/tests/segment_store_implementation_documentation.rs b/xtask/tests/segment_store_implementation_documentation.rs index 7698434d..991faba0 100644 --- a/xtask/tests/segment_store_implementation_documentation.rs +++ b/xtask/tests/segment_store_implementation_documentation.rs @@ -9,9 +9,9 @@ const RECOVERY: &str = include_str!("../../docs/formats/segment-store-v1/recover const CORPUS_README: &str = include_str!("../../conformance/segment-store/v1/README.md"); #[test] -fn version_one_crash_evidence_is_distinguished_from_the_complete_command() { +fn catalog_protocol_crash_evidence_is_distinguished_from_the_complete_command() { assert!( - RECOVERY.contains("105 version-one cases are a subset"), + RECOVERY.contains("90 catalog-protocol cases are a subset"), "the unqualified crash command also executes version-two cases" ); } diff --git a/xtask/tests/segment_store_protocol_contract/publication_laws.rs b/xtask/tests/segment_store_protocol_contract/publication_laws.rs index bd43027b..8193b979 100644 --- a/xtask/tests/segment_store_protocol_contract/publication_laws.rs +++ b/xtask/tests/segment_store_protocol_contract/publication_laws.rs @@ -31,22 +31,19 @@ fn immutable_pool_links_are_verified_after_namespace_resolution() { } #[test] -fn initialization_is_writer_locked_idempotent_and_root_synchronized() { +fn store_initialization_routes_to_the_single_format_protocol() { for required in [ - "## Crash-safe initialization", - "acquires the exclusive advisory lock before\n\ - creating any directory", - "partial canonical initialization set", - "synchronizes the store\n\ - root (`KEEP-CRASH-035`)", - "Only the completed root synchronization admits an\n\ - uninitialized store", + "## Store initialization", + "(../segment-store-v2/initialization.md)", + "`KEEP-CRASH-081`–`KEEP-CRASH-091`", + "`KEEP-CRASH-031`–`KEEP-CRASH-035`; the", ] { assert!( SPECIFICATION.contains(required), - "missing crash-safe initialization law: {required}" + "missing single-format initialization route: {required}" ); } + assert!(!SPECIFICATION.contains("## Crash-safe initialization")); } #[test] diff --git a/xtask/tests/segment_store_protocol_contract/transition_laws.rs b/xtask/tests/segment_store_protocol_contract/transition_laws.rs index 401571e0..98d774b0 100644 --- a/xtask/tests/segment_store_protocol_contract/transition_laws.rs +++ b/xtask/tests/segment_store_protocol_contract/transition_laws.rs @@ -47,30 +47,13 @@ const EXACT_TRANSITIONS: &[&str] = &[ unlinked-next-head\t\ named-unpublishable-next-head-or-discarded-next-head\t\ discarded-next-head\treport-explicit-next-head-discard", - "KEEP-CRASH-031\tinitialization\testablish-writer-lock\t\ - capability-probed-initialization-root\trecoverable-initialization\t\ - locked-initialization-root\treopen-or-create-and-lock", - "KEEP-CRASH-032\tinitialization\tcreate-staging-directory\t\ - locked-initialization-root\trecoverable-initialization\t\ - staging-directory-present\tverify-canonical-initialization-set", - "KEEP-CRASH-033\tinitialization\tcreate-segment-directory\t\ - staging-directory-present\trecoverable-initialization\t\ - segment-directory-present\tverify-canonical-initialization-set", - "KEEP-CRASH-034\tinitialization\tcreate-catalog-directory\t\ - segment-directory-present\trecoverable-initialization\t\ - complete-unsynchronized-initialization-set\t\ - verify-canonical-initialization-set", - "KEEP-CRASH-035\tinitialization\tsync-root-after-initialization\t\ - complete-unsynchronized-initialization-set\t\ - recoverable-initialization-or-uninitialized-store\t\ - uninitialized-store-admitted\treport-initialization", ]; #[test] -fn conformance_guide_routes_recovery_and_initialization_transitions() { +fn conformance_guide_routes_recovery_and_retires_version_one_initialization() { for required in [ "`KEEP-CRASH-027`–`KEEP-CRASH-030` own explicit recovery discard", - "`KEEP-CRASH-031`–`KEEP-CRASH-035` own crash-safe initialization", + "`KEEP-CRASH-031`–`KEEP-CRASH-035` are retired", ] { assert!( CONFORMANCE_GUIDE.contains(required), @@ -103,7 +86,7 @@ fn durable_transition_ledger_is_complete_and_stable() -> Result<(), String> { .checked_add(1) .ok_or("transition count overflow")?; } - assert_eq!(row_count, 35); + assert_eq!(row_count, 30); for exact_transition in EXACT_TRANSITIONS { assert!( TRANSITIONS From 4dca2b621a48319ca91191e34d9c4010bd8c9b6c Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 10:39:43 -0700 Subject: [PATCH 20/36] Refactor: name the store-format types without "version two" (#198) With version 1 deleted there is one format, so "VersionTwo" no longer distinguishes anything and reads as if a second format existed. Refactor only: no behavior or test law changes; Display messages that said "version-two" now name the store instead. Public API: - FilesystemVersionTwoAdmission -> FilesystemStoreAdmission - FilesystemVersionTwoRecoveryAdmission -> FilesystemRecoveryAdmission - FilesystemVersionTwoCatalogPublisher -> FilesystemCatalogPublisher - FilesystemVersionTwoCatalogStages -> FilesystemRetainedCatalogStages - VersionTwoInitialization{,Boundary,Directory,Error,Observer,Phase,Prefix, Refusal} -> StoreInitialization{...} - VersionTwoRecordRefusal -> StoreRecordRefusal - FilesystemPlatformAdmissionError -> FilesystemStoreAdmissionError - classify_version_two_recovery_names -> classify_recovery_names - open_version_two{,_unchecked_for_repository_tasks} -> open{,...} on the recovery tools and inventory reader Kept: StoreFormatDefinitionDigest::VERSION_TWO and CanonicalStoreFormatMarker::version_two(), which name the format's own version number (keep.segment-store/v2, marker version 2), and the "v2" record-grammar and corpus names. Modules and files follow the types (filesystem_store_admission, filesystem_recovery_admission, filesystem_catalog_publisher, filesystem_retained_catalog_stages, filesystem_store_namespace, store_initialization/, tests/catalog_writer, tests/store_admission_contract). Crash-matrix sequences version-two-initialization and version-two-catalog become initialization and retained-catalog, and points VersionTwo* become Retained*; their KEEP-CRASH identifiers are unchanged. Comments and living docs drop "version-two" where it meant "the store"; changelog, ADRs, and testing-evidence records keep their historical names. --- README.md | 7 +- conformance/segment-store/v2/README.md | 2 +- docs/formats/segment-store-v1/publication.md | 12 +- docs/formats/segment-store-v1/rationale.md | 2 +- docs/formats/segment-store-v1/recovery.md | 8 +- docs/formats/segment-store-v1/requirements.md | 4 +- docs/formats/segment-store-v2/README.md | 6 +- .../segment-store-v2/catalog-publication.md | 22 +-- .../segment-store-v2/initialization.md | 8 +- docs/formats/segment-store-v2/rationale.md | 6 +- docs/formats/segment-store-v2/recovery.md | 4 +- docs/formats/segment-store-v2/requirements.md | 12 +- .../segment-store-v2/retention-publication.md | 2 +- .../segment-store-v2/retired-identifiers.md | 2 +- src/adapters/catalog_restart_error.rs | 6 +- src/adapters/catalog_restart_loader.rs | 8 +- src/adapters/durable/error.rs | 2 +- src/adapters/durable/snapshot.rs | 4 +- src/adapters/durable/store.rs | 6 +- src/adapters/exports.rs | 26 +-- ...her.rs => filesystem_catalog_publisher.rs} | 28 +-- ...> filesystem_catalog_publisher_storage.rs} | 10 +- .../filesystem_catalog_publisher_tests.rs | 28 ++- ...s => filesystem_catalog_recovery_tests.rs} | 14 +- src/adapters/filesystem_format_marker.rs | 8 +- src/adapters/filesystem_namespace_refusal.rs | 4 +- src/adapters/filesystem_platform_profile.rs | 16 +- .../filesystem_platform_profile_tests.rs | 4 +- ...on.rs => filesystem_recovery_admission.rs} | 33 ++-- .../filesystem_recovery_inventory_reader.rs | 36 ++-- .../filesystem_recovery_inventory_tests.rs | 19 +- ...m_recovery_next_head_finalization_tests.rs | 2 +- .../fixture.rs | 15 +- ...filesystem_recovery_next_head_finalizer.rs | 12 +- ...ilesystem_recovery_segment_resume_tests.rs | 6 +- .../fixture.rs | 19 +- .../filesystem_recovery_segment_resumer.rs | 14 +- .../filesystem_recovery_stage_completer.rs | 10 +- ...esystem_recovery_stage_completion_tests.rs | 4 +- .../fixture.rs | 22 +-- ...filesystem_recovery_stage_discard_tests.rs | 4 +- .../fixture.rs | 18 +- .../filesystem_recovery_stage_discarder.rs | 14 +- .../filesystem_recovery_stage_tests.rs | 12 +- ... => filesystem_retained_catalog_stages.rs} | 14 +- .../filesystem_segment_stage_tests.rs | 14 +- ...ssion.rs => filesystem_store_admission.rs} | 102 +++++----- ...rs => filesystem_store_admission_error.rs} | 10 +- ...space.rs => filesystem_store_namespace.rs} | 48 ++--- ....rs => filesystem_store_record_refusal.rs} | 27 ++- src/adapters/mod.rs | 22 +-- src/adapters/recovery.rs | 2 +- src/adapters/recovery/recovery_entry_role.rs | 10 +- .../recovery/recovery_name_classification.rs | 6 +- ...ry_next_head_finalization_storage_error.rs | 2 +- .../recovery/recovery_required_entry.rs | 2 +- src/adapters/retention.rs | 4 +- .../retention/durable_view_law_tests.rs | 4 +- .../retention/filesystem_retained_evidence.rs | 2 +- .../filesystem_retention_authority.rs | 6 +- .../filesystem_retention_fifo_tests.rs | 4 +- ...esystem_retention_recovery_effect_tests.rs | 4 +- ..._retention_recovery_storage_error_tests.rs | 2 +- .../filesystem_retention_short_count_tests.rs | 2 +- .../filesystem_retention_snapshot.rs | 14 +- .../filesystem_retention_snapshot_error.rs | 6 +- .../filesystem_retention_test_fixture.rs | 13 +- .../filesystem_retention_verification.rs | 16 +- ...rs => filesystem_store_admission_tests.rs} | 100 +++++----- src/adapters/retention/reader_fence.rs | 2 +- .../verification_store_admission_tests.rs | 14 +- src/adapters/segment_publication.rs | 2 +- ...tialization.rs => store_initialization.rs} | 22 +-- .../initialization_entries.rs | 2 +- .../initialization_error.rs | 28 +-- .../initialization_execution.rs | 6 +- .../initialization_namespace.rs | 4 +- .../initialization_observation.rs | 6 +- .../initialization_phase.rs | 16 +- .../initialization_prefix.rs | 6 +- .../initialization_prefix_tests.rs | 4 +- .../initialization_reader_fence.rs | 0 .../initialization_receipt.rs | 24 +-- .../initialization_refusal.rs | 6 +- .../initialization_resume_tests.rs | 23 ++- .../initialization_test_fixture.rs | 29 ++- src/adapters/verification_failure_class.rs | 10 +- src/lib.rs | 31 ++- tests/adapters_layout_contract.rs | 12 +- tests/catalog_restart.rs | 6 +- ...wo_catalog_writer.rs => catalog_writer.rs} | 6 +- .../finalization_laws.rs | 0 .../format_marker_laws.rs | 30 ++- .../forward_laws.rs | 2 +- .../recovery_fixture.rs | 48 +++-- .../refusal_laws.rs | 2 +- .../stage_recovery_laws.rs | 16 +- .../suite.rs | 2 +- .../writer_fixture.rs | 14 +- tests/observed_segment_stage.rs | 14 +- tests/reader_fence_process/fixture.rs | 10 +- tests/recovery_inventory.rs | 2 +- tests/recovery_name_classification.rs | 4 +- .../grammar_laws.rs | 4 +- .../refusal_laws.rs | 10 +- tests/recovery_name_classification_memory.rs | 4 +- tests/segment_filesystem_stage.rs | 16 +- tests/single_format.rs | 4 +- tests/single_format/acceptance_laws.rs | 22 +-- tests/single_format/initialization_laws.rs | 41 ++-- tests/single_format/portability_laws.rs | 27 ++- tests/single_format/suite.rs | 4 +- tests/store_admission_contract.rs | 34 ++++ tests/support/admission_refusal.rs | 6 +- tests/support/populated_store.rs | 38 ++-- tests/support/successor_catalog.rs | 24 +-- tests/version_two_admission_contract.rs | 39 ---- xtask/src/durability_crash_matrix/error.rs | 6 +- .../durability_crash_matrix/error/display.rs | 2 +- .../production_protocol.rs | 12 +- .../production_protocol/crash_blob.rs | 12 +- .../production_protocol/initialization.rs | 178 ++++++++++++++++-- .../production_protocol/populated_store.rs | 30 ++- .../production_protocol/publication.rs | 18 +- .../production_protocol/recovery.rs | 19 +- ...ion_two_catalog.rs => retained_catalog.rs} | 60 +++--- .../production_protocol/retention.rs | 18 +- .../version_two_initialization.rs | 178 ------------------ .../production_protocol/writer.rs | 33 ++++ xtask/src/durability_crash_matrix/restart.rs | 14 +- .../restart/expectation.rs | 4 +- ...wo_initialization.rs => initialization.rs} | 41 ++-- ...ion_two_catalog.rs => retained_catalog.rs} | 48 +++-- .../restart/semantic.rs | 13 +- ...sion_two_recovery.rs => stage_recovery.rs} | 65 +++---- xtask/src/durability_crash_point.rs | 52 ++--- xtask/src/durability_crash_point_identity.rs | 8 +- xtask/tests/durability_crash_case_contract.rs | 3 +- xtask/tests/durability_crash_documentation.rs | 2 +- .../tests/durability_crash_point_contract.rs | 48 ++--- .../durability_crash_production_contract.rs | 4 +- .../transition_laws.rs | 4 +- 142 files changed, 1187 insertions(+), 1298 deletions(-) rename src/adapters/{filesystem_version_two_catalog_publisher.rs => filesystem_catalog_publisher.rs} (81%) rename src/adapters/{filesystem_version_two_catalog_storage.rs => filesystem_catalog_publisher_storage.rs} (62%) rename src/adapters/{filesystem_version_two_catalog_recovery_tests.rs => filesystem_catalog_recovery_tests.rs} (89%) rename src/adapters/{filesystem_version_two_recovery_admission.rs => filesystem_recovery_admission.rs} (61%) rename src/adapters/{filesystem_version_two_catalog_stages.rs => filesystem_retained_catalog_stages.rs} (95%) rename src/adapters/{filesystem_version_two_admission.rs => filesystem_store_admission.rs} (62%) rename src/adapters/{filesystem_platform_admission_error.rs => filesystem_store_admission_error.rs} (83%) rename src/adapters/{filesystem_initialization_namespace.rs => filesystem_store_namespace.rs} (81%) rename src/adapters/{filesystem_version_two_record_refusal.rs => filesystem_store_record_refusal.rs} (67%) rename src/adapters/retention/{filesystem_version_two_admission_tests.rs => filesystem_store_admission_tests.rs} (56%) rename src/adapters/{version_two_initialization.rs => store_initialization.rs} (68%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_entries.rs (98%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_error.rs (71%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_execution.rs (95%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_namespace.rs (87%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_observation.rs (97%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_phase.rs (92%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_prefix.rs (89%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_prefix_tests.rs (97%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_reader_fence.rs (100%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_receipt.rs (57%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_refusal.rs (95%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_resume_tests.rs (87%) rename src/adapters/{version_two_initialization => store_initialization}/initialization_test_fixture.rs (74%) rename tests/{version_two_catalog_writer.rs => catalog_writer.rs} (58%) rename tests/{version_two_catalog_writer => catalog_writer}/finalization_laws.rs (100%) rename tests/{version_two_catalog_writer => catalog_writer}/format_marker_laws.rs (78%) rename tests/{version_two_catalog_writer => catalog_writer}/forward_laws.rs (97%) rename tests/{version_two_catalog_writer => catalog_writer}/recovery_fixture.rs (69%) rename tests/{version_two_catalog_writer => catalog_writer}/refusal_laws.rs (97%) rename tests/{version_two_catalog_writer => catalog_writer}/stage_recovery_laws.rs (85%) rename tests/{version_two_catalog_writer => catalog_writer}/suite.rs (93%) rename tests/{version_two_catalog_writer => catalog_writer}/writer_fixture.rs (89%) create mode 100644 tests/store_admission_contract.rs delete mode 100644 tests/version_two_admission_contract.rs rename xtask/src/durability_crash_matrix/production_protocol/{version_two_catalog.rs => retained_catalog.rs} (67%) delete mode 100644 xtask/src/durability_crash_matrix/production_protocol/version_two_initialization.rs create mode 100644 xtask/src/durability_crash_matrix/production_protocol/writer.rs rename xtask/src/durability_crash_matrix/restart/{version_two_initialization.rs => initialization.rs} (88%) rename xtask/src/durability_crash_matrix/restart/{version_two_catalog.rs => retained_catalog.rs} (83%) rename xtask/src/durability_crash_matrix/restart/{version_two_recovery.rs => stage_recovery.rs} (69%) diff --git a/README.md b/README.md index afa5b222..14cf2631 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Keep is required to refuse all three, before mutating anything. at 156 before/during/after coordinates (`KEEP-CRASH-001`–`052`) and verifies the store lands in exactly one documented lawful state each time, for version-1 publication and for version-2 retention publication. -- **One store format, created directly.** `FilesystemVersionTwoAdmission::initialize` +- **One store format, created directly.** `FilesystemStoreAdmission::initialize` creates a version-2 store through a recoverable state machine with one durable commit point; an interrupted call resumes, and any other nonempty directory refuses before it is touched. A 41-case process-death matrix @@ -237,10 +237,9 @@ Select a crash campaign with `--sequence NAME` using these exact CLI names: | `catalog` | Catalog publication | | `head` | Publication-head replacement | | `recovery-discard` | Explicit recovery evidence discard | -| `initialization` | Writer-locked initialization | | `retention` | Retention root, manifest, and head publication | -| `version-two-catalog` | Version-two catalog writer and stage recovery | -| `version-two-initialization` | Direct initialization and its recovery | +| `retained-catalog` | Successor catalog publication beside a retained root, and its stage recovery | +| `initialization` | Direct initialization and its recovery | ```bash cargo xtask durability-crash-matrix --sequence retention diff --git a/conformance/segment-store/v2/README.md b/conformance/segment-store/v2/README.md index 836f0964..42d415ca 100644 --- a/conformance/segment-store/v2/README.md +++ b/conformance/segment-store/v2/README.md @@ -57,7 +57,7 @@ rows whose interruption may leave an incomplete pre-commit stage and therefore the only rows that plan a discard; `-089` and `-090` admit the committed store. -The `cargo xtask durability-crash-matrix --sequence version-two-initialization` +The `cargo xtask durability-crash-matrix --sequence initialization` harness executes 41 canonical initialization process-death cases at the same 11 boundaries: each at three positions plus one `during` case per directory for `KEEP-CRASH-086`. It kills an isolated writer process group, compares the diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/segment-store-v1/publication.md index a76c1f72..558c5b35 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/segment-store-v1/publication.md @@ -106,8 +106,8 @@ process-scoped exclusion are unsupported. `FilesystemWriterLock::try_acquire` opens the existing regular `writer.lock` without following symbolic links and acquires its exclusive advisory lock without blocking. That lock alone cannot construct a publisher. -`FilesystemVersionTwoCatalogPublisher::open` consumes -`FilesystemVersionTwoAdmission`, which owns the lock after the production +`FilesystemCatalogPublisher::open` consumes +`FilesystemStoreAdmission`, which owns the lock after the production platform, the exact committed namespace, and the `FORMAT` marker are admitted, then pins the store root plus `staging`, `segments`, and `catalogs` and the retention directories. Both operations perform blocking filesystem I/O. @@ -115,9 +115,9 @@ Neither operation repairs, enumerates, or removes protocol state. The admission has private fields, so only Keep's own initialization and admission boundary can produce a value. A new store obtains one through -`FilesystemVersionTwoAdmission::initialize`, which runs the +`FilesystemStoreAdmission::initialize`, which runs the [initialization protocol](../segment-store-v2/initialization.md); a committed -store reacquires one through `FilesystemVersionTwoAdmission::reopen`, which +store reacquires one through `FilesystemStoreAdmission::reopen`, which mutates nothing. A root without the `FORMAT` marker refuses at every open path. The version-1 initializer, `FilesystemPlatformAdmission`, and their rows `KEEP-CRASH-031`–`-035` are retired. @@ -138,7 +138,7 @@ protocol advances. Publisher teardown closes retained writable handles and pinned directory capabilities before releasing the writer lock. Publishing a new filesystem segment additionally requires -`FilesystemVersionTwoCatalogPublisher::select_segment`. The method consumes the +`FilesystemCatalogPublisher::select_segment`. The method consumes the `SealedSegment`, drops Keep's owned writable stage, proves that this publisher created the stage, and binds its record count, byte length, and digest to the exact `AdmittedSegment` bytes. A storage-agnostic @@ -146,7 +146,7 @@ exact `AdmittedSegment` bytes. A storage-agnostic adapters, but its handle-free `ClosedSegment` receipt cannot authorize `staging/current.seg`. Catalog publication cannot select an unrelated or still-open sealed stage. -`FilesystemVersionTwoCatalogPublisher::create_segment_stage` is the only public +`FilesystemCatalogPublisher::create_segment_stage` is the only public filesystem-stage constructor. Its returned lifetime keeps the acquired writer authority borrowed while `current.seg` remains writable. diff --git a/docs/formats/segment-store-v1/rationale.md b/docs/formats/segment-store-v1/rationale.md index 27aa0faf..1b77fa3d 100644 --- a/docs/formats/segment-store-v1/rationale.md +++ b/docs/formats/segment-store-v1/rationale.md @@ -152,7 +152,7 @@ availability. The writer lock proves process-scoped exclusion; it does not prove case-sensitive names, single-host ownership, hard-link and replacement semantics, or directory durability. The writer therefore consumes an opaque -admission value, now `FilesystemVersionTwoAdmission`, that owns the acquired +admission value, now `FilesystemStoreAdmission`, that owns the acquired lock. The proof is bound to that exact root authority and cannot be constructed from metadata or caller assertion. diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/segment-store-v1/recovery.md index 9c47f0e5..7e50f7c3 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/segment-store-v1/recovery.md @@ -29,7 +29,7 @@ the configured and protocol ceilings, count stability, duplicate refusal, and namespace-plus-raw-byte ordering. `FilesystemRecoveryInventoryReader` pins the root and three no-follow child directories, bounds each scan by the remaining global budget, preserves raw Linux name bytes, and verifies child-directory -identity before and after inventory. `classify_version_two_recovery_names` +identity before and after inventory. `classify_recovery_names` requires every root entry of an initialized store, types each fixed name and immutable-pool coordinate, and refuses an unknown or conflicting name without artifact I/O. @@ -39,9 +39,9 @@ maximum, and returns its exact observed length and `KEEP:RECOVERY:STAGE\0` fingerprint. New stores obtain writer authority through -`FilesystemVersionTwoAdmission::initialize`. A committed store reacquires -authority through `FilesystemVersionTwoAdmission::reopen`, and every recovery -tool through `FilesystemVersionTwoRecoveryAdmission::reopen`; each performs no +`FilesystemStoreAdmission::initialize`. A committed store reacquires +authority through `FilesystemStoreAdmission::reopen`, and every recovery +tool through `FilesystemRecoveryAdmission::reopen`; each performs no protocol mutation, admits the production platform, acquires the existing writer lock, and requires the exact committed namespace and the `FORMAT` marker, with `HEAD` optional and, for recovery only, a retained `head.next`. diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/segment-store-v1/requirements.md index 0adbceac..5d3b66be 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/segment-store-v1/requirements.md @@ -59,7 +59,7 @@ retention, or garbage collection. Issue #16 implements catalog-generation admission, writer-locked filesystem publication mechanics, and immutable reader snapshots. Publisher construction -now requires the single-format `FilesystemVersionTwoAdmission` (#198), which +now requires the single-format `FilesystemStoreAdmission` (#198), which admits the platform, namespace, and `FORMAT` marker under the writer lock. Explicit recovery is implemented in issue #17; its evidence is the recovery table below. @@ -74,7 +74,7 @@ below. | `KEEP-CATALOG-004` | Every catalog location equals a verified top-level record span in the exact named segment; construction and admission require every supplied segment to be referenced, and admission scans each referenced segment once | Bounded grouped lookup plan and golden artifacts | `tests/catalog_encoding.rs`, `tests/catalog_locations.rs` | Implemented in #16 | | `KEEP-CATALOG-005` | Publication admits only the exact expected successor and reports expected and observed generation and digest on staleness | Generation transition model | `tests/catalog_transition.rs` | Implemented in #16 | | `KEEP-CATALOG-006` | A reader retains one complete catalog generation and never combines it with a concurrent head | Immutable snapshot model | `tests/catalog_snapshot.rs` | Implemented in #16 | -| `KEEP-CATALOG-007` | Retained kernel locks on the pinned store root and persistent writer file exclude a second cooperative writer even if the directory entry is replaced; neither lock is deleted on release, and lock ownership alone cannot construct a publisher without platform admission | Multi-handle lock model, replacement fixture, and public unsupported-platform refusal | `tests/catalog_writer_lock.rs`; the `compile_fail` doctests on `FilesystemVersionTwoCatalogPublisher::open` and `FilesystemRetentionPublicationAuthority::open` refuse a bare `FilesystemWriterLock`; `initialization_refuses_an_unsupported_platform_before_any_mutation` in `tests/single_format.rs`; historical [admission evidence](../../testing-evidence/catalog-platform-admission.md) | Implemented in #16; replacement-hardened in #17; repository-task admission corrected for #150; single-format admission since #198 | +| `KEEP-CATALOG-007` | Retained kernel locks on the pinned store root and persistent writer file exclude a second cooperative writer even if the directory entry is replaced; neither lock is deleted on release, and lock ownership alone cannot construct a publisher without platform admission | Multi-handle lock model, replacement fixture, and public unsupported-platform refusal | `tests/catalog_writer_lock.rs`; the `compile_fail` doctests on `FilesystemCatalogPublisher::open` and `FilesystemRetentionPublicationAuthority::open` refuse a bare `FilesystemWriterLock`; `initialization_refuses_an_unsupported_platform_before_any_mutation` in `tests/single_format.rs`; historical [admission evidence](../../testing-evidence/catalog-platform-admission.md) | Implemented in #16; replacement-hardened in #17; repository-task admission corrected for #150; single-format admission since #198 | | `KEEP-CATALOG-008` | Segment, catalog, and head publication follows the documented synchronization order; retained fixed-name recovery state refuses before mutation; an absent head requires empty immutable pools; retry of an already-current candidate performs no publication mutation and re-synchronizes the root | Fault-recording port and filesystem fixtures | `tests/catalog_publication.rs`, `src/adapters/filesystem_catalog_publisher_tests.rs` | Implemented in #16 | | `KEEP-CATALOG-009` | Restart loading refuses corrupt, unsupported, noncanonical, dangling, and conflicting catalog state | Corruption matrix | `tests/catalog_restart.rs` | Implemented in #16 | | `KEEP-CATALOG-010` | Model-based transitions and lookups agree with a deterministic `BTreeMap` catalog | Boring reference catalog | `tests/catalog_model.rs`, `tests/catalog_model/generated_histories.rs`, `tests/catalog_model/transition_refusals.rs`; bounded oracle and calibration in [model evidence](../../testing-evidence/catalog-model-histories.md) | Implemented in #16; generated model evidence in #166 | diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md index 56b20499..e999d66f 100644 --- a/docs/formats/segment-store-v2/README.md +++ b/docs/formats/segment-store-v2/README.md @@ -88,9 +88,9 @@ generations, realization profiles, closure policies, anchors, and roots; canonical root, manifest, head, and marker codecs matching their golden records; storage-independent transition planning, bounded closure verification against one pinned catalog, preflight, preparation, and the -17-phase publication port; `FilesystemVersionTwoAdmission::initialize`, which +17-phase publication port; `FilesystemStoreAdmission::initialize`, which creates or resumes a store through the initialization protocol and its 41-case -process-death matrix; `FilesystemVersionTwoAdmission::reopen`, which admits +process-death matrix; `FilesystemStoreAdmission::reopen`, which admits the committed namespace and the marker and pins the retention directories it admitted; and `FilesystemRetentionPublicationAuthority`, which publishes initial and successor generations against the observed head, binds this @@ -111,7 +111,7 @@ Writer authority coordinates cooperating writers in a managed namespace; it does `FilesystemRetentionSnapshot` binds a catalog, retention head and manifest under a shared reader fence and verifies selected roots on demand. `DurableStore` composes that view with authenticated whole-blob and exact-range reads, delivered in [PR #164](https://github.com/flyingrobots/keep/pull/164); the [read contract](../../invariants/authenticated-reconstruction/README.md) records proof and allocation limits. -The candidate-catalog retained-closure gate is implemented over its storage port ([#125](https://github.com/flyingrobots/keep/issues/125)). `FilesystemVersionTwoCatalogPublisher` ingests one segment and publishes a catalog successor through it, including the first catalog of an initialized store, and version-two next-head finalization runs the same admission before replacing `HEAD` ([#198](https://github.com/flyingrobots/keep/issues/198)); see [catalog publication and stage recovery](catalog-publication.md). Verifying a new retention root against the current catalog is a different operation. +The candidate-catalog retained-closure gate is implemented over its storage port ([#125](https://github.com/flyingrobots/keep/issues/125)). `FilesystemCatalogPublisher` ingests one segment and publishes a catalog successor through it, including the first catalog of an initialized store, and version-two next-head finalization runs the same admission before replacing `HEAD` ([#198](https://github.com/flyingrobots/keep/issues/198)); see [catalog publication and stage recovery](catalog-publication.md). Verifying a new retention root against the current catalog is a different operation. [Durable verification reports](../../invariants/verification/README.md) are delivered in [PR #165](https://github.com/flyingrobots/keep/pull/165), with explicit subject/depth evidence and typed non-success outcomes. diff --git a/docs/formats/segment-store-v2/catalog-publication.md b/docs/formats/segment-store-v2/catalog-publication.md index 36c6a347..251cc76c 100644 --- a/docs/formats/segment-store-v2/catalog-publication.md +++ b/docs/formats/segment-store-v2/catalog-publication.md @@ -24,15 +24,15 @@ created by [direct initialization](initialization.md). ## Authority -`FilesystemVersionTwoAdmission::reopen` admits the root and acquires the writer -lock. `FilesystemVersionTwoCatalogPublisher::open` consumes that admission. It +`FilesystemStoreAdmission::reopen` admits the root and acquires the writer +lock. `FilesystemCatalogPublisher::open` consumes that admission. It pins the store root, `staging`, `segments`, `catalogs`, `retention`, `retention/roots`, and `retention/manifests`. A bare writer lock cannot open it. The publisher implements only `RetainedCatalogPublicationStorage`, so it publishes only through `publish_retained_catalog_generation`. Its stage type, -`FilesystemVersionTwoCatalogStages`, has private fields and no public +`FilesystemRetainedCatalogStages`, has private fields and no public constructor. It is reachable only through `catalog_stages`, which requires the gate's `RetainedClosureAdmission`. The ungated `publish_catalog_generation` therefore cannot drive version-2 stages. @@ -102,21 +102,21 @@ must resolve it first. ### Recovery admission -`FilesystemVersionTwoRecoveryAdmission::reopen` applies the writer admission +`FilesystemRecoveryAdmission::reopen` applies the writer admission with one difference: the root may also hold `head.next`. It requires the platform profile, the writer lock, the exact version-2 namespace and protocol directories, and the canonical `FORMAT` marker. The recovery tools take it through their -`open_version_two` constructors: +`open` constructors: - `FilesystemRecoveryStageDiscarder`; - `FilesystemRecoveryStageCompleter`; - `FilesystemRecoverySegmentResumer`; and - `FilesystemRecoveryNextHeadFinalizer`. -The read-only `FilesystemRecoveryInventoryReader::open_version_two` applies +The read-only `FilesystemRecoveryInventoryReader::open` applies the same namespace and marker laws without the writer lock. -`classify_version_two_recovery_names` classifies `reader.lock`, `FORMAT`, +`classify_recovery_names` classifies `reader.lock`, `FORMAT`, `retention`, `gc`, and `recovery` as known roles and requires each of them. It does not require `HEAD`: the first publication of an initialized store can be interrupted before `HEAD` exists. `FORMAT.next` is an unknown entry. @@ -144,7 +144,7 @@ pools hold artifacts, refuses in the same way. restart, and the retention and durable readers refuse a root without `FORMAT` as `MissingFormatMarker`, even when every other entry and record of a published, retained store is intact. -- `classify_version_two_recovery_names` requires every root entry of an +- `classify_recovery_names` requires every root entry of an initialized store and refuses a missing one as a missing required entry. - A segment stage sealed under one publisher's authority refuses selection by any other publisher. @@ -154,7 +154,7 @@ are deleted (#198), so no cross-version entry remains. ## Process-death matrix -`cargo xtask durability-crash-matrix --sequence version-two-catalog` runs the +`cargo xtask durability-crash-matrix --sequence retained-catalog` runs the production version-2 writer in a child process. The child blocks at the selected boundary through an observer on the writer's own stage type. Restart then requires the exact stage residue, recovers it with the public version-2 @@ -179,7 +179,7 @@ The implemented rows are 12 cases. CI runs them with the complete matrix in debug and release builds. They establish recovery after process death; they do not simulate power loss. In-process laws add an injected storage-port failure after segment link, catalog-stage creation, and `head.next` synchronization -(`filesystem_version_two_catalog_recovery_tests`), plus hand-staged complete +(`filesystem_catalog_recovery_tests`), plus hand-staged complete `head.next` candidates that preserve or drop retained content -(`tests/version_two_catalog_writer.rs`). Other boundaries rely on the shared +(`tests/catalog_writer.rs`). Other boundaries rely on the shared version-1 stage implementation and its `KEEP-CRASH-001` to `030` evidence. diff --git a/docs/formats/segment-store-v2/initialization.md b/docs/formats/segment-store-v2/initialization.md index fcc0816e..8cf60098 100644 --- a/docs/formats/segment-store-v2/initialization.md +++ b/docs/formats/segment-store-v2/initialization.md @@ -41,7 +41,7 @@ admission and the reader fence. ## Forward protocol -`FilesystemVersionTwoAdmission::initialize` runs under exclusive writer +`FilesystemStoreAdmission::initialize` runs under exclusive writer authority. It performs these phases in order: | Phase | Operation | Durable effect | @@ -110,10 +110,10 @@ an existing `reader.lock` or directory is admitted by kind, length, and membership and its parent is synchronized again, and the link never replaces an entry. Replay therefore never depends on knowing which synchronization completed before process death. A synchronization failure returns -`VersionTwoInitializationError::Io` at that phase and leaves the state on the +`StoreInitializationError::Io` at that phase and leaves the state on the table, so a retry resumes and repeats the synchronization. -Refusals carry their evidence as `VersionTwoInitializationRefusal`: +Refusals carry their evidence as `StoreInitializationRefusal`: - an entry outside the initialization namespace, a directory or `reader.lock` without `FORMAT.next`, a directory out of order, or an entry inside a @@ -165,7 +165,7 @@ path refuses it as `MissingFormatMarker`. A root holding only an empty ## Process-death matrix -`cargo xtask durability-crash-matrix --sequence version-two-initialization` +`cargo xtask durability-crash-matrix --sequence initialization` runs the production initializer in a subprocess and kills it before, during, and after each boundary. The parent then checks the exact inventory and bytes, the admission refusal or result, and runs `initialize` again as recovery. It diff --git a/docs/formats/segment-store-v2/rationale.md b/docs/formats/segment-store-v2/rationale.md index 6c5636ac..8e5f806f 100644 --- a/docs/formats/segment-store-v2/rationale.md +++ b/docs/formats/segment-store-v2/rationale.md @@ -187,7 +187,7 @@ transitions as version 1. A copy would let the two formats drift in exactly the steps that decide durability. `FilesystemCatalogStages` therefore holds the pinned catalog directories, the writable stages, and the stage-authority token, and it is the only `CatalogPublicationStorage` implementation of those -transitions. `FilesystemVersionTwoCatalogStages` wraps one under the writer lock +transitions. `FilesystemRetainedCatalogStages` wraps one under the writer lock and has no public constructor; it forwards transitions unchanged. (Until #198 deleted version 1, the version-1 `FilesystemCatalogPublisher` owned a second one beside its own writer lock.) @@ -203,7 +203,7 @@ stop readers for the whole window in which publication stages a head. Writer admission still refuses it, so a writer never starts over unrecovered state. Catalog-stage recovery on version 2 reuses the version-1 executors under a -separate `FilesystemVersionTwoRecoveryAdmission`. That admission differs from +separate `FilesystemRecoveryAdmission`. That admission differs from writer admission only in admitting `head.next`. Next-head finalization is the one recovery route that changes `HEAD`, so a version-2 finalizer runs the gate's own `admit_retained_closures` before every candidate verification that @@ -215,7 +215,7 @@ Rejected alternatives: - Copying the stage code into a version-2 publisher. The formats would drift. - Implementing the ungated port for the version-2 publisher behind a runtime check. The type system would no longer prevent ungated publication. -- Letting `FilesystemVersionTwoAdmission` admit `head.next`. Writers and +- Letting `FilesystemStoreAdmission` admit `head.next`. Writers and retention publication would then start over unrecovered catalog state. - Gating only the planner. Planning is pure and runs before the writer lock. The retention head can change between planning and replacement. diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/segment-store-v2/recovery.md index 4955c88f..addfd11c 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/segment-store-v2/recovery.md @@ -70,7 +70,7 @@ identifies the format contract, not the software release that wrote the store. `AdmittedStoreFormatMarker` admits its framing, checksum, definition, and namespace bound. Writer admission, catalog-stage recovery, recovery inventory, and reader admission each read `FORMAT` no-follow, bounded to its exact length, -and refuse a trailing byte. `FilesystemPlatformAdmissionError::FormatMarker` +and refuse a trailing byte. `FilesystemStoreAdmissionError::FormatMarker` carries the decoder's typed refusal as its source. ## Reader fence @@ -140,6 +140,6 @@ binding satisfied. `GcRetirementIntent`, `GcRetirementReceipt`, and `RecoveryDispositionReceipt` are owned by the [GC specification](gc.md). Until issue #21 implements them, any such artifact is unsupported and refuses. Initialization recovery is part of the initialization protocol: an interrupted -call is resumed by calling `FilesystemVersionTwoAdmission::initialize` again, +call is resumed by calling `FilesystemStoreAdmission::initialize` again, which classifies the root and replays the idempotent remaining phases. Its process death states are specified in [Direct initialization](initialization.md). diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/segment-store-v2/requirements.md index 5fb71bc5..1b016bfa 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/segment-store-v2/requirements.md @@ -20,9 +20,9 @@ case is not evidence. | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | all three-operation histories over initial publications of two namespaces, successor, release, restore, byte-identical retry and stale initial (343 histories, each in a fresh initialized store) compare fenced namespace generations, anchor sets and liveness against an operation-derived model after every step; exact typed refusals remain checked. The count describes exploration, not correctness. See [release/restore evidence](../../testing-evidence/retention-release-restore-model.md). Core architecture checks remain separate static evidence. | Implemented; release/restore model coverage added for #128 | | `KEEP-RETENTION-011` | Catalog publication over retention state admits the current retention head, manifest, and every manifest root under writer authority; verifies every retained closure against the candidate before any catalog storage call; and refuses `HEAD` replacement if the retention head changed | port-level laws in `tests/retained_catalog_publication.rs`: dropped and substituted members refuse with the exact missing identity and no catalog stage; every manifest root is verified wherever it sorts; absent, corrupt, substituted, and contradictory head, manifest, and root evidence refuse with typed expected and observed state; a head changed, removed, or published before commit refuses after `head.next` verification without `replace_head`; an exhaustive 24-case candidate-by-root-set model and a repeat-run determinism check | Implemented at the storage port; the filesystem version-2 publisher and version-2 next-head finalization use it (#198, `KEEP-RETENTION-012` and `013`) | -| `KEEP-RETENTION-012` | Under version-2 writer authority, one staged segment and a catalog successor publish only through the retained-closure gate, with no-follow bounded retention reads, head absence admitted only when uncontradicted, fixed retention stages refused, and no reader fence taken | `FilesystemVersionTwoCatalogPublisher`; `tests/version_two_catalog_writer.rs`: a preserving successor publishes, reopens, and reads both blobs while the retention root still verifies; a pinned reader view keeps its generation across publication; a candidate that drops retained content, a contradicted head absence, and a fixed retention stage each refuse with the exact typed error and an unchanged store witness; runtime RED for the contradiction and stage checks | Implemented for one staged segment per publication; segment rollover is #121 | -| `KEEP-RETENTION-013` | Version-2 segment and catalog stage recovery admits version-2 root entries as known, reuses the version-1 executors, and finalizes a staged `head.next` only after the retained-closure gate admits it | `FilesystemVersionTwoRecoveryAdmission`, `classify_version_two_recovery_names`, and the `open_version_two` constructors; `tests/version_two_catalog_writer.rs` covers classification, completion, discard, reusable-prefix segment resume, and finalization that preserves or would lose retained content (runtime RED with the gate result ignored); `filesystem_version_two_catalog_recovery_tests` recovers injected interruptions after segment link, catalog-stage creation, and `head.next` synchronization | Implemented | -| `KEEP-RETENTION-015` | Process death in the version-2 writer and its recovery reaches documented recoverable states | `cargo xtask durability-crash-matrix --sequence version-two-catalog` runs `KEEP-CRASH-074..=077`, 12 cases, through recovery and durable reads; `KEEP-CRASH-078..=080` (process death inside version-2 recovery) are planned | In progress: writer rows implemented, recovery rows planned | +| `KEEP-RETENTION-012` | Under version-2 writer authority, one staged segment and a catalog successor publish only through the retained-closure gate, with no-follow bounded retention reads, head absence admitted only when uncontradicted, fixed retention stages refused, and no reader fence taken | `FilesystemCatalogPublisher`; `tests/catalog_writer.rs`: a preserving successor publishes, reopens, and reads both blobs while the retention root still verifies; a pinned reader view keeps its generation across publication; a candidate that drops retained content, a contradicted head absence, and a fixed retention stage each refuse with the exact typed error and an unchanged store witness; runtime RED for the contradiction and stage checks | Implemented for one staged segment per publication; segment rollover is #121 | +| `KEEP-RETENTION-013` | Version-2 segment and catalog stage recovery admits version-2 root entries as known, reuses the version-1 executors, and finalizes a staged `head.next` only after the retained-closure gate admits it | `FilesystemRecoveryAdmission`, `classify_recovery_names`, and the `open` constructors; `tests/catalog_writer.rs` covers classification, completion, discard, reusable-prefix segment resume, and finalization that preserves or would lose retained content (runtime RED with the gate result ignored); `filesystem_catalog_recovery_tests` recovers injected interruptions after segment link, catalog-stage creation, and `head.next` synchronization | Implemented | +| `KEEP-RETENTION-015` | Process death in the version-2 writer and its recovery reaches documented recoverable states | `cargo xtask durability-crash-matrix --sequence retained-catalog` runs `KEEP-CRASH-074..=077`, 12 cases, through recovery and durable reads; `KEEP-CRASH-078..=080` (process death inside version-2 recovery) are planned | In progress: writer rows implemented, recovery rows planned | @@ -32,7 +32,7 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | -| `KEEP-FORMAT-001` | `FORMAT` has a complete fixed byte table, named domains, a checksum, the registered definition digest, and exact admission; every admission reads it no-follow, bounded, and refuses trailing bytes | `tests/store_format_marker.rs`; independent golden oracle `xtask/tests/retention_store_v2_format_oracle.rs`; corrupt and oversized marker refusals in `filesystem_version_two_admission_tests` and `verification_store_admission_tests`; seeded `format_marker` fuzz target | Implemented (#198); marker obligation of retired `KEEP-MIGRATION-002` | +| `KEEP-FORMAT-001` | `FORMAT` has a complete fixed byte table, named domains, a checksum, the registered definition digest, and exact admission; every admission reads it no-follow, bounded, and refuses trailing bytes | `tests/store_format_marker.rs`; independent golden oracle `xtask/tests/retention_store_v2_format_oracle.rs`; corrupt and oversized marker refusals in `filesystem_store_admission_tests` and `verification_store_admission_tests`; seeded `format_marker` fuzz target | Implemented (#198); marker obligation of retired `KEEP-MIGRATION-002` | @@ -44,8 +44,8 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-INITIALIZATION-001` | Direct initialization creates exactly the committed namespace under exclusive writer authority, links `FORMAT` from a durable `FORMAT.next` without replacement, and returns authority only after the commit and cleanup synchronizations and complete admission | `a_fresh_root_commits_to_exactly_the_initialized_namespace` in `initialization_resume_tests`; production-profile laws in `tests/single_format.rs` | Implemented (#198) | | `KEEP-INITIALIZATION-002` | Every interrupted state classifies as exactly one prefix and resumes idempotently; every other nonempty root, including a committed one, refuses with typed evidence before any mutation, writer-lock creation included | `every_interrupted_boundary_classifies_and_resumes_to_one_store` (every boundary of a recorded run, against a model of the protocol table) and the incomplete-stage and sync-failure laws in `initialization_resume_tests`; foreign, out-of-order, stray-retention, foreign-stage, nonempty-lock, linked-directory, committed, and copied-marker refusals with unchanged witnesses in `initialization_prefix_tests` | Implemented (#198) | -| `KEEP-INITIALIZATION-003` | Admission requires a committed marker: writers and readers refuse `FORMAT.next` as `InitializationIncomplete`, and every open path (writer, recovery, recovery inventory, catalog restart, retention reader, durable reader) refuses a missing `FORMAT` as `MissingFormatMarker`, so no root without the single format's marker is admitted; a committed store without catalog `HEAD` admits writers, refuses readers as `Unpublished`, and its first publication proves both absences instead of assuming them | `filesystem_version_two_admission_tests`; `every_open_path_refuses_a_root_without_the_format_marker` in `tests/version_two_catalog_writer.rs`; `tests/single_format.rs` initialization laws, including a first publication refused by a stray retention root with an unchanged store | Implemented (#198); every open path since version-1 admission was deleted | -| `KEEP-INITIALIZATION-004` | Process death before, during, and after every initialization boundary reaches a documented state that `initialize` recovers | `cargo xtask durability-crash-matrix --sequence version-two-initialization` runs 41 cases at `KEEP-CRASH-081..=091`, checks the exact inventory, marker bytes and inode, admission result, and resumed state, then publishes, retains, and reconstructs the first blob | Implemented (#198); process death only, not power loss | +| `KEEP-INITIALIZATION-003` | Admission requires a committed marker: writers and readers refuse `FORMAT.next` as `InitializationIncomplete`, and every open path (writer, recovery, recovery inventory, catalog restart, retention reader, durable reader) refuses a missing `FORMAT` as `MissingFormatMarker`, so no root without the single format's marker is admitted; a committed store without catalog `HEAD` admits writers, refuses readers as `Unpublished`, and its first publication proves both absences instead of assuming them | `filesystem_store_admission_tests`; `every_open_path_refuses_a_root_without_the_format_marker` in `tests/catalog_writer.rs`; `tests/single_format.rs` initialization laws, including a first publication refused by a stray retention root with an unchanged store | Implemented (#198); every open path since version-1 admission was deleted | +| `KEEP-INITIALIZATION-004` | Process death before, during, and after every initialization boundary reaches a documented state that `initialize` recovers | `cargo xtask durability-crash-matrix --sequence initialization` runs 41 cases at `KEEP-CRASH-081..=091`, checks the exact inventory, marker bytes and inode, admission result, and resumed state, then publishes, retains, and reconstructs the first blob | Implemented (#198); process death only, not power loss | | `KEEP-INITIALIZATION-005` | Concurrent initialization attempts have exactly one initializer: the other refuses on writer authority and, after commit, as already initialized | `concurrent_attempts_have_exactly_one_initializer` in `initialization_resume_tests` | Implemented (#198) | | `KEEP-INITIALIZATION-006` | One public-API sequence initializes, ingests, reopens, reconstructs exact bytes, retains, publishes a second catalog, interrupts publication, recovers, and reads both retained contents; a candidate that would lose retained content refuses with its evidence intact | `the_single_format_serves_every_operation_from_initialization_to_recovery` in `tests/single_format.rs` | Implemented (#198) | diff --git a/docs/formats/segment-store-v2/retention-publication.md b/docs/formats/segment-store-v2/retention-publication.md index ddae0ac3..929f8ef0 100644 --- a/docs/formats/segment-store-v2/retention-publication.md +++ b/docs/formats/segment-store-v2/retention-publication.md @@ -30,7 +30,7 @@ Version-2 catalog publication must hold the same writer authority and prove ever After `head.next` verification, it rereads the retention head and refuses `HEAD` replacement if the head changed. Its receipt binds the preserved manifest generation and digest. The [format rationale](rationale.md#gate-catalog-publication-on-every-retained-closure) records the crash states and the recovery obligation. -`FilesystemVersionTwoCatalogPublisher` publishes through the gate, and version-2 next-head finalization runs the same retained-closure admission against a staged candidate ([#198](https://github.com/flyingrobots/keep/issues/198)). [Catalog publication and stage recovery](catalog-publication.md) owns that writer and its recovery. Retention publication and complete-stage retention recovery separately reverify a new root's closure against the current catalog. +`FilesystemCatalogPublisher` publishes through the gate, and version-2 next-head finalization runs the same retained-closure admission against a staged candidate ([#198](https://github.com/flyingrobots/keep/issues/198)). [Catalog publication and stage recovery](catalog-publication.md) owns that writer and its recovery. Retention publication and complete-stage retention recovery separately reverify a new root's closure against the current catalog. ## Generation transition diff --git a/docs/formats/segment-store-v2/retired-identifiers.md b/docs/formats/segment-store-v2/retired-identifiers.md index 7ab41257..212b7b49 100644 --- a/docs/formats/segment-store-v2/retired-identifiers.md +++ b/docs/formats/segment-store-v2/retired-identifiers.md @@ -129,7 +129,7 @@ boundary no longer exists is retired here and never reused. | `KEEP-STORE-020` | Initialization is writer-locked, idempotent, recoverable from every partial canonical namespace set, and admitted only after root synchronization | `docs/formats/segment-store-v1/requirements.md` and rows `KEEP-CRASH-031`–`035` at `e03c2a1` | Moved to `KEEP-INITIALIZATION-001`, `-002`, and `-004` | | `KEEP-RECOVERY-002` | Initialization admits the platform, locks the writer file, and admits the three directories in order with exact failure phases | `tests/store_initialization.rs` at `c60def9` | Moved to `KEEP-INITIALIZATION-001` and `-002`; every initialization phase keeps its typed failure | | `KEEP-RECOVERY-003` | Production initialization and published-store reopen admit only the Linux ext4 profile for every protocol directory and the exact version-1 namespace | `src/adapters/filesystem_store_initializer_tests.rs` at `ecf4f7e`; `tests/store_initialization.rs` at `c60def9` | The profile obligation moved to single-format admission: `filesystem_platform_profile` admits every protocol directory, and `initialization_refuses_an_unsupported_platform_before_any_mutation` in `tests/single_format.rs` refuses tmpfs before mutation. The version-1 namespace disappeared | -| `KEEP-RETENTION-014` | Version-1 entry points refuse version-2 roots and version-2 entry points refuse version-1 roots | Never on `main`; introduced and retired within #198 (`tests/version_two_catalog_writer.rs` version laws at `e3930f0`) | The version-1 half disappeared. The version-2 half moved to `KEEP-INITIALIZATION-003`, evidenced by `every_open_path_refuses_a_root_without_the_format_marker` | +| `KEEP-RETENTION-014` | Version-1 entry points refuse version-2 roots and version-2 entry points refuse version-1 roots | Never on `main`; introduced and retired within #198 (`tests/catalog_writer.rs` version laws at `e3930f0`) | The version-1 half disappeared. The version-2 half moved to `KEEP-INITIALIZATION-003`, evidenced by `every_open_path_refuses_a_root_without_the_format_marker` | diff --git a/src/adapters/catalog_restart_error.rs b/src/adapters/catalog_restart_error.rs index 3e345d25..1b868ceb 100644 --- a/src/adapters/catalog_restart_error.rs +++ b/src/adapters/catalog_restart_error.rs @@ -7,8 +7,8 @@ use std::io; use super::{ CatalogAdmissionError, CatalogDecodeError, CatalogRestartArtifact, CatalogRestartPhase, - CatalogSnapshotError, FilesystemPlatformAdmissionError, PublicationHeadDecodeError, - SegmentDigest, SegmentReadError, + CatalogSnapshotError, FilesystemStoreAdmissionError, PublicationHeadDecodeError, SegmentDigest, + SegmentReadError, }; use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; @@ -19,7 +19,7 @@ pub enum CatalogRestartError { /// marker refused before any catalog artifact was opened. Admission { /// Preserved namespace or format-marker refusal. - source: Box, + source: Box, }, /// One capability-relative filesystem operation failed. Io { diff --git a/src/adapters/catalog_restart_loader.rs b/src/adapters/catalog_restart_loader.rs index a0bf6a3e..deed3cd6 100644 --- a/src/adapters/catalog_restart_loader.rs +++ b/src/adapters/catalog_restart_loader.rs @@ -9,8 +9,8 @@ use crate::CatalogLength; use super::{ CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, ChecksummedCatalog, ChecksummedPublicationHead, FilesystemCatalogSnapshot, catalog_restart_io, - catalog_restart_segments, filesystem_initialization_namespace, - filesystem_version_two_admission, physical_pool_name, + catalog_restart_segments, filesystem_store_admission, filesystem_store_namespace, + physical_pool_name, }; const HEAD_NAME: &str = "HEAD"; @@ -28,9 +28,9 @@ pub(super) fn load( policy: CatalogRestartPolicy, ) -> Result { let directory = catalog_restart_io::open_root(root)?; - filesystem_version_two_admission::admit_root( + filesystem_store_admission::admit_root( &directory, - filesystem_initialization_namespace::admit_version_two_with_catalog_stage, + filesystem_store_namespace::admit_store_with_catalog_stage, ) .map_err(|source| CatalogRestartError::Admission { source: Box::new(source), diff --git a/src/adapters/durable/error.rs b/src/adapters/durable/error.rs index 596bfeb1..3301057f 100644 --- a/src/adapters/durable/error.rs +++ b/src/adapters/durable/error.rs @@ -20,7 +20,7 @@ pub enum DurableStoreError { /// Original path-resolution failure, preserved without stringification. source: io::Error, }, - /// The root did not admit as a version-two store, the fence could not + /// The root did not admit as a store, the fence could not /// be taken, or one consistent view could not be collected. Snapshot(Box), /// The pinned catalog could not be re-admitted. diff --git a/src/adapters/durable/snapshot.rs b/src/adapters/durable/snapshot.rs index dca8c1e3..5d3b3fd2 100644 --- a/src/adapters/durable/snapshot.rs +++ b/src/adapters/durable/snapshot.rs @@ -14,7 +14,7 @@ use crate::adapters::{ use crate::authenticated_read::{ChunkSource, read_admitted, reconstruct_admitted}; use crate::{AdmittedLayout, BlobId, ByteRange, ChunkId, LayoutId}; -/// One consistent, fenced view of a version-two store. +/// One consistent, fenced view of a store. /// /// The snapshot owns the shared reader fence for its lifetime, so no /// collector can retire a segment it may read; every read borrows the @@ -72,7 +72,7 @@ impl ChunkSource for CatalogChunks<'_, '_, '_, '_> { } impl DurableSnapshot { - /// Admits `store_root` as a version-two store, acquires the shared + /// Admits `store_root` as a store, acquires the shared /// reader fence, double-collects one consistent view within `limit` /// attempts, and verifies every retained root's closure. /// diff --git a/src/adapters/durable/store.rs b/src/adapters/durable/store.rs index ee4a23fe..cede95ef 100644 --- a/src/adapters/durable/store.rs +++ b/src/adapters/durable/store.rs @@ -1,4 +1,4 @@ -//! This boundary module owns the durable store handle: a version-two root +//! This boundary module owns the durable store handle: a store root //! that pins snapshots on demand. use std::io::Write; @@ -11,7 +11,7 @@ use super::{ use crate::adapters::{CatalogRestartPolicy, ReaderAttemptLimit}; use crate::{BlobId, ByteRange, LayoutId}; -/// One committed version-two store to read from. +/// One committed store to read from. /// /// The handle holds no fence and no view; every read pins a fresh /// [`DurableSnapshot`] unless the caller pins one with [`Self::snapshot`] @@ -27,7 +27,7 @@ use crate::{BlobId, ByteRange, LayoutId}; /// /// # Example /// -/// On Linux, read an initialized version-two store with a retained blob. +/// On Linux, read an initialized store with a retained blob. /// The caller owns output visibility: a failed write can leave an untrusted /// prefix. The receipt names the view; it does not extend retention after the /// snapshot is dropped. This example limits aggregate catalog-selected segment bytes; diff --git a/src/adapters/exports.rs b/src/adapters/exports.rs index 027d4295..41d43000 100644 --- a/src/adapters/exports.rs +++ b/src/adapters/exports.rs @@ -37,8 +37,9 @@ pub use super::checksummed_publication_head::ChecksummedPublicationHead; pub use super::checksummed_segment_record::ChecksummedSegmentRecord; pub use super::closed_segment::ClosedSegment; pub use super::filesystem_catalog_publication_error::FilesystemCatalogPublicationError; +pub use super::filesystem_catalog_publisher::FilesystemCatalogPublisher; pub use super::filesystem_catalog_snapshot::FilesystemCatalogSnapshot; -pub use super::filesystem_platform_admission_error::FilesystemPlatformAdmissionError; +pub use super::filesystem_recovery_admission::FilesystemRecoveryAdmission; pub use super::filesystem_recovery_inventory_reader::FilesystemRecoveryInventoryReader; pub use super::filesystem_recovery_next_head_finalization_open_error::FilesystemRecoveryNextHeadFinalizationOpenError; pub use super::filesystem_recovery_next_head_finalizer::FilesystemRecoveryNextHeadFinalizer; @@ -52,16 +53,15 @@ pub use super::filesystem_recovery_stage_discarder::FilesystemRecoveryStageDisca pub use super::filesystem_recovery_stage_error::{ FilesystemRecoveryStageError, RecoveryStageNamespacePhase, }; -pub use super::filesystem_segment_stage::FilesystemSegmentStage; -pub use super::filesystem_version_two_admission::FilesystemVersionTwoAdmission; -pub use super::filesystem_version_two_catalog_publisher::FilesystemVersionTwoCatalogPublisher; -pub use super::filesystem_version_two_catalog_stages::FilesystemVersionTwoCatalogStages; +pub use super::filesystem_retained_catalog_stages::FilesystemRetainedCatalogStages; #[cfg(any(test, feature = "repository-tasks"))] -pub use super::filesystem_version_two_catalog_stages::{ +pub use super::filesystem_retained_catalog_stages::{ CatalogPublicationBoundary, CatalogPublicationObserver, }; -pub use super::filesystem_version_two_record_refusal::VersionTwoRecordRefusal; -pub use super::filesystem_version_two_recovery_admission::FilesystemVersionTwoRecoveryAdmission; +pub use super::filesystem_segment_stage::FilesystemSegmentStage; +pub use super::filesystem_store_admission::FilesystemStoreAdmission; +pub use super::filesystem_store_admission_error::FilesystemStoreAdmissionError; +pub use super::filesystem_store_record_refusal::StoreRecordRefusal; pub use super::filesystem_writer_lock::FilesystemWriterLock; pub use super::layout_decode_error::LayoutDecodeError; pub use super::layout_decode_policy::LayoutDecodePolicy; @@ -109,11 +109,11 @@ pub use super::staged_segment::StagedSegment; pub use super::storage_profile_id_text_error::StorageProfileIdParseError; pub use super::store_format::*; #[cfg(any(test, feature = "repository-tasks"))] -pub use super::version_two_initialization::VersionTwoInitializationObserver; -pub use super::version_two_initialization::{ - VersionTwoInitialization, VersionTwoInitializationBoundary, VersionTwoInitializationDirectory, - VersionTwoInitializationError, VersionTwoInitializationPhase, VersionTwoInitializationPrefix, - VersionTwoInitializationRefusal, +pub use super::store_initialization::StoreInitializationObserver; +pub use super::store_initialization::{ + StoreInitialization, StoreInitializationBoundary, StoreInitializationDirectory, + StoreInitializationError, StoreInitializationPhase, StoreInitializationPrefix, + StoreInitializationRefusal, }; pub use super::writer_lock_acquire_error::WriterLockAcquireError; pub use super::writer_lock_acquire_phase::WriterLockAcquirePhase; diff --git a/src/adapters/filesystem_version_two_catalog_publisher.rs b/src/adapters/filesystem_catalog_publisher.rs similarity index 81% rename from src/adapters/filesystem_version_two_catalog_publisher.rs rename to src/adapters/filesystem_catalog_publisher.rs index 079c833b..be3618f7 100644 --- a/src/adapters/filesystem_version_two_catalog_publisher.rs +++ b/src/adapters/filesystem_catalog_publisher.rs @@ -1,22 +1,22 @@ -//! This module owns writer-locked version-two catalog publication state. +//! This module owns writer-locked catalog publication state. use std::io; use super::filesystem_catalog_stages::FilesystemCatalogStages; #[cfg(feature = "repository-tasks")] -use super::filesystem_version_two_catalog_stages::CatalogPublicationObserver; +use super::filesystem_retained_catalog_stages::CatalogPublicationObserver; use super::retention::FilesystemRetainedEvidence; use super::{ - AdmittedSegment, CatalogRestartPolicy, FilesystemSegmentStage, FilesystemVersionTwoAdmission, - FilesystemVersionTwoCatalogStages, FilesystemWriterLock, SealedSegment, SegmentPublication, + AdmittedSegment, CatalogRestartPolicy, FilesystemRetainedCatalogStages, FilesystemSegmentStage, + FilesystemStoreAdmission, FilesystemWriterLock, SealedSegment, SegmentPublication, SegmentPublicationError, SegmentStageCreateError, }; -/// Exclusive version-two authority to ingest one segment and publish a catalog. +/// Exclusive store authority to ingest one segment and publish a catalog. /// /// The publisher owns the writer lock, the pinned store root, the catalog /// protocol directories, and the `retention`, `retention/roots`, and -/// `retention/manifests` capabilities that version-two admission pinned. It +/// `retention/manifests` capabilities that store admission pinned. It /// publishes only through /// [`publish_retained_catalog_generation`](crate::publish_retained_catalog_generation): /// it implements [`RetainedCatalogPublicationStorage`](crate::RetainedCatalogPublicationStorage) @@ -25,7 +25,7 @@ use super::{ /// /// ```compile_fail /// fn publish( -/// mut publisher: keep::FilesystemVersionTwoCatalogPublisher, +/// mut publisher: keep::FilesystemCatalogPublisher, /// catalog: &keep::CanonicalCatalog, /// ) { /// let _ = keep::publish_catalog_generation( @@ -43,24 +43,24 @@ use super::{ /// before releasing the writer lock, but never publishes, removes, truncates, /// or repairs protocol state. #[must_use] -pub struct FilesystemVersionTwoCatalogPublisher { +pub struct FilesystemCatalogPublisher { pub(super) evidence: FilesystemRetainedEvidence, - pub(super) stages: FilesystemVersionTwoCatalogStages, + pub(super) stages: FilesystemRetainedCatalogStages, // Fields drop in declaration order. Writer authority must outlive every // directory capability and retained writable stage. _lock: FilesystemWriterLock, } -impl FilesystemVersionTwoCatalogPublisher { +impl FilesystemCatalogPublisher { /// Pins the catalog and retention directories under admitted writer authority. /// - /// Only [`FilesystemVersionTwoAdmission`] is accepted, so a bare writer + /// Only [`FilesystemStoreAdmission`] is accepted, so a bare writer /// lock, which proves neither the namespace nor the `FORMAT` marker, cannot /// reach this publisher: /// /// ```compile_fail /// fn open(lock: keep::FilesystemWriterLock, policy: keep::CatalogRestartPolicy) { - /// let _ = keep::FilesystemVersionTwoCatalogPublisher::open(lock, policy); + /// let _ = keep::FilesystemCatalogPublisher::open(lock, policy); /// } /// ``` /// @@ -73,7 +73,7 @@ impl FilesystemVersionTwoCatalogPublisher { /// returns [`io::ErrorKind::NotADirectory`]. A failure releases the writer /// lock. pub fn open( - admission: FilesystemVersionTwoAdmission, + admission: FilesystemStoreAdmission, policy: CatalogRestartPolicy, ) -> io::Result { let (lock, retention, roots, manifests) = admission.into_parts(); @@ -81,7 +81,7 @@ impl FilesystemVersionTwoCatalogPublisher { let stages = FilesystemCatalogStages::open(&pinned_root, policy)?; Ok(Self { evidence: FilesystemRetainedEvidence::new(pinned_root, retention, roots, manifests), - stages: FilesystemVersionTwoCatalogStages::new(stages), + stages: FilesystemRetainedCatalogStages::new(stages), _lock: lock, }) } diff --git a/src/adapters/filesystem_version_two_catalog_storage.rs b/src/adapters/filesystem_catalog_publisher_storage.rs similarity index 62% rename from src/adapters/filesystem_version_two_catalog_storage.rs rename to src/adapters/filesystem_catalog_publisher_storage.rs index e7f58241..1807c66a 100644 --- a/src/adapters/filesystem_version_two_catalog_storage.rs +++ b/src/adapters/filesystem_catalog_publisher_storage.rs @@ -1,15 +1,15 @@ -//! This module binds version-two writer authority to the retained-closure port. +//! This module binds writer authority to the retained-closure port. use std::io; use super::{ - FilesystemVersionTwoCatalogPublisher, FilesystemVersionTwoCatalogStages, - RetainedCatalogPublicationStorage, RetainedClosureAdmission, + FilesystemCatalogPublisher, FilesystemRetainedCatalogStages, RetainedCatalogPublicationStorage, + RetainedClosureAdmission, }; use crate::{RetentionHead, RetentionManifestEntry}; -impl RetainedCatalogPublicationStorage for FilesystemVersionTwoCatalogPublisher { - type Stages = FilesystemVersionTwoCatalogStages; +impl RetainedCatalogPublicationStorage for FilesystemCatalogPublisher { + type Stages = FilesystemRetainedCatalogStages; fn read_retention_head(&mut self) -> io::Result>> { self.evidence.read_head() diff --git a/src/adapters/filesystem_catalog_publisher_tests.rs b/src/adapters/filesystem_catalog_publisher_tests.rs index b84e1f05..fe96742e 100644 --- a/src/adapters/filesystem_catalog_publisher_tests.rs +++ b/src/adapters/filesystem_catalog_publisher_tests.rs @@ -18,9 +18,9 @@ use super::test_support::decode_hex; use crate::{ AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationOutcome, - CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogSnapshot, - FilesystemPlatformAdmissionError, FilesystemSegmentStage, FilesystemVersionTwoAdmission, - FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, RetainedCatalogPublicationError, + CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, + FilesystemCatalogSnapshot, FilesystemSegmentStage, FilesystemStoreAdmission, + FilesystemStoreAdmissionError, LayoutEntryLimit, RetainedCatalogPublicationError, SealedSegment, SegmentPublication, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, publish_retained_catalog_generation, }; @@ -111,11 +111,7 @@ struct StoreFixture { impl StoreFixture { fn create(name: &str) -> Result> { let sandbox = TestDirectory::create(name)?; - drop( - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( - sandbox.path(), - )?, - ); + drop(FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(sandbox.path())?); let segments = sandbox.path().join("segments"); let catalogs = sandbox.path().join("catalogs"); Ok(Self { @@ -149,15 +145,13 @@ impl StoreFixture { /// Reacquires single-format writer authority over the initialized store. fn admit_writer( store: &StoreFixture, -) -> Result { - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store.path()) +) -> Result { + FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(store.path()) } /// Reopens the initialized store and pins its single-format writer. -fn open_publisher( - store: &StoreFixture, -) -> Result> { - Ok(FilesystemVersionTwoCatalogPublisher::open( +fn open_publisher(store: &StoreFixture) -> Result> { + Ok(FilesystemCatalogPublisher::open( admit_writer(store)?, restart_policy()?, )?) @@ -181,7 +175,7 @@ fn restart_policy() -> Result> { } fn stage_one_zero<'publisher>( - publisher: &'publisher FilesystemVersionTwoCatalogPublisher, + publisher: &'publisher FilesystemCatalogPublisher, store: &StoreFixture, ) -> Result, Box> { let stage = publisher.create_segment_stage()?; @@ -204,9 +198,9 @@ fn fixture(hex: &str) -> Result, Box> { #[test] fn publisher_drop_closes_writable_stages_before_releasing_writer_authority() -> Result<(), Box> { - let source = include_str!("filesystem_version_two_catalog_publisher.rs"); + let source = include_str!("filesystem_catalog_publisher.rs"); let stages = source - .find("pub(super) stages: FilesystemVersionTwoCatalogStages,") + .find("pub(super) stages: FilesystemRetainedCatalogStages,") .ok_or("publisher must retain the gated catalog stages")?; let writer_lock = source .find(" _lock: FilesystemWriterLock,") diff --git a/src/adapters/filesystem_version_two_catalog_recovery_tests.rs b/src/adapters/filesystem_catalog_recovery_tests.rs similarity index 89% rename from src/adapters/filesystem_version_two_catalog_recovery_tests.rs rename to src/adapters/filesystem_catalog_recovery_tests.rs index c6125964..818040a3 100644 --- a/src/adapters/filesystem_version_two_catalog_recovery_tests.rs +++ b/src/adapters/filesystem_catalog_recovery_tests.rs @@ -1,7 +1,7 @@ -//! Interrupted version-two publication and its stage recovery. +//! Interrupted publication and its stage recovery. //! -//! Each law fails the real version-two writer at one storage-port boundary, -//! then recovers with the public version-two recovery tools and requires the +//! Each law fails the real writer at one storage-port boundary, +//! then recovers with the public recovery tools and requires the //! store to read both blobs exactly with every retained root preserved. #[path = "../../tests/golden_file_worldline/durable_fixture.rs"] @@ -10,7 +10,7 @@ reason = "interruption laws use complete stores; golden laws cover partial records" )] mod durable_fixture; -#[path = "../../tests/version_two_catalog_writer/recovery_fixture.rs"] +#[path = "../../tests/catalog_writer/recovery_fixture.rs"] #[allow( dead_code, reason = "interruption laws finalize and complete; the public suite also inspects roles" @@ -22,7 +22,7 @@ mod recovery_fixture; reason = "interruption laws publish and republish; the public suite also stages by hand" )] mod successor_catalog; -#[path = "../../tests/version_two_catalog_writer/writer_fixture.rs"] +#[path = "../../tests/catalog_writer/writer_fixture.rs"] #[allow( dead_code, reason = "interruption laws ingest and retain; the public suite also stages by hand" @@ -37,14 +37,14 @@ use super::populated_store; use crate::{CatalogPublicationError, CatalogPublicationPhase, RetainedCatalogPublicationError}; -use super::filesystem_version_two_catalog_stages::CatalogPublicationBoundary; +use super::filesystem_retained_catalog_stages::CatalogPublicationBoundary; use recovery_fixture::{complete, discard, finalize, stage_roles}; use writer_fixture::{ Predecessor, TestResult, assert_reads, first_blob, open_publisher, pool_segments, publish, republish, retain, second_blob, }; -const INJECTED: &str = "injected version-two publication interruption"; +const INJECTED: &str = "injected publication interruption"; /// Publishes the second blob, failing after `phase` completes. fn interrupted_after(root: &Path, phase: CatalogPublicationPhase) -> TestResult<()> { diff --git a/src/adapters/filesystem_format_marker.rs b/src/adapters/filesystem_format_marker.rs index bf86aad8..489ac005 100644 --- a/src/adapters/filesystem_format_marker.rs +++ b/src/adapters/filesystem_format_marker.rs @@ -4,19 +4,19 @@ use std::io; use cap_std::fs::Dir; -use super::VersionTwoRecordRefusal as Refusal; +use super::StoreRecordRefusal as Refusal; use super::filesystem_exact_record::{self as exact_record, ExactRecordError, ExactRecordRefusal}; use super::{AdmittedStoreFormatMarker, FORMAT_MARKER_LENGTH}; const MARKER_NAME: &str = "FORMAT"; -/// Reads and admits `FORMAT` as the one registered version-two marker. +/// Reads and admits `FORMAT` as the one registered format marker. /// /// The record is reopened without following links, bounded to its exact -/// canonical length, and decoded. Writer authority over a version-two root +/// canonical length, and decoded. Writer authority over a store root /// must not be returned before this admission succeeds. Every refusal is an /// `InvalidData` error whose source is a -/// [`VersionTwoRecordRefusal`](super::VersionTwoRecordRefusal). Nothing about +/// [`StoreRecordRefusal`](super::StoreRecordRefusal). Nothing about /// the root directory's own identity is compared: the marker binds the format /// contract, not a location. pub(super) fn admit(root: &Dir) -> io::Result<()> { diff --git a/src/adapters/filesystem_namespace_refusal.rs b/src/adapters/filesystem_namespace_refusal.rs index 36f27d4f..bb4ed27a 100644 --- a/src/adapters/filesystem_namespace_refusal.rs +++ b/src/adapters/filesystem_namespace_refusal.rs @@ -39,10 +39,10 @@ impl FilesystemEntryKind { pub enum FilesystemNamespaceRefusal { /// An observed entry is outside the admitted protocol membership. UnexpectedEntry, - /// The root holds no `FORMAT` marker, so it is not a committed version-two store. + /// The root holds no `FORMAT` marker, so it is not a committed store. MissingFormatMarker, /// The root still holds `FORMAT.next`: initialization has not committed. - /// Resume it with `FilesystemVersionTwoAdmission::initialize`. + /// Resume it with `FilesystemStoreAdmission::initialize`. InitializationIncomplete, /// A no-follow metadata observation established the wrong entry kind. WrongKind { diff --git a/src/adapters/filesystem_platform_profile.rs b/src/adapters/filesystem_platform_profile.rs index c06f5321..669e5313 100644 --- a/src/adapters/filesystem_platform_profile.rs +++ b/src/adapters/filesystem_platform_profile.rs @@ -5,13 +5,13 @@ use std::path::Path; use cap_std::fs::Dir; -/// Every version-two protocol directory, including nested immutable pools. +/// Every protocol directory, including nested immutable pools. /// -/// A version-two root receives writer authority only when each of these shares +/// A store root receives writer authority only when each of these shares /// the root's filesystem type, device, and mount identity and is not /// casefolded or read-only. #[cfg(target_os = "linux")] -const VERSION_TWO_PROTOCOL_DIRECTORIES: [&str; 9] = [ +const PROTOCOL_DIRECTORIES: [&str; 9] = [ "staging", "segments", "catalogs", @@ -34,13 +34,13 @@ struct LinuxDirectoryProperties { mount_id: u64, } -/// Opens one version-two store root under the admitted Linux profile. +/// Opens one store root under the admitted Linux profile. /// /// The root is opened without following links, and every protocol directory /// that exists must satisfy the same filesystem, mount, and inode-flag laws as /// the root; absence is left to namespace admission. #[cfg(target_os = "linux")] -pub(super) fn open_version_two(store_root: &Path) -> io::Result { +pub(super) fn open(store_root: &Path) -> io::Result { use std::fs::File; use rustix::fs::{CWD, Mode, OFlags, ResolveFlags, openat2}; @@ -53,15 +53,15 @@ pub(super) fn open_version_two(store_root: &Path) -> io::Result { ResolveFlags::NO_MAGICLINKS | ResolveFlags::NO_SYMLINKS, )?; let directory = Dir::from_std_file(File::from(descriptor)); - admit_linux_profile(&directory, &VERSION_TWO_PROTOCOL_DIRECTORIES)?; + admit_linux_profile(&directory, &PROTOCOL_DIRECTORIES)?; Ok(directory) } #[cfg(not(target_os = "linux"))] -pub(super) fn open_version_two(_store_root: &Path) -> io::Result { +pub(super) fn open(_store_root: &Path) -> io::Result { Err(io::Error::new( io::ErrorKind::Unsupported, - "version-two reopen currently requires the admitted Linux ext4 profile", + "store reopen currently requires the admitted Linux ext4 profile", )) } diff --git a/src/adapters/filesystem_platform_profile_tests.rs b/src/adapters/filesystem_platform_profile_tests.rs index 81994487..dc4a3209 100644 --- a/src/adapters/filesystem_platform_profile_tests.rs +++ b/src/adapters/filesystem_platform_profile_tests.rs @@ -1,7 +1,7 @@ //! Linux filesystem platform-profile laws. use super::{ - LinuxDirectoryProperties, VERSION_TWO_PROTOCOL_DIRECTORIES, admit_linux_child_properties, + LinuxDirectoryProperties, PROTOCOL_DIRECTORIES, admit_linux_child_properties, admit_linux_properties, }; use rustix::fs::{NFS_SUPER_MAGIC, StatVfsMountFlags}; @@ -75,7 +75,7 @@ const fn properties( #[test] fn admission_covers_every_protocol_directory() { assert_eq!( - VERSION_TWO_PROTOCOL_DIRECTORIES, + PROTOCOL_DIRECTORIES, [ "staging", "segments", diff --git a/src/adapters/filesystem_version_two_recovery_admission.rs b/src/adapters/filesystem_recovery_admission.rs similarity index 61% rename from src/adapters/filesystem_version_two_recovery_admission.rs rename to src/adapters/filesystem_recovery_admission.rs index 5641b2da..77f2ff51 100644 --- a/src/adapters/filesystem_version_two_recovery_admission.rs +++ b/src/adapters/filesystem_recovery_admission.rs @@ -1,4 +1,4 @@ -//! This module owns proof that a version-two root admits catalog-stage recovery. +//! This module owns proof that a store root admits catalog-stage recovery. use std::path::Path; @@ -6,10 +6,9 @@ use std::path::Path; use cap_std::ambient_authority; use cap_std::fs::Dir; -use super::filesystem_version_two_admission::{AdmittedVersionTwoParts, admit_locked}; +use super::filesystem_store_admission::{AdmittedStoreParts, admit_locked}; use super::{ - FilesystemPlatformAdmissionError, filesystem_initialization_namespace, - filesystem_platform_profile, + FilesystemStoreAdmissionError, filesystem_platform_profile, filesystem_store_namespace, }; /// Exclusive writer authority over a committed root for stage recovery. @@ -25,23 +24,23 @@ use super::{ /// Every recovery tool that mutates a stage consumes this value, so recovery /// never runs against a root without the marker. #[must_use] -pub struct FilesystemVersionTwoRecoveryAdmission { - parts: AdmittedVersionTwoParts, +pub struct FilesystemRecoveryAdmission { + parts: AdmittedStoreParts, } -impl FilesystemVersionTwoRecoveryAdmission { +impl FilesystemRecoveryAdmission { /// Acquires writer authority over one committed store for stage recovery. /// /// The call mutates no protocol state and may block on filesystem I/O. /// /// # Errors /// - /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, + /// Returns [`FilesystemStoreAdmissionError`] with the exact platform, /// writer-lock, namespace, or format-marker boundary and preserved /// source. - pub fn reopen(store_root: &Path) -> Result { - let root = filesystem_platform_profile::open_version_two(store_root) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + pub fn reopen(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open(store_root) + .map_err(|source| FilesystemStoreAdmissionError::Platform { source })?; Self::admit(root) } @@ -52,26 +51,26 @@ impl FilesystemVersionTwoRecoveryAdmission { /// /// # Errors /// - /// Returns [`FilesystemPlatformAdmissionError`] exactly as [`Self::reopen`] + /// Returns [`FilesystemStoreAdmissionError`] exactly as [`Self::reopen`] /// does for every boundary after platform admission. #[cfg(any(test, feature = "repository-tasks"))] pub fn reopen_unchecked_for_repository_tasks( store_root: &Path, - ) -> Result { + ) -> Result { let root = Dir::open_ambient_dir(store_root, ambient_authority()) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + .map_err(|source| FilesystemStoreAdmissionError::Platform { source })?; Self::admit(root) } /// Releases the writer lock and the three pinned retention capabilities. - pub(super) fn into_parts(self) -> AdmittedVersionTwoParts { + pub(super) fn into_parts(self) -> AdmittedStoreParts { self.parts } - fn admit(root: Dir) -> Result { + fn admit(root: Dir) -> Result { admit_locked( root, - filesystem_initialization_namespace::admit_version_two_with_catalog_stage, + filesystem_store_namespace::admit_store_with_catalog_stage, ) .map(|parts| Self { parts }) } diff --git a/src/adapters/filesystem_recovery_inventory_reader.rs b/src/adapters/filesystem_recovery_inventory_reader.rs index 91764be8..9c2e8a19 100644 --- a/src/adapters/filesystem_recovery_inventory_reader.rs +++ b/src/adapters/filesystem_recovery_inventory_reader.rs @@ -7,15 +7,14 @@ use std::path::Path; use cap_std::ambient_authority; use cap_std::fs::Dir; -use super::filesystem_version_two_admission::admit_root; +use super::filesystem_store_admission::admit_root; use super::{ FilesystemRecoveryStageError, RecoveryEntryName, RecoveryInventory, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryOperation, RecoveryInventoryStorage, RecoveryNamespace, RecoveryStage, RecoveryStageCompletionPool, RecoveryStageEvidence, - RecoveryStageNamespacePhase, RecoveryStageParent, filesystem_initialization_namespace, - filesystem_platform_profile, filesystem_recovery_inventory_scan, - filesystem_recovery_namespace::PinnedRecoveryDirectory, filesystem_recovery_stage, - read_recovery_inventory, + RecoveryStageNamespacePhase, RecoveryStageParent, filesystem_platform_profile, + filesystem_recovery_inventory_scan, filesystem_recovery_namespace::PinnedRecoveryDirectory, + filesystem_recovery_stage, filesystem_store_namespace, read_recovery_inventory, }; const STAGING_NAME: &str = "staging"; @@ -44,19 +43,19 @@ impl FilesystemRecoveryInventoryReader { /// optional retained catalog `head.next`, and the canonical `FORMAT` marker. /// No writer lock is taken /// and no protocol state is mutated. Classify the resulting inventory with - /// [`classify_version_two_recovery_names`](crate::classify_version_two_recovery_names). + /// [`classify_recovery_names`](crate::classify_recovery_names). /// /// # Errors /// /// Returns [`RecoveryInventoryError::Io`] for the store root when platform - /// or version-two admission refuses; the source carries the exact - /// [`FilesystemPlatformAdmissionError`](crate::FilesystemPlatformAdmissionError). + /// or store admission refuses; the source carries the exact + /// [`FilesystemStoreAdmissionError`](crate::FilesystemStoreAdmissionError). /// A root without the `FORMAT` marker refuses. - pub fn open_version_two(store_root: &Path) -> Result { - let root = filesystem_platform_profile::open_version_two(store_root).map_err(root_error)?; + pub fn open(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open(store_root).map_err(root_error)?; admit_root( &root, - filesystem_initialization_namespace::admit_version_two_with_catalog_stage, + filesystem_store_namespace::admit_store_with_catalog_stage, ) .map_err(|source| root_error(io::Error::new(io::ErrorKind::InvalidData, source)))?; Self::pin(root) @@ -64,30 +63,29 @@ impl FilesystemRecoveryInventoryReader { /// Opens a committed root without platform admission for repository tasks. /// - /// Every namespace, record, and identity law of [`Self::open_version_two`] + /// Every namespace, record, and identity law of [`Self::open`] /// still applies. /// /// # Errors /// - /// As [`Self::open_version_two`] after platform admission. + /// As [`Self::open`] after platform admission. #[cfg(any(test, feature = "repository-tasks"))] #[doc(hidden)] - pub fn open_version_two_unchecked_for_repository_tasks( + pub fn open_unchecked_for_repository_tasks( store_root: &Path, ) -> Result { let root = Dir::open_ambient_dir(store_root, ambient_authority()).map_err(root_error)?; admit_root( &root, - filesystem_initialization_namespace::admit_version_two_with_catalog_stage, + filesystem_store_namespace::admit_store_with_catalog_stage, ) .map_err(|source| root_error(io::Error::new(io::ErrorKind::InvalidData, source)))?; Self::pin(root) } - /// Pins an already admitted, writer-locked version-two root. - pub(super) fn from_version_two_root(root: Dir) -> Result { - filesystem_initialization_namespace::admit_version_two_with_catalog_stage(&root) - .map_err(root_error)?; + /// Pins an already admitted, writer-locked store root. + pub(super) fn from_admitted_root(root: Dir) -> Result { + filesystem_store_namespace::admit_store_with_catalog_stage(&root).map_err(root_error)?; Self::pin(root) } diff --git a/src/adapters/filesystem_recovery_inventory_tests.rs b/src/adapters/filesystem_recovery_inventory_tests.rs index dd6c9bbe..54b2df3f 100644 --- a/src/adapters/filesystem_recovery_inventory_tests.rs +++ b/src/adapters/filesystem_recovery_inventory_tests.rs @@ -9,8 +9,8 @@ use std::os::unix::ffi::OsStringExt; use super::filesystem_test_sandbox::TestDirectory; use super::{ - FilesystemEntryKind, FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError, - FilesystemRecoveryInventoryReader, FilesystemVersionTwoAdmission, RecoveryInventoryError, + FilesystemEntryKind, FilesystemNamespaceRefusal, FilesystemRecoveryInventoryReader, + FilesystemStoreAdmission, FilesystemStoreAdmissionError, RecoveryInventoryError, RecoveryInventoryLimit, RecoveryInventoryOperation, RecoveryNamespace, }; @@ -99,13 +99,12 @@ fn filesystem_inventory_never_follows_protocol_directory_links() -> Result<(), B let sandbox = TestDirectory::create("recovery-inventory-symlink")?; let store = sandbox.path().join("store"); fs::create_dir(&store)?; - drop(FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(&store)?); + drop(FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(&store)?); fs::create_dir(sandbox.path().join("target"))?; fs::remove_dir(store.join(STAGING_NAME))?; symlink("../target", store.join(STAGING_NAME))?; - let Err(error) = - FilesystemRecoveryInventoryReader::open_version_two_unchecked_for_repository_tasks(&store) + let Err(error) = FilesystemRecoveryInventoryReader::open_unchecked_for_repository_tasks(&store) else { return Err("filesystem inventory followed a staging-directory link".into()); }; @@ -157,24 +156,22 @@ fn replaced_protocol_directory_refuses_the_pinned_inventory() -> Result<(), Box< /// A store committed by direct initialization. fn initialized_namespace(name: &str) -> Result> { let sandbox = TestDirectory::create(name)?; - drop(FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(sandbox.path())?); + drop(FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(sandbox.path())?); Ok(sandbox) } fn open_reader( sandbox: &TestDirectory, ) -> Result { - FilesystemRecoveryInventoryReader::open_version_two_unchecked_for_repository_tasks( - sandbox.path(), - ) + FilesystemRecoveryInventoryReader::open_unchecked_for_repository_tasks(sandbox.path()) } /// The namespace refusal one root admission failure carries, if any. fn namespace_refusal(source: &std::io::Error) -> Option { let admission = source .get_ref()? - .downcast_ref::()?; - let FilesystemPlatformAdmissionError::Namespace { source } = admission else { + .downcast_ref::()?; + let FilesystemStoreAdmissionError::Namespace { source } = admission else { return None; }; source diff --git a/src/adapters/filesystem_recovery_next_head_finalization_tests.rs b/src/adapters/filesystem_recovery_next_head_finalization_tests.rs index f6d439a0..1810b965 100644 --- a/src/adapters/filesystem_recovery_next_head_finalization_tests.rs +++ b/src/adapters/filesystem_recovery_next_head_finalization_tests.rs @@ -57,7 +57,7 @@ fn one_writer_excludes_a_second_finalizer() -> Result<(), Box> { assert!(matches!( error, - super::FilesystemPlatformAdmissionError::WriterLock { + super::FilesystemStoreAdmissionError::WriterLock { source: super::WriterLockAcquireError::Busy, } )); diff --git a/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs b/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs index 07fabd8d..de300a9b 100644 --- a/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs +++ b/src/adapters/filesystem_recovery_next_head_finalization_tests/fixture.rs @@ -8,9 +8,8 @@ use crate::LayoutEntryLimit; use super::super::{ AdmittedSegment, CatalogPublicationExpectation, CatalogRestartByteLimit, CatalogRestartPolicy, - CatalogSnapshot, ChecksummedCatalog, ChecksummedPublicationHead, - FilesystemPlatformAdmissionError, FilesystemRecoveryNextHeadFinalizer, - FilesystemVersionTwoAdmission, FilesystemVersionTwoRecoveryAdmission, + CatalogSnapshot, ChecksummedCatalog, ChecksummedPublicationHead, FilesystemRecoveryAdmission, + FilesystemRecoveryNextHeadFinalizer, FilesystemStoreAdmission, FilesystemStoreAdmissionError, RecoveryNextHeadFinalizationRequest, RecoveryStage, RecoveryStageMetadata, SegmentReadPolicy, SegmentRecordLimit, admit_recovery_stage_bytes, assess_recovery_stage, filesystem_test_sandbox::TestDirectory, fingerprint_recovery_stage, physical_pool_name, @@ -36,9 +35,7 @@ impl FinalizationFixture { pub(super) fn new(name: &str) -> Result> { let directory = TestDirectory::create(name)?; drop( - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( - directory.path(), - )?, + FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(directory.path())?, ); Ok(Self { directory }) } @@ -107,12 +104,12 @@ impl FinalizationFixture { /// Acquires single-format recovery admission, the writer authority finalization runs under. pub(super) fn recovery_admission( &self, - ) -> Result { - FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) + ) -> Result { + FilesystemRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) } pub(super) fn finalizer(&self) -> Result> { - Ok(FilesystemRecoveryNextHeadFinalizer::open_version_two( + Ok(FilesystemRecoveryNextHeadFinalizer::open( self.recovery_admission()?, policy()?, )?) diff --git a/src/adapters/filesystem_recovery_next_head_finalizer.rs b/src/adapters/filesystem_recovery_next_head_finalizer.rs index 64d05a95..3a36c887 100644 --- a/src/adapters/filesystem_recovery_next_head_finalizer.rs +++ b/src/adapters/filesystem_recovery_next_head_finalizer.rs @@ -4,9 +4,9 @@ use cap_std::fs::Dir; use super::retention::FilesystemRetainedEvidence; use super::{ - CatalogRestartPolicy, FilesystemRecoveryNextHeadFinalizationOpenError, - FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, - FilesystemVersionTwoRecoveryAdmission, RecoveryStageParent, + CatalogRestartPolicy, FilesystemRecoveryAdmission, + FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryStageDiscardOpenError, + FilesystemRecoveryStageDiscarder, RecoveryStageParent, }; /// Writer-authorized pinned filesystem adapter for exact next-head finalization. @@ -40,15 +40,15 @@ impl FilesystemRecoveryNextHeadFinalizer { /// /// Returns [`FilesystemRecoveryNextHeadFinalizationOpenError`] on root-clone /// or namespace pinning failure. - pub fn open_version_two( - admission: FilesystemVersionTwoRecoveryAdmission, + pub fn open( + admission: FilesystemRecoveryAdmission, policy: CatalogRestartPolicy, ) -> Result { let (lock, retention, roots, manifests) = admission.into_parts(); let root = lock .clone_directory() .map_err(|source| FilesystemRecoveryStageDiscardOpenError::CloneRoot { source })?; - let discarder = FilesystemRecoveryStageDiscarder::from_version_two_lock(lock)?; + let discarder = FilesystemRecoveryStageDiscarder::from_lock(lock)?; Ok(Self { discarder, policy, diff --git a/src/adapters/filesystem_recovery_segment_resume_tests.rs b/src/adapters/filesystem_recovery_segment_resume_tests.rs index 8aff8679..3de5c85c 100644 --- a/src/adapters/filesystem_recovery_segment_resume_tests.rs +++ b/src/adapters/filesystem_recovery_segment_resume_tests.rs @@ -4,8 +4,8 @@ use std::error::Error; use std::fs; use super::{ - AdmittedSegment, AdmittedSegmentRecord, FilesystemPlatformAdmissionError, - WriterLockAcquireError, execute_recovery_segment_resume, + AdmittedSegment, AdmittedSegmentRecord, FilesystemStoreAdmissionError, WriterLockAcquireError, + execute_recovery_segment_resume, }; mod fixture; @@ -50,7 +50,7 @@ fn exact_prefix_resumes_seals_and_retains_writer_authority() -> Result<(), Box Result> { let directory = TestDirectory::create(name)?; drop( - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( - directory.path(), - )?, + FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(directory.path())?, ); Ok(Self { directory }) } @@ -80,12 +77,12 @@ impl ResumeFixture { /// Acquires single-format recovery admission, the writer authority continuation runs under. pub(super) fn recovery_admission( &self, - ) -> Result { - FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) + ) -> Result { + FilesystemRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) } pub(super) fn resumer(&self) -> Result> { - Ok(FilesystemRecoverySegmentResumer::open_version_two( + Ok(FilesystemRecoverySegmentResumer::open( self.recovery_admission()?, )?) } diff --git a/src/adapters/filesystem_recovery_segment_resumer.rs b/src/adapters/filesystem_recovery_segment_resumer.rs index 5f67a98d..8b9158d6 100644 --- a/src/adapters/filesystem_recovery_segment_resumer.rs +++ b/src/adapters/filesystem_recovery_segment_resumer.rs @@ -1,8 +1,8 @@ //! This module owns pinned writer authority for filesystem segment continuation. use super::{ - FilesystemRecoverySegmentResumeOpenError, FilesystemRecoveryStageDiscarder, - FilesystemVersionTwoRecoveryAdmission, + FilesystemRecoveryAdmission, FilesystemRecoverySegmentResumeOpenError, + FilesystemRecoveryStageDiscarder, }; /// Writer-authorized pinned filesystem adapter for reusable segment recovery. @@ -27,10 +27,10 @@ impl FilesystemRecoverySegmentResumer { /// /// Returns [`FilesystemRecoverySegmentResumeOpenError`] on root-clone or /// namespace pinning failure. - pub fn open_version_two( - admission: FilesystemVersionTwoRecoveryAdmission, + pub fn open( + admission: FilesystemRecoveryAdmission, ) -> Result { - FilesystemRecoveryStageDiscarder::open_version_two(admission) + FilesystemRecoveryStageDiscarder::open(admission) .map(Self::from_discarder) .map_err(Into::into) } @@ -45,13 +45,13 @@ impl FilesystemRecoverySegmentResumer { #[cfg(test)] pub(super) fn open_before_handoff_for_tests( - admission: FilesystemVersionTwoRecoveryAdmission, + admission: FilesystemRecoveryAdmission, before_handoff: F, ) -> Result where F: FnOnce() + 'static, { - let mut resumer = Self::open_version_two(admission)?; + let mut resumer = Self::open(admission)?; resumer.before_handoff = Some(Box::new(before_handoff)); Ok(resumer) } diff --git a/src/adapters/filesystem_recovery_stage_completer.rs b/src/adapters/filesystem_recovery_stage_completer.rs index 83182218..b014c54b 100644 --- a/src/adapters/filesystem_recovery_stage_completer.rs +++ b/src/adapters/filesystem_recovery_stage_completer.rs @@ -1,8 +1,8 @@ //! This module owns pinned writer authority for filesystem stage completion. use super::{ - FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscarder, - FilesystemVersionTwoRecoveryAdmission, + FilesystemRecoveryAdmission, FilesystemRecoveryStageCompletionOpenError, + FilesystemRecoveryStageDiscarder, }; /// Writer-authorized pinned filesystem adapter for exact stage completion. @@ -26,10 +26,10 @@ impl FilesystemRecoveryStageCompleter { /// /// Returns [`FilesystemRecoveryStageCompletionOpenError`] on root-clone or /// namespace pinning failure. - pub fn open_version_two( - admission: FilesystemVersionTwoRecoveryAdmission, + pub fn open( + admission: FilesystemRecoveryAdmission, ) -> Result { - FilesystemRecoveryStageDiscarder::open_version_two(admission) + FilesystemRecoveryStageDiscarder::open(admission) .map(|discarder| Self { discarder }) .map_err(Into::into) } diff --git a/src/adapters/filesystem_recovery_stage_completion_tests.rs b/src/adapters/filesystem_recovery_stage_completion_tests.rs index 7414810c..7cd18dd1 100644 --- a/src/adapters/filesystem_recovery_stage_completion_tests.rs +++ b/src/adapters/filesystem_recovery_stage_completion_tests.rs @@ -4,7 +4,7 @@ use std::error::Error; use std::fs; use super::{ - FilesystemPlatformAdmissionError, RecoveryStage, RecoveryStageDiscardOutcome, + FilesystemStoreAdmissionError, RecoveryStage, RecoveryStageDiscardOutcome, RecoveryStagePoolOutcome, RecoveryStageSynchronizationOutcome, WriterLockAcquireError, execute_recovery_stage_completion, }; @@ -89,7 +89,7 @@ fn retained_completer_authority_excludes_a_second_writer() -> Result<(), Box Result> { let directory = TestDirectory::create(name)?; drop( - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( - directory.path(), - )?, + FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(directory.path())?, ); Ok(Self { directory }) } @@ -95,12 +93,12 @@ impl CompletionFixture { /// Acquires single-format recovery admission, the writer authority completion runs under. pub(super) fn recovery_admission( &self, - ) -> Result { - FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) + ) -> Result { + FilesystemRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) } pub(super) fn completer(&self) -> Result> { - Ok(FilesystemRecoveryStageCompleter::open_version_two( + Ok(FilesystemRecoveryStageCompleter::open( self.recovery_admission()?, )?) } diff --git a/src/adapters/filesystem_recovery_stage_discard_tests.rs b/src/adapters/filesystem_recovery_stage_discard_tests.rs index 3627ffe5..06921420 100644 --- a/src/adapters/filesystem_recovery_stage_discard_tests.rs +++ b/src/adapters/filesystem_recovery_stage_discard_tests.rs @@ -4,7 +4,7 @@ use std::error::Error; use std::fs; use super::{ - FilesystemPlatformAdmissionError, FilesystemRecoveryStageError, RecoveryStage, + FilesystemRecoveryStageError, FilesystemStoreAdmissionError, RecoveryStage, RecoveryStageDiscardError, RecoveryStageDiscardOutcome, RecoveryStageDiscardStorageError, WriterLockAcquireError, execute_recovery_stage_discard, }; @@ -196,7 +196,7 @@ fn retained_discarder_authority_excludes_a_second_writer() -> Result<(), Box Result> { let directory = TestDirectory::create(name)?; drop( - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( - directory.path(), - )?, + FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(directory.path())?, ); Ok(Self { directory }) } @@ -83,12 +81,12 @@ impl DiscardFixture { /// Acquires single-format recovery admission, the writer authority discard runs under. pub(super) fn recovery_admission( &self, - ) -> Result { - FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) + ) -> Result { + FilesystemRecoveryAdmission::reopen_unchecked_for_repository_tasks(self.root()) } pub(super) fn discarder(&self) -> Result> { - Ok(FilesystemRecoveryStageDiscarder::open_version_two( + Ok(FilesystemRecoveryStageDiscarder::open( self.recovery_admission()?, )?) } diff --git a/src/adapters/filesystem_recovery_stage_discarder.rs b/src/adapters/filesystem_recovery_stage_discarder.rs index 7366872b..af0dad18 100644 --- a/src/adapters/filesystem_recovery_stage_discarder.rs +++ b/src/adapters/filesystem_recovery_stage_discarder.rs @@ -1,8 +1,8 @@ //! This module owns pinned writer authority for filesystem stage discard. use super::{ - FilesystemRecoveryInventoryReader, FilesystemRecoveryStageDiscardOpenError, - FilesystemVersionTwoRecoveryAdmission, FilesystemWriterLock, + FilesystemRecoveryAdmission, FilesystemRecoveryInventoryReader, + FilesystemRecoveryStageDiscardOpenError, FilesystemWriterLock, }; /// Writer-authorized pinned filesystem adapter for exact stage discard. @@ -30,20 +30,20 @@ impl FilesystemRecoveryStageDiscarder { /// /// Returns [`FilesystemRecoveryStageDiscardOpenError`] on root-clone or /// namespace pinning failure. - pub fn open_version_two( - admission: FilesystemVersionTwoRecoveryAdmission, + pub fn open( + admission: FilesystemRecoveryAdmission, ) -> Result { let (lock, _retention, _roots, _manifests) = admission.into_parts(); - Self::from_version_two_lock(lock) + Self::from_lock(lock) } - pub(super) fn from_version_two_lock( + pub(super) fn from_lock( authority: FilesystemWriterLock, ) -> Result { let inventory_root = authority .clone_directory() .map_err(|source| FilesystemRecoveryStageDiscardOpenError::CloneRoot { source })?; - let inventory = FilesystemRecoveryInventoryReader::from_version_two_root(inventory_root) + let inventory = FilesystemRecoveryInventoryReader::from_admitted_root(inventory_root) .map_err(|source| FilesystemRecoveryStageDiscardOpenError::Namespace { source })?; Ok(Self { inventory, diff --git a/src/adapters/filesystem_recovery_stage_tests.rs b/src/adapters/filesystem_recovery_stage_tests.rs index 3ea83e4f..79419398 100644 --- a/src/adapters/filesystem_recovery_stage_tests.rs +++ b/src/adapters/filesystem_recovery_stage_tests.rs @@ -4,7 +4,7 @@ use std::error::Error; use std::fs; use super::{ - FilesystemRecoveryInventoryReader, FilesystemRecoveryStageError, FilesystemVersionTwoAdmission, + FilesystemRecoveryInventoryReader, FilesystemRecoveryStageError, FilesystemStoreAdmission, RecoveryStage, RecoveryStageMetadataError, filesystem_test_sandbox::TestDirectory, }; @@ -175,9 +175,7 @@ impl StageFixture { fn new(name: &str) -> Result> { let directory = TestDirectory::create(name)?; drop( - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( - directory.path(), - )?, + FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(directory.path())?, ); Ok(Self { directory }) } @@ -199,11 +197,7 @@ impl StageFixture { } fn reader(&self) -> Result> { - Ok( - FilesystemRecoveryInventoryReader::open_version_two_unchecked_for_repository_tasks( - self.root(), - )?, - ) + Ok(FilesystemRecoveryInventoryReader::open_unchecked_for_repository_tasks(self.root())?) } fn remove(self) -> std::io::Result<()> { diff --git a/src/adapters/filesystem_version_two_catalog_stages.rs b/src/adapters/filesystem_retained_catalog_stages.rs similarity index 95% rename from src/adapters/filesystem_version_two_catalog_stages.rs rename to src/adapters/filesystem_retained_catalog_stages.rs index 2b908724..d23dbd38 100644 --- a/src/adapters/filesystem_version_two_catalog_stages.rs +++ b/src/adapters/filesystem_retained_catalog_stages.rs @@ -1,13 +1,13 @@ //! This module owns the catalog stages reachable only after retained-closure admission. //! -//! Version-two catalog publication executes the shared +//! Catalog publication executes the shared //! [`FilesystemCatalogStages`] implementation through this type. It has no //! public constructor and is reachable only through //! [`RetainedCatalogPublicationStorage::catalog_stages`](crate::RetainedCatalogPublicationStorage::catalog_stages), //! which requires a [`RetainedClosureAdmission`](crate::RetainedClosureAdmission) //! that only the gate creates. The ungated //! [`publish_catalog_generation`](crate::publish_catalog_generation) can -//! therefore never drive version-two stages. +//! therefore never drive gated stages. use std::io; @@ -29,7 +29,7 @@ pub enum CatalogPublicationBoundary { After, } -/// Repository-task observer of version-two catalog transitions. +/// Repository-task observer of catalog transitions. /// /// Returning an error fails the transition at that boundary exactly as a /// storage failure would; the crash matrix blocks inside it for process death. @@ -38,19 +38,19 @@ pub enum CatalogPublicationBoundary { pub type CatalogPublicationObserver = Box io::Result<()>>; -/// Version-two catalog protocol transitions guarded by retained-closure admission. +/// Catalog protocol transitions guarded by retained-closure admission. /// /// Every transition is the shared stage implementation, executed under the /// publisher's writer authority. Fields are private and /// no constructor is public. #[must_use] -pub struct FilesystemVersionTwoCatalogStages { +pub struct FilesystemRetainedCatalogStages { stages: FilesystemCatalogStages, #[cfg(any(test, feature = "repository-tasks"))] observer: Option, } -impl FilesystemVersionTwoCatalogStages { +impl FilesystemRetainedCatalogStages { pub(super) const fn new(stages: FilesystemCatalogStages) -> Self { Self { stages, @@ -99,7 +99,7 @@ impl FilesystemVersionTwoCatalogStages { } } -impl CatalogPublicationStorage for FilesystemVersionTwoCatalogStages { +impl CatalogPublicationStorage for FilesystemRetainedCatalogStages { fn verify_current( &mut self, expected: CatalogPublicationExpectation, diff --git a/src/adapters/filesystem_segment_stage_tests.rs b/src/adapters/filesystem_segment_stage_tests.rs index 59348220..89815f08 100644 --- a/src/adapters/filesystem_segment_stage_tests.rs +++ b/src/adapters/filesystem_segment_stage_tests.rs @@ -8,8 +8,8 @@ use super::filesystem_test_sandbox::TestDirectory; use super::test_support::decode_hex; use crate::{ AdmittedSegmentRecord, CatalogRestartByteLimit, CatalogRestartPolicy, - FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, - SegmentHeader, SegmentReadPolicy, SegmentRecordLimit, SegmentStageCreateError, StagedSegment, + FilesystemCatalogPublisher, FilesystemStoreAdmission, LayoutEntryLimit, SegmentHeader, + SegmentReadPolicy, SegmentRecordLimit, SegmentStageCreateError, StagedSegment, }; const ONE_ZERO_SEGMENT_HEX: &str = @@ -109,17 +109,13 @@ fn dropping_an_unsealed_stage_preserves_the_reusable_prefix() -> Result<(), Box< } /// Initializes a single-format store in `sandbox` and opens its writer. -fn open_publisher( - sandbox: &TestDirectory, -) -> Result> { +fn open_publisher(sandbox: &TestDirectory) -> Result> { let admission = - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(sandbox.path())? + FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(sandbox.path())? .into_admission(); let policy = CatalogRestartPolicy::new( SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), CatalogRestartByteLimit::new(1_048_576)?, ); - Ok(FilesystemVersionTwoCatalogPublisher::open( - admission, policy, - )?) + Ok(FilesystemCatalogPublisher::open(admission, policy)?) } diff --git a/src/adapters/filesystem_version_two_admission.rs b/src/adapters/filesystem_store_admission.rs similarity index 62% rename from src/adapters/filesystem_version_two_admission.rs rename to src/adapters/filesystem_store_admission.rs index f59e3dc3..913ba5c1 100644 --- a/src/adapters/filesystem_version_two_admission.rs +++ b/src/adapters/filesystem_store_admission.rs @@ -1,4 +1,4 @@ -//! This module owns proof that a filesystem root passed version-two admission. +//! This module owns proof that a filesystem root passed store admission. use std::path::Path; @@ -8,12 +8,12 @@ use cap_std::ambient_authority; use cap_std::fs::Dir; #[cfg(any(test, feature = "repository-tasks"))] -use super::VersionTwoInitializationObserver; -use super::version_two_initialization::{self as initialization, PhaseObserver}; +use super::StoreInitializationObserver; +use super::store_initialization::{self as initialization, PhaseObserver}; use super::{ - FilesystemPlatformAdmissionError, FilesystemWriterLock, VersionTwoInitialization, - VersionTwoInitializationError, filesystem_format_marker, filesystem_initialization_namespace, - filesystem_platform_profile, + FilesystemStoreAdmissionError, FilesystemWriterLock, StoreInitialization, + StoreInitializationError, filesystem_format_marker, filesystem_platform_profile, + filesystem_store_namespace, }; /// Exclusive writer authority over a committed store root. @@ -31,15 +31,15 @@ use super::{ /// it is placed. Live authority is the writer lock and the pinned /// capabilities of this process. #[must_use] -pub struct FilesystemVersionTwoAdmission { +pub struct FilesystemStoreAdmission { lock: FilesystemWriterLock, retention: Dir, roots: Dir, manifests: Dir, } -impl FilesystemVersionTwoAdmission { - /// Initializes a version-two store at `store_root`, or resumes an interrupted initialization. +impl FilesystemStoreAdmission { + /// Initializes a store at `store_root`, or resumes an interrupted initialization. /// /// The root must exist. An empty root, or a recoverable prefix that an /// interrupted call left, becomes a committed store with no catalog @@ -52,16 +52,14 @@ impl FilesystemVersionTwoAdmission { /// /// # Errors /// - /// Returns [`VersionTwoInitializationError`] with the exact platform, + /// Returns [`StoreInitializationError`] with the exact platform, /// writer-lock, refusal, phase, or admission failure. A committed store /// refuses as - /// [`AlreadyInitialized`](super::VersionTwoInitializationRefusal::AlreadyInitialized); + /// [`AlreadyInitialized`](super::StoreInitializationRefusal::AlreadyInitialized); /// a concurrent attempt refuses with a busy writer lock. - pub fn initialize( - store_root: &Path, - ) -> Result { - let root = filesystem_platform_profile::open_version_two(store_root) - .map_err(|source| VersionTwoInitializationError::Platform { source })?; + pub fn initialize(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open(store_root) + .map_err(|source| StoreInitializationError::Platform { source })?; initialize_root(&root, &mut |_, _| Ok(())) } @@ -72,12 +70,12 @@ impl FilesystemVersionTwoAdmission { /// /// # Errors /// - /// Returns [`VersionTwoInitializationError`] exactly as [`Self::initialize`] + /// Returns [`StoreInitializationError`] exactly as [`Self::initialize`] /// does after platform admission. #[cfg(any(test, feature = "repository-tasks"))] pub fn initialize_unchecked_for_repository_tasks( store_root: &Path, - ) -> Result { + ) -> Result { initialize_root(&open_unchecked(store_root)?, &mut |_, _| Ok(())) } @@ -88,22 +86,22 @@ impl FilesystemVersionTwoAdmission { /// /// # Errors /// - /// Returns [`VersionTwoInitializationError`] as + /// Returns [`StoreInitializationError`] as /// [`Self::initialize_unchecked_for_repository_tasks`] does, or the /// observer's error at its phase. #[cfg(any(test, feature = "repository-tasks"))] #[doc(hidden)] pub fn initialize_observed_for_repository_tasks( store_root: &Path, - mut observer: VersionTwoInitializationObserver, - ) -> Result { + mut observer: StoreInitializationObserver, + ) -> Result { initialize_root(&open_unchecked(store_root)?, &mut *observer) } - /// Reacquires writer authority over one committed version-two store. + /// Reacquires writer authority over one committed store. /// /// The call mutates no protocol state. It admits the production platform - /// for every version-two protocol directory, acquires the existing writer + /// for every protocol directory, acquires the existing writer /// lock, requires the exact committed root namespace, and admits the /// canonical `FORMAT` marker. A store with no catalog `HEAD` admits. /// Retention adapters perform content-level validation under the @@ -111,13 +109,13 @@ impl FilesystemVersionTwoAdmission { /// /// # Errors /// - /// Returns [`FilesystemPlatformAdmissionError`] with the exact platform, + /// Returns [`FilesystemStoreAdmissionError`] with the exact platform, /// writer-lock, namespace, or format-marker boundary and preserved source. /// A root whose initialization has not committed refuses at the namespace /// boundary. - pub fn reopen(store_root: &Path) -> Result { - let root = filesystem_platform_profile::open_version_two(store_root) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + pub fn reopen(store_root: &Path) -> Result { + let root = filesystem_platform_profile::open(store_root) + .map_err(|source| FilesystemStoreAdmissionError::Platform { source })?; Self::admit(root) } @@ -129,14 +127,14 @@ impl FilesystemVersionTwoAdmission { /// /// # Errors /// - /// Returns [`FilesystemPlatformAdmissionError`] exactly as [`Self::reopen`] + /// Returns [`FilesystemStoreAdmissionError`] exactly as [`Self::reopen`] /// does for every boundary after platform admission. #[cfg(any(test, feature = "repository-tasks"))] pub fn reopen_unchecked_for_repository_tasks( store_root: &Path, - ) -> Result { + ) -> Result { let root = Dir::open_ambient_dir(store_root, ambient_authority()) - .map_err(|source| FilesystemPlatformAdmissionError::Platform { source })?; + .map_err(|source| FilesystemStoreAdmissionError::Platform { source })?; Self::admit(root) } @@ -145,12 +143,11 @@ impl FilesystemVersionTwoAdmission { (self.lock, self.retention, self.roots, self.manifests) } - fn admit(root: Dir) -> Result { - admit_locked(root, filesystem_initialization_namespace::admit_version_two) - .map(Self::from_parts) + fn admit(root: Dir) -> Result { + admit_locked(root, filesystem_store_namespace::admit_store).map(Self::from_parts) } - fn from_parts((lock, retention, roots, manifests): AdmittedVersionTwoParts) -> Self { + fn from_parts((lock, retention, roots, manifests): AdmittedStoreParts) -> Self { Self { lock, retention, @@ -161,50 +158,50 @@ impl FilesystemVersionTwoAdmission { } /// Writer authority and the three pinned retention capabilities of one admitted root. -pub(super) type AdmittedVersionTwoParts = (FilesystemWriterLock, Dir, Dir, Dir); +pub(super) type AdmittedStoreParts = (FilesystemWriterLock, Dir, Dir, Dir); #[cfg(any(test, feature = "repository-tasks"))] -fn open_unchecked(store_root: &Path) -> Result { +fn open_unchecked(store_root: &Path) -> Result { Dir::open_ambient_dir(store_root, ambient_authority()) - .map_err(|source| VersionTwoInitializationError::Platform { source }) + .map_err(|source| StoreInitializationError::Platform { source }) } /// Runs the initialization protocol, then admits the committed root under its lock. fn initialize_root( root: &Dir, observer: PhaseObserver<'_>, -) -> Result { +) -> Result { let (lock, found) = initialization::initialize(root, observer)?; - let parts = admit_under_lock(lock, filesystem_initialization_namespace::admit_version_two) - .map_err(|source| VersionTwoInitializationError::Admission { source })?; - Ok(VersionTwoInitialization::new( - FilesystemVersionTwoAdmission::from_parts(parts), + let parts = admit_under_lock(lock, filesystem_store_namespace::admit_store) + .map_err(|source| StoreInitializationError::Admission { source })?; + Ok(StoreInitialization::new( + FilesystemStoreAdmission::from_parts(parts), found, )) } /// Acquires writer authority, then admits the root under `namespace`. /// -/// Forward writers pass the exact version-two namespace law; catalog-stage +/// Forward writers pass the exact store namespace law; catalog-stage /// recovery passes the law that also admits a retained `head.next`. Every /// other check is identical: the `FORMAT` marker and the pinned retention /// directories. pub(super) fn admit_locked( root: Dir, namespace: fn(&Dir) -> std::io::Result<()>, -) -> Result { +) -> Result { let lock = FilesystemWriterLock::try_acquire_in(root) - .map_err(|source| FilesystemPlatformAdmissionError::WriterLock { source })?; + .map_err(|source| FilesystemStoreAdmissionError::WriterLock { source })?; admit_under_lock(lock, namespace) } fn admit_under_lock( lock: FilesystemWriterLock, namespace: fn(&Dir) -> std::io::Result<()>, -) -> Result { +) -> Result { let directory = lock .clone_directory() - .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; + .map_err(|source| FilesystemStoreAdmissionError::Namespace { source })?; admit_root(&directory, namespace)?; let retention = pin(&directory, "retention")?; let roots = pin(&retention, "roots")?; @@ -216,16 +213,15 @@ fn admit_under_lock( pub(super) fn admit_root( directory: &Dir, namespace: fn(&Dir) -> std::io::Result<()>, -) -> Result<(), FilesystemPlatformAdmissionError> { - namespace(directory) - .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source })?; +) -> Result<(), FilesystemStoreAdmissionError> { + namespace(directory).map_err(|source| FilesystemStoreAdmissionError::Namespace { source })?; filesystem_format_marker::admit(directory) - .map_err(|source| FilesystemPlatformAdmissionError::FormatMarker { source }) + .map_err(|source| FilesystemStoreAdmissionError::FormatMarker { source }) } /// Pins one admitted protocol directory without following links. -fn pin(parent: &Dir, name: &str) -> Result { +fn pin(parent: &Dir, name: &str) -> Result { parent .open_dir_nofollow(name) - .map_err(|source| FilesystemPlatformAdmissionError::Namespace { source }) + .map_err(|source| FilesystemStoreAdmissionError::Namespace { source }) } diff --git a/src/adapters/filesystem_platform_admission_error.rs b/src/adapters/filesystem_store_admission_error.rs similarity index 83% rename from src/adapters/filesystem_platform_admission_error.rs rename to src/adapters/filesystem_store_admission_error.rs index ad906ab8..0910e41f 100644 --- a/src/adapters/filesystem_platform_admission_error.rs +++ b/src/adapters/filesystem_store_admission_error.rs @@ -9,7 +9,7 @@ use super::WriterLockAcquireError; /// Failure to reacquire writer authority over one published filesystem store. #[derive(Debug)] #[non_exhaustive] -pub enum FilesystemPlatformAdmissionError { +pub enum FilesystemStoreAdmissionError { /// The store root does not satisfy the production platform profile. Platform { /// Preserved platform-admission failure. @@ -25,25 +25,25 @@ pub enum FilesystemPlatformAdmissionError { /// Preserved namespace-admission failure. source: io::Error, }, - /// The version-two `FORMAT` marker failed exact admission. + /// The `FORMAT` marker failed exact admission. FormatMarker { /// Preserved record-admission failure. source: io::Error, }, } -impl fmt::Display for FilesystemPlatformAdmissionError { +impl fmt::Display for FilesystemStoreAdmissionError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { Self::Platform { .. } => "published store platform admission failed", Self::WriterLock { .. } => "published store writer-lock acquisition failed", Self::Namespace { .. } => "published store namespace admission failed", - Self::FormatMarker { .. } => "version-two format marker admission failed", + Self::FormatMarker { .. } => "format marker admission failed", }) } } -impl Error for FilesystemPlatformAdmissionError { +impl Error for FilesystemStoreAdmissionError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::Platform { source } diff --git a/src/adapters/filesystem_initialization_namespace.rs b/src/adapters/filesystem_store_namespace.rs similarity index 81% rename from src/adapters/filesystem_initialization_namespace.rs rename to src/adapters/filesystem_store_namespace.rs index d92acd4b..f03c172c 100644 --- a/src/adapters/filesystem_initialization_namespace.rs +++ b/src/adapters/filesystem_store_namespace.rs @@ -23,7 +23,7 @@ const RECOVERY_NAME: &str = "recovery"; const ROOTS_NAME: &str = "roots"; const MANIFESTS_NAME: &str = "manifests"; const DISPOSITIONS_NAME: &str = "dispositions"; -const VERSION_TWO_NAMES: [&str; 10] = [ +const STORE_NAMES: [&str; 10] = [ LOCK_NAME, STAGING_NAME, SEGMENTS_NAME, @@ -36,7 +36,7 @@ const VERSION_TWO_NAMES: [&str; 10] = [ RECOVERY_NAME, ]; const NEXT_HEAD_NAME: &str = "head.next"; -const VERSION_TWO_STAGED_NAMES: [&str; 11] = [ +const STAGED_STORE_NAMES: [&str; 11] = [ LOCK_NAME, STAGING_NAME, SEGMENTS_NAME, @@ -50,7 +50,7 @@ const VERSION_TWO_STAGED_NAMES: [&str; 11] = [ RECOVERY_NAME, ]; -/// Admits the exact committed version-2 root namespace. +/// Admits the exact committed store root namespace. /// /// `writer.lock`, `staging`, `segments`, `catalogs`, the persistent reader /// fence, the format marker, and all three protocol directories must be @@ -58,33 +58,33 @@ const VERSION_TWO_STAGED_NAMES: [&str; 11] = [ /// none until its first publication. A root that still holds `FORMAT.next` /// refuses as `InitializationIncomplete` and a root without `FORMAT` as /// `MissingFormatMarker`. Any other entry is unrecoverable ambiguity. -pub(super) fn admit_version_two(directory: &Dir) -> io::Result<()> { - admit_version_two_required(directory)?; - admit_membership(directory, &VERSION_TWO_NAMES)?; - admit_version_two_protocol_directories(directory) +pub(super) fn admit_store(directory: &Dir) -> io::Result<()> { + admit_store_required(directory)?; + admit_membership(directory, &STORE_NAMES)?; + admit_protocol_directories(directory) } -/// Admits a committed version-2 root that may retain a catalog `head.next`. +/// Admits a committed store root that may retain a catalog `head.next`. /// /// `head.next` is the catalog-protocol head stage. A reader selects only /// `HEAD`, so a retained or in-flight `head.next` does not change its view, and /// catalog-stage recovery must be able to open the store that holds one. Writer admission keeps the -/// exact law of [`admit_version_two`] and refuses it. Every other entry and -/// every protocol directory is admitted exactly as [`admit_version_two`] does. -pub(super) fn admit_version_two_with_catalog_stage(directory: &Dir) -> io::Result<()> { - admit_version_two_required(directory)?; +/// exact law of [`admit_store`] and refuses it. Every other entry and +/// every protocol directory is admitted exactly as [`admit_store`] does. +pub(super) fn admit_store_with_catalog_stage(directory: &Dir) -> io::Result<()> { + admit_store_required(directory)?; admit_optional_file(directory, NEXT_HEAD_NAME)?; - admit_membership(directory, &VERSION_TWO_STAGED_NAMES)?; - admit_version_two_protocol_directories(directory) + admit_membership(directory, &STAGED_STORE_NAMES)?; + admit_protocol_directories(directory) } -fn admit_version_two_required(directory: &Dir) -> io::Result<()> { +fn admit_store_required(directory: &Dir) -> io::Result<()> { admit_committed(directory)?; - admit_version_two_layout(directory) + admit_store_layout(directory) } /// Admits every committed root entry by kind; catalog `HEAD` is optional. -fn admit_version_two_layout(directory: &Dir) -> io::Result<()> { +fn admit_store_layout(directory: &Dir) -> io::Result<()> { admit_required_file(directory, LOCK_NAME)?; admit_required_directory(directory, STAGING_NAME)?; admit_required_directory(directory, SEGMENTS_NAME)?; @@ -97,14 +97,14 @@ fn admit_version_two_layout(directory: &Dir) -> io::Result<()> { admit_required_directory(directory, RECOVERY_NAME) } -/// Admits the nested version-2 protocol directories initialization created. +/// Admits the nested protocol directories initialization created. /// /// `retention` must carry both immutable pools (its head and stages belong to /// retention publication); `gc` must be empty until `KEEP-GC-001` implements /// its records; `recovery` must hold exactly an empty `dispositions`. A root /// that drifted after initialization refuses here rather than as a later /// pinning failure. -fn admit_version_two_protocol_directories(directory: &Dir) -> io::Result<()> { +fn admit_protocol_directories(directory: &Dir) -> io::Result<()> { let retention = directory.open_dir_nofollow(RETENTION_NAME)?; admit_required_directory(&retention, ROOTS_NAME)?; admit_required_directory(&retention, MANIFESTS_NAME)?; @@ -180,16 +180,16 @@ fn ambiguous_namespace() -> io::Error { /// /// Initialization uses this law only after it has proven that `FORMAT` and /// `FORMAT.next` name one canonical marker inode. Every committed entry is -/// admitted as [`admit_version_two_with_catalog_stage`] admits it, and +/// admitted as [`admit_store_with_catalog_stage`] admits it, and /// `FORMAT.next` is additionally required. -pub(super) fn admit_version_two_marker_linked(directory: &Dir) -> io::Result<()> { +pub(super) fn admit_store_marker_linked(directory: &Dir) -> io::Result<()> { admit_required_file(directory, MARKER_STAGE_NAME)?; - admit_version_two_layout(directory)?; + admit_store_layout(directory)?; admit_optional_file(directory, NEXT_HEAD_NAME)?; - let mut allowed: Vec<&str> = VERSION_TWO_STAGED_NAMES.to_vec(); + let mut allowed: Vec<&str> = STAGED_STORE_NAMES.to_vec(); allowed.push(MARKER_STAGE_NAME); admit_membership(directory, &allowed)?; - admit_version_two_protocol_directories(directory) + admit_protocol_directories(directory) } /// Refuses an uncommitted root with a typed reason before any other law. diff --git a/src/adapters/filesystem_version_two_record_refusal.rs b/src/adapters/filesystem_store_record_refusal.rs similarity index 67% rename from src/adapters/filesystem_version_two_record_refusal.rs rename to src/adapters/filesystem_store_record_refusal.rs index e8927272..df1d6ab3 100644 --- a/src/adapters/filesystem_version_two_record_refusal.rs +++ b/src/adapters/filesystem_store_record_refusal.rs @@ -1,4 +1,4 @@ -//! This module owns the typed refusal behind version-two record admission. +//! This module owns the typed refusal behind store record admission. use std::error::Error; use std::fmt; @@ -9,12 +9,12 @@ use super::StoreFormatMarkerDecodeError; /// Why the `FORMAT` record refused admission. /// /// Carried as the source of the `InvalidData` error behind -/// [`FilesystemPlatformAdmissionError::FormatMarker`](super::FilesystemPlatformAdmissionError::FormatMarker), +/// [`FilesystemStoreAdmissionError::FormatMarker`](super::FilesystemStoreAdmissionError::FormatMarker), /// so a caller can recover the decoder's own /// diagnosis instead of a rendered string. #[derive(Debug)] #[non_exhaustive] -pub enum VersionTwoRecordRefusal { +pub enum StoreRecordRefusal { /// The record's canonical length exceeds the platform's addressable range. LengthOverflow { /// The record's fixed name. @@ -30,47 +30,42 @@ pub enum VersionTwoRecordRefusal { /// The record's fixed name. name: &'static str, }, - /// `FORMAT` did not decode as a canonical version-two marker. + /// `FORMAT` did not decode as a canonical format marker. Marker { /// The exact decode refusal. source: StoreFormatMarkerDecodeError, }, } -impl VersionTwoRecordRefusal { +impl StoreRecordRefusal { /// Wraps the refusal as the `InvalidData` error record admission returns. pub(super) fn into_io(self) -> io::Error { io::Error::new(io::ErrorKind::InvalidData, self) } } -impl fmt::Display for VersionTwoRecordRefusal { +impl fmt::Display for StoreRecordRefusal { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::LengthOverflow { name } => write!( formatter, - "version-two record {name} has a canonical length beyond the addressable range" + "store record {name} has a canonical length beyond the addressable range" ), Self::KindOrLength { name } => { write!( formatter, - "version-two record {name} has the wrong kind or length" + "store record {name} has the wrong kind or length" ) } Self::TrailingBytes { name } => { - write!( - formatter, - "version-two record {name} carried trailing bytes" - ) - } - Self::Marker { .. } => { - formatter.write_str("version-two record FORMAT refused admission") + write!(formatter, "store record {name} carried trailing bytes") } + Self::Marker { .. } => formatter.write_str("store record FORMAT refused admission"), } } } -impl Error for VersionTwoRecordRefusal { +impl Error for StoreRecordRefusal { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::Marker { source } => Some(source), diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 4997b37a..416cc40e 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -82,19 +82,22 @@ mod filesystem_catalog_catalog; mod filesystem_catalog_current; mod filesystem_catalog_head; mod filesystem_catalog_publication_error; +mod filesystem_catalog_publisher; +mod filesystem_catalog_publisher_storage; #[cfg(test)] mod filesystem_catalog_publisher_tests; +#[cfg(all(test, target_os = "linux"))] +mod filesystem_catalog_recovery_tests; mod filesystem_catalog_segment; mod filesystem_catalog_snapshot; mod filesystem_catalog_stages; mod filesystem_catalog_storage; mod filesystem_exact_record; mod filesystem_format_marker; -mod filesystem_initialization_namespace; mod filesystem_namespace_refusal; -mod filesystem_platform_admission_error; mod filesystem_platform_profile; mod filesystem_publisher_authority; +mod filesystem_recovery_admission; mod filesystem_recovery_inventory_reader; mod filesystem_recovery_inventory_scan; #[cfg(all(test, unix))] @@ -130,20 +133,17 @@ mod filesystem_recovery_stage_materialization; mod filesystem_recovery_stage_sync; #[cfg(all(test, unix))] mod filesystem_recovery_stage_tests; +mod filesystem_retained_catalog_stages; mod filesystem_segment_stage; #[cfg(test)] mod filesystem_segment_stage_tests; +mod filesystem_store_admission; +mod filesystem_store_admission_error; +mod filesystem_store_namespace; +mod filesystem_store_record_refusal; #[cfg(test)] #[path = "../../tests/segment_filesystem_stage/sandbox.rs"] mod filesystem_test_sandbox; -mod filesystem_version_two_admission; -mod filesystem_version_two_catalog_publisher; -#[cfg(all(test, target_os = "linux"))] -mod filesystem_version_two_catalog_recovery_tests; -mod filesystem_version_two_catalog_stages; -mod filesystem_version_two_catalog_storage; -mod filesystem_version_two_record_refusal; -mod filesystem_version_two_recovery_admission; mod filesystem_writer_lock; mod framed_blake3; mod layout_decode_error; @@ -240,6 +240,7 @@ mod staged_segment; mod storage_profile_id_text; mod storage_profile_id_text_error; mod store_format; +mod store_initialization; mod sync_capable_directory; #[cfg(test)] #[path = "../../tests/support/mod.rs"] @@ -249,7 +250,6 @@ mod verification_decode_error; mod verification_error; mod verification_failure_class; mod verification_ingress; -mod version_two_initialization; mod writer_lock_acquire_error; mod writer_lock_acquire_phase; diff --git a/src/adapters/recovery.rs b/src/adapters/recovery.rs index 1af60de2..ab1d3263 100644 --- a/src/adapters/recovery.rs +++ b/src/adapters/recovery.rs @@ -107,7 +107,7 @@ pub use recovery_inventory_error::RecoveryInventoryError; pub use recovery_inventory_limit::{RecoveryInventoryLimit, RecoveryInventoryLimitError}; pub use recovery_inventory_operation::RecoveryInventoryOperation; pub use recovery_inventory_storage::RecoveryInventoryStorage; -pub use recovery_name_classification::classify_version_two_recovery_names; +pub use recovery_name_classification::classify_recovery_names; pub use recovery_name_classification_error::RecoveryNameClassificationError; pub use recovery_name_manifest::{RecoveryNameManifest, RecoveryNamedEntry}; pub use recovery_namespace::RecoveryNamespace; diff --git a/src/adapters/recovery/recovery_entry_role.rs b/src/adapters/recovery/recovery_entry_role.rs index 05b4d50d..6dafe556 100644 --- a/src/adapters/recovery/recovery_entry_role.rs +++ b/src/adapters/recovery/recovery_entry_role.rs @@ -34,15 +34,15 @@ pub enum RecoveryEntryRole { /// Physical catalog digest parsed from the name. digest: CatalogDigest, }, - /// Version-two persistent reader fence `reader.lock`. + /// Persistent reader fence `reader.lock`. ReaderFence, - /// Version-two format marker `FORMAT`. + /// Format marker `FORMAT`. FormatMarker, - /// Version-two `retention` protocol directory, admitted by retention recovery. + /// The `retention` protocol directory, admitted by retention recovery. RetentionDirectory, - /// Version-two reserved `gc` protocol directory. + /// Reserved `gc` protocol directory. GcDirectory, - /// Version-two reserved `recovery` protocol directory. + /// Reserved `recovery` protocol directory. RecoveryDirectory, } diff --git a/src/adapters/recovery/recovery_name_classification.rs b/src/adapters/recovery/recovery_name_classification.rs index 2ab61cd3..b54bd149 100644 --- a/src/adapters/recovery/recovery_name_classification.rs +++ b/src/adapters/recovery/recovery_name_classification.rs @@ -6,7 +6,7 @@ use super::{ recovery_pool_name, }; -/// Classifies one complete committed version-two inventory without opening bytes. +/// Classifies one complete committed inventory without opening bytes. /// /// The operation preserves inventory order and allocates at most one manifest /// entry per inventory entry. It rejects unknown names, missing root entries, @@ -17,13 +17,13 @@ use super::{ /// store can be interrupted before it exists. `FORMAT.next` is unknown: a /// committed root never holds it. /// The contents of `retention`, `gc`, and `recovery` are not inventoried here; -/// version-two admission and retention recovery own them. +/// store admission and retention recovery own them. /// /// # Errors /// /// Returns [`RecoveryNameClassificationError`] at the first deterministic /// namespace, grammar, required-entry, or stage-conflict failure. -pub fn classify_version_two_recovery_names( +pub fn classify_recovery_names( inventory: RecoveryInventory, ) -> Result { let mut required = RequiredEntries::default(); diff --git a/src/adapters/recovery/recovery_next_head_finalization_storage_error.rs b/src/adapters/recovery/recovery_next_head_finalization_storage_error.rs index 9b3e2b54..baee0e22 100644 --- a/src/adapters/recovery/recovery_next_head_finalization_storage_error.rs +++ b/src/adapters/recovery/recovery_next_head_finalization_storage_error.rs @@ -61,7 +61,7 @@ pub enum RecoveryNextHeadFinalizationStorageError { }, /// The candidate would lose content a current retention root retains. /// - /// Only version-two finalization reports this. The source is the forward + /// Finalization reports this. The source is the forward /// catalog gate's own evidence, binding, or closure refusal; its `Catalog` /// and `RetentionChanged` variants never arise here. RetainedClosure { diff --git a/src/adapters/recovery/recovery_required_entry.rs b/src/adapters/recovery/recovery_required_entry.rs index 550fd7ec..3b5ca971 100644 --- a/src/adapters/recovery/recovery_required_entry.rs +++ b/src/adapters/recovery/recovery_required_entry.rs @@ -5,7 +5,7 @@ use std::fmt; /// One fixed entry required in an initialized store root. /// /// The first four are required in every root. The remaining entries are -/// required only when a version-two root is classified. Catalog `HEAD` is +/// required only when a store root is classified. Catalog `HEAD` is /// required by neither: a store holds none until its first publication. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RecoveryRequiredEntry { diff --git a/src/adapters/retention.rs b/src/adapters/retention.rs index b9e76660..24526dba 100644 --- a/src/adapters/retention.rs +++ b/src/adapters/retention.rs @@ -128,9 +128,9 @@ mod filesystem_retention_successor_tests; mod filesystem_retention_test_fixture; mod filesystem_retention_verification; #[cfg(test)] -mod filesystem_verification_law_tests; +mod filesystem_store_admission_tests; #[cfg(test)] -mod filesystem_version_two_admission_tests; +mod filesystem_verification_law_tests; mod head_decode_error; mod head_decode_error_display; mod head_decoder; diff --git a/src/adapters/retention/durable_view_law_tests.rs b/src/adapters/retention/durable_view_law_tests.rs index e64a7eaa..50ce576b 100644 --- a/src/adapters/retention/durable_view_law_tests.rs +++ b/src/adapters/retention/durable_view_law_tests.rs @@ -17,7 +17,7 @@ use crate::{ AdmittedRetentionManifest, AdmittedRetentionRoot, CanonicalRetentionRoot, CatalogRestartByteLimit, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, DurableSnapshot, DurableStoreError, FilesystemRetentionPublicationAuthority, - FilesystemRetentionSnapshotError, FilesystemVersionTwoAdmission, LayoutEntryLimit, + FilesystemRetentionSnapshotError, FilesystemStoreAdmission, LayoutEntryLimit, ReaderAttemptLimit, RetentionPolicy, RetentionRoot, SegmentReadPolicy, SegmentRecordLimit, execute_retention_publication, }; @@ -48,7 +48,7 @@ fn a_pinned_read_keeps_its_retained_view_after_release_publication() -> Result<( .ok_or("golden anchor absent")?; let pinned = snapshot(sandbox.path())?; let admission = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; + FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; let mut authority = FilesystemRetentionPublicationAuthority::open(admission)?; let state = authority.observe_current()?.ok_or("retention absent")?; let manifest = AdmittedRetentionManifest::decode(state.manifest_bytes())?; diff --git a/src/adapters/retention/filesystem_retained_evidence.rs b/src/adapters/retention/filesystem_retained_evidence.rs index 65449832..207b8321 100644 --- a/src/adapters/retention/filesystem_retained_evidence.rs +++ b/src/adapters/retention/filesystem_retained_evidence.rs @@ -1,6 +1,6 @@ //! This module owns bounded no-follow reads of retained-closure evidence. //! -//! Version-two catalog publication and catalog-head recovery finalization read +//! Catalog publication and catalog-head recovery finalization read //! the current retention head, its manifest, and every root it selects through //! this one reader before `HEAD` may change. The reader returns exact bytes; //! the retained-closure gate decodes and binds every record itself. diff --git a/src/adapters/retention/filesystem_retention_authority.rs b/src/adapters/retention/filesystem_retention_authority.rs index c10fc379..2a133933 100644 --- a/src/adapters/retention/filesystem_retention_authority.rs +++ b/src/adapters/retention/filesystem_retention_authority.rs @@ -10,7 +10,7 @@ use super::filesystem_retention_authority_error::{ }; use super::filesystem_retention_current::{self, ObservedRetentionState}; use super::filesystem_retention_recovery::RetentionRecoveryContext; -use crate::adapters::{FilesystemVersionTwoAdmission, FilesystemWriterLock}; +use crate::adapters::{FilesystemStoreAdmission, FilesystemWriterLock}; /// Exclusive authority to publish retention transitions on one pinned root. /// @@ -42,7 +42,7 @@ pub struct FilesystemRetentionPublicationAuthority { impl FilesystemRetentionPublicationAuthority { /// Pins one admitted root for retention publication. /// - /// Only [`FilesystemVersionTwoAdmission`] is accepted, so a bare writer + /// Only [`FilesystemStoreAdmission`] is accepted, so a bare writer /// lock, which proves neither the namespace nor the `FORMAT` marker, can /// never reach retention publication; the type system refuses it: /// @@ -60,7 +60,7 @@ impl FilesystemRetentionPublicationAuthority { /// Returns [`FilesystemRetentionAuthorityError`](super::FilesystemRetentionAuthorityError) /// when the root capability cannot be cloned. The retention namespace and /// both immutable pools arrive already pinned by admission. - pub fn open(admission: FilesystemVersionTwoAdmission) -> Result { + pub fn open(admission: FilesystemStoreAdmission) -> Result { let (lock, retention, roots, manifests) = admission.into_parts(); let root = lock.clone_directory().map_err(|source| Error::Directory { directory: Directory::Root, diff --git a/src/adapters/retention/filesystem_retention_fifo_tests.rs b/src/adapters/retention/filesystem_retention_fifo_tests.rs index 1f219b2d..a9e55f80 100644 --- a/src/adapters/retention/filesystem_retention_fifo_tests.rs +++ b/src/adapters/retention/filesystem_retention_fifo_tests.rs @@ -11,7 +11,7 @@ use std::time::Duration; use super::filesystem_retention_test_fixture::{ ROOT_HEX, fixture, initial_preparation, initialized_store, manifest_pool_path, open_authority, }; -use crate::adapters::FilesystemVersionTwoAdmission; +use crate::adapters::FilesystemStoreAdmission; use crate::execute_retention_publication; const DEADLINE: Duration = Duration::from_secs(5); @@ -35,7 +35,7 @@ fn a_fifo_at_the_format_marker_refuses_instead_of_blocking() -> Result<(), Box Result<(), Box> { ); drop(authority); let mut restarted = FilesystemRetentionPublicationAuthority::open( - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?, + FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?, )?; let receipt = restarted.recover()?; assert_eq!( diff --git a/src/adapters/retention/filesystem_retention_recovery_storage_error_tests.rs b/src/adapters/retention/filesystem_retention_recovery_storage_error_tests.rs index bccfd390..8691fce6 100644 --- a/src/adapters/retention/filesystem_retention_recovery_storage_error_tests.rs +++ b/src/adapters/retention/filesystem_retention_recovery_storage_error_tests.rs @@ -240,7 +240,7 @@ fn cleanup_sync_failure_reports_removed_stage_and_preserved_pool() -> Result<(), ); drop(authority); let admission = - crate::adapters::FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks( + crate::adapters::FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks( sandbox.path(), )?; let mut restarted = diff --git a/src/adapters/retention/filesystem_retention_short_count_tests.rs b/src/adapters/retention/filesystem_retention_short_count_tests.rs index 5edfa28a..33ebe585 100644 --- a/src/adapters/retention/filesystem_retention_short_count_tests.rs +++ b/src/adapters/retention/filesystem_retention_short_count_tests.rs @@ -10,7 +10,7 @@ use super::{ }; use std::{error::Error, fs}; -// Size: medium. Oracle: version-two root/manifest count ceilings apply before stage discard. +// Size: medium. Oracle: store root/manifest count ceilings apply before stage discard. // Delete only when stronger public recovery coverage subsumes excessive-count preservation. #[test] fn excessive_counts_in_short_headers_preserve_evidence() -> Result<(), Box> { diff --git a/src/adapters/retention/filesystem_retention_snapshot.rs b/src/adapters/retention/filesystem_retention_snapshot.rs index 6332d99b..8db57435 100644 --- a/src/adapters/retention/filesystem_retention_snapshot.rs +++ b/src/adapters/retention/filesystem_retention_snapshot.rs @@ -1,4 +1,4 @@ -//! This module owns one fenced, double-collected reader view of a version-two store. +//! This module owns one fenced, double-collected reader view of a store. use std::io; use std::path::Path; @@ -18,8 +18,8 @@ use crate::adapters::filesystem_exact_record::{ }; use crate::adapters::{ CatalogRestartError, CatalogRestartPolicy, ChecksummedPublicationHead, - FilesystemCatalogSnapshot, filesystem_format_marker, filesystem_initialization_namespace, - filesystem_platform_profile, publication_head_decoder, + FilesystemCatalogSnapshot, filesystem_format_marker, filesystem_platform_profile, + filesystem_store_namespace, publication_head_decoder, }; use crate::{RetentionHead, RetentionManifest, RetentionNamespaceDigest}; @@ -105,7 +105,7 @@ impl RetentionViewSource for Source { } impl FilesystemRetentionSnapshot { - /// Admits the production filesystem profile and version-two root, acquires the reader fence, and + /// Admits the production filesystem profile and store root, acquires the reader fence, and /// double-collects one consistent view within `limit` attempts. /// Admission requires the committed namespace and the canonical `FORMAT` /// marker before fencing; nothing binds the store to the directory it is in. @@ -115,7 +115,7 @@ impl FilesystemRetentionSnapshot { /// returned as `Admission` with the original I/O cause. It may also block /// while collection holds the fence exclusively. /// Platform admission requires the existing local writable, case-sensitive - /// Linux ext4 profile across every present version-two protocol directory. + /// Linux ext4 profile across every present protocol directory. /// The same opened root capability is retained through namespace, marker, /// fence, and coordinate admission; the ambient path is not reopened. A /// committed store with no catalog `HEAD` refuses as `Unpublished` once the @@ -147,9 +147,9 @@ impl FilesystemRetentionSnapshot { limit: ReaderAttemptLimit, collect: impl FnOnce(&mut Source, ReaderAttemptLimit) -> Result, ) -> Result { - let root = filesystem_platform_profile::open_version_two(store_root) + let root = filesystem_platform_profile::open(store_root) .map_err(|source| Error::Admission { source })?; - filesystem_initialization_namespace::admit_version_two_with_catalog_stage(&root) + filesystem_store_namespace::admit_store_with_catalog_stage(&root) .map_err(|source| Error::Admission { source })?; filesystem_format_marker::admit(&root).map_err(|source| Error::Admission { source })?; let fence = ReaderFence::acquire(&root).map_err(|source| Error::Fence { source })?; diff --git a/src/adapters/retention/filesystem_retention_snapshot_error.rs b/src/adapters/retention/filesystem_retention_snapshot_error.rs index d04eae78..de9d1398 100644 --- a/src/adapters/retention/filesystem_retention_snapshot_error.rs +++ b/src/adapters/retention/filesystem_retention_snapshot_error.rs @@ -7,11 +7,11 @@ use std::io; use super::RetentionViewError; use crate::adapters::CatalogRestartError; -/// Why a reader could not bind one consistent version-two view. +/// Why a reader could not bind one consistent view. #[derive(Debug)] #[non_exhaustive] pub enum FilesystemRetentionSnapshotError { - /// The platform profile or exact version-two root admission failed. + /// The platform profile or exact store root admission failed. Admission { /// The exact platform I/O, namespace, or record failure. source: io::Error, @@ -46,7 +46,7 @@ pub enum FilesystemRetentionSnapshotError { impl fmt::Display for FilesystemRetentionSnapshotError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str(match self { - Self::Admission { .. } => "version-two reader admission refused", + Self::Admission { .. } => "reader admission refused", Self::Unpublished => "the store has published no catalog", Self::Fence { .. } => "reader fence acquisition failed", Self::View { .. } => "reader view collection refused", diff --git a/src/adapters/retention/filesystem_retention_test_fixture.rs b/src/adapters/retention/filesystem_retention_test_fixture.rs index bbf9c173..bd118775 100644 --- a/src/adapters/retention/filesystem_retention_test_fixture.rs +++ b/src/adapters/retention/filesystem_retention_test_fixture.rs @@ -20,8 +20,7 @@ use crate::adapters::populated_store::{self, FirstCatalogRecords, StoreAdmission use crate::adapters::test_support::decode_hex; use crate::adapters::{ AdmittedCatalog, AdmittedSegment, CatalogSnapshot, ChecksummedCatalog, - ChecksummedPublicationHead, FilesystemVersionTwoAdmission, SegmentReadPolicy, - SegmentRecordLimit, + ChecksummedPublicationHead, FilesystemStoreAdmission, SegmentReadPolicy, SegmentRecordLimit, }; use crate::{ RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionRoot, @@ -53,14 +52,14 @@ pub(super) const CATALOG_NAME: &str = /// Builds one initialized version-2 store and pins its retention authority. /// /// The fixture initializes the store, publishes the exact bundle corpus as -/// its first catalog through the version-two writer, releases writer +/// its first catalog through the writer, releases writer /// authority, then reopens the admitted root for retention publication. pub(super) fn open_authority( name: &str, ) -> Result<(TestDirectory, FilesystemRetentionPublicationAuthority), Box> { let sandbox = initialized_store(name)?; let admission = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; + FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; let authority = FilesystemRetentionPublicationAuthority::open(admission)?; Ok((sandbox, authority)) } @@ -247,13 +246,11 @@ fn hex(bytes: &[u8; 32]) -> String { const UNCHECKED: StoreAdmission = StoreAdmission { initialize: |root| { Ok( - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(root)? + FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(root)? .into_admission(), ) }, - reopen_version_two: |root| { - Ok(FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(root)?) - }, + reopen: |root| Ok(FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(root)?), }; /// Builds one version-2 store holding the frozen bundle, with writer authority released. diff --git a/src/adapters/retention/filesystem_retention_verification.rs b/src/adapters/retention/filesystem_retention_verification.rs index e98f2441..11062939 100644 --- a/src/adapters/retention/filesystem_retention_verification.rs +++ b/src/adapters/retention/filesystem_retention_verification.rs @@ -10,9 +10,9 @@ use crate::adapters::filesystem_exact_record::ExactRecordError; use crate::adapters::{verification_admission, verification_ingress}; use crate::{ CatalogRestartPolicy, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, - ReaderAttemptLimit, RetentionNamespaceDigest, RetentionRootDecodeError, VerificationDepth, - VerificationError, VerificationRefusal, VerificationReport, VerificationSource, - VerificationSubject, VersionTwoRecordRefusal, + ReaderAttemptLimit, RetentionNamespaceDigest, RetentionRootDecodeError, StoreRecordRefusal, + VerificationDepth, VerificationError, VerificationRefusal, VerificationReport, + VerificationSource, VerificationSubject, }; use std::io; @@ -145,12 +145,12 @@ fn admission_is_corrupt(error: &io::Error) -> bool { { return true; } - if let Some(record) = source.downcast_ref::() { + if let Some(record) = source.downcast_ref::() { return match record { - VersionTwoRecordRefusal::LengthOverflow { .. } => false, - VersionTwoRecordRefusal::KindOrLength { .. } - | VersionTwoRecordRefusal::TrailingBytes { .. } - | VersionTwoRecordRefusal::Marker { .. } => true, + StoreRecordRefusal::LengthOverflow { .. } => false, + StoreRecordRefusal::KindOrLength { .. } + | StoreRecordRefusal::TrailingBytes { .. } + | StoreRecordRefusal::Marker { .. } => true, }; } false diff --git a/src/adapters/retention/filesystem_version_two_admission_tests.rs b/src/adapters/retention/filesystem_store_admission_tests.rs similarity index 56% rename from src/adapters/retention/filesystem_version_two_admission_tests.rs rename to src/adapters/retention/filesystem_store_admission_tests.rs index cfaec3ea..1a7b4683 100644 --- a/src/adapters/retention/filesystem_version_two_admission_tests.rs +++ b/src/adapters/retention/filesystem_store_admission_tests.rs @@ -1,4 +1,4 @@ -//! Version-two reopen laws: writer authority requires a committed namespace and the canonical marker. +//! Store reopen laws: writer authority requires a committed namespace and the canonical marker. use std::error::Error; use std::fs; @@ -12,14 +12,14 @@ use super::{ }; use crate::adapters::filesystem_test_sandbox::TestDirectory; use crate::adapters::{ - FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, - VersionTwoRecordRefusal, + FilesystemNamespaceRefusal, FilesystemStoreAdmission, FilesystemStoreAdmissionError, + StoreRecordRefusal, }; use crate::execute_retention_publication; #[test] -fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box> { - let sandbox = initialized_store("version-two-admission-corrupt-marker")?; +fn reopen_refuses_a_corrupt_format_marker() -> Result<(), Box> { + let sandbox = initialized_store("store-admission-corrupt-marker")?; let marker = sandbox.path().join("FORMAT"); let mut bytes = fs::read(&marker)?; *bytes @@ -27,49 +27,47 @@ fn version_two_reopen_refuses_a_corrupt_format_marker() -> Result<(), Box()), - Some(VersionTwoRecordRefusal::Marker { .. }) + .and_then(|refusal| refusal.downcast_ref::()), + Some(StoreRecordRefusal::Marker { .. }) )); Ok(()) } #[test] -fn version_two_reopen_refuses_an_oversized_format_marker() -> Result<(), Box> { - let sandbox = initialized_store("version-two-admission-oversized-marker")?; +fn reopen_refuses_an_oversized_format_marker() -> Result<(), Box> { + let sandbox = initialized_store("store-admission-oversized-marker")?; let marker = sandbox.path().join("FORMAT"); let mut bytes = fs::read(&marker)?; bytes.push(0); fs::write(&marker, &bytes)?; - let error = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path()) - .err() - .ok_or("oversized format marker was unexpectedly admitted")?; + let error = FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path()) + .err() + .ok_or("oversized format marker was unexpectedly admitted")?; assert!(matches!( error, - FilesystemPlatformAdmissionError::FormatMarker { .. } + FilesystemStoreAdmissionError::FormatMarker { .. } )); Ok(()) } #[test] -fn version_two_reopen_admits_a_canonical_marker() -> Result<(), Box> { - let sandbox = initialized_store("version-two-admission-exact")?; +fn reopen_admits_a_canonical_marker() -> Result<(), Box> { + let sandbox = initialized_store("store-admission-exact")?; let admission = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; + FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; drop(admission); Ok(()) @@ -77,31 +75,30 @@ fn version_two_reopen_admits_a_canonical_marker() -> Result<(), Box> #[cfg(target_os = "linux")] #[test] -fn production_version_two_reopen_refuses_an_aliased_protocol_directory() --> Result<(), Box> { - let sandbox = initialized_store("version-two-admission-aliased-gc")?; +fn production_reopen_refuses_an_aliased_protocol_directory() -> Result<(), Box> { + let sandbox = initialized_store("store-admission-aliased-gc")?; let alias_target = sandbox.path().join("elsewhere"); fs::create_dir(&alias_target)?; fs::remove_dir(sandbox.path().join("gc"))?; std::os::unix::fs::symlink(&alias_target, sandbox.path().join("gc"))?; - let error = FilesystemVersionTwoAdmission::reopen(sandbox.path()) + let error = FilesystemStoreAdmission::reopen(sandbox.path()) .err() .ok_or("aliased gc protocol directory was unexpectedly admitted")?; assert!(matches!( error, - FilesystemPlatformAdmissionError::Platform { .. } + FilesystemStoreAdmissionError::Platform { .. } )); Ok(()) } #[cfg(target_os = "linux")] #[test] -fn production_version_two_reopen_admits_an_exact_initialized_store() -> Result<(), Box> { - let sandbox = initialized_store("version-two-admission-production-exact")?; +fn production_reopen_admits_an_exact_initialized_store() -> Result<(), Box> { + let sandbox = initialized_store("store-admission-production-exact")?; - let admission = FilesystemVersionTwoAdmission::reopen(sandbox.path())?; + let admission = FilesystemStoreAdmission::reopen(sandbox.path())?; drop(admission); Ok(()) @@ -114,45 +111,44 @@ fn refuses_namespace( let sandbox = initialized_store(name)?; mutate(sandbox.path())?; - let error = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path()) - .err() - .ok_or_else(|| format!("{name}: version-two root was unexpectedly admitted"))?; + let error = FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path()) + .err() + .ok_or_else(|| format!("{name}: store root was unexpectedly admitted"))?; assert!( - matches!(error, FilesystemPlatformAdmissionError::Namespace { .. }), + matches!(error, FilesystemStoreAdmissionError::Namespace { .. }), "{name}: expected a Namespace refusal, got {error:?}" ); Ok(()) } #[test] -fn version_two_reopen_refuses_a_missing_dispositions_directory() -> Result<(), Box> { - refuses_namespace("version-two-admission-no-dispositions", |root| { +fn reopen_refuses_a_missing_dispositions_directory() -> Result<(), Box> { + refuses_namespace("store-admission-no-dispositions", |root| { fs::remove_dir(root.join("recovery").join("dispositions"))?; Ok(()) }) } #[test] -fn version_two_reopen_refuses_a_stray_gc_entry() -> Result<(), Box> { - refuses_namespace("version-two-admission-gc-junk", |root| { +fn reopen_refuses_a_stray_gc_entry() -> Result<(), Box> { + refuses_namespace("store-admission-gc-junk", |root| { fs::write(root.join("gc").join("junk"), b"")?; Ok(()) }) } #[test] -fn version_two_reopen_refuses_a_stray_recovery_entry() -> Result<(), Box> { - refuses_namespace("version-two-admission-recovery-junk", |root| { +fn reopen_refuses_a_stray_recovery_entry() -> Result<(), Box> { + refuses_namespace("store-admission-recovery-junk", |root| { fs::write(root.join("recovery").join("junk"), b"")?; Ok(()) }) } #[test] -fn version_two_reopen_refuses_a_missing_retention_pool() -> Result<(), Box> { - refuses_namespace("version-two-admission-no-roots", |root| { +fn reopen_refuses_a_missing_retention_pool() -> Result<(), Box> { + refuses_namespace("store-admission-no-roots", |root| { fs::remove_dir(root.join("retention").join("roots"))?; Ok(()) }) @@ -161,13 +157,13 @@ fn version_two_reopen_refuses_a_missing_retention_pool() -> Result<(), Box Result<(), Box> { - let (sandbox, mut first) = open_authority("version-two-admission-replaced-retention")?; + let (sandbox, mut first) = open_authority("store-admission-replaced-retention")?; let root_bytes = fixture(ROOT_HEX)?; let preparation = initial_preparation(&root_bytes)?; let _published = execute_retention_publication(&mut first, &preparation)?; drop(first); let admission = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; + FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; fs::rename( sandbox.path().join("retention"), sandbox.path().join("retention.moved"), @@ -197,12 +193,12 @@ fn a_protocol_directory_replaced_after_reopen_is_neither_opened_nor_published_in #[test] fn an_initialized_store_without_a_published_catalog_reopens() -> Result<(), Box> { - let sandbox = TestDirectory::create("version-two-admission-unpublished")?; - drop(FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(sandbox.path())?); + let sandbox = TestDirectory::create("store-admission-unpublished")?; + drop(FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(sandbox.path())?); assert!(!sandbox.path().join("HEAD").exists()); let admission = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; + FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?; drop(admission); Ok(()) @@ -210,7 +206,7 @@ fn an_initialized_store_without_a_published_catalog_reopens() -> Result<(), Box< #[test] fn reopen_refuses_a_root_whose_marker_stage_remains() -> Result<(), Box> { - let sandbox = initialized_store("version-two-admission-marker-stage")?; + let sandbox = initialized_store("store-admission-marker-stage")?; fs::hard_link( sandbox.path().join("FORMAT"), sandbox.path().join("FORMAT.next"), @@ -224,7 +220,7 @@ fn reopen_refuses_a_root_whose_marker_stage_remains() -> Result<(), Box Result<(), Box> { - let sandbox = initialized_store("version-two-admission-no-marker")?; + let sandbox = initialized_store("store-admission-no-marker")?; fs::remove_file(sandbox.path().join("FORMAT"))?; let error = reopen_namespace_refusal(sandbox.path())?; @@ -236,10 +232,10 @@ fn reopen_refuses_a_root_without_a_marker() -> Result<(), Box> { fn reopen_namespace_refusal( root: &std::path::Path, ) -> Result> { - let error = FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(root) + let error = FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(root) .err() .ok_or("an uncommitted root was unexpectedly admitted")?; - let FilesystemPlatformAdmissionError::Namespace { source } = error else { + let FilesystemStoreAdmissionError::Namespace { source } = error else { return Err(format!("expected a namespace refusal, got {error:?}").into()); }; source diff --git a/src/adapters/retention/reader_fence.rs b/src/adapters/retention/reader_fence.rs index 9218aff3..104c1529 100644 --- a/src/adapters/retention/reader_fence.rs +++ b/src/adapters/retention/reader_fence.rs @@ -1,4 +1,4 @@ -//! This module owns the shared reader fence over one version-two store root. +//! This module owns the shared reader fence over one store root. use std::io; diff --git a/src/adapters/retention/verification_store_admission_tests.rs b/src/adapters/retention/verification_store_admission_tests.rs index c6a156a9..f91f48bf 100644 --- a/src/adapters/retention/verification_store_admission_tests.rs +++ b/src/adapters/retention/verification_store_admission_tests.rs @@ -4,8 +4,8 @@ use super::filesystem_retention_test_fixture::initialized_store; use crate::{ CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, ReaderAttemptLimit, SegmentReadPolicy, - StoreFormatMarkerDecodeError, VerificationError, VerificationObservation, VerificationRefusal, - VerificationSource, VerificationSubject, VersionTwoRecordRefusal, + StoreFormatMarkerDecodeError, StoreRecordRefusal, VerificationError, VerificationObservation, + VerificationRefusal, VerificationSource, VerificationSubject, }; use std::{error::Error, fs, io}; @@ -32,8 +32,8 @@ fn a_malformed_format_marker_is_corrupt_with_the_original_magic() -> ResultOf<() )?; let typed = source .get_ref() - .and_then(|source| source.downcast_ref::()); - let Some(VersionTwoRecordRefusal::Marker { + .and_then(|source| source.downcast_ref::()); + let Some(StoreRecordRefusal::Marker { source: StoreFormatMarkerDecodeError::InvalidMagic { observed }, }) = typed else { @@ -84,7 +84,7 @@ fn policy() -> ResultOf { fn inconclusive_admission_causes_remain_operational_without_losing_the_source() -> ResultOf<()> { let source = io::Error::new( io::ErrorKind::InvalidData, - VersionTwoRecordRefusal::LengthOverflow { name: "FORMAT" }, + StoreRecordRefusal::LengthOverflow { name: "FORMAT" }, ); let error = VerificationError::from(FilesystemRetentionSnapshotError::Admission { source }); let source = operational_cause(error)?; @@ -92,8 +92,8 @@ fn inconclusive_admission_causes_remain_operational_without_losing_the_source() matches!( source .get_ref() - .and_then(|source| source.downcast_ref::()), - Some(VersionTwoRecordRefusal::LengthOverflow { name: "FORMAT" }) + .and_then(|source| source.downcast_ref::()), + Some(StoreRecordRefusal::LengthOverflow { name: "FORMAT" }) ), "host-width cause must survive: {source:?}" ); diff --git a/src/adapters/segment_publication.rs b/src/adapters/segment_publication.rs index 3a64a9e6..31987541 100644 --- a/src/adapters/segment_publication.rs +++ b/src/adapters/segment_publication.rs @@ -29,7 +29,7 @@ impl<'selection, 'records> SegmentPublication<'selection, 'records> { /// Binds one storage-agnostic closed stage to its exact admitted bytes. /// /// This selection carries no filesystem-publisher authority. Use - /// [`crate::FilesystemVersionTwoCatalogPublisher::select_segment`] for filesystem + /// [`crate::FilesystemCatalogPublisher::select_segment`] for filesystem /// publication. /// /// # Errors diff --git a/src/adapters/version_two_initialization.rs b/src/adapters/store_initialization.rs similarity index 68% rename from src/adapters/version_two_initialization.rs rename to src/adapters/store_initialization.rs index f45067e2..11094649 100644 --- a/src/adapters/version_two_initialization.rs +++ b/src/adapters/store_initialization.rs @@ -1,4 +1,4 @@ -//! This module owns direct initialization of a version-two store under writer authority. +//! This module owns direct initialization of a store under writer authority. //! //! Initialization is the only way a store comes into existence. It is a //! recoverable state machine: every interrupted state is a classified prefix @@ -24,18 +24,18 @@ mod initialization_resume_tests; #[cfg(test)] mod initialization_test_fixture; -pub use initialization_error::VersionTwoInitializationError; -pub use initialization_phase::{VersionTwoInitializationDirectory, VersionTwoInitializationPhase}; -pub use initialization_prefix::VersionTwoInitializationPrefix; -pub use initialization_receipt::VersionTwoInitialization; -pub use initialization_refusal::VersionTwoInitializationRefusal; +pub use initialization_error::StoreInitializationError; +pub use initialization_phase::{StoreInitializationDirectory, StoreInitializationPhase}; +pub use initialization_prefix::StoreInitializationPrefix; +pub use initialization_receipt::StoreInitialization; +pub use initialization_refusal::StoreInitializationRefusal; pub(super) use initialization_execution::{PhaseObserver, initialize}; /// Where an initialization observer runs relative to one phase. #[doc(hidden)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum VersionTwoInitializationBoundary { +pub enum StoreInitializationBoundary { /// Before the phase begins. Before, /// After the phase returned successfully. @@ -48,9 +48,5 @@ pub enum VersionTwoInitializationBoundary { /// failure would; the crash matrix blocks inside it for process death. #[cfg(any(test, feature = "repository-tasks"))] #[doc(hidden)] -pub type VersionTwoInitializationObserver = Box< - dyn FnMut( - VersionTwoInitializationPhase, - VersionTwoInitializationBoundary, - ) -> std::io::Result<()>, ->; +pub type StoreInitializationObserver = + Box std::io::Result<()>>; diff --git a/src/adapters/version_two_initialization/initialization_entries.rs b/src/adapters/store_initialization/initialization_entries.rs similarity index 98% rename from src/adapters/version_two_initialization/initialization_entries.rs rename to src/adapters/store_initialization/initialization_entries.rs index cb70c399..b8c88fc4 100644 --- a/src/adapters/version_two_initialization/initialization_entries.rs +++ b/src/adapters/store_initialization/initialization_entries.rs @@ -9,7 +9,7 @@ use std::io::{self, Write}; use cap_fs_ext::DirExt; use cap_std::fs::{Dir, File}; -use super::VersionTwoInitializationDirectory as Directory; +use super::StoreInitializationDirectory as Directory; use super::initialization_namespace::{MARKER, MARKER_STAGE}; use crate::adapters::filesystem_exact_record::{ self as exact_record, EntryIdentity, ExactRecordError, diff --git a/src/adapters/version_two_initialization/initialization_error.rs b/src/adapters/store_initialization/initialization_error.rs similarity index 71% rename from src/adapters/version_two_initialization/initialization_error.rs rename to src/adapters/store_initialization/initialization_error.rs index a81bf63d..96461cbb 100644 --- a/src/adapters/version_two_initialization/initialization_error.rs +++ b/src/adapters/store_initialization/initialization_error.rs @@ -4,17 +4,17 @@ use std::error::Error; use std::fmt; use std::io; -use super::{VersionTwoInitializationPhase, VersionTwoInitializationRefusal}; -use crate::adapters::{FilesystemPlatformAdmissionError, WriterLockAcquireError}; +use super::{StoreInitializationPhase, StoreInitializationRefusal}; +use crate::adapters::{FilesystemStoreAdmissionError, WriterLockAcquireError}; -/// Failure of direct version-two initialization. +/// Failure of direct initialization. /// /// A failure leaves the root in a state that the protocol classifies: either /// unchanged, or a recoverable prefix that a later call resumes. No variant /// implies cleanup was attempted. #[derive(Debug)] #[non_exhaustive] -pub enum VersionTwoInitializationError { +pub enum StoreInitializationError { /// The root could not be opened or does not satisfy the platform profile. Platform { /// Preserved platform-admission failure. @@ -28,33 +28,33 @@ pub enum VersionTwoInitializationError { /// The root is not a recoverable initialization prefix. Refused { /// The exact refusal and its evidence. - refusal: VersionTwoInitializationRefusal, + refusal: StoreInitializationRefusal, }, /// One phase failed with an I/O error or a concurrent change it detected. Io { /// The exact failed phase. - phase: VersionTwoInitializationPhase, + phase: StoreInitializationPhase, /// Preserved storage failure. source: io::Error, }, - /// The committed store failed version-two admission. + /// The committed store failed store admission. Admission { /// Preserved admission failure. - source: FilesystemPlatformAdmissionError, + source: FilesystemStoreAdmissionError, }, } -impl VersionTwoInitializationError { - pub(super) const fn refused(refusal: VersionTwoInitializationRefusal) -> Self { +impl StoreInitializationError { + pub(super) const fn refused(refusal: StoreInitializationRefusal) -> Self { Self::Refused { refusal } } - pub(super) const fn io(phase: VersionTwoInitializationPhase, source: io::Error) -> Self { + pub(super) const fn io(phase: StoreInitializationPhase, source: io::Error) -> Self { Self::Io { phase, source } } } -impl fmt::Display for VersionTwoInitializationError { +impl fmt::Display for StoreInitializationError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Platform { .. } => { @@ -66,13 +66,13 @@ impl fmt::Display for VersionTwoInitializationError { Self::Refused { refusal } => write!(formatter, "initialization refused: {refusal}"), Self::Io { phase, .. } => write!(formatter, "initialization failed during {phase}"), Self::Admission { .. } => { - formatter.write_str("initialized store failed version-two admission") + formatter.write_str("initialized store failed store admission") } } } } -impl Error for VersionTwoInitializationError { +impl Error for StoreInitializationError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { Self::Platform { source } | Self::Io { source, .. } => Some(source), diff --git a/src/adapters/version_two_initialization/initialization_execution.rs b/src/adapters/store_initialization/initialization_execution.rs similarity index 95% rename from src/adapters/version_two_initialization/initialization_execution.rs rename to src/adapters/store_initialization/initialization_execution.rs index b1ad79e2..b148886b 100644 --- a/src/adapters/version_two_initialization/initialization_execution.rs +++ b/src/adapters/store_initialization/initialization_execution.rs @@ -12,9 +12,9 @@ use cap_std::fs::Dir; use super::initialization_entries::{self as entries, MarkerStage}; use super::initialization_reader_fence::admit_reader_fence; use super::{ - VersionTwoInitializationBoundary as Boundary, VersionTwoInitializationDirectory as Directory, - VersionTwoInitializationError as Error, VersionTwoInitializationPhase as Phase, - VersionTwoInitializationPrefix as Prefix, initialization_observation, + StoreInitializationBoundary as Boundary, StoreInitializationDirectory as Directory, + StoreInitializationError as Error, StoreInitializationPhase as Phase, + StoreInitializationPrefix as Prefix, initialization_observation, }; use crate::adapters::FilesystemWriterLock; diff --git a/src/adapters/version_two_initialization/initialization_namespace.rs b/src/adapters/store_initialization/initialization_namespace.rs similarity index 87% rename from src/adapters/version_two_initialization/initialization_namespace.rs rename to src/adapters/store_initialization/initialization_namespace.rs index 0c9caf09..e46a1ac1 100644 --- a/src/adapters/version_two_initialization/initialization_namespace.rs +++ b/src/adapters/store_initialization/initialization_namespace.rs @@ -5,7 +5,7 @@ use std::io; use cap_std::fs::Dir; -use crate::adapters::filesystem_initialization_namespace; +use crate::adapters::filesystem_store_namespace; pub(super) const WRITER_LOCK: &str = "writer.lock"; pub(super) const READER_LOCK: &str = "reader.lock"; @@ -36,5 +36,5 @@ pub(super) fn is_root_name(name: &OsStr) -> bool { /// Admits the committed namespace that may still hold `FORMAT.next`. pub(super) fn admit_linked_namespace(root: &Dir) -> io::Result<()> { - filesystem_initialization_namespace::admit_version_two_marker_linked(root) + filesystem_store_namespace::admit_store_marker_linked(root) } diff --git a/src/adapters/version_two_initialization/initialization_observation.rs b/src/adapters/store_initialization/initialization_observation.rs similarity index 97% rename from src/adapters/version_two_initialization/initialization_observation.rs rename to src/adapters/store_initialization/initialization_observation.rs index e1ee607a..6f5ae464 100644 --- a/src/adapters/version_two_initialization/initialization_observation.rs +++ b/src/adapters/store_initialization/initialization_observation.rs @@ -12,9 +12,9 @@ use cap_fs_ext::DirExt; use cap_std::fs::Dir; use super::{ - VersionTwoInitializationDirectory as Directory, VersionTwoInitializationError as Error, - VersionTwoInitializationPhase as Phase, VersionTwoInitializationPrefix as Prefix, - VersionTwoInitializationRefusal as Refusal, initialization_namespace as names, + StoreInitializationDirectory as Directory, StoreInitializationError as Error, + StoreInitializationPhase as Phase, StoreInitializationPrefix as Prefix, + StoreInitializationRefusal as Refusal, initialization_namespace as names, }; use crate::adapters::filesystem_exact_record::{self as exact_record, EntryIdentity}; use crate::adapters::filesystem_namespace_refusal::FilesystemEntryKind; diff --git a/src/adapters/version_two_initialization/initialization_phase.rs b/src/adapters/store_initialization/initialization_phase.rs similarity index 92% rename from src/adapters/version_two_initialization/initialization_phase.rs rename to src/adapters/store_initialization/initialization_phase.rs index cc31c0bf..009143f9 100644 --- a/src/adapters/version_two_initialization/initialization_phase.rs +++ b/src/adapters/store_initialization/initialization_phase.rs @@ -1,4 +1,4 @@ -//! This module owns the ordered phases of direct version-two initialization. +//! This module owns the ordered phases of direct initialization. use std::fmt; @@ -7,7 +7,7 @@ use std::fmt; /// A nested directory is created only after its parent, and every directory /// is created only after the one before it in [`Self::ALL`]. #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -pub enum VersionTwoInitializationDirectory { +pub enum StoreInitializationDirectory { /// `staging`. Staging, /// `segments`. @@ -28,7 +28,7 @@ pub enum VersionTwoInitializationDirectory { RecoveryDispositions, } -impl VersionTwoInitializationDirectory { +impl StoreInitializationDirectory { /// Every directory in creation order. pub const ALL: [Self; 9] = [ Self::Staging, @@ -89,9 +89,9 @@ impl VersionTwoInitializationDirectory { } } -/// One exact operation of direct version-two initialization. +/// One exact operation of direct initialization. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum VersionTwoInitializationPhase { +pub enum StoreInitializationPhase { /// Open the root without following links and admit the platform profile. AdmitPlatform, /// Classify the root without mutation. @@ -110,7 +110,7 @@ pub enum VersionTwoInitializationPhase { /// Create or admit zero-length `reader.lock` and synchronize the root. AdmitReaderFence, /// Create or admit one protocol directory and synchronize its parent. - AdmitDirectory(VersionTwoInitializationDirectory), + AdmitDirectory(StoreInitializationDirectory), /// Hard-link `FORMAT.next` to `FORMAT` without replacement. LinkMarker, /// Synchronize the root after the link: the commit point. @@ -121,7 +121,7 @@ pub enum VersionTwoInitializationPhase { SynchronizeRootAfterCleanup, } -impl fmt::Display for VersionTwoInitializationPhase { +impl fmt::Display for StoreInitializationPhase { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::AdmitDirectory(directory) => { @@ -132,7 +132,7 @@ impl fmt::Display for VersionTwoInitializationPhase { } } -impl VersionTwoInitializationPhase { +impl StoreInitializationPhase { const fn label(self) -> &'static str { match self { Self::AdmitPlatform => "platform admission", diff --git a/src/adapters/version_two_initialization/initialization_prefix.rs b/src/adapters/store_initialization/initialization_prefix.rs similarity index 89% rename from src/adapters/version_two_initialization/initialization_prefix.rs rename to src/adapters/store_initialization/initialization_prefix.rs index a12ea3e4..68be66fc 100644 --- a/src/adapters/version_two_initialization/initialization_prefix.rs +++ b/src/adapters/store_initialization/initialization_prefix.rs @@ -1,6 +1,6 @@ //! This module owns the lawful states a root can hold before initialization commits. -use super::VersionTwoInitializationDirectory; +use super::StoreInitializationDirectory; /// One recoverable initialization state, observed under writer authority. /// @@ -9,7 +9,7 @@ use super::VersionTwoInitializationDirectory; /// state is refused, and a committed root is refused as already initialized, /// so no variant describes a committed store. #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum VersionTwoInitializationPrefix { +pub enum StoreInitializationPrefix { /// The root was empty. Empty, /// The root held only a zero-length `writer.lock`. @@ -26,7 +26,7 @@ pub enum VersionTwoInitializationPrefix { /// Every directory up to and including `last` existed, in order. Directories { /// The last directory present in creation order. - last: VersionTwoInitializationDirectory, + last: StoreInitializationDirectory, }, /// `FORMAT` named the same inode as the canonical `FORMAT.next`. MarkerLinked, diff --git a/src/adapters/version_two_initialization/initialization_prefix_tests.rs b/src/adapters/store_initialization/initialization_prefix_tests.rs similarity index 97% rename from src/adapters/version_two_initialization/initialization_prefix_tests.rs rename to src/adapters/store_initialization/initialization_prefix_tests.rs index c1f609a2..50dc8ece 100644 --- a/src/adapters/version_two_initialization/initialization_prefix_tests.rs +++ b/src/adapters/store_initialization/initialization_prefix_tests.rs @@ -6,9 +6,7 @@ use std::path::PathBuf; use super::initialization_test_fixture::{ TestResult, initialize, initialize_until, require_refusal, witness, }; -use super::{ - VersionTwoInitializationDirectory as Directory, VersionTwoInitializationRefusal as Refusal, -}; +use super::{StoreInitializationDirectory as Directory, StoreInitializationRefusal as Refusal}; use crate::adapters::filesystem_test_sandbox::TestDirectory; use crate::adapters::{CanonicalStoreFormatMarker, FORMAT_MARKER_LENGTH, FilesystemEntryKind}; diff --git a/src/adapters/version_two_initialization/initialization_reader_fence.rs b/src/adapters/store_initialization/initialization_reader_fence.rs similarity index 100% rename from src/adapters/version_two_initialization/initialization_reader_fence.rs rename to src/adapters/store_initialization/initialization_reader_fence.rs diff --git a/src/adapters/version_two_initialization/initialization_receipt.rs b/src/adapters/store_initialization/initialization_receipt.rs similarity index 57% rename from src/adapters/version_two_initialization/initialization_receipt.rs rename to src/adapters/store_initialization/initialization_receipt.rs index 7bd30f56..a8ade1fd 100644 --- a/src/adapters/version_two_initialization/initialization_receipt.rs +++ b/src/adapters/store_initialization/initialization_receipt.rs @@ -1,38 +1,38 @@ //! This module owns the result of one completed direct initialization. -use super::VersionTwoInitializationPrefix; -use crate::adapters::FilesystemVersionTwoAdmission; +use super::StoreInitializationPrefix; +use crate::adapters::FilesystemStoreAdmission; /// Writer authority over a store whose initialization has durably committed. /// /// Returned only after the commit synchronization, the removal of -/// `FORMAT.next`, the final root synchronization, and complete version-two +/// `FORMAT.next`, the final root synchronization, and complete store /// admission of the committed root. The store holds no catalog `HEAD` until /// the first publication. #[must_use] -pub struct VersionTwoInitialization { - admission: FilesystemVersionTwoAdmission, - found: VersionTwoInitializationPrefix, +pub struct StoreInitialization { + admission: FilesystemStoreAdmission, + found: StoreInitializationPrefix, } -impl VersionTwoInitialization { +impl StoreInitialization { pub(in crate::adapters) const fn new( - admission: FilesystemVersionTwoAdmission, - found: VersionTwoInitializationPrefix, + admission: FilesystemStoreAdmission, + found: StoreInitializationPrefix, ) -> Self { Self { admission, found } } /// The state the call found under writer authority. /// - /// A fresh root reports [`VersionTwoInitializationPrefix::Empty`]; any + /// A fresh root reports [`StoreInitializationPrefix::Empty`]; any /// other value names the interrupted state the call resumed. - pub const fn found(&self) -> VersionTwoInitializationPrefix { + pub const fn found(&self) -> StoreInitializationPrefix { self.found } /// Releases the retained writer authority for forward publication. - pub fn into_admission(self) -> FilesystemVersionTwoAdmission { + pub fn into_admission(self) -> FilesystemStoreAdmission { self.admission } } diff --git a/src/adapters/version_two_initialization/initialization_refusal.rs b/src/adapters/store_initialization/initialization_refusal.rs similarity index 95% rename from src/adapters/version_two_initialization/initialization_refusal.rs rename to src/adapters/store_initialization/initialization_refusal.rs index 22be977b..c38cf39b 100644 --- a/src/adapters/version_two_initialization/initialization_refusal.rs +++ b/src/adapters/store_initialization/initialization_refusal.rs @@ -13,7 +13,7 @@ use crate::adapters::FilesystemEntryKind; /// under writer authority before any later phase. #[derive(Clone, Debug, Eq, PartialEq)] #[non_exhaustive] -pub enum VersionTwoInitializationRefusal { +pub enum StoreInitializationRefusal { /// The root holds `FORMAT` without `FORMAT.next`: initialization has /// already committed. Reopen the store instead. AlreadyInitialized, @@ -52,7 +52,7 @@ pub enum VersionTwoInitializationRefusal { MarkerLinkMismatch, } -impl fmt::Display for VersionTwoInitializationRefusal { +impl fmt::Display for StoreInitializationRefusal { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::AlreadyInitialized => formatter.write_str("the store is already initialized"), @@ -88,4 +88,4 @@ impl fmt::Display for VersionTwoInitializationRefusal { } } -impl Error for VersionTwoInitializationRefusal {} +impl Error for StoreInitializationRefusal {} diff --git a/src/adapters/version_two_initialization/initialization_resume_tests.rs b/src/adapters/store_initialization/initialization_resume_tests.rs similarity index 87% rename from src/adapters/version_two_initialization/initialization_resume_tests.rs rename to src/adapters/store_initialization/initialization_resume_tests.rs index 6485a2bb..02a99b20 100644 --- a/src/adapters/version_two_initialization/initialization_resume_tests.rs +++ b/src/adapters/store_initialization/initialization_resume_tests.rs @@ -9,14 +9,14 @@ use super::initialization_test_fixture::{ require_refusal, }; use super::{ - VersionTwoInitializationBoundary as Boundary, VersionTwoInitializationError as Error, - VersionTwoInitializationObserver, VersionTwoInitializationPhase as Phase, - VersionTwoInitializationPrefix as Prefix, VersionTwoInitializationRefusal as Refusal, + StoreInitializationBoundary as Boundary, StoreInitializationError as Error, + StoreInitializationObserver, StoreInitializationPhase as Phase, + StoreInitializationPrefix as Prefix, StoreInitializationRefusal as Refusal, initialization_observation, }; use crate::adapters::filesystem_test_sandbox::TestDirectory; use crate::adapters::{ - CanonicalStoreFormatMarker, FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, + CanonicalStoreFormatMarker, FilesystemStoreAdmission, FilesystemStoreAdmissionError, WriterLockAcquireError, }; @@ -116,7 +116,7 @@ fn an_incomplete_marker_stage_is_discarded_and_rewritten() -> TestResult<()> { #[test] fn a_failed_commit_synchronization_leaves_an_unadmitted_resumable_store() -> TestResult<()> { let sandbox = TestDirectory::create("initialization-commit-sync-failure")?; - let observer: VersionTwoInitializationObserver = Box::new(|phase, boundary| { + let observer: StoreInitializationObserver = Box::new(|phase, boundary| { if phase == Phase::SynchronizeRootAfterMarker && boundary == Boundary::Before { Err(std::io::Error::other("injected synchronization failure")) } else { @@ -124,7 +124,7 @@ fn a_failed_commit_synchronization_leaves_an_unadmitted_resumable_store() -> Tes } }); - let failed = FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks( + let failed = FilesystemStoreAdmission::initialize_observed_for_repository_tasks( sandbox.path(), observer, ); @@ -136,14 +136,13 @@ fn a_failed_commit_synchronization_leaves_an_unadmitted_resumable_store() -> Tes .. }) )); - let refused = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path()); + let refused = FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path()); assert!(matches!( refused, - Err(FilesystemPlatformAdmissionError::Namespace { .. }) + Err(FilesystemStoreAdmissionError::Namespace { .. }) )); assert_eq!(initialize(sandbox.path())?.found(), Prefix::MarkerLinked); - drop(FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?); + drop(FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?); Ok(()) } @@ -156,14 +155,14 @@ fn concurrent_attempts_have_exactly_one_initializer() -> TestResult<()> { let (reached, paused) = mpsc::sync_channel(0); let (resume, resumed) = mpsc::sync_channel::<()>(0); let first = thread::spawn(move || { - let observer: VersionTwoInitializationObserver = Box::new(move |phase, boundary| { + let observer: StoreInitializationObserver = Box::new(move |phase, boundary| { if phase == Phase::AdmitReaderFence && boundary == Boundary::Before { reached.send(()).map_err(std::io::Error::other)?; resumed.recv().map_err(std::io::Error::other)?; } Ok(()) }); - FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks(&root, observer) + FilesystemStoreAdmission::initialize_observed_for_repository_tasks(&root, observer) .map(|initialized| initialized.found()) }); paused.recv()?; diff --git a/src/adapters/version_two_initialization/initialization_test_fixture.rs b/src/adapters/store_initialization/initialization_test_fixture.rs similarity index 74% rename from src/adapters/version_two_initialization/initialization_test_fixture.rs rename to src/adapters/store_initialization/initialization_test_fixture.rs index 9938c580..6a2435cb 100644 --- a/src/adapters/version_two_initialization/initialization_test_fixture.rs +++ b/src/adapters/store_initialization/initialization_test_fixture.rs @@ -8,11 +8,10 @@ use std::path::Path; use std::sync::{Arc, Mutex}; use super::{ - VersionTwoInitialization, VersionTwoInitializationBoundary as Boundary, - VersionTwoInitializationError, VersionTwoInitializationObserver, - VersionTwoInitializationPhase as Phase, VersionTwoInitializationRefusal, + StoreInitialization, StoreInitializationBoundary as Boundary, StoreInitializationError, + StoreInitializationObserver, StoreInitializationPhase as Phase, StoreInitializationRefusal, }; -use crate::adapters::FilesystemVersionTwoAdmission; +use crate::adapters::FilesystemStoreAdmission; pub(super) type TestResult = Result>; @@ -32,10 +31,8 @@ pub(super) const COMMITTED: [&str; 12] = [ "writer.lock", ]; -pub(super) fn initialize( - root: &Path, -) -> Result { - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(root) +pub(super) fn initialize(root: &Path) -> Result { + FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(root) } /// Initializes with an observer that fails the `stop`-th boundary notification. @@ -45,9 +42,9 @@ pub(super) fn initialize( pub(super) fn initialize_until( root: &Path, stop: usize, -) -> Result { +) -> Result { let mut seen = 0_usize; - let observer: VersionTwoInitializationObserver = Box::new(move |_phase, _boundary| { + let observer: StoreInitializationObserver = Box::new(move |_phase, _boundary| { let index = seen; seen = seen.saturating_add(1); if index == stop { @@ -56,20 +53,20 @@ pub(super) fn initialize_until( Ok(()) } }); - FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks(root, observer) + FilesystemStoreAdmission::initialize_observed_for_repository_tasks(root, observer) } /// Records every boundary notification of one complete initialization. pub(super) fn recorded_boundaries(root: &Path) -> TestResult> { let record = Arc::new(Mutex::new(Vec::new())); let sink = Arc::clone(&record); - let observer: VersionTwoInitializationObserver = Box::new(move |phase, boundary| { + let observer: StoreInitializationObserver = Box::new(move |phase, boundary| { sink.lock() .map_err(|_| io::Error::other("boundary record poisoned"))? .push((phase, boundary)); Ok(()) }); - drop(FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks(root, observer)?); + drop(FilesystemStoreAdmission::initialize_observed_for_repository_tasks(root, observer)?); let boundaries = record .lock() .map_err(|_| "boundary record poisoned")? @@ -115,11 +112,11 @@ pub(super) fn witness(root: &Path) -> TestResult> { /// Requires `result` to be exactly `expected` as an initialization refusal. pub(super) fn require_refusal( - result: Result, - expected: &VersionTwoInitializationRefusal, + result: Result, + expected: &StoreInitializationRefusal, ) -> TestResult<()> { match result { - Err(VersionTwoInitializationError::Refused { refusal }) if &refusal == expected => Ok(()), + Err(StoreInitializationError::Refused { refusal }) if &refusal == expected => Ok(()), Err(other) => Err(format!("expected {expected:?}, got {other:?}").into()), Ok(_) => Err(format!("expected {expected:?}, but initialization admitted").into()), } diff --git a/src/adapters/verification_failure_class.rs b/src/adapters/verification_failure_class.rs index 3ede3a18..6dd0a7cd 100644 --- a/src/adapters/verification_failure_class.rs +++ b/src/adapters/verification_failure_class.rs @@ -1,8 +1,8 @@ //! This module owns resource-versus-content classification at durable ingress. use crate::{ - CatalogAdmissionError, CatalogRestartError, FilesystemPlatformAdmissionError, - LayoutDecodeError, SegmentReadError, SegmentRecordAdmissionError, + CatalogAdmissionError, CatalogRestartError, FilesystemStoreAdmissionError, LayoutDecodeError, + SegmentReadError, SegmentRecordAdmissionError, }; #[derive(Clone, Copy)] @@ -119,10 +119,10 @@ fn admission_class(error: &CatalogAdmissionError) -> FailureClass { /// A namespace or marker refusal is demonstrated corruption; anything else /// that stopped admission is an operational failure. -fn store_admission_class(error: &FilesystemPlatformAdmissionError) -> FailureClass { +fn store_admission_class(error: &FilesystemStoreAdmissionError) -> FailureClass { match error { - FilesystemPlatformAdmissionError::Namespace { source } - | FilesystemPlatformAdmissionError::FormatMarker { source } + FilesystemStoreAdmissionError::Namespace { source } + | FilesystemStoreAdmissionError::FormatMarker { source } if source.kind() == std::io::ErrorKind::InvalidData => { FailureClass::Corrupt diff --git a/src/lib.rs b/src/lib.rs index 54bf8f02..d4a9a616 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,7 +31,7 @@ //! Ordered publication revalidates authority, executes all durability phases, //! and returns a complete receipt. The exact version-2 store-format marker has //! canonical encoding, registered-definition admission, checksum verification, -//! and domain-separated identity. [`FilesystemVersionTwoAdmission::initialize`] +//! and domain-separated identity. [`FilesystemStoreAdmission::initialize`] //! creates a store directly under writer authority through a recoverable //! state machine with one durable commit point; an interrupted call resumes //! and every other nonempty root refuses before mutation. Filesystem @@ -77,7 +77,7 @@ pub use adapters::{ #[cfg(any(test, feature = "repository-tasks"))] #[doc(hidden)] -pub use adapters::VersionTwoInitializationObserver; +pub use adapters::StoreInitializationObserver; pub use adapters::{ AdmittedCatalog, AdmittedRecoveryStageBytes, AdmittedSegment, AdmittedSegmentRecord, AdmittedStoreFormatMarker, BlobIdBinaryParseError, BlobIdTextParseError, CanonicalCatalog, @@ -90,14 +90,14 @@ pub use adapters::{ CatalogRestartPhase, CatalogRestartPolicy, CatalogSnapshot, CatalogSnapshotError, CatalogSuccessor, CatalogTransitionError, ChecksummedCatalog, ChecksummedPublicationHead, ChecksummedSegmentRecord, ClosedSegment, FilesystemCatalogPublicationError, - FilesystemCatalogSnapshot, FilesystemPlatformAdmissionError, FilesystemRecoveryInventoryReader, - FilesystemRecoveryNextHeadFinalizationOpenError, FilesystemRecoveryNextHeadFinalizer, - FilesystemRecoverySegmentResumeOpenError, FilesystemRecoverySegmentResumer, - FilesystemRecoverySegmentStage, FilesystemRecoveryStageCompleter, - FilesystemRecoveryStageCompletionOpenError, FilesystemRecoveryStageDiscardOpenError, - FilesystemRecoveryStageDiscarder, FilesystemRecoveryStageError, FilesystemSegmentStage, - FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, - FilesystemVersionTwoCatalogStages, FilesystemVersionTwoRecoveryAdmission, FilesystemWriterLock, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemRecoveryAdmission, + FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizationOpenError, + FilesystemRecoveryNextHeadFinalizer, FilesystemRecoverySegmentResumeOpenError, + FilesystemRecoverySegmentResumer, FilesystemRecoverySegmentStage, + FilesystemRecoveryStageCompleter, FilesystemRecoveryStageCompletionOpenError, + FilesystemRecoveryStageDiscardOpenError, FilesystemRecoveryStageDiscarder, + FilesystemRecoveryStageError, FilesystemRetainedCatalogStages, FilesystemSegmentStage, + FilesystemStoreAdmission, FilesystemStoreAdmissionError, FilesystemWriterLock, LayoutDecodeError, LayoutDecodePolicy, LayoutEncodeError, LayoutIdBinaryParseError, LayoutIdTextParseError, OpenedReusableSegment, PublicationHeadDecodeError, RecoveryCatalogStage, RecoveryCatalogStageError, RecoveryEntryName, RecoveryEntryNameError, @@ -131,12 +131,11 @@ pub use adapters::{ SegmentRecords, SegmentSeal, SegmentSealError, SegmentStage, SegmentStageCreateError, SegmentWriteError, SegmentWritePhase, StagedSegment, StorageProfileIdParseError, StoreFormatDefinitionDigest, StoreFormatMarkerDecodeError, StoreFormatMarkerDigest, - VersionTwoInitialization, VersionTwoInitializationBoundary, VersionTwoInitializationDirectory, - VersionTwoInitializationError, VersionTwoInitializationPhase, VersionTwoInitializationPrefix, - VersionTwoInitializationRefusal, VersionTwoRecordRefusal, WriterLockAcquireError, - WriterLockAcquirePhase, admit_recovery_stage_bytes, assess_recovery_stage, - classify_recovery_catalog_stage, classify_recovery_next_head_stage, - classify_recovery_segment_stage, classify_version_two_recovery_names, + StoreInitialization, StoreInitializationBoundary, StoreInitializationDirectory, + StoreInitializationError, StoreInitializationPhase, StoreInitializationPrefix, + StoreInitializationRefusal, StoreRecordRefusal, WriterLockAcquireError, WriterLockAcquirePhase, + admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_catalog_stage, + classify_recovery_names, classify_recovery_next_head_stage, classify_recovery_segment_stage, execute_recovery_next_head_finalization, execute_recovery_segment_resume, execute_recovery_stage_completion, execute_recovery_stage_discard, fingerprint_recovery_stage, plan_recovery_next_head_finalization, plan_recovery_segment_resume, diff --git a/tests/adapters_layout_contract.rs b/tests/adapters_layout_contract.rs index f16f9247..3a4cc673 100644 --- a/tests/adapters_layout_contract.rs +++ b/tests/adapters_layout_contract.rs @@ -70,8 +70,8 @@ const RECORD_READERS: [(&str, &str); 4] = [ include_str!("../src/adapters/filesystem_format_marker.rs"), ), ( - "src/adapters/version_two_initialization/initialization_observation.rs", - include_str!("../src/adapters/version_two_initialization/initialization_observation.rs"), + "src/adapters/store_initialization/initialization_observation.rs", + include_str!("../src/adapters/store_initialization/initialization_observation.rs"), ), ]; @@ -139,12 +139,12 @@ const EXACT_RECORD_CONSUMERS: [(&str, &str); 5] = [ include_str!("../src/adapters/retention/filesystem_retention_stage.rs"), ), ( - "src/adapters/version_two_initialization/initialization_entries.rs", - include_str!("../src/adapters/version_two_initialization/initialization_entries.rs"), + "src/adapters/store_initialization/initialization_entries.rs", + include_str!("../src/adapters/store_initialization/initialization_entries.rs"), ), ( - "src/adapters/version_two_initialization/initialization_observation.rs", - include_str!("../src/adapters/version_two_initialization/initialization_observation.rs"), + "src/adapters/store_initialization/initialization_observation.rs", + include_str!("../src/adapters/store_initialization/initialization_observation.rs"), ), ]; diff --git a/tests/catalog_restart.rs b/tests/catalog_restart.rs index ea3afd2a..d65006f0 100644 --- a/tests/catalog_restart.rs +++ b/tests/catalog_restart.rs @@ -17,7 +17,7 @@ use std::path::{Path, PathBuf}; use keep::{ AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, CatalogPublicationExpectation, CatalogRestartByteLimit, CatalogRestartPolicy, ChunkId, - FilesystemCatalogSnapshot, FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, + FilesystemCatalogPublisher, FilesystemCatalogSnapshot, FilesystemStoreAdmission, LayoutEntryLimit, SegmentReadPolicy, SegmentRecordIdentity, SegmentRecordLimit, StagedSegment, publish_retained_catalog_generation, }; @@ -110,8 +110,8 @@ fn fixture(hex: &str) -> Result, Box> { /// Publishes generation one holding only the one-zero chunk record. fn publish_one_zero(root: &Path) -> Result<(), Box> { - let admission = FilesystemVersionTwoAdmission::initialize(root)?.into_admission(); - let mut publisher = FilesystemVersionTwoCatalogPublisher::open(admission, restart_policy()?)?; + let admission = FilesystemStoreAdmission::initialize(root)?.into_admission(); + let mut publisher = FilesystemCatalogPublisher::open(admission, restart_policy()?)?; let sealed = StagedSegment::begin( publisher.create_segment_stage()?, SegmentRecordLimit::MAXIMUM, diff --git a/tests/version_two_catalog_writer.rs b/tests/catalog_writer.rs similarity index 58% rename from tests/version_two_catalog_writer.rs rename to tests/catalog_writer.rs index ea77fd6a..2249b325 100644 --- a/tests/version_two_catalog_writer.rs +++ b/tests/catalog_writer.rs @@ -1,10 +1,10 @@ -//! Version-two catalog writer and stage-recovery laws on initialized stores. +//! Catalog writer and stage-recovery laws on initialized stores. //! //! Every law initializes a real store, publishes its first catalog and a -//! retention root through the public writer, and then drives the version-two writer and recovery tools +//! retention root through the public writer, and then drives the writer and recovery tools //! through their public production entry points. #![cfg(target_os = "linux")] -#[path = "version_two_catalog_writer/suite.rs"] +#[path = "catalog_writer/suite.rs"] mod suite; diff --git a/tests/version_two_catalog_writer/finalization_laws.rs b/tests/catalog_writer/finalization_laws.rs similarity index 100% rename from tests/version_two_catalog_writer/finalization_laws.rs rename to tests/catalog_writer/finalization_laws.rs diff --git a/tests/version_two_catalog_writer/format_marker_laws.rs b/tests/catalog_writer/format_marker_laws.rs similarity index 78% rename from tests/version_two_catalog_writer/format_marker_laws.rs rename to tests/catalog_writer/format_marker_laws.rs index 8bfce719..78f79057 100644 --- a/tests/version_two_catalog_writer/format_marker_laws.rs +++ b/tests/catalog_writer/format_marker_laws.rs @@ -9,12 +9,10 @@ use std::path::Path; use keep::{ CatalogRestartError, DurableStore, DurableStoreError, FilesystemCatalogSnapshot, - FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError, - FilesystemRecoveryInventoryReader, FilesystemRetentionSnapshot, - FilesystemRetentionSnapshotError, FilesystemVersionTwoAdmission, - FilesystemVersionTwoRecoveryAdmission, ReaderAttemptLimit, RecoveryEntryRole, - RecoveryInventoryError, RecoveryInventoryLimit, RecoveryNamespace, - VersionTwoInitializationError, classify_version_two_recovery_names, + FilesystemNamespaceRefusal, FilesystemRecoveryAdmission, FilesystemRecoveryInventoryReader, + FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, FilesystemStoreAdmission, + FilesystemStoreAdmissionError, ReaderAttemptLimit, RecoveryEntryRole, RecoveryInventoryError, + RecoveryInventoryLimit, RecoveryNamespace, StoreInitializationError, classify_recovery_names, }; use super::durable_fixture; @@ -28,9 +26,8 @@ fn limit() -> TestResult { fn every_root_entry_of_a_published_store_is_a_known_role() -> TestResult<()> { let sandbox = durable_fixture::build("format-marker-roles", &[&first_blob()])?; - let inventory = - FilesystemRecoveryInventoryReader::open_version_two(sandbox.path())?.read(limit()?)?; - let manifest = classify_version_two_recovery_names(inventory)?; + let inventory = FilesystemRecoveryInventoryReader::open(sandbox.path())?.read(limit()?)?; + let manifest = classify_recovery_names(inventory)?; for role in [ RecoveryEntryRole::WriterLock, @@ -65,16 +62,15 @@ fn every_open_path_refuses_a_root_without_the_format_marker() -> TestResult<()> } fn require_writer_paths_refuse(root: &Path) -> TestResult<()> { - let Err(writer) = FilesystemVersionTwoAdmission::reopen(root) else { + let Err(writer) = FilesystemStoreAdmission::reopen(root) else { return Err("writer admission admitted a root without FORMAT".into()); }; require_missing_marker(&writer)?; - let Err(recovery) = FilesystemVersionTwoRecoveryAdmission::reopen(root) else { + let Err(recovery) = FilesystemRecoveryAdmission::reopen(root) else { return Err("recovery admission admitted a root without FORMAT".into()); }; require_missing_marker(&recovery)?; - let Err(VersionTwoInitializationError::Refused { .. }) = - FilesystemVersionTwoAdmission::initialize(root) + let Err(StoreInitializationError::Refused { .. }) = FilesystemStoreAdmission::initialize(root) else { return Err("initialization adopted a populated root without FORMAT".into()); }; @@ -86,14 +82,14 @@ fn require_inventory_refuses(root: &Path) -> TestResult<()> { namespace: RecoveryNamespace::Root, source, .. - }) = FilesystemRecoveryInventoryReader::open_version_two(root) + }) = FilesystemRecoveryInventoryReader::open(root) else { return Err("recovery inventory admitted a root without FORMAT".into()); }; require_missing_marker( source .get_ref() - .and_then(|inner| inner.downcast_ref::()) + .and_then(|inner| inner.downcast_ref::()) .ok_or("inventory refusal lost its admission source")?, )?; Ok(()) @@ -124,8 +120,8 @@ fn require_content_readers_refuse(root: &Path) -> TestResult<()> { require_refusal(source) } -fn require_missing_marker(error: &FilesystemPlatformAdmissionError) -> TestResult<()> { - let FilesystemPlatformAdmissionError::Namespace { source } = error else { +fn require_missing_marker(error: &FilesystemStoreAdmissionError) -> TestResult<()> { + let FilesystemStoreAdmissionError::Namespace { source } = error else { return Err(format!("unexpected admission refusal {error:?}").into()); }; require_refusal(source) diff --git a/tests/version_two_catalog_writer/forward_laws.rs b/tests/catalog_writer/forward_laws.rs similarity index 97% rename from tests/version_two_catalog_writer/forward_laws.rs rename to tests/catalog_writer/forward_laws.rs index 0dc36853..30ebc15b 100644 --- a/tests/version_two_catalog_writer/forward_laws.rs +++ b/tests/catalog_writer/forward_laws.rs @@ -1,4 +1,4 @@ -//! This module owns forward version-two ingestion laws. +//! This module owns forward ingestion laws. use keep::{CatalogPublicationOutcome, DurableStore, ReaderAttemptLimit}; diff --git a/tests/version_two_catalog_writer/recovery_fixture.rs b/tests/catalog_writer/recovery_fixture.rs similarity index 69% rename from tests/version_two_catalog_writer/recovery_fixture.rs rename to tests/catalog_writer/recovery_fixture.rs index 5ef56d9d..9d49ccda 100644 --- a/tests/version_two_catalog_writer/recovery_fixture.rs +++ b/tests/catalog_writer/recovery_fixture.rs @@ -1,6 +1,6 @@ -//! This module owns version-two stage recovery through the public recovery tools. +//! This module owns stage recovery through the public recovery tools. //! -//! Every helper reopens the store through version-two recovery admission, +//! Every helper reopens the store through stage recovery admission, //! fingerprints the fixed stage under the pinned inventory, admits exactly the //! fingerprinted bytes, and executes one planned transition. @@ -10,25 +10,25 @@ use std::path::Path; use keep::{ CatalogPublicationExpectation, ChecksummedCatalog, ChecksummedPublicationHead, - FilesystemCatalogSnapshot, FilesystemRecoveryInventoryReader, + FilesystemCatalogSnapshot, FilesystemRecoveryAdmission, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageCompleter, - FilesystemRecoveryStageDiscarder, FilesystemVersionTwoRecoveryAdmission, RecoveryEntryRole, - RecoveryInventoryLimit, RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationReceipt, - RecoveryStage, RecoveryStageCompletionReceipt, RecoveryStageDiscardReceipt, - admit_recovery_stage_bytes, assess_recovery_stage, classify_version_two_recovery_names, - execute_recovery_next_head_finalization, execute_recovery_stage_completion, - execute_recovery_stage_discard, plan_recovery_next_head_finalization, - plan_recovery_stage_completion, plan_recovery_stage_discard, + FilesystemRecoveryStageDiscarder, RecoveryEntryRole, RecoveryInventoryLimit, + RecoveryNextHeadFinalizationError, RecoveryNextHeadFinalizationReceipt, RecoveryStage, + RecoveryStageCompletionReceipt, RecoveryStageDiscardReceipt, admit_recovery_stage_bytes, + assess_recovery_stage, classify_recovery_names, execute_recovery_next_head_finalization, + execute_recovery_stage_completion, execute_recovery_stage_discard, + plan_recovery_next_head_finalization, plan_recovery_stage_completion, + plan_recovery_stage_discard, }; use super::durable_fixture; use super::writer_fixture::{TestResult, admit_all, segment_policy}; -/// Classifies a fresh version-two inventory and returns every stage role. +/// Classifies a fresh inventory and returns every stage role. pub(super) fn stage_roles(root: &Path) -> TestResult> { - let mut reader = FilesystemRecoveryInventoryReader::open_version_two(root)?; + let mut reader = FilesystemRecoveryInventoryReader::open(root)?; let inventory = reader.read(RecoveryInventoryLimit::new(4096)?)?; - let manifest = classify_version_two_recovery_names(inventory)?; + let manifest = classify_recovery_names(inventory)?; Ok(manifest .entries() .iter() @@ -49,14 +49,14 @@ pub(super) fn complete( root: &Path, stage: RecoveryStage, ) -> TestResult { - let reader = FilesystemRecoveryInventoryReader::open_version_two(root)?; + let reader = FilesystemRecoveryInventoryReader::open(root)?; let evidence = reader.fingerprint_stage(stage)?; let bytes = fs::read(stage_path(root, stage))?; let admitted = admit_recovery_stage_bytes(stage, evidence, &bytes)?; let assessment = assess_recovery_stage(&admitted, segment_policy())?; let request = plan_recovery_stage_completion(&assessment)?; - let admission = FilesystemVersionTwoRecoveryAdmission::reopen(root)?; - let mut completer = FilesystemRecoveryStageCompleter::open_version_two(admission)?; + let admission = FilesystemRecoveryAdmission::reopen(root)?; + let mut completer = FilesystemRecoveryStageCompleter::open(admission)?; Ok(execute_recovery_stage_completion(&mut completer, request)?) } @@ -65,14 +65,14 @@ pub(super) fn discard( root: &Path, stage: RecoveryStage, ) -> TestResult { - let reader = FilesystemRecoveryInventoryReader::open_version_two(root)?; + let reader = FilesystemRecoveryInventoryReader::open(root)?; let evidence = reader.fingerprint_stage(stage)?; let bytes = fs::read(stage_path(root, stage))?; let admitted = admit_recovery_stage_bytes(stage, evidence, &bytes)?; let assessment = assess_recovery_stage(&admitted, segment_policy())?; let request = plan_recovery_stage_discard(&assessment)?; - let admission = FilesystemVersionTwoRecoveryAdmission::reopen(root)?; - let mut discarder = FilesystemRecoveryStageDiscarder::open_version_two(admission)?; + let admission = FilesystemRecoveryAdmission::reopen(root)?; + let mut discarder = FilesystemRecoveryStageDiscarder::open(admission)?; Ok(execute_recovery_stage_discard(&mut discarder, request)?) } @@ -85,7 +85,7 @@ pub(super) fn finalize( root: &Path, candidate_segments: &[Vec], ) -> TestResult> { - let reader = FilesystemRecoveryInventoryReader::open_version_two(root)?; + let reader = FilesystemRecoveryInventoryReader::open(root)?; let evidence = reader.fingerprint_stage(RecoveryStage::NextHead)?; drop(reader); let head_bytes = fs::read(root.join("head.next"))?; @@ -98,11 +98,9 @@ pub(super) fn finalize( let current = FilesystemCatalogSnapshot::load(root, durable_fixture::policy()?)?; let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); let request = plan_recovery_next_head_finalization(&assessment, &candidate, expectation)?; - let admission = FilesystemVersionTwoRecoveryAdmission::reopen(root)?; - let mut finalizer = FilesystemRecoveryNextHeadFinalizer::open_version_two( - admission, - durable_fixture::policy()?, - )?; + let admission = FilesystemRecoveryAdmission::reopen(root)?; + let mut finalizer = + FilesystemRecoveryNextHeadFinalizer::open(admission, durable_fixture::policy()?)?; Ok(execute_recovery_next_head_finalization( &mut finalizer, request, diff --git a/tests/version_two_catalog_writer/refusal_laws.rs b/tests/catalog_writer/refusal_laws.rs similarity index 97% rename from tests/version_two_catalog_writer/refusal_laws.rs rename to tests/catalog_writer/refusal_laws.rs index e2f6ea9a..b8e05b46 100644 --- a/tests/version_two_catalog_writer/refusal_laws.rs +++ b/tests/catalog_writer/refusal_laws.rs @@ -1,4 +1,4 @@ -//! This module owns version-two writer refusals that must precede every mutation. +//! This module owns writer refusals that must precede every mutation. use std::fs; use std::io; diff --git a/tests/version_two_catalog_writer/stage_recovery_laws.rs b/tests/catalog_writer/stage_recovery_laws.rs similarity index 85% rename from tests/version_two_catalog_writer/stage_recovery_laws.rs rename to tests/catalog_writer/stage_recovery_laws.rs index d855865b..00bad206 100644 --- a/tests/version_two_catalog_writer/stage_recovery_laws.rs +++ b/tests/catalog_writer/stage_recovery_laws.rs @@ -1,10 +1,10 @@ -//! This module owns version-two segment and catalog stage recovery laws. +//! This module owns segment and catalog stage recovery laws. use std::fs; use keep::{ - FilesystemRecoveryInventoryReader, FilesystemRecoverySegmentResumer, - FilesystemVersionTwoRecoveryAdmission, RecoveryEntryRole, RecoveryStage, + FilesystemRecoveryAdmission, FilesystemRecoveryInventoryReader, + FilesystemRecoverySegmentResumer, RecoveryEntryRole, RecoveryStage, RecoveryStageDiscardOutcome, SegmentRecordLimit, StagedSegment, admit_recovery_stage_bytes, assess_recovery_stage, execute_recovery_segment_resume, plan_recovery_segment_resume, }; @@ -40,7 +40,7 @@ fn a_refused_candidate_leaves_a_segment_stage_recovery_completes() -> TestResult } #[test] -fn an_incomplete_catalog_stage_is_discarded_by_version_two_recovery() -> TestResult<()> { +fn an_incomplete_catalog_stage_is_discarded_by_stage_recovery() -> TestResult<()> { let first = first_blob(); let sandbox = durable_fixture::build("v2-recover-partial-catalog", &[&first])?; let root = sandbox.path(); @@ -58,7 +58,7 @@ fn an_incomplete_catalog_stage_is_discarded_by_version_two_recovery() -> TestRes } #[test] -fn a_reusable_segment_prefix_resumes_under_version_two_authority() -> TestResult<()> { +fn a_reusable_segment_prefix_resumes_under_recovery_admission() -> TestResult<()> { let first = first_blob(); let second = second_blob(); let sandbox = durable_fixture::build("v2-recover-resume", &[&first])?; @@ -76,8 +76,8 @@ fn a_reusable_segment_prefix_resumes_under_version_two_authority() -> TestResult assert_eq!(stage_roles(root)?, [RecoveryEntryRole::SegmentStage]); let request = resume_plan(root)?; - let admission = FilesystemVersionTwoRecoveryAdmission::reopen(root)?; - let resumer = FilesystemRecoverySegmentResumer::open_version_two(admission)?; + let admission = FilesystemRecoveryAdmission::reopen(root)?; + let resumer = FilesystemRecoverySegmentResumer::open(admission)?; let mut continued = execute_recovery_segment_resume(resumer, request)?; for record in records { continued = continued.append(record)?; @@ -93,7 +93,7 @@ fn a_reusable_segment_prefix_resumes_under_version_two_authority() -> TestResult } fn resume_plan(root: &std::path::Path) -> TestResult { - let reader = FilesystemRecoveryInventoryReader::open_version_two(root)?; + let reader = FilesystemRecoveryInventoryReader::open(root)?; let evidence = reader.fingerprint_stage(RecoveryStage::Segment)?; let bytes = fs::read(root.join("staging/current.seg"))?; let admitted = admit_recovery_stage_bytes(RecoveryStage::Segment, evidence, &bytes)?; diff --git a/tests/version_two_catalog_writer/suite.rs b/tests/catalog_writer/suite.rs similarity index 93% rename from tests/version_two_catalog_writer/suite.rs rename to tests/catalog_writer/suite.rs index 63315315..d48f057e 100644 --- a/tests/version_two_catalog_writer/suite.rs +++ b/tests/catalog_writer/suite.rs @@ -1,4 +1,4 @@ -//! This module owns the version-two writer suite's fixture imports. +//! This module owns the writer suite's fixture imports. #[path = "../golden_file_worldline/durable_fixture.rs"] #[allow( diff --git a/tests/version_two_catalog_writer/writer_fixture.rs b/tests/catalog_writer/writer_fixture.rs similarity index 89% rename from tests/version_two_catalog_writer/writer_fixture.rs rename to tests/catalog_writer/writer_fixture.rs index e64cd43c..19f27644 100644 --- a/tests/version_two_catalog_writer/writer_fixture.rs +++ b/tests/catalog_writer/writer_fixture.rs @@ -1,4 +1,4 @@ -//! This module owns the version-two writer suite's blobs and store witnesses. +//! This module owns the writer suite's blobs and store witnesses. //! //! Store construction and successor publication belong to the shared //! `tests/support` builder; this module fixes the suite's blobs and the @@ -10,9 +10,7 @@ use std::fmt::Write as _; use std::fs; use std::path::Path; -use keep::{ - AdmittedSegment, DurableStore, FilesystemVersionTwoCatalogPublisher, ReaderAttemptLimit, -}; +use keep::{AdmittedSegment, DurableStore, FilesystemCatalogPublisher, ReaderAttemptLimit}; use super::populated_store::{self, StoreAdmission}; pub(super) use super::populated_store::{TestResult, segment_policy}; @@ -25,13 +23,13 @@ pub(super) fn first_blob() -> Vec { b"keep version-two writer: first retained blob. ".repeat(3_000) } -/// The content each law ingests through the version-two writer. +/// The content each law ingests through the writer. pub(super) fn second_blob() -> Vec { b"KEEP V2 WRITER -- the second blob, ingested after the first catalog | ".repeat(2_500) } -/// Opens the version-two writer under the production platform profile. -pub(super) fn open_publisher(root: &Path) -> TestResult { +/// Opens the writer under the production platform profile. +pub(super) fn open_publisher(root: &Path) -> TestResult { super::successor_catalog::open_publisher(root, StoreAdmission::PLATFORM) } @@ -89,7 +87,7 @@ pub(super) fn witness(root: &Path) -> TestResult { /// /// Returns the exact sealed bytes. The closed stage remains for the caller. pub(super) fn stage_only( - publisher: &FilesystemVersionTwoCatalogPublisher, + publisher: &FilesystemCatalogPublisher, root: &Path, content: &[u8], ) -> TestResult> { diff --git a/tests/observed_segment_stage.rs b/tests/observed_segment_stage.rs index faea7e18..e58ff195 100644 --- a/tests/observed_segment_stage.rs +++ b/tests/observed_segment_stage.rs @@ -16,10 +16,10 @@ use std::fs; use std::io; use keep::{ - AdmittedSegment, CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemVersionTwoAdmission, - FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, ObservedSegmentStage, - SegmentReadPolicy, SegmentRecordLimit, SegmentStageDurabilityEvent, SegmentStageObserver, - SegmentWriteError, SegmentWritePhase, StagedSegment, + AdmittedSegment, CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, + FilesystemStoreAdmission, LayoutEntryLimit, ObservedSegmentStage, SegmentReadPolicy, + SegmentRecordLimit, SegmentStageDurabilityEvent, SegmentStageObserver, SegmentWriteError, + SegmentWritePhase, StagedSegment, }; #[test] @@ -134,9 +134,9 @@ impl SegmentStageObserver for Observer { fn publisher( directory: &sandbox::TestDirectory, -) -> Result> { - let admission = FilesystemVersionTwoAdmission::initialize(directory.path())?.into_admission(); - Ok(FilesystemVersionTwoCatalogPublisher::open( +) -> Result> { + let admission = FilesystemStoreAdmission::initialize(directory.path())?.into_admission(); + Ok(FilesystemCatalogPublisher::open( admission, CatalogRestartPolicy::new(read_policy(), CatalogRestartByteLimit::new(1_048_576)?), )?) diff --git a/tests/reader_fence_process/fixture.rs b/tests/reader_fence_process/fixture.rs index 6e7eb056..fc4a63c0 100644 --- a/tests/reader_fence_process/fixture.rs +++ b/tests/reader_fence_process/fixture.rs @@ -2,20 +2,18 @@ use super::populated_store::{self, FirstCatalogRecords, StoreAdmission}; use super::sandbox::TestDirectory; -use keep::{FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher}; +use keep::{FilesystemCatalogPublisher, FilesystemStoreAdmission}; use std::error::Error; /// The shared builder's route outside the production platform profile. const UNCHECKED: StoreAdmission = StoreAdmission { initialize: |root| { Ok( - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(root)? + FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(root)? .into_admission(), ) }, - reopen_version_two: |root| { - Ok(FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(root)?) - }, + reopen: |root| Ok(FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(root)?), }; /// Creates the one-zero bundle store and keeps its writer authority. @@ -26,7 +24,7 @@ const UNCHECKED: StoreAdmission = StoreAdmission { /// immediately reacquiring cannot assume all holders have disappeared. pub(super) fn initialized( name: &str, -) -> Result<(TestDirectory, FilesystemVersionTwoCatalogPublisher), Box> { +) -> Result<(TestDirectory, FilesystemCatalogPublisher), Box> { let sandbox = TestDirectory::create(name)?; let (_created, writer) = populated_store::create_holding_authority( sandbox.path(), diff --git a/tests/recovery_inventory.rs b/tests/recovery_inventory.rs index 1efaa704..e6aa6844 100644 --- a/tests/recovery_inventory.rs +++ b/tests/recovery_inventory.rs @@ -66,7 +66,7 @@ fn every_count_precedes_name_retention_in_fixed_namespace_order() -> Result<(), fn unsupported_platform_refuses_before_filesystem_inventory_mutation() -> Result<(), Box> { let sandbox = TestDirectory::create("recovery-inventory-unsupported")?; - let Err(error) = FilesystemRecoveryInventoryReader::open_version_two(sandbox.path()) else { + let Err(error) = FilesystemRecoveryInventoryReader::open(sandbox.path()) else { return Err("filesystem inventory admitted an unsupported platform".into()); }; diff --git a/tests/recovery_name_classification.rs b/tests/recovery_name_classification.rs index e37b2876..0838fc6c 100644 --- a/tests/recovery_name_classification.rs +++ b/tests/recovery_name_classification.rs @@ -12,7 +12,7 @@ use std::error::Error; use inventory_double::InventoryDouble; use keep::{ RecoveryEntryName, RecoveryEntryRole, RecoveryInventory, RecoveryInventoryLimit, - classify_version_two_recovery_names, read_recovery_inventory, + classify_recovery_names, read_recovery_inventory, }; #[test] @@ -26,7 +26,7 @@ fn canonical_names_produce_exact_typed_roles() -> Result<(), Box> { vec![name(catalog_name.as_bytes())?], ])?; - let manifest = classify_version_two_recovery_names(inventory)?; + let manifest = classify_recovery_names(inventory)?; assert!( manifest diff --git a/tests/recovery_name_classification/grammar_laws.rs b/tests/recovery_name_classification/grammar_laws.rs index c4877f33..82877f25 100644 --- a/tests/recovery_name_classification/grammar_laws.rs +++ b/tests/recovery_name_classification/grammar_laws.rs @@ -4,7 +4,7 @@ use std::error::Error; use keep::{ RecoveryNameClassificationError, RecoveryNamespace, RecoveryPoolNameError, - classify_version_two_recovery_names, + classify_recovery_names, }; use super::{initialized_root, inventory, name}; @@ -97,7 +97,7 @@ fn pool_error( return Err("pool grammar test requires an immutable namespace".into()); } }; - let Err(error) = classify_version_two_recovery_names(inventory(entries)?) else { + let Err(error) = classify_recovery_names(inventory(entries)?) else { return Err("name classification admitted a noncanonical pool name".into()); }; match error { diff --git a/tests/recovery_name_classification/refusal_laws.rs b/tests/recovery_name_classification/refusal_laws.rs index 31823b00..edf6290e 100644 --- a/tests/recovery_name_classification/refusal_laws.rs +++ b/tests/recovery_name_classification/refusal_laws.rs @@ -4,14 +4,14 @@ use std::error::Error; use keep::{ RecoveryEntryRole, RecoveryNameClassificationError, RecoveryNamespace, RecoveryPoolNameError, - RecoveryRequiredEntry, classify_version_two_recovery_names, + RecoveryRequiredEntry, classify_recovery_names, }; use super::{INITIALIZED_ROOT, initialized_root, inventory, name, names}; #[test] fn unknown_name_is_refused_with_its_exact_namespace_and_bytes() -> Result<(), Box> { - let Err(error) = classify_version_two_recovery_names(inventory([ + let Err(error) = classify_recovery_names(inventory([ initialized_root()?, Vec::new(), names(&["not-a-segment"])?, @@ -36,7 +36,7 @@ fn unknown_name_is_refused_with_its_exact_namespace_and_bytes() -> Result<(), Bo #[test] fn missing_initialized_root_entry_is_an_exact_refusal() -> Result<(), Box> { - let Err(error) = classify_version_two_recovery_names(inventory([ + let Err(error) = classify_recovery_names(inventory([ names( INITIALIZED_ROOT .get(1..) @@ -60,7 +60,7 @@ fn missing_initialized_root_entry_is_an_exact_refusal() -> Result<(), Box Result<(), Box> { - let Err(error) = classify_version_two_recovery_names(inventory([ + let Err(error) = classify_recovery_names(inventory([ initialized_root()?, Vec::new(), vec![name(format!("{}.seg", "AA".repeat(32)).as_bytes())?], @@ -81,7 +81,7 @@ fn uppercase_pool_digest_is_noncanonical() -> Result<(), Box> { #[test] fn simultaneous_fixed_stages_are_unrecoverable_ambiguity() -> Result<(), Box> { - let Err(error) = classify_version_two_recovery_names(inventory([ + let Err(error) = classify_recovery_names(inventory([ initialized_root()?, names(&["current.seg", "current.cat"])?, Vec::new(), diff --git a/tests/recovery_name_classification_memory.rs b/tests/recovery_name_classification_memory.rs index 008b8cb1..c93956d2 100644 --- a/tests/recovery_name_classification_memory.rs +++ b/tests/recovery_name_classification_memory.rs @@ -9,7 +9,7 @@ use allocation_counter::measure; use inventory_double::InventoryDouble; use keep::{ RecoveryEntryName, RecoveryInventoryLimit, RecoveryNameClassificationError, - classify_version_two_recovery_names, read_recovery_inventory, + classify_recovery_names, read_recovery_inventory, }; #[test] @@ -40,7 +40,7 @@ fn refusal_moves_the_exact_name_without_an_extra_allocation() -> Result<(), Box< let allocations = measure(|| { if let Some(inventory) = input.take() { - result = Some(classify_version_two_recovery_names(inventory)); + result = Some(classify_recovery_names(inventory)); } }); let error = match result.ok_or("name classification did not run")? { diff --git a/tests/segment_filesystem_stage.rs b/tests/segment_filesystem_stage.rs index 48880446..ac45b5fa 100644 --- a/tests/segment_filesystem_stage.rs +++ b/tests/segment_filesystem_stage.rs @@ -14,9 +14,9 @@ use std::io::ErrorKind; use keep::{ AdmittedSegment, AdmittedSegmentRecord, CatalogRestartByteLimit, CatalogRestartPolicy, - FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, - SegmentHeader, SegmentReadError, SegmentReadPolicy, SegmentRecordLimit, - SegmentStageCreateError, StagedSegment, + FilesystemCatalogPublisher, FilesystemStoreAdmission, LayoutEntryLimit, SegmentHeader, + SegmentReadError, SegmentReadPolicy, SegmentRecordLimit, SegmentStageCreateError, + StagedSegment, }; use sandbox::TestDirectory; use support::decode_hex; @@ -133,14 +133,10 @@ fn dropping_an_unsealed_stage_preserves_evidence_without_sealing() -> Result<(), Ok(()) } -fn open_publisher( - sandbox: &TestDirectory, -) -> Result> { - let admission = FilesystemVersionTwoAdmission::initialize(sandbox.path())?.into_admission(); +fn open_publisher(sandbox: &TestDirectory) -> Result> { + let admission = FilesystemStoreAdmission::initialize(sandbox.path())?.into_admission(); let policy = CatalogRestartPolicy::new(read_policy(), CatalogRestartByteLimit::new(1_048_576)?); - Ok(FilesystemVersionTwoCatalogPublisher::open( - admission, policy, - )?) + Ok(FilesystemCatalogPublisher::open(admission, policy)?) } const fn read_policy() -> SegmentReadPolicy { diff --git a/tests/single_format.rs b/tests/single_format.rs index e863133e..7bd0c8d8 100644 --- a/tests/single_format.rs +++ b/tests/single_format.rs @@ -1,8 +1,8 @@ //! Public laws of the single store format: direct initialization, portable //! stores, and the integrated acceptance sequence that replaced migration. //! -//! Every law creates its store through `FilesystemVersionTwoAdmission::initialize` -//! and drives the version-two writer, recovery tools, and durable reader +//! Every law creates its store through `FilesystemStoreAdmission::initialize` +//! and drives the writer, recovery tools, and durable reader //! through their public production entry points. #![cfg(target_os = "linux")] diff --git a/tests/single_format/acceptance_laws.rs b/tests/single_format/acceptance_laws.rs index 8604a77f..f301db27 100644 --- a/tests/single_format/acceptance_laws.rs +++ b/tests/single_format/acceptance_laws.rs @@ -10,11 +10,11 @@ use std::io; use std::path::Path; use keep::{ - CatalogPublicationBoundary, CatalogPublicationPhase, DurableStore, FilesystemCatalogSnapshot, - FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, FilesystemVersionTwoAdmission, - FilesystemVersionTwoCatalogPublisher, ReaderAttemptLimit, RecoveryEntryRole, + CatalogPublicationBoundary, CatalogPublicationPhase, DurableStore, FilesystemCatalogPublisher, + FilesystemCatalogSnapshot, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, + FilesystemStoreAdmission, ReaderAttemptLimit, RecoveryEntryRole, RecoveryNextHeadFinalizationOutcome, RetainedCatalogPublicationError, - VersionTwoInitializationPrefix, + StoreInitializationPrefix, }; use super::durable_fixture; @@ -90,17 +90,15 @@ fn the_single_format_serves_every_operation_from_initialization_to_recovery() -> let (first, second) = (first_blob(), second_blob()); let third = b"keep acceptance: the publication interrupted before HEAD. ".repeat(1_500); - let initialized = FilesystemVersionTwoAdmission::initialize(root)?; - assert_eq!(initialized.found(), VersionTwoInitializationPrefix::Empty); - let mut writer = FilesystemVersionTwoCatalogPublisher::open( - initialized.into_admission(), - populated_store::policy()?, - )?; + let initialized = FilesystemStoreAdmission::initialize(root)?; + assert_eq!(initialized.found(), StoreInitializationPrefix::Empty); + let mut writer = + FilesystemCatalogPublisher::open(initialized.into_admission(), populated_store::policy()?)?; let ingested = populated_store::publish_first(root, &mut writer, &[&first])?; assert_eq!(ingested.generation().get(), 1); drop(writer); - drop(FilesystemVersionTwoAdmission::reopen(root)?); + drop(FilesystemStoreAdmission::reopen(root)?); let layout = populated_store::identify(&first)?.record.id(); let store = DurableStore::open( root, @@ -171,7 +169,7 @@ fn a_backup_copy_reopens_verifies_and_recovers_an_interrupted_write() -> TestRes )); let root = backup.path(); - drop(FilesystemVersionTwoAdmission::reopen(root)?); + drop(FilesystemStoreAdmission::reopen(root)?); assert_reads(root, &first)?; let candidate = interrupted_publication(root, &second_blob())?; recover(root, &candidate, 2)?; diff --git a/tests/single_format/initialization_laws.rs b/tests/single_format/initialization_laws.rs index aaf26711..e0ffecda 100644 --- a/tests/single_format/initialization_laws.rs +++ b/tests/single_format/initialization_laws.rs @@ -4,11 +4,11 @@ use std::fs; use std::io::ErrorKind; use keep::{ - FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError, FilesystemRetentionSnapshot, - FilesystemRetentionSnapshotError, FilesystemVersionTwoAdmission, - FilesystemVersionTwoCatalogPublisher, ReaderAttemptLimit, RetainedCatalogPublicationError, - RetentionCurrentStateRefusal, RetentionEvidencePhase, VersionTwoInitializationError, - VersionTwoInitializationPrefix, VersionTwoInitializationRefusal, + FilesystemCatalogPublisher, FilesystemNamespaceRefusal, FilesystemRetentionSnapshot, + FilesystemRetentionSnapshotError, FilesystemStoreAdmission, FilesystemStoreAdmissionError, + ReaderAttemptLimit, RetainedCatalogPublicationError, RetentionCurrentStateRefusal, + RetentionEvidencePhase, StoreInitializationError, StoreInitializationPrefix, + StoreInitializationRefusal, }; use super::durable_sandbox::TestDirectory; @@ -21,12 +21,12 @@ fn an_initialized_store_has_no_catalog_and_admits_writers_but_not_readers() -> T let sandbox = TestDirectory::create("single-format-unpublished")?; let root = sandbox.path(); - let initialized = FilesystemVersionTwoAdmission::initialize(root)?; + let initialized = FilesystemStoreAdmission::initialize(root)?; - assert_eq!(initialized.found(), VersionTwoInitializationPrefix::Empty); + assert_eq!(initialized.found(), StoreInitializationPrefix::Empty); drop(initialized); assert!(!root.join("HEAD").exists()); - drop(FilesystemVersionTwoAdmission::reopen(root)?); + drop(FilesystemStoreAdmission::reopen(root)?); let reader = FilesystemRetentionSnapshot::load( root, populated_store::policy()?, @@ -43,9 +43,9 @@ fn an_initialized_store_has_no_catalog_and_admits_writers_but_not_readers() -> T fn initialization_refuses_an_unsupported_platform_before_any_mutation() -> TestResult<()> { let directory = UnsupportedDirectory::create()?; - let refused = FilesystemVersionTwoAdmission::initialize(directory.path()); + let refused = FilesystemStoreAdmission::initialize(directory.path()); - let Err(VersionTwoInitializationError::Platform { source }) = refused else { + let Err(StoreInitializationError::Platform { source }) = refused else { return Err("initialization admitted an unsupported platform".into()); }; assert_eq!(source.kind(), ErrorKind::Unsupported); @@ -67,12 +67,12 @@ fn a_second_initialization_of_a_committed_store_refuses_unchanged() -> TestResul drop(writer); let before = witness(root)?; - let refused = FilesystemVersionTwoAdmission::initialize(root); + let refused = FilesystemStoreAdmission::initialize(root); assert!(matches!( refused, - Err(VersionTwoInitializationError::Refused { - refusal: VersionTwoInitializationRefusal::AlreadyInitialized + Err(StoreInitializationError::Refused { + refusal: StoreInitializationRefusal::AlreadyInitialized }) )); assert_eq!(witness(root)?, before); @@ -83,11 +83,11 @@ fn a_second_initialization_of_a_committed_store_refuses_unchanged() -> TestResul fn an_uncommitted_root_is_refused_by_writers_and_readers() -> TestResult<()> { let sandbox = TestDirectory::create("single-format-uncommitted")?; let root = sandbox.path(); - drop(FilesystemVersionTwoAdmission::initialize(root)?); + drop(FilesystemStoreAdmission::initialize(root)?); fs::hard_link(root.join("FORMAT"), root.join("FORMAT.next"))?; - let Err(FilesystemPlatformAdmissionError::Namespace { source }) = - FilesystemVersionTwoAdmission::reopen(root) + let Err(FilesystemStoreAdmissionError::Namespace { source }) = + FilesystemStoreAdmission::reopen(root) else { return Err("an uncommitted root admitted a writer".into()); }; @@ -107,8 +107,8 @@ fn an_uncommitted_root_is_refused_by_writers_and_readers() -> TestResult<()> { Err(FilesystemRetentionSnapshotError::Admission { .. }) )); assert_eq!( - FilesystemVersionTwoAdmission::initialize(root)?.found(), - VersionTwoInitializationPrefix::MarkerLinked + FilesystemStoreAdmission::initialize(root)?.found(), + StoreInitializationPrefix::MarkerLinked ); Ok(()) } @@ -120,12 +120,11 @@ fn an_uncommitted_root_is_refused_by_writers_and_readers() -> TestResult<()> { fn the_first_publication_refuses_retention_artifacts_without_a_head() -> TestResult<()> { let sandbox = TestDirectory::create("single-format-first-contradicted")?; let root = sandbox.path(); - let admission = FilesystemVersionTwoAdmission::initialize(root)?.into_admission(); + let admission = FilesystemStoreAdmission::initialize(root)?.into_admission(); let namespace = root.join("retention/roots").join("a".repeat(64)); fs::create_dir(&namespace)?; let before = witness(root)?; - let mut publisher = - FilesystemVersionTwoCatalogPublisher::open(admission, populated_store::policy()?)?; + let mut publisher = FilesystemCatalogPublisher::open(admission, populated_store::policy()?)?; let refused = populated_store::publish_first(root, &mut publisher, &[&first_blob()]); drop(publisher); diff --git a/tests/single_format/portability_laws.rs b/tests/single_format/portability_laws.rs index ab8cb40f..c3a92ec1 100644 --- a/tests/single_format/portability_laws.rs +++ b/tests/single_format/portability_laws.rs @@ -10,9 +10,9 @@ use std::io; use std::path::Path; use keep::{ - FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, - FilesystemVersionTwoCatalogPublisher, RetainedCatalogPublicationError, - RetentionCurrentStateRefusal, VersionTwoRecordRefusal, WriterLockAcquireError, + FilesystemCatalogPublisher, FilesystemStoreAdmission, FilesystemStoreAdmissionError, + RetainedCatalogPublicationError, RetentionCurrentStateRefusal, StoreRecordRefusal, + WriterLockAcquireError, }; use super::durable_fixture; @@ -37,7 +37,7 @@ fn a_valid_quiescent_copy_opens_and_reads_the_same_content() -> TestResult<()> { let original = durable_fixture::build("portable-original", &[&first])?; let copy = copied("portable-copy", original.path())?; - drop(FilesystemVersionTwoAdmission::reopen(copy.path())?); + drop(FilesystemStoreAdmission::reopen(copy.path())?); assert_reads(copy.path(), &first) } @@ -86,15 +86,15 @@ fn a_replaced_lock_or_protocol_directory_during_an_operation_still_refuses() -> let original = durable_fixture::build("portable-replaced-original", &[&first_blob()])?; let copy = copied("portable-replaced-copy", original.path())?; let root = copy.path(); - let held = FilesystemVersionTwoAdmission::reopen(root)?; + let held = FilesystemStoreAdmission::reopen(root)?; fs::remove_file(root.join("writer.lock"))?; fs::write(root.join("writer.lock"), b"")?; - let contended = FilesystemVersionTwoAdmission::reopen(root); + let contended = FilesystemStoreAdmission::reopen(root); assert!(matches!( contended, - Err(FilesystemPlatformAdmissionError::WriterLock { + Err(FilesystemStoreAdmissionError::WriterLock { source: WriterLockAcquireError::Busy }) )); @@ -104,8 +104,7 @@ fn a_replaced_lock_or_protocol_directory_during_an_operation_still_refuses() -> fs::rename(root.join("retention"), displaced.path().join("retention"))?; fs::create_dir_all(root.join("retention/roots"))?; fs::create_dir(root.join("retention/manifests"))?; - let mut publisher = - FilesystemVersionTwoCatalogPublisher::open(held, populated_store::policy()?)?; + let mut publisher = FilesystemCatalogPublisher::open(held, populated_store::policy()?)?; let outcome = publish(&mut publisher, root, &second_blob(), Predecessor::Kept)?; let Err(RetainedCatalogPublicationError::RetentionStorage { source, .. }) = outcome else { return Err(format!("publication used a replaced retention directory: {outcome:?}").into()); @@ -165,21 +164,21 @@ fn a_copy_and_its_original_diverge_independently_and_keep_shared_identity() -> T } fn marker_refusal(root: &Path) -> TestResult { - let Err(FilesystemPlatformAdmissionError::FormatMarker { source }) = - FilesystemVersionTwoAdmission::reopen(root) + let Err(FilesystemStoreAdmissionError::FormatMarker { source }) = + FilesystemStoreAdmission::reopen(root) else { return Err("a corrupt marker was not refused at the marker boundary".into()); }; let refusal = source .get_ref() - .and_then(|cause| cause.downcast_ref::()) + .and_then(|cause| cause.downcast_ref::()) .ok_or("the marker refusal lost its typed cause")?; Ok(format!("{refusal:?}")) } fn namespace_refusal_kind(root: &Path) -> TestResult { - match FilesystemVersionTwoAdmission::reopen(root) { - Err(FilesystemPlatformAdmissionError::Namespace { source }) => Ok(source.kind()), + match FilesystemStoreAdmission::reopen(root) { + Err(FilesystemStoreAdmissionError::Namespace { source }) => Ok(source.kind()), Err(other) => Err(format!("unexpected incomplete-store refusal: {other:?}").into()), Ok(_) => Err("an incomplete store was admitted".into()), } diff --git a/tests/single_format/suite.rs b/tests/single_format/suite.rs index 21aacba5..13639df1 100644 --- a/tests/single_format/suite.rs +++ b/tests/single_format/suite.rs @@ -25,7 +25,7 @@ mod initialization_laws; mod populated_store; #[path = "portability_laws.rs"] mod portability_laws; -#[path = "../version_two_catalog_writer/recovery_fixture.rs"] +#[path = "../catalog_writer/recovery_fixture.rs"] #[allow(dead_code, reason = "these laws use finalization only")] mod recovery_fixture; #[path = "store_copy.rs"] @@ -38,7 +38,7 @@ mod store_copy; mod successor_catalog; #[path = "unsupported_directory.rs"] mod unsupported_directory; -#[path = "../version_two_catalog_writer/writer_fixture.rs"] +#[path = "../catalog_writer/writer_fixture.rs"] #[allow( dead_code, unused_imports, diff --git a/tests/store_admission_contract.rs b/tests/store_admission_contract.rs new file mode 100644 index 00000000..fc33b9cb --- /dev/null +++ b/tests/store_admission_contract.rs @@ -0,0 +1,34 @@ +//! Writer authority over a store comes only from single-format admission. + +const RETENTION_AUTHORITY: &str = + include_str!("../src/adapters/retention/filesystem_retention_authority.rs"); +const CATALOG_PUBLISHER: &str = include_str!("../src/adapters/filesystem_catalog_publisher.rs"); +const STORE_ADMISSION: &str = include_str!("../src/adapters/filesystem_store_admission.rs"); +const ADMISSION_ERROR: &str = include_str!("../src/adapters/filesystem_store_admission_error.rs"); + +/// The type-level proofs are the `compile_fail` doctests on +/// `FilesystemRetentionPublicationAuthority::open` and +/// `FilesystemCatalogPublisher::open`, which refuse a bare +/// `FilesystemWriterLock`. These markers only keep each writer on the +/// admission side of the boundary without pinning a signature. +#[test] +fn every_writer_consumes_admitted_authority() { + assert!(RETENTION_AUTHORITY.contains("FilesystemStoreAdmission")); + assert!(CATALOG_PUBLISHER.contains("admission: FilesystemStoreAdmission")); +} + +#[test] +fn reopen_admits_the_marker_under_the_production_profile() { + assert!(STORE_ADMISSION.contains("pub struct FilesystemStoreAdmission")); + assert!(STORE_ADMISSION.contains("pub fn reopen(store_root: &Path)")); + assert!(STORE_ADMISSION.contains("pub fn initialize(")); + assert!(STORE_ADMISSION.contains("filesystem_format_marker::admit")); + assert!(STORE_ADMISSION.contains("filesystem_platform_profile::open")); +} + +/// Admission refusals grow with the platform surface, so the public error is +/// non-exhaustive and a new refusal is not a breaking change. +#[test] +fn admission_error_is_non_exhaustive() { + assert!(ADMISSION_ERROR.contains("#[non_exhaustive]\npub enum FilesystemStoreAdmissionError")); +} diff --git a/tests/support/admission_refusal.rs b/tests/support/admission_refusal.rs index 0f610b27..baaf39f2 100644 --- a/tests/support/admission_refusal.rs +++ b/tests/support/admission_refusal.rs @@ -1,6 +1,6 @@ //! This module owns extraction of typed namespace refusals from store admission. -use keep::{FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError}; +use keep::{FilesystemNamespaceRefusal, FilesystemStoreAdmissionError}; /// The namespace refusal one store admission failure carries, if any. /// @@ -8,9 +8,9 @@ use keep::{FilesystemNamespaceRefusal, FilesystemPlatformAdmissionError}; /// so laws assert the exact contradiction instead of an error string. #[must_use] pub(crate) fn namespace_refusal( - error: &FilesystemPlatformAdmissionError, + error: &FilesystemStoreAdmissionError, ) -> Option { - let FilesystemPlatformAdmissionError::Namespace { source } = error else { + let FilesystemStoreAdmissionError::Namespace { source } = error else { return None; }; source diff --git a/tests/support/populated_store.rs b/tests/support/populated_store.rs index e2688d66..5a681c2c 100644 --- a/tests/support/populated_store.rs +++ b/tests/support/populated_store.rs @@ -1,11 +1,11 @@ //! This module owns populated-store construction for test code. //! -//! Every populated version-two test store is created by [`create`] and then -//! extended by [`retain`] and by the version-two writer in `successor_catalog.rs`. +//! Every populated test store is created by [`create`] and then +//! extended by [`retain`] and by the writer in `successor_catalog.rs`. //! Only public production entry points run, so test code is not a second //! implementation of the store format. [`create`] initializes the store //! directly and publishes the first catalog through -//! `FilesystemVersionTwoCatalogPublisher` and the retained-closure gate. +//! `FilesystemCatalogPublisher` and the retained-closure gate. //! //! Consumers include this file by path beside their laws, so it names the //! crate as `keep` and uses `super`-relative visibility. @@ -19,11 +19,11 @@ use keep::{ AdmittedLayout, AdmittedRetentionManifest, AdmittedRetentionRoot, AdmittedSegment, AdmittedSegmentRecord, BlobId, CanonicalCatalog, CanonicalLayoutRecord, CanonicalRetentionRoot, CatalogGeneration, CatalogPublicationExpectation, CatalogRestartByteLimit, - CatalogRestartPolicy, ChunkId, ChunkSpan, FastCdc, FilesystemCatalogSnapshot, - FilesystemRetentionPublicationAuthority, FilesystemVersionTwoAdmission, - FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, RegisteredRetentionProfile, - RegisteredStorageProfile, RetainedCatalogPublicationReceipt, RetentionAnchor, - RetentionClosureLimits, RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, + CatalogRestartPolicy, ChunkId, ChunkSpan, FastCdc, FilesystemCatalogPublisher, + FilesystemCatalogSnapshot, FilesystemRetentionPublicationAuthority, FilesystemStoreAdmission, + LayoutEntryLimit, RegisteredRetentionProfile, RegisteredStorageProfile, + RetainedCatalogPublicationReceipt, RetentionAnchor, RetentionClosureLimits, + RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionPublicationReceipt, RetentionRoot, RootGeneration, SegmentReadPolicy, SegmentRecordLimit, StagedSegment, execute_retention_publication, preflight_retention_transition, prepare_retention_publication, @@ -40,16 +40,16 @@ pub(super) type TestResult = Result>; #[derive(Clone, Copy)] pub(super) struct StoreAdmission { /// Initializes an empty root and returns its writer authority. - pub(super) initialize: fn(&Path) -> TestResult, + pub(super) initialize: fn(&Path) -> TestResult, /// Reacquires authority over the created root. - pub(super) reopen_version_two: fn(&Path) -> TestResult, + pub(super) reopen: fn(&Path) -> TestResult, } impl StoreAdmission { /// The production platform profile. pub(super) const PLATFORM: Self = Self { - initialize: |root| Ok(FilesystemVersionTwoAdmission::initialize(root)?.into_admission()), - reopen_version_two: |root| Ok(FilesystemVersionTwoAdmission::reopen(root)?), + initialize: |root| Ok(FilesystemStoreAdmission::initialize(root)?.into_admission()), + reopen: |root| Ok(FilesystemStoreAdmission::reopen(root)?), }; } @@ -124,7 +124,7 @@ pub(super) const fn segment_policy() -> SegmentReadPolicy { SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) } -/// Initializes a version-two store at the empty directory `root`. +/// Initializes a store at the empty directory `root`. /// /// The store has no catalog `HEAD` and no retention head, so the first /// publication runs with no predecessor and proves both absences through @@ -150,15 +150,14 @@ pub(super) fn create_holding_authority( admission: StoreAdmission, contents: &[&[u8]], selection: FirstCatalogRecords, -) -> TestResult<(CreatedStore, FilesystemVersionTwoCatalogPublisher)> { +) -> TestResult<(CreatedStore, FilesystemCatalogPublisher)> { let identified = contents .iter() .map(|bytes| identify(bytes)) .collect::>>()?; let blobs = contents.iter().copied().zip(&identified); let records = select_records(blobs, selection)?; - let mut publisher = - FilesystemVersionTwoCatalogPublisher::open((admission.initialize)(root)?, policy()?)?; + let mut publisher = FilesystemCatalogPublisher::open((admission.initialize)(root)?, policy()?)?; let catalog = publish_first_catalog(root, &mut publisher, records)?; Ok((CreatedStore { catalog }, publisher)) } @@ -170,7 +169,7 @@ pub(super) fn create_holding_authority( /// `retention/HEAD` exists uncontradicted. pub(super) fn publish_first( root: &Path, - publisher: &mut FilesystemVersionTwoCatalogPublisher, + publisher: &mut FilesystemCatalogPublisher, contents: &[&[u8]], ) -> TestResult { let identified = contents @@ -208,7 +207,7 @@ pub(super) fn select_records<'content>( fn publish_first_catalog( root: &Path, - publisher: &mut FilesystemVersionTwoCatalogPublisher, + publisher: &mut FilesystemCatalogPublisher, records: Vec>, ) -> TestResult { let mut stage = StagedSegment::begin( @@ -267,8 +266,7 @@ pub(super) fn retain( candidate, &snapshot, )?; - let mut authority = - FilesystemRetentionPublicationAuthority::open((admission.reopen_version_two)(root)?)?; + let mut authority = FilesystemRetentionPublicationAuthority::open((admission.reopen)(root)?)?; let observed = authority.observe_current()?; let manifest = observed .as_ref() diff --git a/tests/support/successor_catalog.rs b/tests/support/successor_catalog.rs index a6e3def1..657fa7e1 100644 --- a/tests/support/successor_catalog.rs +++ b/tests/support/successor_catalog.rs @@ -1,6 +1,6 @@ //! This module owns test publication of catalogs after a store's first one. //! -//! Every successor goes through `FilesystemVersionTwoCatalogPublisher` and the +//! Every successor goes through `FilesystemCatalogPublisher` and the //! retained-closure gate, on a store created by `populated_store.rs`. Callers //! pass the publisher so they can observe or interrupt its transitions. //! @@ -11,10 +11,10 @@ use std::path::Path; use keep::{ AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CatalogGeneration, - CatalogPublicationExpectation, FilesystemCatalogSnapshot, FilesystemSegmentStage, - FilesystemVersionTwoCatalogPublisher, RetainedCatalogPublicationError, - RetainedCatalogPublicationReceipt, SealedSegment, SegmentPublication, SegmentRecordLimit, - StagedSegment, publish_retained_catalog_generation, + CatalogPublicationExpectation, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemSegmentStage, RetainedCatalogPublicationError, RetainedCatalogPublicationReceipt, + SealedSegment, SegmentPublication, SegmentRecordLimit, StagedSegment, + publish_retained_catalog_generation, }; use super::populated_store::{ @@ -35,13 +35,13 @@ pub(super) enum Predecessor { Dropped, } -/// Opens the version-two writer over a created store. +/// Opens the writer over a created store. pub(super) fn open_publisher( root: &Path, admission: StoreAdmission, -) -> TestResult { - Ok(FilesystemVersionTwoCatalogPublisher::open( - (admission.reopen_version_two)(root)?, +) -> TestResult { + Ok(FilesystemCatalogPublisher::open( + (admission.reopen)(root)?, policy()?, )?) } @@ -75,7 +75,7 @@ pub(super) fn blob_records<'content>( /// Stages and seals every record of `content` without selecting the stage. pub(super) fn stage_blob<'publisher>( - publisher: &'publisher FilesystemVersionTwoCatalogPublisher, + publisher: &'publisher FilesystemCatalogPublisher, content: &[u8], identified: &Identified, ) -> TestResult>> { @@ -94,7 +94,7 @@ pub(super) fn stage_blob<'publisher>( /// The candidate references the current segments when `predecessor` is /// [`Predecessor::Kept`]. pub(super) fn publish( - publisher: &mut FilesystemVersionTwoCatalogPublisher, + publisher: &mut FilesystemCatalogPublisher, root: &Path, content: &[u8], predecessor: Predecessor, @@ -123,7 +123,7 @@ pub(super) fn publish( /// Publishes the next catalog generation over already pooled segments. pub(super) fn republish( - publisher: &mut FilesystemVersionTwoCatalogPublisher, + publisher: &mut FilesystemCatalogPublisher, root: &Path, ) -> TestResult { let current = FilesystemCatalogSnapshot::load(root, policy()?)?; diff --git a/tests/version_two_admission_contract.rs b/tests/version_two_admission_contract.rs deleted file mode 100644 index a0dbc41e..00000000 --- a/tests/version_two_admission_contract.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! Writer authority over a store comes only from single-format admission. - -const RETENTION_AUTHORITY: &str = - include_str!("../src/adapters/retention/filesystem_retention_authority.rs"); -const CATALOG_PUBLISHER: &str = - include_str!("../src/adapters/filesystem_version_two_catalog_publisher.rs"); -const VERSION_TWO_ADMISSION: &str = - include_str!("../src/adapters/filesystem_version_two_admission.rs"); -const ADMISSION_ERROR: &str = - include_str!("../src/adapters/filesystem_platform_admission_error.rs"); - -/// The type-level proofs are the `compile_fail` doctests on -/// `FilesystemRetentionPublicationAuthority::open` and -/// `FilesystemVersionTwoCatalogPublisher::open`, which refuse a bare -/// `FilesystemWriterLock`. These markers only keep each writer on the -/// admission side of the boundary without pinning a signature. -#[test] -fn every_writer_consumes_admitted_authority() { - assert!(RETENTION_AUTHORITY.contains("FilesystemVersionTwoAdmission")); - assert!(CATALOG_PUBLISHER.contains("admission: FilesystemVersionTwoAdmission")); -} - -#[test] -fn reopen_admits_the_marker_under_the_production_profile() { - assert!(VERSION_TWO_ADMISSION.contains("pub struct FilesystemVersionTwoAdmission")); - assert!(VERSION_TWO_ADMISSION.contains("pub fn reopen(store_root: &Path)")); - assert!(VERSION_TWO_ADMISSION.contains("pub fn initialize(")); - assert!(VERSION_TWO_ADMISSION.contains("filesystem_format_marker::admit")); - assert!(VERSION_TWO_ADMISSION.contains("filesystem_platform_profile::open_version_two")); -} - -/// Admission refusals grow with the platform surface, so the public error is -/// non-exhaustive and a new refusal is not a breaking change. -#[test] -fn admission_error_is_non_exhaustive() { - assert!( - ADMISSION_ERROR.contains("#[non_exhaustive]\npub enum FilesystemPlatformAdmissionError") - ); -} diff --git a/xtask/src/durability_crash_matrix/error.rs b/xtask/src/durability_crash_matrix/error.rs index 986beb50..b4795ced 100644 --- a/xtask/src/durability_crash_matrix/error.rs +++ b/xtask/src/durability_crash_matrix/error.rs @@ -7,7 +7,7 @@ use std::error::Error; use std::io; use std::time::Duration; -use keep::VersionTwoInitializationPrefix; +use keep::StoreInitializationPrefix; use xtask::protocol_admission::HexError; use xtask::{ DurabilityCrashCase, DurabilityCrashCaseError, DurabilityCrashOccurrence, DurabilityCrashPoint, @@ -84,8 +84,8 @@ pub(crate) enum DurabilityCrashMatrixError { point: DurabilityCrashPoint, }, InitializationPrefixMismatch { - expected: Option, - observed: Option, + expected: Option, + observed: Option, }, RepeatedInventoryPath { path: String, diff --git a/xtask/src/durability_crash_matrix/error/display.rs b/xtask/src/durability_crash_matrix/error/display.rs index 00b18d0c..2a6358ee 100644 --- a/xtask/src/durability_crash_matrix/error/display.rs +++ b/xtask/src/durability_crash_matrix/error/display.rs @@ -235,7 +235,7 @@ fn format_command( DurabilityCrashMatrixError::Usage => formatter.write_str( "usage: cargo xtask durability-crash-matrix \ [--case [] \ - | --sequence ]", + | --sequence ]", ), _ => Err(fmt::Error), } diff --git a/xtask/src/durability_crash_matrix/production_protocol.rs b/xtask/src/durability_crash_matrix/production_protocol.rs index a6f7b53c..8dadd465 100644 --- a/xtask/src/durability_crash_matrix/production_protocol.rs +++ b/xtask/src/durability_crash_matrix/production_protocol.rs @@ -8,11 +8,11 @@ pub(super) mod populated_store; mod publication; mod recovery; mod recovery_storage; +pub(super) mod retained_catalog; pub(super) mod retention; mod retention_storage; mod segment_stage; -pub(super) mod version_two_catalog; -pub(super) mod version_two_initialization; +pub(super) mod writer; use std::error::Error; use std::fs; @@ -45,11 +45,11 @@ pub(super) fn run( DurabilityCrashSequence::Retention => { retention::run(&store_root, &mut control)?; } - DurabilityCrashSequence::VersionTwoInitialization => { - version_two_initialization::run(&store_root, control)?; + DurabilityCrashSequence::Initialization => { + initialization::run(&store_root, control)?; } - DurabilityCrashSequence::VersionTwoCatalog => { - version_two_catalog::run(&store_root, control)?; + DurabilityCrashSequence::RetainedCatalog => { + retained_catalog::run(&store_root, control)?; } } Err(DurabilityCrashMatrixError::PointSequenceMismatch { diff --git a/xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs b/xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs index 4ca9eab1..3a331f9d 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/crash_blob.rs @@ -11,8 +11,8 @@ use keep::{ use super::super::DurabilityCrashMatrixError; use super::verification; -/// The exact bytes the version-two writer ingests before process death. -pub(in crate::durability_crash_matrix) const VERSION_TWO_CONTENT: &[u8] = +/// The exact bytes the writer ingests before process death. +pub(in crate::durability_crash_matrix) const SUCCESSOR_CONTENT: &[u8] = b"keep version-two crash campaign: one blob ingested after the first catalog"; /// The synthetic source byte of the frozen one-zero bundle corpus. @@ -27,10 +27,10 @@ pub(in crate::durability_crash_matrix) struct CrashBlob { } impl CrashBlob { - /// The blob the version-two writer ingests before process death. - pub(in crate::durability_crash_matrix) fn version_two() - -> Result { - Self::identify(VERSION_TWO_CONTENT) + /// The blob the writer ingests before process death. + pub(in crate::durability_crash_matrix) fn successor() -> Result + { + Self::identify(SUCCESSOR_CONTENT) } /// The blob whose first catalog is the frozen bundle corpus. diff --git a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs index bf30c95e..4f843e48 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/initialization.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/initialization.rs @@ -1,33 +1,173 @@ -//! This module owns the crash children' initialized writer and read policies. +//! This module owns execution of the production initializer. +//! +//! An observer installed on the initializer's own phases blocks at the +//! selected boundary for process death; no phase is reimplemented here. The +//! recovery row first stops one initialization after its writer lock, with an +//! incomplete `FORMAT.next` in place, then dies inside the resuming call. -use std::path::Path; +use std::fs::OpenOptions; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; use keep::{ - CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemVersionTwoAdmission, - FilesystemVersionTwoCatalogPublisher, LayoutEntryLimit, SegmentReadPolicy, SegmentRecordLimit, + CanonicalStoreFormatMarker, FilesystemStoreAdmission, StoreInitializationBoundary, + StoreInitializationDirectory, StoreInitializationObserver, StoreInitializationPhase, }; +use xtask::{DurabilityCrashOccurrence, DurabilityCrashPoint, DurabilityCrashPosition}; +use super::control::{CrashControl, DuringTiming}; use super::{DurabilityCrashMatrixError, verification}; -const RESTART_BYTE_LIMIT: u64 = 1_048_576; +/// Length of the incomplete marker stage a torn write leaves; a nonempty +/// strict prefix of the 96-byte marker. +pub(in crate::durability_crash_matrix) const INCOMPLETE_STAGE_LENGTH: usize = 48; -/// Initializes the single-format store and opens its writer, as production does. -pub(super) fn publisher( +/// Initializes the store root, dying at the selected coordinate. +pub(super) fn run( store_root: &Path, -) -> Result { - let admission = FilesystemVersionTwoAdmission::initialize(store_root) - .map_err(|source| verification("initialize crash catalog store", source))? - .into_admission(); - FilesystemVersionTwoCatalogPublisher::open(admission, restart_policy()?) - .map_err(|source| DurabilityCrashMatrixError::io("open crash catalog publisher", source)) + control: CrashControl, +) -> Result<(), DurabilityCrashMatrixError> { + if control + .position(DurabilityCrashPoint::InitializeDiscardIncompleteMarkerStage) + .is_some() + { + leave_incomplete_stage(store_root)?; + } + let observer = observer(control, store_root.to_path_buf()); + FilesystemStoreAdmission::initialize_observed_for_repository_tasks(store_root, observer) + .map(drop) + .map_err(|source| verification("execute production initialization", source)) } -pub(super) fn restart_policy() -> Result { - let byte_limit = CatalogRestartByteLimit::new(RESTART_BYTE_LIMIT) - .map_err(|source| verification("construct crash restart byte limit", source))?; - Ok(CatalogRestartPolicy::new(segment_policy(), byte_limit)) +/// Stops a real initialization at its marker write with a torn stage in place. +fn leave_incomplete_stage(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { + let root = store_root.to_path_buf(); + let observer: StoreInitializationObserver = Box::new(move |phase, boundary| { + if phase == StoreInitializationPhase::WriteMarkerStage + && boundary == StoreInitializationBoundary::Before + { + write_incomplete_stage(&root)?; + return Err(io::Error::other("stopped with an incomplete marker stage")); + } + Ok(()) + }); + match FilesystemStoreAdmission::initialize_observed_for_repository_tasks(store_root, observer) { + Err(_) => Ok(()), + Ok(_) => Err(verification( + "prepare an incomplete marker stage", + io::Error::other("initialization completed past the marker write"), + )), + } } -pub(in crate::durability_crash_matrix) const fn segment_policy() -> SegmentReadPolicy { - SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +/// Writes the strict canonical prefix a torn write could leave, unsynchronized. +fn write_incomplete_stage(root: &Path) -> io::Result<()> { + let marker = CanonicalStoreFormatMarker::version_two(); + let prefix = marker + .encoded() + .get(..INCOMPLETE_STAGE_LENGTH) + .ok_or_else(|| io::Error::other("marker shorter than the incomplete stage"))?; + let mut stage = OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join("FORMAT.next"))?; + stage.write_all(prefix)?; + stage.flush() +} + +/// Maps the initializer's own phases onto the eleven initialization crash points. +fn observer(mut control: CrashControl, root: PathBuf) -> StoreInitializationObserver { + Box::new(move |phase, boundary| { + if phase == StoreInitializationPhase::WriteMarkerStage + && boundary == StoreInitializationBoundary::Before + && control.position(DurabilityCrashPoint::InitializeWriteMarkerStage) + == Some(DurabilityCrashPosition::During) + { + write_incomplete_stage(&root)?; + return control.await_process_death(); + } + if let StoreInitializationPhase::AdmitDirectory(directory) = phase { + return directory_boundary(&mut control, directory, boundary); + } + let Some((point, timing)) = crash_point(phase) else { + return Ok(()); + }; + match boundary { + StoreInitializationBoundary::Before => control.before(point, timing), + StoreInitializationBoundary::After => control.after(point, timing), + } + }) +} + +/// Directory admission is one occurrence-counted point: `before` precedes the +/// first directory, `after` follows the last, and `during` occurrence *k* +/// follows directory *k* + 1. +fn directory_boundary( + control: &mut CrashControl, + directory: StoreInitializationDirectory, + boundary: StoreInitializationBoundary, +) -> io::Result<()> { + let point = DurabilityCrashPoint::InitializeAdmitDirectory; + let ordinal = StoreInitializationDirectory::ALL + .iter() + .position(|candidate| *candidate == directory) + .ok_or_else(|| io::Error::other("directory outside the creation order"))?; + let last = StoreInitializationDirectory::ALL.len().saturating_sub(1); + let selected = match (control.position(point), boundary) { + (Some(DurabilityCrashPosition::Before), StoreInitializationBoundary::Before) => { + ordinal == 0 + } + (Some(DurabilityCrashPosition::After), StoreInitializationBoundary::After) => { + ordinal == last + } + (Some(DurabilityCrashPosition::During), StoreInitializationBoundary::After) => { + let occurrence = control + .occurrence() + .map_or(0, DurabilityCrashOccurrence::get); + usize::try_from(occurrence).map_err(io::Error::other)? == ordinal + } + _ => false, + }; + if selected { + control.await_process_death() + } else { + Ok(()) + } +} + +const fn crash_point( + phase: StoreInitializationPhase, +) -> Option<(DurabilityCrashPoint, DuringTiming)> { + use DurabilityCrashPoint as Point; + use StoreInitializationPhase as Phase; + Some(match phase { + Phase::OpenAndLockWriterFile => { + (Point::InitializeOpenAndLockWriterFile, DuringTiming::After) + } + Phase::WriteMarkerStage => (Point::InitializeWriteMarkerStage, DuringTiming::After), + Phase::SynchronizeMarkerStage => ( + Point::InitializeSynchronizeMarkerStage, + DuringTiming::Before, + ), + Phase::SynchronizeRootAfterMarkerStage => ( + Point::InitializeSynchronizeRootAfterMarkerStage, + DuringTiming::Before, + ), + Phase::AdmitReaderFence => (Point::InitializeAdmitReaderFence, DuringTiming::After), + Phase::LinkMarker => (Point::InitializeLinkMarker, DuringTiming::After), + Phase::SynchronizeRootAfterMarker => ( + Point::InitializeSynchronizeRootAfterMarker, + DuringTiming::Before, + ), + Phase::RemoveMarkerStage => (Point::InitializeRemoveMarkerStage, DuringTiming::After), + Phase::SynchronizeRootAfterCleanup => ( + Point::InitializeSynchronizeRootAfterCleanup, + DuringTiming::Before, + ), + Phase::DiscardIncompleteMarkerStage => ( + Point::InitializeDiscardIncompleteMarkerStage, + DuringTiming::After, + ), + _ => return None, + }) } diff --git a/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs b/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs index fb43cfb2..4c3add23 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/populated_store.rs @@ -4,20 +4,20 @@ //! the same public entry points, under the matrix's typed error, because a //! repository task cannot include integration-test support. Creation is direct //! initialization, then the first catalog through -//! `FilesystemVersionTwoCatalogPublisher` and the retained-closure gate. +//! `FilesystemCatalogPublisher` and the retained-closure gate. use std::fs; use std::path::Path; use keep::{ AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationExpectation, - FilesystemSegmentStage, FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, - SealedSegment, SegmentRecordLimit, StagedSegment, publish_retained_catalog_generation, + FilesystemCatalogPublisher, FilesystemSegmentStage, FilesystemStoreAdmission, SealedSegment, + SegmentRecordLimit, StagedSegment, publish_retained_catalog_generation, }; use super::crash_blob::CrashBlob; use super::fixture::{BUNDLE_CATALOG_NAME, BUNDLE_SEGMENT_NAME, GoldenFixture}; -use super::{DurabilityCrashMatrixError, initialization, verification, version_two_catalog}; +use super::{DurabilityCrashMatrixError, retained_catalog, verification, writer}; /// Creates the one-zero bundle store at the empty directory `store_root`. /// @@ -25,10 +25,9 @@ use super::{DurabilityCrashMatrixError, initialization, verification, version_tw /// against the frozen bundle snapshot and retention root, so the pools must /// hold exactly the frozen bundle bytes; any other bytes refuse here. pub(super) fn create_bundle_store(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let admission = - FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks(store_root) - .map_err(|source| verification("initialize crash bundle store", source))? - .into_admission(); + let admission = FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("initialize crash bundle store", source))? + .into_admission(); publish_bundle_catalog(store_root, admission)?; require_frozen_bundle(store_root) } @@ -36,19 +35,16 @@ pub(super) fn create_bundle_store(store_root: &Path) -> Result<(), DurabilityCra /// Publishes the bundle blob as the first catalog of an initialized store. pub(in crate::durability_crash_matrix) fn publish_bundle_catalog( store_root: &Path, - admission: FilesystemVersionTwoAdmission, + admission: FilesystemStoreAdmission, ) -> Result<(), DurabilityCrashMatrixError> { - let mut publisher = - FilesystemVersionTwoCatalogPublisher::open(admission, version_two_catalog::policy()?) - .map_err(|source| verification("open crash bundle publisher", source))?; + let mut publisher = FilesystemCatalogPublisher::open(admission, retained_catalog::policy()?) + .map_err(|source| verification("open crash bundle publisher", source))?; let blob = CrashBlob::bundle()?; let sealed = stage_bundle(&publisher, &blob)?; let bytes = fs::read(store_root.join("staging/current.seg")) .map_err(|source| DurabilityCrashMatrixError::io("read bundle segment", source))?; - let segments = [ - AdmittedSegment::decode(&bytes, initialization::segment_policy()) - .map_err(|source| verification("admit bundle segment", source))?, - ]; + let segments = [AdmittedSegment::decode(&bytes, writer::segment_policy()) + .map_err(|source| verification("admit bundle segment", source))?]; let selection = publisher .select_segment(sealed, &segments[0]) .map_err(|source| verification("select bundle segment stage", source))?; @@ -68,7 +64,7 @@ pub(in crate::durability_crash_matrix) fn publish_bundle_catalog( } fn stage_bundle<'publisher>( - publisher: &'publisher FilesystemVersionTwoCatalogPublisher, + publisher: &'publisher FilesystemCatalogPublisher, blob: &CrashBlob, ) -> Result>, DurabilityCrashMatrixError> { let stage = publisher diff --git a/xtask/src/durability_crash_matrix/production_protocol/publication.rs b/xtask/src/durability_crash_matrix/production_protocol/publication.rs index 14fbd269..bff42696 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/publication.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/publication.rs @@ -1,7 +1,7 @@ //! This module owns execution of the production publication protocols. //! //! The child initializes a store directly and publishes its first catalog -//! through `FilesystemVersionTwoCatalogPublisher` and the retained-closure +//! through `FilesystemCatalogPublisher` and the retained-closure //! gate. Segment staging boundaries are observed through the segment-stage //! port and every pool, catalog, and head boundary through the writer's own //! catalog observer, so no transition is reimplemented here. @@ -13,15 +13,15 @@ use std::path::{Path, PathBuf}; use keep::{ AdmittedSegment, AdmittedSegmentRecord, CanonicalCatalog, CanonicalPublicationHead, CatalogGeneration, CatalogPublicationBoundary, CatalogPublicationExpectation, - CatalogPublicationObserver, CatalogPublicationPhase, FilesystemSegmentStage, - FilesystemVersionTwoCatalogPublisher, ObservedSegmentStage, SealedSegment, SegmentRecordLimit, - StagedSegment, publish_retained_catalog_generation, + CatalogPublicationObserver, CatalogPublicationPhase, FilesystemCatalogPublisher, + FilesystemSegmentStage, ObservedSegmentStage, SealedSegment, SegmentRecordLimit, StagedSegment, + publish_retained_catalog_generation, }; use xtask::{DurabilityCrashPoint, DurabilityCrashPosition}; use super::control::{CrashControl, DuringTiming}; -use super::initialization; use super::segment_stage::CrashSegmentObserver; +use super::writer; use super::{DurabilityCrashMatrixError, verification}; const CATALOG_INTERRUPTION: usize = 176; @@ -33,12 +33,12 @@ pub(super) fn run( store_root: &Path, mut control: CrashControl, ) -> Result<(), DurabilityCrashMatrixError> { - let mut publisher = initialization::publisher(store_root)?; + let mut publisher = writer::publisher(store_root)?; let sealed = stage_observed(&publisher, &mut control)?; let segment_bytes = fs::read(store_root.join("staging/current.seg")) .map_err(|source| DurabilityCrashMatrixError::io("read sealed crash segment", source))?; let segments = [ - AdmittedSegment::decode(&segment_bytes, initialization::segment_policy()) + AdmittedSegment::decode(&segment_bytes, writer::segment_policy()) .map_err(|source| verification("admit sealed crash segment", source))?, ]; let catalog = first_catalog(&segments)?; @@ -70,7 +70,7 @@ pub(super) fn first_catalog( /// Stages and seals the one-zero segment with no crash observation. pub(super) fn stage_one_zero( - publisher: &FilesystemVersionTwoCatalogPublisher, + publisher: &FilesystemCatalogPublisher, ) -> Result>, DurabilityCrashMatrixError> { let stage = publisher .create_segment_stage() @@ -84,7 +84,7 @@ pub(super) fn stage_one_zero( } fn stage_observed<'publisher>( - publisher: &'publisher FilesystemVersionTwoCatalogPublisher, + publisher: &'publisher FilesystemCatalogPublisher, control: &mut CrashControl, ) -> Result>, DurabilityCrashMatrixError> { let point = DurabilityCrashPoint::CreateSegmentStage; diff --git a/xtask/src/durability_crash_matrix/production_protocol/recovery.rs b/xtask/src/durability_crash_matrix/production_protocol/recovery.rs index 21df9d91..d5c4b8eb 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/recovery.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/recovery.rs @@ -7,7 +7,7 @@ use std::path::Path; use keep::{ AdmittedSegment, CanonicalPublicationHead, CatalogPublicationBoundary, CatalogPublicationError, CatalogPublicationExpectation, CatalogPublicationObserver, CatalogPublicationPhase, - FilesystemRecoveryStageDiscarder, FilesystemVersionTwoRecoveryAdmission, RecoveryStage, + FilesystemRecoveryAdmission, FilesystemRecoveryStageDiscarder, RecoveryStage, RecoveryStageDiscardRequest, RecoveryStageMetadata, RetainedCatalogPublicationError, admit_recovery_stage_bytes, assess_recovery_stage, execute_recovery_stage_discard, fingerprint_recovery_stage, plan_recovery_stage_discard, publish_retained_catalog_generation, @@ -18,7 +18,7 @@ use super::control::CrashControl; use super::fixture::GoldenFixture; use super::recovery_storage::CrashRecoveryStorage; use super::{DurabilityCrashMatrixError, verification}; -use super::{initialization, publication}; +use super::{publication, writer}; const SEGMENT_INTERRUPTION: usize = 32; const HEAD_INTERRUPTION: usize = 64; @@ -33,10 +33,9 @@ pub(super) fn run( prepare_head(store_root)? }; let request = discard_request(store_root, stage)?; - let admission = - FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(store_root) - .map_err(|source| verification("admit production recovery store", source))?; - let storage = FilesystemRecoveryStageDiscarder::open_version_two(admission) + let admission = FilesystemRecoveryAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("admit production recovery store", source))?; + let storage = FilesystemRecoveryStageDiscarder::open(admission) .map_err(|source| verification("open production recovery discarder", source))?; let mut storage = CrashRecoveryStorage::new(storage, control); execute_recovery_stage_discard(&mut storage, request) @@ -54,7 +53,7 @@ fn targets_segment(control: &CrashControl) -> bool { } fn prepare_segment(store_root: &Path) -> Result { - let publisher = initialization::publisher(store_root)?; + let publisher = writer::publisher(store_root)?; let mut stage = publisher .create_segment_stage() .map_err(|source| verification("create recovery segment precondition", source))?; @@ -75,12 +74,12 @@ fn prepare_segment(store_root: &Path) -> Result Result { - let mut publisher = initialization::publisher(store_root)?; + let mut publisher = writer::publisher(store_root)?; let sealed = publication::stage_one_zero(&publisher)?; let segment_bytes = fs::read(store_root.join("staging/current.seg")) .map_err(|source| DurabilityCrashMatrixError::io("read recovery head segment", source))?; let segments = [ - AdmittedSegment::decode(&segment_bytes, initialization::segment_policy()) + AdmittedSegment::decode(&segment_bytes, writer::segment_policy()) .map_err(|source| verification("admit recovery head segment", source))?, ]; let catalog = publication::first_catalog(&segments)?; @@ -161,7 +160,7 @@ fn discard_request( .map_err(|source| verification("fingerprint recovery precondition", source))?; let admitted = admit_recovery_stage_bytes(stage, evidence, &bytes) .map_err(|source| verification("admit recovery precondition bytes", source))?; - let assessment = assess_recovery_stage(&admitted, initialization::segment_policy()) + let assessment = assess_recovery_stage(&admitted, writer::segment_policy()) .map_err(|source| verification("assess recovery precondition", source))?; plan_recovery_stage_discard(&assessment) .map_err(|source| verification("plan production recovery discard", source)) diff --git a/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs b/xtask/src/durability_crash_matrix/production_protocol/retained_catalog.rs similarity index 67% rename from xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs rename to xtask/src/durability_crash_matrix/production_protocol/retained_catalog.rs index bc1e0ee3..f4af8267 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/version_two_catalog.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/retained_catalog.rs @@ -1,7 +1,7 @@ -//! This module owns execution of the production version-two catalog writer. +//! This module owns execution of the production catalog writer. //! //! The child initializes the bundle store, publishes retention generation one, -//! then ingests one new blob through the version-two writer. An observer +//! then ingests one new blob through the writer. An observer //! installed on the writer's own stage type blocks at the selected boundary //! for process death; no transition is reimplemented here. @@ -11,55 +11,53 @@ use std::path::Path; use keep::{ AdmittedSegment, CanonicalCatalog, CatalogGeneration, CatalogPublicationBoundary, CatalogPublicationExpectation, CatalogPublicationObserver, CatalogPublicationPhase, - CatalogSnapshot, FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, - SegmentRecordLimit, StagedSegment, execute_retention_publication, - publish_retained_catalog_generation, + CatalogSnapshot, FilesystemCatalogPublisher, FilesystemStoreAdmission, SegmentRecordLimit, + StagedSegment, execute_retention_publication, publish_retained_catalog_generation, }; use xtask::DurabilityCrashPoint; use super::control::{CrashControl, DuringTiming}; use super::crash_blob::CrashBlob; use super::fixture::GoldenFixture; -use super::{DurabilityCrashMatrixError, initialization, retention, verification}; +use super::{DurabilityCrashMatrixError, retention, verification, writer}; -/// Ingests the version-two crash blob, dying at the selected coordinate. +/// Ingests the retained-catalog crash blob, dying at the selected coordinate. pub(super) fn run( store_root: &Path, control: CrashControl, ) -> Result<(), DurabilityCrashMatrixError> { retained_bundle_store(store_root)?; - let admission = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store_root) - .map_err(|source| verification("reopen crash store for version-two writer", source))?; - let mut publisher = FilesystemVersionTwoCatalogPublisher::open(admission, policy()?) - .map_err(|source| verification("open version-two catalog publisher", source))?; + let admission = FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("reopen crash store for writer", source))?; + let mut publisher = FilesystemCatalogPublisher::open(admission, policy()?) + .map_err(|source| verification("open catalog publisher", source))?; publisher.observe_catalog_phases_for_repository_tasks(observer(control)); - let blob = CrashBlob::version_two()?; + let blob = CrashBlob::successor()?; let mut stage = StagedSegment::begin( publisher .create_segment_stage() - .map_err(|source| verification("create version-two segment stage", source))?, + .map_err(|source| verification("create segment stage", source))?, SegmentRecordLimit::MAXIMUM, ) - .map_err(|source| verification("write version-two segment header", source))?; + .map_err(|source| verification("write segment header", source))?; for record in blob.records()? { stage = stage .append(record) - .map_err(|source| verification("append version-two segment record", source))?; + .map_err(|source| verification("append segment record", source))?; } let sealed = stage .seal() - .map_err(|source| verification("seal version-two segment", source))?; + .map_err(|source| verification("seal segment", source))?; let bundle = GoldenFixture::bundle_segment()?; let staged = fs::read(store_root.join("staging/current.seg")) - .map_err(|source| DurabilityCrashMatrixError::io("read version-two segment", source))?; + .map_err(|source| DurabilityCrashMatrixError::io("read segment", source))?; let segments = admit_segments(&[bundle.bytes(), &staged])?; let new_segment = segments .last() .ok_or(DurabilityCrashMatrixError::FixtureRange)?; let selection = publisher .select_segment(sealed, new_segment) - .map_err(|source| verification("select version-two segment stage", source))?; + .map_err(|source| verification("select segment stage", source))?; let head = GoldenFixture::bundle_head()?; let catalog_fixture = GoldenFixture::bundle_catalog()?; let predecessor = segments @@ -68,13 +66,13 @@ pub(super) fn run( let current = bundle_snapshot(head.bytes(), catalog_fixture.bytes(), predecessor)?; let expectation = CatalogPublicationExpectation::successor_of(¤t); let generation = CatalogGeneration::new(2) - .map_err(|source| verification("construct version-two catalog generation", source))?; + .map_err(|source| verification("construct catalog generation", source))?; let catalog = CanonicalCatalog::from_segments(generation, Some(current.catalog_digest()), &segments) - .map_err(|source| verification("encode version-two catalog", source))?; + .map_err(|source| verification("encode catalog", source))?; publish_retained_catalog_generation(&mut publisher, expectation, selection, &catalog, &segments) .map(|_receipt| ()) - .map_err(|source| verification("execute version-two catalog publication", source)) + .map_err(|source| verification("execute catalog publication", source)) } /// Creates the bundle store and publishes its golden retention root. @@ -92,9 +90,9 @@ pub(in crate::durability_crash_matrix) fn retained_bundle_store( pub(in crate::durability_crash_matrix) fn policy() -> Result { let limit = keep::CatalogRestartByteLimit::new(1_048_576) - .map_err(|source| verification("bound version-two catalog read", source))?; + .map_err(|source| verification("bound catalog read", source))?; Ok(keep::CatalogRestartPolicy::new( - initialization::segment_policy(), + writer::segment_policy(), limit, )) } @@ -105,8 +103,8 @@ pub(in crate::durability_crash_matrix) fn admit_segments<'bytes>( encoded .iter() .map(|bytes| { - AdmittedSegment::decode(bytes, initialization::segment_policy()) - .map_err(|source| verification("admit version-two crash segment", source)) + AdmittedSegment::decode(bytes, writer::segment_policy()) + .map_err(|source| verification("admit retained-catalog crash segment", source)) }) .collect() } @@ -126,7 +124,7 @@ fn bundle_snapshot<'bytes>( .map_err(|source| verification("admit bundle snapshot", source)) } -/// Maps the writer's own transitions onto the four version-two crash points. +/// Maps the writer's own transitions onto the four retained-catalog crash points. fn observer(mut control: CrashControl) -> CatalogPublicationObserver { Box::new(move |phase, boundary| { let Some((point, timing)) = crash_point(phase) else { @@ -144,19 +142,19 @@ const fn crash_point( ) -> Option<(DurabilityCrashPoint, DuringTiming)> { match phase { CatalogPublicationPhase::LinkSegment => Some(( - DurabilityCrashPoint::VersionTwoLinkSegment, + DurabilityCrashPoint::RetainedLinkSegment, DuringTiming::After, )), CatalogPublicationPhase::CreateCatalogStage => Some(( - DurabilityCrashPoint::VersionTwoCreateCatalogStage, + DurabilityCrashPoint::RetainedCreateCatalogStage, DuringTiming::After, )), CatalogPublicationPhase::SynchronizeHead => Some(( - DurabilityCrashPoint::VersionTwoSynchronizeHead, + DurabilityCrashPoint::RetainedSynchronizeHead, DuringTiming::Before, )), CatalogPublicationPhase::ReplaceHead => Some(( - DurabilityCrashPoint::VersionTwoReplaceHead, + DurabilityCrashPoint::RetainedReplaceHead, DuringTiming::After, )), _ => None, diff --git a/xtask/src/durability_crash_matrix/production_protocol/retention.rs b/xtask/src/durability_crash_matrix/production_protocol/retention.rs index 2ff9bd99..aa487fa4 100644 --- a/xtask/src/durability_crash_matrix/production_protocol/retention.rs +++ b/xtask/src/durability_crash_matrix/production_protocol/retention.rs @@ -4,16 +4,16 @@ use std::path::Path; use keep::{ AdmittedCatalog, AdmittedRetentionRoot, AdmittedSegment, ChecksummedCatalog, - ChecksummedPublicationHead, FilesystemRetentionPublicationAuthority, - FilesystemVersionTwoAdmission, RetentionGenerationExpectation, RetentionPublicationPreparation, - execute_retention_publication, preflight_retention_transition, prepare_retention_publication, + ChecksummedPublicationHead, FilesystemRetentionPublicationAuthority, FilesystemStoreAdmission, + RetentionGenerationExpectation, RetentionPublicationPreparation, execute_retention_publication, + preflight_retention_transition, prepare_retention_publication, }; use super::control::CrashControl; use super::fixture::GoldenFixture; -use super::initialization; use super::populated_store; use super::retention_storage::CrashRetentionStorage; +use super::writer; use super::{DurabilityCrashMatrixError, verification}; /// Initializes a fresh bundle store and publishes retention generation one, @@ -44,9 +44,8 @@ pub(super) fn bundle_authority( pub(in crate::durability_crash_matrix) fn reopened_authority( store_root: &Path, ) -> Result { - let admission = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store_root) - .map_err(|source| verification("reopen crash store as version two", source))?; + let admission = FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("reopen crash store", source))?; FilesystemRetentionPublicationAuthority::open(admission) .map_err(|source| verification("open crash retention authority", source)) } @@ -61,9 +60,8 @@ pub(in crate::durability_crash_matrix) fn preparation( let head_fixture = GoldenFixture::bundle_head()?; let candidate = AdmittedRetentionRoot::decode(root_bytes) .map_err(|source| verification("decode crash retention root", source))?; - let segment = - AdmittedSegment::decode(segment_fixture.bytes(), initialization::segment_policy()) - .map_err(|source| verification("admit bundle segment", source))?; + let segment = AdmittedSegment::decode(segment_fixture.bytes(), writer::segment_policy()) + .map_err(|source| verification("admit bundle segment", source))?; let segments = [segment]; let catalog: AdmittedCatalog<'_, '_> = ChecksummedCatalog::decode(catalog_fixture.bytes()) .map_err(|source| verification("decode bundle catalog", source))? diff --git a/xtask/src/durability_crash_matrix/production_protocol/version_two_initialization.rs b/xtask/src/durability_crash_matrix/production_protocol/version_two_initialization.rs deleted file mode 100644 index db773d7a..00000000 --- a/xtask/src/durability_crash_matrix/production_protocol/version_two_initialization.rs +++ /dev/null @@ -1,178 +0,0 @@ -//! This module owns execution of the production version-two initializer. -//! -//! An observer installed on the initializer's own phases blocks at the -//! selected boundary for process death; no phase is reimplemented here. The -//! recovery row first stops one initialization after its writer lock, with an -//! incomplete `FORMAT.next` in place, then dies inside the resuming call. - -use std::fs::OpenOptions; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; - -use keep::{ - CanonicalStoreFormatMarker, FilesystemVersionTwoAdmission, VersionTwoInitializationBoundary, - VersionTwoInitializationDirectory, VersionTwoInitializationObserver, - VersionTwoInitializationPhase, -}; -use xtask::{DurabilityCrashOccurrence, DurabilityCrashPoint, DurabilityCrashPosition}; - -use super::control::{CrashControl, DuringTiming}; -use super::{DurabilityCrashMatrixError, verification}; - -/// Length of the incomplete marker stage a torn write leaves; a nonempty -/// strict prefix of the 96-byte marker. -pub(in crate::durability_crash_matrix) const INCOMPLETE_STAGE_LENGTH: usize = 48; - -/// Initializes the store root, dying at the selected coordinate. -pub(super) fn run( - store_root: &Path, - control: CrashControl, -) -> Result<(), DurabilityCrashMatrixError> { - if control - .position(DurabilityCrashPoint::InitializeDiscardIncompleteMarkerStage) - .is_some() - { - leave_incomplete_stage(store_root)?; - } - let observer = observer(control, store_root.to_path_buf()); - FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks(store_root, observer) - .map(drop) - .map_err(|source| verification("execute production initialization", source)) -} - -/// Stops a real initialization at its marker write with a torn stage in place. -fn leave_incomplete_stage(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let root = store_root.to_path_buf(); - let observer: VersionTwoInitializationObserver = Box::new(move |phase, boundary| { - if phase == VersionTwoInitializationPhase::WriteMarkerStage - && boundary == VersionTwoInitializationBoundary::Before - { - write_incomplete_stage(&root)?; - return Err(io::Error::other("stopped with an incomplete marker stage")); - } - Ok(()) - }); - match FilesystemVersionTwoAdmission::initialize_observed_for_repository_tasks( - store_root, observer, - ) { - Err(_) => Ok(()), - Ok(_) => Err(verification( - "prepare an incomplete marker stage", - io::Error::other("initialization completed past the marker write"), - )), - } -} - -/// Writes the strict canonical prefix a torn write could leave, unsynchronized. -fn write_incomplete_stage(root: &Path) -> io::Result<()> { - let marker = CanonicalStoreFormatMarker::version_two(); - let prefix = marker - .encoded() - .get(..INCOMPLETE_STAGE_LENGTH) - .ok_or_else(|| io::Error::other("marker shorter than the incomplete stage"))?; - let mut stage = OpenOptions::new() - .write(true) - .create_new(true) - .open(root.join("FORMAT.next"))?; - stage.write_all(prefix)?; - stage.flush() -} - -/// Maps the initializer's own phases onto the eleven initialization crash points. -fn observer(mut control: CrashControl, root: PathBuf) -> VersionTwoInitializationObserver { - Box::new(move |phase, boundary| { - if phase == VersionTwoInitializationPhase::WriteMarkerStage - && boundary == VersionTwoInitializationBoundary::Before - && control.position(DurabilityCrashPoint::InitializeWriteMarkerStage) - == Some(DurabilityCrashPosition::During) - { - write_incomplete_stage(&root)?; - return control.await_process_death(); - } - if let VersionTwoInitializationPhase::AdmitDirectory(directory) = phase { - return directory_boundary(&mut control, directory, boundary); - } - let Some((point, timing)) = crash_point(phase) else { - return Ok(()); - }; - match boundary { - VersionTwoInitializationBoundary::Before => control.before(point, timing), - VersionTwoInitializationBoundary::After => control.after(point, timing), - } - }) -} - -/// Directory admission is one occurrence-counted point: `before` precedes the -/// first directory, `after` follows the last, and `during` occurrence *k* -/// follows directory *k* + 1. -fn directory_boundary( - control: &mut CrashControl, - directory: VersionTwoInitializationDirectory, - boundary: VersionTwoInitializationBoundary, -) -> io::Result<()> { - let point = DurabilityCrashPoint::InitializeAdmitDirectory; - let ordinal = VersionTwoInitializationDirectory::ALL - .iter() - .position(|candidate| *candidate == directory) - .ok_or_else(|| io::Error::other("directory outside the creation order"))?; - let last = VersionTwoInitializationDirectory::ALL - .len() - .saturating_sub(1); - let selected = match (control.position(point), boundary) { - (Some(DurabilityCrashPosition::Before), VersionTwoInitializationBoundary::Before) => { - ordinal == 0 - } - (Some(DurabilityCrashPosition::After), VersionTwoInitializationBoundary::After) => { - ordinal == last - } - (Some(DurabilityCrashPosition::During), VersionTwoInitializationBoundary::After) => { - let occurrence = control - .occurrence() - .map_or(0, DurabilityCrashOccurrence::get); - usize::try_from(occurrence).map_err(io::Error::other)? == ordinal - } - _ => false, - }; - if selected { - control.await_process_death() - } else { - Ok(()) - } -} - -const fn crash_point( - phase: VersionTwoInitializationPhase, -) -> Option<(DurabilityCrashPoint, DuringTiming)> { - use DurabilityCrashPoint as Point; - use VersionTwoInitializationPhase as Phase; - Some(match phase { - Phase::OpenAndLockWriterFile => { - (Point::InitializeOpenAndLockWriterFile, DuringTiming::After) - } - Phase::WriteMarkerStage => (Point::InitializeWriteMarkerStage, DuringTiming::After), - Phase::SynchronizeMarkerStage => ( - Point::InitializeSynchronizeMarkerStage, - DuringTiming::Before, - ), - Phase::SynchronizeRootAfterMarkerStage => ( - Point::InitializeSynchronizeRootAfterMarkerStage, - DuringTiming::Before, - ), - Phase::AdmitReaderFence => (Point::InitializeAdmitReaderFence, DuringTiming::After), - Phase::LinkMarker => (Point::InitializeLinkMarker, DuringTiming::After), - Phase::SynchronizeRootAfterMarker => ( - Point::InitializeSynchronizeRootAfterMarker, - DuringTiming::Before, - ), - Phase::RemoveMarkerStage => (Point::InitializeRemoveMarkerStage, DuringTiming::After), - Phase::SynchronizeRootAfterCleanup => ( - Point::InitializeSynchronizeRootAfterCleanup, - DuringTiming::Before, - ), - Phase::DiscardIncompleteMarkerStage => ( - Point::InitializeDiscardIncompleteMarkerStage, - DuringTiming::After, - ), - _ => return None, - }) -} diff --git a/xtask/src/durability_crash_matrix/production_protocol/writer.rs b/xtask/src/durability_crash_matrix/production_protocol/writer.rs new file mode 100644 index 00000000..d2c70164 --- /dev/null +++ b/xtask/src/durability_crash_matrix/production_protocol/writer.rs @@ -0,0 +1,33 @@ +//! This module owns the crash children' initialized writer and read policies. + +use std::path::Path; + +use keep::{ + CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemCatalogPublisher, + FilesystemStoreAdmission, LayoutEntryLimit, SegmentReadPolicy, SegmentRecordLimit, +}; + +use super::{DurabilityCrashMatrixError, verification}; + +const RESTART_BYTE_LIMIT: u64 = 1_048_576; + +/// Initializes the single-format store and opens its writer, as production does. +pub(super) fn publisher( + store_root: &Path, +) -> Result { + let admission = FilesystemStoreAdmission::initialize(store_root) + .map_err(|source| verification("initialize crash catalog store", source))? + .into_admission(); + FilesystemCatalogPublisher::open(admission, restart_policy()?) + .map_err(|source| DurabilityCrashMatrixError::io("open crash catalog publisher", source)) +} + +pub(super) fn restart_policy() -> Result { + let byte_limit = CatalogRestartByteLimit::new(RESTART_BYTE_LIMIT) + .map_err(|source| verification("construct crash restart byte limit", source))?; + Ok(CatalogRestartPolicy::new(segment_policy(), byte_limit)) +} + +pub(in crate::durability_crash_matrix) const fn segment_policy() -> SegmentReadPolicy { + SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM) +} diff --git a/xtask/src/durability_crash_matrix/restart.rs b/xtask/src/durability_crash_matrix/restart.rs index 5964c4c7..7810d7c9 100644 --- a/xtask/src/durability_crash_matrix/restart.rs +++ b/xtask/src/durability_crash_matrix/restart.rs @@ -1,13 +1,13 @@ //! This module owns independent post-process-death store verification. mod expectation; +mod initialization; +mod retained_catalog; mod retention; mod retention_incomplete; mod retention_snapshot; mod semantic; -mod version_two_catalog; -mod version_two_initialization; -mod version_two_recovery; +mod stage_recovery; use std::collections::BTreeSet; use std::fs; @@ -25,11 +25,11 @@ pub(super) fn verify( if case.point().sequence() == xtask::DurabilityCrashSequence::Retention { return retention::verify(store_root, case); } - if case.point().sequence() == DurabilityCrashSequence::VersionTwoInitialization { - return version_two_initialization::verify(store_root, case); + if case.point().sequence() == DurabilityCrashSequence::Initialization { + return initialization::verify(store_root, case); } - if case.point().sequence() == DurabilityCrashSequence::VersionTwoCatalog { - return version_two_catalog::verify(store_root, case); + if case.point().sequence() == DurabilityCrashSequence::RetainedCatalog { + return retained_catalog::verify(store_root, case); } let expected = ExpectedStoreState::for_case(case)?; let observed_paths = inventory(store_root)?; diff --git a/xtask/src/durability_crash_matrix/restart/expectation.rs b/xtask/src/durability_crash_matrix/restart/expectation.rs index 6ab3a4b7..6dc746cd 100644 --- a/xtask/src/durability_crash_matrix/restart/expectation.rs +++ b/xtask/src/durability_crash_matrix/restart/expectation.rs @@ -106,8 +106,8 @@ impl ExpectedStoreState { DurabilityCrashSequence::Head => sequence::head(case), DurabilityCrashSequence::RecoveryDiscard => sequence::recovery(case), DurabilityCrashSequence::Retention - | DurabilityCrashSequence::VersionTwoCatalog - | DurabilityCrashSequence::VersionTwoInitialization => { + | DurabilityCrashSequence::RetainedCatalog + | DurabilityCrashSequence::Initialization => { Err(DurabilityCrashMatrixError::PointSequenceMismatch { point: case.point(), }) diff --git a/xtask/src/durability_crash_matrix/restart/version_two_initialization.rs b/xtask/src/durability_crash_matrix/restart/initialization.rs similarity index 88% rename from xtask/src/durability_crash_matrix/restart/version_two_initialization.rs rename to xtask/src/durability_crash_matrix/restart/initialization.rs index d29efaa6..81bad978 100644 --- a/xtask/src/durability_crash_matrix/restart/version_two_initialization.rs +++ b/xtask/src/durability_crash_matrix/restart/initialization.rs @@ -4,7 +4,7 @@ //! inventory and marker bytes that coordinate leaves and the exact admission //! refusal or success, then runs the public initializer again as recovery and //! requires the state it reports. It then reopens the store, publishes the -//! bundle as the first catalog through the version-two writer, retains it, and +//! bundle as the first catalog through the writer, retains it, and //! reads it back through the production durable reader. use std::collections::BTreeSet; @@ -15,10 +15,9 @@ use std::path::Path; use keep::{ BlobId, CanonicalStoreFormatMarker, DurableStore, FilesystemNamespaceRefusal, - FilesystemPlatformAdmissionError, FilesystemVersionTwoAdmission, ReaderAttemptLimit, - VersionTwoInitializationDirectory as Directory, VersionTwoInitializationError, - VersionTwoInitializationPrefix as Prefix, VersionTwoInitializationRefusal, - execute_retention_publication, + FilesystemStoreAdmission, FilesystemStoreAdmissionError, ReaderAttemptLimit, + StoreInitializationDirectory as Directory, StoreInitializationError, + StoreInitializationPrefix as Prefix, StoreInitializationRefusal, execute_retention_publication, }; use xtask::{ DurabilityCrashCase, DurabilityCrashOccurrence, DurabilityCrashPoint as Point, @@ -28,11 +27,11 @@ use xtask::{ use super::super::DurabilityCrashMatrixError; use super::super::production_protocol::crash_blob::CrashBlob; use super::super::production_protocol::fixture::GoldenFixture; +use super::super::production_protocol::initialization::INCOMPLETE_STAGE_LENGTH; use super::super::production_protocol::populated_store; +use super::super::production_protocol::retained_catalog::policy; use super::super::production_protocol::retention::{preparation, reopened_authority}; use super::super::production_protocol::verification; -use super::super::production_protocol::version_two_catalog::policy; -use super::super::production_protocol::version_two_initialization::INCOMPLETE_STAGE_LENGTH; use super::inventory; /// The protocol state one coordinate leaves behind. @@ -56,15 +55,14 @@ pub(super) fn verify( require_inventory(store_root, state)?; require_marker_bytes(store_root, state)?; require_reopen(store_root, state)?; - let resumed = match FilesystemVersionTwoAdmission::initialize_unchecked_for_repository_tasks( - store_root, - ) { - Ok(initialized) => Some(initialized.found()), - Err(VersionTwoInitializationError::Refused { - refusal: VersionTwoInitializationRefusal::AlreadyInitialized, - }) => None, - Err(source) => return Err(verification("recover interrupted initialization", source)), - }; + let resumed = + match FilesystemStoreAdmission::initialize_unchecked_for_repository_tasks(store_root) { + Ok(initialized) => Some(initialized.found()), + Err(StoreInitializationError::Refused { + refusal: StoreInitializationRefusal::AlreadyInitialized, + }) => None, + Err(source) => return Err(verification("recover interrupted initialization", source)), + }; let expected = resumes_from(state)?; if resumed != expected { return Err(DurabilityCrashMatrixError::InitializationPrefixMismatch { @@ -230,11 +228,11 @@ fn metadata(store_root: &Path, name: &str) -> Result Result<(), DurabilityCrashMatrixError> { - let reopened = FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store_root); + let reopened = FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(store_root); let expected = match state { State::Empty => { return match reopened { - Err(FilesystemPlatformAdmissionError::WriterLock { .. }) => Ok(()), + Err(FilesystemStoreAdmissionError::WriterLock { .. }) => Ok(()), Ok(_) => Err(mismatch("an empty root reopened")), Err(other) => Err(mismatch(format!("empty root reopened as {other:?}"))), }; @@ -248,7 +246,7 @@ fn require_reopen(store_root: &Path, state: State) -> Result<(), DurabilityCrash _ => FilesystemNamespaceRefusal::InitializationIncomplete, }; let observed = match &reopened { - Err(FilesystemPlatformAdmissionError::Namespace { source }) => source + Err(FilesystemStoreAdmissionError::Namespace { source }) => source .get_ref() .and_then(|cause| cause.downcast_ref::()) .copied(), @@ -266,9 +264,8 @@ fn require_reopen(store_root: &Path, state: State) -> Result<(), DurabilityCrash /// Publishes and retains the bundle, then reconstructs its exact bytes. fn publish_and_read(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let admission = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store_root) - .map_err(|source| verification("reopen recovered store", source))?; + let admission = FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("reopen recovered store", source))?; populated_store::publish_bundle_catalog(store_root, admission)?; populated_store::require_frozen_bundle(store_root)?; let mut authority = reopened_authority(store_root)?; diff --git a/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs b/xtask/src/durability_crash_matrix/restart/retained_catalog.rs similarity index 83% rename from xtask/src/durability_crash_matrix/restart/version_two_catalog.rs rename to xtask/src/durability_crash_matrix/restart/retained_catalog.rs index 1a089fb8..3e52e9ad 100644 --- a/xtask/src/durability_crash_matrix/restart/version_two_catalog.rs +++ b/xtask/src/durability_crash_matrix/restart/retained_catalog.rs @@ -1,7 +1,7 @@ -//! This module owns post-process-death verification of the version-two writer. +//! This module owns post-process-death verification of the writer. //! //! After the child dies at its coordinate, restart requires the exact stage -//! residue that coordinate leaves, recovers it with the public version-two +//! residue that coordinate leaves, recovers it with the public //! recovery tools, republishes when the candidate never reached `HEAD`, and //! then reads both the retained bundle blob and the ingested blob through //! the production durable reader. @@ -13,23 +13,22 @@ use std::path::Path; use keep::{ AdmittedRetentionManifest, AdmittedRetentionRoot, BlobId, CanonicalCatalog, CanonicalRetentionRoot, CatalogGeneration, CatalogPublicationExpectation, - CatalogPublicationOutcome, DurableStore, FilesystemCatalogSnapshot, - FilesystemVersionTwoAdmission, FilesystemVersionTwoCatalogPublisher, ReaderAttemptLimit, - RecoveryEntryRole, RegisteredRetentionProfile, RetentionClosureLimits, - RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, RetentionRoot, - RootGeneration, SegmentPublication, execute_retention_publication, + CatalogPublicationOutcome, DurableStore, FilesystemCatalogPublisher, FilesystemCatalogSnapshot, + FilesystemStoreAdmission, ReaderAttemptLimit, RecoveryEntryRole, RegisteredRetentionProfile, + RetentionClosureLimits, RetentionGenerationExpectation, RetentionNamespace, RetentionPolicy, + RetentionRoot, RootGeneration, SegmentPublication, execute_retention_publication, preflight_retention_transition, prepare_retention_publication, publish_retained_catalog_generation, }; use xtask::{DurabilityCrashCase, DurabilityCrashPoint, DurabilityCrashPosition}; use super::super::DurabilityCrashMatrixError; -use super::super::production_protocol::crash_blob::{CrashBlob, VERSION_TWO_CONTENT}; +use super::super::production_protocol::crash_blob::{CrashBlob, SUCCESSOR_CONTENT}; use super::super::production_protocol::fixture::GoldenFixture; +use super::super::production_protocol::retained_catalog::{admit_segments, policy}; use super::super::production_protocol::retention::reopened_authority; use super::super::production_protocol::verification; -use super::super::production_protocol::version_two_catalog::{admit_segments, policy}; -use super::version_two_recovery as recovery; +use super::stage_recovery as recovery; /// The catalog-protocol residue one coordinate leaves. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -74,12 +73,12 @@ pub(super) fn verify( fn residue(case: DurabilityCrashCase) -> Result { let before = case.position() == DurabilityCrashPosition::Before; match case.point() { - DurabilityCrashPoint::VersionTwoLinkSegment => Ok(Residue::SegmentStage), - DurabilityCrashPoint::VersionTwoCreateCatalogStage if before => Ok(Residue::Unpublished), - DurabilityCrashPoint::VersionTwoCreateCatalogStage => Ok(Residue::CatalogStage), - DurabilityCrashPoint::VersionTwoSynchronizeHead => Ok(Residue::HeadStage), - DurabilityCrashPoint::VersionTwoReplaceHead if before => Ok(Residue::HeadStage), - DurabilityCrashPoint::VersionTwoReplaceHead => Ok(Residue::Published), + DurabilityCrashPoint::RetainedLinkSegment => Ok(Residue::SegmentStage), + DurabilityCrashPoint::RetainedCreateCatalogStage if before => Ok(Residue::Unpublished), + DurabilityCrashPoint::RetainedCreateCatalogStage => Ok(Residue::CatalogStage), + DurabilityCrashPoint::RetainedSynchronizeHead => Ok(Residue::HeadStage), + DurabilityCrashPoint::RetainedReplaceHead if before => Ok(Residue::HeadStage), + DurabilityCrashPoint::RetainedReplaceHead => Ok(Residue::Published), point => Err(DurabilityCrashMatrixError::PointSequenceMismatch { point }), } } @@ -121,11 +120,10 @@ fn republish(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { let catalog = CanonicalCatalog::from_segments(generation, Some(current.catalog_digest()), &segments) .map_err(|source| verification("encode republished catalog", source))?; - let admission = - FilesystemVersionTwoAdmission::reopen_unchecked_for_repository_tasks(store_root) - .map_err(|source| verification("reopen recovered store for publication", source))?; - let mut publisher = FilesystemVersionTwoCatalogPublisher::open(admission, policy()?) - .map_err(|source| verification("open recovered version-two publisher", source))?; + let admission = FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("reopen recovered store for publication", source))?; + let mut publisher = FilesystemCatalogPublisher::open(admission, policy()?) + .map_err(|source| verification("open recovered publisher", source))?; let receipt = publish_retained_catalog_generation( &mut publisher, expectation, @@ -159,7 +157,7 @@ fn require_generation_two(store_root: &Path) -> Result<(), DurabilityCrashMatrix /// Retains the ingested blob in its own namespace, gated by the current catalog. fn retain_blob(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let blob = CrashBlob::version_two()?; + let blob = CrashBlob::successor()?; let root = RetentionRoot::new( RetentionNamespace::try_from(&b"version-two-crash"[..]) .map_err(|source| verification("name crash retention namespace", source))?, @@ -220,8 +218,8 @@ fn read_back(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { return Err(mismatch("recovered retained blob changed identity".into())); } } - let ingested = reconstruct(&store, CrashBlob::version_two()?.target())?; - if ingested == VERSION_TWO_CONTENT { + let ingested = reconstruct(&store, CrashBlob::successor()?.target())?; + if ingested == SUCCESSOR_CONTENT { Ok(()) } else { Err(mismatch( @@ -260,7 +258,7 @@ fn pool_segments(store_root: &Path) -> Result>, DurabilityCrashMatri fn mismatch(message: String) -> DurabilityCrashMatrixError { verification( - "verify version-two crash recovery", + "verify retained-catalog crash recovery", io::Error::other(message), ) } diff --git a/xtask/src/durability_crash_matrix/restart/semantic.rs b/xtask/src/durability_crash_matrix/restart/semantic.rs index 7903cfe4..31b3d931 100644 --- a/xtask/src/durability_crash_matrix/restart/semantic.rs +++ b/xtask/src/durability_crash_matrix/restart/semantic.rs @@ -6,10 +6,10 @@ use std::path::Path; use keep::{ AdmittedSegment, CatalogRestartByteLimit, CatalogRestartPolicy, ChecksummedCatalog, ChunkId, - FilesystemCatalogSnapshot, FilesystemVersionTwoRecoveryAdmission, LayoutEntryLimit, - RecoveryCatalogStage, RecoveryNextHeadStage, RecoverySegmentStage, SegmentReadPolicy, - SegmentRecordIdentity, SegmentRecordLimit, classify_recovery_catalog_stage, - classify_recovery_next_head_stage, classify_recovery_segment_stage, + FilesystemCatalogSnapshot, FilesystemRecoveryAdmission, LayoutEntryLimit, RecoveryCatalogStage, + RecoveryNextHeadStage, RecoverySegmentStage, SegmentReadPolicy, SegmentRecordIdentity, + SegmentRecordLimit, classify_recovery_catalog_stage, classify_recovery_next_head_stage, + classify_recovery_segment_stage, }; use super::expectation::{ @@ -40,9 +40,8 @@ pub(super) fn verify( /// `FORMAT` marker: every residue a catalog-protocol crash leaves is a state /// of the format that recovery can open. fn verify_recovery_admission(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let admission = - FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(store_root) - .map_err(|source| verification("admit the crash residue for recovery", source))?; + let admission = FilesystemRecoveryAdmission::reopen_unchecked_for_repository_tasks(store_root) + .map_err(|source| verification("admit the crash residue for recovery", source))?; drop(admission); Ok(()) } diff --git a/xtask/src/durability_crash_matrix/restart/version_two_recovery.rs b/xtask/src/durability_crash_matrix/restart/stage_recovery.rs similarity index 69% rename from xtask/src/durability_crash_matrix/restart/version_two_recovery.rs rename to xtask/src/durability_crash_matrix/restart/stage_recovery.rs index 2eae37d4..2275fd2d 100644 --- a/xtask/src/durability_crash_matrix/restart/version_two_recovery.rs +++ b/xtask/src/durability_crash_matrix/restart/stage_recovery.rs @@ -1,6 +1,6 @@ -//! This module owns version-two stage recovery after one writer process death. +//! This module owns stage recovery after one writer process death. //! -//! Each step reopens the store through the production version-two recovery +//! Each step reopens the store through the production stage recovery //! admission (without the platform probe), fingerprints the fixed stage under //! the pinned inventory, admits exactly those bytes, and executes one planned //! transition with the public recovery executors. @@ -11,35 +11,33 @@ use std::path::{Path, PathBuf}; use keep::{ CatalogPublicationExpectation, ChecksummedCatalog, ChecksummedPublicationHead, - FilesystemCatalogSnapshot, FilesystemRecoveryInventoryReader, + FilesystemCatalogSnapshot, FilesystemRecoveryAdmission, FilesystemRecoveryInventoryReader, FilesystemRecoveryNextHeadFinalizer, FilesystemRecoveryStageCompleter, - FilesystemRecoveryStageDiscarder, FilesystemVersionTwoRecoveryAdmission, RecoveryEntryRole, - RecoveryInventoryLimit, RecoveryStage, admit_recovery_stage_bytes, assess_recovery_stage, - classify_version_two_recovery_names, execute_recovery_next_head_finalization, - execute_recovery_stage_completion, execute_recovery_stage_discard, - plan_recovery_next_head_finalization, plan_recovery_stage_completion, - plan_recovery_stage_discard, + FilesystemRecoveryStageDiscarder, RecoveryEntryRole, RecoveryInventoryLimit, RecoveryStage, + admit_recovery_stage_bytes, assess_recovery_stage, classify_recovery_names, + execute_recovery_next_head_finalization, execute_recovery_stage_completion, + execute_recovery_stage_discard, plan_recovery_next_head_finalization, + plan_recovery_stage_completion, plan_recovery_stage_discard, }; use super::super::DurabilityCrashMatrixError; -use super::super::production_protocol::initialization::segment_policy; +use super::super::production_protocol::retained_catalog::{admit_segments, policy}; use super::super::production_protocol::verification; -use super::super::production_protocol::version_two_catalog::{admit_segments, policy}; +use super::super::production_protocol::writer::segment_policy; -/// Every fixed catalog-protocol stage a version-two inventory classifies. +/// Every fixed catalog-protocol stage a inventory classifies. pub(super) fn stage_roles( root: &Path, ) -> Result, DurabilityCrashMatrixError> { - let mut reader = - FilesystemRecoveryInventoryReader::open_version_two_unchecked_for_repository_tasks(root) - .map_err(|source| verification("open version-two recovery inventory", source))?; + let mut reader = FilesystemRecoveryInventoryReader::open_unchecked_for_repository_tasks(root) + .map_err(|source| verification("open recovery inventory", source))?; let limit = RecoveryInventoryLimit::new(4096) - .map_err(|source| verification("bound version-two recovery inventory", source))?; + .map_err(|source| verification("bound recovery inventory", source))?; let inventory = reader .read(limit) - .map_err(|source| verification("read version-two recovery inventory", source))?; - let manifest = classify_version_two_recovery_names(inventory) - .map_err(|source| verification("classify version-two recovery names", source))?; + .map_err(|source| verification("read recovery inventory", source))?; + let manifest = classify_recovery_names(inventory) + .map_err(|source| verification("classify stage recovery names", source))?; Ok(manifest .entries() .iter() @@ -65,8 +63,8 @@ pub(super) fn complete_segment(root: &Path) -> Result<(), DurabilityCrashMatrixE .map_err(|source| verification("assess recovered segment stage", source))?; let request = plan_recovery_stage_completion(&assessment) .map_err(|source| verification("plan recovered segment completion", source))?; - let mut completer = FilesystemRecoveryStageCompleter::open_version_two(admission(root)?) - .map_err(|source| verification("open version-two stage completer", source))?; + let mut completer = FilesystemRecoveryStageCompleter::open(admission(root)?) + .map_err(|source| verification("open stage completer", source))?; execute_recovery_stage_completion(&mut completer, request) .map(|_receipt| ()) .map_err(|source| verification("complete recovered segment stage", source)) @@ -82,8 +80,8 @@ pub(super) fn discard_catalog(root: &Path) -> Result<(), DurabilityCrashMatrixEr .map_err(|source| verification("assess recovered catalog stage", source))?; let request = plan_recovery_stage_discard(&assessment) .map_err(|source| verification("plan recovered catalog discard", source))?; - let mut discarder = FilesystemRecoveryStageDiscarder::open_version_two(admission(root)?) - .map_err(|source| verification("open version-two stage discarder", source))?; + let mut discarder = FilesystemRecoveryStageDiscarder::open(admission(root)?) + .map_err(|source| verification("open stage discarder", source))?; execute_recovery_stage_discard(&mut discarder, request) .map(|_receipt| ()) .map_err(|source| verification("discard recovered catalog stage", source)) @@ -123,34 +121,31 @@ pub(super) fn finalize_head( ); let request = plan_recovery_next_head_finalization(&assessment, &candidate, expectation) .map_err(|source| verification("plan recovered next-head finalization", source))?; - let mut finalizer = - FilesystemRecoveryNextHeadFinalizer::open_version_two(admission(root)?, policy()?) - .map_err(|source| verification("open version-two head finalizer", source))?; + let mut finalizer = FilesystemRecoveryNextHeadFinalizer::open(admission(root)?, policy()?) + .map_err(|source| verification("open head finalizer", source))?; execute_recovery_next_head_finalization(&mut finalizer, request) .map(|_receipt| ()) .map_err(|source| verification("finalize recovered next head", source)) } -fn admission( - root: &Path, -) -> Result { - FilesystemVersionTwoRecoveryAdmission::reopen_unchecked_for_repository_tasks(root) - .map_err(|source| verification("admit version-two stage recovery", source)) +fn admission(root: &Path) -> Result { + FilesystemRecoveryAdmission::reopen_unchecked_for_repository_tasks(root) + .map_err(|source| verification("admit stage recovery", source)) } fn evidence( root: &Path, stage: RecoveryStage, ) -> Result { - FilesystemRecoveryInventoryReader::open_version_two_unchecked_for_repository_tasks(root) - .map_err(|source| verification("open version-two stage inventory", source))? + FilesystemRecoveryInventoryReader::open_unchecked_for_repository_tasks(root) + .map_err(|source| verification("open stage inventory", source))? .fingerprint_stage(stage) - .map_err(|source| verification("fingerprint version-two recovery stage", source)) + .map_err(|source| verification("fingerprint stage recovery stage", source)) } fn stage_bytes(root: &Path, stage: RecoveryStage) -> Result, DurabilityCrashMatrixError> { fs::read(stage_path(root, stage)) - .map_err(|source| DurabilityCrashMatrixError::io("read version-two recovery stage", source)) + .map_err(|source| DurabilityCrashMatrixError::io("read stage recovery stage", source)) } fn stage_path(root: &Path, stage: RecoveryStage) -> PathBuf { diff --git a/xtask/src/durability_crash_point.rs b/xtask/src/durability_crash_point.rs index f6a8f35e..9259bee2 100644 --- a/xtask/src/durability_crash_point.rs +++ b/xtask/src/durability_crash_point.rs @@ -11,14 +11,14 @@ pub enum DurabilityCrashSequence { Head, /// Explicit discard of fingerprint-bound recovery evidence. RecoveryDiscard, - /// Version-two retention publication, `KEEP-CRASH-036` through `052`. + /// Retention publication, `KEEP-CRASH-036` through `052`. Retention, - /// Direct version-two initialization and its recovery, `KEEP-CRASH-081` + /// Direct initialization and its recovery, `KEEP-CRASH-081` /// through `091`. - VersionTwoInitialization, - /// Version-two catalog publication and its stage recovery, `KEEP-CRASH-074` + Initialization, + /// Catalog publication and its stage recovery, `KEEP-CRASH-074` /// through `077`. - VersionTwoCatalog, + RetainedCatalog, } impl DurabilityCrashSequence { @@ -29,8 +29,8 @@ impl DurabilityCrashSequence { Self::Head, Self::RecoveryDiscard, Self::Retention, - Self::VersionTwoCatalog, - Self::VersionTwoInitialization, + Self::RetainedCatalog, + Self::Initialization, ]; /// Returns the stable identifier used by the crash-matrix command line. @@ -42,8 +42,8 @@ impl DurabilityCrashSequence { Self::Head => "head", Self::RecoveryDiscard => "recovery-discard", Self::Retention => "retention", - Self::VersionTwoCatalog => "version-two-catalog", - Self::VersionTwoInitialization => "version-two-initialization", + Self::RetainedCatalog => "retained-catalog", + Self::Initialization => "initialization", } } @@ -153,14 +153,14 @@ pub enum DurabilityCrashPoint { RemoveManifestStage, /// Retention cleanup synchronization. SynchronizeRetentionCleanup, - /// Link the version-two writer's sealed segment into the immutable pool. - VersionTwoLinkSegment, - /// Exclusively create the version-two writer's `staging/current.cat`. - VersionTwoCreateCatalogStage, - /// Synchronize the version-two writer's complete `head.next`. - VersionTwoSynchronizeHead, - /// Replace `HEAD` after the version-two retained-closure revalidation. - VersionTwoReplaceHead, + /// Link the writer's sealed segment into the immutable pool. + RetainedLinkSegment, + /// Exclusively create the writer's `staging/current.cat`. + RetainedCreateCatalogStage, + /// Synchronize the writer's complete `head.next`. + RetainedSynchronizeHead, + /// Replace `HEAD` after the retained-closure revalidation. + RetainedReplaceHead, /// Create or open `writer.lock`, lock it, and synchronize it and the root. InitializeOpenAndLockWriterFile, /// Exclusively create `FORMAT.next` and write the canonical marker; the @@ -238,10 +238,10 @@ impl DurabilityCrashPoint { Self::RemoveRootStage, Self::RemoveManifestStage, Self::SynchronizeRetentionCleanup, - Self::VersionTwoLinkSegment, - Self::VersionTwoCreateCatalogStage, - Self::VersionTwoSynchronizeHead, - Self::VersionTwoReplaceHead, + Self::RetainedLinkSegment, + Self::RetainedCreateCatalogStage, + Self::RetainedSynchronizeHead, + Self::RetainedReplaceHead, Self::InitializeOpenAndLockWriterFile, Self::InitializeWriteMarkerStage, Self::InitializeSynchronizeMarkerStage, @@ -329,12 +329,12 @@ impl DurabilityCrashPoint { | Self::InitializeRemoveMarkerStage | Self::InitializeSynchronizeRootAfterCleanup | Self::InitializeDiscardIncompleteMarkerStage => { - DurabilityCrashSequence::VersionTwoInitialization + DurabilityCrashSequence::Initialization } - Self::VersionTwoLinkSegment - | Self::VersionTwoCreateCatalogStage - | Self::VersionTwoSynchronizeHead - | Self::VersionTwoReplaceHead => DurabilityCrashSequence::VersionTwoCatalog, + Self::RetainedLinkSegment + | Self::RetainedCreateCatalogStage + | Self::RetainedSynchronizeHead + | Self::RetainedReplaceHead => DurabilityCrashSequence::RetainedCatalog, } } diff --git a/xtask/src/durability_crash_point_identity.rs b/xtask/src/durability_crash_point_identity.rs index 54693b15..49e31563 100644 --- a/xtask/src/durability_crash_point_identity.rs +++ b/xtask/src/durability_crash_point_identity.rs @@ -54,10 +54,10 @@ impl DurabilityCrashPoint { Self::RemoveRootStage => "KEEP-CRASH-050", Self::RemoveManifestStage => "KEEP-CRASH-051", Self::SynchronizeRetentionCleanup => "KEEP-CRASH-052", - Self::VersionTwoLinkSegment => "KEEP-CRASH-074", - Self::VersionTwoCreateCatalogStage => "KEEP-CRASH-075", - Self::VersionTwoSynchronizeHead => "KEEP-CRASH-076", - Self::VersionTwoReplaceHead => "KEEP-CRASH-077", + Self::RetainedLinkSegment => "KEEP-CRASH-074", + Self::RetainedCreateCatalogStage => "KEEP-CRASH-075", + Self::RetainedSynchronizeHead => "KEEP-CRASH-076", + Self::RetainedReplaceHead => "KEEP-CRASH-077", Self::InitializeOpenAndLockWriterFile => "KEEP-CRASH-081", Self::InitializeWriteMarkerStage => "KEEP-CRASH-082", Self::InitializeSynchronizeMarkerStage => "KEEP-CRASH-083", diff --git a/xtask/tests/durability_crash_case_contract.rs b/xtask/tests/durability_crash_case_contract.rs index c1fd036b..84701313 100644 --- a/xtask/tests/durability_crash_case_contract.rs +++ b/xtask/tests/durability_crash_case_contract.rs @@ -105,8 +105,7 @@ fn directory_occurrences_are_admitted_only_within_the_protocol_fixed_range() #[test] fn the_initialization_sequence_runs_its_documented_forty_one_cases() { let cases: Vec<_> = - DurabilityCrashCase::in_sequence(DurabilityCrashSequence::VersionTwoInitialization) - .collect(); + DurabilityCrashCase::in_sequence(DurabilityCrashSequence::Initialization).collect(); assert_eq!(cases.len(), 41); let directory_cases = cases .iter() diff --git a/xtask/tests/durability_crash_documentation.rs b/xtask/tests/durability_crash_documentation.rs index 90e25f49..e813d213 100644 --- a/xtask/tests/durability_crash_documentation.rs +++ b/xtask/tests/durability_crash_documentation.rs @@ -23,7 +23,7 @@ fn living_documentation_routes_the_complete_crash_matrix_and_its_limits() { ), ( INITIALIZATION, - "cargo xtask durability-crash-matrix --sequence version-two-initialization", + "cargo xtask durability-crash-matrix --sequence initialization", ), (V2_REQUIREMENTS, "`KEEP-INITIALIZATION-004`"), ] { diff --git a/xtask/tests/durability_crash_point_contract.rs b/xtask/tests/durability_crash_point_contract.rs index 4c0ac3ce..62c720b3 100644 --- a/xtask/tests/durability_crash_point_contract.rs +++ b/xtask/tests/durability_crash_point_contract.rs @@ -5,7 +5,7 @@ use xtask::{DurabilityCrashPoint, DurabilityCrashSequence}; use DurabilityCrashSequence::{ - Catalog, Head, RecoveryDiscard, Retention, Segment, VersionTwoCatalog, VersionTwoInitialization, + Catalog, Head, Initialization, RecoveryDiscard, RetainedCatalog, Retention, Segment, }; const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ @@ -221,79 +221,79 @@ const EXPECTED: &[(DurabilityCrashPoint, &str, DurabilityCrashSequence)] = &[ Retention, ), ( - DurabilityCrashPoint::VersionTwoLinkSegment, + DurabilityCrashPoint::RetainedLinkSegment, "KEEP-CRASH-074", - VersionTwoCatalog, + RetainedCatalog, ), ( - DurabilityCrashPoint::VersionTwoCreateCatalogStage, + DurabilityCrashPoint::RetainedCreateCatalogStage, "KEEP-CRASH-075", - VersionTwoCatalog, + RetainedCatalog, ), ( - DurabilityCrashPoint::VersionTwoSynchronizeHead, + DurabilityCrashPoint::RetainedSynchronizeHead, "KEEP-CRASH-076", - VersionTwoCatalog, + RetainedCatalog, ), ( - DurabilityCrashPoint::VersionTwoReplaceHead, + DurabilityCrashPoint::RetainedReplaceHead, "KEEP-CRASH-077", - VersionTwoCatalog, + RetainedCatalog, ), ( DurabilityCrashPoint::InitializeOpenAndLockWriterFile, "KEEP-CRASH-081", - VersionTwoInitialization, + Initialization, ), ( DurabilityCrashPoint::InitializeWriteMarkerStage, "KEEP-CRASH-082", - VersionTwoInitialization, + Initialization, ), ( DurabilityCrashPoint::InitializeSynchronizeMarkerStage, "KEEP-CRASH-083", - VersionTwoInitialization, + Initialization, ), ( DurabilityCrashPoint::InitializeSynchronizeRootAfterMarkerStage, "KEEP-CRASH-084", - VersionTwoInitialization, + Initialization, ), ( DurabilityCrashPoint::InitializeAdmitReaderFence, "KEEP-CRASH-085", - VersionTwoInitialization, + Initialization, ), ( DurabilityCrashPoint::InitializeAdmitDirectory, "KEEP-CRASH-086", - VersionTwoInitialization, + Initialization, ), ( DurabilityCrashPoint::InitializeLinkMarker, "KEEP-CRASH-087", - VersionTwoInitialization, + Initialization, ), ( DurabilityCrashPoint::InitializeSynchronizeRootAfterMarker, "KEEP-CRASH-088", - VersionTwoInitialization, + Initialization, ), ( DurabilityCrashPoint::InitializeRemoveMarkerStage, "KEEP-CRASH-089", - VersionTwoInitialization, + Initialization, ), ( DurabilityCrashPoint::InitializeSynchronizeRootAfterCleanup, "KEEP-CRASH-090", - VersionTwoInitialization, + Initialization, ), ( DurabilityCrashPoint::InitializeDiscardIncompleteMarkerStage, "KEEP-CRASH-091", - VersionTwoInitialization, + Initialization, ), ]; @@ -338,7 +338,7 @@ fn initialization_directory_admission_runs_one_during_case_per_directory() { } assert_eq!( DurabilityCrashPoint::INITIALIZATION_DIRECTORIES, - u32::try_from(keep::VersionTwoInitializationDirectory::ALL.len()).unwrap_or(u32::MAX) + u32::try_from(keep::StoreInitializationDirectory::ALL.len()).unwrap_or(u32::MAX) ); } @@ -358,7 +358,7 @@ fn retired_crash_identifiers_are_never_reassigned() { fn initialization_boundaries_take_consecutive_new_identifiers() { let initialization: Vec<_> = DurabilityCrashPoint::ALL .into_iter() - .filter(|point| point.sequence() == VersionTwoInitialization) + .filter(|point| point.sequence() == Initialization) .map(DurabilityCrashPoint::identifier) .collect(); let expected: Vec<_> = (81..=91) @@ -378,8 +378,8 @@ fn crash_matrix_cli_admits_its_stable_sequence_names() { ("head", Head), ("recovery-discard", RecoveryDiscard), ("retention", Retention), - ("version-two-catalog", VersionTwoCatalog), - ("version-two-initialization", VersionTwoInitialization), + ("retained-catalog", RetainedCatalog), + ("initialization", Initialization), ] { assert_eq!( DurabilityCrashSequence::from_identifier(name), diff --git a/xtask/tests/durability_crash_production_contract.rs b/xtask/tests/durability_crash_production_contract.rs index 15fe61e7..b1d72226 100644 --- a/xtask/tests/durability_crash_production_contract.rs +++ b/xtask/tests/durability_crash_production_contract.rs @@ -14,8 +14,8 @@ fn crash_children_execute_every_claimed_production_protocol() -> Result<(), Box< "durability_crash_matrix/production_protocol.rs", "durability_crash_matrix/production_protocol/publication.rs", "durability_crash_matrix/production_protocol/recovery.rs", - "durability_crash_matrix/production_protocol/version_two_initialization.rs", - "durability_crash_matrix/production_protocol/version_two_catalog.rs", + "durability_crash_matrix/production_protocol/initialization.rs", + "durability_crash_matrix/production_protocol/retained_catalog.rs", ] .into_iter() .map(|path| fs::read_to_string(source_root.join(path))) diff --git a/xtask/tests/retention_store_v2_protocol_contract/transition_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/transition_laws.rs index 89a3b718..65235eec 100644 --- a/xtask/tests/retention_store_v2_protocol_contract/transition_laws.rs +++ b/xtask/tests/retention_store_v2_protocol_contract/transition_laws.rs @@ -38,7 +38,7 @@ enum LedgerRefusal { fn initialization_transition_coordinates_are_complete_and_ordered() { let points = DurabilityCrashPoint::ALL .into_iter() - .filter(|point| point.sequence() == DurabilityCrashSequence::VersionTwoInitialization) + .filter(|point| point.sequence() == DurabilityCrashSequence::Initialization) .count(); assert_eq!(OPERATIONS.len(), points); assert_eq!(admit(TRANSITIONS), Ok(())); @@ -119,7 +119,7 @@ fn admit_row(offset: usize, row: &str) -> Result<(), LedgerRefusal> { .ok_or(LedgerRefusal::Coordinate { row: offset })?; if id != format!("KEEP-CRASH-{ordinal:03}") || DurabilityCrashPoint::from_identifier(id).map(DurabilityCrashPoint::sequence) - != Some(DurabilityCrashSequence::VersionTwoInitialization) + != Some(DurabilityCrashSequence::Initialization) || phase != "initialization" || Some(&operation) != OPERATIONS.get(offset) { From fc4ac15b11a802f16de8d0614f92e95fbe4ac162 Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 10:50:12 -0700 Subject: [PATCH 21/36] Docs: consolidate the store format pages under docs/formats/store (#198) ADR-0010 asks for one normative set of pages for the one store format. docs/formats/segment-store-v1 and segment-store-v2 become docs/formats/store: - segment.md, catalog.md, and publication.md move unchanged in grammar; their intros name the store format and the record-grammar version their own magic fields carry; - the version-1 recovery page becomes stage-recovery.md, its rationale catalog-protocol-rationale.md, and its ledger catalog-protocol-requirements.md, beside the store's own recovery.md, rationale.md, and requirements.md (merging the ledgers would exceed the 300-line page threshold); - one README replaces the two indexes and keeps both core-law sections and every route; the format registry has one row and states that keep.segment-store/v1 is retired as a store format. Every heading other pages link to is kept, including recovery.md#root-identity-across-restart. Links across the repository, including historical evidence pages, point to the new paths; the retirement ledger keeps the old paths it records as history. The conformance corpora keep their v1/v2 directory names, which are the record-grammar versions in the artifacts' own bytes, so definition.tsv and the definition digest are unchanged. Prose that used "version 2" to mean "the store beside version 1" now names the store. The compatibility section follows ADR-0010 (no in-place conversion; an upgrade tool after the first release) and states that definition.tsv does not yet bind the segment, catalog, and head grammars. KEEP-CRASH-078, reserved for discard under version-2 recovery admission and never implemented, is retired: KEEP-CRASH-027..030 now run under recovery admission. The changelog records the unit. --- CHANGELOG.md | 2 + README.md | 10 +- conformance/segment-store/v1/README.md | 15 +- conformance/segment-store/v2/ORIGIN.md | 2 +- conformance/segment-store/v2/README.md | 2 +- .../0005-durable-segment-store-protocol.md | 2 +- docs/adr/retention-reader-root-identity.md | 2 +- docs/formats/README.md | 7 +- docs/formats/segment-store-v1/README.md | 55 ------ docs/formats/segment-store-v2/README.md | 122 ------------ docs/formats/store/README.md | 187 ++++++++++++++++++ .../catalog-protocol-rationale.md} | 9 +- .../catalog-protocol-requirements.md} | 37 ++-- .../catalog-publication.md | 52 ++--- .../{segment-store-v1 => store}/catalog.md | 5 +- .../closure-corruption.md | 0 .../{segment-store-v2 => store}/closure.md | 0 .../formats/{segment-store-v2 => store}/gc.md | 0 .../initialization.md | 2 +- .../publication.md | 23 ++- .../{segment-store-v2 => store}/rationale.md | 0 .../{segment-store-v2 => store}/recovery.md | 15 +- .../requirements.md | 17 +- .../retention-publication.md | 4 +- .../retention-recovery.md | 0 .../{segment-store-v2 => store}/retention.md | 0 .../retired-identifiers.md | 13 +- .../{segment-store-v1 => store}/segment.md | 4 +- .../recovery.md => store/stage-recovery.md} | 8 +- .../current-durable-surfaces.md | 8 +- .../retention-reader-root-identity.md | 2 +- .../retention-stage-prefix-evidence.md | 2 +- src/adapters/retained_catalog_publication.rs | 2 +- src/adapters/store_initialization.rs | 2 +- .../initialization_execution.rs | 2 +- .../initialization_prefix.rs | 2 +- xtask/tests/durability_crash_documentation.rs | 9 +- xtask/tests/layout_format_contract.rs | 11 +- .../retention_store_v2_protocol_contract.rs | 4 +- .../parser_fuzz_laws.rs | 2 +- ...ment_store_implementation_documentation.rs | 11 +- .../tests/segment_store_protocol_contract.rs | 30 +-- .../documentation_laws.rs | 4 +- .../parser_fuzz_laws.rs | 3 +- .../publication_laws.rs | 2 +- 45 files changed, 371 insertions(+), 320 deletions(-) delete mode 100644 docs/formats/segment-store-v1/README.md delete mode 100644 docs/formats/segment-store-v2/README.md create mode 100644 docs/formats/store/README.md rename docs/formats/{segment-store-v1/rationale.md => store/catalog-protocol-rationale.md} (97%) rename docs/formats/{segment-store-v1/requirements.md => store/catalog-protocol-requirements.md} (95%) rename docs/formats/{segment-store-v2 => store}/catalog-publication.md (82%) rename docs/formats/{segment-store-v1 => store}/catalog.md (96%) rename docs/formats/{segment-store-v2 => store}/closure-corruption.md (100%) rename docs/formats/{segment-store-v2 => store}/closure.md (100%) rename docs/formats/{segment-store-v2 => store}/gc.md (100%) rename docs/formats/{segment-store-v2 => store}/initialization.md (99%) rename docs/formats/{segment-store-v1 => store}/publication.md (94%) rename docs/formats/{segment-store-v2 => store}/rationale.md (100%) rename docs/formats/{segment-store-v2 => store}/recovery.md (93%) rename docs/formats/{segment-store-v2 => store}/requirements.md (86%) rename docs/formats/{segment-store-v2 => store}/retention-publication.md (86%) rename docs/formats/{segment-store-v2 => store}/retention-recovery.md (100%) rename docs/formats/{segment-store-v2 => store}/retention.md (100%) rename docs/formats/{segment-store-v2 => store}/retired-identifiers.md (96%) rename docs/formats/{segment-store-v1 => store}/segment.md (96%) rename docs/formats/{segment-store-v1/recovery.md => store/stage-recovery.md} (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ef27b17..04f6aeb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ after its public API and format compatibility policies are established. ## [Unreleased] +- Version 1 is deleted: every open path refuses a root without the `FORMAT` marker, `KEEP-CRASH-001`–`030` run on initialized stores, `KEEP-CRASH-031`–`035` are retired, the `VersionTwo*` types take format-neutral names (`FilesystemStoreAdmission`, `FilesystemCatalogPublisher`, `StoreInitialization`, `classify_recovery_names`), and the format pages are one set under `docs/formats/store/` (#198). + - Keep has one storage format: `FilesystemVersionTwoAdmission::initialize` creates a store directly through a recoverable protocol with one durable commit point and a 41-case crash matrix (`KEEP-CRASH-081`–`091`), migration and its public API, records, golden fixtures, and crash rows `KEEP-CRASH-053`–`073` are deleted, the persisted root device-and-inode binding is removed so quiescent copies and backups open, and the format-definition digest changes to `f1a0ce10…361d` (#198). - Version-2 stores can ingest: `FilesystemVersionTwoCatalogPublisher` stages one segment and publishes a catalog successor through the retained-closure gate, version-2 catalog-stage recovery classifies version-2 roots and finalizes a staged head only after the same gate admits it, and `KEEP-CRASH-074` through `077` cover writer process death (#198). diff --git a/README.md b/README.md index 14cf2631..60969281 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Keep is required to refuse all three, before mutating anything. (`KEEP-CRASH-081`–`091`) covers every boundary. Nothing persisted binds a store to its directory: a quiescent copy or restored backup opens wherever it is placed. There is no migration; see the - [retirement ledger](docs/formats/segment-store-v2/retired-identifiers.md). + [retirement ledger](docs/formats/store/retired-identifiers.md). - **Version-2 retention, forward path.** Explicit retention roots, deterministic closure verification, the gated segment and catalog writer, and a 17-phase retention publication, all with production filesystem @@ -78,7 +78,7 @@ Version-2 complete-stage retention recovery, fenced retention snapshots and mode The retention process-death sequence checks the recovered head generation and exact selected-root bytes before retry; its [evidence receipt](docs/testing-evidence/retention-crash-reader-oracle.md) bounds that claim to the declared initial-publication crash coordinates. -Recovery failures retain their typed cause and report known effects separately from uncertain effects or durability under the [approved recovery contract](docs/formats/segment-store-v2/retention-recovery.md). +Recovery failures retain their typed cause and report known effects separately from uncertain effects or durability under the [approved recovery contract](docs/formats/store/retention-recovery.md). Writer authority coordinates cooperating writers in a managed namespace; it does not isolate arbitrary concurrent out-of-band filesystem mutation. @@ -278,8 +278,8 @@ has one job; this one is the front door. | Understand what is proved and what is not | [`docs/invariants/`](docs/invariants/) | | Read the byte-level formats | [`docs/formats/`](docs/formats/) | | See the architecture and port boundaries | [`docs/architecture/`](docs/architecture/) | -| Follow the crash and recovery rules | [`segment-store-v1/recovery.md`](docs/formats/segment-store-v1/recovery.md) · [`segment-store-v2/recovery.md`](docs/formats/segment-store-v2/recovery.md) · [`segment-store-v2/initialization.md`](docs/formats/segment-store-v2/initialization.md) | -| See how a retention generation is published | [`segment-store-v2/retention-publication.md`](docs/formats/segment-store-v2/retention-publication.md) | +| Follow the crash and recovery rules | [`store/stage-recovery.md`](docs/formats/store/stage-recovery.md) · [`store/recovery.md`](docs/formats/store/recovery.md) · [`store/initialization.md`](docs/formats/store/initialization.md) | +| See how a retention generation is published | [`store/retention-publication.md`](docs/formats/store/retention-publication.md) | | Check reproducible performance evidence | [`docs/benchmarks/`](docs/benchmarks/) | | Run the language-neutral corpora | [`conformance/`](conformance/) | | See what changed | [`CHANGELOG.md`](CHANGELOG.md) | @@ -296,4 +296,4 @@ plaintext content, keys, or sensitive paths in a public issue. Licensed under the [Apache License 2.0](LICENSE). [reconstruction]: docs/invariants/authenticated-reconstruction/README.md -[ledger]: docs/formats/segment-store-v2/requirements.md +[ledger]: docs/formats/store/requirements.md diff --git a/conformance/segment-store/v1/README.md b/conformance/segment-store/v1/README.md index e5bc7a39..680a3062 100644 --- a/conformance/segment-store/v1/README.md +++ b/conformance/segment-store/v1/README.md @@ -1,12 +1,15 @@ # Durable Segment Store Corpus v1 -This directory is the implementation-independent corpus for -`keep.segment-store/v1`. It freezes canonical segment, catalog-generation, -and publication-head bytes plus the stable durable-transition ledger -independently of any production implementation. +This directory is the implementation-independent corpus for the segment, +catalog-generation, and publication-head grammars, first specified as +`keep.segment-store/v1` and carried unchanged by the one Keep store format. It +freezes their canonical bytes plus the stable catalog-protocol transition +ledger independently of any production implementation. The directory keeps its +`v1` name because every artifact here carries grammar version 1 in its own +magic and version fields. The normative protocol is the -[Durable Segment Store v1 specification](../../../docs/formats/segment-store-v1/README.md). +[Keep store format](../../../docs/formats/store/README.md). ADR-0005 records the cross-cutting decision. Issue [#15](https://github.com/flyingrobots/keep/issues/15) owns production segment writing and reading, issue @@ -100,7 +103,7 @@ initializer, which was deleted when the single store format became the only format. A store is now created by direct initialization, whose rows are `KEEP-CRASH-081`–`KEEP-CRASH-091` in the [version-2 corpus](../v2/transitions.tsv). The -[retirement ledger](../../../docs/formats/segment-store-v2/retired-identifiers.md) +[retirement ledger](../../../docs/formats/store/retired-identifiers.md) records their last evidence. No row reuses a retired identifier. The `interrupted_class` column names the weakest state that interruption may diff --git a/conformance/segment-store/v2/ORIGIN.md b/conformance/segment-store/v2/ORIGIN.md index 92490cc6..20c74cca 100644 --- a/conformance/segment-store/v2/ORIGIN.md +++ b/conformance/segment-store/v2/ORIGIN.md @@ -9,7 +9,7 @@ The corpus was constructed on 2026-07-29 with: `transitions.tsv` was first added on 2026-09-30 for the 21 migration boundaries. On 2026-10-27 its migration rows were removed and its 11 rows were transcribed from the initialization crash points `KEEP-CRASH-081` to `-091` -and the state table in `docs/formats/segment-store-v2/initialization.md`. No +and the state table in `docs/formats/store/initialization.md`. No independent oracle constructs this ledger. Its committed-byte guard is `transition_laws.rs` under `xtask/tests/retention_store_v2_protocol_contract/`. The crash matrix checks runtime behavior at the same boundaries without reading the TSV. diff --git a/conformance/segment-store/v2/README.md b/conformance/segment-store/v2/README.md index 42d415ca..192592b0 100644 --- a/conformance/segment-store/v2/README.md +++ b/conformance/segment-store/v2/README.md @@ -28,7 +28,7 @@ In `artifacts.tsv`, `bound_digest_hex` is the marker content digest for The migration records, their inventory and source tables, and the `KEEP-CRASH-053` to `-073` transition rows were removed with migration; the -[retirement ledger](../../../docs/formats/segment-store-v2/retired-identifiers.md) +[retirement ledger](../../../docs/formats/store/retired-identifiers.md) records their last evidence. ## Frozen identities diff --git a/docs/adr/0005-durable-segment-store-protocol.md b/docs/adr/0005-durable-segment-store-protocol.md index d641c9ce..00ad4386 100644 --- a/docs/adr/0005-durable-segment-store-protocol.md +++ b/docs/adr/0005-durable-segment-store-protocol.md @@ -35,7 +35,7 @@ segment writer, catalog publisher, and recovery engine can be implemented. ## Decision Keep adopts `keep.segment-store/v1`, specified in the -[version-1 format and state protocol](../formats/segment-store-v1/README.md). +[version-1 format and state protocol](../formats/store/README.md). The decision is one inseparable contract with three parts: 1. canonical immutable segment, catalog-generation, and publication-head diff --git a/docs/adr/retention-reader-root-identity.md b/docs/adr/retention-reader-root-identity.md index fc0a6ba0..76d26c16 100644 --- a/docs/adr/retention-reader-root-identity.md +++ b/docs/adr/retention-reader-root-identity.md @@ -1,6 +1,6 @@ # Retention reader root identity -Status: Superseded by [ADR-0010](https://github.com/flyingrobots/keep/pull/199) (#198). The persisted binding this record relied on lived in `migration.intent`, which was deleted with migration; readers now admit the committed namespace and the `FORMAT` marker, and a store is portable. The live checks that remain are listed under [Portable stores](../formats/segment-store-v2/recovery.md#portable-stores). The text below is kept as the record of the retired decision. +Status: Superseded by [ADR-0010](https://github.com/flyingrobots/keep/pull/199) (#198). The persisted binding this record relied on lived in `migration.intent`, which was deleted with migration; readers now admit the committed namespace and the `FORMAT` marker, and a store is portable. The live checks that remain are listed under [Portable stores](../formats/store/recovery.md#portable-stores). The text below is kept as the record of the retired decision. Joint admission of `FORMAT`, `migration.intent`, and `migration.receipt` establishes that those records agree with each other. It does not establish that the directory receiving the read request is the directory the migration intent names. Reader admission must compare the opened root's physical identity before accepting the version-two read capability. diff --git a/docs/formats/README.md b/docs/formats/README.md index 3ccbaaf3..11075ffb 100644 --- a/docs/formats/README.md +++ b/docs/formats/README.md @@ -8,8 +8,11 @@ admitted merely because one Rust type can serialize and deserialize it. | Format | Coordinate | Status | Evidence | | --- | --- | --- | --- | | [Flat Chunk Layout v1](flat-chunk-layout-v1/README.md) | `keep.flat-chunks/v1` | Implemented through verified reconstruction in issues #10 and #13 | [Golden corpus](../../conformance/layout/v1/README.md) | -| [Durable Segment Store v1](segment-store-v1/README.md) | `keep.segment-store/v1` | Implemented through initialization, publication, restart, and recovery in issues #14–#17 | [Golden corpus](../../conformance/segment-store/v1/README.md) | -| [Durable Segment Store v2](segment-store-v2/README.md) | `keep.segment-store/v2` | The one store format: direct initialization, reopen, the gated catalog writer and its recovery, and retention publication and recovery implemented; collection planned in issue #21 | [Golden corpus](../../conformance/segment-store/v2/README.md) | +| [Keep store format](store/README.md) | `keep.segment-store/v2` | The one store format: direct initialization and reopen, the segment, catalog, publication, and stage-recovery protocol of issues #14–#17, the gated catalog writer, and retention publication and recovery implemented; collection planned in issue #21 | [Segment, catalog, and head corpus](../../conformance/segment-store/v1/README.md) · [Store corpus](../../conformance/segment-store/v2/README.md) | + +`keep.segment-store/v1` is retired as a store format (#198): no code admits a +version-1 root. Its segment, catalog, and publication-head grammars continue +unchanged inside the store format. The registry records protocol specifications, including formats whose implementation is still planned. Each format page states its exact proof diff --git a/docs/formats/segment-store-v1/README.md b/docs/formats/segment-store-v1/README.md deleted file mode 100644 index 0194600e..00000000 --- a/docs/formats/segment-store-v1/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Durable Segment Store Version 1 - -`keep.segment-store/v1` is the accepted physical protocol for Keep's first -durable segment store. It specifies bytes, publication, crash states, reader -visibility, and recovery as one contract. - -ADR-0005 records the cross-cutting decision. These pages are a protocol -commitment. Segment writing and verified reading are implemented in issue #15. -Catalog generation, writer-locked publication, and immutable restart snapshots -are implemented in issue #16. Store initialization, platform admission, -explicit recovery, and the 105-case process-death crash matrix are implemented -in issue #17. Every page below describes the behaviour of `main`; the -[requirements ledger](requirements.md) names the test behind each claim. - -## Core law - -For a given logical content identity, Keep returns exactly the bytes named by -that identity or refuses. - -Durable version 1 adds these physical laws: - -- file existence proves no identity, completeness, publication, retention, or - durability claim; -- only one verified publication head selects one verified immutable catalog; -- only that catalog maps logical identities to immutable physical records; -- a segment is immutable after its complete seal is synchronized; -- a generation is published only after all referenced immutable artifacts are - synchronized and the new head replacement is directory-synchronized; and -- uncertain, conflicting, malformed, unsupported, or corrupt state is - represented and refused, never guessed or silently repaired. - -Physical locations and digests are not `BlobId`, `ChunkId`, or `LayoutId`. -Rechunking, migration, and later compaction may move physical evidence without -moving logical identity. - -## Normative pages - -The following pages form one versioned protocol: - -- [Segment bytes](segment.md) owns canonical primitives, record framing, and - immutable segment bytes. -- [Catalog and publication-head bytes](catalog.md) owns catalog generations, - publication-head framing, and fixed bounds. -- [Publication and reader visibility](publication.md) owns the physical - namespace, forward protocol, writer exclusion, and reader snapshots. -- [Recovery and platform contract](recovery.md) owns recovery classification, - refusal precedence, and supported filesystem semantics. -- [Requirements, compatibility, and evidence](requirements.md) owns the - requirement ledger, compatibility boundary, security nonclaims, and golden - evidence. -- The [colocated rationale](rationale.md) records format-local tradeoffs. - -No page is independently optional. A version-1 implementation conforms only -when it satisfies the complete linked protocol and the -[durable transition ledger](../../../conformance/segment-store/v1/transitions.tsv). diff --git a/docs/formats/segment-store-v2/README.md b/docs/formats/segment-store-v2/README.md deleted file mode 100644 index e999d66f..00000000 --- a/docs/formats/segment-store-v2/README.md +++ /dev/null @@ -1,122 +0,0 @@ -# Durable Segment Store Version 2 - -`keep.segment-store/v2` is the successor to `keep.segment-store/v1` and the -one store format Keep creates. It keeps the version-1 segment, catalog, and -publication-head grammars byte for byte and adds the `FORMAT` marker, the -reader fence, explicit retention state, and reserved GC and -recovery-disposition namespaces. - -ADR-0009 owns the cross-cutting retention and liveness decision. -[ADR-0010](https://github.com/flyingrobots/keep/pull/199) makes this the only -storage format and replaces migration with direct initialization. These pages -own the durable representation. - -The [requirements ledger](requirements.md) records evidence and remaining obligations; the laws below are normative requirements, not a claim that every corresponding runtime surface exists. - -## Core laws - -Version 2 retains every version-1 physical law and adds these: - -- the format version is explicit and cannot be inferred from path existence; -- a store is created only by direct initialization, a recoverable state - machine under writer authority with one durable commit point, and it does - not admit before that point; -- an initialized store with no published catalog is a defined, admitted - state; -- nothing persisted binds a store to its directory: a quiescent copy is the - same store, and live authority comes from the opening process; -- retention authority exists only through one verified retention head and its - complete immutable manifest; -- each manifest binds every admitted namespace to one exact root generation - and canonical digest; -- root closure is derived from a verified catalog, never from paths, caller - claims, recent access, or application identity; -- catalog publication preserves every current retained closure before - replacing the catalog head; -- readers acquire the version-2 reader fence before opening the catalog head; - and -- ambiguous, corrupt, missing, excessive, or unsupported evidence refuses - before mutation. - -## Normative pages - -The following pages form one protocol: - -- [Direct initialization](initialization.md) owns how a store comes into - existence: the forward protocol, its commit point, every interrupted state - and its one result, and the initialization process-death boundaries. -- [Format, namespace, and portability](recovery.md) owns the exact root - namespace, the `FORMAT` marker, the reader fence, the GC and - recovery-disposition reservation, and the laws that keep a store portable. -- [Retention records](retention.md) owns canonical namespace, root-generation, - manifest, and retention-head rules. -- [Retention publication](retention-publication.md) owns closure admission and - the generation transition. -- [Catalog publication and stage recovery](catalog-publication.md) owns the - version-2 segment and catalog writer, its crash states, and version-2 - catalog-stage recovery, including gated next-head finalization. -- [Retention publication recovery](retention-recovery.md) owns interrupted stages, restart publication, and retention process-death boundaries. -- [Closure verification](closure.md) owns deterministic traversal, exact - resource accounting, authenticated reconstruction, and closure evidence. -- [Closure corruption boundary](closure-corruption.md) owns the admitted-record - ingress proof and its exact refusal evidence. -- [GC and disposition records](gc.md) owns the canonical planned intent, - completion, and recovery-disposition byte grammars. -- [Requirements and evidence](requirements.md) owns stable requirement and - crash identifiers, evidence status, compatibility, and nonclaims. -- [Retired identifiers](retired-identifiers.md) maps every retired - requirement and crash identifier to its last evidence. -- [Format rationale](rationale.md) records format-local choices and rejected - alternatives. - -The [version-2 golden corpus](../../../conformance/segment-store/v2/README.md) -freezes independent definition, profile, marker, and retention record bytes. -It is format evidence, not production-writer evidence. - -The version-1 [segment](../segment-store-v1/segment.md), -[catalog](../segment-store-v1/catalog.md), and -[publication-head](../segment-store-v1/catalog.md#publication-head) -grammars remain byte-for-byte authoritative. Version 2 does not reinterpret or -re-encode them. - -## Status - -The format contract is frozen by ADR-0009, ADR-0010, and this specification. - -Implemented with executable evidence: public core types for namespaces, -generations, realization profiles, closure policies, anchors, and roots; -canonical root, manifest, head, and marker codecs matching their golden -records; storage-independent transition planning, bounded closure -verification against one pinned catalog, preflight, preparation, and the -17-phase publication port; `FilesystemStoreAdmission::initialize`, which -creates or resumes a store through the initialization protocol and its 41-case -process-death matrix; `FilesystemStoreAdmission::reopen`, which admits -the committed namespace and the marker and pins the retention directories it -admitted; and `FilesystemRetentionPublicationAuthority`, which publishes -initial and successor generations against the observed head, binds this -store's catalog head and the catalog it selects, and refuses superseded -candidates, retained stages, replaced protocol directories, and every -namespace or capacity violation before mutation, each as a typed -`RetentionCurrentStateRefusal`. - -Retention publication recovery, fenced retention snapshots and model-based transition evidence are implemented on main. - -The [landing ledger](../../testing-evidence/retention-landing.md) records the accepted scope and evidence for merged PR #99. - -Incomplete retention stages are preserved pending explicit disposition; automatic disposal remains deferred in [#155](https://github.com/flyingrobots/keep/issues/155). - -Execution failures preserve the typed cause and report known effects separately from uncertain effects or durability, as specified by the [retention recovery contract](retention-recovery.md). - -Writer authority coordinates cooperating writers in a managed namespace; it does not isolate arbitrary out-of-band filesystem mutation or make pathname unlink conditional on inode identity. - -`FilesystemRetentionSnapshot` binds a catalog, retention head and manifest under a shared reader fence and verifies selected roots on demand. `DurableStore` composes that view with authenticated whole-blob and exact-range reads, delivered in [PR #164](https://github.com/flyingrobots/keep/pull/164); the [read contract](../../invariants/authenticated-reconstruction/README.md) records proof and allocation limits. - -The candidate-catalog retained-closure gate is implemented over its storage port ([#125](https://github.com/flyingrobots/keep/issues/125)). `FilesystemCatalogPublisher` ingests one segment and publishes a catalog successor through it, including the first catalog of an initialized store, and version-two next-head finalization runs the same admission before replacing `HEAD` ([#198](https://github.com/flyingrobots/keep/issues/198)); see [catalog publication and stage recovery](catalog-publication.md). Verifying a new retention root against the current catalog is a different operation. - -[Durable verification reports](../../invariants/verification/README.md) are delivered in [PR #165](https://github.com/flyingrobots/keep/pull/165), with explicit subject/depth evidence and typed non-success outcomes. - -Production durable ingestion ([#82](https://github.com/flyingrobots/keep/issues/82)) builds on the version-two segment and catalog writer but is not delivered by it. Durable ingestion, garbage collection and compaction ([#21](https://github.com/flyingrobots/keep/issues/21)) remain absent from main; those prepared portions of unmerged PR #107 are not delivered APIs here. - -The `KEEP-CRASH-036..052` initial-publication process-death sequence includes independent recovered-reader checks; its [evidence receipt](../../testing-evidence/retention-crash-reader-oracle.md) records the assertions, calibration, and scope. - -Version-1 admission is deleted ([#198](https://github.com/flyingrobots/keep/issues/198)): this is the only format, and every open path refuses a root without its `FORMAT` marker. No conversion from version 1 exists. diff --git a/docs/formats/store/README.md b/docs/formats/store/README.md new file mode 100644 index 00000000..16949e86 --- /dev/null +++ b/docs/formats/store/README.md @@ -0,0 +1,187 @@ +# Keep Store Format + +`keep.segment-store/v2` is the one store format Keep creates, admits, writes, +and recovers. It is the successor to `keep.segment-store/v1`: it keeps the +version-1 segment, catalog, and publication-head grammars byte for byte and +adds the `FORMAT` marker, the reader fence, explicit retention state, and +reserved GC and recovery-disposition namespaces. Version 1 is no longer a +store format: its admission, initializer, and recovery entry points were +deleted in [#198](https://github.com/flyingrobots/keep/issues/198), and every +open path refuses a root without the `FORMAT` marker. No conversion from +version 1 exists. + +ADR-0005 records the segment, catalog, and publication-head protocol. +ADR-0009 owns the retention and liveness decision. +[ADR-0010](https://github.com/flyingrobots/keep/pull/199) makes this the only +storage format and replaces migration with direct initialization. These pages +are one normative set; every page describes the behaviour of `main`, and the +two requirement ledgers name the test behind each claim. + +Segment writing and verified reading are implemented in issue #15. Catalog +generation, writer-locked publication, and immutable restart snapshots are +implemented in issue #16. Explicit stage recovery and the process-death crash +matrix are implemented in issue #17; since #198 they run on stores created +by direct initialization. + +## Core law + +For a given logical content identity, Keep returns exactly the bytes named by +that identity or refuses. + +The catalog protocol adds these physical laws: + +- file existence proves no identity, completeness, publication, retention, or + durability claim; +- only one verified publication head selects one verified immutable catalog; +- only that catalog maps logical identities to immutable physical records; +- a segment is immutable after its complete seal is synchronized; +- a generation is published only after all referenced immutable artifacts are + synchronized and the new head replacement is directory-synchronized; and +- uncertain, conflicting, malformed, unsupported, or corrupt state is + represented and refused, never guessed or silently repaired. + +Physical locations and digests are not `BlobId`, `ChunkId`, or `LayoutId`. +Rechunking and later compaction may move physical evidence without moving +logical identity. + +## Store laws + +The store adds these laws to the catalog protocol: + +- the format version is explicit and cannot be inferred from path existence; +- a store is created only by direct initialization, a recoverable state + machine under writer authority with one durable commit point, and it does + not admit before that point; +- an initialized store with no published catalog is a defined, admitted + state; +- nothing persisted binds a store to its directory: a quiescent copy is the + same store, and live authority comes from the opening process; +- retention authority exists only through one verified retention head and its + complete immutable manifest; +- each manifest binds every admitted namespace to one exact root generation + and canonical digest; +- root closure is derived from a verified catalog, never from paths, caller + claims, recent access, or application identity; +- catalog publication preserves every current retained closure before + replacing the catalog head; +- readers acquire the reader fence before opening the catalog head; and +- ambiguous, corrupt, missing, excessive, or unsupported evidence refuses + before mutation. + +## Normative pages + +Record grammars: + +- [Segment bytes](segment.md) owns canonical primitives, record framing, and + immutable segment bytes. +- [Catalog and publication-head bytes](catalog.md) owns catalog generations, + publication-head framing, and fixed bounds. +- [Retention records](retention.md) owns canonical namespace, root-generation, + manifest, and retention-head rules. +- [GC and disposition records](gc.md) owns the canonical planned intent, + completion, and recovery-disposition byte grammars. + +Store lifecycle: + +- [Direct initialization](initialization.md) owns how a store comes into + existence: the forward protocol, its commit point, every interrupted state + and its one result, and the initialization process-death boundaries. +- [Format, namespace, and portability](recovery.md) owns the exact root + namespace, the `FORMAT` marker, the reader fence, the GC and + recovery-disposition reservation, and the laws that keep a store portable. +- [Publication and reader visibility](publication.md) owns the physical + namespace, forward protocol, writer exclusion, and reader snapshots of the + segment, catalog, and head transitions. +- [Catalog publication and stage recovery](catalog-publication.md) owns the + gated segment and catalog writer, its crash states, and catalog-stage + recovery, including gated next-head finalization. +- [Catalog-protocol stage recovery and platform contract](stage-recovery.md) + owns recovery classification, refusal precedence, the stage recovery tools, + the catalog-protocol crash matrix, and supported filesystem semantics. +- [Retention publication](retention-publication.md) owns closure admission and + the generation transition. +- [Retention publication recovery](retention-recovery.md) owns interrupted + stages, restart publication, and retention process-death boundaries. +- [Closure verification](closure.md) owns deterministic traversal, exact + resource accounting, authenticated reconstruction, and closure evidence. +- [Closure corruption boundary](closure-corruption.md) owns the admitted-record + ingress proof and its exact refusal evidence. + +Evidence and decisions: + +- [Requirements and evidence](requirements.md) owns the store, retention, + initialization, and portability requirement and crash identifiers, evidence + status, compatibility, and nonclaims. +- [Catalog-protocol requirements, compatibility, and evidence](catalog-protocol-requirements.md) + owns the segment, catalog, and stage-recovery requirement ledger, security + nonclaims, and golden evidence. +- [Retired identifiers](retired-identifiers.md) maps every retired + requirement and crash identifier to its last evidence. +- [Format rationale](rationale.md) and the + [catalog-protocol rationale](catalog-protocol-rationale.md) record + format-local choices and rejected alternatives. + +No page is independently optional. An implementation conforms only when it +satisfies the complete linked protocol, the +[catalog-protocol transition ledger](../../../conformance/segment-store/v1/transitions.tsv), +and the +[initialization transition ledger](../../../conformance/segment-store/v2/transitions.tsv). + +## Golden corpora + +The [segment, catalog, and head corpus](../../../conformance/segment-store/v1/README.md) +and the [store corpus](../../../conformance/segment-store/v2/README.md) freeze +independent record bytes, the format definition, the realization profile, and +the marker. Their directory names carry the record-grammar version that each +artifact's own magic and version fields name, so they are not renamed with +these pages. They are format evidence, not production-writer evidence. + +## Status + +The format contract is frozen by ADR-0005, ADR-0009, ADR-0010, and these +pages. + +Implemented with executable evidence: the segment, catalog, and head codecs +and the shared catalog stages; public core types for namespaces, generations, +realization profiles, closure policies, anchors, and roots; canonical root, +manifest, head, and marker codecs matching their golden records; +storage-independent transition planning, bounded closure verification against +one pinned catalog, preflight, preparation, and the 17-phase publication +port; `FilesystemStoreAdmission::initialize`, which creates or resumes a +store through the initialization protocol and its 41-case process-death +matrix; `FilesystemStoreAdmission::reopen`, which admits the committed +namespace and the marker and pins the retention directories it admitted; +`FilesystemCatalogPublisher`, which ingests one segment and publishes a +catalog through the retained-closure gate +([#125](https://github.com/flyingrobots/keep/issues/125)), including the first +catalog of an initialized store; the stage recovery tools, whose next-head +finalization runs the same admission before replacing `HEAD`; and +`FilesystemRetentionPublicationAuthority`, which publishes and recovers +retention generations and refuses every namespace or capacity violation before +mutation. + +`FilesystemRetentionSnapshot` binds a catalog, retention head and manifest +under a shared reader fence and verifies selected roots on demand. +`DurableStore` composes that view with authenticated whole-blob and +exact-range reads ([PR #164](https://github.com/flyingrobots/keep/pull/164)); +the [read contract](../../invariants/authenticated-reconstruction/README.md) +records proof and allocation limits. +[Durable verification reports](../../invariants/verification/README.md) are +delivered in [PR #165](https://github.com/flyingrobots/keep/pull/165). + +Incomplete retention stages are preserved pending explicit disposition; +automatic disposal remains deferred in +[#155](https://github.com/flyingrobots/keep/issues/155). Writer authority +coordinates cooperating writers in a managed namespace; it does not isolate +arbitrary out-of-band filesystem mutation or make pathname unlink conditional +on inode identity. Production durable ingestion +([#82](https://github.com/flyingrobots/keep/issues/82)), garbage collection, +and compaction ([#21](https://github.com/flyingrobots/keep/issues/21)) remain +absent from main. + +The `KEEP-CRASH-036..052` initial-publication process-death sequence includes +independent recovered-reader checks; its +[evidence receipt](../../testing-evidence/retention-crash-reader-oracle.md) +records the assertions, calibration, and scope. The +[landing ledger](../../testing-evidence/retention-landing.md) records the +accepted scope and evidence for merged PR #99. diff --git a/docs/formats/segment-store-v1/rationale.md b/docs/formats/store/catalog-protocol-rationale.md similarity index 97% rename from docs/formats/segment-store-v1/rationale.md rename to docs/formats/store/catalog-protocol-rationale.md index 1b77fa3d..4369a5ed 100644 --- a/docs/formats/segment-store-v1/rationale.md +++ b/docs/formats/store/catalog-protocol-rationale.md @@ -1,7 +1,8 @@ -# Durable Segment Store Version 1 Rationale +# Catalog-Protocol Rationale -This note records the format-local choices behind -`keep.segment-store/v1`. ADR-0005 owns the cross-cutting decision to bind +This note records the format-local choices behind the segment, catalog, and +publication-head protocol, first specified as `keep.segment-store/v1` and +unchanged in the [Keep store format](README.md). ADR-0005 owns the cross-cutting decision to bind format, publication, crash states, and recovery into one protocol. ## Three artifacts instead of one mutable file @@ -163,7 +164,7 @@ Initialization then refuses any root that is not an initialization prefix before mutation, acquires and retains `writer.lock`, and returns the proof only after its commit point; the version-1 initializer that created just the three catalog-protocol directories is retired -([Initialization](../segment-store-v2/initialization.md)). +([Initialization](initialization.md)). The profile treats local single-host mounting as an environmental precondition, not as a fact derivable from `statfs`. A shared or multiply mounted ext4 volume diff --git a/docs/formats/segment-store-v1/requirements.md b/docs/formats/store/catalog-protocol-requirements.md similarity index 95% rename from docs/formats/segment-store-v1/requirements.md rename to docs/formats/store/catalog-protocol-requirements.md index 5d3b66be..6288f04f 100644 --- a/docs/formats/segment-store-v1/requirements.md +++ b/docs/formats/store/catalog-protocol-requirements.md @@ -1,7 +1,10 @@ -# Requirements, Compatibility, and Evidence +# Catalog-Protocol Requirements, Compatibility, and Evidence -This page owns the requirement ledger, compatibility boundary, security -nonclaims, and golden evidence for `keep.segment-store/v1`. +This page owns the segment, catalog, publication, and stage-recovery +requirement ledger, its compatibility boundary, security nonclaims, and golden +evidence for the [Keep store format](README.md). The store, retention, +initialization, and portability ledger is [Requirements and +evidence](requirements.md). ## Requirement ledger @@ -103,12 +106,11 @@ may now authorize a transition-checked finalization through a semantic storage port, and the filesystem finalizer binds that transition to pinned writer-authorized storage. These slices now include reusable-stage continuation and the complete process-death crash matrix. Retention, compaction, garbage -collection, and host-power-loss simulation remain outside version 1: -retention belongs to -[`keep.segment-store/v2`](../segment-store-v2/README.md), compaction and -garbage collection are tracked in issue #21. The process-death matrix does -not establish host-power-loss behavior; that requires separate filesystem and -device fault evidence. +collection, and host-power-loss simulation remain outside this protocol: +retention belongs to the [retention records](retention.md) of the store +format, and compaction and garbage collection are tracked in issue #21. The +process-death matrix does not establish host-power-loss behavior; that +requires separate filesystem and device fault evidence. @@ -136,17 +138,20 @@ device fault evidence. -## Compatibility and migration +## Compatibility and format changes The byte grammars, magic values, field widths and order, endianness, kinds, flags, algorithm coordinates, bounds, checksum domains, catalog ordering, physical-name grammar, generation law, crash-point identifiers, publication order, and recovery classifications are compatibility commitments. -Changing any of them requires a new store protocol version and an explicit -migration decision. A migration writes and verifies new immutable artifacts, -publishes a new compatible head under its accepted protocol, and never -reinterprets existing bytes in place. +Changing any of them changes the store format. Before the first release that +regenerates the affected golden fixtures and needs no conversion; after it, a +format change ships ADR-0010's separately invoked upgrade tool, which writes a +new store from the old one and never converts or reinterprets existing bytes +in place. The format definition (`definition.tsv`) does not yet enumerate +these segment, catalog, and head grammars, so its digest does not change when +they do; that binding is not claimed. Logical `BlobId`, `ChunkId`, and `LayoutId` remain stable when their exact logical bytes and canonical plans remain unchanged. @@ -196,7 +201,7 @@ single-format stores and retires the version-1 initializer. `KEEP-STORE-020`, `KEEP-RECOVERY-002`, and `KEEP-RECOVERY-003` named the version-1 initializer and its platform-checked reopen. They are retired, never reused, and mapped to their successors in the -[retirement ledger](../segment-store-v2/retired-identifiers.md). +[retirement ledger](retired-identifiers.md). The format-local tradeoffs are recorded in the -[colocated rationale](rationale.md). +[colocated rationale](catalog-protocol-rationale.md). diff --git a/docs/formats/segment-store-v2/catalog-publication.md b/docs/formats/store/catalog-publication.md similarity index 82% rename from docs/formats/segment-store-v2/catalog-publication.md rename to docs/formats/store/catalog-publication.md index 251cc76c..d2adb2d3 100644 --- a/docs/formats/segment-store-v2/catalog-publication.md +++ b/docs/formats/store/catalog-publication.md @@ -1,11 +1,12 @@ # Catalog Publication and Stage Recovery -This page owns segment ingestion and catalog publication on a committed -`keep.segment-store/v2` store, and recovery of the catalog-protocol -stages that publication leaves. It adds no byte format: the segment, catalog, -and publication-head grammars stay the version-1 grammars, and the retention -records stay those of [Retention records](retention.md). Version-2 stores are -created by [direct initialization](initialization.md). +This page owns segment ingestion and catalog publication on a committed store, +and recovery of the catalog-protocol stages that publication leaves. It adds +no byte format: the segment, catalog, and publication-head grammars are those +of [Segment bytes](segment.md) and [Catalog and publication-head +bytes](catalog.md), and the retention records stay those of [Retention +records](retention.md). Stores are created by [direct +initialization](initialization.md). ## Supported envelope @@ -14,10 +15,10 @@ created by [direct initialization](initialization.md). segment is outside this envelope. Segment rollover ([#121](https://github.com/flyingrobots/keep/issues/121)) is not claimed. - The caller supplies the complete candidate catalog and every segment it - references. The version-1 catalog, segment, and head bounds apply unchanged. + references. The catalog, segment, and head bounds apply unchanged. - Retention evidence is bounded by one manifest of at most 4,096 entries and one root at a time. Each closure is bounded by its root's admitted limits. -- The store must be a committed version-2 root. A freshly initialized store +- The store must be a committed root. A freshly initialized store has no `HEAD`; its first publication uses the uninitialized expectation, has no predecessor, and is admitted only when `segments`, `catalogs`, and both retention pools are empty. @@ -35,11 +36,11 @@ publishes only through `publish_retained_catalog_generation`. Its stage type, `FilesystemRetainedCatalogStages`, has private fields and no public constructor. It is reachable only through `catalog_stages`, which requires the gate's `RetainedClosureAdmission`. The ungated `publish_catalog_generation` -therefore cannot drive version-2 stages. +therefore cannot drive the gated stages. -The stage transitions are the version-1 implementation. One -`FilesystemCatalogStages` value runs them under either writer authority, so the -two formats cannot diverge in their segment, catalog, or head steps. +The stage transitions are the one `FilesystemCatalogStages` implementation +that [Publication and reader visibility](publication.md) specifies; the gated +stages forward to it unchanged. Publication never takes the reader fence. Readers hold `reader.lock` shared and select only `HEAD`. A retained or in-flight catalog `head.next` is admitted @@ -104,7 +105,7 @@ must resolve it first. `FilesystemRecoveryAdmission::reopen` applies the writer admission with one difference: the root may also hold `head.next`. It requires the -platform profile, the writer lock, the exact version-2 namespace and protocol +platform profile, the writer lock, the exact store namespace and protocol directories, and the canonical `FORMAT` marker. The recovery tools take it through their `open` constructors: @@ -120,15 +121,15 @@ the same namespace and marker laws without the writer lock. `retention`, `gc`, and `recovery` as known roles and requires each of them. It does not require `HEAD`: the first publication of an initialized store can be interrupted before `HEAD` exists. `FORMAT.next` is an unknown entry. -Version-2 admission owns the contents of the three protocol -directories, and retention recovery owns `retention`. +Store admission owns the contents of the three protocol directories, and +retention recovery owns `retention`. -Stage discard, completion, and segment resume execute the version-1 -executors unchanged. None of them changes `HEAD`. +Stage discard, completion, and segment resume execute the executors of +[Catalog-protocol stage recovery](stage-recovery.md) unchanged. None of them changes `HEAD`. ### Gated next-head finalization -A version-2 finalizer retains the admitted retention directories. Before every +A finalizer retains the admitted retention directories. Before every candidate verification that can precede `HEAD` replacement, including the one immediately before the rename, it runs the forward gate's retained-closure admission against the staged candidate catalog. A candidate that would lose @@ -155,9 +156,9 @@ are deleted (#198), so no cross-version entry remains. ## Process-death matrix `cargo xtask durability-crash-matrix --sequence retained-catalog` runs the -production version-2 writer in a child process. The child blocks at the +production writer in a child process. The child blocks at the selected boundary through an observer on the writer's own stage type. Restart -then requires the exact stage residue, recovers it with the public version-2 +then requires the exact stage residue, recovers it with the public recovery tools, retains the ingested blob, and reads both blobs through `DurableStore`. @@ -169,9 +170,9 @@ recovery tools, retains the ingested blob, and reads both blobs through | `KEEP-CRASH-075` | catalog-stage creation | before, during, after | none and republication before; empty `current.cat`, discard, and republication otherwise | Implemented | | `KEEP-CRASH-076` | `head.next` synchronization | before, during, after | complete `head.next`; gated finalization | Implemented | | `KEEP-CRASH-077` | `HEAD` replacement | before, during, after | complete `head.next` and gated finalization before; generation two published otherwise | Implemented | -| `KEEP-CRASH-078` | version-2 recovery stage removal | before, during, after | the version-1 discard executor's law under version-2 admission | Planned | -| `KEEP-CRASH-079` | version-2 recovery stage completion link | before, during, after | the version-1 completion executor's law under version-2 admission | Planned | -| `KEEP-CRASH-080` | version-2 gated finalization replacement | before, during, after | finalized or retryable `head.next`; the gate reruns on retry | Planned | +| `KEEP-CRASH-078` | recovery stage removal under recovery admission | before, during, after | covered by `KEEP-CRASH-027`–`030`, which now run under recovery admission | Retired (#198) | +| `KEEP-CRASH-079` | recovery stage completion link | before, during, after | the completion executor's law under recovery admission | Planned | +| `KEEP-CRASH-080` | gated finalization replacement | before, during, after | finalized or retryable `head.next`; the gate reruns on retry | Planned | @@ -181,5 +182,6 @@ not simulate power loss. In-process laws add an injected storage-port failure after segment link, catalog-stage creation, and `head.next` synchronization (`filesystem_catalog_recovery_tests`), plus hand-staged complete `head.next` candidates that preserve or drop retained content -(`tests/catalog_writer.rs`). Other boundaries rely on the shared -version-1 stage implementation and its `KEEP-CRASH-001` to `030` evidence. +(`tests/catalog_writer.rs`). Every other boundary of the shared stages is +covered by `KEEP-CRASH-001` to `030`, which run on stores created by direct +initialization. diff --git a/docs/formats/segment-store-v1/catalog.md b/docs/formats/store/catalog.md similarity index 96% rename from docs/formats/segment-store-v1/catalog.md rename to docs/formats/store/catalog.md index 9f9d26c5..fdd01971 100644 --- a/docs/formats/segment-store-v1/catalog.md +++ b/docs/formats/store/catalog.md @@ -1,8 +1,9 @@ # Catalog and Publication-Head Bytes This page owns catalog generations, publication-head framing, and the fixed -bounds for `keep.segment-store/v1`. It uses the canonical primitives defined -by the [segment-byte grammar](segment.md#canonical-primitives). +bounds of the [Keep store format](README.md). Both records are grammar +version 1, unchanged since `keep.segment-store/v1`. It uses the canonical +primitives defined by the [segment-byte grammar](segment.md#canonical-primitives). ## Catalog generation diff --git a/docs/formats/segment-store-v2/closure-corruption.md b/docs/formats/store/closure-corruption.md similarity index 100% rename from docs/formats/segment-store-v2/closure-corruption.md rename to docs/formats/store/closure-corruption.md diff --git a/docs/formats/segment-store-v2/closure.md b/docs/formats/store/closure.md similarity index 100% rename from docs/formats/segment-store-v2/closure.md rename to docs/formats/store/closure.md diff --git a/docs/formats/segment-store-v2/gc.md b/docs/formats/store/gc.md similarity index 100% rename from docs/formats/segment-store-v2/gc.md rename to docs/formats/store/gc.md diff --git a/docs/formats/segment-store-v2/initialization.md b/docs/formats/store/initialization.md similarity index 99% rename from docs/formats/segment-store-v2/initialization.md rename to docs/formats/store/initialization.md index 8cf60098..4d795164 100644 --- a/docs/formats/segment-store-v2/initialization.md +++ b/docs/formats/store/initialization.md @@ -169,7 +169,7 @@ path refuses it as `MissingFormatMarker`. A root holding only an empty runs the production initializer in a subprocess and kills it before, during, and after each boundary. The parent then checks the exact inventory and bytes, the admission refusal or result, and runs `initialize` again as recovery. It -then requires writer reopen, publishes a first catalog through the version-2 +then requires writer reopen, publishes a first catalog through the writer, and reads the blob back. | ID | Boundary | During | diff --git a/docs/formats/segment-store-v1/publication.md b/docs/formats/store/publication.md similarity index 94% rename from docs/formats/segment-store-v1/publication.md rename to docs/formats/store/publication.md index 558c5b35..02e29518 100644 --- a/docs/formats/segment-store-v1/publication.md +++ b/docs/formats/store/publication.md @@ -1,12 +1,14 @@ # Publication and Reader Visibility This page owns the physical namespace, visibility states, writer exclusion, -forward publication order, and reader snapshots for -`keep.segment-store/v1`. +forward publication order, and reader snapshots of the segment, catalog, and +head transitions of the [Keep store format](README.md). ## Physical namespace -The version-1 filesystem adapter owns this relative namespace: +The filesystem adapter owns this relative catalog-protocol namespace; the +store root holds the further entries that +[Format, namespace, and portability](recovery.md) specifies: ```text writer.lock @@ -54,7 +56,7 @@ silently interpreted as an empty store. ## Store initialization A store is created only by direct initialization of the single store format, -specified in [Initialization](../segment-store-v2/initialization.md) and +specified in [Initialization](initialization.md) and crash-tested at `KEEP-CRASH-081`–`KEEP-CRASH-091`. It commits `writer.lock`, the catalog-protocol directories `staging`, `segments`, and `catalogs`, and the format's marker, reader fence, and retention, GC, and recovery directories @@ -64,7 +66,7 @@ publication. The version-1 initializer, which created only `writer.lock` and the three catalog-protocol directories, is retired with its crash rows `KEEP-CRASH-031`–`KEEP-CRASH-035`; the -[retirement ledger](../segment-store-v2/retired-identifiers.md) records them. +[retirement ledger](retired-identifiers.md) records them. ## State and visibility @@ -78,7 +80,7 @@ catalog-protocol directories, is retired with its crash rows | Sealed stage | complete verified seal in staging | no | immutable bytes, not yet durable pool evidence | | Valid orphan | verified immutable artifact not selected by current head | no | preserve; neither promote nor delete silently | | Published | selected through one verified head and catalog | yes | exact immutable reader snapshot | -| Retired | selected by an older catalog but not the current head | no for new readers | existing pinned readers may finish; no deletion in v1 | +| Retired | selected by an older catalog but not the current head | no for new readers | existing pinned readers may finish; no deletion until garbage collection ([#21](https://github.com/flyingrobots/keep/issues/21)) | | Corrupt sealed | seal, checksum, digest, or framing disagreement | no | refuse; never truncate or reinterpret | | Stale generation | candidate does not extend current generation exactly | no | refuse expected/observed generation | | Unrecoverable ambiguity | conflicting or insufficient durable evidence | no | represent uncertainty and refuse | @@ -87,7 +89,7 @@ catalog-protocol directories, is retired with its crash rows ## Writer exclusion -Version 1 is a one-writer/many-reader protocol. +The store is a one-writer/many-reader protocol. The writer opens `writer.lock` without following links and acquires one exclusive kernel advisory lock before recovery planning. It holds the lock @@ -116,7 +118,7 @@ Neither operation repairs, enumerates, or removes protocol state. The admission has private fields, so only Keep's own initialization and admission boundary can produce a value. A new store obtains one through `FilesystemStoreAdmission::initialize`, which runs the -[initialization protocol](../segment-store-v2/initialization.md); a committed +[initialization protocol](initialization.md); a committed store reacquires one through `FilesystemStoreAdmission::reopen`, which mutates nothing. A root without the `FORMAT` marker refuses at every open path. The version-1 initializer, `FilesystemPlatformAdmission`, and their @@ -162,7 +164,7 @@ proves the exact canonical directories and the persistent lock file before a publisher opens. Any retained `head.next` or `current.cat`, and any `current.seg` not owned by the selected staged segment, causes publication to refuse before mutation; the explicit recovery boundaries in -[Recovery and platform contract](recovery.md) classify and resolve that +[Recovery and platform contract](stage-recovery.md) classify and resolve that residue. When `HEAD` is absent, the publisher probes both immutable pools and admits first publication only when both are empty; any entry is preserved as recovery evidence and refuses the operation. An already-current retry @@ -271,7 +273,8 @@ catalogs. If head replacement races the initial read, atomic replacement gives the reader either the complete old head or complete new head. Any other observation is an unsupported-platform or corruption refusal. -Version 1 never deletes immutable artifacts. A reader holding an older +No implemented protocol deletes immutable artifacts; collection is planned +in [GC and disposition records](gc.md). A reader holding an older verified generation can therefore finish while a new generation is published. Retention and GC must define later deletion safety before this can change. diff --git a/docs/formats/segment-store-v2/rationale.md b/docs/formats/store/rationale.md similarity index 100% rename from docs/formats/segment-store-v2/rationale.md rename to docs/formats/store/rationale.md diff --git a/docs/formats/segment-store-v2/recovery.md b/docs/formats/store/recovery.md similarity index 93% rename from docs/formats/segment-store-v2/recovery.md rename to docs/formats/store/recovery.md index addfd11c..d36130ef 100644 --- a/docs/formats/segment-store-v2/recovery.md +++ b/docs/formats/store/recovery.md @@ -1,14 +1,15 @@ # Namespace, Format Marker, and Recovery -This page owns the exact version-2 root namespace, the `FORMAT` marker, the +This page owns the exact store root namespace, the `FORMAT` marker, the reader fence, the GC and recovery-disposition reservation, and the laws that make a store independent of the directory it lives in. [Direct initialization](initialization.md) owns how a store comes into -existence; every version-2 store is created by direct initialization. +existence; every store is created by direct initialization. ## Exact filesystem namespace -Version 2 keeps the version-1 files and directories and adds: +The store keeps the catalog-protocol files and directories of +[Publication and reader visibility](publication.md) and adds: ```text reader.lock @@ -28,9 +29,9 @@ recovery/dispositions/.receipt Catalog `HEAD` is optional: an initialized store holds none until its first catalog publication. `retention/HEAD`, every fixed `.next` stage, `gc/intent`, and `gc/receipt` are optional according to the exact state tables of their -pages. The version-1 catalog `head.next` stays a version-1 file: writer -admission refuses it as recovery state, while readers and version-2 -catalog-stage recovery admit it, as +pages. The catalog `head.next` is a catalog-protocol stage: writer admission +refuses it as recovery state, while readers and catalog-stage recovery admit +it, as [catalog publication and stage recovery](catalog-publication.md) specifies. `FORMAT.next` exists only while initialization has not committed; every admission refuses it as `InitializationIncomplete`, and a root without `FORMAT` @@ -78,7 +79,7 @@ carries the decoder's typed refusal as its source. `reader.lock` is a persistent regular zero-length file. Its contents and existence alone prove nothing. -A version-2 reader acquires a kernel-managed shared lock on `reader.lock` +A reader acquires a kernel-managed shared lock on `reader.lock` before opening catalog `HEAD` or `retention/HEAD`. The returned `ReaderFence` owns that lock for the complete snapshot lifetime. Close, drop, or process death releases only the kernel lock and never deletes the persistent file. diff --git a/docs/formats/segment-store-v2/requirements.md b/docs/formats/store/requirements.md similarity index 86% rename from docs/formats/segment-store-v2/requirements.md rename to docs/formats/store/requirements.md index 1b016bfa..fcef0962 100644 --- a/docs/formats/segment-store-v2/requirements.md +++ b/docs/formats/store/requirements.md @@ -20,9 +20,9 @@ case is not evidence. | `KEEP-RETENTION-009` | Exact already-committed retry is idempotent only while its successor remains current | byte-identical planning in `tests/retention_transition.rs`; authority-revalidated zero-mutation retry receipt in `tests/retention_publication_execution.rs`; exact already-committed filesystem retry with a byte-identical retention witness in `filesystem_retention_storage_tests`; superseded-candidate filesystem refusal with zero mutation in `filesystem_retention_successor_tests`; committed retry reopens the head-selected manifest entry and root pool bytes, refusing absent, changed, or corrupt evidence in `filesystem_retention_current_tests`; every refusal is a typed `RetentionCurrentStateRefusal` source, with superseded, committed-root-absent, committed-root-changed, and head-absent-with-artifacts pinned by downcast | Implemented | | `KEEP-RETENTION-010` | Model operation sequences agree with a deterministic namespace-to-anchor-set map and never admit caller identity, paths, clocks, or application policy | all three-operation histories over initial publications of two namespaces, successor, release, restore, byte-identical retry and stale initial (343 histories, each in a fresh initialized store) compare fenced namespace generations, anchor sets and liveness against an operation-derived model after every step; exact typed refusals remain checked. The count describes exploration, not correctness. See [release/restore evidence](../../testing-evidence/retention-release-restore-model.md). Core architecture checks remain separate static evidence. | Implemented; release/restore model coverage added for #128 | | `KEEP-RETENTION-011` | Catalog publication over retention state admits the current retention head, manifest, and every manifest root under writer authority; verifies every retained closure against the candidate before any catalog storage call; and refuses `HEAD` replacement if the retention head changed | port-level laws in `tests/retained_catalog_publication.rs`: dropped and substituted members refuse with the exact missing identity and no catalog stage; every manifest root is verified wherever it sorts; absent, corrupt, substituted, and contradictory head, manifest, and root evidence refuse with typed expected and observed state; a head changed, removed, or published before commit refuses after `head.next` verification without `replace_head`; an exhaustive 24-case candidate-by-root-set model and a repeat-run determinism check | Implemented at the storage port; the filesystem version-2 publisher and version-2 next-head finalization use it (#198, `KEEP-RETENTION-012` and `013`) | -| `KEEP-RETENTION-012` | Under version-2 writer authority, one staged segment and a catalog successor publish only through the retained-closure gate, with no-follow bounded retention reads, head absence admitted only when uncontradicted, fixed retention stages refused, and no reader fence taken | `FilesystemCatalogPublisher`; `tests/catalog_writer.rs`: a preserving successor publishes, reopens, and reads both blobs while the retention root still verifies; a pinned reader view keeps its generation across publication; a candidate that drops retained content, a contradicted head absence, and a fixed retention stage each refuse with the exact typed error and an unchanged store witness; runtime RED for the contradiction and stage checks | Implemented for one staged segment per publication; segment rollover is #121 | -| `KEEP-RETENTION-013` | Version-2 segment and catalog stage recovery admits version-2 root entries as known, reuses the version-1 executors, and finalizes a staged `head.next` only after the retained-closure gate admits it | `FilesystemRecoveryAdmission`, `classify_recovery_names`, and the `open` constructors; `tests/catalog_writer.rs` covers classification, completion, discard, reusable-prefix segment resume, and finalization that preserves or would lose retained content (runtime RED with the gate result ignored); `filesystem_catalog_recovery_tests` recovers injected interruptions after segment link, catalog-stage creation, and `head.next` synchronization | Implemented | -| `KEEP-RETENTION-015` | Process death in the version-2 writer and its recovery reaches documented recoverable states | `cargo xtask durability-crash-matrix --sequence retained-catalog` runs `KEEP-CRASH-074..=077`, 12 cases, through recovery and durable reads; `KEEP-CRASH-078..=080` (process death inside version-2 recovery) are planned | In progress: writer rows implemented, recovery rows planned | +| `KEEP-RETENTION-012` | Under writer authority, one staged segment and a catalog successor publish only through the retained-closure gate, with no-follow bounded retention reads, head absence admitted only when uncontradicted, fixed retention stages refused, and no reader fence taken | `FilesystemCatalogPublisher`; `tests/catalog_writer.rs`: a preserving successor publishes, reopens, and reads both blobs while the retention root still verifies; a pinned reader view keeps its generation across publication; a candidate that drops retained content, a contradicted head absence, and a fixed retention stage each refuse with the exact typed error and an unchanged store witness; runtime RED for the contradiction and stage checks | Implemented for one staged segment per publication; segment rollover is #121 | +| `KEEP-RETENTION-013` | Segment and catalog stage recovery admits every store root entry as known, reuses the catalog-protocol executors, and finalizes a staged `head.next` only after the retained-closure gate admits it | `FilesystemRecoveryAdmission`, `classify_recovery_names`, and the `open` constructors; `tests/catalog_writer.rs` covers classification, completion, discard, reusable-prefix segment resume, and finalization that preserves or would lose retained content (runtime RED with the gate result ignored); `filesystem_catalog_recovery_tests` recovers injected interruptions after segment link, catalog-stage creation, and `head.next` synchronization | Implemented | +| `KEEP-RETENTION-015` | Process death in the gated writer and its recovery reaches documented recoverable states | `cargo xtask durability-crash-matrix --sequence retained-catalog` runs `KEEP-CRASH-074..=077`, 12 cases, through recovery and durable reads; stage discard under recovery admission is `KEEP-CRASH-027..=030` (so the planned `-078` is retired); `KEEP-CRASH-079..=080` (completion and gated finalization) are planned | In progress: writer rows implemented, recovery rows planned | @@ -85,12 +85,15 @@ evidence and states where its obligation went. ## Compatibility and nonclaims -- Version 2 keeps the version-1 segment, catalog, and publication-head - grammars byte for byte. A version-1 store is not converted: version-2 code - refuses it, and no migration or downgrade exists. +- The store keeps the segment, catalog, and publication-head grammars first + specified for `keep.segment-store/v1` byte for byte. A version-1 store is + not converted: every open path refuses a root without `FORMAT`, and no + migration or downgrade exists. After the first release, a format change + follows ADR-0010's separately invoked upgrade tool; normal opening never + converts a store. - Retention evidence proves a bounded physical reconstruction claim, not application meaning, causal ownership, future policy, or secure erasure. -- A fresh forward writer is not proof that version 2 is restart-safe or +- A fresh forward writer is not proof that the store is restart-safe or production-admitted; partial-prefix recovery and crash evidence remain mandatory. This applies to retention publication exactly as it applies to initialization: the filesystem publication writer never continues a retained diff --git a/docs/formats/segment-store-v2/retention-publication.md b/docs/formats/store/retention-publication.md similarity index 86% rename from docs/formats/segment-store-v2/retention-publication.md rename to docs/formats/store/retention-publication.md index 929f8ef0..9963f40d 100644 --- a/docs/formats/segment-store-v2/retention-publication.md +++ b/docs/formats/store/retention-publication.md @@ -20,7 +20,7 @@ the current manifest and derives exact canonical successors. ordered durability phases, and returns the complete receipt only after cleanup; exact already-committed retry revalidates authority and performs no mutation. -Version-2 catalog publication must hold the same writer authority and prove every current retained closure against its candidate catalog before replacing the catalog `HEAD`. +Catalog publication must hold the same writer authority and prove every current retained closure against its candidate catalog before replacing the catalog `HEAD`. `publish_retained_catalog_generation` implements that gate over the `RetainedCatalogPublicationStorage` port ([#125](https://github.com/flyingrobots/keep/issues/125)). Under writer authority, before its first catalog storage call, it: @@ -30,7 +30,7 @@ Version-2 catalog publication must hold the same writer authority and prove ever After `head.next` verification, it rereads the retention head and refuses `HEAD` replacement if the head changed. Its receipt binds the preserved manifest generation and digest. The [format rationale](rationale.md#gate-catalog-publication-on-every-retained-closure) records the crash states and the recovery obligation. -`FilesystemCatalogPublisher` publishes through the gate, and version-2 next-head finalization runs the same retained-closure admission against a staged candidate ([#198](https://github.com/flyingrobots/keep/issues/198)). [Catalog publication and stage recovery](catalog-publication.md) owns that writer and its recovery. Retention publication and complete-stage retention recovery separately reverify a new root's closure against the current catalog. +`FilesystemCatalogPublisher` publishes through the gate, and next-head finalization runs the same retained-closure admission against a staged candidate ([#198](https://github.com/flyingrobots/keep/issues/198)). [Catalog publication and stage recovery](catalog-publication.md) owns that writer and its recovery. Retention publication and complete-stage retention recovery separately reverify a new root's closure against the current catalog. ## Generation transition diff --git a/docs/formats/segment-store-v2/retention-recovery.md b/docs/formats/store/retention-recovery.md similarity index 100% rename from docs/formats/segment-store-v2/retention-recovery.md rename to docs/formats/store/retention-recovery.md diff --git a/docs/formats/segment-store-v2/retention.md b/docs/formats/store/retention.md similarity index 100% rename from docs/formats/segment-store-v2/retention.md rename to docs/formats/store/retention.md diff --git a/docs/formats/segment-store-v2/retired-identifiers.md b/docs/formats/store/retired-identifiers.md similarity index 96% rename from docs/formats/segment-store-v2/retired-identifiers.md rename to docs/formats/store/retired-identifiers.md index 212b7b49..6f2cbbcb 100644 --- a/docs/formats/segment-store-v2/retired-identifiers.md +++ b/docs/formats/store/retired-identifiers.md @@ -120,7 +120,7 @@ single-format writer runs, so they keep their identifiers and their crash children now run on stores created by direct initialization. A row whose boundary no longer exists is retired here and never reused. -### Requirements +### Version-1 requirements @@ -139,7 +139,7 @@ boundary no longer exists is retired here and never reused. ownership alone still cannot construct a publisher, now because only single-format admission produces publisher authority. -### Crash identifiers +### Version-1 crash identifiers `KEEP-CRASH-031` to `-035` named the version-1 initializer. Their last evidence is the initialization crash child at `1fc1ad0` and `f7956c0`, and the @@ -161,7 +161,14 @@ transition ledger at `e03c2a1`. publication with a retained root, a scenario distinct from the first publication that `KEEP-CRASH-009`, `-013`, `-024`, and `-025` cover. -### Code and evidence removed +`KEEP-CRASH-078` was reserved, never implemented, for process death in stage +discard under version-2 recovery admission. It is retired: `KEEP-CRASH-027` to +`-030` are that boundary and now run under single-format recovery admission. +`KEEP-CRASH-079` and `-080` (stage completion and gated finalization under +recovery admission) stay reserved and planned in +[catalog publication and stage recovery](catalog-publication.md). + +### Version-1 code and evidence removed - `src/adapters/filesystem_platform_admission.rs`, `filesystem_catalog_publisher.rs`, `filesystem_catalog_publisher_storage.rs`, diff --git a/docs/formats/segment-store-v1/segment.md b/docs/formats/store/segment.md similarity index 96% rename from docs/formats/segment-store-v1/segment.md rename to docs/formats/store/segment.md index 445d71a2..c2c29eeb 100644 --- a/docs/formats/segment-store-v1/segment.md +++ b/docs/formats/store/segment.md @@ -1,7 +1,9 @@ # Segment Bytes This page owns the canonical primitives, record framing, and immutable segment -grammar for `keep.segment-store/v1`. +grammar of the [Keep store format](README.md). The segment records are +grammar version 1, named by their own magic and version fields; they were first +specified for `keep.segment-store/v1` and are unchanged. ## Canonical primitives diff --git a/docs/formats/segment-store-v1/recovery.md b/docs/formats/store/stage-recovery.md similarity index 98% rename from docs/formats/segment-store-v1/recovery.md rename to docs/formats/store/stage-recovery.md index 7e50f7c3..fed6062d 100644 --- a/docs/formats/segment-store-v1/recovery.md +++ b/docs/formats/store/stage-recovery.md @@ -1,7 +1,9 @@ -# Recovery and Platform Contract +# Catalog-Protocol Stage Recovery and Platform Contract -This page owns recovery classification, deterministic refusal precedence, and -the minimum filesystem semantics for `keep.segment-store/v1`. +This page owns recovery classification, deterministic refusal precedence, the +segment, catalog, and head stage recovery tools, the catalog-protocol crash +matrix, and the minimum filesystem semantics of the +[Keep store format](README.md). ## Recovery diff --git a/docs/testing-evidence/current-durable-surfaces.md b/docs/testing-evidence/current-durable-surfaces.md index bf1fee5c..362fa64f 100644 --- a/docs/testing-evidence/current-durable-surfaces.md +++ b/docs/testing-evidence/current-durable-surfaces.md @@ -8,15 +8,15 @@ The initial reconciliation used main integration `6051abb25a9fd33ae7ee0de5614514 | Claim | Owning source or evidence | Documentation disposition | | --- | --- | --- | -| Migration can resume admitted residue under writer authority | Retired with migration by ADR-0010 (#198); see the [retirement ledger](../formats/segment-store-v2/retired-identifiers.md) | Superseded: stores are created by [direct initialization](../formats/segment-store-v2/initialization.md), which resumes its own interrupted states. | +| Migration can resume admitted residue under writer authority | Retired with migration by ADR-0010 (#198); see the [retirement ledger](../formats/store/retired-identifiers.md) | Superseded: stores are created by [direct initialization](../formats/store/initialization.md), which resumes its own interrupted states. | | Retention recovery executes complete-stage plans and preserves incomplete stages | [Filesystem recovery](../../src/adapters/retention/filesystem_retention_recovery.rs), [planner](../../src/adapters/retention/recovery_planner.rs), [accepted landing evidence](retention-landing.md) | Remove obsolete absence and pending-#99 claims; keep incomplete-stage disposition deferred to #155. | -| Execution failure is distinct from pre-effect refusal | [Execution](../../src/adapters/retention/recovery_execution.rs), [progress](../../src/adapters/retention/retention_storage_progress.rs), [contract](../formats/segment-store-v2/retention-recovery.md) | Preserve typed causes, known/uncertain effects, and durability distinctions. | +| Execution failure is distinct from pre-effect refusal | [Execution](../../src/adapters/retention/recovery_execution.rs), [progress](../../src/adapters/retention/retention_storage_progress.rs), [contract](../formats/store/retention-recovery.md) | Preserve typed causes, known/uncertain effects, and durability distinctions. | | Retention snapshots bind a fenced view and admit selected roots | [Snapshot](../../src/adapters/retention/filesystem_retention_snapshot.rs), [collector](../../src/adapters/retention/retention_view_collector.rs) | Keep the snapshot contract distinct from the now-delivered `DurableStore` composition in #164. [Reader-fence process evidence](reader-fence-process.md) records the merged #160 lifecycle laws, not power-loss guarantees. | | Forward retention publication and complete-root recovery reverify live closure | [Closure admission](../../src/adapters/retention/filesystem_retention_closure_admission.rs), [forward verification](../../src/adapters/retention/filesystem_retention_catalog.rs) | Remove the stale planned member-reverification claim. | -| General candidate-catalog admission must preserve every retained root | [Catalog preflight](../../src/adapters/filesystem_catalog_current.rs), [catalog storage](../../src/adapters/filesystem_catalog_storage.rs), [publication requirement](../formats/segment-store-v2/retention-publication.md#closure-admission) | Keep the normative requirement; explicitly identify its implementation gap in #125. Current-root verification against the current catalog does not satisfy this different requirement. | +| General candidate-catalog admission must preserve every retained root | [Catalog preflight](../../src/adapters/filesystem_catalog_current.rs), [catalog storage](../../src/adapters/filesystem_catalog_storage.rs), [publication requirement](../formats/store/retention-publication.md#closure-admission) | Keep the normative requirement; explicitly identify its implementation gap in #125. Current-root verification against the current catalog does not satisfy this different requirement. | | Durable authenticated reads are delivered | [Durable store](../../src/adapters/durable/store.rs), [snapshot](../../src/adapters/durable/snapshot.rs), [read contract](../invariants/authenticated-reconstruction/README.md), [evidence](durable-authenticated-reads.md) | Describe whole-blob and exact-range proof boundaries, stable locator, fence lifetime and separate allocation limits; #164 supplies mainline integration. | | Explicit verification reports are delivered | [Report](../../src/verification/report.rs), [durable ingress](../../src/adapters/verification_ingress.rs), [subject/depth matrix](../invariants/verification/README.md), [evidence](durable-verification.md) | Describe named subject/depth evidence and precise non-success outcomes; #165 supplies mainline integration. Reports grant no retention authority or future SnapshotBinding proof. | -| Production durable ingestion, GC and compaction are not delivered | [Crate public surface](../../src/lib.rs), [GC ledger](../formats/segment-store-v2/requirements.md#garbage-collection-reservation) | Track #82 and #21; do not import PR #107's remaining prepared implementations. | +| Production durable ingestion, GC and compaction are not delivered | [Crate public surface](../../src/lib.rs), [GC ledger](../formats/store/requirements.md#garbage-collection-reservation) | Track #82 and #21; do not import PR #107's remaining prepared implementations. | The managed-namespace concurrency contract remains the [accepted decision](../adr/retention-bounded-recovery-landing.md): cooperating writers under Keep authority, without a guarantee against arbitrary concurrent raw namespace mutation. diff --git a/docs/testing-evidence/retention-reader-root-identity.md b/docs/testing-evidence/retention-reader-root-identity.md index 6d20b627..94363c34 100644 --- a/docs/testing-evidence/retention-reader-root-identity.md +++ b/docs/testing-evidence/retention-reader-root-identity.md @@ -1,6 +1,6 @@ # Retention reader root-identity evidence -Superseded (#198): ADR-0010 deleted migration and the persisted root binding this evidence tested; the law and its test were removed. The [retirement ledger](../formats/segment-store-v2/retired-identifiers.md) records the last evidence. This page is kept as history. +Superseded (#198): ADR-0010 deleted migration and the persisted root binding this evidence tested; the law and its test were removed. The [retirement ledger](../formats/store/retired-identifiers.md) records the last evidence. This page is kept as history. Change kind: bug fix. Subject: Keep runtime fenced snapshot admission. Size: medium. Oracle: mutually consistent migration records are admissible for reading only when their restart-stable device and inode binding names the opened root; identity diagnostics preserve the bound and observed values. diff --git a/docs/testing/retention-stage-prefix-evidence.md b/docs/testing/retention-stage-prefix-evidence.md index 096718ac..7ce88dbf 100644 --- a/docs/testing/retention-stage-prefix-evidence.md +++ b/docs/testing/retention-stage-prefix-evidence.md @@ -62,7 +62,7 @@ Adding stage synchronization before root namespace creation/link, manifest link, Hosted validation of `fb34603` exposed a stale-document-copy gap in the earlier local full-workspace claim: the Docker checkout did not contain the latest recovery document. The hosted documentation-size guard refused its 306 lines. The earlier pass therefore proves the executed Rust suite against that local document snapshot, not full current-tree validation. Current-tree verification must use a complete source copy and resolve the document ownership issue before claiming the broad gate is green. -The retention restart section now lives in [retention-recovery.md](../formats/segment-store-v2/retention-recovery.md), linked from the namespace page and protocol index; its grammar and boundaries are preserved. The existing documentation guards read the new owner explicitly and remain documentation checks. Obsolete inline line-length overrides were removed from the touched pages so the accepted global line-length policy governs prose. A complete copy of the changed source and documents passed both `cargo test --workspace --all-features` and its release counterpart, including the OS-boundary regressions. All-feature Clippy, source structure, formatting, and Markdown passed. A preliminary complete-copy run failed Git's ownership check after archive extraction changed the owned sandbox directory's UID; correcting that sandbox ownership, without global Git exceptions or source changes, allowed the controlled run. Physical power loss, kernel I/O error injection, and the remaining PR review queue are still not certified by these results. +The retention restart section now lives in [retention-recovery.md](../formats/store/retention-recovery.md), linked from the namespace page and protocol index; its grammar and boundaries are preserved. The existing documentation guards read the new owner explicitly and remain documentation checks. Obsolete inline line-length overrides were removed from the touched pages so the accepted global line-length policy governs prose. A complete copy of the changed source and documents passed both `cargo test --workspace --all-features` and its release counterpart, including the OS-boundary regressions. All-feature Clippy, source structure, formatting, and Markdown passed. A preliminary complete-copy run failed Git's ownership check after archive extraction changed the owned sandbox directory's UID; correcting that sandbox ownership, without global Git exceptions or source changes, allowed the controlled run. Physical power loss, kernel I/O error injection, and the remaining PR review queue are still not certified by these results. ## Namespace refusal before recovery effects diff --git a/src/adapters/retained_catalog_publication.rs b/src/adapters/retained_catalog_publication.rs index 171b9f95..c4337be1 100644 --- a/src/adapters/retained_catalog_publication.rs +++ b/src/adapters/retained_catalog_publication.rs @@ -6,7 +6,7 @@ //! then verifies each root's closure against the candidate before the first //! catalog mutation. It confirms the retention head is unchanged again //! immediately before `HEAD` replacement. The decision and its crash states are -//! recorded in `docs/formats/segment-store-v2/rationale.md`. +//! recorded in `docs/formats/store/rationale.md`. mod preserved_retention_manifest; mod retained_catalog_publication_error; diff --git a/src/adapters/store_initialization.rs b/src/adapters/store_initialization.rs index 11094649..21b24e1e 100644 --- a/src/adapters/store_initialization.rs +++ b/src/adapters/store_initialization.rs @@ -5,7 +5,7 @@ //! that a later call resumes, every other nonempty root is refused before any //! mutation, and the store admits only after one named durable commit point. //! The protocol, its states, and its crash matrix are specified in -//! `docs/formats/segment-store-v2/initialization.md`. +//! `docs/formats/store/initialization.md`. mod initialization_entries; mod initialization_error; diff --git a/src/adapters/store_initialization/initialization_execution.rs b/src/adapters/store_initialization/initialization_execution.rs index b148886b..2f6cb0f5 100644 --- a/src/adapters/store_initialization/initialization_execution.rs +++ b/src/adapters/store_initialization/initialization_execution.rs @@ -1,7 +1,7 @@ //! This module owns the ordered execution of direct initialization from one observed prefix. //! //! Forward protocol, crash states, and recovery are specified in -//! `docs/formats/segment-store-v2/initialization.md`. Every phase runs +//! `docs/formats/store/initialization.md`. Every phase runs //! between two observer notifications, so the crash matrix kills the process //! at an exact boundary of this code rather than of a reimplementation. diff --git a/src/adapters/store_initialization/initialization_prefix.rs b/src/adapters/store_initialization/initialization_prefix.rs index 68be66fc..056fed7b 100644 --- a/src/adapters/store_initialization/initialization_prefix.rs +++ b/src/adapters/store_initialization/initialization_prefix.rs @@ -5,7 +5,7 @@ use super::StoreInitializationDirectory; /// One recoverable initialization state, observed under writer authority. /// /// Each variant is exactly one forward state of the protocol in -/// `docs/formats/segment-store-v2/initialization.md`. A root in any other +/// `docs/formats/store/initialization.md`. A root in any other /// state is refused, and a committed root is refused as already initialized, /// so no variant describes a committed store. #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/xtask/tests/durability_crash_documentation.rs b/xtask/tests/durability_crash_documentation.rs index e813d213..3a1ee0dd 100644 --- a/xtask/tests/durability_crash_documentation.rs +++ b/xtask/tests/durability_crash_documentation.rs @@ -3,12 +3,13 @@ #![cfg(feature = "repository-tasks")] const ROOT_README: &str = include_str!("../../README.md"); -const RECOVERY: &str = include_str!("../../docs/formats/segment-store-v1/recovery.md"); -const REQUIREMENTS: &str = include_str!("../../docs/formats/segment-store-v1/requirements.md"); +const RECOVERY: &str = include_str!("../../docs/formats/store/stage-recovery.md"); +const REQUIREMENTS: &str = + include_str!("../../docs/formats/store/catalog-protocol-requirements.md"); const CORPUS_README: &str = include_str!("../../conformance/segment-store/v1/README.md"); const V2_CORPUS_README: &str = include_str!("../../conformance/segment-store/v2/README.md"); -const INITIALIZATION: &str = include_str!("../../docs/formats/segment-store-v2/initialization.md"); -const V2_REQUIREMENTS: &str = include_str!("../../docs/formats/segment-store-v2/requirements.md"); +const INITIALIZATION: &str = include_str!("../../docs/formats/store/initialization.md"); +const V2_REQUIREMENTS: &str = include_str!("../../docs/formats/store/requirements.md"); #[test] fn living_documentation_routes_the_complete_crash_matrix_and_its_limits() { diff --git a/xtask/tests/layout_format_contract.rs b/xtask/tests/layout_format_contract.rs index 71ccd526..bd33ce00 100644 --- a/xtask/tests/layout_format_contract.rs +++ b/xtask/tests/layout_format_contract.rs @@ -90,10 +90,13 @@ fn format_registry_reports_flat_layout_as_implemented() { #[test] fn format_registry_reports_the_segment_store_implementation_boundary() { const EXPECTED_ROW: &str = "\ -| [Durable Segment Store v1](segment-store-v1/README.md) | \ -`keep.segment-store/v1` | Implemented through initialization, publication, \ -restart, and recovery in issues #14–#17 | \ -[Golden corpus](../../conformance/segment-store/v1/README.md) |"; +| [Keep store format](store/README.md) | `keep.segment-store/v2` | \ +The one store format: direct initialization and reopen, the segment, catalog, \ +publication, and stage-recovery protocol of issues #14–#17, the gated catalog \ +writer, and retention publication and recovery implemented; collection planned \ +in issue #21 | \ +[Segment, catalog, and head corpus](../../conformance/segment-store/v1/README.md) \ +· [Store corpus](../../conformance/segment-store/v2/README.md) |"; assert!( FORMAT_REGISTRY.lines().any(|line| line == EXPECTED_ROW), diff --git a/xtask/tests/retention_store_v2_protocol_contract.rs b/xtask/tests/retention_store_v2_protocol_contract.rs index d923da6b..8433a1e7 100644 --- a/xtask/tests/retention_store_v2_protocol_contract.rs +++ b/xtask/tests/retention_store_v2_protocol_contract.rs @@ -15,7 +15,7 @@ use std::fs; use std::io; use std::path::{Path, PathBuf}; -const FORMAT_ROOT: &str = "docs/formats/segment-store-v2"; +const FORMAT_ROOT: &str = "docs/formats/store"; const DOCUMENT_REVIEW_LIMIT_LINES: usize = 300; fn repository_root() -> Result { @@ -40,7 +40,7 @@ fn version_two_is_one_routed_protocol() -> Result<(), Box let overview = normalized(&read(&format!("{FORMAT_ROOT}/README.md"))?); assert!( - format_index.contains("segment-store-v2/README.md"), + format_index.contains("store/README.md"), "format index does not route to segment-store v2" ); assert!( diff --git a/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs index 22ca52fd..e3be9b25 100644 --- a/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs +++ b/xtask/tests/retention_store_v2_protocol_contract/parser_fuzz_laws.rs @@ -5,7 +5,7 @@ use std::path::Path; const FUZZ_MANIFEST: &str = include_str!("../../../fuzz/Cargo.toml"); const FUZZ_GUIDE: &str = include_str!("../../../fuzz/README.md"); -const REQUIREMENTS: &str = include_str!("../../../docs/formats/segment-store-v2/requirements.md"); +const REQUIREMENTS: &str = include_str!("../../../docs/formats/store/requirements.md"); #[test] fn retention_decoders_have_registered_seeded_fuzz_evidence() -> Result<(), Box> { diff --git a/xtask/tests/segment_store_implementation_documentation.rs b/xtask/tests/segment_store_implementation_documentation.rs index 991faba0..b9092ee0 100644 --- a/xtask/tests/segment_store_implementation_documentation.rs +++ b/xtask/tests/segment_store_implementation_documentation.rs @@ -2,10 +2,11 @@ const ROOT_README: &str = include_str!("../../README.md"); const FORMAT_REGISTRY: &str = include_str!("../../docs/formats/README.md"); -const FORMAT_README: &str = include_str!("../../docs/formats/segment-store-v1/README.md"); -const REQUIREMENTS: &str = include_str!("../../docs/formats/segment-store-v1/requirements.md"); -const PUBLICATION: &str = include_str!("../../docs/formats/segment-store-v1/publication.md"); -const RECOVERY: &str = include_str!("../../docs/formats/segment-store-v1/recovery.md"); +const FORMAT_README: &str = include_str!("../../docs/formats/store/README.md"); +const REQUIREMENTS: &str = + include_str!("../../docs/formats/store/catalog-protocol-requirements.md"); +const PUBLICATION: &str = include_str!("../../docs/formats/store/publication.md"); +const RECOVERY: &str = include_str!("../../docs/formats/store/stage-recovery.md"); const CORPUS_README: &str = include_str!("../../conformance/segment-store/v1/README.md"); #[test] @@ -23,7 +24,7 @@ fn living_documentation_names_the_implemented_segment_boundary() { (ROOT_README, "`AdmittedSegment`"), ( FORMAT_REGISTRY, - "Implemented through initialization, publication, restart, and recovery in issues #14–#17", + "the segment, catalog, publication, and stage-recovery protocol of issues #14–#17", ), ( FORMAT_README, diff --git a/xtask/tests/segment_store_protocol_contract.rs b/xtask/tests/segment_store_protocol_contract.rs index f9334de2..569abefa 100644 --- a/xtask/tests/segment_store_protocol_contract.rs +++ b/xtask/tests/segment_store_protocol_contract.rs @@ -19,16 +19,16 @@ mod transition_laws; const ADR_INDEX: &str = include_str!("../../docs/adr/README.md"); const ADR: &str = include_str!("../../docs/adr/0005-durable-segment-store-protocol.md"); -const SPECIFICATION_INDEX: &str = include_str!("../../docs/formats/segment-store-v1/README.md"); +const SPECIFICATION_INDEX: &str = include_str!("../../docs/formats/store/README.md"); const SPECIFICATION: &str = concat!( - include_str!("../../docs/formats/segment-store-v1/README.md"), - include_str!("../../docs/formats/segment-store-v1/segment.md"), - include_str!("../../docs/formats/segment-store-v1/catalog.md"), - include_str!("../../docs/formats/segment-store-v1/publication.md"), - include_str!("../../docs/formats/segment-store-v1/recovery.md"), - include_str!("../../docs/formats/segment-store-v1/requirements.md"), + include_str!("../../docs/formats/store/README.md"), + include_str!("../../docs/formats/store/segment.md"), + include_str!("../../docs/formats/store/catalog.md"), + include_str!("../../docs/formats/store/publication.md"), + include_str!("../../docs/formats/store/stage-recovery.md"), + include_str!("../../docs/formats/store/catalog-protocol-requirements.md"), ); -const RATIONALE: &str = include_str!("../../docs/formats/segment-store-v1/rationale.md"); +const RATIONALE: &str = include_str!("../../docs/formats/store/catalog-protocol-rationale.md"); const CONFORMANCE_GUIDE: &str = include_str!("../../conformance/segment-store/v1/README.md"); const CONFORMANCE_ORIGIN: &str = include_str!("../../conformance/segment-store/v1/ORIGIN.md"); const TRANSITIONS: &str = include_str!("../../conformance/segment-store/v1/transitions.tsv"); @@ -36,13 +36,13 @@ const CONTRACT_SOURCE: &str = include_str!("segment_store_protocol_contract.rs") const REQUIRED_PROTOCOL_PAGES: &[&str] = &[ "docs/adr/0005-durable-segment-store-protocol.md", - "docs/formats/segment-store-v1/README.md", - "docs/formats/segment-store-v1/segment.md", - "docs/formats/segment-store-v1/catalog.md", - "docs/formats/segment-store-v1/publication.md", - "docs/formats/segment-store-v1/recovery.md", - "docs/formats/segment-store-v1/requirements.md", - "docs/formats/segment-store-v1/rationale.md", + "docs/formats/store/README.md", + "docs/formats/store/segment.md", + "docs/formats/store/catalog.md", + "docs/formats/store/publication.md", + "docs/formats/store/stage-recovery.md", + "docs/formats/store/catalog-protocol-requirements.md", + "docs/formats/store/catalog-protocol-rationale.md", "conformance/segment-store/v1/README.md", "conformance/segment-store/v1/ORIGIN.md", ]; diff --git a/xtask/tests/segment_store_protocol_contract/documentation_laws.rs b/xtask/tests/segment_store_protocol_contract/documentation_laws.rs index 6d261540..c977df12 100644 --- a/xtask/tests/segment_store_protocol_contract/documentation_laws.rs +++ b/xtask/tests/segment_store_protocol_contract/documentation_laws.rs @@ -53,8 +53,8 @@ fn protocol_index_routes_each_semantic_owner() { "[Segment bytes](segment.md)", "[Catalog and publication-head bytes](catalog.md)", "[Publication and reader visibility](publication.md)", - "[Recovery and platform contract](recovery.md)", - "[Requirements, compatibility, and evidence](requirements.md)", + "[Catalog-protocol stage recovery and platform contract](stage-recovery.md)", + "[Catalog-protocol requirements, compatibility, and evidence](catalog-protocol-requirements.md)", ] { assert!( SPECIFICATION_INDEX.contains(route), diff --git a/xtask/tests/segment_store_protocol_contract/parser_fuzz_laws.rs b/xtask/tests/segment_store_protocol_contract/parser_fuzz_laws.rs index 10602b99..99be0bd6 100644 --- a/xtask/tests/segment_store_protocol_contract/parser_fuzz_laws.rs +++ b/xtask/tests/segment_store_protocol_contract/parser_fuzz_laws.rs @@ -5,7 +5,8 @@ use std::path::Path; const FUZZ_MANIFEST: &str = include_str!("../../../fuzz/Cargo.toml"); const FUZZ_GUIDE: &str = include_str!("../../../fuzz/README.md"); -const REQUIREMENTS: &str = include_str!("../../../docs/formats/segment-store-v1/requirements.md"); +const REQUIREMENTS: &str = + include_str!("../../../docs/formats/store/catalog-protocol-requirements.md"); #[test] fn catalog_and_head_decoders_have_registered_seeded_fuzz_evidence() -> Result<(), Box> { diff --git a/xtask/tests/segment_store_protocol_contract/publication_laws.rs b/xtask/tests/segment_store_protocol_contract/publication_laws.rs index 8193b979..d7ea8447 100644 --- a/xtask/tests/segment_store_protocol_contract/publication_laws.rs +++ b/xtask/tests/segment_store_protocol_contract/publication_laws.rs @@ -34,7 +34,7 @@ fn immutable_pool_links_are_verified_after_namespace_resolution() { fn store_initialization_routes_to_the_single_format_protocol() { for required in [ "## Store initialization", - "(../segment-store-v2/initialization.md)", + "(initialization.md)", "`KEEP-CRASH-081`–`KEEP-CRASH-091`", "`KEEP-CRASH-031`–`KEEP-CRASH-035`; the", ] { From ee5d60aedd071e495d24e8520606365779a71a9e Mon Sep 17 00:00:00 2001 From: James Ross Date: Tue, 27 Oct 2026 10:55:46 -0700 Subject: [PATCH 22/36] Test: require every initialized root entry in name classification (#198) The deleted cross-version law showed that classification refuses a former version-1 inventory as missing reader.lock. Generalize it: removing any one of the nine root entries of an initialized store refuses as exactly that missing required entry, so a classifier that stops requiring FORMAT or the reader fence is caught. --- .../refusal_laws.rs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/recovery_name_classification/refusal_laws.rs b/tests/recovery_name_classification/refusal_laws.rs index edf6290e..a78385ff 100644 --- a/tests/recovery_name_classification/refusal_laws.rs +++ b/tests/recovery_name_classification/refusal_laws.rs @@ -99,3 +99,44 @@ fn simultaneous_fixed_stages_are_unrecoverable_ambiguity() -> Result<(), Box Result<(), Box> { + let required = [ + RecoveryRequiredEntry::WriterLock, + RecoveryRequiredEntry::StagingDirectory, + RecoveryRequiredEntry::SegmentPoolDirectory, + RecoveryRequiredEntry::CatalogPoolDirectory, + RecoveryRequiredEntry::ReaderFence, + RecoveryRequiredEntry::FormatMarker, + RecoveryRequiredEntry::RetentionDirectory, + RecoveryRequiredEntry::GcDirectory, + RecoveryRequiredEntry::RecoveryDirectory, + ]; + for (missing, expected) in INITIALIZED_ROOT.iter().zip(required) { + let present: Vec<&str> = INITIALIZED_ROOT + .iter() + .copied() + .filter(|entry| entry != missing) + .collect(); + let Err(error) = classify_recovery_names(inventory([ + names(&present)?, + Vec::new(), + Vec::new(), + Vec::new(), + ])?) else { + return Err(format!("classification admitted a root without {missing}").into()); + }; + assert!( + matches!( + error, + RecoveryNameClassificationError::Missing { required } if required == expected + ), + "a root without {missing} refused as {error:?}" + ); + } + Ok(()) +} From 9a44159ce382a37d411cef99a243de4626e0e41b Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 4 Nov 2026 17:34:15 -0800 Subject: [PATCH 23/36] Feat!: require an expected-state choice to open a store (#198) ADR-0010 removes the persisted root binding, which never detected a swapped or rolled-back store. Each public open now takes a StoreExpectation with no default: an exact catalog refuses anything else with expected and observed values, and NoExpectation states the absence of a check at the call site. Enumerated laws cover the two copies ADR-0010 requires before migration deletion: every mix of two published states and every partly copied file refuses or returns exact bytes, and a restored older snapshot refuses newer expected state. --- CHANGELOG.md | 2 + README.md | 4 +- docs/adr/0010-one-storage-format.md | 4 +- .../store/catalog-protocol-requirements.md | 2 + docs/formats/store/recovery.md | 21 +- docs/formats/store/requirements.md | 5 +- src/adapters/catalog_restart_loader.rs | 48 +++- src/adapters/durable/snapshot.rs | 9 +- src/adapters/durable/store.rs | 15 +- src/adapters/exports.rs | 1 + src/adapters/filesystem_store_admission.rs | 44 +++- .../filesystem_store_admission_error.rs | 18 +- src/adapters/mod.rs | 1 + .../retention/durable_read_law_tests.rs | 1 + .../retention/durable_view_law_tests.rs | 1 + ...esystem_retention_recovery_prefix_tests.rs | 7 +- .../filesystem_retention_snapshot.rs | 30 ++- .../filesystem_retention_snapshot_error.rs | 9 +- ...lesystem_retention_snapshot_error_tests.rs | 8 +- .../filesystem_retention_snapshot_tests.rs | 36 ++- .../filesystem_retention_verification.rs | 14 +- .../filesystem_store_admission_tests.rs | 10 +- .../reader_platform_authority_tests.rs | 8 +- .../retention/reader_platform_law_tests.rs | 23 +- .../retention/retention_model_tests.rs | 1 + .../verification_selection_law_tests.rs | 8 +- src/adapters/store_expectation.rs | 95 ++++++++ src/lib.rs | 2 + tests/catalog_writer/format_marker_laws.rs | 20 +- tests/catalog_writer/forward_laws.rs | 1 + tests/catalog_writer/writer_fixture.rs | 1 + tests/darwin_fifo_refusal.rs | 8 +- tests/darwin_volume_reattach.rs | 1 + .../durable_assertions.rs | 14 +- .../durable_closure_refusal.rs | 7 +- .../durable_corruption_laws.rs | 14 +- .../durable_diagnostic_laws.rs | 15 +- .../durable_layout_laws.rs | 54 ++++- .../durable_locator_laws.rs | 14 +- .../durable_namespace_laws.rs | 22 +- .../durable_output_laws.rs | 54 ++++- .../durable_range_properties.rs | 36 ++- .../durable_read_memory.rs | 9 +- .../durable_refusal_laws.rs | 45 +++- .../durable_writer_failures.rs | 36 ++- tests/reader_fence_process/reader.rs | 7 +- tests/single_format/acceptance_laws.rs | 12 +- tests/single_format/expectation_laws.rs | 208 ++++++++++++++++++ tests/single_format/initialization_laws.rs | 11 +- tests/single_format/portability_laws.rs | 13 +- tests/single_format/suite.rs | 4 + tests/single_format/torn_copy_laws.rs | 165 ++++++++++++++ tests/support/populated_store.rs | 7 +- .../restart/initialization.rs | 9 +- .../restart/retained_catalog.rs | 9 +- .../restart/retention_snapshot.rs | 9 +- 56 files changed, 1070 insertions(+), 162 deletions(-) create mode 100644 src/adapters/store_expectation.rs create mode 100644 tests/single_format/expectation_laws.rs create mode 100644 tests/single_format/torn_copy_laws.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ff923e8..7c6a4e72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ after its public API and format compatibility policies are established. ## [Unreleased] +- Opening a store requires a typed choice: `FilesystemStoreAdmission::reopen`, `FilesystemRetentionSnapshot::load`, `DurableSnapshot::open`, and `DurableStore::open` take a `StoreExpectation` with no default. `Catalog` refuses any other published catalog with the expected and observed values, and `NoExpectation` accepts any valid store at the path. Enumerated laws show that a copy torn by a concurrent writer refuses or returns exact bytes. + - Stop naming Cargo workspace members as Dependabot update directories. Dependabot refuses to update from inside a member and the job for the workspace root already updates member manifests, so those entries failed the scheduled run without adding coverage. The dependency policy check now reads the root workspace member list, refuses a member listed as an update directory, and still requires an entry for every standalone manifest. - Admit macOS APFS as a production storage platform, on both default and case-sensitive volumes. Keep requires every existing protocol name byte-for-byte in a listing of its parent, so `head` never answers for `HEAD`. Adds a native macOS CI job; no format, dependency, or unsafe change. The crash matrix passes natively, but no power-loss experiment has been run on APFS. diff --git a/README.md b/README.md index 5147af05..f02a79ae 100644 --- a/README.md +++ b/README.md @@ -212,14 +212,14 @@ fn copy_retained_blob( ) -> Result> { use keep::{ CatalogRestartByteLimit, CatalogRestartPolicy, DurableStore, LayoutEntryLimit, - ReaderAttemptLimit, SegmentReadPolicy, SegmentRecordLimit, + ReaderAttemptLimit, SegmentReadPolicy, SegmentRecordLimit, StoreExpectation, }; // Limit catalog-selected segment bytes; catalog and metadata allocate separately. let policy = CatalogRestartPolicy::new( SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), CatalogRestartByteLimit::new(16_777_216)?, ); - let store = DurableStore::open(root, policy, ReaderAttemptLimit::DEFAULT)?; + let store = DurableStore::open(root, policy, ReaderAttemptLimit::DEFAULT, StoreExpectation::NoExpectation)?; let snapshot = store.snapshot()?; Ok(snapshot.reconstruct(target, output)?) } diff --git a/docs/adr/0010-one-storage-format.md b/docs/adr/0010-one-storage-format.md index d39fdd82..3ff48125 100644 --- a/docs/adr/0010-one-storage-format.md +++ b/docs/adr/0010-one-storage-format.md @@ -153,7 +153,9 @@ Before the first release, opening a store therefore requires a typed choice: the caller passes the state it expects, and a mismatch refuses, or the caller explicitly states that it has no expectation. Neither is a default, so removing the binding never silently removes the question. Keep does not store the -expectation. +expectation. An expected state names an exact catalog generation and digest, so it +does not detect a rollback that leaves the catalog unchanged. Recovery and +verification take no expectation, because they examine whatever state is there. Keep persists no store identifier. The only one, the content-derived identifier in the migration records, is deleted with migration, and diff --git a/docs/formats/store/catalog-protocol-requirements.md b/docs/formats/store/catalog-protocol-requirements.md index 6288f04f..1d68aa53 100644 --- a/docs/formats/store/catalog-protocol-requirements.md +++ b/docs/formats/store/catalog-protocol-requirements.md @@ -32,6 +32,8 @@ evidence](requirements.md). | `KEEP-STORE-018` | Truncated-stage discard has explicit unlink and directory-sync crash boundaries | `KEEP-CRASH-027`–`028` | Specified in #14 | | `KEEP-STORE-019` | Leftover next-head recovery either finalizes an exact successor or discards explicit evidence durably | `KEEP-CRASH-025`–`026`, `029`–`030` | Specified in #14 | | `KEEP-STORE-021` | Explicit recovery completes a durable fixed-name stage into its immutable pool without publishing a head | `KEEP-CRASH-008`–`012`, `016`–`020` | Specified in #14 | +| `KEEP-STORE-022` | Opening a store requires a typed expectation choice with no default; an expected catalog refuses any other published catalog, or none, with the expected and observed values | `tests/single_format/expectation_laws.rs` | Implemented in #198 | +| `KEEP-STORE-023` | Every mix of two published states and every partly copied file refuses or returns exactly the bytes an identity names | `tests/single_format/torn_copy_laws.rs` | Implemented in #198 | diff --git a/docs/formats/store/recovery.md b/docs/formats/store/recovery.md index b6a3b1da..8cff195e 100644 --- a/docs/formats/store/recovery.md +++ b/docs/formats/store/recovery.md @@ -114,14 +114,29 @@ Live authority is unchanged: applies, so a corrupt or incomplete copy refuses exactly as the original would. +A copy taken while a writer publishes is outside this guarantee, but it cannot make +Keep return other bytes. It may hold any mix of two published states and a partly +copied file, and every such mix refuses or returns exactly the bytes an identity +names. It need not open at all, and it can lose the newer state. + A copy and its original are two stores. Each takes its own locks, and their histories diverge independently while unchanged content keeps its identity. Copies are not replication and are never coordinated. -Admission does not detect an unintended but valid store, or an older valid +Admission alone does not detect an unintended but valid store, or an older valid state of the intended one. Detecting either needs an expectation the caller -supplies and keeps outside the checked state, such as an exact catalog -generation and digest. Keep does not claim either check. +supplies and keeps outside the checked state. `FilesystemStoreAdmission::reopen`, +`FilesystemRetentionSnapshot::load`, `DurableSnapshot::open`, and +`DurableStore::open` therefore require a `StoreExpectation`, with no default: + +- `Catalog` names an exact catalog generation and digest, and any other published + catalog, or none, refuses with the expected and observed values; and +- `NoExpectation` accepts any valid store at the path, and says so at the call site. + +The writer compares under the writer lock and the reader compares its collected +view under the fence. `Catalog` does not detect a rollback that leaves the catalog +unchanged, such as restored retention state alone. Recovery and verification opens +take no expectation: they examine whatever state the path holds. ### Root identity across restart diff --git a/docs/formats/store/requirements.md b/docs/formats/store/requirements.md index fcef0962..60b50ec2 100644 --- a/docs/formats/store/requirements.md +++ b/docs/formats/store/requirements.md @@ -104,7 +104,8 @@ evidence and states where its obligation went. - Publication binds this store's catalog `HEAD` to the verified closure and reopens the head-selected catalog and its segments under authority. Forward publication and recovery both replay the bounded live closure before effects. -- Admission does not detect an unintended but valid store or an older valid - state; a copy of a store is a second, independent store. +- Admission alone does not detect an unintended but valid store or an older + valid state, so every open requires an explicit expectation choice; a copy of + a store is a second, independent store. - Benchmarks are required before performance-sensitive retention or initialization optimization. diff --git a/src/adapters/catalog_restart_loader.rs b/src/adapters/catalog_restart_loader.rs index d6c07d37..ebefe7e1 100644 --- a/src/adapters/catalog_restart_loader.rs +++ b/src/adapters/catalog_restart_loader.rs @@ -6,9 +6,9 @@ use crate::CatalogLength; use super::{ CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, - ChecksummedCatalog, ChecksummedPublicationHead, FilesystemCatalogSnapshot, catalog_restart_io, - catalog_restart_segments, filesystem_store_admission, filesystem_store_namespace, - physical_pool_name, + ChecksummedCatalog, ChecksummedPublicationHead, ExpectedCatalog, FilesystemCatalogSnapshot, + catalog_restart_io, catalog_restart_segments, filesystem_store_admission, + filesystem_store_namespace, physical_pool_name, }; const HEAD_NAME: &str = "HEAD"; @@ -36,11 +36,35 @@ pub(super) fn load( load_from_directory(&directory, HEAD_NAME, policy) } -pub(super) fn load_from_directory( +/// Reads the published catalog's coordinates from `HEAD`, or `None` when none is published. +/// +/// The checksummed head names the catalog; this neither opens nor verifies it. +pub(super) fn observe_published_catalog( + directory: &crate::adapters::filesystem_directory::FilesystemDirectory, +) -> Result, CatalogRestartError> { + match directory.symlink_metadata(HEAD_NAME) { + Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(source) => { + return Err(CatalogRestartError::io( + CatalogRestartPhase::OpenHead, + source, + )); + } + Ok(_) => {} + } + let head_bytes = read_head_bytes(directory, HEAD_NAME)?; + let head = ChecksummedPublicationHead::decode(&head_bytes) + .map_err(|source| CatalogRestartError::Head { source })?; + Ok(Some(ExpectedCatalog::new( + head.generation(), + head.catalog_digest(), + ))) +} + +fn read_head_bytes( directory: &crate::adapters::filesystem_directory::FilesystemDirectory, head_name: &str, - policy: CatalogRestartPolicy, -) -> Result { +) -> Result, CatalogRestartError> { let (head_file, observed_head_length) = catalog_restart_io::open_regular( directory, head_name, @@ -55,12 +79,20 @@ pub(super) fn load_from_directory( observed: observed_head_length, }); } - let head_bytes = catalog_restart_io::read_exact( + catalog_restart_io::read_exact( head_file, CatalogRestartArtifact::Head, CatalogRestartPhase::ReadHead, HEAD_LENGTH, - )?; + ) +} + +pub(super) fn load_from_directory( + directory: &crate::adapters::filesystem_directory::FilesystemDirectory, + head_name: &str, + policy: CatalogRestartPolicy, +) -> Result { + let head_bytes = read_head_bytes(directory, head_name)?; let head = ChecksummedPublicationHead::decode(&head_bytes) .map_err(|source| CatalogRestartError::Head { source })?; let catalogs = directory diff --git a/src/adapters/durable/snapshot.rs b/src/adapters/durable/snapshot.rs index 6f28eb35..65785c31 100644 --- a/src/adapters/durable/snapshot.rs +++ b/src/adapters/durable/snapshot.rs @@ -9,7 +9,7 @@ use super::{ }; use crate::adapters::{ AdmittedSegmentRecord, CatalogRestartPolicy, CatalogSnapshot, FilesystemRetentionSnapshot, - LayoutDecodePolicy, ReaderAttemptLimit, SegmentRecordIdentity, + LayoutDecodePolicy, ReaderAttemptLimit, SegmentRecordIdentity, StoreExpectation, }; use crate::authenticated_read::{ChunkSource, read_admitted, reconstruct_admitted}; use crate::{AdmittedLayout, BlobId, ByteRange, ChunkId, LayoutId}; @@ -91,14 +91,15 @@ impl DurableSnapshot { /// /// # Errors /// - /// Returns [`DurableStoreError`] at the exact admission, fence, - /// collection, or retained-root refusal. + /// Returns [`DurableStoreError`] at the exact admission, fence, collection, + /// expectation, or retained-root refusal. pub fn open( store_root: &Path, policy: CatalogRestartPolicy, limit: ReaderAttemptLimit, + expectation: StoreExpectation, ) -> Result { - let view = FilesystemRetentionSnapshot::load(store_root, policy, limit) + let view = FilesystemRetentionSnapshot::load(store_root, policy, limit, expectation) .map_err(|source| DurableStoreError::Snapshot(Box::new(source)))?; let coordinates = DurableView::new( view.catalog().generation(), diff --git a/src/adapters/durable/store.rs b/src/adapters/durable/store.rs index 065b3c5a..cae28c9d 100644 --- a/src/adapters/durable/store.rs +++ b/src/adapters/durable/store.rs @@ -8,7 +8,7 @@ use super::{ DurableRangeReadReceipt, DurableReadError, DurableReconstructionReceipt, DurableSnapshot, DurableStoreError, }; -use crate::adapters::{CatalogRestartPolicy, ReaderAttemptLimit}; +use crate::adapters::{CatalogRestartPolicy, ReaderAttemptLimit, StoreExpectation}; use crate::{BlobId, ByteRange, LayoutId}; /// One committed store to read from. @@ -42,14 +42,14 @@ use crate::{BlobId, ByteRange, LayoutId}; /// ) -> Result> { /// use keep::{ /// CatalogRestartByteLimit, CatalogRestartPolicy, DurableStore, LayoutEntryLimit, -/// ReaderAttemptLimit, SegmentReadPolicy, SegmentRecordLimit, +/// ReaderAttemptLimit, SegmentReadPolicy, SegmentRecordLimit, StoreExpectation, /// }; /// // Limit catalog-selected segment bytes; catalog and metadata allocate separately. /// let policy = CatalogRestartPolicy::new( /// SegmentReadPolicy::new(SegmentRecordLimit::MAXIMUM, LayoutEntryLimit::MAXIMUM), /// CatalogRestartByteLimit::new(16_777_216)?, /// ); -/// let store = DurableStore::open(root, policy, ReaderAttemptLimit::DEFAULT)?; +/// let store = DurableStore::open(root, policy, ReaderAttemptLimit::DEFAULT, StoreExpectation::NoExpectation)?; /// let snapshot = store.snapshot()?; /// Ok(snapshot.reconstruct(target, output)?) /// } @@ -60,6 +60,7 @@ pub struct DurableStore { root: PathBuf, policy: CatalogRestartPolicy, limit: ReaderAttemptLimit, + expectation: StoreExpectation, } impl DurableStore { @@ -70,7 +71,9 @@ impl DurableStore { /// relative path. It does not open the store or prove that it exists. /// Later working-directory changes cannot retarget this handle. The path /// remains a locator, not a content identity or a pinned directory handle; - /// snapshot admission still checks the named store on every call. + /// snapshot admission still checks the named store on every call, including + /// `expectation`, so a handle expecting one catalog refuses once the store + /// publishes another. /// /// # Errors /// @@ -81,6 +84,7 @@ impl DurableStore { root: &Path, policy: CatalogRestartPolicy, limit: ReaderAttemptLimit, + expectation: StoreExpectation, ) -> Result { let root = std::path::absolute(root).map_err(|source| DurableStoreError::Locator { source })?; @@ -88,6 +92,7 @@ impl DurableStore { root, policy, limit, + expectation, }) } @@ -104,7 +109,7 @@ impl DurableStore { /// Returns [`DurableStoreError`] at the exact admission, fence, /// collection, or retained-root refusal. pub fn snapshot(&self) -> Result { - DurableSnapshot::open(&self.root, self.policy, self.limit) + DurableSnapshot::open(&self.root, self.policy, self.limit, self.expectation) } /// Whether some retained root anchors `target` in a fresh view. diff --git a/src/adapters/exports.rs b/src/adapters/exports.rs index 41d43000..5a6980bb 100644 --- a/src/adapters/exports.rs +++ b/src/adapters/exports.rs @@ -107,6 +107,7 @@ pub use super::segment_write_error::SegmentWriteError; pub use super::segment_write_phase::{SegmentDurabilityPhase, SegmentWritePhase}; pub use super::staged_segment::StagedSegment; pub use super::storage_profile_id_text_error::StorageProfileIdParseError; +pub use super::store_expectation::{ExpectedCatalog, StoreExpectation, StoreExpectationMismatch}; pub use super::store_format::*; #[cfg(any(test, feature = "repository-tasks"))] pub use super::store_initialization::StoreInitializationObserver; diff --git a/src/adapters/filesystem_store_admission.rs b/src/adapters/filesystem_store_admission.rs index 82a585ad..f654b552 100644 --- a/src/adapters/filesystem_store_admission.rs +++ b/src/adapters/filesystem_store_admission.rs @@ -10,9 +10,9 @@ use cap_std::ambient_authority; use super::StoreInitializationObserver; use super::store_initialization::{self as initialization, PhaseObserver}; use super::{ - FilesystemStoreAdmissionError, FilesystemWriterLock, StoreInitialization, - StoreInitializationError, filesystem_format_marker, filesystem_platform_profile, - filesystem_store_namespace, + FilesystemStoreAdmissionError, FilesystemWriterLock, StoreExpectation, StoreInitialization, + StoreInitializationError, catalog_restart_loader, filesystem_format_marker, + filesystem_platform_profile, filesystem_store_namespace, }; /// Exclusive writer authority over a committed store root. @@ -102,20 +102,28 @@ impl FilesystemStoreAdmission { /// The call mutates no protocol state. It admits the production platform /// for every protocol directory, acquires the existing writer /// lock, requires the exact committed root namespace, and admits the - /// canonical `FORMAT` marker. A store with no catalog `HEAD` admits. + /// canonical `FORMAT` marker. A store with no catalog `HEAD` admits under + /// [`StoreExpectation::NoExpectation`]. Any other `expectation` reads `HEAD` + /// under the lock and compares its generation and digest. /// Retention adapters perform content-level validation under the /// returned authority. The synchronous call may block on filesystem I/O. /// /// # Errors /// /// Returns [`FilesystemStoreAdmissionError`] with the exact platform, - /// writer-lock, namespace, or format-marker boundary and preserved source. + /// writer-lock, namespace, format-marker, or expectation boundary and + /// preserved source. /// A root whose initialization has not committed refuses at the namespace /// boundary. - pub fn reopen(store_root: &Path) -> Result { + pub fn reopen( + store_root: &Path, + expectation: StoreExpectation, + ) -> Result { let root = filesystem_platform_profile::open(store_root) .map_err(|source| FilesystemStoreAdmissionError::Platform { source })?; - Self::admit(root) + let admission = Self::admit(root)?; + admission.require_expectation(expectation)?; + Ok(admission) } /// Reopens a committed root without platform admission for repository tasks. @@ -137,6 +145,28 @@ impl FilesystemStoreAdmission { Self::admit(root) } + fn require_expectation( + &self, + expectation: StoreExpectation, + ) -> Result<(), FilesystemStoreAdmissionError> { + if expectation == StoreExpectation::NoExpectation { + return Ok(()); + } + let directory = self + .lock + .clone_directory() + .map_err(|source| FilesystemStoreAdmissionError::Namespace { source })?; + let observed = + catalog_restart_loader::observe_published_catalog(&directory).map_err(|source| { + FilesystemStoreAdmissionError::ExpectationHead { + source: Box::new(source), + } + })?; + expectation + .require(observed) + .map_err(|source| FilesystemStoreAdmissionError::Expectation { source }) + } + /// Releases the writer lock and the three pinned retention capabilities. pub(super) fn into_parts(self) -> (FilesystemWriterLock, Dir, Dir, Dir) { (self.lock, self.retention, self.roots, self.manifests) diff --git a/src/adapters/filesystem_store_admission_error.rs b/src/adapters/filesystem_store_admission_error.rs index 0910e41f..04ca81b7 100644 --- a/src/adapters/filesystem_store_admission_error.rs +++ b/src/adapters/filesystem_store_admission_error.rs @@ -4,7 +4,7 @@ use std::error::Error; use std::fmt; use std::io; -use super::WriterLockAcquireError; +use super::{CatalogRestartError, StoreExpectationMismatch, WriterLockAcquireError}; /// Failure to reacquire writer authority over one published filesystem store. #[derive(Debug)] @@ -30,6 +30,16 @@ pub enum FilesystemStoreAdmissionError { /// Preserved record-admission failure. source: io::Error, }, + /// The published `HEAD` needed to compare the caller's expectation refused. + ExpectationHead { + /// Preserved head read or decode failure. + source: Box, + }, + /// The store does not hold the catalog the caller expected. + Expectation { + /// The expected and observed catalogs. + source: StoreExpectationMismatch, + }, } impl fmt::Display for FilesystemStoreAdmissionError { @@ -39,6 +49,10 @@ impl fmt::Display for FilesystemStoreAdmissionError { Self::WriterLock { .. } => "published store writer-lock acquisition failed", Self::Namespace { .. } => "published store namespace admission failed", Self::FormatMarker { .. } => "format marker admission failed", + Self::ExpectationHead { .. } => { + "published head could not be compared with the expectation" + } + Self::Expectation { .. } => "the store does not hold the expected catalog", }) } } @@ -50,6 +64,8 @@ impl Error for FilesystemStoreAdmissionError { | Self::Namespace { source } | Self::FormatMarker { source } => Some(source), Self::WriterLock { source } => Some(source), + Self::ExpectationHead { source } => Some(source), + Self::Expectation { source } => Some(source), } } } diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index bc1d363c..a520465e 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -249,6 +249,7 @@ mod segment_write_phase; mod staged_segment; mod storage_profile_id_text; mod storage_profile_id_text_error; +mod store_expectation; mod store_format; mod store_initialization; mod sync_capable_directory; diff --git a/src/adapters/retention/durable_read_law_tests.rs b/src/adapters/retention/durable_read_law_tests.rs index fb33db05..6aa9c036 100644 --- a/src/adapters/retention/durable_read_law_tests.rs +++ b/src/adapters/retention/durable_read_law_tests.rs @@ -23,6 +23,7 @@ fn store(root: &std::path::Path) -> Result> { CatalogRestartByteLimit::new(1_048_576)?, ), ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, )?) } diff --git a/src/adapters/retention/durable_view_law_tests.rs b/src/adapters/retention/durable_view_law_tests.rs index 50ce576b..5f9cab28 100644 --- a/src/adapters/retention/durable_view_law_tests.rs +++ b/src/adapters/retention/durable_view_law_tests.rs @@ -30,6 +30,7 @@ fn snapshot(path: &std::path::Path) -> Result> { CatalogRestartByteLimit::new(1_048_576)?, ), ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, )?) } diff --git a/src/adapters/retention/filesystem_retention_recovery_prefix_tests.rs b/src/adapters/retention/filesystem_retention_recovery_prefix_tests.rs index 4abba79d..71bb1092 100644 --- a/src/adapters/retention/filesystem_retention_recovery_prefix_tests.rs +++ b/src/adapters/retention/filesystem_retention_recovery_prefix_tests.rs @@ -265,7 +265,12 @@ fn require_successor_view( SegmentReadPolicy::MAXIMUM, CatalogRestartByteLimit::new(1_048_576)?, ); - let view = FilesystemRetentionSnapshot::load(root, policy, ReaderAttemptLimit::DEFAULT)?; + let view = FilesystemRetentionSnapshot::load( + root, + policy, + ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, + )?; let expected_root = AdmittedRetentionRoot::decode(expected)?; let head = view .retention_head() diff --git a/src/adapters/retention/filesystem_retention_snapshot.rs b/src/adapters/retention/filesystem_retention_snapshot.rs index 39bcd82b..cdde597a 100644 --- a/src/adapters/retention/filesystem_retention_snapshot.rs +++ b/src/adapters/retention/filesystem_retention_snapshot.rs @@ -16,9 +16,9 @@ use crate::adapters::filesystem_exact_record::{ self as exact_record, ExactRecordError, ExactRecordRefusal, }; use crate::adapters::{ - CatalogRestartError, CatalogRestartPolicy, ChecksummedPublicationHead, - FilesystemCatalogSnapshot, filesystem_format_marker, filesystem_platform_profile, - filesystem_store_namespace, publication_head_decoder, + CatalogRestartError, CatalogRestartPolicy, ChecksummedPublicationHead, ExpectedCatalog, + FilesystemCatalogSnapshot, StoreExpectation, filesystem_format_marker, + filesystem_platform_profile, filesystem_store_namespace, publication_head_decoder, }; use crate::{RetentionHead, RetentionManifest, RetentionNamespaceDigest}; @@ -121,19 +121,21 @@ impl FilesystemRetentionSnapshot { /// fence is held. /// A retained or in-flight catalog `head.next` is admitted and ignored: the /// view selects only `HEAD`, so catalog publication runs beside readers. + /// `expectation` is compared with the collected view under the fence. /// /// # Errors /// /// Returns [`FilesystemRetentionSnapshotError`](super::FilesystemRetentionSnapshotError) - /// at the exact admission, fence, collection, or catalog refusal. + /// at the exact admission, fence, collection, catalog, or expectation refusal. /// A catalog admission result is returned only after both coordinate /// reads agree; a moving head discards that result and retries. pub fn load( store_root: &Path, policy: CatalogRestartPolicy, limit: ReaderAttemptLimit, + expectation: StoreExpectation, ) -> Result { - Self::load_with(store_root, policy, limit, |source, limit| { + Self::load_with(store_root, policy, limit, expectation, |source, limit| { collect_retention_view(source, limit) .map_err(|source| Error::View { source })? .map_err(|source| Error::Catalog { source }) @@ -144,6 +146,7 @@ impl FilesystemRetentionSnapshot { store_root: &Path, policy: CatalogRestartPolicy, limit: ReaderAttemptLimit, + expectation: StoreExpectation, collect: impl FnOnce(&mut Source, ReaderAttemptLimit) -> Result, ) -> Result { let root = filesystem_platform_profile::open(store_root) @@ -152,7 +155,7 @@ impl FilesystemRetentionSnapshot { .map_err(|source| Error::Admission { source })?; filesystem_format_marker::admit(&root).map_err(|source| Error::Admission { source })?; let fence = ReaderFence::acquire(&root).map_err(|source| Error::Fence { source })?; - require_published(&root)?; + require_published(&root, expectation)?; let retention = root .open_dir_nofollow(pool_name::RETENTION) .map_err(|source| Error::Admission { source })?; @@ -169,6 +172,12 @@ impl FilesystemRetentionSnapshot { policy, }; let view = collect(&mut source, limit)?; + expectation + .require(Some(ExpectedCatalog::new( + view.catalog.generation(), + view.catalog.catalog_digest(), + ))) + .map_err(|source| Error::Expectation { source })?; Ok(Self { _fence: fence, roots, @@ -321,10 +330,15 @@ fn selected_namespace_directory( /// /// Checked under the fence, after admission: absence of `HEAD` is a defined /// state of an initialized store, reported as such instead of as a failed read. -fn require_published(root: &Dir) -> Result<(), Error> { +fn require_published(root: &Dir, expectation: StoreExpectation) -> Result<(), Error> { match root.symlink_metadata(HEAD_NAME) { Ok(_) => Ok(()), - Err(source) if source.kind() == io::ErrorKind::NotFound => Err(Error::Unpublished), + Err(source) if source.kind() == io::ErrorKind::NotFound => { + expectation + .require(None) + .map_err(|source| Error::Expectation { source })?; + Err(Error::Unpublished) + } Err(source) => Err(Error::Admission { source }), } } diff --git a/src/adapters/retention/filesystem_retention_snapshot_error.rs b/src/adapters/retention/filesystem_retention_snapshot_error.rs index de9d1398..adb048b5 100644 --- a/src/adapters/retention/filesystem_retention_snapshot_error.rs +++ b/src/adapters/retention/filesystem_retention_snapshot_error.rs @@ -5,7 +5,7 @@ use std::fmt; use std::io; use super::RetentionViewError; -use crate::adapters::CatalogRestartError; +use crate::adapters::{CatalogRestartError, StoreExpectationMismatch}; /// Why a reader could not bind one consistent view. #[derive(Debug)] @@ -36,6 +36,11 @@ pub enum FilesystemRetentionSnapshotError { /// The exact restart refusal. source: CatalogRestartError, }, + /// The store does not hold the catalog the caller expected. + Expectation { + /// The expected and observed catalogs. + source: StoreExpectationMismatch, + }, /// A selected root pool entry is absent, unreadable, or not the manifest's. Root { /// The exact filesystem or decode refusal. @@ -52,6 +57,7 @@ impl fmt::Display for FilesystemRetentionSnapshotError { Self::View { .. } => "reader view collection refused", Self::Catalog { .. } => "catalog snapshot refused", Self::Root { .. } => "selected retention root refused", + Self::Expectation { .. } => "the store does not hold the expected catalog", }) } } @@ -64,6 +70,7 @@ impl Error for FilesystemRetentionSnapshotError { } Self::View { source } => Some(source), Self::Catalog { source } => Some(source), + Self::Expectation { source } => Some(source), Self::Unpublished => None, } } diff --git a/src/adapters/retention/filesystem_retention_snapshot_error_tests.rs b/src/adapters/retention/filesystem_retention_snapshot_error_tests.rs index 9a8a8fbd..4fd08a85 100644 --- a/src/adapters/retention/filesystem_retention_snapshot_error_tests.rs +++ b/src/adapters/retention/filesystem_retention_snapshot_error_tests.rs @@ -115,5 +115,11 @@ fn refusal(path: &Path) -> Result, Box> { SegmentReadPolicy::MAXIMUM, CatalogRestartByteLimit::new(1_048_576)?, ); - Ok(FilesystemRetentionSnapshot::load(path, policy, ReaderAttemptLimit::DEFAULT).err()) + Ok(FilesystemRetentionSnapshot::load( + path, + policy, + ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, + ) + .err()) } diff --git a/src/adapters/retention/filesystem_retention_snapshot_tests.rs b/src/adapters/retention/filesystem_retention_snapshot_tests.rs index 87bec53c..9685fd73 100644 --- a/src/adapters/retention/filesystem_retention_snapshot_tests.rs +++ b/src/adapters/retention/filesystem_retention_snapshot_tests.rs @@ -30,8 +30,12 @@ fn policy() -> Result> { fn an_initialized_store_snapshot_binds_the_catalog_and_no_retention_head() -> Result<(), Box> { let sandbox = initialized_store("filesystem-retention-snapshot-empty")?; - let snapshot = - FilesystemRetentionSnapshot::load(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + let snapshot = FilesystemRetentionSnapshot::load( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, + )?; assert_eq!(snapshot.catalog().generation().get(), 1); assert!(snapshot.retention_head().is_none()); assert!(snapshot.manifest().is_none()); @@ -48,8 +52,12 @@ fn a_published_generation_is_read_and_its_root_verified() -> Result<(), Box Result<(), Box Result fn a_non_empty_reader_lock_refuses_the_fence() -> Result<(), Box> { let sandbox = initialized_store("filesystem-retention-snapshot-bad-fence")?; fs::write(sandbox.path().join("reader.lock"), b"not empty")?; - let error = - FilesystemRetentionSnapshot::load(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT) - .err() - .ok_or("a non-empty reader.lock was accepted as the fence")?; + let error = FilesystemRetentionSnapshot::load( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, + ) + .err() + .ok_or("a non-empty reader.lock was accepted as the fence")?; assert!( matches!(&error, FilesystemRetentionSnapshotError::Fence { source } if source.kind() == std::io::ErrorKind::InvalidData), diff --git a/src/adapters/retention/filesystem_retention_verification.rs b/src/adapters/retention/filesystem_retention_verification.rs index 11062939..59d0557f 100644 --- a/src/adapters/retention/filesystem_retention_verification.rs +++ b/src/adapters/retention/filesystem_retention_verification.rs @@ -36,10 +36,16 @@ impl FilesystemRetentionSnapshot { policy: CatalogRestartPolicy, limit: ReaderAttemptLimit, ) -> Result { - Self::load_with(root, policy, limit, |source, limit| { - super::collect_verification_view(source, limit)? - .map_err(verification_ingress::catalog_error) - }) + Self::load_with( + root, + policy, + limit, + crate::StoreExpectation::NoExpectation, + |source, limit| { + super::collect_verification_view(source, limit)? + .map_err(verification_ingress::catalog_error) + }, + ) } /// Verifies the exact root selected for `namespace` in this fenced view. diff --git a/src/adapters/retention/filesystem_store_admission_tests.rs b/src/adapters/retention/filesystem_store_admission_tests.rs index b54313c5..97a5c73e 100644 --- a/src/adapters/retention/filesystem_store_admission_tests.rs +++ b/src/adapters/retention/filesystem_store_admission_tests.rs @@ -82,9 +82,10 @@ fn production_reopen_refuses_an_aliased_protocol_directory() -> Result<(), Box Result<(), Box Result<(), Box> { let sandbox = initialized_store("store-admission-production-exact")?; - let admission = FilesystemStoreAdmission::reopen(sandbox.path())?; + let admission = + FilesystemStoreAdmission::reopen(sandbox.path(), crate::StoreExpectation::NoExpectation)?; drop(admission); Ok(()) diff --git a/src/adapters/retention/reader_platform_authority_tests.rs b/src/adapters/retention/reader_platform_authority_tests.rs index 542ba7db..01985143 100644 --- a/src/adapters/retention/reader_platform_authority_tests.rs +++ b/src/adapters/retention/reader_platform_authority_tests.rs @@ -20,8 +20,12 @@ fn reader_platform_admission_does_not_acquire_writer_authority() -> Result<(), B CatalogRestartByteLimit::new(1_048_576)?, ); - let snapshot = - FilesystemRetentionSnapshot::load(store.path(), policy, ReaderAttemptLimit::DEFAULT)?; + let snapshot = FilesystemRetentionSnapshot::load( + store.path(), + policy, + ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, + )?; assert_eq!( snapshot.catalog().generation().get(), diff --git a/src/adapters/retention/reader_platform_law_tests.rs b/src/adapters/retention/reader_platform_law_tests.rs index e0da62d4..ca3c71ae 100644 --- a/src/adapters/retention/reader_platform_law_tests.rs +++ b/src/adapters/retention/reader_platform_law_tests.rs @@ -20,10 +20,14 @@ use crate::{ #[test] fn a_canonical_tmpfs_store_refuses_the_public_reader_profile() -> Result<(), Box> { let store = TmpfsStore::create("direct")?; - let refusal = - FilesystemRetentionSnapshot::load(&store.0, policy()?, ReaderAttemptLimit::DEFAULT) - .err() - .ok_or("valid store bytes admitted an unsupported reader filesystem")?; + let refusal = FilesystemRetentionSnapshot::load( + &store.0, + policy()?, + ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, + ) + .err() + .ok_or("valid store bytes admitted an unsupported reader filesystem")?; assert!( matches!(&refusal, FilesystemRetentionSnapshotError::Admission { source } if source.kind() == io::ErrorKind::Unsupported), @@ -35,9 +39,14 @@ fn a_canonical_tmpfs_store_refuses_the_public_reader_profile() -> Result<(), Box #[test] fn a_canonical_tmpfs_store_refuses_durable_snapshot_admission() -> Result<(), Box> { let store = TmpfsStore::create("durable")?; - let refusal = DurableSnapshot::open(&store.0, policy()?, ReaderAttemptLimit::DEFAULT) - .err() - .ok_or("durable snapshot accepted an unsupported filesystem")?; + let refusal = DurableSnapshot::open( + &store.0, + policy()?, + ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, + ) + .err() + .ok_or("durable snapshot accepted an unsupported filesystem")?; assert!( matches!(&refusal, DurableStoreError::Snapshot(source) if matches!(source.as_ref(), FilesystemRetentionSnapshotError::Admission { source } diff --git a/src/adapters/retention/retention_model_tests.rs b/src/adapters/retention/retention_model_tests.rs index 11f7f38a..bbf89c17 100644 --- a/src/adapters/retention/retention_model_tests.rs +++ b/src/adapters/retention/retention_model_tests.rs @@ -132,6 +132,7 @@ fn snapshot(store: &Store) -> Result &store.path, policy()?, ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, )?) } diff --git a/src/adapters/retention/verification_selection_law_tests.rs b/src/adapters/retention/verification_selection_law_tests.rs index 10001b40..4076155d 100644 --- a/src/adapters/retention/verification_selection_law_tests.rs +++ b/src/adapters/retention/verification_selection_law_tests.rs @@ -118,8 +118,12 @@ fn a_selected_root_coordinate_refusal_keeps_its_typed_evidence() -> Result<(), B replacement.encoded(), )?; let before = retention_witness(sandbox.path())?; - let view = - FilesystemRetentionSnapshot::load(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + let view = FilesystemRetentionSnapshot::load( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + crate::StoreExpectation::NoExpectation, + )?; let error = view .retained_root(expected.root().namespace().digest()) .err() diff --git a/src/adapters/store_expectation.rs b/src/adapters/store_expectation.rs new file mode 100644 index 00000000..64493842 --- /dev/null +++ b/src/adapters/store_expectation.rs @@ -0,0 +1,95 @@ +//! This module owns the state a caller expects before trusting an opened store. + +use std::error::Error; +use std::fmt; + +use crate::{CatalogDigest, CatalogGeneration}; + +/// The exact published catalog a caller recorded outside the store. +/// +/// A publication receipt or a pinned view supplies both values. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExpectedCatalog { + generation: CatalogGeneration, + digest: CatalogDigest, +} + +impl ExpectedCatalog { + /// Names one exact catalog generation and digest. + #[must_use] + pub const fn new(generation: CatalogGeneration, digest: CatalogDigest) -> Self { + Self { generation, digest } + } + + /// The expected catalog generation. + pub const fn generation(self) -> CatalogGeneration { + self.generation + } + + /// The expected catalog digest. + pub const fn digest(self) -> CatalogDigest { + self.digest + } +} + +/// What a caller knows about the store it is opening. +/// +/// A store's own records cannot show that it is the store the caller meant, or +/// that newer history existed, because a restored store carries consistent old +/// records. Only state kept outside the store can. Every open therefore requires +/// this choice, and it has no default, so no caller loses the check by omission. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StoreExpectation { + /// Refuse unless the store's published catalog is exactly this one. + /// + /// This does not detect a rollback that leaves the catalog unchanged, such as + /// restoring only older retention state. + Catalog(ExpectedCatalog), + /// The caller has no expectation and accepts any valid store at this path, + /// including a different store or an older state of the intended one. + NoExpectation, +} + +impl StoreExpectation { + /// Compares the observed published catalog, `None` meaning none is published. + pub(super) fn require( + self, + observed: Option, + ) -> Result<(), StoreExpectationMismatch> { + match self { + Self::Catalog(expected) if observed != Some(expected) => { + Err(StoreExpectationMismatch { expected, observed }) + } + Self::Catalog(_) | Self::NoExpectation => Ok(()), + } + } +} + +/// An opened store does not hold the catalog its caller expected. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct StoreExpectationMismatch { + expected: ExpectedCatalog, + observed: Option, +} + +impl StoreExpectationMismatch { + /// The catalog the caller expected. + #[must_use] + pub const fn expected(self) -> ExpectedCatalog { + self.expected + } + + /// The catalog the store has published, or `None` when it has published none. + #[must_use] + pub const fn observed(self) -> Option { + self.observed + } +} + +impl fmt::Display for StoreExpectationMismatch { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("the store does not hold the expected published catalog") + } +} + +impl Error for StoreExpectationMismatch {} diff --git a/src/lib.rs b/src/lib.rs index ae0697a8..67575bd9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -70,6 +70,8 @@ mod retention; mod segment_digest; mod verification; +pub use adapters::{ExpectedCatalog, StoreExpectation, StoreExpectationMismatch}; + pub use adapters::{ DurableOutcome, DurableRangeReadReceipt, DurableReadError, DurableReconstructionReceipt, DurableSnapshot, DurableStore, DurableStoreError, DurableView, diff --git a/tests/catalog_writer/format_marker_laws.rs b/tests/catalog_writer/format_marker_laws.rs index 78f79057..17acf3f6 100644 --- a/tests/catalog_writer/format_marker_laws.rs +++ b/tests/catalog_writer/format_marker_laws.rs @@ -62,7 +62,8 @@ fn every_open_path_refuses_a_root_without_the_format_marker() -> TestResult<()> } fn require_writer_paths_refuse(root: &Path) -> TestResult<()> { - let Err(writer) = FilesystemStoreAdmission::reopen(root) else { + let Err(writer) = FilesystemStoreAdmission::reopen(root, keep::StoreExpectation::NoExpectation) + else { return Err("writer admission admitted a root without FORMAT".into()); }; require_missing_marker(&writer)?; @@ -104,14 +105,23 @@ fn require_content_readers_refuse(root: &Path) -> TestResult<()> { }; require_missing_marker(&source)?; let Err(FilesystemRetentionSnapshotError::Admission { source }) = - FilesystemRetentionSnapshot::load(root, policy, ReaderAttemptLimit::DEFAULT) + FilesystemRetentionSnapshot::load( + root, + policy, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + ) else { return Err("the retention reader admitted a root without FORMAT".into()); }; require_refusal(&source)?; - let Err(DurableStoreError::Snapshot(snapshot)) = - DurableStore::open(root, policy, ReaderAttemptLimit::DEFAULT)?.snapshot() - else { + let Err(DurableStoreError::Snapshot(snapshot)) = DurableStore::open( + root, + policy, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot() else { return Err("the durable reader admitted a root without FORMAT".into()); }; let FilesystemRetentionSnapshotError::Admission { source } = snapshot.as_ref() else { diff --git a/tests/catalog_writer/forward_laws.rs b/tests/catalog_writer/forward_laws.rs index 30ebc15b..029a0e23 100644 --- a/tests/catalog_writer/forward_laws.rs +++ b/tests/catalog_writer/forward_laws.rs @@ -48,6 +48,7 @@ fn publication_runs_beside_a_pinned_reader_view() -> TestResult<()> { root, durable_fixture::policy()?, ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, )?; let view = store.snapshot()?; diff --git a/tests/catalog_writer/writer_fixture.rs b/tests/catalog_writer/writer_fixture.rs index 19f27644..7cd409c1 100644 --- a/tests/catalog_writer/writer_fixture.rs +++ b/tests/catalog_writer/writer_fixture.rs @@ -46,6 +46,7 @@ pub(super) fn assert_reads(root: &Path, content: &[u8]) -> TestResult<()> { root, populated_store::policy()?, ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, )?; let mut output = Vec::new(); let _receipt = store.reconstruct(target, &mut output)?; diff --git a/tests/darwin_fifo_refusal.rs b/tests/darwin_fifo_refusal.rs index 042bcca5..2036e41f 100644 --- a/tests/darwin_fifo_refusal.rs +++ b/tests/darwin_fifo_refusal.rs @@ -47,7 +47,13 @@ fn a_fifo_at_the_head_refuses_without_blocking_the_reader() -> Result<(), Box Result<(), Box> { root, policy, ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, )?); Ok(()) } diff --git a/tests/golden_file_worldline/durable_assertions.rs b/tests/golden_file_worldline/durable_assertions.rs index c9f1fafe..72466ebc 100644 --- a/tests/golden_file_worldline/durable_assertions.rs +++ b/tests/golden_file_worldline/durable_assertions.rs @@ -21,7 +21,12 @@ fn reopened_durable_views_reconstruct_every_worldline_identity() -> Result<(), B let sources = bytes.iter().map(Vec::as_slice).collect::>(); let sandbox = build("durable-worldline-reopen", &sources)?; for (case, expected) in cases.iter().zip(&bytes) { - let store = DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + let store = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )?; let target = case.expected_id()?; let mut output = Vec::new(); let receipt = store.reconstruct(target, &mut output)?; @@ -59,7 +64,12 @@ fn durable_worldline_ranges_match_source_slices_after_reopen() -> Result<(), Box let sources = bytes.iter().map(Vec::as_slice).collect::>(); let sandbox = build("durable-worldline-ranges", &sources)?; for (case, expected) in cases.iter().zip(&bytes) { - let store = DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + let store = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )?; let snapshot = store.snapshot()?; let target = case.expected_id()?; let length = u64::try_from(expected.len())?; diff --git a/tests/golden_file_worldline/durable_closure_refusal.rs b/tests/golden_file_worldline/durable_closure_refusal.rs index 8bdc4a9c..0c8a5104 100644 --- a/tests/golden_file_worldline/durable_closure_refusal.rs +++ b/tests/golden_file_worldline/durable_closure_refusal.rs @@ -43,7 +43,12 @@ fn an_unsatisfied_retained_closure_refuses_snapshot_admission_before_output() .iter() .map(fs::read) .collect::, _>>()?; - let store = DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + let store = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )?; let failure = store .snapshot() .err() diff --git a/tests/golden_file_worldline/durable_corruption_laws.rs b/tests/golden_file_worldline/durable_corruption_laws.rs index e617d798..234b992c 100644 --- a/tests/golden_file_worldline/durable_corruption_laws.rs +++ b/tests/golden_file_worldline/durable_corruption_laws.rs @@ -43,7 +43,12 @@ fn a_corrupt_selected_chunk_preserves_exact_record_refusal_before_output() .ok_or("record preimage absent")?; let expected = record_checksum(covered)?; fs::write(segment_path, bytes)?; - let store = DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + let store = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )?; let mut output = vec![0xAB]; let failure = store .reconstruct(target, &mut output) @@ -89,7 +94,12 @@ fn a_corrupt_selected_chunk_refuses_a_range_before_output() -> Result<(), Box Result<(), Box> { let bytes = b"diagnostic whole output"; let sandbox = build("durable-whole-diagnostic", &[bytes])?; - let store = DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + let store = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )?; let failure = store .reconstruct(identify(bytes)?.target, &mut ZeroWriter) .err() @@ -35,7 +40,12 @@ fn durable_whole_failure_renders_each_boundary_once() -> Result<(), Box Result<(), Box> { let bytes = b"diagnostic range output"; let sandbox = build("durable-range-diagnostic", &[bytes])?; - let store = DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + let store = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )?; let range = ByteRange::new(ByteOffset::new(0), ByteLength::new(1))?; let failure = store .read_range(identify(bytes)?.target, range, &mut ZeroWriter) @@ -62,6 +72,7 @@ fn durable_admission_failure_renders_its_boundary_once() -> Result<(), Box Resul let identified = identify(bytes)?; let decode = LayoutDecodePolicy::new(LayoutEntryLimit::MAXIMUM); let layout = AdmittedLayout::decode_record(identified.record.bytes(), decode)?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let mut admitted_output = Vec::new(); let admitted = snapshot.reconstruct_admitted_layout(&layout, &mut admitted_output)?; let mut record_output = Vec::new(); @@ -59,8 +64,13 @@ fn a_wrong_whole_blob_claim_refuses_before_any_durable_output() -> Result<(), Bo LayoutEntryLimit::MAXIMUM, )?; let layout_id = layout.encode_record()?.id(); - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let mut output = vec![0xAB]; let failure = snapshot .reconstruct_admitted_layout(&layout, &mut output) @@ -90,8 +100,13 @@ fn supplied_range_layouts_cannot_replace_the_catalogued_target_binding() LayoutEntryLimit::MAXIMUM, )?; let record = layout.encode_record()?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let requested = ByteRange::new(ByteOffset::new(0), ByteLength::new(1))?; let mut output = vec![0xAB]; let failure = snapshot @@ -128,8 +143,13 @@ fn supplied_corrupt_whole_record_reports_input_decode_before_output() -> Result< let expected: [u8; 32] = encoded.get(offset..).ok_or("checksum absent")?.try_into()?; *encoded.last_mut().ok_or("record empty")? ^= 1; let observed: [u8; 32] = encoded.get(offset..).ok_or("checksum absent")?.try_into()?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let decode = LayoutDecodePolicy::new(LayoutEntryLimit::MAXIMUM); let mut output = vec![0xAB]; @@ -158,8 +178,13 @@ fn supplied_corrupt_range_record_reports_input_decode_before_output() -> Result< let expected: [u8; 32] = encoded.get(offset..).ok_or("checksum absent")?.try_into()?; *encoded.last_mut().ok_or("record empty")? ^= 1; let observed: [u8; 32] = encoded.get(offset..).ok_or("checksum absent")?.try_into()?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let decode = LayoutDecodePolicy::new(LayoutEntryLimit::MAXIMUM); let mut output = vec![0xAB]; @@ -189,8 +214,13 @@ fn a_content_correct_durable_layout_refuses_false_profile_boundaries() -> Result let first = vec![0_u8; 262_143]; let last = [0_u8; 2]; let sandbox = build("durable-false-profile", &[&first, &last])?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let mut output = vec![0xAB]; let failure = snapshot .reconstruct_record( diff --git a/tests/golden_file_worldline/durable_locator_laws.rs b/tests/golden_file_worldline/durable_locator_laws.rs index a3c5b0a4..cca16911 100644 --- a/tests/golden_file_worldline/durable_locator_laws.rs +++ b/tests/golden_file_worldline/durable_locator_laws.rs @@ -81,7 +81,12 @@ fn deleted_current_directory() -> Result<(), Box> { std::env::set_current_dir(sandbox.path())?; std::fs::remove_dir(sandbox.path())?; - let result = DurableStore::open(Path::new("."), policy()?, ReaderAttemptLimit::DEFAULT); + let result = DurableStore::open( + Path::new("."), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + ); restore.restore()?; let refusal = result @@ -111,7 +116,12 @@ fn change_directory_after_open() -> Result<(), Box> { let second_blob = identify(b"second store")?.target; let restore = WorkingDirectory(std::env::current_dir()?); std::env::set_current_dir(first.path())?; - let store = DurableStore::open(Path::new("."), policy()?, ReaderAttemptLimit::DEFAULT)?; + let store = DurableStore::open( + Path::new("."), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )?; assert!( store.contains_blob(first_blob)?, "initial store must contain its own blob" diff --git a/tests/golden_file_worldline/durable_namespace_laws.rs b/tests/golden_file_worldline/durable_namespace_laws.rs index 4e530aa9..4fcb5a2f 100644 --- a/tests/golden_file_worldline/durable_namespace_laws.rs +++ b/tests/golden_file_worldline/durable_namespace_laws.rs @@ -25,8 +25,12 @@ use super::durable_fixture::{build, identify, policy}; fn a_selected_root_from_another_namespace_refuses_direct_read() -> Result<(), Box> { let sandbox = build("durable-direct-foreign-root", &[b"namespace-bound"])?; let namespace = install_foreign_selection(sandbox.path())?; - let snapshot = - FilesystemRetentionSnapshot::load(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?; + let snapshot = FilesystemRetentionSnapshot::load( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )?; let refusal = snapshot .retained_root(namespace) @@ -42,7 +46,12 @@ fn a_foreign_retained_namespace_refuses_durable_output() -> Result<(), Box Result> { - let view = FilesystemRetentionSnapshot::load(path, policy()?, ReaderAttemptLimit::DEFAULT)?; + let view = FilesystemRetentionSnapshot::load( + path, + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )?; let selected = view .manifest() .ok_or("manifest absent")? diff --git a/tests/golden_file_worldline/durable_output_laws.rs b/tests/golden_file_worldline/durable_output_laws.rs index a9141dc0..4d93ae31 100644 --- a/tests/golden_file_worldline/durable_output_laws.rs +++ b/tests/golden_file_worldline/durable_output_laws.rs @@ -19,8 +19,13 @@ fn durable_reconstruction_completes_short_and_interrupted_writes() -> Result<(), let bytes = b"authenticated durable bytes through short writes"; let sandbox = build("durable-short-writes", &[bytes])?; let identified = identify(bytes)?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let mut output = PartitionWriter::new(&[1, 7, 3])?; let receipt = snapshot.reconstruct(identified.target, &mut output)?; assert_eq!(output.bytes(), bytes); @@ -37,8 +42,13 @@ fn durable_reconstruction_preserves_a_failed_writers_accepted_prefix() -> Result let bytes = b"a failed output has no successful receipt"; let sandbox = build("durable-output-prefix", &[bytes])?; let identified = identify(bytes)?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let mut output = PrefixThenFailWriter::new(5)?; let failure = snapshot .reconstruct(identified.target, &mut output) @@ -60,8 +70,13 @@ fn durable_range_preserves_a_failed_writers_accepted_prefix() -> Result<(), Box< let bytes = b"a range failure preserves its exact output prefix"; let sandbox = build("durable-range-prefix", &[bytes])?; let identified = identify(bytes)?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let requested = ByteRange::new(ByteOffset::new(3), ByteLength::new(13))?; let mut output = PrefixThenFailWriter::new(5)?; let failure = snapshot @@ -84,8 +99,13 @@ fn a_zero_progress_durable_writer_refuses_at_the_output_boundary() -> Result<(), let bytes = b"nonempty output must make progress"; let sandbox = build("durable-output-zero", &[bytes])?; let identified = identify(bytes)?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let failure = snapshot .reconstruct(identified.target, &mut ZeroWriter) .err() @@ -104,8 +124,13 @@ fn durable_ranges_complete_short_and_interrupted_writes() -> Result<(), Box Result<() let bytes = b"nonempty range must make progress"; let sandbox = build("durable-range-zero", &[bytes])?; let identified = identify(bytes)?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let requested = ByteRange::new(ByteOffset::new(3), ByteLength::new(13))?; let failure = snapshot .read_range(identified.target, requested, &mut ZeroWriter) diff --git a/tests/golden_file_worldline/durable_range_properties.rs b/tests/golden_file_worldline/durable_range_properties.rs index 6126e276..4502ed2d 100644 --- a/tests/golden_file_worldline/durable_range_properties.rs +++ b/tests/golden_file_worldline/durable_range_properties.rs @@ -17,8 +17,13 @@ use keep::{ fn every_short_durable_range_equals_its_source_slice() -> Result<(), Box> { let source = (0_u8..64).collect::>(); let sandbox = build("durable-all-short-ranges", &[&source])?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let target = BlobId::hash_bytes(&source)?; let size = u64::try_from(source.len())?; for length in 0..=size { @@ -39,8 +44,13 @@ fn multichunk_durable_boundary_ranges_equal_the_frozen_worldline_source() let case = find_case("large-ramp")?; let source = case.bytes()?; let sandbox = build("durable-multichunk-boundaries", &[&source])?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let size = u64::try_from(source.len())?; let positions = [ 0, @@ -99,8 +109,13 @@ fn generated_multichunk_durable_ranges_equal_the_reference_domain() -> Result<() source.push(u8::try_from(value)?); } let sandbox = build("durable-reference-range-domain", &[&source])?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let target = BlobId::hash_bytes(&source)?; let coordinate_count = u64::try_from(source.len())? .checked_add(1) @@ -142,8 +157,13 @@ fn a_durable_range_needs_no_nonoverlapping_chunk_records() -> Result<(), Box Result let case = find_case("large-ramp")?; let bytes = case.bytes()?; let sandbox = build("durable-read-allocation", &[&bytes])?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let target = case.expected_id()?; let mut output = io::sink(); let mut result = None; diff --git a/tests/golden_file_worldline/durable_refusal_laws.rs b/tests/golden_file_worldline/durable_refusal_laws.rs index 3ea12d13..87b7a417 100644 --- a/tests/golden_file_worldline/durable_refusal_laws.rs +++ b/tests/golden_file_worldline/durable_refusal_laws.rs @@ -23,8 +23,13 @@ fn a_durable_layout_naming_an_absent_chunk_refuses_before_reconstruction_output( .first() .ok_or("nonempty input has no chunk")? .id(); - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let mut output = vec![0xAB]; let failure = snapshot .reconstruct_layout(expected.record.id(), &mut output) @@ -54,8 +59,13 @@ fn a_durable_range_naming_an_absent_chunk_refuses_before_output() -> Result<(), .first() .ok_or("nonempty input has no chunk")? .id(); - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let requested = ByteRange::new(ByteOffset::new(1), ByteLength::new(3))?; let mut output = vec![0xAB]; let failure = snapshot @@ -78,8 +88,13 @@ fn an_out_of_bounds_durable_range_refuses_the_exact_requested_coordinates() let bytes = b"one"; let sandbox = build("durable-range-outside", &[bytes])?; let expected = identify(bytes)?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let requested = ByteRange::new(ByteOffset::new(2), ByteLength::new(2))?; let mut output = vec![0xAB]; let failure = snapshot @@ -101,8 +116,13 @@ fn an_absent_durable_blob_refuses_range_output_with_its_exact_identity() -> Result<(), Box> { let sandbox = build("durable-absent-blob", &[b"present"])?; let absent = identify(b"absent")?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let requested = ByteRange::new(ByteOffset::new(0), ByteLength::new(1))?; let mut output = vec![0xAB]; let failure = snapshot @@ -120,8 +140,13 @@ fn an_absent_durable_blob_refuses_range_output_with_its_exact_identity() fn an_absent_durable_layout_refuses_before_reconstruction_output() -> Result<(), Box> { let sandbox = build("durable-absent-layout", &[b"present"])?; let absent = identify(b"absent")?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let mut output = vec![0xAB]; let failure = snapshot .reconstruct_layout(absent.record.id(), &mut output) diff --git a/tests/golden_file_worldline/durable_writer_failures.rs b/tests/golden_file_worldline/durable_writer_failures.rs index 5737a4cb..98284cf6 100644 --- a/tests/golden_file_worldline/durable_writer_failures.rs +++ b/tests/golden_file_worldline/durable_writer_failures.rs @@ -17,8 +17,13 @@ fn durable_reconstruction_preserves_immediate_output_failure() -> Result<(), Box let bytes = b"exact output capability failure"; let sandbox = build("durable-reconstruction-failing", &[bytes])?; let identified = identify(bytes)?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let failure = snapshot .reconstruct(identified.target, &mut FailingWriter) .err() @@ -38,8 +43,13 @@ fn durable_reconstruction_rejects_impossible_write_counts() -> Result<(), Box Result<(), Box Result<(), Box> let bytes = b"exact output capability failure"; let sandbox = build("durable-range-lying", &[bytes])?; let identified = identify(bytes)?; - let snapshot = - DurableStore::open(sandbox.path(), policy()?, ReaderAttemptLimit::DEFAULT)?.snapshot()?; + let snapshot = DurableStore::open( + sandbox.path(), + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )? + .snapshot()?; let requested = ByteRange::new(ByteOffset::new(3), ByteLength::new(11))?; let failure = snapshot .read_range(identified.target, requested, &mut LyingWriter) diff --git a/tests/reader_fence_process/reader.rs b/tests/reader_fence_process/reader.rs index e9ec7ce8..420d5634 100644 --- a/tests/reader_fence_process/reader.rs +++ b/tests/reader_fence_process/reader.rs @@ -34,7 +34,12 @@ pub(super) fn serve() -> Result<(), Box> { SegmentReadPolicy::MAXIMUM, CatalogRestartByteLimit::new(1_048_576)?, ); - let snapshot = FilesystemRetentionSnapshot::load(&root, policy, ReaderAttemptLimit::DEFAULT)?; + let snapshot = FilesystemRetentionSnapshot::load( + &root, + policy, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + )?; assert_eq!( snapshot.catalog().generation().get(), 1, diff --git a/tests/single_format/acceptance_laws.rs b/tests/single_format/acceptance_laws.rs index f301db27..954fb79c 100644 --- a/tests/single_format/acceptance_laws.rs +++ b/tests/single_format/acceptance_laws.rs @@ -98,12 +98,16 @@ fn the_single_format_serves_every_operation_from_initialization_to_recovery() -> assert_eq!(ingested.generation().get(), 1); drop(writer); - drop(FilesystemStoreAdmission::reopen(root)?); + drop(FilesystemStoreAdmission::reopen( + root, + keep::StoreExpectation::NoExpectation, + )?); let layout = populated_store::identify(&first)?.record.id(); let store = DurableStore::open( root, populated_store::policy()?, ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, )?; let mut output = Vec::new(); let _receipt = store.reconstruct_layout(layout, &mut output)?; @@ -162,6 +166,7 @@ fn a_backup_copy_reopens_verifies_and_recovers_an_interrupted_write() -> TestRes original.path(), populated_store::policy()?, ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, ); assert!(matches!( unavailable, @@ -169,7 +174,10 @@ fn a_backup_copy_reopens_verifies_and_recovers_an_interrupted_write() -> TestRes )); let root = backup.path(); - drop(FilesystemStoreAdmission::reopen(root)?); + drop(FilesystemStoreAdmission::reopen( + root, + keep::StoreExpectation::NoExpectation, + )?); assert_reads(root, &first)?; let candidate = interrupted_publication(root, &second_blob())?; recover(root, &candidate, 2)?; diff --git a/tests/single_format/expectation_laws.rs b/tests/single_format/expectation_laws.rs new file mode 100644 index 00000000..892001a4 --- /dev/null +++ b/tests/single_format/expectation_laws.rs @@ -0,0 +1,208 @@ +//! This module owns the ADR-0010 expected-state laws at every public open. +//! +//! A restored store's records are all consistent, so only state the caller kept +//! outside the store can refuse it. Each law opens through a production entry +//! point and asserts the exact expected and observed catalogs. + +use std::fs; +use std::path::Path; + +use keep::{ + DurableSnapshot, DurableStore, DurableStoreError, ExpectedCatalog, FilesystemRetentionSnapshot, + FilesystemRetentionSnapshotError, FilesystemStoreAdmission, FilesystemStoreAdmissionError, + ReaderAttemptLimit, StoreExpectation, StoreExpectationMismatch, +}; + +use super::durable_fixture; +use super::durable_sandbox::TestDirectory; +use super::populated_store; +use super::store_copy::copy_store; +use super::writer_fixture::{ + Predecessor, TestResult, assert_reads, first_blob, open_publisher, publish, retain, second_blob, +}; + +/// The catalog a fresh view of `root` selects, as a caller would record it. +fn published(root: &Path) -> TestResult { + let snapshot = DurableSnapshot::open( + root, + populated_store::policy()?, + ReaderAttemptLimit::DEFAULT, + StoreExpectation::NoExpectation, + )?; + let view = snapshot.view(); + Ok(ExpectedCatalog::new( + view.catalog_generation(), + view.catalog_digest(), + )) +} + +fn publish_second(root: &Path) -> TestResult<()> { + let second = second_blob(); + let mut writer = open_publisher(root)?; + let _receipt = publish(&mut writer, root, &second, Predecessor::Kept)??; + drop(writer); + retain(root, b"second", &second) +} + +/// Requires every public open of `root` to refuse with exactly this mismatch. +fn every_open_refuses( + root: &Path, + expected: ExpectedCatalog, + observed: Option, +) -> TestResult<()> { + let expectation = StoreExpectation::Catalog(expected); + let policy = populated_store::policy()?; + let limit = ReaderAttemptLimit::DEFAULT; + let reader = FilesystemRetentionSnapshot::load(root, policy, limit, expectation); + let Err(FilesystemRetentionSnapshotError::Expectation { source }) = reader else { + return Err("the reader did not refuse at the expectation boundary".into()); + }; + require_mismatch(source, expected, observed); + let store = DurableStore::open(root, policy, limit, expectation)?; + let Err(DurableStoreError::Snapshot(error)) = store.snapshot() else { + return Err("the durable store did not refuse its snapshot".into()); + }; + let FilesystemRetentionSnapshotError::Expectation { source } = *error else { + return Err("the durable store lost the expectation boundary".into()); + }; + require_mismatch(source, expected, observed); + let writer = FilesystemStoreAdmission::reopen(root, expectation); + let Err(FilesystemStoreAdmissionError::Expectation { source }) = writer else { + return Err("the writer did not refuse at the expectation boundary".into()); + }; + require_mismatch(source, expected, observed); + Ok(()) +} + +fn require_mismatch( + mismatch: StoreExpectationMismatch, + expected: ExpectedCatalog, + observed: Option, +) { + assert_eq!(mismatch.expected(), expected); + assert_eq!(mismatch.observed(), observed); +} + +fn every_open_admits(root: &Path, expectation: StoreExpectation) -> TestResult<()> { + let policy = populated_store::policy()?; + let limit = ReaderAttemptLimit::DEFAULT; + drop(FilesystemRetentionSnapshot::load( + root, + policy, + limit, + expectation, + )?); + drop(DurableStore::open(root, policy, limit, expectation)?.snapshot()?); + drop(FilesystemStoreAdmission::reopen(root, expectation)?); + Ok(()) +} + +/// Replaces every entry under `root` with the entries of `snapshot`. +fn restore(snapshot: &Path, root: &Path) -> TestResult<()> { + for entry in fs::read_dir(root)? { + let path = entry?.path(); + if path.is_dir() { + fs::remove_dir_all(&path)?; + } else { + fs::remove_file(&path)?; + } + } + copy_store(snapshot, root) +} + +#[test] +fn an_older_snapshot_restored_in_place_refuses_newer_expected_state() -> TestResult<()> { + let first = first_blob(); + let store = durable_fixture::build("expected-rollback", &[&first])?; + let snapshot = TestDirectory::create("expected-rollback-snapshot")?; + copy_store(store.path(), snapshot.path())?; + let older = published(store.path())?; + publish_second(store.path())?; + let newer = published(store.path())?; + assert_ne!(older, newer); + every_open_admits(store.path(), StoreExpectation::Catalog(newer))?; + every_open_refuses(store.path(), older, Some(newer))?; + + restore(snapshot.path(), store.path())?; + + every_open_refuses(store.path(), newer, Some(older))?; + // The limit the typed choice exists to make visible: without an expectation, + // nothing in a consistent restored store shows that newer history existed. + every_open_admits(store.path(), StoreExpectation::NoExpectation)?; + assert_reads(store.path(), &first) +} + +#[test] +fn a_different_valid_store_refuses_the_expected_catalog() -> TestResult<()> { + let intended = durable_fixture::build("expected-intended", &[&first_blob()])?; + let other = durable_fixture::build("expected-other", &[&second_blob()])?; + let wanted = published(intended.path())?; + let found = published(other.path())?; + assert_eq!(wanted.generation(), found.generation()); + + every_open_refuses(other.path(), wanted, Some(found)) +} + +#[test] +fn a_store_with_no_catalog_refuses_an_expected_catalog() -> TestResult<()> { + let published_store = durable_fixture::build("expected-published", &[&first_blob()])?; + let wanted = published(published_store.path())?; + let empty = TestDirectory::create("expected-unpublished")?; + drop(FilesystemStoreAdmission::initialize(empty.path())?); + + every_open_refuses(empty.path(), wanted, None)?; + assert!(matches!( + FilesystemRetentionSnapshot::load( + empty.path(), + populated_store::policy()?, + ReaderAttemptLimit::DEFAULT, + StoreExpectation::NoExpectation, + ), + Err(FilesystemRetentionSnapshotError::Unpublished) + )); + drop(FilesystemStoreAdmission::reopen( + empty.path(), + StoreExpectation::NoExpectation, + )?); + Ok(()) +} + +#[test] +fn a_damaged_head_refuses_the_writer_expectation_at_its_own_boundary() -> TestResult<()> { + let store = durable_fixture::build("expected-damaged-head", &[&first_blob()])?; + let wanted = published(store.path())?; + let head = store.path().join("HEAD"); + let mut bytes = fs::read(&head)?; + *bytes.first_mut().ok_or("empty HEAD")? ^= 1; + fs::write(&head, &bytes)?; + + let refused = FilesystemStoreAdmission::reopen(store.path(), StoreExpectation::Catalog(wanted)); + + assert!(matches!( + refused, + Err(FilesystemStoreAdmissionError::ExpectationHead { .. }) + )); + Ok(()) +} + +/// Pins the documented limit: this expectation covers the catalog, so restoring +/// only older retention state under an unchanged catalog is not detected. +#[test] +fn a_catalog_expectation_does_not_detect_a_retention_only_rollback() -> TestResult<()> { + let first = first_blob(); + let store = durable_fixture::build("expected-retention-only", &[&first])?; + let snapshot = TestDirectory::create("expected-retention-only-snapshot")?; + copy_store(store.path(), snapshot.path())?; + let wanted = published(store.path())?; + retain(store.path(), b"later", &first)?; + assert_eq!(published(store.path())?, wanted); + + fs::remove_dir_all(store.path().join("retention"))?; + fs::create_dir(store.path().join("retention"))?; + copy_store( + &snapshot.path().join("retention"), + &store.path().join("retention"), + )?; + + every_open_admits(store.path(), StoreExpectation::Catalog(wanted)) +} diff --git a/tests/single_format/initialization_laws.rs b/tests/single_format/initialization_laws.rs index 63767ee6..111e6835 100644 --- a/tests/single_format/initialization_laws.rs +++ b/tests/single_format/initialization_laws.rs @@ -27,11 +27,15 @@ fn an_initialized_store_has_no_catalog_and_admits_writers_but_not_readers() -> T assert_eq!(initialized.found(), StoreInitializationPrefix::Empty); drop(initialized); assert!(!root.join("HEAD").exists()); - drop(FilesystemStoreAdmission::reopen(root)?); + drop(FilesystemStoreAdmission::reopen( + root, + keep::StoreExpectation::NoExpectation, + )?); let reader = FilesystemRetentionSnapshot::load( root, populated_store::policy()?, ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, ); assert!(matches!( reader, @@ -102,7 +106,7 @@ fn an_uncommitted_root_is_refused_by_writers_and_readers() -> TestResult<()> { fs::hard_link(root.join("FORMAT"), root.join("FORMAT.next"))?; let Err(FilesystemStoreAdmissionError::Namespace { source }) = - FilesystemStoreAdmission::reopen(root) + FilesystemStoreAdmission::reopen(root, keep::StoreExpectation::NoExpectation) else { return Err("an uncommitted root admitted a writer".into()); }; @@ -117,7 +121,8 @@ fn an_uncommitted_root_is_refused_by_writers_and_readers() -> TestResult<()> { FilesystemRetentionSnapshot::load( root, populated_store::policy()?, - ReaderAttemptLimit::DEFAULT + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation ), Err(FilesystemRetentionSnapshotError::Admission { .. }) )); diff --git a/tests/single_format/portability_laws.rs b/tests/single_format/portability_laws.rs index c3a92ec1..a4911f56 100644 --- a/tests/single_format/portability_laws.rs +++ b/tests/single_format/portability_laws.rs @@ -37,7 +37,10 @@ fn a_valid_quiescent_copy_opens_and_reads_the_same_content() -> TestResult<()> { let original = durable_fixture::build("portable-original", &[&first])?; let copy = copied("portable-copy", original.path())?; - drop(FilesystemStoreAdmission::reopen(copy.path())?); + drop(FilesystemStoreAdmission::reopen( + copy.path(), + keep::StoreExpectation::NoExpectation, + )?); assert_reads(copy.path(), &first) } @@ -86,11 +89,11 @@ fn a_replaced_lock_or_protocol_directory_during_an_operation_still_refuses() -> let original = durable_fixture::build("portable-replaced-original", &[&first_blob()])?; let copy = copied("portable-replaced-copy", original.path())?; let root = copy.path(); - let held = FilesystemStoreAdmission::reopen(root)?; + let held = FilesystemStoreAdmission::reopen(root, keep::StoreExpectation::NoExpectation)?; fs::remove_file(root.join("writer.lock"))?; fs::write(root.join("writer.lock"), b"")?; - let contended = FilesystemStoreAdmission::reopen(root); + let contended = FilesystemStoreAdmission::reopen(root, keep::StoreExpectation::NoExpectation); assert!(matches!( contended, @@ -165,7 +168,7 @@ fn a_copy_and_its_original_diverge_independently_and_keep_shared_identity() -> T fn marker_refusal(root: &Path) -> TestResult { let Err(FilesystemStoreAdmissionError::FormatMarker { source }) = - FilesystemStoreAdmission::reopen(root) + FilesystemStoreAdmission::reopen(root, keep::StoreExpectation::NoExpectation) else { return Err("a corrupt marker was not refused at the marker boundary".into()); }; @@ -177,7 +180,7 @@ fn marker_refusal(root: &Path) -> TestResult { } fn namespace_refusal_kind(root: &Path) -> TestResult { - match FilesystemStoreAdmission::reopen(root) { + match FilesystemStoreAdmission::reopen(root, keep::StoreExpectation::NoExpectation) { Err(FilesystemStoreAdmissionError::Namespace { source }) => Ok(source.kind()), Err(other) => Err(format!("unexpected incomplete-store refusal: {other:?}").into()), Ok(_) => Err("an incomplete store was admitted".into()), diff --git a/tests/single_format/suite.rs b/tests/single_format/suite.rs index 49a3dbe7..2f2d7e3a 100644 --- a/tests/single_format/suite.rs +++ b/tests/single_format/suite.rs @@ -15,6 +15,8 @@ mod durable_fixture; reason = "these laws use Drop cleanup; other filesystem laws check explicit removal" )] mod durable_sandbox; +#[path = "expectation_laws.rs"] +mod expectation_laws; #[path = "initialization_laws.rs"] mod initialization_laws; #[path = "../support/populated_store.rs"] @@ -36,6 +38,8 @@ mod store_copy; reason = "these laws publish successors only through `publish`" )] mod successor_catalog; +#[path = "torn_copy_laws.rs"] +mod torn_copy_laws; #[cfg(target_os = "linux")] #[path = "unsupported_directory.rs"] mod unsupported_directory; diff --git a/tests/single_format/torn_copy_laws.rs b/tests/single_format/torn_copy_laws.rs new file mode 100644 index 00000000..81623f38 --- /dev/null +++ b/tests/single_format/torn_copy_laws.rs @@ -0,0 +1,165 @@ +//! This module owns the ADR-0010 law for a copy torn by a concurrent writer. +//! +//! A copy taken while a writer publishes can hold any mix of the states before +//! and after that publication, and a partly copied file. These laws enumerate +//! those mixes directly, so they need no scheduling or timing. Every mix must +//! refuse or return exactly the bytes a content identity names. + +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use keep::{DurableStore, ReaderAttemptLimit, StoreExpectation}; + +use super::durable_fixture; +use super::durable_sandbox::TestDirectory; +use super::populated_store; +use super::store_copy::copy_store; +use super::writer_fixture::{ + Predecessor, TestResult, first_blob, open_publisher, publish, retain, second_blob, +}; + +/// Every changed file between the two states costs a doubling of the mixes. +const MAXIMUM_CHANGED_FILES: usize = 12; + +type Files = BTreeMap>; + +/// How many mixes refused and how many read, so the laws cannot pass vacuously. +#[derive(Default)] +struct Outcomes { + refused: usize, + read: usize, +} + +fn files(root: &Path) -> TestResult { + let mut found = Files::new(); + let mut pending = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + for entry in fs::read_dir(&directory)? { + let path = entry?.path(); + if path.is_dir() { + pending.push(path); + } else { + let bytes = fs::read(&path)?; + found.insert(path.strip_prefix(root)?.to_path_buf(), bytes); + } + } + } + Ok(found) +} + +/// Builds a store, records it, and publishes a successor: the two torn-copy ends. +fn two_states(name: &str) -> TestResult<(TestDirectory, Files, Vec, Files)> { + let store = durable_fixture::build(name, &[&first_blob()])?; + let before_copy = TestDirectory::create(&format!("{name}-before"))?; + copy_store(store.path(), before_copy.path())?; + let second = second_blob(); + let mut writer = open_publisher(store.path())?; + let _receipt = publish(&mut writer, store.path(), &second, Predecessor::Kept)??; + drop(writer); + retain(store.path(), b"second", &second)?; + let after = files(store.path())?; + let before = files(before_copy.path())?; + let changed: Vec = after + .iter() + .filter(|(path, bytes)| before.get(*path) != Some(*bytes)) + .map(|(path, _)| path.clone()) + .collect(); + assert!(!changed.is_empty(), "the publication changed no file"); + assert!( + changed.len() <= MAXIMUM_CHANGED_FILES, + "{} changed files would make the enumeration too large", + changed.len() + ); + Ok((before_copy, before, changed, after)) +} + +/// Overlays `after`'s version of each selected file onto a copy of the base. +fn torn(name: &str, base: &Path, overlay: &[(&Path, &[u8])]) -> TestResult { + let copy = TestDirectory::create(name)?; + copy_store(base, copy.path())?; + for (path, bytes) in overlay { + let target = copy.path().join(path); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + fs::write(target, bytes)?; + } + Ok(copy) +} + +/// Requires `content` to reconstruct exactly, or to refuse with a verified prefix. +fn exact_or_refused(root: &Path, content: &[u8], outcomes: &mut Outcomes) -> TestResult<()> { + let target = populated_store::identify(content)?.target; + let store = DurableStore::open( + root, + populated_store::policy()?, + ReaderAttemptLimit::DEFAULT, + StoreExpectation::NoExpectation, + )?; + let mut output = Vec::new(); + match store.reconstruct(target, &mut output) { + Ok(_receipt) => { + assert_eq!(output, content, "a torn copy returned other bytes"); + outcomes.read = outcomes.read.saturating_add(1); + } + Err(_refusal) => { + assert!( + content.starts_with(&output), + "a refused read emitted bytes its identity does not name" + ); + outcomes.refused = outcomes.refused.saturating_add(1); + } + } + Ok(()) +} + +fn both_blobs(root: &Path, outcomes: &mut Outcomes) -> TestResult<()> { + exact_or_refused(root, &first_blob(), outcomes)?; + exact_or_refused(root, &second_blob(), outcomes) +} + +#[test] +fn every_mix_of_two_published_states_refuses_or_returns_exact_bytes() -> TestResult<()> { + let (base, _before, changed, after) = two_states("torn-mix")?; + let mut outcomes = Outcomes::default(); + for mask in 0_u32..(1_u32 << changed.len()) { + let mut overlay: Vec<(&Path, &[u8])> = Vec::new(); + for (bit, path) in changed.iter().enumerate() { + if mask & (1_u32 << bit) != 0 { + let bytes = after.get(path).ok_or("changed file left the state")?; + overlay.push((path, bytes)); + } + } + let copy = torn("torn-mix-copy", base.path(), &overlay)?; + both_blobs(copy.path(), &mut outcomes)?; + } + assert!(outcomes.read > 0, "no torn mix read anything"); + assert!(outcomes.refused > 0, "no torn mix refused anything"); + Ok(()) +} + +#[test] +fn every_partly_copied_file_refuses_or_returns_exact_bytes() -> TestResult<()> { + let (base, before, changed, after) = two_states("torn-partial")?; + let mut outcomes = Outcomes::default(); + let whole: Vec<(&Path, &[u8])> = after + .iter() + .filter(|(path, _)| changed.contains(path)) + .map(|(path, bytes)| (path.as_path(), bytes.as_slice())) + .collect(); + for path in before.keys().chain(changed.iter()) { + let source = after.get(path).or_else(|| before.get(path)); + let bytes = source.ok_or("the file left both states")?; + let half = bytes + .get(..bytes.len() >> 1) + .ok_or("half is out of range")?; + let mut overlay = whole.clone(); + overlay.retain(|(candidate, _)| *candidate != path.as_path()); + overlay.push((path.as_path(), half)); + let copy = torn("torn-partial-copy", base.path(), &overlay)?; + both_blobs(copy.path(), &mut outcomes)?; + } + assert!(outcomes.refused > 0, "no partial copy refused anything"); + Ok(()) +} diff --git a/tests/support/populated_store.rs b/tests/support/populated_store.rs index 5a681c2c..f134c1bc 100644 --- a/tests/support/populated_store.rs +++ b/tests/support/populated_store.rs @@ -49,7 +49,12 @@ impl StoreAdmission { /// The production platform profile. pub(super) const PLATFORM: Self = Self { initialize: |root| Ok(FilesystemStoreAdmission::initialize(root)?.into_admission()), - reopen: |root| Ok(FilesystemStoreAdmission::reopen(root)?), + reopen: |root| { + Ok(FilesystemStoreAdmission::reopen( + root, + keep::StoreExpectation::NoExpectation, + )?) + }, }; } diff --git a/xtask/src/durability_crash_matrix/restart/initialization.rs b/xtask/src/durability_crash_matrix/restart/initialization.rs index 81bad978..0343a492 100644 --- a/xtask/src/durability_crash_matrix/restart/initialization.rs +++ b/xtask/src/durability_crash_matrix/restart/initialization.rs @@ -275,8 +275,13 @@ fn publish_and_read(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> .map(drop) .map_err(|source| verification("retain recovered bundle", source))?; drop(authority); - let store = DurableStore::open(store_root, policy()?, ReaderAttemptLimit::DEFAULT) - .map_err(|source| verification("open recovered durable store", source))?; + let store = DurableStore::open( + store_root, + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + ) + .map_err(|source| verification("open recovered durable store", source))?; let bundle = CrashBlob::bundle()?; let mut output = Vec::new(); store diff --git a/xtask/src/durability_crash_matrix/restart/retained_catalog.rs b/xtask/src/durability_crash_matrix/restart/retained_catalog.rs index 3e52e9ad..aa8ea4f8 100644 --- a/xtask/src/durability_crash_matrix/restart/retained_catalog.rs +++ b/xtask/src/durability_crash_matrix/restart/retained_catalog.rs @@ -205,8 +205,13 @@ fn retain_blob(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { /// Reads the retained bundle blob and the ingested blob through the durable reader. fn read_back(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let store = DurableStore::open(store_root, policy()?, ReaderAttemptLimit::DEFAULT) - .map_err(|source| verification("open recovered durable store", source))?; + let store = DurableStore::open( + store_root, + policy()?, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + ) + .map_err(|source| verification("open recovered durable store", source))?; let golden = GoldenFixture::retention_root()?; let retained = AdmittedRetentionRoot::decode(golden.bytes()) .map_err(|source| verification("decode golden retention root", source))?; diff --git a/xtask/src/durability_crash_matrix/restart/retention_snapshot.rs b/xtask/src/durability_crash_matrix/restart/retention_snapshot.rs index 1b875f67..e802d5ca 100644 --- a/xtask/src/durability_crash_matrix/restart/retention_snapshot.rs +++ b/xtask/src/durability_crash_matrix/restart/retention_snapshot.rs @@ -22,8 +22,13 @@ pub(super) fn verify( let limit = CatalogRestartByteLimit::new(1_048_576) .map_err(|source| verification("bound recovered catalog read", source))?; let policy = CatalogRestartPolicy::new(SegmentReadPolicy::MAXIMUM, limit); - let view = FilesystemRetentionSnapshot::load(store_root, policy, ReaderAttemptLimit::DEFAULT) - .map_err(|source| verification("load recovered retention snapshot", source))?; + let view = FilesystemRetentionSnapshot::load( + store_root, + policy, + ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, + ) + .map_err(|source| verification("load recovered retention snapshot", source))?; let observed_generation = view.retention_head().map(|head| head.generation().get()); if observed_generation != expected_generation { return Err(verification( From cfdf67377f1111a2166212ba7fda698d0ebabcf3 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 4 Nov 2026 17:34:15 -0800 Subject: [PATCH 24/36] Test: pin the exact-name check on the single-format store paths (#198) The namespace check refuses a foreign spelling first, so the public laws do not witness the lookup check. These laws call the marker, head and protocol-directory lookups directly, and the public laws add the writer and initialization routes. --- .../filesystem-platform-adapters.md | 8 ++ src/adapters/darwin_filesystem_tests.rs | 77 +++++++++++++++++++ tests/darwin_store_spelling.rs | 27 ++++++- 3 files changed, 110 insertions(+), 2 deletions(-) diff --git a/docs/testing-evidence/filesystem-platform-adapters.md b/docs/testing-evidence/filesystem-platform-adapters.md index 9aab866f..3d60c3c1 100644 --- a/docs/testing-evidence/filesystem-platform-adapters.md +++ b/docs/testing-evidence/filesystem-platform-adapters.md @@ -49,6 +49,14 @@ Calibration with the check disabled, on default APFS: - `darwin_store_spelling` laws **still pass**. Namespace admission independently refuses the foreign entry, so those laws do not witness the per-lookup check. +## Single-format store paths + +The initialization, marker, admission, and namespace code from ADR-0010 uses the +same exact-name directory as every other path. `darwin_filesystem_tests` pins the +per-lookup check below namespace admission for `FORMAT`, `HEAD`, and a protocol +directory, and `tests/darwin_store_spelling.rs` covers the public reader, writer, +and initialization routes. + ## Volume reattachment Darwin assigns `st_dev` per mount and reuses a freed number. Mounting another diff --git a/src/adapters/darwin_filesystem_tests.rs b/src/adapters/darwin_filesystem_tests.rs index d449f60e..0ae65b07 100644 --- a/src/adapters/darwin_filesystem_tests.rs +++ b/src/adapters/darwin_filesystem_tests.rs @@ -10,6 +10,7 @@ use crate::adapters::filesystem_directory::FilesystemDirectory; use crate::adapters::filesystem_entry_spelling::EntrySpellingMismatch; use crate::adapters::filesystem_test_sandbox::TestDirectory; use crate::adapters::platform_filesystem::PlatformFilesystem; +use crate::adapters::{catalog_restart_loader, filesystem_format_marker, sync_capable_directory}; fn open(path: &Path) -> io::Result { FilesystemDirectory::open_ambient_dir(path, ambient_authority()) @@ -191,3 +192,79 @@ fn read_only_apfs_volumes_are_refused() -> Result<(), Box> { assert!(matches!(refusal, Err(source) if source.kind() == io::ErrorKind::Unsupported)); Ok(()) } + +/// These protocol lookups run below namespace admission, so only the per-lookup +/// check can refuse a foreign spelling here. +fn spelling_refusal<'a>(source: &'a (dyn Error + 'static)) -> Option<&'a EntrySpellingMismatch> { + let mut cause = Some(source); + while let Some(current) = cause { + let inner = current + .downcast_ref::() + .and_then(io::Error::get_ref) + .and_then(|inner| inner.downcast_ref::()); + if inner.is_some() { + return inner; + } + cause = current.source(); + } + None +} + +fn assert_spelling_source(source: &(dyn Error + 'static), expected: &str) { + assert_eq!( + spelling_refusal(source), + Some(&EntrySpellingMismatch { + expected: expected.into() + }) + ); +} + +#[test] +fn the_format_marker_lookup_never_accepts_a_foreign_spelling() -> Result<(), Box> { + let sandbox = TestDirectory::create("darwin-marker-alias")?; + let folds = folds_case(sandbox.path())?; + fs::write(sandbox.path().join("format"), b"foreign")?; + let directory = open(sandbox.path())?; + let refusal = filesystem_format_marker::admit(&directory) + .err() + .ok_or("a foreign marker spelling admitted")?; + if folds { + assert_spelling_source(&refusal, "FORMAT"); + } else { + assert_eq!(refusal.kind(), io::ErrorKind::NotFound); + } + Ok(()) +} + +#[test] +fn the_expected_catalog_lookup_never_accepts_a_foreign_spelling() -> Result<(), Box> { + let sandbox = TestDirectory::create("darwin-head-alias")?; + let folds = folds_case(sandbox.path())?; + fs::write(sandbox.path().join("head"), [0_u8; 128])?; + let directory = open(sandbox.path())?; + let observed = catalog_restart_loader::observe_published_catalog(&directory); + if folds { + let refusal = observed.err().ok_or("a foreign head spelling admitted")?; + assert_spelling_source(&refusal, "HEAD"); + } else { + assert!(matches!(observed, Ok(None))); + } + Ok(()) +} + +#[test] +fn a_protocol_directory_lookup_never_accepts_a_foreign_spelling() -> Result<(), Box> { + let sandbox = TestDirectory::create("darwin-directory-alias")?; + let folds = folds_case(sandbox.path())?; + fs::create_dir(sandbox.path().join("Staging"))?; + let directory = open(sandbox.path())?; + let refusal = sync_capable_directory::open(&directory, "staging") + .err() + .ok_or("a foreign directory spelling opened")?; + if folds { + assert_spelling_source(&refusal, "staging"); + } else { + assert_eq!(refusal.kind(), io::ErrorKind::NotFound); + } + Ok(()) +} diff --git a/tests/darwin_store_spelling.rs b/tests/darwin_store_spelling.rs index fc58c84a..dea7b080 100644 --- a/tests/darwin_store_spelling.rs +++ b/tests/darwin_store_spelling.rs @@ -26,8 +26,9 @@ pub mod sandbox; use std::{error::Error, fs, path::Path}; use keep::{ - CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemRetentionSnapshot, ReaderAttemptLimit, - SegmentReadPolicy, + CatalogRestartByteLimit, CatalogRestartPolicy, FilesystemRetentionSnapshot, + FilesystemStoreAdmission, ReaderAttemptLimit, SegmentReadPolicy, StoreExpectation, + StoreInitializationError, StoreInitializationRefusal, }; fn load(root: &Path) -> Result> { @@ -39,6 +40,7 @@ fn load(root: &Path) -> Result> { root, policy, ReaderAttemptLimit::DEFAULT, + keep::StoreExpectation::NoExpectation, )?) } @@ -51,6 +53,10 @@ fn assert_respelling_refuses(law: &str, stored: &str, foreign: &str) -> Result<( load(store.path()).is_err(), "{foreign} answered for {stored}" ); + assert!( + FilesystemStoreAdmission::reopen(store.path(), StoreExpectation::NoExpectation).is_err(), + "{foreign} gave the writer authority for {stored}" + ); fs::rename(store.path().join(foreign), store.path().join(stored))?; drop(load(store.path())?); Ok(()) @@ -71,6 +77,23 @@ fn a_respelled_segment_pool_refuses_until_its_exact_name_returns() -> Result<(), assert_respelling_refuses("darwin-respelled-pool", "segments", "Segments") } +#[test] +fn initialization_refuses_a_foreign_spelling_and_leaves_it_in_place() -> Result<(), Box> +{ + let sandbox = sandbox::TestDirectory::create("darwin-initialization-alias")?; + fs::write(sandbox.path().join("Format"), b"foreign")?; + let refused = FilesystemStoreAdmission::initialize(sandbox.path()); + assert!(matches!( + refused, + Err(StoreInitializationError::Refused { + refusal: StoreInitializationRefusal::NotAPrefix { .. } + }) + )); + assert_eq!(fs::read(sandbox.path().join("Format"))?, b"foreign"); + assert_eq!(fs::read_dir(sandbox.path())?.count(), 1); + Ok(()) +} + // Finder can write this into any folder it displays. Refusal is deliberate until // an owner decides otherwise: the namespace admits no entry it cannot account for. #[test] From add45c881bb02ee79c45b1322d58dd0d340ce158 Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 4 Nov 2026 17:39:18 -0800 Subject: [PATCH 25/36] Test: name the expectation in the store admission contract (#198) --- tests/store_admission_contract.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/store_admission_contract.rs b/tests/store_admission_contract.rs index fc33b9cb..e9370557 100644 --- a/tests/store_admission_contract.rs +++ b/tests/store_admission_contract.rs @@ -20,7 +20,9 @@ fn every_writer_consumes_admitted_authority() { #[test] fn reopen_admits_the_marker_under_the_production_profile() { assert!(STORE_ADMISSION.contains("pub struct FilesystemStoreAdmission")); - assert!(STORE_ADMISSION.contains("pub fn reopen(store_root: &Path)")); + assert!(STORE_ADMISSION.contains( + "pub fn reopen(\n store_root: &Path,\n expectation: StoreExpectation," + )); assert!(STORE_ADMISSION.contains("pub fn initialize(")); assert!(STORE_ADMISSION.contains("filesystem_format_marker::admit")); assert!(STORE_ADMISSION.contains("filesystem_platform_profile::open")); From fb355d664ab8eb926f42eddf6d3cee4cc0c9cc9f Mon Sep 17 00:00:00 2001 From: James Ross Date: Wed, 4 Nov 2026 18:14:37 -0800 Subject: [PATCH 26/36] Fix: repair Linux test imports and roadmap links after the merge (#198) --- ROADMAP.md | 4 ++-- src/adapters/retention/reader_platform_law_tests.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 447f03da..6994a006 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -4,8 +4,8 @@ This page has one job: to list every feature Keep has, is building, or intends, and to break the unfinished ones into tasks precise enough to start. It is a plan. It is not evidence. The authoritative status of every requirement, with the test that proves it, remains the requirement ledgers: -[`segment-store-v1/requirements.md`](docs/formats/segment-store-v1/requirements.md), -[`segment-store-v2/requirements.md`](docs/formats/segment-store-v2/requirements.md), +[`segment-store-v1/requirements.md`](docs/formats/store/catalog-protocol-requirements.md), +[`segment-store-v2/requirements.md`](docs/formats/store/requirements.md), [`authenticated-reconstruction/requirements.md`](docs/invariants/authenticated-reconstruction/requirements.md), and [`verification/requirements.md`](docs/invariants/verification/requirements.md). diff --git a/src/adapters/retention/reader_platform_law_tests.rs b/src/adapters/retention/reader_platform_law_tests.rs index ca3c71ae..62fd8445 100644 --- a/src/adapters/retention/reader_platform_law_tests.rs +++ b/src/adapters/retention/reader_platform_law_tests.rs @@ -10,11 +10,11 @@ use std::fs; use std::io; use std::path::PathBuf; -use super::filesystem_retention_test_fixture::{initialized_store, populate_bundle}; +use super::filesystem_retention_test_fixture::populate_bundle; use crate::{ CatalogRestartByteLimit, CatalogRestartPolicy, DurableSnapshot, DurableStoreError, - FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, FilesystemWriterLock, - ReaderAttemptLimit, SegmentReadPolicy, + FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, ReaderAttemptLimit, + SegmentReadPolicy, }; #[test] From 7c03b179edff6c744cd8e1a6bf127e64bee8ede7 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 5 Nov 2026 01:06:34 -0800 Subject: [PATCH 27/36] Test: witness the initialization commit synchronization (#198) Initialization delegates every root synchronization to one synchronizer, which production binds to the real call. Two laws substitute it: one requires the commit synchronization after the marker link and before stage removal, while both names exist; the other fails it and requires an unadmitted, resumable store. The law previously named for a failed commit synchronization failed its observer before the synchronization ran. It is renamed for what it does. A mutant that made the synchronization a no-op passed the complete suite before this change. These laws witness that the protocol requests the synchronization. That the production call reaches stable storage is still established by reading it. --- src/adapters/store_initialization.rs | 2 + .../initialization_commit_tests.rs | 162 ++++++++++++++++++ .../initialization_execution.rs | 54 ++++-- .../initialization_resume_tests.rs | 20 +-- 4 files changed, 215 insertions(+), 23 deletions(-) create mode 100644 src/adapters/store_initialization/initialization_commit_tests.rs diff --git a/src/adapters/store_initialization.rs b/src/adapters/store_initialization.rs index 21b24e1e..2a2368cd 100644 --- a/src/adapters/store_initialization.rs +++ b/src/adapters/store_initialization.rs @@ -7,6 +7,8 @@ //! The protocol, its states, and its crash matrix are specified in //! `docs/formats/store/initialization.md`. +#[cfg(test)] +mod initialization_commit_tests; mod initialization_entries; mod initialization_error; mod initialization_execution; diff --git a/src/adapters/store_initialization/initialization_commit_tests.rs b/src/adapters/store_initialization/initialization_commit_tests.rs new file mode 100644 index 00000000..15bbadf6 --- /dev/null +++ b/src/adapters/store_initialization/initialization_commit_tests.rs @@ -0,0 +1,162 @@ +//! Laws: the commit point is one root synchronization between the marker link and stage removal. +//! +//! A real directory synchronization leaves nothing a test can observe, so these +//! laws substitute the root synchronizer. They witness that the protocol +//! requests the synchronization, in order, and what a failed one leaves. That +//! the production synchronizer reaches stable storage is not witnessed here. + +use std::cell::RefCell; +use std::io; +use std::path::Path; + +use cap_std::ambient_authority; + +use super::initialization_entries; +use super::initialization_execution::initialize_with; +use super::initialization_test_fixture::{TestResult, initialize}; +use super::{ + StoreInitializationBoundary as Boundary, StoreInitializationError as Error, + StoreInitializationPhase as Phase, StoreInitializationPrefix as Prefix, +}; +use crate::adapters::filesystem_directory::FilesystemDirectory as Dir; +use crate::adapters::filesystem_namespace_refusal::FilesystemNamespaceRefusal; +use crate::adapters::filesystem_test_sandbox::TestDirectory; +use crate::adapters::{FilesystemStoreAdmission, FilesystemStoreAdmissionError}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Event { + Boundary(Phase, Boundary), + /// One root synchronization and the marker names the root held when it ran. + RootSynchronization { + marker: bool, + stage: bool, + }, +} + +fn present(root: &Dir, name: &str) -> io::Result { + match root.symlink_metadata(name) { + Ok(_) => Ok(true), + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(false), + Err(source) => Err(source), + } +} + +/// What one initialization returned and every boundary and root synchronization it ran. +type Observed = (Result, Vec); + +/// Initializes `path`, failing the `fail`-th root synchronization when given. +fn observed(path: &Path, fail: Option) -> TestResult { + let root = Dir::open_ambient_dir(path, ambient_authority())?; + let events = RefCell::new(Vec::new()); + let mut observer = |phase: Phase, boundary: Boundary| -> io::Result<()> { + events.borrow_mut().push(Event::Boundary(phase, boundary)); + Ok(()) + }; + let mut synchronize = |root: &Dir| -> io::Result<()> { + let event = Event::RootSynchronization { + marker: present(root, "FORMAT")?, + stage: present(root, "FORMAT.next")?, + }; + let index = events + .borrow() + .iter() + .filter(|seen| matches!(seen, Event::RootSynchronization { .. })) + .count(); + events.borrow_mut().push(event); + if fail == Some(index) { + Err(io::Error::other("injected root synchronization failure")) + } else { + initialization_entries::synchronize_root(root) + } + }; + let outcome = + initialize_with(&root, &mut observer, &mut synchronize).map(|(_lock, found)| found); + Ok((outcome, events.into_inner())) +} + +const COMMIT: Event = Event::RootSynchronization { + marker: true, + stage: true, +}; + +#[test] +fn the_commit_synchronization_runs_after_the_link_and_before_stage_removal() -> TestResult<()> { + let sandbox = TestDirectory::create("initialization-commit-order")?; + + let (outcome, events) = observed(sandbox.path(), None)?; + + assert_eq!(outcome?, Prefix::Empty); + let link = events + .iter() + .position(|event| *event == Event::Boundary(Phase::LinkMarker, Boundary::After)) + .ok_or("the marker was never linked")?; + let commit: Vec = events.iter().skip(link).take(5).copied().collect(); + assert_eq!( + commit, + [ + Event::Boundary(Phase::LinkMarker, Boundary::After), + Event::Boundary(Phase::SynchronizeRootAfterMarker, Boundary::Before), + COMMIT, + Event::Boundary(Phase::SynchronizeRootAfterMarker, Boundary::After), + Event::Boundary(Phase::RemoveMarkerStage, Boundary::Before), + ] + ); + let synchronizations: Vec = events + .iter() + .filter(|event| matches!(event, Event::RootSynchronization { .. })) + .copied() + .collect(); + let root = |marker, stage| Event::RootSynchronization { marker, stage }; + assert_eq!( + synchronizations, + [ + root(false, false), + root(false, true), + COMMIT, + root(true, false) + ], + "each of the four root-synchronization phases requests exactly one" + ); + Ok(()) +} + +#[test] +fn a_failed_commit_synchronization_leaves_an_unadmitted_resumable_store() -> TestResult<()> { + let sandbox = TestDirectory::create("initialization-commit-sync-failure")?; + + let (outcome, events) = observed(sandbox.path(), Some(2))?; + + assert!(matches!( + outcome, + Err(Error::Io { + phase: Phase::SynchronizeRootAfterMarker, + .. + }) + )); + assert_eq!( + events.last(), + Some(&COMMIT), + "nothing may follow the failed commit synchronization" + ); + require_incomplete(sandbox.path())?; + assert_eq!(initialize(sandbox.path())?.found(), Prefix::MarkerLinked); + drop(FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?); + Ok(()) +} + +/// Requires admission to refuse `path` as an initialization that has not committed. +pub(super) fn require_incomplete(path: &Path) -> TestResult<()> { + let Err(FilesystemStoreAdmissionError::Namespace { source }) = + FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(path) + else { + return Err("an uncommitted initialization was not refused at the namespace".into()); + }; + let refusal = source + .get_ref() + .and_then(|cause| cause.downcast_ref::()); + assert_eq!( + refusal, + Some(&FilesystemNamespaceRefusal::InitializationIncomplete) + ); + Ok(()) +} diff --git a/src/adapters/store_initialization/initialization_execution.rs b/src/adapters/store_initialization/initialization_execution.rs index 7520b3e3..605e0e44 100644 --- a/src/adapters/store_initialization/initialization_execution.rs +++ b/src/adapters/store_initialization/initialization_execution.rs @@ -22,6 +22,14 @@ use crate::adapters::FilesystemWriterLock; pub(in crate::adapters) type PhaseObserver<'observer> = &'observer mut dyn FnMut(Phase, Boundary) -> io::Result<()>; +/// Synchronizes the store root. +/// +/// Production binds [`entries::synchronize_root`]. A law substitutes a +/// recording or failing one, because a real directory synchronization leaves +/// nothing a test can observe short of power loss. +pub(super) type RootSynchronizer<'synchronizer> = + &'synchronizer mut dyn FnMut(&Dir) -> io::Result<()>; + /// Classifies, locks, reclassifies, and completes initialization of `root`. /// /// Returns the retained writer lock and the state the call found. The first @@ -35,9 +43,27 @@ pub(in crate::adapters) type PhaseObserver<'observer> = pub(in crate::adapters) fn initialize( root: &Dir, observer: PhaseObserver<'_>, +) -> Result<(FilesystemWriterLock, Prefix), Error> { + initialize_with(root, observer, &mut entries::synchronize_root) +} + +/// Runs [`initialize`] with every root synchronization delegated to `synchronize_root`. +/// +/// # Errors +/// +/// Returns exactly what [`initialize`] returns; a synchronizer error fails the +/// phase that requested it. +pub(super) fn initialize_with( + root: &Dir, + observer: PhaseObserver<'_>, + synchronize_root: RootSynchronizer<'_>, ) -> Result<(FilesystemWriterLock, Prefix), Error> { let unlocked = initialization_observation::observe(root)?; - let mut execution = Execution { root, observer }; + let mut execution = Execution { + root, + observer, + synchronize_root, + }; let lock = execution.lock()?; let locked = initialization_observation::observe(root)?; execution.resume(locked)?; @@ -49,12 +75,13 @@ pub(in crate::adapters) fn initialize( Ok((lock, found)) } -struct Execution<'root, 'observer> { +struct Execution<'root, 'observer, 'synchronizer> { root: &'root Dir, observer: PhaseObserver<'observer>, + synchronize_root: RootSynchronizer<'synchronizer>, } -impl Execution<'_, '_> { +impl Execution<'_, '_, '_> { fn phase( &mut self, phase: Phase, @@ -66,6 +93,13 @@ impl Execution<'_, '_> { Ok(value) } + /// Runs one root synchronization as its own observed phase. + fn synchronize(&mut self, phase: Phase) -> Result<(), Error> { + (self.observer)(phase, Boundary::Before).map_err(|source| Error::io(phase, source))?; + (self.synchronize_root)(self.root).map_err(|source| Error::io(phase, source))?; + (self.observer)(phase, Boundary::After).map_err(|source| Error::io(phase, source)) + } + fn lock(&mut self) -> Result { let phase = Phase::OpenAndLockWriterFile; (self.observer)(phase, Boundary::Before).map_err(|source| Error::io(phase, source))?; @@ -75,7 +109,7 @@ impl Execution<'_, '_> { .map_err(|source| Error::io(phase, source))?; let lock = FilesystemWriterLock::initialize_in(directory) .map_err(|source| Error::WriterLock { source })?; - entries::synchronize_root(self.root).map_err(|source| Error::io(phase, source))?; + (self.synchronize_root)(self.root).map_err(|source| Error::io(phase, source))?; (self.observer)(phase, Boundary::After).map_err(|source| Error::io(phase, source))?; Ok(lock) } @@ -113,10 +147,7 @@ impl Execution<'_, '_> { self.phase(Phase::SynchronizeMarkerStage, |root| { stage.synchronize(root) })?; - self.phase( - Phase::SynchronizeRootAfterMarkerStage, - entries::synchronize_root, - )?; + self.synchronize(Phase::SynchronizeRootAfterMarkerStage)?; self.phase(Phase::AdmitReaderFence, admit_reader_fence)?; for directory in Directory::ALL { self.phase(Phase::AdmitDirectory(directory), |root| { @@ -128,12 +159,9 @@ impl Execution<'_, '_> { /// Commits with the root synchronization after the link, then removes the stage. fn commit(&mut self, stage: MarkerStage) -> Result<(), Error> { - self.phase(Phase::SynchronizeRootAfterMarker, entries::synchronize_root)?; + self.synchronize(Phase::SynchronizeRootAfterMarker)?; self.phase(Phase::RemoveMarkerStage, |root| stage.remove(root))?; - self.phase( - Phase::SynchronizeRootAfterCleanup, - entries::synchronize_root, - ) + self.synchronize(Phase::SynchronizeRootAfterCleanup) } } diff --git a/src/adapters/store_initialization/initialization_resume_tests.rs b/src/adapters/store_initialization/initialization_resume_tests.rs index 46e3897c..20942232 100644 --- a/src/adapters/store_initialization/initialization_resume_tests.rs +++ b/src/adapters/store_initialization/initialization_resume_tests.rs @@ -4,6 +4,7 @@ use std::fs; use std::sync::mpsc; use std::thread; +use super::initialization_commit_tests::require_incomplete; use super::initialization_test_fixture::{ COMMITTED, TestResult, initialize, initialize_until, inventory, recorded_boundaries, require_refusal, @@ -16,8 +17,7 @@ use super::{ }; use crate::adapters::filesystem_test_sandbox::TestDirectory; use crate::adapters::{ - CanonicalStoreFormatMarker, FilesystemStoreAdmission, FilesystemStoreAdmissionError, - WriterLockAcquireError, + CanonicalStoreFormatMarker, FilesystemStoreAdmission, WriterLockAcquireError, }; /// The state a root holds after exactly `completed` phases, by the protocol table. @@ -113,12 +113,16 @@ fn an_incomplete_marker_stage_is_discarded_and_rewritten() -> TestResult<()> { Ok(()) } +/// The observer fails before the commit synchronization is requested, so this +/// witnesses the linked prefix. The synchronization's own failure is witnessed +/// in `initialization_commit_tests`. #[test] -fn a_failed_commit_synchronization_leaves_an_unadmitted_resumable_store() -> TestResult<()> { - let sandbox = TestDirectory::create("initialization-commit-sync-failure")?; +fn an_interruption_before_the_commit_synchronization_leaves_an_unadmitted_resumable_store() +-> TestResult<()> { + let sandbox = TestDirectory::create("initialization-commit-interrupted")?; let observer: StoreInitializationObserver = Box::new(|phase, boundary| { if phase == Phase::SynchronizeRootAfterMarker && boundary == Boundary::Before { - Err(std::io::Error::other("injected synchronization failure")) + Err(std::io::Error::other("injected interruption")) } else { Ok(()) } @@ -136,11 +140,7 @@ fn a_failed_commit_synchronization_leaves_an_unadmitted_resumable_store() -> Tes .. }) )); - let refused = FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path()); - assert!(matches!( - refused, - Err(FilesystemStoreAdmissionError::Namespace { .. }) - )); + require_incomplete(sandbox.path())?; assert_eq!(initialize(sandbox.path())?.found(), Prefix::MarkerLinked); drop(FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks(sandbox.path())?); Ok(()) From 525071e33b043817bc0300113f37a4264b873f0a Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 5 Nov 2026 01:06:34 -0800 Subject: [PATCH 28/36] Test: run the catalog recovery laws on macOS (#198) The module was compiled only on Linux. Its three laws pass natively on default and case-sensitive APFS, so the gate goes. --- src/adapters/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index a520465e..a342a5a5 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -88,7 +88,7 @@ mod filesystem_catalog_publisher; mod filesystem_catalog_publisher_storage; #[cfg(test)] mod filesystem_catalog_publisher_tests; -#[cfg(all(test, target_os = "linux"))] +#[cfg(test)] mod filesystem_catalog_recovery_tests; mod filesystem_catalog_segment; mod filesystem_catalog_snapshot; From 0694ebff5a0973d27ab405d8a7b1f3d986f8fb85 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 5 Nov 2026 01:06:34 -0800 Subject: [PATCH 29/36] Test: type torn-copy refusals and count only real mixes (#198) The guards could pass on the two whole states, any error counted as a refusal, and the names claimed more than the laws enumerate. Refusals must now be a typed absent-or-malformed cause, the guards count only real mixes, empty files are skipped as untorn, and the names and header say what is and is not covered. --- tests/single_format/torn_copy_laws.rs | 91 +++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 13 deletions(-) diff --git a/tests/single_format/torn_copy_laws.rs b/tests/single_format/torn_copy_laws.rs index 81623f38..36d37c2b 100644 --- a/tests/single_format/torn_copy_laws.rs +++ b/tests/single_format/torn_copy_laws.rs @@ -1,15 +1,24 @@ //! This module owns the ADR-0010 law for a copy torn by a concurrent writer. //! -//! A copy taken while a writer publishes can hold any mix of the states before +//! A copy taken while a writer publishes can hold a mix of the states before //! and after that publication, and a partly copied file. These laws enumerate -//! those mixes directly, so they need no scheduling or timing. Every mix must +//! every mix of two quiescent published states, and one half-length file over +//! the newer state, so they need no scheduling or timing. Every such copy must //! refuse or return exactly the bytes a content identity names. +//! +//! They do not cover a copy that holds a writer's in-flight stage, a file at +//! another length than half, or a tear over the older state. use std::collections::BTreeMap; use std::fs; +use std::io; use std::path::{Path, PathBuf}; -use keep::{DurableStore, ReaderAttemptLimit, StoreExpectation}; +use keep::{ + CatalogRestartError, DurableOutcome, DurableReadError, DurableStore, DurableStoreError, + FilesystemRetentionSnapshotError, ReaderAttemptLimit, RetentionClosureVerificationError, + RetentionViewError, StoreExpectation, +}; use super::durable_fixture; use super::durable_sandbox::TestDirectory; @@ -24,7 +33,7 @@ const MAXIMUM_CHANGED_FILES: usize = 12; type Files = BTreeMap>; -/// How many mixes refused and how many read, so the laws cannot pass vacuously. +/// How many reads refused and how many returned bytes, over torn copies only. #[derive(Default)] struct Outcomes { refused: usize, @@ -88,7 +97,45 @@ fn torn(name: &str, base: &Path, overlay: &[(&Path, &[u8])]) -> TestResult bool { + matches!( + source.kind(), + io::ErrorKind::NotFound | io::ErrorKind::InvalidData + ) +} + +/// Whether `refusal` is evidence that the copy is incomplete or damaged. +/// +/// Anything else, such as a permission, resource, or locator failure, is an +/// operational failure of the test and must not count as a refusal. +fn lawful(refusal: &DurableOutcome) -> bool { + use FilesystemRetentionSnapshotError as Snapshot; + match refusal { + DurableOutcome::Read(DurableReadError::BlobMissing { .. }) => true, + DurableOutcome::Store(DurableStoreError::Closure { source, .. }) => matches!( + **source, + RetentionClosureVerificationError::MissingMember { .. } + ), + DurableOutcome::Store(DurableStoreError::Snapshot(source)) => match &**source { + Snapshot::Admission { source } + | Snapshot::Root { source } + | Snapshot::View { + source: RetentionViewError::Io { source, .. }, + } => evidenced(source), + Snapshot::Catalog { source } => match source { + CatalogRestartError::Io { source, .. } + | CatalogRestartError::SegmentIo { source, .. } => evidenced(source), + CatalogRestartError::Length { .. } | CatalogRestartError::Segment { .. } => true, + _ => false, + }, + _ => false, + }, + _ => false, + } +} + +/// Requires `content` to reconstruct exactly, or to refuse lawfully with a verified prefix. fn exact_or_refused(root: &Path, content: &[u8], outcomes: &mut Outcomes) -> TestResult<()> { let target = populated_store::identify(content)?.target; let store = DurableStore::open( @@ -103,7 +150,11 @@ fn exact_or_refused(root: &Path, content: &[u8], outcomes: &mut Outcomes) -> Tes assert_eq!(output, content, "a torn copy returned other bytes"); outcomes.read = outcomes.read.saturating_add(1); } - Err(_refusal) => { + Err(refusal) => { + assert!( + lawful(&refusal), + "a torn copy failed for another cause than its own state: {refusal:?}" + ); assert!( content.starts_with(&output), "a refused read emitted bytes its identity does not name" @@ -120,10 +171,11 @@ fn both_blobs(root: &Path, outcomes: &mut Outcomes) -> TestResult<()> { } #[test] -fn every_mix_of_two_published_states_refuses_or_returns_exact_bytes() -> TestResult<()> { +fn every_mix_of_two_quiescent_published_states_refuses_or_returns_exact_bytes() -> TestResult<()> { let (base, _before, changed, after) = two_states("torn-mix")?; - let mut outcomes = Outcomes::default(); - for mask in 0_u32..(1_u32 << changed.len()) { + let newer = (1_u32 << changed.len()).saturating_sub(1); + let mut torn_outcomes = Outcomes::default(); + for mask in 0_u32..=newer { let mut overlay: Vec<(&Path, &[u8])> = Vec::new(); for (bit, path) in changed.iter().enumerate() { if mask & (1_u32 << bit) != 0 { @@ -132,15 +184,24 @@ fn every_mix_of_two_published_states_refuses_or_returns_exact_bytes() -> TestRes } } let copy = torn("torn-mix-copy", base.path(), &overlay)?; - both_blobs(copy.path(), &mut outcomes)?; + // The two ends are whole states. They must hold the law, but only a + // real mix may satisfy the guards below. + let mut whole_state = Outcomes::default(); + let outcomes = if mask == 0 || mask == newer { + &mut whole_state + } else { + &mut torn_outcomes + }; + both_blobs(copy.path(), outcomes)?; } - assert!(outcomes.read > 0, "no torn mix read anything"); - assert!(outcomes.refused > 0, "no torn mix refused anything"); + assert!(torn_outcomes.read > 0, "no torn mix read anything"); + assert!(torn_outcomes.refused > 0, "no torn mix refused anything"); Ok(()) } #[test] -fn every_partly_copied_file_refuses_or_returns_exact_bytes() -> TestResult<()> { +fn a_half_length_copy_of_any_one_file_over_the_newer_state_refuses_or_returns_exact_bytes() +-> TestResult<()> { let (base, before, changed, after) = two_states("torn-partial")?; let mut outcomes = Outcomes::default(); let whole: Vec<(&Path, &[u8])> = after @@ -151,6 +212,10 @@ fn every_partly_copied_file_refuses_or_returns_exact_bytes() -> TestResult<()> { for path in before.keys().chain(changed.iter()) { let source = after.get(path).or_else(|| before.get(path)); let bytes = source.ok_or("the file left both states")?; + if bytes.is_empty() { + // Half of an empty lock file is the whole file: not a tear. + continue; + } let half = bytes .get(..bytes.len() >> 1) .ok_or("half is out of range")?; From a389a80123b96d078f9cebd9af08d997641bbdfd Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 5 Nov 2026 01:06:34 -0800 Subject: [PATCH 30/36] Test: keep process-group socket paths under the macOS sun_path bound Two process-group laws failed under a long temporary directory because their Unix socket path exceeded macOS's 104-byte sun_path bound, and the child panicked before readiness. Their sibling already used short names. With short names both pass natively; ProcessGroup itself was not at fault. --- .../bounded_process/process_group/tests.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/xtask/src/bounded_process/process_group/tests.rs b/xtask/src/bounded_process/process_group/tests.rs index 1e5017a6..26dcbcdb 100644 --- a/xtask/src/bounded_process/process_group/tests.rs +++ b/xtask/src/bounded_process/process_group/tests.rs @@ -58,10 +58,12 @@ fn spawn_ready_descendant( #[test] fn terminal_interrupt_terminates_the_isolated_descendant_group() -> Result<(), Box> { - let directory = TestDirectory::create("process-group-interrupt")?; - let ready = directory.path().join("ready"); - let child_ready = directory.path().join("child-ready"); - let socket = directory.path().join("descendant.sock"); + // Short names keep the socket path under the 104-byte `sun_path` bound on + // macOS when the temporary directory is itself a long path. + let directory = TestDirectory::create("pi")?; + let ready = directory.path().join("r"); + let child_ready = directory.path().join("cr"); + let socket = directory.path().join("d"); let listener = readiness_listener(&ready)?; let executable = env::current_exe()?; let mut supervisor = Command::new(executable) @@ -121,10 +123,11 @@ fn inherited_descendant_pipe_obeys_the_process_deadline() -> Result<(), Box Result<(), Box> { - let directory = TestDirectory::create("process-group-cleanup")?; - let ready = directory.path().join("ready"); - let child_ready = directory.path().join("child-ready"); - let socket = directory.path().join("descendant.sock"); + // Short names for the `sun_path` bound, as above. + let directory = TestDirectory::create("pc")?; + let ready = directory.path().join("r"); + let child_ready = directory.path().join("cr"); + let socket = directory.path().join("d"); let listener = readiness_listener(&ready)?; let executable = env::current_exe()?; let mut command = Command::new(executable); From 0037255f19a54eaa1723d7657f9fdb80b532080f Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 5 Nov 2026 01:06:34 -0800 Subject: [PATCH 31/36] Feat!: require an expectation on the catalog snapshot load (#198) FilesystemCatalogSnapshot::load was a public open that took no StoreExpectation, so a caller could lose the check by using it. It now takes one and refuses with CatalogRestartError::Expectation. Verification loads still name none. An expectation can now be kept across processes. ExpectedCatalog is the caller's claim: from_recorded rebuilds it from a generation and 32 digest bytes, and digest_bytes returns the bytes to record. It no longer hands out a CatalogDigest, which would have let a claim pass as a verified digest. The observed side of a mismatch is the new ObservedCatalog, and the mismatch's Display reports both. The retained-catalog crash sequence now requires the catalog recorded before an interrupted publication to admit while generation one is published and to refuse, reporting generation two, once HEAD moves. Keep cannot tell that successor from another store, so accepting it stays the caller's decision. BREAKING CHANGE: FilesystemCatalogSnapshot::load takes a StoreExpectation. ExpectedCatalog::digest is replaced by digest_bytes, and StoreExpectationMismatch::observed returns Option. --- src/adapters/catalog_restart_error.rs | 11 +- src/adapters/catalog_restart_loader.rs | 46 ++++- src/adapters/durable/store.rs | 13 +- src/adapters/exports.rs | 4 +- .../filesystem_catalog_publisher_tests.rs | 6 +- src/adapters/filesystem_catalog_snapshot.rs | 23 ++- ...recovery_next_head_finalization_storage.rs | 14 +- .../filesystem_retention_snapshot.rs | 6 +- src/adapters/store_expectation.rs | 178 ++++++++++++++++-- src/adapters/verification_failure_class.rs | 5 +- src/adapters/verification_ingress.rs | 3 +- src/lib.rs | 2 +- .../refusal_laws.rs | 6 +- tests/catalog_restart.rs | 6 +- tests/catalog_restart/refusal_laws.rs | 42 ++++- tests/catalog_writer/finalization_laws.rs | 6 +- tests/catalog_writer/format_marker_laws.rs | 2 +- tests/catalog_writer/recovery_fixture.rs | 6 +- tests/single_format/acceptance_laws.rs | 12 +- tests/single_format/expectation_laws.rs | 56 +++++- tests/support/populated_store.rs | 3 +- tests/support/successor_catalog.rs | 6 +- xtask/src/durability_crash_matrix/restart.rs | 1 + .../restart/recorded_catalog.rs | 56 ++++++ .../restart/retained_catalog.rs | 22 ++- .../restart/semantic.rs | 8 +- .../restart/stage_recovery.rs | 5 +- 27 files changed, 460 insertions(+), 88 deletions(-) create mode 100644 xtask/src/durability_crash_matrix/restart/recorded_catalog.rs diff --git a/src/adapters/catalog_restart_error.rs b/src/adapters/catalog_restart_error.rs index 1b868ceb..36a38341 100644 --- a/src/adapters/catalog_restart_error.rs +++ b/src/adapters/catalog_restart_error.rs @@ -8,7 +8,7 @@ use std::io; use super::{ CatalogAdmissionError, CatalogDecodeError, CatalogRestartArtifact, CatalogRestartPhase, CatalogSnapshotError, FilesystemStoreAdmissionError, PublicationHeadDecodeError, SegmentDigest, - SegmentReadError, + SegmentReadError, StoreExpectationMismatch, }; use crate::{CatalogDigest, CatalogGeneration, CatalogLength}; @@ -141,6 +141,11 @@ pub enum CatalogRestartError { /// Preserved snapshot refusal. source: CatalogSnapshotError, }, + /// The loaded store does not hold the catalog its caller expected. + Expectation { + /// The expected and observed published catalogs. + source: StoreExpectationMismatch, + }, } impl CatalogRestartError { @@ -196,6 +201,9 @@ impl fmt::Display for CatalogRestartError { } Self::CatalogAdmission { .. } => formatter.write_str("catalog record binding failed"), Self::Snapshot { .. } => formatter.write_str("restart snapshot binding failed"), + Self::Expectation { .. } => { + formatter.write_str("the store does not hold the expected published catalog") + } } } } @@ -215,6 +223,7 @@ impl Error for CatalogRestartError { Self::Segment { source, .. } => Some(source), Self::CatalogAdmission { source } => Some(source), Self::Snapshot { source } => Some(source), + Self::Expectation { source } => Some(source), Self::NotRegular { .. } | Self::Length { .. } | Self::LengthArithmetic { .. } diff --git a/src/adapters/catalog_restart_loader.rs b/src/adapters/catalog_restart_loader.rs index ebefe7e1..b654de80 100644 --- a/src/adapters/catalog_restart_loader.rs +++ b/src/adapters/catalog_restart_loader.rs @@ -6,8 +6,8 @@ use crate::CatalogLength; use super::{ CatalogRestartArtifact, CatalogRestartError, CatalogRestartPhase, CatalogRestartPolicy, - ChecksummedCatalog, ChecksummedPublicationHead, ExpectedCatalog, FilesystemCatalogSnapshot, - catalog_restart_io, catalog_restart_segments, filesystem_store_admission, + ChecksummedCatalog, ChecksummedPublicationHead, FilesystemCatalogSnapshot, ObservedCatalog, + StoreExpectation, catalog_restart_io, catalog_restart_segments, filesystem_store_admission, filesystem_store_namespace, physical_pool_name, }; @@ -20,10 +20,12 @@ const HEAD_LENGTH: u64 = 128; /// /// Admission is the reader law: the exact namespace, which may hold an /// in-flight `head.next`, and the canonical `FORMAT` marker. A root without the -/// marker refuses before any catalog artifact is opened. +/// marker refuses before any catalog artifact is opened. `expectation` is +/// compared with the loaded snapshot, or with the absence of `HEAD`. pub(super) fn load( root: &Path, policy: CatalogRestartPolicy, + expectation: StoreExpectation, ) -> Result { let directory = catalog_restart_io::open_root(root)?; filesystem_store_admission::admit_root( @@ -33,7 +35,39 @@ pub(super) fn load( .map_err(|source| CatalogRestartError::Admission { source: Box::new(source), })?; - load_from_directory(&directory, HEAD_NAME, policy) + match load_from_directory(&directory, HEAD_NAME, policy) { + Ok(snapshot) => { + let observed = ObservedCatalog::new(snapshot.generation(), snapshot.catalog_digest()); + require(expectation, Some(observed))?; + Ok(snapshot) + } + Err(source) => { + if missing_head(&source) { + require(expectation, None)?; + } + Err(source) + } + } +} + +fn require( + expectation: StoreExpectation, + observed: Option, +) -> Result<(), CatalogRestartError> { + expectation + .require(observed) + .map_err(|source| CatalogRestartError::Expectation { source }) +} + +/// Whether `error` is exactly the absence of the published head. +pub(super) fn missing_head(error: &CatalogRestartError) -> bool { + matches!( + error, + CatalogRestartError::Io { + phase: CatalogRestartPhase::OpenHead, + source, + } if source.kind() == std::io::ErrorKind::NotFound + ) } /// Reads the published catalog's coordinates from `HEAD`, or `None` when none is published. @@ -41,7 +75,7 @@ pub(super) fn load( /// The checksummed head names the catalog; this neither opens nor verifies it. pub(super) fn observe_published_catalog( directory: &crate::adapters::filesystem_directory::FilesystemDirectory, -) -> Result, CatalogRestartError> { +) -> Result, CatalogRestartError> { match directory.symlink_metadata(HEAD_NAME) { Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(source) => { @@ -55,7 +89,7 @@ pub(super) fn observe_published_catalog( let head_bytes = read_head_bytes(directory, HEAD_NAME)?; let head = ChecksummedPublicationHead::decode(&head_bytes) .map_err(|source| CatalogRestartError::Head { source })?; - Ok(Some(ExpectedCatalog::new( + Ok(Some(ObservedCatalog::new( head.generation(), head.catalog_digest(), ))) diff --git a/src/adapters/durable/store.rs b/src/adapters/durable/store.rs index cae28c9d..78d7ced2 100644 --- a/src/adapters/durable/store.rs +++ b/src/adapters/durable/store.rs @@ -31,7 +31,9 @@ use crate::{BlobId, ByteRange, LayoutId}; /// The caller owns output visibility: a failed write can leave an untrusted /// prefix. The receipt names the view; it does not extend retention after the /// snapshot is dropped. This example limits aggregate catalog-selected segment bytes; -/// catalog bytes and decoded metadata allocate separately. +/// catalog bytes and decoded metadata allocate separately. It names no +/// expectation, which accepts any valid store at the path; a caller that has +/// recorded a catalog names it instead, as [`StoreExpectation`] describes. /// /// ```no_run /// #[cfg(any(target_os = "linux", target_os = "macos"))] @@ -64,10 +66,11 @@ pub struct DurableStore { } impl DurableStore { - /// Names the store at `root`, reading under `policy` and collecting a - /// consistent view within `limit` attempts. + /// Names the store at `root` without opening it: nothing, including + /// `expectation`, is checked until a snapshot is pinned. /// - /// Allocates an absolute locator and queries the current directory for a + /// Reads use `policy` and collect a consistent view within `limit` + /// attempts. Allocates an absolute locator and queries the current directory for a /// relative path. It does not open the store or prove that it exists. /// Later working-directory changes cannot retarget this handle. The path /// remains a locator, not a content identity or a pinned directory handle; @@ -107,7 +110,7 @@ impl DurableStore { /// # Errors /// /// Returns [`DurableStoreError`] at the exact admission, fence, - /// collection, or retained-root refusal. + /// collection, expectation, or retained-root refusal. pub fn snapshot(&self) -> Result { DurableSnapshot::open(&self.root, self.policy, self.limit, self.expectation) } diff --git a/src/adapters/exports.rs b/src/adapters/exports.rs index 5a6980bb..091ce13a 100644 --- a/src/adapters/exports.rs +++ b/src/adapters/exports.rs @@ -107,7 +107,9 @@ pub use super::segment_write_error::SegmentWriteError; pub use super::segment_write_phase::{SegmentDurabilityPhase, SegmentWritePhase}; pub use super::staged_segment::StagedSegment; pub use super::storage_profile_id_text_error::StorageProfileIdParseError; -pub use super::store_expectation::{ExpectedCatalog, StoreExpectation, StoreExpectationMismatch}; +pub use super::store_expectation::{ + ExpectedCatalog, ObservedCatalog, StoreExpectation, StoreExpectationMismatch, +}; pub use super::store_format::*; #[cfg(any(test, feature = "repository-tasks"))] pub use super::store_initialization::StoreInitializationObserver; diff --git a/src/adapters/filesystem_catalog_publisher_tests.rs b/src/adapters/filesystem_catalog_publisher_tests.rs index fe96742e..b85f8f1a 100644 --- a/src/adapters/filesystem_catalog_publisher_tests.rs +++ b/src/adapters/filesystem_catalog_publisher_tests.rs @@ -63,7 +63,11 @@ fn successful_publication_materializes_only_the_exact_durable_view() -> Result<( assert!(!store.staging().join("current.seg").exists()); assert!(!store.staging().join("current.cat").exists()); assert!(!store.path().join("head.next").exists()); - let loaded = FilesystemCatalogSnapshot::load(store.path(), restart_policy()?)?; + let loaded = FilesystemCatalogSnapshot::load( + store.path(), + restart_policy()?, + crate::StoreExpectation::NoExpectation, + )?; assert_eq!(loaded.catalog_digest(), receipt.catalog_digest()); store.remove() } diff --git a/src/adapters/filesystem_catalog_snapshot.rs b/src/adapters/filesystem_catalog_snapshot.rs index f27fbe30..a83841c9 100644 --- a/src/adapters/filesystem_catalog_snapshot.rs +++ b/src/adapters/filesystem_catalog_snapshot.rs @@ -5,7 +5,7 @@ use std::path::Path; use super::loaded_segment::LoadedSegment; use super::{ AdmittedSegment, CatalogRestartError, CatalogRestartPolicy, CatalogSnapshot, - ChecksummedCatalog, ChecksummedPublicationHead, catalog_restart_loader, + ChecksummedCatalog, ChecksummedPublicationHead, StoreExpectation, catalog_restart_loader, }; use crate::{CatalogDigest, CatalogGeneration}; @@ -29,18 +29,27 @@ impl FilesystemCatalogSnapshot { /// /// Admission requires the exact store namespace, which may hold an in-flight /// `head.next`, and the canonical `FORMAT` marker; it takes no lock and no - /// reader fence. The returned owner retains bounded segment bytes so every - /// later logical lookup remains pinned to this immutable generation. + /// reader fence, and it does not admit a platform profile. The returned + /// owner retains bounded segment bytes so every later logical lookup remains + /// pinned to this immutable generation. `expectation` is compared with the + /// loaded generation and digest after the snapshot verifies. /// /// # Errors /// /// Returns [`CatalogRestartError::Admission`] for a root that is not a - /// committed store, including one without `FORMAT`, and otherwise + /// committed store, including one without `FORMAT`; + /// [`CatalogRestartError::Expectation`] when the store publishes another + /// catalog than [`StoreExpectation::Catalog`] names, or none; and otherwise /// [`CatalogRestartError`] for filesystem refusal, malformed or /// noncanonical bytes, coordinate disagreement, resource-limit refusal, or - /// failed catalog-to-record admission. - pub fn load(root: &Path, policy: CatalogRestartPolicy) -> Result { - catalog_restart_loader::load(root, policy) + /// failed catalog-to-record admission. A damaged store refuses at its own + /// boundary before the expectation is compared. + pub fn load( + root: &Path, + policy: CatalogRestartPolicy, + expectation: StoreExpectation, + ) -> Result { + catalog_restart_loader::load(root, policy, expectation) } /// Returns the exact generation selected during restart. diff --git a/src/adapters/filesystem_recovery_next_head_finalization_storage.rs b/src/adapters/filesystem_recovery_next_head_finalization_storage.rs index 6c80a93c..9fda5b67 100644 --- a/src/adapters/filesystem_recovery_next_head_finalization_storage.rs +++ b/src/adapters/filesystem_recovery_next_head_finalization_storage.rs @@ -3,7 +3,7 @@ use std::io; use super::{ - CatalogRestartError, CatalogRestartPhase, FilesystemRecoveryNextHeadFinalizer, + CatalogRestartError, FilesystemRecoveryNextHeadFinalizer, RecoveryNextHeadFinalizationReadiness, RecoveryNextHeadFinalizationRequest, RecoveryNextHeadFinalizationStorage, RecoveryNextHeadFinalizationStorageError, RecoveryNextHeadFinalizationTarget, RecoveryStageNamespacePhase, catalog_restart_loader, @@ -91,7 +91,7 @@ fn current_readiness( }) } } - Err(source) if missing_head(&source) => { + Err(source) if catalog_restart_loader::missing_head(&source) => { if request.expectation().current_generation().is_none() { Ok(RecoveryNextHeadFinalizationReadiness::Ready) } else { @@ -105,16 +105,6 @@ fn current_readiness( } } -fn missing_head(error: &CatalogRestartError) -> bool { - matches!( - error, - CatalogRestartError::Io { - phase: CatalogRestartPhase::OpenHead, - source, - } if source.kind() == io::ErrorKind::NotFound - ) -} - fn current_view(source: CatalogRestartError) -> RecoveryNextHeadFinalizationStorageError { RecoveryNextHeadFinalizationStorageError::CurrentView { source: Box::new(source), diff --git a/src/adapters/retention/filesystem_retention_snapshot.rs b/src/adapters/retention/filesystem_retention_snapshot.rs index cdde597a..4ad2c5ec 100644 --- a/src/adapters/retention/filesystem_retention_snapshot.rs +++ b/src/adapters/retention/filesystem_retention_snapshot.rs @@ -16,8 +16,8 @@ use crate::adapters::filesystem_exact_record::{ self as exact_record, ExactRecordError, ExactRecordRefusal, }; use crate::adapters::{ - CatalogRestartError, CatalogRestartPolicy, ChecksummedPublicationHead, ExpectedCatalog, - FilesystemCatalogSnapshot, StoreExpectation, filesystem_format_marker, + CatalogRestartError, CatalogRestartPolicy, ChecksummedPublicationHead, + FilesystemCatalogSnapshot, ObservedCatalog, StoreExpectation, filesystem_format_marker, filesystem_platform_profile, filesystem_store_namespace, publication_head_decoder, }; use crate::{RetentionHead, RetentionManifest, RetentionNamespaceDigest}; @@ -173,7 +173,7 @@ impl FilesystemRetentionSnapshot { }; let view = collect(&mut source, limit)?; expectation - .require(Some(ExpectedCatalog::new( + .require(Some(ObservedCatalog::new( view.catalog.generation(), view.catalog.catalog_digest(), ))) diff --git a/src/adapters/store_expectation.rs b/src/adapters/store_expectation.rs index 64493842..e9e7911e 100644 --- a/src/adapters/store_expectation.rs +++ b/src/adapters/store_expectation.rs @@ -5,19 +5,57 @@ use std::fmt; use crate::{CatalogDigest, CatalogGeneration}; -/// The exact published catalog a caller recorded outside the store. +/// The published catalog a caller recorded outside the store. /// -/// A publication receipt or a pinned view supplies both values. +/// This is the caller's claim, not a verified digest: Keep only compares it +/// with what an opened store publishes. A publication receipt or a pinned view +/// supplies both values to [`Self::new`]. To carry the expectation into a later +/// process, record [`Self::generation`] as its `u64` and [`Self::digest_bytes`], +/// then rebuild it with [`Self::from_recorded`]. Keep stores neither value for +/// the caller; where they are kept, and how that record is protected, is the +/// caller's responsibility. +/// +/// ``` +/// use keep::{CatalogGeneration, ExpectedCatalog, StoreExpectation}; +/// +/// # fn main() -> Result<(), Box> { +/// // Recorded by an earlier process, from a publication receipt or a view. +/// let recorded_generation: u64 = 7; +/// let recorded_digest: [u8; 32] = [0x5a; 32]; +/// +/// let expected = ExpectedCatalog::from_recorded( +/// CatalogGeneration::new(recorded_generation)?, +/// recorded_digest, +/// ); +/// let expectation = StoreExpectation::Catalog(expected); +/// assert_eq!(expected.generation().get(), recorded_generation); +/// assert_eq!(expected.digest_bytes(), recorded_digest); +/// # let _ = expectation; +/// # Ok(()) +/// # } +/// ``` #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ExpectedCatalog { generation: CatalogGeneration, - digest: CatalogDigest, + digest: [u8; 32], } impl ExpectedCatalog { - /// Names one exact catalog generation and digest. + /// Names the exact catalog a receipt or view reported in this process. #[must_use] pub const fn new(generation: CatalogGeneration, digest: CatalogDigest) -> Self { + Self { + generation, + digest: *digest.as_bytes(), + } + } + + /// Rebuilds an expectation from the two values a caller recorded earlier. + /// + /// Nothing is verified here. A wrong record refuses the right store, and a + /// record an attacker can rewrite protects nothing. + #[must_use] + pub const fn from_recorded(generation: CatalogGeneration, digest: [u8; 32]) -> Self { Self { generation, digest } } @@ -26,7 +64,35 @@ impl ExpectedCatalog { self.generation } - /// The expected catalog digest. + /// The expected catalog digest, as the exact bytes to record. + #[must_use] + pub const fn digest_bytes(self) -> [u8; 32] { + self.digest + } + + fn names(self, observed: ObservedCatalog) -> bool { + self.generation == observed.generation && self.digest == *observed.digest.as_bytes() + } +} + +/// The catalog an opened store's checksummed `HEAD` names. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ObservedCatalog { + generation: CatalogGeneration, + digest: CatalogDigest, +} + +impl ObservedCatalog { + pub(super) const fn new(generation: CatalogGeneration, digest: CatalogDigest) -> Self { + Self { generation, digest } + } + + /// The published catalog generation. + pub const fn generation(self) -> CatalogGeneration { + self.generation + } + + /// The published catalog digest. pub const fn digest(self) -> CatalogDigest { self.digest } @@ -36,8 +102,65 @@ impl ExpectedCatalog { /// /// A store's own records cannot show that it is the store the caller meant, or /// that newer history existed, because a restored store carries consistent old -/// records. Only state kept outside the store can. Every open therefore requires -/// this choice, and it has no default, so no caller loses the check by omission. +/// records. Only state kept outside the store can. Every normal open therefore +/// requires this choice, and it has no default: [`FilesystemStoreAdmission::reopen`], +/// [`FilesystemCatalogSnapshot::load`], [`FilesystemRetentionSnapshot::load`], +/// [`DurableSnapshot::open`], and [`DurableStore::open`]. Recovery tools, +/// verification loads, and initialization take none, so a caller of those routes +/// does not get this check. +/// +/// # Keeping an expectation current +/// +/// An expectation names one catalog, so it must follow every publication: after +/// a receipt, record the receipt's generation and digest before relying on the +/// old ones again. Publication and that record are two steps, and a process can +/// die between them. The store then holds the successor while the caller still +/// holds its predecessor, and every open refuses with +/// [`StoreExpectationMismatch`]. Keep cannot tell that successor from a +/// different store through this interface. Accepting what +/// [`StoreExpectationMismatch::observed`] reports is the caller's own trust +/// decision, made with evidence from outside the store; see "Recovering a stale +/// expectation" in `docs/formats/store/recovery.md`. +/// +/// ```no_run +/// # #[cfg(any(target_os = "linux", target_os = "macos"))] +/// # fn later_open( +/// # root: &std::path::Path, +/// # policy: keep::CatalogRestartPolicy, +/// # recorded: (u64, [u8; 32]), +/// # ) -> Result<(), Box> { +/// use keep::{ +/// CatalogGeneration, DurableSnapshot, DurableStoreError, ExpectedCatalog, +/// FilesystemRetentionSnapshotError, ReaderAttemptLimit, StoreExpectation, +/// }; +/// +/// let expected = ExpectedCatalog::from_recorded(CatalogGeneration::new(recorded.0)?, recorded.1); +/// let expectation = StoreExpectation::Catalog(expected); +/// match DurableSnapshot::open(root, policy, ReaderAttemptLimit::DEFAULT, expectation) { +/// // The store holds exactly the recorded catalog. +/// Ok(snapshot) => assert_eq!(snapshot.view().catalog_generation(), expected.generation()), +/// Err(DurableStoreError::Snapshot(error)) => match *error { +/// // Another catalog, or none. `source.observed()` reports which; whether +/// // to trust and record it is decided outside Keep. +/// FilesystemRetentionSnapshotError::Expectation { source } => return Err(source.into()), +/// other => return Err(other.into()), +/// }, +/// Err(other) => return Err(other.into()), +/// } +/// # Ok(()) +/// # } +/// ``` +/// +/// # Limits +/// +/// Two stores that publish the same catalog bytes are indistinguishable, and a +/// store expected to have published nothing has no representation here. +/// +/// [`FilesystemStoreAdmission::reopen`]: crate::FilesystemStoreAdmission::reopen +/// [`FilesystemCatalogSnapshot::load`]: crate::FilesystemCatalogSnapshot::load +/// [`FilesystemRetentionSnapshot::load`]: crate::FilesystemRetentionSnapshot::load +/// [`DurableSnapshot::open`]: crate::DurableSnapshot::open +/// [`DurableStore::open`]: crate::DurableStore::open #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum StoreExpectation { /// Refuse unless the store's published catalog is exactly this one. @@ -47,6 +170,10 @@ pub enum StoreExpectation { Catalog(ExpectedCatalog), /// The caller has no expectation and accepts any valid store at this path, /// including a different store or an older state of the intended one. + /// + /// This is the honest choice for a first open, before anything is recorded. + /// A store that has published no catalog still refuses on the read path, + /// which has nothing to read. NoExpectation, } @@ -54,13 +181,14 @@ impl StoreExpectation { /// Compares the observed published catalog, `None` meaning none is published. pub(super) fn require( self, - observed: Option, + observed: Option, ) -> Result<(), StoreExpectationMismatch> { - match self { - Self::Catalog(expected) if observed != Some(expected) => { + match (self, observed) { + (Self::NoExpectation, _) => Ok(()), + (Self::Catalog(expected), Some(found)) if expected.names(found) => Ok(()), + (Self::Catalog(expected), observed) => { Err(StoreExpectationMismatch { expected, observed }) } - Self::Catalog(_) | Self::NoExpectation => Ok(()), } } } @@ -69,7 +197,7 @@ impl StoreExpectation { #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct StoreExpectationMismatch { expected: ExpectedCatalog, - observed: Option, + observed: Option, } impl StoreExpectationMismatch { @@ -81,15 +209,37 @@ impl StoreExpectationMismatch { /// The catalog the store has published, or `None` when it has published none. #[must_use] - pub const fn observed(self) -> Option { + pub const fn observed(self) -> Option { self.observed } } impl fmt::Display for StoreExpectationMismatch { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("the store does not hold the expected published catalog") + write!( + formatter, + "expected published catalog generation {} digest ", + self.expected.generation.get() + )?; + write_hex(formatter, &self.expected.digest)?; + match self.observed { + None => formatter.write_str(", but the store has published no catalog"), + Some(observed) => { + write!( + formatter, + ", but the store holds generation {} digest ", + observed.generation.get() + )?; + write_hex(formatter, observed.digest.as_bytes()) + } + } } } +fn write_hex(formatter: &mut fmt::Formatter<'_>, bytes: &[u8; 32]) -> fmt::Result { + bytes + .iter() + .try_for_each(|byte| write!(formatter, "{byte:02x}")) +} + impl Error for StoreExpectationMismatch {} diff --git a/src/adapters/verification_failure_class.rs b/src/adapters/verification_failure_class.rs index 6dd0a7cd..b59d4824 100644 --- a/src/adapters/verification_failure_class.rs +++ b/src/adapters/verification_failure_class.rs @@ -87,7 +87,10 @@ pub(super) fn catalog_class(error: &CatalogRestartError) -> FailureClass { | CatalogRestartError::SegmentIndexLength | CatalogRestartError::SegmentIndexAllocation { .. } | CatalogRestartError::RetainedSegmentBytes { .. } - | CatalogRestartError::RetainedSegmentByteArithmetic { .. } => FailureClass::Operational, + | CatalogRestartError::RetainedSegmentByteArithmetic { .. } + // Verification loads name no expectation, so this cannot arise from one. A + // mismatch is in any case not evidence of damage. + | CatalogRestartError::Expectation { .. } => FailureClass::Operational, CatalogRestartError::Admission { source } => store_admission_class(source), CatalogRestartError::Segment { source, .. } => segment_class(source), CatalogRestartError::CatalogAdmission { source } => admission_class(source), diff --git a/src/adapters/verification_ingress.rs b/src/adapters/verification_ingress.rs index 5748489c..a9e7f821 100644 --- a/src/adapters/verification_ingress.rs +++ b/src/adapters/verification_ingress.rs @@ -60,7 +60,8 @@ impl FilesystemCatalogSnapshot { root: &Path, policy: CatalogRestartPolicy, ) -> Result { - Self::load(root, policy).map_err(catalog_error) + // Verification examines whatever the path holds, so it names no expectation. + Self::load(root, policy, crate::StoreExpectation::NoExpectation).map_err(catalog_error) } /// Reports the requested catalog evidence over this exact owned view. diff --git a/src/lib.rs b/src/lib.rs index 67575bd9..1108f8bd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -70,7 +70,7 @@ mod retention; mod segment_digest; mod verification; -pub use adapters::{ExpectedCatalog, StoreExpectation, StoreExpectationMismatch}; +pub use adapters::{ExpectedCatalog, ObservedCatalog, StoreExpectation, StoreExpectationMismatch}; pub use adapters::{ DurableOutcome, DurableRangeReadReceipt, DurableReadError, DurableReconstructionReceipt, diff --git a/tests/catalog_filesystem_publication/refusal_laws.rs b/tests/catalog_filesystem_publication/refusal_laws.rs index 7ab258db..837eb834 100644 --- a/tests/catalog_filesystem_publication/refusal_laws.rs +++ b/tests/catalog_filesystem_publication/refusal_laws.rs @@ -31,7 +31,11 @@ fn conflicting_immutable_pool_bytes_refuse_before_visibility() -> Result<(), Box &initial_segments, )?; let initial_head = fs::read(store.path().join("HEAD"))?; - let current = FilesystemCatalogSnapshot::load(store.path(), restart_policy()?)?; + let current = FilesystemCatalogSnapshot::load( + store.path(), + restart_policy()?, + keep::StoreExpectation::NoExpectation, + )?; let current_snapshot = current.snapshot()?; let expectation = CatalogPublicationExpectation::successor_of(¤t_snapshot); let (sealed, segment_bytes) = stage_one_zero(&publisher, &store)?; diff --git a/tests/catalog_restart.rs b/tests/catalog_restart.rs index d6640cad..b99cc838 100644 --- a/tests/catalog_restart.rs +++ b/tests/catalog_restart.rs @@ -34,7 +34,11 @@ const RETAINED_SEGMENT_LIMIT: u64 = 1_048_576; #[test] fn restart_reconstructs_the_exact_published_snapshot() -> Result<(), Box> { let store = StoreFixture::create("restart-published")?; - let loaded = FilesystemCatalogSnapshot::load(store.path(), restart_policy()?)?; + let loaded = FilesystemCatalogSnapshot::load( + store.path(), + restart_policy()?, + keep::StoreExpectation::NoExpectation, + )?; let snapshot = loaded.snapshot()?; let identity = SegmentRecordIdentity::Chunk(ChunkId::hash_bytes(&[0])?); diff --git a/tests/catalog_restart/refusal_laws.rs b/tests/catalog_restart/refusal_laws.rs index 5cd66bb3..497d3fd3 100644 --- a/tests/catalog_restart/refusal_laws.rs +++ b/tests/catalog_restart/refusal_laws.rs @@ -26,7 +26,11 @@ fn corrupt_and_unsupported_heads_refuse_at_the_head_boundary() -> Result<(), Box *bytes.last_mut().ok_or("head fixture is empty")? ^= 1; fs::write(corrupt.path().join("HEAD"), bytes)?; let error = require_error( - FilesystemCatalogSnapshot::load(corrupt.path(), restart_policy()?), + FilesystemCatalogSnapshot::load( + corrupt.path(), + restart_policy()?, + keep::StoreExpectation::NoExpectation, + ), "corrupt head was loaded", )?; assert!(matches!( @@ -44,7 +48,11 @@ fn corrupt_and_unsupported_heads_refuse_at_the_head_boundary() -> Result<(), Box .ok_or("head lacks version field")? = 2; fs::write(unsupported.path().join("HEAD"), bytes)?; let error = require_error( - FilesystemCatalogSnapshot::load(unsupported.path(), restart_policy()?), + FilesystemCatalogSnapshot::load( + unsupported.path(), + restart_policy()?, + keep::StoreExpectation::NoExpectation, + ), "unsupported head was loaded", )?; assert!(matches!( @@ -65,7 +73,11 @@ fn noncanonical_catalog_refuses_before_segment_loading() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box TestResult<()> { fn require_content_readers_refuse(root: &Path) -> TestResult<()> { let policy = durable_fixture::policy()?; let Err(CatalogRestartError::Admission { source }) = - FilesystemCatalogSnapshot::load(root, policy) + FilesystemCatalogSnapshot::load(root, policy, keep::StoreExpectation::NoExpectation) else { return Err("catalog restart admitted a root without FORMAT".into()); }; diff --git a/tests/catalog_writer/recovery_fixture.rs b/tests/catalog_writer/recovery_fixture.rs index 9d49ccda..8425fdce 100644 --- a/tests/catalog_writer/recovery_fixture.rs +++ b/tests/catalog_writer/recovery_fixture.rs @@ -95,7 +95,11 @@ pub(super) fn finalize( let candidate = head.admit(ChecksummedCatalog::decode(&catalog_bytes)?.admit(&segments)?)?; let admitted = admit_recovery_stage_bytes(RecoveryStage::NextHead, evidence, &head_bytes)?; let assessment = assess_recovery_stage(&admitted, segment_policy())?; - let current = FilesystemCatalogSnapshot::load(root, durable_fixture::policy()?)?; + let current = FilesystemCatalogSnapshot::load( + root, + durable_fixture::policy()?, + keep::StoreExpectation::NoExpectation, + )?; let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); let request = plan_recovery_next_head_finalization(&assessment, &candidate, expectation)?; let admission = FilesystemRecoveryAdmission::reopen(root)?; diff --git a/tests/single_format/acceptance_laws.rs b/tests/single_format/acceptance_laws.rs index 954fb79c..73a60a87 100644 --- a/tests/single_format/acceptance_laws.rs +++ b/tests/single_format/acceptance_laws.rs @@ -72,11 +72,13 @@ fn recover(root: &Path, candidate: &[Vec], generation: u64) -> TestResult<() } fn current_generation(root: &Path) -> TestResult { - Ok( - FilesystemCatalogSnapshot::load(root, populated_store::policy()?)? - .generation() - .get(), - ) + Ok(FilesystemCatalogSnapshot::load( + root, + populated_store::policy()?, + keep::StoreExpectation::NoExpectation, + )? + .generation() + .get()) } /// ADR-0010 step 4: initialize, ingest, reopen, reconstruct exact bytes, diff --git a/tests/single_format/expectation_laws.rs b/tests/single_format/expectation_laws.rs index 892001a4..7aa14569 100644 --- a/tests/single_format/expectation_laws.rs +++ b/tests/single_format/expectation_laws.rs @@ -1,14 +1,17 @@ -//! This module owns the ADR-0010 expected-state laws at every public open. +//! This module owns the ADR-0010 expected-state laws at every normal open. //! //! A restored store's records are all consistent, so only state the caller kept -//! outside the store can refuse it. Each law opens through a production entry -//! point and asserts the exact expected and observed catalogs. +//! outside the store can refuse it. Each law opens through the five production +//! entry points that take a `StoreExpectation` and asserts the exact expected +//! and observed catalogs. Recovery tools, verification loads, and initialization +//! take no expectation and are outside these laws. use std::fs; use std::path::Path; use keep::{ - DurableSnapshot, DurableStore, DurableStoreError, ExpectedCatalog, FilesystemRetentionSnapshot, + CatalogGeneration, CatalogRestartError, DurableSnapshot, DurableStore, DurableStoreError, + ExpectedCatalog, FilesystemCatalogSnapshot, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, FilesystemStoreAdmission, FilesystemStoreAdmissionError, ReaderAttemptLimit, StoreExpectation, StoreExpectationMismatch, }; @@ -44,7 +47,7 @@ fn publish_second(root: &Path) -> TestResult<()> { retain(root, b"second", &second) } -/// Requires every public open of `root` to refuse with exactly this mismatch. +/// Requires every normal open of `root` to refuse with exactly this mismatch. fn every_open_refuses( root: &Path, expected: ExpectedCatalog, @@ -71,6 +74,11 @@ fn every_open_refuses( return Err("the writer did not refuse at the expectation boundary".into()); }; require_mismatch(source, expected, observed); + let catalog = FilesystemCatalogSnapshot::load(root, policy, expectation); + let Err(CatalogRestartError::Expectation { source }) = catalog else { + return Err("the catalog snapshot did not refuse at the expectation boundary".into()); + }; + require_mismatch(source, expected, observed); Ok(()) } @@ -80,7 +88,10 @@ fn require_mismatch( observed: Option, ) { assert_eq!(mismatch.expected(), expected); - assert_eq!(mismatch.observed(), observed); + let found = mismatch + .observed() + .map(|found| ExpectedCatalog::new(found.generation(), found.digest())); + assert_eq!(found, observed); } fn every_open_admits(root: &Path, expectation: StoreExpectation) -> TestResult<()> { @@ -94,6 +105,7 @@ fn every_open_admits(root: &Path, expectation: StoreExpectation) -> TestResult<( )?); drop(DurableStore::open(root, policy, limit, expectation)?.snapshot()?); drop(FilesystemStoreAdmission::reopen(root, expectation)?); + drop(FilesystemCatalogSnapshot::load(root, policy, expectation)?); Ok(()) } @@ -160,6 +172,15 @@ fn a_store_with_no_catalog_refuses_an_expected_catalog() -> TestResult<()> { ), Err(FilesystemRetentionSnapshotError::Unpublished) )); + // The catalog snapshot has nothing to load, and says so at the head boundary. + assert!(matches!( + FilesystemCatalogSnapshot::load( + empty.path(), + populated_store::policy()?, + StoreExpectation::NoExpectation, + ), + Err(CatalogRestartError::Io { source, .. }) if source.kind() == std::io::ErrorKind::NotFound + )); drop(FilesystemStoreAdmission::reopen( empty.path(), StoreExpectation::NoExpectation, @@ -206,3 +227,26 @@ fn a_catalog_expectation_does_not_detect_a_retention_only_rollback() -> TestResu every_open_admits(store.path(), StoreExpectation::Catalog(wanted)) } + +/// A later process holds only what the caller recorded: a `u64` and 32 bytes. +/// The expectation rebuilt from them admits the recorded catalog and refuses +/// its successor through every normal open. +#[test] +fn an_expectation_rebuilt_from_recorded_values_admits_and_refuses_as_the_original() -> TestResult<()> +{ + let store = durable_fixture::build("expected-recorded", &[&first_blob()])?; + let original = published(store.path())?; + let recorded: (u64, [u8; 32]) = (original.generation().get(), original.digest_bytes()); + + let rebuilt = ExpectedCatalog::from_recorded(CatalogGeneration::new(recorded.0)?, recorded.1); + + assert_eq!(rebuilt, original); + every_open_admits(store.path(), StoreExpectation::Catalog(rebuilt))?; + publish_second(store.path())?; + let successor = published(store.path())?; + every_open_refuses(store.path(), rebuilt, Some(successor))?; + let mut damaged = recorded.1; + *damaged.first_mut().ok_or("empty digest")? ^= 1; + let wrong = ExpectedCatalog::from_recorded(successor.generation(), damaged); + every_open_refuses(store.path(), wrong, Some(successor)) +} diff --git a/tests/support/populated_store.rs b/tests/support/populated_store.rs index f134c1bc..496d0c4a 100644 --- a/tests/support/populated_store.rs +++ b/tests/support/populated_store.rs @@ -263,7 +263,8 @@ pub(super) fn retain( )?; let canonical = CanonicalRetentionRoot::from_root(&retained)?; let candidate = AdmittedRetentionRoot::decode(canonical.encoded())?; - let current = FilesystemCatalogSnapshot::load(root, policy()?)?; + let current = + FilesystemCatalogSnapshot::load(root, policy()?, keep::StoreExpectation::NoExpectation)?; let snapshot = current.snapshot()?; let preflight = preflight_retention_transition( RetentionGenerationExpectation::Absent, diff --git a/tests/support/successor_catalog.rs b/tests/support/successor_catalog.rs index 657fa7e1..81b05536 100644 --- a/tests/support/successor_catalog.rs +++ b/tests/support/successor_catalog.rs @@ -100,7 +100,8 @@ pub(super) fn publish( predecessor: Predecessor, ) -> TestResult { let identified = identify(content)?; - let current = FilesystemCatalogSnapshot::load(root, policy()?)?; + let current = + FilesystemCatalogSnapshot::load(root, policy()?, keep::StoreExpectation::NoExpectation)?; let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); let mut encoded = match predecessor { Predecessor::Kept => pool_segments(root)?, @@ -126,7 +127,8 @@ pub(super) fn republish( publisher: &mut FilesystemCatalogPublisher, root: &Path, ) -> TestResult { - let current = FilesystemCatalogSnapshot::load(root, policy()?)?; + let current = + FilesystemCatalogSnapshot::load(root, policy()?, keep::StoreExpectation::NoExpectation)?; let expectation = CatalogPublicationExpectation::successor_of(¤t.snapshot()?); let encoded = pool_segments(root)?; let segments = admit_all(&encoded)?; diff --git a/xtask/src/durability_crash_matrix/restart.rs b/xtask/src/durability_crash_matrix/restart.rs index 7810d7c9..dd0b0a8f 100644 --- a/xtask/src/durability_crash_matrix/restart.rs +++ b/xtask/src/durability_crash_matrix/restart.rs @@ -2,6 +2,7 @@ mod expectation; mod initialization; +mod recorded_catalog; mod retained_catalog; mod retention; mod retention_incomplete; diff --git a/xtask/src/durability_crash_matrix/restart/recorded_catalog.rs b/xtask/src/durability_crash_matrix/restart/recorded_catalog.rs new file mode 100644 index 00000000..832b1551 --- /dev/null +++ b/xtask/src/durability_crash_matrix/restart/recorded_catalog.rs @@ -0,0 +1,56 @@ +//! This module owns the crash law for a catalog a caller recorded before publication. +//! +//! A caller that names generation one in its `StoreExpectation` and dies during +//! a publication cannot know whether `HEAD` moved. The store must admit that +//! caller while generation one is published and refuse it, reporting what it +//! holds, once generation two is. Keep cannot tell the caller's own successor +//! from another store; accepting the observed catalog is the caller's decision. + +use std::io; +use std::path::Path; + +use keep::{ + CatalogRestartError, ChecksummedPublicationHead, ExpectedCatalog, FilesystemCatalogSnapshot, + StoreExpectation, +}; + +use super::super::DurabilityCrashMatrixError; +use super::super::production_protocol::fixture::GoldenFixture; +use super::super::production_protocol::retained_catalog::policy; +use super::super::production_protocol::verification; + +/// The generation-one catalog of the bundle store, as a caller would have recorded it. +pub(super) fn generation_one() -> Result { + let fixture = GoldenFixture::bundle_head()?; + let head = ChecksummedPublicationHead::decode(fixture.bytes()) + .map_err(|source| verification("decode recorded generation-one head", source))?; + Ok(ExpectedCatalog::new( + head.generation(), + head.catalog_digest(), + )) +} + +/// Requires the recorded generation-one catalog to refuse against generation two. +pub(super) fn require_refused_at_generation_two( + store_root: &Path, +) -> Result<(), DurabilityCrashMatrixError> { + let recorded = generation_one()?; + let loaded = + FilesystemCatalogSnapshot::load(store_root, policy()?, StoreExpectation::Catalog(recorded)); + match loaded { + Err(CatalogRestartError::Expectation { source }) + if source.expected() == recorded + && source.observed().map(|found| found.generation().get()) == Some(2) => + { + Ok(()) + } + Err(source) => Err(verification( + "refuse the recorded generation-one catalog", + source, + )), + Ok(_admitted) => Err(verification( + "refuse the recorded generation-one catalog", + io::Error::other("generation two admitted a generation-one expectation"), + )), + } +} diff --git a/xtask/src/durability_crash_matrix/restart/retained_catalog.rs b/xtask/src/durability_crash_matrix/restart/retained_catalog.rs index aa8ea4f8..0850f622 100644 --- a/xtask/src/durability_crash_matrix/restart/retained_catalog.rs +++ b/xtask/src/durability_crash_matrix/restart/retained_catalog.rs @@ -28,6 +28,7 @@ use super::super::production_protocol::fixture::GoldenFixture; use super::super::production_protocol::retained_catalog::{admit_segments, policy}; use super::super::production_protocol::retention::reopened_authority; use super::super::production_protocol::verification; +use super::recorded_catalog; use super::stage_recovery as recovery; /// The catalog-protocol residue one coordinate leaves. @@ -66,6 +67,7 @@ pub(super) fn verify( } recover(store_root, residue)?; require_generation_two(store_root)?; + recorded_catalog::require_refused_at_generation_two(store_root)?; retain_blob(store_root)?; read_back(store_root) } @@ -108,7 +110,9 @@ fn republish(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { let pool = pool_segments(store_root)?; let encoded: Vec<&[u8]> = pool.iter().map(Vec::as_slice).collect(); let segments = admit_segments(&encoded)?; - let current = FilesystemCatalogSnapshot::load(store_root, policy()?) + // The publication never reached `HEAD`, so the catalog recorded before it still admits. + let recorded = keep::StoreExpectation::Catalog(recorded_catalog::generation_one()?); + let current = FilesystemCatalogSnapshot::load(store_root, policy()?, recorded) .map_err(|source| verification("load generation-one catalog", source))?; let expectation = CatalogPublicationExpectation::successor_of( ¤t @@ -143,8 +147,12 @@ fn republish(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { } fn require_generation_two(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { - let current = FilesystemCatalogSnapshot::load(store_root, policy()?) - .map_err(|source| verification("load recovered catalog", source))?; + let current = FilesystemCatalogSnapshot::load( + store_root, + policy()?, + keep::StoreExpectation::NoExpectation, + ) + .map_err(|source| verification("load recovered catalog", source))?; if current.generation().get() == 2 { Ok(()) } else { @@ -175,8 +183,12 @@ fn retain_blob(store_root: &Path) -> Result<(), DurabilityCrashMatrixError> { .map_err(|source| verification("encode crash retention root", source))?; let candidate = AdmittedRetentionRoot::decode(canonical.encoded()) .map_err(|source| verification("admit crash retention root", source))?; - let current = FilesystemCatalogSnapshot::load(store_root, policy()?) - .map_err(|source| verification("load recovered catalog for retention", source))?; + let current = FilesystemCatalogSnapshot::load( + store_root, + policy()?, + keep::StoreExpectation::NoExpectation, + ) + .map_err(|source| verification("load recovered catalog for retention", source))?; let snapshot = current .snapshot() .map_err(|source| verification("admit recovered catalog for retention", source))?; diff --git a/xtask/src/durability_crash_matrix/restart/semantic.rs b/xtask/src/durability_crash_matrix/restart/semantic.rs index 31b3d931..7d6a081d 100644 --- a/xtask/src/durability_crash_matrix/restart/semantic.rs +++ b/xtask/src/durability_crash_matrix/restart/semantic.rs @@ -166,8 +166,12 @@ fn verify_published_snapshot( if expected.artifact(HEAD).is_none() { return Ok(()); } - let loaded = FilesystemCatalogSnapshot::load(store_root, restart_policy()?) - .map_err(|source| verification("load published restart snapshot", source))?; + let loaded = FilesystemCatalogSnapshot::load( + store_root, + restart_policy()?, + keep::StoreExpectation::NoExpectation, + ) + .map_err(|source| verification("load published restart snapshot", source))?; let observed_generation = loaded.generation().get(); if observed_generation != 1 { return Err(DurabilityCrashMatrixError::SnapshotGenerationMismatch { diff --git a/xtask/src/durability_crash_matrix/restart/stage_recovery.rs b/xtask/src/durability_crash_matrix/restart/stage_recovery.rs index 2275fd2d..e0da07f4 100644 --- a/xtask/src/durability_crash_matrix/restart/stage_recovery.rs +++ b/xtask/src/durability_crash_matrix/restart/stage_recovery.rs @@ -112,8 +112,9 @@ pub(super) fn finalize_head( .map_err(|source| verification("admit recovered next-head stage", source))?; let assessment = assess_recovery_stage(&admitted, segment_policy()) .map_err(|source| verification("assess recovered next-head stage", source))?; - let current = FilesystemCatalogSnapshot::load(root, policy()?) - .map_err(|source| verification("load recovered current catalog", source))?; + let current = + FilesystemCatalogSnapshot::load(root, policy()?, keep::StoreExpectation::NoExpectation) + .map_err(|source| verification("load recovered current catalog", source))?; let expectation = CatalogPublicationExpectation::successor_of( ¤t .snapshot() From bea75ae9f162f56d2374272fe7d256e835676566 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 5 Nov 2026 01:06:34 -0800 Subject: [PATCH 32/36] Docs: correct the single-format record after review (#198) The review of PR #220 found statements that claimed more than the code and laws establish. This corrects them: which opens take an expectation and which routes take none, how an expectation is kept and what a crash does to it, what the torn-copy and commit-point laws witness, the current crash-row counts, the retirement ledger, the changelog, and the platform pages. It records two owner decisions of 2026-11-05: KEEP-CRASH-079 and -080 follow version 1's deletion and block the first release in #221, and the GC salvage renumbers from KEEP-CRASH-092. The mutation record holds the six hand-run mutants and what they showed. The protocol contract law that pinned the old case-sensitivity wording now pins the rule that replaced it. --- CHANGELOG.md | 11 +- README.md | 19 ++-- ROADMAP.md | 53 ++++++--- docs/adr/0010-one-storage-format.md | 45 ++++++-- docs/adr/filesystem-platform-adapters.md | 5 +- docs/architecture/filesystem-platforms.md | 70 ++++++++---- docs/audits/pr-107-salvage.md | 2 +- docs/formats/store/README.md | 37 ++++--- .../store/catalog-protocol-requirements.md | 4 +- docs/formats/store/catalog-publication.md | 47 +++++++- docs/formats/store/initialization.md | 25 ++++- docs/formats/store/publication.md | 24 +++-- docs/formats/store/rationale.md | 9 +- docs/formats/store/recovery.md | 101 ++++++++++++++++-- docs/formats/store/requirements.md | 29 +++-- docs/formats/store/retired-identifiers.md | 69 +++++++----- docs/formats/store/stage-recovery.md | 22 +++- .../filesystem-platform-adapters.md | 67 ++++++++++-- .../testing-evidence/single-format-mutants.md | 43 ++++++++ .../publication_laws.rs | 7 +- 20 files changed, 542 insertions(+), 147 deletions(-) create mode 100644 docs/testing-evidence/single-format-mutants.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c6a4e72..b0f7ab26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ after its public API and format compatibility policies are established. ## [Unreleased] -- Opening a store requires a typed choice: `FilesystemStoreAdmission::reopen`, `FilesystemRetentionSnapshot::load`, `DurableSnapshot::open`, and `DurableStore::open` take a `StoreExpectation` with no default. `Catalog` refuses any other published catalog with the expected and observed values, and `NoExpectation` accepts any valid store at the path. Enumerated laws show that a copy torn by a concurrent writer refuses or returns exact bytes. +- Opening a store requires a typed choice (#198): `FilesystemStoreAdmission::reopen`, `FilesystemCatalogSnapshot::load`, `FilesystemRetentionSnapshot::load`, `DurableSnapshot::open`, and `DurableStore::open` take a `StoreExpectation` with no default. `DurableStore::open` checks nothing until a snapshot is pinned. `Catalog` refuses any other published catalog, or none, with the expected and observed values, and `NoExpectation` accepts any valid store at the path. `ExpectedCatalog` is the caller's claim, not a verified digest: `new` takes the generation and digest a receipt or view reports, `from_recorded` rebuilds it in a later process from a recorded generation and 32 digest bytes, and `digest_bytes` returns the bytes to record. `ObservedCatalog` is the observed side of `StoreExpectationMismatch`. A mismatch from `FilesystemCatalogSnapshot::load` is the new `CatalogRestartError::Expectation`, which exhaustive matches on that error must admit. Keep stores no expectation for the caller. Recovery tools, verification loads, initialization, and the repository-task reopen take none, so their callers do not get this check. Limits: without an expectation, a torn copy may silently be the older state. A `Catalog` expectation misses rollbacks that leave the catalog unchanged. After a crash between a publication and the caller's record of it, the old expectation refuses, and accepting the observed catalog is the caller's own trust decision; every `retained-catalog` crash case (`KEEP-CRASH-074` to `-077`) asserts that refusal once `HEAD` names the successor. Enumerated laws show that every mix of two quiescent published states, and a half-length copy of any one file over the newer state, refuses with a typed absent-or-malformed cause or returns exact bytes. They do not cover a copy that holds a writer's in-flight stage, other truncation lengths, or a tear over the older state. - Stop naming Cargo workspace members as Dependabot update directories. Dependabot refuses to update from inside a member and the job for the workspace root already updates member manifests, so those entries failed the scheduled run without adding coverage. The dependency policy check now reads the root workspace member list, refuses a member listed as an update directory, and still requires an entry for every standalone manifest. @@ -26,14 +26,13 @@ after its public API and format compatibility policies are established. - Add field-mutation laws for selected version-2 retention root, manifest, and head fields: rows mutate the frozen corpus record, reseal every digest and checksum they did not target, and assert the named refusal variant. KEEP-RETENTION-003 remains in progress (#212). Salvaged from the closed #107 branch; tests and ledger only. -- ADR-0010 commits Keep to one current storage format: direct initialization replaces version-1 initialization plus migration, complete stores become portable while live authority checks remain, opening requires an explicit expected-state choice before the first release, and after it Keep reads released sealed segments while writing only the current format, with explicit metadata upgrades and optional, resumable segment rewriting. Main's behavior is unchanged until the units tracked in #198 land (#198). -- Version 1 is deleted: every open path refuses a root without the `FORMAT` marker, `KEEP-CRASH-001`–`030` run on initialized stores, `KEEP-CRASH-031`–`035` are retired, the `VersionTwo*` types take format-neutral names (`FilesystemStoreAdmission`, `FilesystemCatalogPublisher`, `StoreInitialization`, `classify_recovery_names`), and the format pages are one set under `docs/formats/store/` (#198). +- ADR-0010 commits Keep to one current storage format: direct initialization replaces version-1 initialization plus migration, complete stores become portable while live authority checks remain, opening requires an explicit expected-state choice before the first release, and after it Keep reads released sealed segments while writing only the current format, with explicit metadata upgrades and optional, resumable segment rewriting (#198). -- Keep has one storage format: `FilesystemVersionTwoAdmission::initialize` creates a store directly through a recoverable protocol with one durable commit point and a 41-case crash matrix (`KEEP-CRASH-081`–`091`), migration and its public API, records, golden fixtures, and crash rows `KEEP-CRASH-053`–`073` are deleted, the persisted root device-and-inode binding is removed so quiescent copies and backups open, and the format-definition digest changes to `f1a0ce10…361d` (#198). +- Keep has one storage format (#198, #125). In the order its units were built: `publish_retained_catalog_generation` refuses a candidate catalog that loses retained content, verifying every current retained closure against the candidate before any catalog mutation and revalidating the retention head before `HEAD` replacement. `FilesystemCatalogPublisher` stages one segment and publishes a catalog successor through that gate, catalog-stage recovery finalizes a staged head only after the same gate admits it, and `KEEP-CRASH-074` to `-077` cover writer process death. `FilesystemStoreAdmission::initialize` creates a store directly through a recoverable protocol with one durable commit point and a 41-case crash matrix (`KEEP-CRASH-081`–`091`); laws witness the order and the failure of the commit-point synchronization at an injected synchronizer, and that the production call reaches stable storage is by inspection, with no power-loss test. Migration and its public API, records, golden fixtures, and crash rows `KEEP-CRASH-053`–`073` are deleted, the persisted root device-and-inode binding is removed so quiescent copies and backups open, and the format-definition digest changes to `f1a0ce10…361d`. Version 1 is deleted: every store open refuses a root without the `FORMAT` marker, `KEEP-CRASH-001`–`030` run on initialized stores, `KEEP-CRASH-031`–`035` are retired, the `VersionTwo*` types take format-neutral names (`FilesystemStoreAdmission`, `FilesystemCatalogPublisher`, `StoreInitialization`, `classify_recovery_names`), and the format pages are one set under `docs/formats/store/`. -- Version-2 stores can ingest: `FilesystemVersionTwoCatalogPublisher` stages one segment and publishes a catalog successor through the retained-closure gate, version-2 catalog-stage recovery classifies version-2 roots and finalizes a staged head only after the same gate admits it, and `KEEP-CRASH-074` through `077` cover writer process death (#198). +- Platform coordinates are live observations only (#198): the internal `FilesystemDirectoryCoordinate` carries a directory's device and mount for comparison within one operation, and the object coordinate and the Darwin coordinate domain, which existed for the persisted root binding, are removed. Nothing persists a coordinate. -- Version-2 catalog publication can refuse candidates that lose retained content: `publish_retained_catalog_generation` verifies every current retained closure against the candidate before any catalog mutation and revalidates the retention head before `HEAD` replacement (#125). +- Not yet implemented: `KEEP-CRASH-079` and `-080`, process death in recovery's stage-completion link and in gated finalization's `HEAD` replacement, are reserved. The plan in #198 placed them before version 1's deletion. The owner accepted on 2026-11-05 that they follow it; #221 tracks them and blocks the first release together with #205. - Update the repository-only serde_json parser to 1.0.154 in both locked workspaces and refresh its dependency admission; the resolved transitive graph, features, license and MSRV are unchanged (#186). diff --git a/README.md b/README.md index f02a79ae..9736a0a8 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Keep is required to refuse all three, before mutating anything. single byte reaches the caller. It also provides authenticated exact byte-range reads that load only the overlapping chunks and state their narrower claim explicitly. -- **Durable version-1 segment store.** `StagedSegment` writes only +- **Durable segment store.** `StagedSegment` writes only content-admitted records; `AdmittedSegment` exposes payloads only after complete framing, checksum, and identity verification. Immutable segments, generation-versioned catalogs, and a fixed-width `HEAD` are published @@ -53,16 +53,23 @@ Keep is required to refuse all three, before mutating anything. Platform admission is non-casefolded Linux ext4 or local writable macOS APFS, one writer. - **Process-death recovery evidence.** The crash matrix kills real writer processes - at 156 before/during/after coordinates (`KEEP-CRASH-001`–`052`) and - verifies the store lands in exactly one documented lawful state each time, - for version-1 publication and for version-2 retention publication. + at 194 before/during/after coordinates over 62 boundaries and verifies the + store lands in exactly one documented lawful state each time: 141 for + segment, catalog and head publication, recovery discard, and retention + publication (the 47 live rows of `KEEP-CRASH-001`–`052`), 12 for catalog + publication beside a retained root (`KEEP-CRASH-074`–`077`), and 41 for + initialization (`KEEP-CRASH-081`–`091`). Process death is not power loss, + which nothing here tests. - **One store format, created directly.** `FilesystemStoreAdmission::initialize` creates a version-2 store through a recoverable state machine with one durable commit point; an interrupted call resumes, and any other nonempty directory refuses before it is touched. A 41-case process-death matrix (`KEEP-CRASH-081`–`091`) covers every boundary. Nothing persisted binds a store to its directory: a quiescent copy or restored backup opens wherever - it is placed. There is no migration; see the + it is placed, as a second, independent store. Nothing in a valid store + shows that it is the wrong one or an older state, so every normal open + takes a `StoreExpectation`: the exact catalog the caller recorded, or an + explicit statement that it has none. There is no migration; see the [retirement ledger](docs/formats/store/retired-identifiers.md). - **Version-2 retention, forward path.** Explicit retention roots, deterministic closure verification, the gated segment and catalog writer, @@ -201,7 +208,7 @@ assert_eq!(output, b"exact bytes, or nothing"); # Ok::<(), Box>(()) ``` -On an admitted Linux ext4 or macOS APFS volume, read an initialized store whose retention roots anchor the requested blob. The following example is also compiled in the `DurableStore` API documentation. Its 16 MiB limit applies to aggregate catalog-selected segment bytes; catalog bytes and decoded metadata allocate separately. +On an admitted Linux ext4 or macOS APFS volume, read an initialized store whose retention roots anchor the requested blob. The following example is also compiled in the `DurableStore` API documentation. Its 16 MiB limit applies to aggregate catalog-selected segment bytes; catalog bytes and decoded metadata allocate separately. It passes `StoreExpectation::NoExpectation`, which accepts any valid store at the path, including a different store or an older state of the intended one. That is the right choice for a first open. A caller that has recorded a catalog from a publication receipt or a view passes `StoreExpectation::Catalog` instead, rebuilt with `ExpectedCatalog::from_recorded`, and the open refuses with the expected and observed catalogs when they differ. ```rust #[cfg(any(target_os = "linux", target_os = "macos"))] diff --git a/ROADMAP.md b/ROADMAP.md index 6994a006..3eae8cfe 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -4,8 +4,8 @@ This page has one job: to list every feature Keep has, is building, or intends, and to break the unfinished ones into tasks precise enough to start. It is a plan. It is not evidence. The authoritative status of every requirement, with the test that proves it, remains the requirement ledgers: -[`segment-store-v1/requirements.md`](docs/formats/store/catalog-protocol-requirements.md), -[`segment-store-v2/requirements.md`](docs/formats/store/requirements.md), +[`store/catalog-protocol-requirements.md`](docs/formats/store/catalog-protocol-requirements.md), +[`store/requirements.md`](docs/formats/store/requirements.md), [`authenticated-reconstruction/requirements.md`](docs/invariants/authenticated-reconstruction/requirements.md), and [`verification/requirements.md`](docs/invariants/verification/requirements.md). @@ -147,7 +147,7 @@ names; use those in code, tests, and commits. - [ ] [F-39 Compression representation codec](#f-39-compression-representation-codec) — Proposed - [ ] [F-40 Additional chunking profiles and hierarchical layouts](#f-40-additional-chunking-profiles-and-hierarchical-layouts) — Proposed -- [ ] [F-41 Platform adapters beyond Linux ext4](#f-41-platform-adapters-beyond-linux-ext4) — In progress; macOS APFS done +- [ ] [F-41 Platform adapters beyond Linux ext4](#f-41-platform-adapters-beyond-linux-ext4) — In progress; macOS APFS admitted, its ledger row owed - [ ] [F-42 Operator surfaces](#f-42-operator-surfaces) — Proposed (CLI and MCP adapters) - [ ] [F-43 Public release and API stability](#f-43-public-release-and-api-stability) — Proposed - [ ] [F-44 Multi-writer, replication, and remote tiers](#f-44-multi-writer-replication-and-remote-tiers) — Out of scope @@ -1246,10 +1246,15 @@ the `main` ledger marks `KEEP-GC-001` and `KEEP-GC-002` "Planned in #21", and `recovery.md` says any GC or disposition artifact is unsupported and refuses until #21 implements it. All three codecs (T-22.1, T-22.1a), the deterministic planner (T-22.2), explicit orphan disposition (T-22.5), -retirement with recovery and its `KEEP-CRASH-074` to `-087` process-death -matrix (T-22.4), and identity-preserving compaction (T-22.3) exist only on +retirement with recovery and its 42-case process-death matrix (T-22.4), and +identity-preserving compaction (T-22.3) exist only on the archived #107 branch (`feature/roadmap-m4-tasks` at `57b6c37`); salvage is tracked in the #107 salvage ledger, and a salvage of the T-22.1 codecs is in progress. +The branch numbered that matrix `KEEP-CRASH-074` to `-087`. The single format +(#198) has since taken `KEEP-CRASH-074` to `-091`: `-074` to `-077` and `-081` +to `-091` are implemented, `-078` is retired, and `-079` and `-080` are +reserved (#221). Retired and assigned identifiers are never reused, so a GC +salvage renumbers its crash points from `KEEP-CRASH-092`. The branch itself still owed the disposition-phase and compaction process-death sequences, the 65,536-candidate stress run, compaction benchmarks, and re-encoding. @@ -1392,7 +1397,9 @@ disposition receipt exists. `BlobId`, `ChunkId`, and `LayoutId` never move. benchmarks report amplification, sync count, reclaimed bytes, latency, and peak temporary space. - **Scope:** in — compaction planner, executor, storage port, filesystem - adapter, crash points `KEEP-CRASH-074` onward. Out — re-encoding + adapter, and new crash points numbered after the GC rows (the branch + wrote `KEEP-CRASH-074` onward, which the single format now uses; see + the F-22 status). Out — re-encoding (representation change) which waits for F-28 or F-39. - **User stories:** Human — an operator reclaims space from a store whose segments are half released. API user — `compact(plan)` returns a @@ -1412,14 +1419,16 @@ disposition receipt exists. `BlobId`, `ChunkId`, and `LayoutId` never move. - **Documentation:** `gc.md` compaction section; ADR-0002 compaction example cross-linked. - **Dependencies:** T-22.2. -- [ ] T-22.4 GC execution, retirement, and recovery — not on `main`; no - `KEEP-CRASH-074` or later point exists there. On the archived #107 branch +- [ ] T-22.4 GC execution, retirement, and recovery — not on `main`; no GC + crash point exists there. On the archived #107 branch only: `FilesystemGcAuthority::{prepare, execute, recover}`, the 14-phase `GcExecutionPhase` protocol over the `GcExecutionStorage` port, `GcResidue` and `plan_gc_recovery`, the registered proof, pool, and disposition-set derivations, `filesystem_gc_tests` (every prefix, both - truncated stages, readers, stale plan, exclusion), and the - `KEEP-CRASH-074..087` matrix (`--sequence gc`, 42 cases). `KEEP-GC-002` + truncated stages, readers, stale plan, exclusion), and the 42-case matrix + (`--sequence gc`), which the branch numbered `KEEP-CRASH-074..087`. Those + identifiers now belong to the single format (#198), so a salvage renumbers + the matrix from `KEEP-CRASH-092`; see the F-22 status. `KEEP-GC-002` stays In progress until compaction (T-22.3); the 65,536-candidate stress run and the disposition-phase process-death matrix promised by T-22.5 are still owed under #21. Original task fields: @@ -2397,17 +2406,29 @@ plan or needs bounded plan streaming. ### F-41 Platform adapters beyond Linux ext4 **Status:** In progress. `recovery.md` defers Windows "until an adapter and -crash harness prove equivalent semantics"; macOS APFS is admitted (T-41.1); -the crash matrix does not simulate power loss. - -- [x] T-41.1 macOS APFS production admission. +crash harness prove equivalent semantics"; macOS APFS is admitted (T-41.1, +whose ledger row is still owed); the crash matrix does not simulate power loss. + +- [ ] T-41.1 macOS APFS production admission — admission shipped in #216, + and the single-format store paths (#198) run on it: default and + case-sensitive APFS admit, with evidence in the + [platform adapter evidence record](docs/testing-evidence/filesystem-platform-adapters.md) + and the `Native macOS storage` CI job, which runs the complete + process-death crash matrix natively. Still owed: no requirement ledger + names macOS. The original criterion asked for a macOS row under + `KEEP-RECOVERY-003`, which #198 retired; the + [retirement ledger](docs/formats/store/retired-identifiers.md) moves that + profile obligation to single-format admission, where no live row cites + macOS evidence yet. - **Requirements:** a platform profile that probes APFS for the same capabilities (atomic no-clobber link, atomic same-filesystem replacement, durable `fsync` semantics including `F_FULLFSYNC`, advisory locks, device and inode identity); refuses anything else; the crash matrix runs on macOS in CI. - - **Acceptance criteria:** `KEEP-RECOVERY-003` gains a macOS row; the - 105-case matrix (and the v2 matrices) green on a macOS runner. + - **Acceptance criteria:** a live requirement row in + [`store/requirements.md`](docs/formats/store/requirements.md) names + macOS APFS admission and its evidence; the complete process-death crash + matrix is green on a macOS runner (met by the CI job above). - **Scope:** in — `filesystem_platform_profile.rs` sibling. Out — network volumes. Case-insensitive volumes are in, with exact-spelling checks (see the platform adapter decision). diff --git a/docs/adr/0010-one-storage-format.md b/docs/adr/0010-one-storage-format.md index 3ff48125..bfc7c67d 100644 --- a/docs/adr/0010-one-storage-format.md +++ b/docs/adr/0010-one-storage-format.md @@ -1,7 +1,7 @@ # ADR-0010: One Current Storage Format - Status: Accepted -- Date: 2026-10-27; amended 2026-11-04 +- Date: 2026-10-27; amended 2026-11-04 and 2026-11-05 - Owners: Keep format, initialization, publication, recovery, and retention boundaries - Related issue: @@ -73,8 +73,9 @@ version-2 record set and namespace without migration artifacts: ADR-0005 and the version-1 pages define them; - `FORMAT`, `reader.lock`, retention records, GC records, and recovery-disposition records, as the version-2 pages define them; -- no `migration.intent`, `migration.receipt`, their `.next` stages, or - `FORMAT.next`. +- no `migration.intent`, `migration.receipt`, or their `.next` stages; and +- no `FORMAT.next` in a committed store. `FORMAT.next` exists only as direct + initialization's stage, and normal opening refuses a root that still holds it. Its documentation is consolidated into one normative set of pages for the Keep store format. @@ -154,8 +155,24 @@ the caller passes the state it expects, and a mismatch refuses, or the caller explicitly states that it has no expectation. Neither is a default, so removing the binding never silently removes the question. Keep does not store the expectation. An expected state names an exact catalog generation and digest, so it -does not detect a rollback that leaves the catalog unchanged. Recovery and -verification take no expectation, because they examine whatever state is there. +does not detect a rollback that leaves the catalog unchanged. + +The choice is required by the writer's reopen +(`FilesystemStoreAdmission::reopen`), the catalog snapshot load +(`FilesystemCatalogSnapshot::load`), the fenced retention snapshot load +(`FilesystemRetentionSnapshot::load`), and both durable readers +(`DurableSnapshot::open` and `DurableStore::open`, which checks when a snapshot +is pinned). Four routes take no expectation, so their callers do not get this +check: + +- recovery tools, because they examine whatever state is there; +- verification loads, for the same reason; +- initialization, including a call that resumes an interrupted one. Its + marker-linked state admits a populated committed namespace + ([initialization](../formats/store/initialization.md)), so that call can finish + the cleanup of a store that already holds content; and +- the repository-task reopen, which is compiled only for tests and the + `repository-tasks` feature and also skips platform admission. Keep persists no store identifier. The only one, the content-derived identifier in the migration records, is deleted with migration, and @@ -189,6 +206,22 @@ everything they do today. The order, tracked in #198, is: newer expected state. 5. Version-1 admission is deleted and the format pages are consolidated. +One departure from this order is recorded. Step 2's stage recovery reserved two +process-death rows that are not implemented: `KEEP-CRASH-079`, death during +recovery's stage-completion link, and `KEEP-CRASH-080`, death during gated +finalization's `HEAD` replacement. The order placed them before step 5, and +version 1 was deleted without them. The owner accepted that on 2026-11-05: the +rows never existed on `main`, nothing is released, and both routes refuse when +their state is ambiguous. Until they land, process death during recovery, and +the gate running again when a finalization is retried after one, are untested. +[#221](https://github.com/flyingrobots/keep/issues/221) tracks the rows and must +close before the first release, as #205 must. + +Step 4's torn-copy law is met narrowly. The laws enumerate every mix of two +quiescent published states, and a half-length copy of any one file over the +newer state. A copy that holds a writer's in-flight stage, a file cut at another +length, and a tear over the older state are untested. + Retired requirement and crash identifiers are never reused. A retirement ledger maps each one to its last evidence and states whether its obligation disappeared with migration or moved to the single format. @@ -329,4 +362,4 @@ lands rather than specified here: - Security: removing a persistent binding does not widen filesystem authority. All access stays capability-relative and no-follow. The stronger wrong-store and rollback claims hold only for a caller that passes expected - state, and every caller must choose whether to. + state, and every caller of an open that takes the choice must make it. diff --git a/docs/adr/filesystem-platform-adapters.md b/docs/adr/filesystem-platform-adapters.md index bff15302..73ff0c16 100644 --- a/docs/adr/filesystem-platform-adapters.md +++ b/docs/adr/filesystem-platform-adapters.md @@ -79,7 +79,10 @@ of descriptors and filesystem-library types. Adding an OS entails one adapter, one selector entry, and its conformance evidence, rather than changing every durability protocol. Filesystem support stays explicit and fail-closed. -Existing Linux wire bytes remain unchanged. Native coordinates are live observations +This decision changed no wire bytes and no golden fixture. +[ADR-0010](0010-one-storage-format.md) separately changed the format-definition +digest and the `FORMAT` marker fixture and deleted the migration fixtures, on +every platform alike. Native coordinates are live observations that keep every protocol directory on the root's device and mount within one operation, and nothing persists them. Darwin uses `st_dev` as both device and mount, because each mounted APFS volume has its own `st_dev`. Darwin renumbers `st_dev` when diff --git a/docs/architecture/filesystem-platforms.md b/docs/architecture/filesystem-platforms.md index 9cf79e93..c4516914 100644 --- a/docs/architecture/filesystem-platforms.md +++ b/docs/architecture/filesystem-platforms.md @@ -36,7 +36,7 @@ The two boundaries have different purposes: | --- | --- | --- | | Semantic protocol ports | Required effects, ordering obligations, typed outcomes | OS selection, descriptors, filesystem detection | | Common filesystem adapters | Canonical namespace, pinned authority, exact verification, protocol effects | Linux or Darwin syscall policy | -| `PlatformFilesystem` | Root traversal, descriptor observations, durability, native locking semantics | Catalog generations, retention policy, codecs, recovery decisions | +| `PlatformFilesystem` | Root traversal; the profile and live coordinates of an opened directory | Catalog generations, retention policy, codecs, recovery decisions | | Filesystem profile admission | Which observed configurations satisfy the supported contract | Caller-selected claims of safety, best-effort fallback | The platform boundary is an internal infrastructure interface, not a domain @@ -70,22 +70,30 @@ requires a profile and its evidence, not another implementation of every port. ## Platform interface The implementation separates shared profile admission from native observations. -The interface has these semantic responsibilities; final Rust names may be -refined without changing the contract: +The `PlatformFilesystem` trait has two operations: | Operation | Required result | | --- | --- | -| Open a selected root | Pinned readable directory, no symlink traversal | -| Observe a directory | Filesystem kind, writable/local observations, naming behavior, device/mount/object coordinates | -| Observe an existing object | Exact descriptor-bound identity; no path reopening as a substitute | -| Synchronize a regular file | All prior writes reach the platform's documented durable boundary, or an exact error | -| Synchronize a directory | Prior namespace effects reach that boundary, or an exact error | -| Acquire a native lock | Nonblocking kernel exclusion tied to the retained handle; typed contention | -| Validate entry spelling where necessary | The actual namespace entry has the exact protocol spelling | - -Use concrete observation records, checked constructors, and explicit enums for -naming and mount behavior. No `supports_durability: bool` supplied by a caller, -no catch-all string capability map, and no public boolean parameters. +| Open a selected root (`open_root`) | Pinned readable directory, no symlink traversal | +| Admit a directory (`admit_directory`) | A supported writable local profile, or refusal; the directory's live device and mount coordinates | + +The coordinates carry no object identity. Nothing persists them, and common code +compares them only within one operation. + +These obligations hold on every platform but are not trait operations, because +one shared implementation serves both platforms: + +| Obligation | Required result | Where it lives | +| --- | --- | --- | +| Observe an existing object | Exact descriptor-bound identity; no path reopening as a substitute | Common exact-record and lock code | +| Synchronize a regular file | All prior writes reach the platform's documented durable boundary, or an exact error | The standard library's `sync_all` on the retained handle | +| Synchronize a directory | Prior namespace effects reach that boundary, or an exact error | The same call on a directory handle | +| Acquire a native lock | Nonblocking kernel exclusion tied to the retained handle; typed contention | Common writer-lock and reader-fence code | +| Validate entry spelling | The actual namespace entry has the exact protocol spelling | The common directory wrapper; the check is compiled on macOS only | + +Use concrete observation records, checked constructors, and typed refusals. No +`supports_durability: bool` supplied by a caller, no catch-all string capability +map, and no public boolean parameters. Common code owns the protocol-directory list, checks every existing child, and compares its device and mount with the root. Native implementations own @@ -100,7 +108,11 @@ allocation, or reference-counted writer authority is needed for OS selection. ## Admission and continued authority -Admission is ordered as follows: +Platform admission covers the routes that admit a profile: +`FilesystemStoreAdmission::initialize` and `reopen`, +`FilesystemRetentionSnapshot::load` and the durable readers built on it, and +recovery admission and inventory. On those routes admission is ordered as +follows: 1. Pin the root through a no-follow traversal. 2. Observe its supported filesystem profile and every existing protocol child. @@ -115,6 +127,14 @@ Subsequent protocol admission must not revert to ambient path lookup. Newly created children are verified through retained parent capabilities before use. Existing substitution and uncertain-recovery refusals remain mandatory. +Three public routes admit no profile. `FilesystemCatalogSnapshot::load`, +`FilesystemCatalogSnapshot::load_for_verification`, and +`FilesystemWriterLock::try_acquire` open the selected path ambiently, so they +apply no filesystem-kind check and do not refuse a symbolic link in that path. +Every lookup beneath the opened root is still capability-relative, and the +exact-name check still applies to it. The repository-task entry points, compiled +only for tests and the `repository-tasks` feature, skip the profile by design. + An adapter's successful admission does not prove that the operator has not shared a block device with another host, or that hardware honestly implements flush commands. Local exclusive administration remains a deployment condition. @@ -149,10 +169,11 @@ case-insensitive variant and case-sensitive APFS. HFS+ and other filesystems remain explicit future profiles. APFS snapshots and read-only mounts refuse writer admission. -Root traversal uses descriptor-relative, no-follow component opens. Handle +On the routes that admit a profile, root traversal uses descriptor-relative, +no-follow component opens. Handle identity uses Darwin observations; it must not pretend that device identity is independently observed mount identity. Symbolic links in the selected path -remain a refusal. The documentation must explain that `/tmp` on macOS is a +remain a refusal on those routes. The documentation must explain that `/tmp` on macOS is a symlink and use the actual selected directory instead of silently resolving it. Finder can write `.DS_Store` into a folder it displays. That entry refuses admission @@ -182,7 +203,11 @@ repeats `st_dev` beside the filesystem type, so it is not read. ## Physical coordinates and durable compatibility Content identities, canonical segment/catalog bytes, and storage profiles do -not depend on the OS. Existing Linux bytes and golden fixtures remain stable. +not depend on the OS. The platform work changed no bytes and no golden fixture. +[ADR-0010](../adr/0010-one-storage-format.md) did, on every platform alike: it +changed the format-definition digest and the `FORMAT` marker fixture and deleted +the migration fixtures. The segment, catalog, and publication-head bytes are +unchanged. Physical coordinates are separate from logical content identity, and a store persists none of them ([ADR-0010](../adr/0010-one-storage-format.md)). They exist @@ -232,8 +257,9 @@ an unfinished backend in production: An independently useful new adapter must pass the same semantic conformance laws. Test both success and the boundary failures: symlinked roots/children, case aliases, missing fields, read-only mounts, foreign devices/mounts, -replaced lock files, copied roots, failed file/directory synchronization, and -each affected crash/recovery prefix. Calibrate refusal assertions against +replaced lock files, failed file/directory synchronization, and +each affected crash/recovery prefix. A quiescent copy of a complete root is not +a failure: since ADR-0010 it must open. Calibrate refusal assertions against deliberately weakened evidence, separately from product execution results. Build output is bounded within the aggregate 20 GiB Keep budget. Test stores, @@ -246,7 +272,9 @@ usage, free space, lock, enforced limits, timeout, and descendant cleanup. - [Hexagonal boundaries](https://github.com/flyingrobots/keep/blob/3887f2a/docs/adr/0004-hexagonal-boundary-architecture.md). - [Platform adapter decision](../adr/filesystem-platform-adapters.md). -- [Current filesystem contract](https://github.com/flyingrobots/keep/blob/3887f2a/docs/formats/segment-store-v1/recovery.md#platform-contract). +- [Current filesystem contract](../formats/store/stage-recovery.md#platform-contract). +- [Filesystem contract at the design baseline](https://github.com/flyingrobots/keep/blob/3887f2a/docs/formats/segment-store-v1/recovery.md#platform-contract), + on a page that has since been deleted. - [Apple APFS naming variants](https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/APFS_Guide/FAQ/FAQ.html). - [Apple file synchronization](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fsync.2.html). - [Apple descriptor controls](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/fcntl.2.html). diff --git a/docs/audits/pr-107-salvage.md b/docs/audits/pr-107-salvage.md index cab06e36..15d87453 100644 --- a/docs/audits/pr-107-salvage.md +++ b/docs/audits/pr-107-salvage.md @@ -59,7 +59,7 @@ means it has no remaining value. | Branch commits | Outcome | Blocked by | | --- | --- | --- | | `668fd5f`, `5bba27e` | Deterministic GC planning from a fenced liveness snapshot | Main lacks `CatalogSnapshot::record_segments`; whether main's live closure-member verification serves the planner is unassessed; main's snapshot load adds a root-identity check the planner must adopt. Lands after the codecs. | -| `9a10b22`, landing `8828db1`, `20dec6d`, `f6aca41` | GC execution, retirement, recovery, crash rows `KEEP-CRASH-074`–`087`, failing-capability reporting | Exclusive reader-fence API absent on main; typed `RetentionStorageError`; KEEP-GC-001 namespace admission must change from "empty `gc/`"; the `gc/intent.next` and `gc/receipt.next` stage names and rename-replacement of the receipt need a protocol ruling against main's "fixed names are never replaced" rule. | +| `9a10b22`, landing `8828db1`, `20dec6d`, `f6aca41` | GC execution, retirement, recovery, crash rows `KEEP-CRASH-074`–`087`, failing-capability reporting | **Crash identifiers must be renumbered.** The single format (#198) has taken `KEEP-CRASH-074` to `-091` (`-074` to `-077` and `-081` to `-091` implemented, `-078` retired, `-079` and `-080` reserved in #221), and identifiers are never reused, so this port renumbers its 14 rows from `KEEP-CRASH-092`. Exclusive reader-fence API absent on main; typed `RetentionStorageError`; KEEP-GC-001 namespace admission must change from "empty `gc/`"; the `gc/intent.next` and `gc/receipt.next` stage names and rename-replacement of the receipt need a protocol ruling against main's "fixed names are never replaced" rule. | | `2859f68` | Disposition enumerations and receipt codec | **Format decision.** The branch adds seven rows to `docs/formats/segment-store-v2/definition.tsv`, changing `StoreFormatDefinitionDigest::VERSION_TWO` from main's `32381f1a…` to `a4a010ce…`. Stores migrated by either side would refuse the other. v2 is unreleased, so a bump is possible, but only as an explicit decision record. | | `aa371ae`, landing `f21589f` | Explicit finalize-or-retire disposition of protected retention orphans | The format decision above; main's retention authority and storage port changed; overlaps #155. The landing RED test is reusable as a test idea only. | | `393accb`, landing `43e20c4`, `5a67c24`, `5f37c7e`, `c5b3e34`, `e49fde5`, `9731d50`, `92aafcd` | Identity-preserving compaction and its later recovery fixes | GC planning (compaction imports `plan_gc` and the liveness observation); v2 publisher and stage-discarder entry points absent on main. | diff --git a/docs/formats/store/README.md b/docs/formats/store/README.md index 15968d1a..63e056cb 100644 --- a/docs/formats/store/README.md +++ b/docs/formats/store/README.md @@ -14,8 +14,10 @@ ADR-0005 records the segment, catalog, and publication-head protocol. ADR-0009 owns the retention and liveness decision. [ADR-0010](../../adr/0010-one-storage-format.md) makes this the only storage format and replaces migration with direct initialization. These pages -are one normative set; every page describes the behaviour of `main`, and the -two requirement ledgers name the test behind each claim. +are one normative set. Every page describes `main` after #198, the two +requirement ledgers name the test behind each claim, and the +[nonclaims](requirements.md#compatibility-and-nonclaims) list what is not +established. Segment writing and verified reading are implemented in issue #15. Catalog generation, writer-locked publication, and immutable restart snapshots are @@ -54,8 +56,11 @@ The store adds these laws to the catalog protocol: not admit before that point; - an initialized store with no published catalog is a defined, admitted state; -- nothing persisted binds a store to its directory: a quiescent copy is the - same store, and live authority comes from the opening process; +- nothing persisted binds a store to its directory: a quiescent copy opens + and holds the same content, and it is a second, independent store; live + authority comes from the opening process; +- admission cannot tell the intended store from another valid one or an older + state, so every normal open takes a `StoreExpectation` with no default; - retention authority exists only through one verified retention head and its complete immutable manifest; - each manifest binds every admitted namespace to one exact root generation @@ -116,7 +121,7 @@ Evidence and decisions: owns the segment, catalog, and stage-recovery requirement ledger, security nonclaims, and golden evidence. - [Retired identifiers](retired-identifiers.md) maps every retired - requirement and crash identifier to its last evidence. + requirement and crash identifier to evidence on `main`. - [Format rationale](rationale.md) and the [catalog-protocol rationale](catalog-protocol-rationale.md) record format-local choices and rejected alternatives. @@ -138,8 +143,11 @@ these pages. They are format evidence, not production-writer evidence. ## Status -The format contract is frozen by ADR-0005, ADR-0009, ADR-0010, and these -pages. +ADR-0005, ADR-0009, ADR-0010, and these pages are the current format +contract. Nothing is released, so until the first release a grammar change +regenerates the fixtures. [#205](https://github.com/flyingrobots/keep/issues/205) +must close before then: the format-definition digest does not yet cover the +segment, catalog, and head grammars. Implemented with executable evidence: the segment, catalog, and head codecs and the shared catalog stages; public core types for namespaces, generations, @@ -150,7 +158,8 @@ one pinned catalog, preflight, preparation, and the 17-phase publication port; `FilesystemStoreAdmission::initialize`, which creates or resumes a store through the initialization protocol and its 41-case process-death matrix; `FilesystemStoreAdmission::reopen`, which admits the committed -namespace and the marker and pins the retention directories it admitted; +namespace and the marker, compares the caller's `StoreExpectation` with `HEAD` +([Portable stores](recovery.md#portable-stores)), and pins its retention directories; `FilesystemCatalogPublisher`, which ingests one segment and publishes a catalog through the retained-closure gate ([#125](https://github.com/flyingrobots/keep/issues/125)), including the first @@ -174,10 +183,14 @@ automatic disposal remains deferred in [#155](https://github.com/flyingrobots/keep/issues/155). Writer authority coordinates cooperating writers in a managed namespace; it does not isolate arbitrary out-of-band filesystem mutation or make pathname unlink conditional -on inode identity. Production durable ingestion -([#82](https://github.com/flyingrobots/keep/issues/82)), garbage collection, -and compaction ([#21](https://github.com/flyingrobots/keep/issues/21)) remain -absent from main. +on inode identity. + +`FilesystemCatalogPublisher` publishes at most one new segment and one +caller-supplied catalog per call. Production durable ingestion +([#82](https://github.com/flyingrobots/keep/issues/82)), garbage collection, and +compaction ([#21](https://github.com/flyingrobots/keep/issues/21)) remain absent +from main. The recovery crash rows `KEEP-CRASH-079` and `-080` are planned in +[#221](https://github.com/flyingrobots/keep/issues/221). The `KEEP-CRASH-036..052` initial-publication process-death sequence includes independent recovered-reader checks; its diff --git a/docs/formats/store/catalog-protocol-requirements.md b/docs/formats/store/catalog-protocol-requirements.md index 1d68aa53..5264489a 100644 --- a/docs/formats/store/catalog-protocol-requirements.md +++ b/docs/formats/store/catalog-protocol-requirements.md @@ -32,8 +32,8 @@ evidence](requirements.md). | `KEEP-STORE-018` | Truncated-stage discard has explicit unlink and directory-sync crash boundaries | `KEEP-CRASH-027`–`028` | Specified in #14 | | `KEEP-STORE-019` | Leftover next-head recovery either finalizes an exact successor or discards explicit evidence durably | `KEEP-CRASH-025`–`026`, `029`–`030` | Specified in #14 | | `KEEP-STORE-021` | Explicit recovery completes a durable fixed-name stage into its immutable pool without publishing a head | `KEEP-CRASH-008`–`012`, `016`–`020` | Specified in #14 | -| `KEEP-STORE-022` | Opening a store requires a typed expectation choice with no default; an expected catalog refuses any other published catalog, or none, with the expected and observed values | `tests/single_format/expectation_laws.rs` | Implemented in #198 | -| `KEEP-STORE-023` | Every mix of two published states and every partly copied file refuses or returns exactly the bytes an identity names | `tests/single_format/torn_copy_laws.rs` | Implemented in #198 | +| `KEEP-STORE-022` | Every normal open (`FilesystemStoreAdmission::reopen`, `FilesystemCatalogSnapshot::load`, `FilesystemRetentionSnapshot::load`, `DurableSnapshot::open`, and `DurableStore::open` at each snapshot it pins) requires a typed expectation choice with no default; an expected catalog refuses any other published catalog, or none, with the expected and observed values, and an expectation rebuilt from a recorded generation and digest behaves as the original. Recovery tools, verification loads, initialization, and the repository-task reopen take no expectation, so their callers do not get this check | `tests/single_format/expectation_laws.rs`; after process death in the gated writer, the catalog recorded before the publication refuses at generation two in every `retained-catalog` crash case and admits at generation one in those that recover by republication (`xtask/src/durability_crash_matrix/restart/recorded_catalog.rs`) | Implemented in #198 | +| `KEEP-STORE-023` | Every mix of two adjacent quiescent published states, and each nonempty file at half length over the newer state, refuses or returns exactly the bytes an identity names | `every_mix_of_two_quiescent_published_states_refuses_or_returns_exact_bytes` and `a_half_length_copy_of_any_one_file_over_the_newer_state_refuses_or_returns_exact_bytes` in `tests/single_format/torn_copy_laws.rs`; a refusal counts only with a typed absent-or-malformed cause | Implemented in #198 for these cases; a copy holding an in-flight stage, other truncation lengths, and tears over the older state are untested | diff --git a/docs/formats/store/catalog-publication.md b/docs/formats/store/catalog-publication.md index d2adb2d3..804aa055 100644 --- a/docs/formats/store/catalog-publication.md +++ b/docs/formats/store/catalog-publication.md @@ -25,8 +25,10 @@ initialization](initialization.md). ## Authority -`FilesystemStoreAdmission::reopen` admits the root and acquires the writer -lock. `FilesystemCatalogPublisher::open` consumes that admission. It +`FilesystemStoreAdmission::reopen` admits the root, acquires the writer +lock, and compares the caller's `StoreExpectation` with `HEAD` under it +([Portable stores](recovery.md#portable-stores)). +`FilesystemCatalogPublisher::open` consumes that admission. It pins the store root, `staging`, `segments`, `catalogs`, `retention`, `retention/roots`, and `retention/manifests`. A bare writer lock cannot open it. @@ -139,6 +141,28 @@ the gate's exact refusal, and `head.next`, the pools, and `HEAD` stay unchanged. A retained retention stage, or a retention head absent while the pools hold artifacts, refuses in the same way. +The gate runs at three entries: the first verification, the verification that +ends candidate synchronization, and the verification inside `HEAD` +replacement. When `HEAD` already names the target and no `head.next` remains, +finalization reports `AlreadyFinalized`: it runs no gate, moves nothing, and +only synchronizes the root. + +The laws hold these entries in place unevenly: + +- the first entry refuses a candidate that would lose retained content, with + the store unchanged + (`recovery_refuses_a_staged_head_that_would_lose_retained_content` in + `tests/catalog_writer.rs`); +- the entry inside replacement is the same verification, and + `changed_candidate_is_refused_at_the_replacement_boundary` fails if it is + removed, because a `head.next` changed after the first verification must + refuse there; and +- no law changes the retention head or its roots between candidate + synchronization and replacement, so that the last entry refuses for a + retained closure specifically is established by reading the shared + verification, not by a test. Process death at this boundary is + `KEEP-CRASH-080`, which is planned. + ## Roots without the marker - Writer admission, recovery admission, the recovery inventory, catalog @@ -171,14 +195,27 @@ recovery tools, retains the ingested blob, and reads both blobs through | `KEEP-CRASH-076` | `head.next` synchronization | before, during, after | complete `head.next`; gated finalization | Implemented | | `KEEP-CRASH-077` | `HEAD` replacement | before, during, after | complete `head.next` and gated finalization before; generation two published otherwise | Implemented | | `KEEP-CRASH-078` | recovery stage removal under recovery admission | before, during, after | covered by `KEEP-CRASH-027`–`030`, which now run under recovery admission | Retired (#198) | -| `KEEP-CRASH-079` | recovery stage completion link | before, during, after | the completion executor's law under recovery admission | Planned | -| `KEEP-CRASH-080` | gated finalization replacement | before, during, after | finalized or retryable `head.next`; the gate reruns on retry | Planned | +| `KEEP-CRASH-079` | recovery stage completion link | before, during, after | the completion executor's law under recovery admission | Planned in [#221](https://github.com/flyingrobots/keep/issues/221); not implemented | +| `KEEP-CRASH-080` | gated finalization replacement | before, during, after | finalized or retryable `head.next`; the gate reruns on retry | Planned in [#221](https://github.com/flyingrobots/keep/issues/221); not implemented | The implemented rows are 12 cases. CI runs them with the complete matrix in debug and release builds. They establish recovery after process death; they do -not simulate power loss. In-process laws add an injected storage-port failure +not simulate power loss. Restart also holds the generation-one catalog a +caller would have recorded before the publication. In the cases that recover +by republication it must admit while `HEAD` names generation one, and in every +case it must refuse, reporting generation two, once `HEAD` names generation +two +([Recovering a stale expectation](recovery.md#recovering-a-stale-expectation)). + +`KEEP-CRASH-079` and `-080` have no implementation. The plan placed them +before version 1 was deleted; the owner accepted on 2026-11-05 that they +follow that deletion, and +[#221](https://github.com/flyingrobots/keep/issues/221) blocks the first +release on them. Until then, process death during recovery's completion link +and during gated `HEAD` replacement, and the gate rerunning on a retry after +such a death, are unproven. In-process laws add an injected storage-port failure after segment link, catalog-stage creation, and `head.next` synchronization (`filesystem_catalog_recovery_tests`), plus hand-staged complete `head.next` candidates that preserve or drop retained content diff --git a/docs/formats/store/initialization.md b/docs/formats/store/initialization.md index 9dcef6f0..6cec2d8e 100644 --- a/docs/formats/store/initialization.md +++ b/docs/formats/store/initialization.md @@ -27,7 +27,8 @@ recovery/dispositions/ Catalog `HEAD` is absent until the first catalog is published. Each directory is empty, and `writer.lock` and `reader.lock` are zero-length regular files. Nothing in the store records the root directory's device, inode, mount, or -path, so a quiescent copy of a committed store is the same store; see +path, so a quiescent copy of a committed store opens and holds the same +content, and it is a second, independent store; see [Portable stores](recovery.md#portable-stores). A committed store with no catalog `HEAD` is a defined state, not an inferred @@ -83,6 +84,28 @@ synchronization returned, so a root holding `FORMAT` without `FORMAT.next` can be observed only after the commit is durable. Initialization returns writer authority only after `SynchronizeRootAfterCleanup`. +What the laws witness is the order of requests, not durability. The four +root-synchronization phases call one synchronizer, which +`initialization_commit_tests` substitutes: + +- `the_commit_synchronization_runs_after_the_link_and_before_stage_removal` + requires the commit synchronization to be requested after `LinkMarker` + returned and before `RemoveMarkerStage` begins, while both `FORMAT` and + `FORMAT.next` exist; and +- `a_failed_commit_synchronization_leaves_an_unadmitted_resumable_store` fails + that request and requires `StoreInitializationError::Io` at + `SynchronizeRootAfterMarker`, no later phase, an admission refusal as + `InitializationIncomplete`, and a retry that resumes from the marker-linked + state. + +That the production synchronizer makes the root durable is established by +reading it, not by a test. A synchronizer changed to do nothing passed the +complete native test suite when tried against `fb355d66`. The process-death +matrix was not run against it and is not expected to expose it either, because +the page cache survives process death. No power-loss test exists. The +[mutation record](../../testing-evidence/single-format-mutants.md) holds both +runs. + ## States and results Initialization classifies the root before it creates `writer.lock`, and again diff --git a/docs/formats/store/publication.md b/docs/formats/store/publication.md index fa4e3ece..159f300f 100644 --- a/docs/formats/store/publication.md +++ b/docs/formats/store/publication.md @@ -26,9 +26,14 @@ capability-relative and refuse symlinks, nonregular files, alternate spellings, unknown entries in protocol-owned directories, and replacement of the opened store root. -The filesystem must expose case-sensitive, byte-preserving directory names. -The capability probe refuses case-folding or normalization aliases before a -store root is initialized or admitted. +No protocol name may be answered by another spelling. On Linux, platform +admission refuses a casefolded ext4 root or protocol directory before a store +is initialized or admitted. On macOS, APFS is admitted in both naming +variants, including the case-insensitive default, so a lookup can resolve a +case-folded or normalized spelling. Every lookup of an existing protocol +entry therefore requires that exact name, byte for byte, in a listing of its +parent, and refuses otherwise. The [platform contract](stage-recovery.md#platform-contract) names +the entry points that run platform admission. The names are physical adapter coordinates, not stable public handles. `writer.lock` persists; its existence and contents prove nothing. @@ -120,9 +125,10 @@ admission boundary can produce a value. A new store obtains one through `FilesystemStoreAdmission::initialize`, which runs the [initialization protocol](initialization.md); a committed store reacquires one through `FilesystemStoreAdmission::reopen`, which -mutates nothing. A root without the `FORMAT` marker refuses at every open -path. The version-1 initializer, `FilesystemPlatformAdmission`, and their -rows `KEEP-CRASH-031`–`-035` are retired. +mutates nothing and takes the caller's `StoreExpectation`. A root without +the `FORMAT` marker refuses at every open path. The version-1 initializer, +`FilesystemPlatformAdmission`, and their rows `KEEP-CRASH-031`–`-035` are +retired. The crash matrix initializes its store and opens the writer through these production entry points, then observes the writer's own transitions. @@ -153,7 +159,11 @@ still-open sealed stage. filesystem-stage constructor. Its returned lifetime keeps the acquired writer authority borrowed while `current.seg` remains writable. -`FilesystemCatalogSnapshot::load` is the observational reader boundary. Its +`FilesystemCatalogSnapshot::load` is the unfenced reader boundary: it takes +no writer lock, no reader fence, and no platform admission. It is a normal +open, so it requires a `StoreExpectation` and compares it with the snapshot +after that snapshot verifies; a mismatch is `CatalogRestartError::Expectation` +([Portable stores](recovery.md#portable-stores)). Its `CatalogRestartPolicy` combines segment parser limits with a positive maximum for aggregate retained segment bytes. The loader follows only exact head-selected coordinates, refuses symbolic links and nonregular artifacts, diff --git a/docs/formats/store/rationale.md b/docs/formats/store/rationale.md index 8e5f806f..a8c1d3bf 100644 --- a/docs/formats/store/rationale.md +++ b/docs/formats/store/rationale.md @@ -95,6 +95,9 @@ intent once carried a deterministic, content-derived identifier; it was deleted with migration, and initialization records none. Two stores with the same bytes are indistinguishable by design; telling an intended store from a valid stranger needs an expectation the caller keeps, as ADR-0010 states. +Keep stores no part of that expectation: the caller records a catalog +generation and digest and names them at every normal open +([Portable stores](recovery.md#portable-stores)). ## Reserve GC names but refuse their state @@ -118,9 +121,9 @@ same-process protocol-directory binding, and no-follow capability access. ADR-0009 requires version-2 catalog publication to verify every current retained closure against the candidate catalog before it replaces `HEAD`. `publish_retained_catalog_generation` implements that rule over the -`RetainedCatalogPublicationStorage` port (#125). It is the entry point a -version-2 catalog writer plugs into; this change adds no filesystem version-2 -catalog publisher. +`RetainedCatalogPublicationStorage` port (#125). `FilesystemCatalogPublisher` +is the filesystem writer that plugs into it, and it implements no other +publication port ([catalog publication](catalog-publication.md#authority)). Under writer authority, the gate runs the pure catalog preflight first. It then reads the retention head, the manifest that head names, and every root diff --git a/docs/formats/store/recovery.md b/docs/formats/store/recovery.md index 8cff195e..0afe0c8a 100644 --- a/docs/formats/store/recovery.md +++ b/docs/formats/store/recovery.md @@ -114,10 +114,25 @@ Live authority is unchanged: applies, so a corrupt or incomplete copy refuses exactly as the original would. -A copy taken while a writer publishes is outside this guarantee, but it cannot make -Keep return other bytes. It may hold any mix of two published states and a partly -copied file, and every such mix refuses or returns exactly the bytes an identity -names. It need not open at all, and it can lose the newer state. +A copy taken while a writer publishes is outside this guarantee. Such a copy +may hold a mix of the states before and after the publication, a partly copied +file, and the writer's in-flight stages. It need not open at all, and it can +lose the newer state. The law it must still obey is the core law: it refuses +or returns exactly the bytes an identity names. + +That law is tested for part of what a torn copy may contain +(`tests/single_format/torn_copy_laws.rs`, `KEEP-STORE-023`): + +- every mix of two adjacent quiescent published states, file by file + (`every_mix_of_two_quiescent_published_states_refuses_or_returns_exact_bytes`); + and +- one file at half its length over the newer state, for each nonempty file in + turn + (`a_half_length_copy_of_any_one_file_over_the_newer_state_refuses_or_returns_exact_bytes`). + +A refusal counts only when its typed cause is an absent or malformed artifact; +any other failure fails the law. A copy that holds an in-flight stage, a file +at another length than half, or a tear over the older state is not tested. A copy and its original are two stores. Each takes its own locks, and their histories diverge independently while unchanged content keeps its identity. @@ -125,18 +140,82 @@ Copies are not replication and are never coordinated. Admission alone does not detect an unintended but valid store, or an older valid state of the intended one. Detecting either needs an expectation the caller -supplies and keeps outside the checked state. `FilesystemStoreAdmission::reopen`, -`FilesystemRetentionSnapshot::load`, `DurableSnapshot::open`, and -`DurableStore::open` therefore require a `StoreExpectation`, with no default: +supplies and keeps outside the checked state. Every normal open therefore +requires a `StoreExpectation`, with no default: + +- `FilesystemStoreAdmission::reopen`, which compares `HEAD` under the writer + lock; +- `FilesystemCatalogSnapshot::load`, which compares the snapshot it loaded + after that snapshot verifies, with no lock and no fence; +- `FilesystemRetentionSnapshot::load` and `DurableSnapshot::open`, which + compare the collected view under the reader fence; and +- `DurableStore::open`, which only records the choice: it checks nothing until + a snapshot is pinned, and every snapshot then compares as + `DurableSnapshot::open` does. + +The choice is one of: - `Catalog` names an exact catalog generation and digest, and any other published catalog, or none, refuses with the expected and observed values; and - `NoExpectation` accepts any valid store at the path, and says so at the call site. -The writer compares under the writer lock and the reader compares its collected -view under the fence. `Catalog` does not detect a rollback that leaves the catalog -unchanged, such as restored retention state alone. Recovery and verification opens -take no expectation: they examine whatever state the path holds. +`Catalog` does not detect a rollback that leaves the catalog unchanged, such +as restored retention state alone. Two stores that publish the same catalog +bytes are indistinguishable, and "expected to have published nothing" has no +representation. + +These routes take no expectation, so a caller of them does not get this check: + +- the recovery tools (`FilesystemRecoveryAdmission::reopen` and the + constructors that consume it, and `FilesystemRecoveryInventoryReader::open`); +- verification loads (`FilesystemCatalogSnapshot::load_for_verification` and + `FilesystemRetentionSnapshot::load_for_verification`), which examine + whatever state the path holds; +- `FilesystemStoreAdmission::initialize`, including when it resumes a + marker-linked root that already holds published state + ([States and results](initialization.md#states-and-results)); and +- the repository-task entry point + `FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks`, which + exists only under the `repository-tasks` feature. + +### Keeping an expectation + +Keep stores nothing for the caller. `ExpectedCatalog` is the caller's claim, +not a verified digest. Within one process, `ExpectedCatalog::new` takes the +generation and digest that a publication receipt or a pinned view reports. To +carry it into a later process, the caller records the generation as its `u64` +and the 32 digest bytes from `ExpectedCatalog::digest_bytes`, and rebuilds the +value with `ExpectedCatalog::from_recorded`. Where that record lives, and how +it is protected, is the caller's responsibility: a record an attacker can +rewrite protects nothing. +`an_expectation_rebuilt_from_recorded_values_admits_and_refuses_as_the_original` +shows that the rebuilt value admits and refuses exactly as the original does. + +### Recovering a stale expectation + +An expectation names one catalog, so the caller must record the new generation +and digest after every publication. Publication and that record are two steps. +A process that dies between them, or after gated next-head finalization moved +`HEAD`, leaves the store at the successor while the caller still holds the +predecessor. Every normal open then refuses at its expectation boundary with +a `StoreExpectationMismatch`. That refusal is the expected result, not damage. + +The mismatch reports the expected catalog and, as `ObservedCatalog`, what the +store publishes now. Keep cannot tell the caller's own successor from a +different store through this interface: both are a valid store that publishes +another catalog. Recording what the mismatch observed, or opening with +`NoExpectation`, is therefore an explicit trust decision that the caller makes +with evidence from outside the store. Keep never makes it, and no open accepts +a successor on its own. An expectation that also accepts the direct successor +of the recorded catalog is not implemented. + +The crash matrix pins this for the gated writer. In every `retained-catalog` +case (`KEEP-CRASH-074` to `-077`), restart builds the generation-one catalog a +caller would have recorded before the interrupted publication +(`xtask/src/durability_crash_matrix/restart/recorded_catalog.rs`). In the +cases that recover by republication, `HEAD` still names generation one, and +the load under that expectation is admitted. In every case, once `HEAD` names +generation two, the same expectation must refuse and report generation two. ### Root identity across restart diff --git a/docs/formats/store/requirements.md b/docs/formats/store/requirements.md index 60b50ec2..45f1d12e 100644 --- a/docs/formats/store/requirements.md +++ b/docs/formats/store/requirements.md @@ -22,7 +22,7 @@ case is not evidence. | `KEEP-RETENTION-011` | Catalog publication over retention state admits the current retention head, manifest, and every manifest root under writer authority; verifies every retained closure against the candidate before any catalog storage call; and refuses `HEAD` replacement if the retention head changed | port-level laws in `tests/retained_catalog_publication.rs`: dropped and substituted members refuse with the exact missing identity and no catalog stage; every manifest root is verified wherever it sorts; absent, corrupt, substituted, and contradictory head, manifest, and root evidence refuse with typed expected and observed state; a head changed, removed, or published before commit refuses after `head.next` verification without `replace_head`; an exhaustive 24-case candidate-by-root-set model and a repeat-run determinism check | Implemented at the storage port; the filesystem version-2 publisher and version-2 next-head finalization use it (#198, `KEEP-RETENTION-012` and `013`) | | `KEEP-RETENTION-012` | Under writer authority, one staged segment and a catalog successor publish only through the retained-closure gate, with no-follow bounded retention reads, head absence admitted only when uncontradicted, fixed retention stages refused, and no reader fence taken | `FilesystemCatalogPublisher`; `tests/catalog_writer.rs`: a preserving successor publishes, reopens, and reads both blobs while the retention root still verifies; a pinned reader view keeps its generation across publication; a candidate that drops retained content, a contradicted head absence, and a fixed retention stage each refuse with the exact typed error and an unchanged store witness; runtime RED for the contradiction and stage checks | Implemented for one staged segment per publication; segment rollover is #121 | | `KEEP-RETENTION-013` | Segment and catalog stage recovery admits every store root entry as known, reuses the catalog-protocol executors, and finalizes a staged `head.next` only after the retained-closure gate admits it | `FilesystemRecoveryAdmission`, `classify_recovery_names`, and the `open` constructors; `tests/catalog_writer.rs` covers classification, completion, discard, reusable-prefix segment resume, and finalization that preserves or would lose retained content (runtime RED with the gate result ignored); `filesystem_catalog_recovery_tests` recovers injected interruptions after segment link, catalog-stage creation, and `head.next` synchronization | Implemented | -| `KEEP-RETENTION-015` | Process death in the gated writer and its recovery reaches documented recoverable states | `cargo xtask durability-crash-matrix --sequence retained-catalog` runs `KEEP-CRASH-074..=077`, 12 cases, through recovery and durable reads; stage discard under recovery admission is `KEEP-CRASH-027..=030` (so the planned `-078` is retired); `KEEP-CRASH-079..=080` (completion and gated finalization) are planned | In progress: writer rows implemented, recovery rows planned | +| `KEEP-RETENTION-015` | Process death in the gated writer and its recovery reaches documented recoverable states | `cargo xtask durability-crash-matrix --sequence retained-catalog` runs `KEEP-CRASH-074..=077`, 12 cases, through recovery and durable reads; stage discard under recovery admission is `KEEP-CRASH-027..=030` (so the planned `-078` is retired); `KEEP-CRASH-079..=080` (completion and gated finalization under recovery admission) have no implementation | In progress: writer rows implemented; recovery rows planned in [#221](https://github.com/flyingrobots/keep/issues/221), which blocks the first release. The owner accepted on 2026-11-05 that they follow the deletion of version 1 | @@ -42,12 +42,12 @@ case is not evidence. | ID | Requirement | Evidence | Status | | --- | --- | --- | --- | -| `KEEP-INITIALIZATION-001` | Direct initialization creates exactly the committed namespace under exclusive writer authority, links `FORMAT` from a durable `FORMAT.next` without replacement, and returns authority only after the commit and cleanup synchronizations and complete admission | `a_fresh_root_commits_to_exactly_the_initialized_namespace` in `initialization_resume_tests`; production-profile laws in `tests/single_format.rs` | Implemented (#198) | -| `KEEP-INITIALIZATION-002` | Every interrupted state classifies as exactly one prefix and resumes idempotently; every other nonempty root, including a committed one, refuses with typed evidence before any mutation, writer-lock creation included | `every_interrupted_boundary_classifies_and_resumes_to_one_store` (every boundary of a recorded run, against a model of the protocol table) and the incomplete-stage and sync-failure laws in `initialization_resume_tests`; foreign, out-of-order, stray-retention, foreign-stage, nonempty-lock, linked-directory, committed, and copied-marker refusals with unchanged witnesses in `initialization_prefix_tests` | Implemented (#198) | +| `KEEP-INITIALIZATION-001` | Direct initialization creates exactly the committed namespace under exclusive writer authority, links `FORMAT` from a durable `FORMAT.next` without replacement, and returns authority only after the commit and cleanup synchronizations and complete admission | `a_fresh_root_commits_to_exactly_the_initialized_namespace` in `initialization_resume_tests` (namespace and bytes); `the_commit_synchronization_runs_after_the_link_and_before_stage_removal` in `initialization_commit_tests`, which substitutes the synchronizer of the root-synchronization phases and requires exactly four requests: after the lock, after the stage name, at the commit point between the link and the stage removal while both marker names exist, and after cleanup; production-profile laws in `tests/single_format.rs` | Implemented (#198). The order of the requested synchronizations is witnessed; that the production synchronizer reaches stable storage is by inspection, and no power-loss test exists | +| `KEEP-INITIALIZATION-002` | Every interrupted state classifies as exactly one prefix and resumes idempotently; every other nonempty root, including a committed one, refuses with typed evidence before any mutation, writer-lock creation included | `every_interrupted_boundary_classifies_and_resumes_to_one_store` (every boundary of a recorded run, against a model of the protocol table) the incomplete-stage and interrupted-before-commit laws in `initialization_resume_tests`, and `a_failed_commit_synchronization_leaves_an_unadmitted_resumable_store` in `initialization_commit_tests`, where the injected synchronizer fails at the commit point; foreign, out-of-order, stray-retention, foreign-stage, nonempty-lock, linked-directory, committed, and copied-marker refusals with unchanged witnesses in `initialization_prefix_tests` | Implemented (#198) | | `KEEP-INITIALIZATION-003` | Admission requires a committed marker: writers and readers refuse `FORMAT.next` as `InitializationIncomplete`, and every open path (writer, recovery, recovery inventory, catalog restart, retention reader, durable reader) refuses a missing `FORMAT` as `MissingFormatMarker`, so no root without the single format's marker is admitted; a committed store without catalog `HEAD` admits writers, refuses readers as `Unpublished`, and its first publication proves both absences instead of assuming them | `filesystem_store_admission_tests`; `every_open_path_refuses_a_root_without_the_format_marker` in `tests/catalog_writer.rs`; `tests/single_format.rs` initialization laws, including a first publication refused by a stray retention root with an unchanged store | Implemented (#198); every open path since version-1 admission was deleted | | `KEEP-INITIALIZATION-004` | Process death before, during, and after every initialization boundary reaches a documented state that `initialize` recovers | `cargo xtask durability-crash-matrix --sequence initialization` runs 41 cases at `KEEP-CRASH-081..=091`, checks the exact inventory, marker bytes and inode, admission result, and resumed state, then publishes, retains, and reconstructs the first blob | Implemented (#198); process death only, not power loss | | `KEEP-INITIALIZATION-005` | Concurrent initialization attempts have exactly one initializer: the other refuses on writer authority and, after commit, as already initialized | `concurrent_attempts_have_exactly_one_initializer` in `initialization_resume_tests` | Implemented (#198) | -| `KEEP-INITIALIZATION-006` | One public-API sequence initializes, ingests, reopens, reconstructs exact bytes, retains, publishes a second catalog, interrupts publication, recovers, and reads both retained contents; a candidate that would lose retained content refuses with its evidence intact | `the_single_format_serves_every_operation_from_initialization_to_recovery` in `tests/single_format.rs` | Implemented (#198) | +| `KEEP-INITIALIZATION-006` | One public-API sequence initializes, ingests, reopens, reconstructs exact bytes, retains, publishes a second catalog, interrupts publication, recovers, and reads both retained contents; a candidate that would lose retained content refuses with its evidence intact | `the_single_format_serves_every_operation_from_initialization_to_recovery` in `tests/single_format.rs`, which runs only with the `repository-tasks` feature | Implemented (#198) | @@ -59,18 +59,21 @@ case is not evidence. | --- | --- | --- | --- | | `KEEP-PORTABILITY-001` | A valid quiescent copy opens and reads the same content; nothing persisted binds a store to its directory | `a_valid_quiescent_copy_opens_and_reads_the_same_content` in `tests/single_format.rs` | Implemented (#198) | | `KEEP-PORTABILITY-002` | A corrupt or incomplete copy refuses exactly as the original with the same damage would | `a_corrupt_copy_refuses_exactly_as_the_corrupt_original_would` and `an_incomplete_copy_refuses_exactly_as_the_incomplete_original_would` | Implemented (#198) | -| `KEEP-PORTABILITY-003` | A lock file or protocol directory replaced during an operation still refuses | `a_replaced_lock_or_protocol_directory_during_an_operation_still_refuses` | Implemented (#198) | +| `KEEP-PORTABILITY-003` | A lock file or protocol directory replaced during an operation still refuses | `a_replaced_lock_or_protocol_directory_during_an_operation_still_refuses` | Implemented (#198). The directory half asserts `ProtocolDirectoryReplaced`. The lock half asserts `Busy`, which the held root-directory lock produces with or without the replacement, so that half does not isolate lock-file replacement | | `KEEP-PORTABILITY-004` | A copy and its original take independent locks and diverge independently while unchanged content keeps its identity and bytes | `a_copy_and_its_original_diverge_independently_and_keep_shared_identity` | Implemented (#198) | -| `KEEP-PORTABILITY-005` | Backup round trip: close, copy, make the original unavailable, reopen and verify the copy, interrupt a new write on the copy, and recover it | `a_backup_copy_reopens_verifies_and_recovers_an_interrupted_write` | Implemented (#198) | +| `KEEP-PORTABILITY-005` | Backup round trip: close, copy, make the original unavailable, reopen and verify the copy, interrupt a new write on the copy, and recover it | `a_backup_copy_reopens_verifies_and_recovers_an_interrupted_write`, which runs only with the `repository-tasks` feature | Implemented (#198). "Verify" is writer reopen and an authenticated read of the retained blob; the law runs no verification report | ## Retired identifiers The `KEEP-MIGRATION-001` to `-008` requirements and the `KEEP-CRASH-053` to -`-073` process-death rows were retired with migration. They are never reused. -The [retirement ledger](retired-identifiers.md) maps each one to its last -evidence and states where its obligation went. +`-073` process-death rows were retired with migration. `KEEP-STORE-020`, +`KEEP-RECOVERY-002` and `-003`, `KEEP-RETENTION-014`, and the +`KEEP-CRASH-031` to `-035` rows were retired with version-1 admission, and +the reserved `KEEP-CRASH-078` was retired unimplemented. None is ever reused. +The [retirement ledger](retired-identifiers.md) maps each one to evidence on +`main` and states where its obligation went. ## Garbage collection reservation @@ -105,7 +108,11 @@ evidence and states where its obligation went. reopens the head-selected catalog and its segments under authority. Forward publication and recovery both replay the bounded live closure before effects. - Admission alone does not detect an unintended but valid store or an older - valid state, so every open requires an explicit expectation choice; a copy of - a store is a second, independent store. + valid state, so every normal open requires an explicit expectation choice. + Recovery tools, verification loads, initialization, and the repository-task + reopen take none, so their callers do not get this check + ([Portable stores](recovery.md#portable-stores)). Without an expectation, a + restored or torn copy may silently be an older state. A copy of a store is + a second, independent store. - Benchmarks are required before performance-sensitive retention or initialization optimization. diff --git a/docs/formats/store/retired-identifiers.md b/docs/formats/store/retired-identifiers.md index 4bef458b..ebf58700 100644 --- a/docs/formats/store/retired-identifiers.md +++ b/docs/formats/store/retired-identifiers.md @@ -4,9 +4,12 @@ This ledger records requirement and crash identifiers that no longer name anything in the code. A retired identifier is never reused, even for a similar obligation; a new obligation takes a new identifier. -Each row names the last commit on `main` that carried the identifier's -evidence, and where its obligation went: it either disappeared with the -subject it constrained, or moved to a current identifier. +Each row names a commit on `main` at which the identifier's evidence can be +read, and where its obligation went: it either disappeared with the subject it +constrained, or moved to a current identifier. The commit is not necessarily +the last one that carried the evidence: the evidence stayed on `main` until +the change that retired its identifier. One row, `KEEP-RETENTION-014`, names +a commit of that change itself and says so. ## Migration (ADR-0010, #198) @@ -21,29 +24,29 @@ is retired with it. -| ID | Obligation | Last evidence on `main` | Disposition | +| ID | Obligation | Evidence on `main` | Disposition | | --- | --- | --- | --- | -| `KEEP-MIGRATION-001` | Version-1 stores stay admitted until a durable migration artifact exists | `tests/store_migration_recovery.rs` at `a58f237`; migration recovery laws under `src/adapters/store_migration/` at `6e7e2d6` | Disappeared: no migration artifact exists. Version-2 code refuses version-1 roots under `KEEP-RETENTION-014` until version-1 admission is deleted | +| `KEEP-MIGRATION-001` | Version-1 stores stay admitted until a durable migration artifact exists | `tests/store_migration_recovery.rs` at `a58f237`; migration recovery laws under `src/adapters/store_migration/` at `6e7e2d6` | Disappeared: no migration artifact exists, and version-1 admission is deleted. A root without `FORMAT` refuses under `KEEP-INITIALIZATION-003` | | `KEEP-MIGRATION-002` | Marker, intent, and receipt byte tables, domains, bounds, checksums, store identity, initial-state digests | `tests/store_format_marker.rs` at `aabfb89`; `tests/store_migration_intent.rs` at `fc20097`; `tests/store_migration_receipt.rs` at `94092ac`; `migration_format` fuzz target at `85919fb` | Marker part moved to `KEEP-FORMAT-001`. Intent, receipt, store identifier, and initial-state digests disappeared with their records | | `KEEP-MIGRATION-003` | Migration revalidates the version-1 head, catalog, pools, root identity, and writer authority before mutation | `tests/store_migration_inventory.rs` at `89429a5`; `tests/store_migration_execution.rs` at `3fc9664`; authority laws at `6e7e2d6` | Disappeared: there is no version-1 source to revalidate. The persisted root-identity binding is retired by ADR-0010; live checks are `KEEP-PORTABILITY-003` | | `KEEP-MIGRATION-004` | Every partial migration prefix continues idempotently, comparing restart-stable root coordinates | `tests/store_migration_recovery.rs` at `a58f237`; prefix and truncation laws at `6e7e2d6` | Prefix continuation moved to `KEEP-INITIALIZATION-002`. The root-coordinate comparison disappeared (ADR-0010) | | `KEEP-MIGRATION-005` | Unknown, out-of-order, substituted, corrupt, or changed evidence is unrecoverable ambiguity | restart matrix laws at `6e7e2d6`; `docs/testing-evidence/migration-restart-matrix.md` at `e206298` | Moved to `KEEP-INITIALIZATION-002`: a root that is not an initialization prefix refuses unchanged | | `KEEP-MIGRATION-006` | Migration never rewrites or deletes admitted version-1 immutable bytes | storage and recovery witnesses at `6e7e2d6`; migration crash restart witnesses at `c2414bf` | Disappeared: nothing converts a store | | `KEEP-MIGRATION-007` | Process death at every migration boundary reaches a documented lawful state | `cargo xtask durability-crash-matrix --sequence migration` at `c2414bf` | Moved to `KEEP-INITIALIZATION-004` under new crash identifiers | -| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact | `tests/store_migration_compatibility.rs` at `85919fb`; `docs/testing-evidence/migration-compatibility-fuzz.md` at `1c886fd` | The version-2 part continues under `KEEP-RETENTION-014` until version-1 admission is deleted; the partial-migration part disappeared | +| `KEEP-MIGRATION-008` | Version-1 admission refuses every version-2 or partial-migration artifact | `tests/store_migration_compatibility.rs` at `85919fb`; `docs/testing-evidence/migration-compatibility-fuzz.md` at `1c886fd` | Disappeared: version-1 admission is deleted, so nothing remains to refuse version-2 artifacts, and no partial migration can exist. The single format's refusal of a root without `FORMAT` is `KEEP-INITIALIZATION-003` | The persisted reader root-identity requirement recorded in [Retention reader root identity](../../adr/retention-reader-root-identity.md) is superseded by ADR-0010 and replaced by the `KEEP-PORTABILITY-*` laws. Its -last evidence on `main` is +evidence on `main` is `src/adapters/retention/filesystem_retention_snapshot_identity_tests.rs` at `5c32126`. ### Crash identifiers -`KEEP-CRASH-053` to `-073` named the 21 migration boundaries. The last +`KEEP-CRASH-053` to `-073` named the 21 migration boundaries. The evidence for every row is the migration crash child and restart model at `abbfe30` and `c2414bf`, and the transition ledger at `c2414bf`. @@ -66,7 +69,7 @@ evidence for every row is the migration crash child and restart model at | `KEEP-CRASH-065` | synchronize the root after the marker | Moved to `KEEP-CRASH-088`, now the commit point | | `KEEP-CRASH-066` | remove the marker stage | Moved to `KEEP-CRASH-089` | | `KEEP-CRASH-067` | synchronize the root after marker cleanup | Moved to `KEEP-CRASH-090` | -| `KEEP-CRASH-068` | write `migration.receipt.next` | Disappeared: the marker link is the commit point | +| `KEEP-CRASH-068` | write `migration.receipt.next` | Disappeared: the root synchronization after the marker link is the commit point | | `KEEP-CRASH-069` | synchronize the receipt stage | Disappeared | | `KEEP-CRASH-070` | link `migration.receipt` | Disappeared | | `KEEP-CRASH-071` | synchronize the root after the receipt | Disappeared | @@ -80,15 +83,19 @@ evidence for every row is the migration crash child and restart model at new identifier, not the migration boundary renamed. `KEEP-CRASH-084` (the root synchronization after the marker stage), `-081` (the writer lock), and `-091` (discarding an incomplete stage inside recovery) have no migration -predecessor. `KEEP-CRASH-078` to `-080` are reserved by version-2 catalog -recovery and are not assigned here. +predecessor. `KEEP-CRASH-078` to `-080` were reserved for catalog-stage +recovery and name no initialization boundary: `-078` is retired and `-079` +and `-080` stay reserved, as +[Version-1 crash identifiers](#version-1-crash-identifiers) records. ### Code and evidence removed - `src/adapters/store_migration/`: the migration records, inventory, authority, recovery planner and executor, and their unit laws. - `tests/store_migration_*`, `tests/migration_descriptor_exhaustion.rs`, and - the migration-completion and reader root-identity unit laws. + the migration-completion and reader root-identity unit laws. The + descriptor-exhaustion law has no successor: no remaining law exhausts file + descriptors at an admission boundary. - The `migration_format` fuzz target and its seeds, replaced by `format_marker` for the marker decoder that remains. - The migration crash child, restart model, and transition rows. @@ -105,10 +112,13 @@ refusals at the migration storage boundaries; those parts are moot. ## Version-1 admission (ADR-0010 step 5, #198) The version-1 initializer, `FilesystemPlatformAdmission`, the version-1 -`FilesystemCatalogPublisher`, the version-1 recovery openers, and -`classify_recovery_names` were deleted once the single format could do -everything they did. A root without the `FORMAT` marker now refuses at every -open path (`KEEP-INITIALIZATION-003`). +`FilesystemCatalogPublisher`, the version-1 recovery openers, and the +version-1 `classify_recovery_names` were deleted once the single format could +do everything they did. `FilesystemCatalogPublisher` and +`classify_recovery_names` still exist: they are the format-neutral successors +that took these names when the version-2 types were renamed. A root without +the `FORMAT` marker now refuses at every open path +(`KEEP-INITIALIZATION-003`). ### Rule for crash identifiers @@ -124,24 +134,24 @@ boundary no longer exists is retired here and never reused. -| ID | Obligation | Last evidence on `main` | Disposition | +| ID | Obligation | Evidence on `main` | Disposition | | --- | --- | --- | --- | | `KEEP-STORE-020` | Initialization is writer-locked, idempotent, recoverable from every partial canonical namespace set, and admitted only after root synchronization | `docs/formats/segment-store-v1/requirements.md` and rows `KEEP-CRASH-031`–`035` at `e03c2a1` | Moved to `KEEP-INITIALIZATION-001`, `-002`, and `-004` | | `KEEP-RECOVERY-002` | Initialization admits the platform, locks the writer file, and admits the three directories in order with exact failure phases | `tests/store_initialization.rs` at `c60def9` | Moved to `KEEP-INITIALIZATION-001` and `-002`; every initialization phase keeps its typed failure | | `KEEP-RECOVERY-003` | Production initialization and published-store reopen admit only the Linux ext4 profile for every protocol directory and the exact version-1 namespace | `src/adapters/filesystem_store_initializer_tests.rs` at `ecf4f7e`; `tests/store_initialization.rs` at `c60def9` | The profile obligation moved to single-format admission: `filesystem_platform_profile` admits every protocol directory, and `initialization_refuses_an_unsupported_platform_before_any_mutation` in `tests/single_format.rs` refuses tmpfs before mutation. The version-1 namespace disappeared | -| `KEEP-RETENTION-014` | Version-1 entry points refuse version-2 roots and version-2 entry points refuse version-1 roots | Never on `main`; introduced and retired within #198 (`tests/catalog_writer.rs` version laws at `e3930f0`) | The version-1 half disappeared. The version-2 half moved to `KEEP-INITIALIZATION-003`, evidenced by `every_open_path_refuses_a_root_without_the_format_marker` | +| `KEEP-RETENTION-014` | Version-1 entry points refuse version-2 roots and version-2 entry points refuse version-1 roots | Introduced and retired within the #198 change series, so `main` never held it as a live requirement (`tests/version_two_catalog_writer/version_laws.rs` at `e3930f0`) | The version-1 half disappeared. The version-2 half moved to `KEEP-INITIALIZATION-003`, evidenced by `every_open_path_refuses_a_root_without_the_format_marker` | -`KEEP-MIGRATION-001` and `-008` above deferred their version-2 refusal to -`KEEP-RETENTION-014` until this step; that obligation now lives in +During #198, `KEEP-MIGRATION-001` and `-008` passed their version-2 refusal +through `KEEP-RETENTION-014`; that obligation now lives in `KEEP-INITIALIZATION-003`. `KEEP-CATALOG-007` keeps its identifier: lock ownership alone still cannot construct a publisher, now because only single-format admission produces publisher authority. ### Version-1 crash identifiers -`KEEP-CRASH-031` to `-035` named the version-1 initializer. Their last +`KEEP-CRASH-031` to `-035` named the version-1 initializer. Their evidence is the initialization crash child at `1fc1ad0` and `f7956c0`, and the transition ledger at `e03c2a1`. @@ -165,15 +175,24 @@ publication that `KEEP-CRASH-009`, `-013`, `-024`, and `-025` cover. discard under version-2 recovery admission. It is retired: `KEEP-CRASH-027` to `-030` are that boundary and now run under single-format recovery admission. `KEEP-CRASH-079` and `-080` (stage completion and gated finalization under -recovery admission) stay reserved and planned in -[catalog publication and stage recovery](catalog-publication.md). +recovery admission) stay reserved and are not implemented. The plan placed +them before version 1 was deleted. The owner accepted on 2026-11-05 that they +follow that deletion, tracked by +[#221](https://github.com/flyingrobots/keep/issues/221), which blocks the +first release. Until they land, process death during recovery's completion +link and during gated `HEAD` replacement, and the gate rerunning on a retry +after such a death, are unproven. Their rows are in +[catalog publication and stage recovery](catalog-publication.md#process-death-matrix). +Garbage-collection crash rows take `KEEP-CRASH-092` and later. ### Version-1 code and evidence removed - `src/adapters/filesystem_platform_admission.rs`, - `filesystem_catalog_publisher.rs`, `filesystem_catalog_publisher_storage.rs`, `filesystem_store_initializer.rs`, `filesystem_initialization_storage.rs`, - `repository_initialization_storage.rs`, and `store_initialization*.rs`; the + and `repository_initialization_storage.rs`; the version-1 contents of + `filesystem_catalog_publisher.rs`, `filesystem_catalog_publisher_storage.rs`, + and `store_initialization*.rs`, whose names the format-neutral successors + now hold; the version-1 namespace laws; the version-1 recovery openers; and the `Platform` and `WriterLock` variants of the four recovery open errors, which no remaining constructor can produce. diff --git a/docs/formats/store/stage-recovery.md b/docs/formats/store/stage-recovery.md index bf4b88d4..48ed1133 100644 --- a/docs/formats/store/stage-recovery.md +++ b/docs/formats/store/stage-recovery.md @@ -381,8 +381,11 @@ The Linux production adapter is supported when it proves: - capability-relative no-follow access to regular files and directories; - an ext4 store root whose inode does not enable ext4 casefolding; -- `staging`, `segments`, and `catalogs` are independently writable, - non-casefolded ext4 directories on the root's device and mount; +- each of the nine protocol directories that exists (`staging`, `segments`, + `catalogs`, `retention`, `retention/roots`, `retention/manifests`, `gc`, + `recovery`, and `recovery/dispositions`) is an independently writable, + non-casefolded ext4 directory on the root's device and mount; an absent one + is left to namespace admission; - a writable mount and successful file and directory synchronization calls; - atomic same-filesystem no-clobber hard-link creation; - atomic same-filesystem replacement of one regular file by another; @@ -409,6 +412,21 @@ each existing protocol name byte-for-byte in a listing of its parent before reading, reusing, or mutating it. File and directory synchronization use `F_FULLFSYNC`. +Platform admission runs at the entry points that take store authority or the +reader fence: `FilesystemStoreAdmission::initialize` and `reopen`, +`FilesystemRecoveryAdmission::reopen`, +`FilesystemRecoveryInventoryReader::open`, and +`FilesystemRetentionSnapshot::load` with the durable readers built on it. The +refusals above, including a symlinked root path, hold on those routes only. +`FilesystemCatalogSnapshot::load`, its `load_for_verification`, +`FilesystemWriterLock::try_acquire`, and the repository-task entry points open +the root path ambiently and check no filesystem type, mount, device, or +casefold flag. Every entry below the opened root is still opened without +following links, and on macOS the byte-for-byte name requirement still +applies, because it belongs to the directory capability and not to platform +admission. Whether a casefolded ext4 directory can answer another spelling on +the unadmitted Linux routes has not been tested. + The protocol cannot compensate for hardware or an operating system that acknowledges synchronization without honoring it. Documentation and receipts state this assumption; they do not upgrade it into proof. diff --git a/docs/testing-evidence/filesystem-platform-adapters.md b/docs/testing-evidence/filesystem-platform-adapters.md index 3d60c3c1..3e0c965c 100644 --- a/docs/testing-evidence/filesystem-platform-adapters.md +++ b/docs/testing-evidence/filesystem-platform-adapters.md @@ -19,15 +19,21 @@ Darwin 27.0.0, arm64, pinned `rustc 1.96.0`, under `scripts/native/check.sh`. | Check | Default APFS | Case-sensitive APFS | | --- | --- | --- | -| `cargo test --all-features` (keep package, debug) | 963 passed, 0 failed | 963 passed, 0 failed | +| `cargo test --all-features` (keep package, debug) | 937 passed, 0 failed | 937 passed, 0 failed | | `cargo xtask durability-crash-matrix` | exit 0 | exit 0 | +Both rows are at `fb355d66`, and CI run 41953913643 reported the same 937 at that +head. Later commits add laws, so a later head has a different count; this table +does not state it. + The volumes are sparse images: default APFS `data` and case-sensitive APFS `data-cs`, 256 MiB each, selected with `KEEP_NATIVE_DATA_IMAGE`. The worker points `KEEP_TEST_SCRATCH_ROOT` at the selected image and sets `KEEP_TEST_EXPECT_APFS_VARIANT`, and the spelling laws fail if the volume is the other variant. Release-profile tests and the optimized crash matrix run in the -`Native macOS storage` CI job. `xtask` tests do not run on macOS. +`Native macOS storage` CI job. CI does not run `xtask` tests on macOS; +[local `xtask` tests on macOS](#local-xtask-tests-on-macos) records what a local +run shows. Two earlier sets of native runs are not counted. One used Homebrew's Rust 1.98 ahead of rustup on `PATH`; the worker now selects the pinned toolchain and asserts @@ -54,8 +60,14 @@ Calibration with the check disabled, on default APFS: The initialization, marker, admission, and namespace code from ADR-0010 uses the same exact-name directory as every other path. `darwin_filesystem_tests` pins the per-lookup check below namespace admission for `FORMAT`, `HEAD`, and a protocol -directory, and `tests/darwin_store_spelling.rs` covers the public reader, writer, -and initialization routes. +directory. Those three unit laws are the per-lookup witnesses for these paths. + +`tests/darwin_store_spelling.rs` respells `HEAD`, `FORMAT`, and `segments` and +requires the public reader and writer to refuse, and requires initialization to +refuse a root that holds `Format`. As the calibration above shows, namespace +admission refuses those entries first, so these laws witness the public refusal +and not the per-lookup check. No public-route law respells a pool entry, such as +a segment or catalog file name. ## Volume reattachment @@ -68,15 +80,28 @@ requires the store to reopen. Reboot itself has not been exercised. ## Laws that remain Linux-only, and why -- tmpfs refusal (`reader_platform_law_tests`, `a_writer_lock_cannot_mint_...`): - no tmpfs. Darwin refusal of non-APFS uses devfs and of read-only APFS uses the - sealed system volume, both at the adapter rather than the public boundary. +- tmpfs refusal (`reader_platform_law_tests`, + `initialization_refuses_an_unsupported_platform_before_any_mutation`): no tmpfs. + On Darwin, non-APFS refusal uses devfs, at the adapter + (`filesystems_other_than_apfs_are_refused`) and at the public boundary + (`initialization_refuses_devfs_at_platform_admission` calls + `FilesystemStoreAdmission::initialize` on `/dev`). Read-only APFS refusal uses + the sealed system volume, at the adapter only. - Raw non-UTF-8 names: APFS refuses to create them. - `filesystem_retention_fifo_tests`: Rustix omits `mknodat` on Apple targets and `std`'s `mkfifo` is unstable. `tests/darwin_fifo_refusal.rs` covers a FIFO at `HEAD` only. - O_PATH directory synchronization and ext4 profile laws: Linux mechanisms. +Two laws left this list: + +- `a_writer_lock_cannot_mint_...` was deleted with + `tests/catalog_platform_admission.rs`. Its successor is the `compile_fail` + doctest on `FilesystemCatalogPublisher::open`, which shows on every platform + that a bare writer lock cannot open a publisher. +- `filesystem_catalog_recovery_tests` was compiled only on Linux. The gate is + removed, and its three laws pass natively on default and case-sensitive APFS. + ## Weaker on macOS - The reader-fence law cannot observe a queued `flock`, because Darwin has no @@ -116,7 +141,33 @@ This is the per-open overhead only, not a store workload. - `F_FULLFSYNC` durability. A successful call is not proof that hardware honored it. -- A real non-APFS or network volume at the public boundary. +- A network volume, or a non-APFS volume other than devfs, at the public + boundary. +- A read-only APFS volume at the public boundary. + +## Local `xtask` tests on macOS + +CI runs `xtask` tests on Linux only, and Linux passing clears no macOS-only +cause. A local run under `scripts/native/check.sh`'s environment at `fb355d66` +failed nine `xtask` laws: + +- **Two, diagnosed by experiment and fixed on this branch.** + `terminal_interrupt_terminates_the_isolated_descendant_group` and + `cleanup_terminates_the_entire_child_process_group`, in + `bounded_process::process_group`. The runner's scratch directory is a long + path, and the laws' Unix socket path exceeded the 104-byte `sun_path` bound on + macOS, so the child exited before it was ready. With short names both pass + natively. `ProcessGroup` was not at fault. This matters because the crash + matrix terminates its children through the same `ProcessGroup`. +- **One, a missing tool.** `external_oracle_agrees_on_the_identity_preimage` + needs the `b3sum` binary, which is not installed on this machine. It fails + the same way on unchanged `main`. +- **Six, caused by the runner, read from their messages.** Five `cli_contract` + laws and `successful_verification_runs_every_documentation_tool_silently`. + The runner sets ambient `CARGO_*` build settings that the tool refuses, and + its sandbox denies writes outside its images. One law's message names the + refused setting, one reports `PermissionDenied`, and four report only a + failed command. None has been re-run without those two conditions. ## Runner bounds diff --git a/docs/testing-evidence/single-format-mutants.md b/docs/testing-evidence/single-format-mutants.md new file mode 100644 index 00000000..24c0de0a --- /dev/null +++ b/docs/testing-evidence/single-format-mutants.md @@ -0,0 +1,43 @@ +# Single-format mutation record + +Change kind: evidence calibration for #198, on PR #220. Owner: `@flyingrobots`. Six source mutants were run by hand to find out whether named laws fail when the behavior they describe is removed. No mutant was committed. This is not a mutation score: the mutants were chosen because a review predicted they would survive. + +## Why these mutants + +The review of PR #220 at `fb355d66` predicted that three laws passed for a reason other than the one they state. Running the prediction was cheaper than arguing it. The first round ran against `fb355d66`. The second round ran against the repaired tree, to check that the repairs fail on their stated cause. + +## First round, against `fb355d66` + +All three mutants were applied together and the complete root-package suite ran once, natively on default APFS with Rust 1.96.0: `cargo test --all-features --locked --offline --no-fail-fast`. The mutants touch separate subsystems, so one failing law identifies its mutant. Result: 936 passed, 1 failed. + +| Mutant | Outcome | What it showed | +| --- | --- | --- | +| `initialization_entries::synchronize_root` returns `Ok(())` without synchronizing | Survived | No law observed any root synchronization during initialization, including the commit point. The law then named for a failed commit synchronization failed its observer before the synchronization ran. | +| `replace_head` in `filesystem_recovery_next_head_finalization_storage.rs` renames without calling `verify_current` again | Killed by `changed_candidate_is_refused_at_the_replacement_boundary` | The gate's re-entry at replacement is held in place. The killing law changes the candidate. No law changes the retention head between `synchronize_candidate` and `replace_head`. | +| The lock replacement removed from `a_replaced_lock_or_protocol_directory_during_an_operation_still_refuses` | Survived | The second writer is refused as `Busy` by the root-inode lock, which the first writer holds whether or not `writer.lock` is replaced. The assertion cannot tell the two cases apart. | + +## Second round, against the repaired tree + +All three were applied together and `cargo test --all-features --locked --offline --no-fail-fast --lib --test single_format --test catalog_restart` ran once, natively on default APFS. Result: library 362 passed and 2 failed; `single_format` 15 passed and 5 failed; `catalog_restart` 10 passed. + +| Mutant | Outcome | Killing laws | +| --- | --- | --- | +| `Execution::synchronize` skips the root synchronizer and keeps both observer notifications | Killed | `the_commit_synchronization_runs_after_the_link_and_before_stage_removal` and `a_failed_commit_synchronization_leaves_an_unadmitted_resumable_store`, and no other law | +| `acquire_root_lock` takes a shared lock, so the root lock excludes nobody | Killed | `a_replaced_lock_or_protocol_directory_during_an_operation_still_refuses`, and no other law | +| `catalog_restart_loader::load` ignores its expectation | Killed | `a_store_with_no_catalog_refuses_an_expected_catalog`, `a_different_valid_store_refuses_the_expected_catalog`, `an_older_snapshot_restored_in_place_refuses_newer_expected_state`, and `an_expectation_rebuilt_from_recorded_values_admits_and_refuses_as_the_original` | + +The third mutant was also run alone against the crash matrix, because the matrix is silent on success: `cargo xtask durability-crash-matrix --sequence retained-catalog` stopped at its first case with "`KEEP-CRASH-074` before: post-crash verification failed while attempting to refuse the recorded generation-one catalog: generation two admitted a generation-one expectation". So the recorded-catalog check in that sequence runs and can fail. + +The second row answers the first round's third row. With the lock replacement in place, the portability law does fail when the mechanism that makes a replaced `writer.lock` harmless is removed. It is also the only law in the suite that failed for a non-exclusive root lock: the in-crate writer-lock laws passed. + +## What this does not establish + +- The first round's first mutant still survives. The commit laws substitute the root synchronizer, so they witness that the protocol requests the synchronization, in order, and what a failed request leaves. That `filesystem_catalog_artifact::synchronize_directory` reaches stable storage is established by reading it. Process death cannot expose a missing synchronization, because the page cache survives it, and no power-loss test exists. +- The crash matrix was run against one mutant only, as described above. +- The retained-closure check on the gate's second entry has no law of its own. It shares one function with the candidate check that the killing law exercises. +- Only default APFS was used. The mutated code is not platform-specific, but that is an argument, not a run. +- Six hand-picked mutants say nothing about the rest of the suite. + +## Replay + +Apply one mutant to a clean tree, run the command for its round under `scripts/native/check.sh`'s bounds, and restore the file. The laws live in `src/adapters/store_initialization/initialization_commit_tests.rs`, `tests/single_format/portability_laws.rs`, `tests/single_format/expectation_laws.rs` and `src/adapters/filesystem_recovery_next_head_finalization_tests/refusal_laws.rs`. diff --git a/xtask/tests/segment_store_protocol_contract/publication_laws.rs b/xtask/tests/segment_store_protocol_contract/publication_laws.rs index d7ea8447..dac390e0 100644 --- a/xtask/tests/segment_store_protocol_contract/publication_laws.rs +++ b/xtask/tests/segment_store_protocol_contract/publication_laws.rs @@ -3,10 +3,11 @@ use super::SPECIFICATION; #[test] -fn physical_namespace_refuses_aliasing_filesystems() { +fn physical_namespace_never_answers_for_another_spelling() { for required in [ - "case-sensitive, byte-preserving directory names", - "case-folding or normalization aliases", + "No protocol name may be answered by another spelling", + "refuses a casefolded ext4 root or protocol directory", + "requires that exact name, byte for byte, in a listing of its", ] { assert!( SPECIFICATION.contains(required), From a172b4cdf4e831ef8e1e0cb8dd0fc0a29bcab993 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 5 Nov 2026 01:50:18 -0800 Subject: [PATCH 33/36] Fix: compare the catalog snapshot's expectation at HEAD (#198) FilesystemCatalogSnapshot::load compared after loading the whole store, so a store that published another catalog and was also damaged, or over the caller's limits, reported that instead of the mismatch. The writer's reopen reported the mismatch for the same store. The load now compares what the checksummed HEAD names before it reads the catalog or any segment, and a law pins that order. Two assertions in the expectation laws were looser than their comments: one did not pin the head phase, and one flipped a bit in a digest that would have refused unflipped. --- src/adapters/catalog_restart_loader.rs | 38 ++++++++++++----- src/adapters/filesystem_catalog_snapshot.rs | 10 +++-- src/adapters/store_expectation.rs | 5 ++- tests/single_format/expectation_laws.rs | 45 ++++++++++++++++++--- 4 files changed, 77 insertions(+), 21 deletions(-) diff --git a/src/adapters/catalog_restart_loader.rs b/src/adapters/catalog_restart_loader.rs index b654de80..860c9fb8 100644 --- a/src/adapters/catalog_restart_loader.rs +++ b/src/adapters/catalog_restart_loader.rs @@ -21,7 +21,7 @@ const HEAD_LENGTH: u64 = 128; /// Admission is the reader law: the exact namespace, which may hold an /// in-flight `head.next`, and the canonical `FORMAT` marker. A root without the /// marker refuses before any catalog artifact is opened. `expectation` is -/// compared with the loaded snapshot, or with the absence of `HEAD`. +/// compared with what the checksummed `HEAD` names, or with its absence. pub(super) fn load( root: &Path, policy: CatalogRestartPolicy, @@ -35,18 +35,12 @@ pub(super) fn load( .map_err(|source| CatalogRestartError::Admission { source: Box::new(source), })?; - match load_from_directory(&directory, HEAD_NAME, policy) { - Ok(snapshot) => { - let observed = ObservedCatalog::new(snapshot.generation(), snapshot.catalog_digest()); - require(expectation, Some(observed))?; - Ok(snapshot) - } - Err(source) => { - if missing_head(&source) { - require(expectation, None)?; - } + match load_expected(&directory, HEAD_NAME, policy, expectation) { + Err(source) if missing_head(&source) => { + require(expectation, None)?; Err(source) } + loaded => loaded, } } @@ -125,10 +119,32 @@ pub(super) fn load_from_directory( directory: &crate::adapters::filesystem_directory::FilesystemDirectory, head_name: &str, policy: CatalogRestartPolicy, +) -> Result { + load_expected( + directory, + head_name, + policy, + StoreExpectation::NoExpectation, + ) +} + +/// Loads the snapshot `head_name` selects, comparing `expectation` with the head first. +/// +/// The comparison runs on the checksummed head, before the catalog or any +/// segment is read. A store that publishes another catalog therefore refuses as +/// a mismatch whatever its catalog and segments hold, and a damaged head +/// refuses at its own boundary before the comparison. +fn load_expected( + directory: &crate::adapters::filesystem_directory::FilesystemDirectory, + head_name: &str, + policy: CatalogRestartPolicy, + expectation: StoreExpectation, ) -> Result { let head_bytes = read_head_bytes(directory, head_name)?; let head = ChecksummedPublicationHead::decode(&head_bytes) .map_err(|source| CatalogRestartError::Head { source })?; + let observed = ObservedCatalog::new(head.generation(), head.catalog_digest()); + require(expectation, Some(observed))?; let catalogs = directory .open_dir_nofollow(CATALOGS_NAME) .map_err(|source| { diff --git a/src/adapters/filesystem_catalog_snapshot.rs b/src/adapters/filesystem_catalog_snapshot.rs index a83841c9..d56fda17 100644 --- a/src/adapters/filesystem_catalog_snapshot.rs +++ b/src/adapters/filesystem_catalog_snapshot.rs @@ -32,7 +32,8 @@ impl FilesystemCatalogSnapshot { /// reader fence, and it does not admit a platform profile. The returned /// owner retains bounded segment bytes so every later logical lookup remains /// pinned to this immutable generation. `expectation` is compared with the - /// loaded generation and digest after the snapshot verifies. + /// generation and digest the checksummed `HEAD` names, before the catalog + /// or any segment is read. /// /// # Errors /// @@ -42,8 +43,11 @@ impl FilesystemCatalogSnapshot { /// catalog than [`StoreExpectation::Catalog`] names, or none; and otherwise /// [`CatalogRestartError`] for filesystem refusal, malformed or /// noncanonical bytes, coordinate disagreement, resource-limit refusal, or - /// failed catalog-to-record admission. A damaged store refuses at its own - /// boundary before the expectation is compared. + /// failed catalog-to-record admission. A damaged `HEAD` refuses at its own + /// boundary before the expectation is compared. A mismatch refuses before + /// anything `HEAD` selects is opened, so it is reported whatever the + /// catalog and segments hold, and an absent `HEAD` is a mismatch with + /// nothing observed. pub fn load( root: &Path, policy: CatalogRestartPolicy, diff --git a/src/adapters/store_expectation.rs b/src/adapters/store_expectation.rs index e9e7911e..e9465514 100644 --- a/src/adapters/store_expectation.rs +++ b/src/adapters/store_expectation.rs @@ -92,7 +92,10 @@ impl ObservedCatalog { self.generation } - /// The published catalog digest. + /// The published catalog digest, as `HEAD` names it. + /// + /// The comparison reads `HEAD` only, so the catalog this names may not + /// have been opened or verified when a mismatch is reported. pub const fn digest(self) -> CatalogDigest { self.digest } diff --git a/tests/single_format/expectation_laws.rs b/tests/single_format/expectation_laws.rs index 7aa14569..beb40964 100644 --- a/tests/single_format/expectation_laws.rs +++ b/tests/single_format/expectation_laws.rs @@ -10,8 +10,8 @@ use std::fs; use std::path::Path; use keep::{ - CatalogGeneration, CatalogRestartError, DurableSnapshot, DurableStore, DurableStoreError, - ExpectedCatalog, FilesystemCatalogSnapshot, FilesystemRetentionSnapshot, + CatalogGeneration, CatalogRestartError, CatalogRestartPhase, DurableSnapshot, DurableStore, + DurableStoreError, ExpectedCatalog, FilesystemCatalogSnapshot, FilesystemRetentionSnapshot, FilesystemRetentionSnapshotError, FilesystemStoreAdmission, FilesystemStoreAdmissionError, ReaderAttemptLimit, StoreExpectation, StoreExpectationMismatch, }; @@ -179,7 +179,8 @@ fn a_store_with_no_catalog_refuses_an_expected_catalog() -> TestResult<()> { populated_store::policy()?, StoreExpectation::NoExpectation, ), - Err(CatalogRestartError::Io { source, .. }) if source.kind() == std::io::ErrorKind::NotFound + Err(CatalogRestartError::Io { phase: CatalogRestartPhase::OpenHead, source }) + if source.kind() == std::io::ErrorKind::NotFound )); drop(FilesystemStoreAdmission::reopen( empty.path(), @@ -245,8 +246,40 @@ fn an_expectation_rebuilt_from_recorded_values_admits_and_refuses_as_the_origina publish_second(store.path())?; let successor = published(store.path())?; every_open_refuses(store.path(), rebuilt, Some(successor))?; - let mut damaged = recorded.1; - *damaged.first_mut().ok_or("empty digest")? ^= 1; - let wrong = ExpectedCatalog::from_recorded(successor.generation(), damaged); + // The right generation with one digest bit wrong must refuse too. + let mut near = successor.digest_bytes(); + *near.first_mut().ok_or("empty digest")? ^= 1; + let wrong = ExpectedCatalog::from_recorded(successor.generation(), near); every_open_refuses(store.path(), wrong, Some(successor)) } + +/// The catalog snapshot compares `HEAD` before it opens what `HEAD` selects, so +/// a store that publishes another catalog refuses as a mismatch even when that +/// catalog is absent, exactly as the writer does. +#[test] +fn a_mismatch_refuses_the_catalog_snapshot_before_its_catalog_is_read() -> TestResult<()> { + let intended = durable_fixture::build("expected-precedence-intended", &[&first_blob()])?; + let other = durable_fixture::build("expected-precedence-other", &[&second_blob()])?; + let wanted = published(intended.path())?; + let found = published(other.path())?; + for entry in fs::read_dir(other.path().join("catalogs"))? { + fs::remove_file(entry?.path())?; + } + let policy = populated_store::policy()?; + + let mismatched = + FilesystemCatalogSnapshot::load(other.path(), policy, StoreExpectation::Catalog(wanted)); + let unchecked = + FilesystemCatalogSnapshot::load(other.path(), policy, StoreExpectation::NoExpectation); + + let Err(CatalogRestartError::Expectation { source }) = mismatched else { + return Err("a mismatch was not reported before the absent catalog".into()); + }; + require_mismatch(source, wanted, Some(found)); + assert!(matches!( + unchecked, + Err(CatalogRestartError::Io { phase: CatalogRestartPhase::OpenCatalog, source }) + if source.kind() == std::io::ErrorKind::NotFound + )); + Ok(()) +} From e1fb91c3931b509a8ba0049b0d075774db14df2f Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 5 Nov 2026 01:50:18 -0800 Subject: [PATCH 34/36] Test: count only malformed segments and real damage in torn copies (#198) The refusal list accepted every segment error, including policy and resource limits, which its own comment excluded. It now accepts only errors about the segment's bytes. A mix that behaves as the older whole state could satisfy both guards, because a blob that state never held refuses there too. The laws now also require a refusal at an absent or malformed artifact. --- tests/single_format/torn_copy_laws.rs | 37 ++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/single_format/torn_copy_laws.rs b/tests/single_format/torn_copy_laws.rs index 36d37c2b..19051c4f 100644 --- a/tests/single_format/torn_copy_laws.rs +++ b/tests/single_format/torn_copy_laws.rs @@ -17,7 +17,7 @@ use std::path::{Path, PathBuf}; use keep::{ CatalogRestartError, DurableOutcome, DurableReadError, DurableStore, DurableStoreError, FilesystemRetentionSnapshotError, ReaderAttemptLimit, RetentionClosureVerificationError, - RetentionViewError, StoreExpectation, + RetentionViewError, SegmentReadError, StoreExpectation, }; use super::durable_fixture; @@ -37,6 +37,9 @@ type Files = BTreeMap>; #[derive(Default)] struct Outcomes { refused: usize, + /// Refusals at an absent or malformed artifact. A blob the selected state + /// never held is refused by a whole state too, so it does not count here. + damaged: usize, read: usize, } @@ -105,6 +108,20 @@ fn evidenced(source: &io::Error) -> bool { ) } +/// Whether a segment refused for its own bytes and not for a policy or resource limit. +const fn malformed(source: &SegmentReadError) -> bool { + matches!( + source, + SegmentReadError::WrongLength { .. } + | SegmentReadError::Header { .. } + | SegmentReadError::Seal { .. } + | SegmentReadError::RecordHeaderTruncated { .. } + | SegmentReadError::RecordHeader { .. } + | SegmentReadError::RecordTruncated { .. } + | SegmentReadError::RecordDecode { .. } + ) +} + /// Whether `refusal` is evidence that the copy is incomplete or damaged. /// /// Anything else, such as a permission, resource, or locator failure, is an @@ -126,7 +143,8 @@ fn lawful(refusal: &DurableOutcome) -> bool { Snapshot::Catalog { source } => match source { CatalogRestartError::Io { source, .. } | CatalogRestartError::SegmentIo { source, .. } => evidenced(source), - CatalogRestartError::Length { .. } | CatalogRestartError::Segment { .. } => true, + CatalogRestartError::Length { .. } => true, + CatalogRestartError::Segment { source, .. } => malformed(source), _ => false, }, _ => false, @@ -160,6 +178,12 @@ fn exact_or_refused(root: &Path, content: &[u8], outcomes: &mut Outcomes) -> Tes "a refused read emitted bytes its identity does not name" ); outcomes.refused = outcomes.refused.saturating_add(1); + if !matches!( + refusal, + DurableOutcome::Read(DurableReadError::BlobMissing { .. }) + ) { + outcomes.damaged = outcomes.damaged.saturating_add(1); + } } } Ok(()) @@ -196,6 +220,10 @@ fn every_mix_of_two_quiescent_published_states_refuses_or_returns_exact_bytes() } assert!(torn_outcomes.read > 0, "no torn mix read anything"); assert!(torn_outcomes.refused > 0, "no torn mix refused anything"); + assert!( + torn_outcomes.damaged > 0, + "no torn mix refused at an absent or malformed artifact" + ); Ok(()) } @@ -225,6 +253,9 @@ fn a_half_length_copy_of_any_one_file_over_the_newer_state_refuses_or_returns_ex let copy = torn("torn-partial-copy", base.path(), &overlay)?; both_blobs(copy.path(), &mut outcomes)?; } - assert!(outcomes.refused > 0, "no partial copy refused anything"); + assert!( + outcomes.damaged > 0, + "no partial copy refused at a malformed artifact" + ); Ok(()) } From 33a99772a442e1591eb3352db823a5b3a0ad8727 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 5 Nov 2026 01:50:18 -0800 Subject: [PATCH 35/36] Test: admit the recorded catalog before recovery in the crash matrix (#198) The recorded generation-one catalog was only shown to admit on the republish path, after recovery had cleaned the stage, which is six of the twelve retained-catalog cases. Restart now requires it to admit before any recovery in every case that left HEAD at generation one, beside whatever stage the publication left. --- .../restart/recorded_catalog.rs | 13 +++++++++++++ .../restart/retained_catalog.rs | 3 +++ 2 files changed, 16 insertions(+) diff --git a/xtask/src/durability_crash_matrix/restart/recorded_catalog.rs b/xtask/src/durability_crash_matrix/restart/recorded_catalog.rs index 832b1551..9fda1a5c 100644 --- a/xtask/src/durability_crash_matrix/restart/recorded_catalog.rs +++ b/xtask/src/durability_crash_matrix/restart/recorded_catalog.rs @@ -54,3 +54,16 @@ pub(super) fn require_refused_at_generation_two( )), } } + +/// Requires the recorded generation-one catalog to admit while `HEAD` still names it. +/// +/// Called before any recovery, so the catalog snapshot opens beside whatever +/// stage the interrupted publication left. +pub(super) fn require_admitted_at_generation_one( + store_root: &Path, +) -> Result<(), DurabilityCrashMatrixError> { + let recorded = generation_one()?; + FilesystemCatalogSnapshot::load(store_root, policy()?, StoreExpectation::Catalog(recorded)) + .map(|_snapshot| ()) + .map_err(|source| verification("admit the recorded generation-one catalog", source)) +} diff --git a/xtask/src/durability_crash_matrix/restart/retained_catalog.rs b/xtask/src/durability_crash_matrix/restart/retained_catalog.rs index 0850f622..2dc94369 100644 --- a/xtask/src/durability_crash_matrix/restart/retained_catalog.rs +++ b/xtask/src/durability_crash_matrix/restart/retained_catalog.rs @@ -65,6 +65,9 @@ pub(super) fn verify( case.position() ))); } + if residue != Residue::Published { + recorded_catalog::require_admitted_at_generation_one(store_root)?; + } recover(store_root, residue)?; require_generation_two(store_root)?; recorded_catalog::require_refused_at_generation_two(store_root)?; From 0ab39da659815638faec3856621034e4d3012d64 Mon Sep 17 00:00:00 2001 From: James Ross Date: Thu, 5 Nov 2026 01:50:18 -0800 Subject: [PATCH 36/36] Docs: state which synchronizations and entry points the laws cover (#198) An independent read of the repair commits found wording that claimed slightly more than the code. Only the lock phase and the three root-synchronization phases go through the injected synchronizer; the fence, the directory admissions and the stage discard synchronize their parent directly. Five repository-task entry points take no expectation, not one. One of the six mutants was a test edit. The docs also follow the three preceding commits. --- README.md | 2 +- docs/adr/0010-one-storage-format.md | 4 ++-- docs/formats/store/catalog-publication.md | 7 +++---- docs/formats/store/initialization.md | 7 +++++-- docs/formats/store/publication.md | 5 +++-- docs/formats/store/recovery.md | 21 +++++++++++-------- .../testing-evidence/single-format-mutants.md | 2 +- .../initialization_commit_tests.rs | 2 +- .../initialization_execution.rs | 6 +++++- 9 files changed, 33 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 9736a0a8..b1235205 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Keep is required to refuse all three, before mutating anything. segment, catalog and head publication, recovery discard, and retention publication (the 47 live rows of `KEEP-CRASH-001`–`052`), 12 for catalog publication beside a retained root (`KEEP-CRASH-074`–`077`), and 41 for - initialization (`KEEP-CRASH-081`–`091`). Process death is not power loss, + initialization (`KEEP-CRASH-081`–`091`, with one case per directory). Process death is not power loss, which nothing here tests. - **One store format, created directly.** `FilesystemStoreAdmission::initialize` creates a version-2 store through a recoverable state machine with one diff --git a/docs/adr/0010-one-storage-format.md b/docs/adr/0010-one-storage-format.md index bfc7c67d..ad609456 100644 --- a/docs/adr/0010-one-storage-format.md +++ b/docs/adr/0010-one-storage-format.md @@ -171,8 +171,8 @@ check: marker-linked state admits a populated committed namespace ([initialization](../formats/store/initialization.md)), so that call can finish the cleanup of a store that already holds content; and -- the repository-task reopen, which is compiled only for tests and the - `repository-tasks` feature and also skips platform admission. +- the repository-task entry points, which are compiled only for tests and the + `repository-tasks` feature and also skip platform admission. Keep persists no store identifier. The only one, the content-derived identifier in the migration records, is deleted with migration, and diff --git a/docs/formats/store/catalog-publication.md b/docs/formats/store/catalog-publication.md index 804aa055..2291f346 100644 --- a/docs/formats/store/catalog-publication.md +++ b/docs/formats/store/catalog-publication.md @@ -203,10 +203,9 @@ recovery tools, retains the ingested blob, and reads both blobs through The implemented rows are 12 cases. CI runs them with the complete matrix in debug and release builds. They establish recovery after process death; they do not simulate power loss. Restart also holds the generation-one catalog a -caller would have recorded before the publication. In the cases that recover -by republication it must admit while `HEAD` names generation one, and in every -case it must refuse, reporting generation two, once `HEAD` names generation -two +caller would have recorded before the publication. Before any recovery it must +admit in every case that left `HEAD` at generation one, and in every case it +must refuse, reporting generation two, once `HEAD` names generation two ([Recovering a stale expectation](recovery.md#recovering-a-stale-expectation)). `KEEP-CRASH-079` and `-080` have no implementation. The plan placed them diff --git a/docs/formats/store/initialization.md b/docs/formats/store/initialization.md index 6cec2d8e..94389cc0 100644 --- a/docs/formats/store/initialization.md +++ b/docs/formats/store/initialization.md @@ -84,8 +84,8 @@ synchronization returned, so a root holding `FORMAT` without `FORMAT.next` can be observed only after the commit is durable. Initialization returns writer authority only after `SynchronizeRootAfterCleanup`. -What the laws witness is the order of requests, not durability. The four -root-synchronization phases call one synchronizer, which +What the laws witness is the order of requests, not durability. The lock +phase and the three root-synchronization phases call one synchronizer, which `initialization_commit_tests` substitutes: - `the_commit_synchronization_runs_after_the_link_and_before_stage_removal` @@ -98,6 +98,9 @@ root-synchronization phases call one synchronizer, which `InitializationIncomplete`, and a retry that resumes from the marker-linked state. +The reader fence, each directory admission, and the discard of an incomplete +stage synchronize their parent directly. No law observes those calls. + That the production synchronizer makes the root durable is established by reading it, not by a test. A synchronizer changed to do nothing passed the complete native test suite when tried against `fb355d66`. The process-death diff --git a/docs/formats/store/publication.md b/docs/formats/store/publication.md index 159f300f..b9c6af5f 100644 --- a/docs/formats/store/publication.md +++ b/docs/formats/store/publication.md @@ -161,8 +161,9 @@ authority borrowed while `current.seg` remains writable. `FilesystemCatalogSnapshot::load` is the unfenced reader boundary: it takes no writer lock, no reader fence, and no platform admission. It is a normal -open, so it requires a `StoreExpectation` and compares it with the snapshot -after that snapshot verifies; a mismatch is `CatalogRestartError::Expectation` +open, so it requires a `StoreExpectation` and compares it with what the +checksummed `HEAD` names, before it reads the catalog or any segment; a +mismatch is `CatalogRestartError::Expectation` ([Portable stores](recovery.md#portable-stores)). Its `CatalogRestartPolicy` combines segment parser limits with a positive maximum for aggregate retained segment bytes. The loader follows only exact diff --git a/docs/formats/store/recovery.md b/docs/formats/store/recovery.md index 0afe0c8a..bfc6a701 100644 --- a/docs/formats/store/recovery.md +++ b/docs/formats/store/recovery.md @@ -145,8 +145,9 @@ requires a `StoreExpectation`, with no default: - `FilesystemStoreAdmission::reopen`, which compares `HEAD` under the writer lock; -- `FilesystemCatalogSnapshot::load`, which compares the snapshot it loaded - after that snapshot verifies, with no lock and no fence; +- `FilesystemCatalogSnapshot::load`, which compares what the checksummed + `HEAD` names before it reads the catalog or any segment, with no lock and + no fence; - `FilesystemRetentionSnapshot::load` and `DurableSnapshot::open`, which compare the collected view under the reader fence; and - `DurableStore::open`, which only records the choice: it checks nothing until @@ -174,9 +175,10 @@ These routes take no expectation, so a caller of them does not get this check: - `FilesystemStoreAdmission::initialize`, including when it resumes a marker-linked root that already holds published state ([States and results](initialization.md#states-and-results)); and -- the repository-task entry point - `FilesystemStoreAdmission::reopen_unchecked_for_repository_tasks`, which - exists only under the `repository-tasks` feature. +- the repository-task entry points, which exist only under the + `repository-tasks` feature: the two unchecked initializers and + `reopen_unchecked_for_repository_tasks` on `FilesystemStoreAdmission`, and + the unchecked variants of the recovery admission and inventory reader. ### Keeping an expectation @@ -212,10 +214,11 @@ of the recorded catalog is not implemented. The crash matrix pins this for the gated writer. In every `retained-catalog` case (`KEEP-CRASH-074` to `-077`), restart builds the generation-one catalog a caller would have recorded before the interrupted publication -(`xtask/src/durability_crash_matrix/restart/recorded_catalog.rs`). In the -cases that recover by republication, `HEAD` still names generation one, and -the load under that expectation is admitted. In every case, once `HEAD` names -generation two, the same expectation must refuse and report generation two. +(`xtask/src/durability_crash_matrix/restart/recorded_catalog.rs`). In every +case where the crash left `HEAD` at generation one, the load under that +expectation must admit before any recovery runs, beside whatever stage the +publication left. In every case, once `HEAD` names generation two, the same +expectation must refuse and report generation two. ### Root identity across restart diff --git a/docs/testing-evidence/single-format-mutants.md b/docs/testing-evidence/single-format-mutants.md index 24c0de0a..95d35b28 100644 --- a/docs/testing-evidence/single-format-mutants.md +++ b/docs/testing-evidence/single-format-mutants.md @@ -1,6 +1,6 @@ # Single-format mutation record -Change kind: evidence calibration for #198, on PR #220. Owner: `@flyingrobots`. Six source mutants were run by hand to find out whether named laws fail when the behavior they describe is removed. No mutant was committed. This is not a mutation score: the mutants were chosen because a review predicted they would survive. +Change kind: evidence calibration for #198, on PR #220. Owner: `@flyingrobots`. Six mutants, five in source and one in a test body, were run by hand to find out whether named laws fail when the behavior they describe is removed. No mutant was committed. This is not a mutation score: the mutants were chosen because a review predicted they would survive. ## Why these mutants diff --git a/src/adapters/store_initialization/initialization_commit_tests.rs b/src/adapters/store_initialization/initialization_commit_tests.rs index 15bbadf6..ce0624f7 100644 --- a/src/adapters/store_initialization/initialization_commit_tests.rs +++ b/src/adapters/store_initialization/initialization_commit_tests.rs @@ -115,7 +115,7 @@ fn the_commit_synchronization_runs_after_the_link_and_before_stage_removal() -> COMMIT, root(true, false) ], - "each of the four root-synchronization phases requests exactly one" + "the lock phase and the three root-synchronization phases each request exactly one" ); Ok(()) } diff --git a/src/adapters/store_initialization/initialization_execution.rs b/src/adapters/store_initialization/initialization_execution.rs index 605e0e44..2326b19e 100644 --- a/src/adapters/store_initialization/initialization_execution.rs +++ b/src/adapters/store_initialization/initialization_execution.rs @@ -47,7 +47,11 @@ pub(in crate::adapters) fn initialize( initialize_with(root, observer, &mut entries::synchronize_root) } -/// Runs [`initialize`] with every root synchronization delegated to `synchronize_root`. +/// Runs [`initialize`] with its four requested root synchronizations delegated to +/// `synchronize_root`: the one in the lock phase and the three `SynchronizeRoot*` phases. +/// +/// The reader fence, each directory admission, and the discard of an incomplete +/// stage synchronize their parent directly and are not delegated. /// /// # Errors ///