From 0d052298018a0664adf2444563afbc1403a54781 Mon Sep 17 00:00:00 2001 From: Uday Chandra Date: Tue, 20 Oct 2026 22:45:53 +0000 Subject: [PATCH 1/4] A node never says in sync with a main that isn't the one it synced with Spec sail-main-rebuilt. M1: every database is minted a random catalog id (one migration, kept by upgrades and copies); main's welcome names it and a node records the first one at session open. M2/M3: NodeRound.admit refuses any other catalog, or none once one is recorded, before anything is pushed, adopted or recorded, with exit 1 and the restore sentence, so "Already in sync" follows only a round with the recorded main. M4: every join prints the pinned fingerprint and how to check it on main; rebuiltRecovery names a rebuild from a copy. M5: the re-join probe prints nothing of its own. M6: README and ARCHITECTURE.md say main's database is the org, how to back it up and restore it; invariant I5 names its tests. NativeFleetIT rebuilds main empty and from a copy over real ssh. --- ARCHITECTURE.md | 34 +- CHANGELOG.md | 1 + README.md | 24 ++ .../ai/singlr/sail/store/DatabaseCatalog.java | 46 +++ .../ai/singlr/sail/store/SchemaManager.java | 16 +- .../java/ai/singlr/sail/sync/NodeRound.java | 33 ++ .../ai/singlr/sail/sync/PagedSyncSession.java | 8 + .../ai/singlr/sail/sync/SyncRpcServer.java | 4 +- .../java/ai/singlr/sail/sync/SyncSession.java | 3 + .../java/ai/singlr/sail/sync/SyncWire.java | 29 +- .../sail/store/DatabaseCatalogTest.java | 60 ++++ .../singlr/sail/store/SchemaManagerTest.java | 20 ++ .../singlr/sail/sync/NodeRoundAdmitTest.java | 97 +++++ .../ai/singlr/sail/sync/SyncWireTest.java | 15 + .../sail/sync/UploadFailureRepliesTest.java | 24 +- .../ai/singlr/sail/commands/JoinCommand.java | 13 +- .../ai/singlr/sail/engine/MainHostKey.java | 19 +- .../ai/singlr/sail/engine/SyncOperations.java | 28 +- .../singlr/sail/commands/JoinCommandTest.java | 69 +++- .../sail/commands/MainRebuiltRoundTest.java | 333 ++++++++++++++++++ .../ai/singlr/sail/commands/NativeFleet.java | 50 ++- .../singlr/sail/commands/NativeFleetIT.java | 102 +++++- .../sail/commands/OperatorHintsTest.java | 15 +- .../ai/singlr/sail/commands/SyncCliTest.java | 1 + .../singlr/sail/engine/MainHostKeyTest.java | 36 +- .../sail/engine/SyncDiagnosticTest.java | 10 +- 26 files changed, 1015 insertions(+), 75 deletions(-) create mode 100644 sail-core/src/main/java/ai/singlr/sail/store/DatabaseCatalog.java create mode 100644 sail-core/src/test/java/ai/singlr/sail/store/DatabaseCatalogTest.java create mode 100644 sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java create mode 100644 sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3b2640c38..df0207fef 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -75,6 +75,19 @@ node` with `main: `, or unset for standalone. `sail host sync --as-m `sail host sync --main ` set it. Main never initiates. Nodes pull from and push to main. Main failover is manual and out of scope for v1. +Main's database is the org. Back up `/var/lib/sail/sail.db` (`sqlite3 /var/lib/sail/sail.db +".backup "`, or the file with sail-api stopped). Every database carries a catalog id +(`DatabaseCatalog`), random, minted with the database and kept by every upgrade and every copy of +the file, and main's welcome names it. A node records the catalog of the first main database it +syncs with and refuses any other before anything moves (`NodeRound.admit`): "Main at is +not the database this box synced with (catalog , now ). Restore main's database +(/var/lib/sail/sail.db) from a copy, then sync again." So a main set up again from a copy (`sudo +sail init --as-main`, the copy put in place of its new database, then `sudo sail migrate`, which +also restores the sail user's keys from it) is the same main to every node, which then pushes +whatever it wrote since the copy; a main set up empty is a new org, and its nodes refuse it. A +copy taken before the release that mints catalog ids is minted a new one when restored, so take a +fresh copy after upgrading. Main is never rebuilt from its nodes. + ## Module layout Three Maven modules, Java 25, one native binary. @@ -187,6 +200,7 @@ and the spec that closes it. | I1 | Every write on every box is made by exactly one bound `Actor`, whichever door it came through. | `ActorTest`, `OperationsTakeNoActorTest` | | I2 | On a node only the box's FDE, its runs and FDE-less credentials write; any other credential reads. | `RoleRuleTest`, `NodeWritesTest`, `DeniedSyncTest` | | I3 | Main and the node agree who the node is, and one box syncs as each FDE. | `NodeIdentitySyncTest`, `SyncServerCommandTest` (main's own FDE), `FdeCommandTest` (`release-box`) | +| I5 | A node syncs only with the main database it first synced with: a welcome naming another catalog, or none once one is recorded, ends the round before any row moves, and "Already in sync" is said only after a round with the recorded one. | `DatabaseCatalogTest` (minted per database, kept by upgrades and copies, the first main recorded), `SchemaManagerTest` (a database from before catalogs), `SyncWireTest`, `NodeRoundAdmitTest`, `MainRebuiltRoundTest` (an empty main under a new or the same box id refused with nothing moved either way; a main restored from a copy takes the node's newer work; a node from before catalogs records main's), `NativeFleetIT` (an empty main set up again at the node's address; one restored from a copy) | | I4 | Every run a box executes carries that box's handle as `node` and `owner`. | `BoxRunsSyncTest`, `RunAuthorityTest`, `RunStoreTest` (stamps; never another box's run), `SyncConfigTest` (one handle), `HandleChangeTest`, `JoinCommandTest`, `HostConfigSetCommandTest`, `HostSyncCommandTest`, `RoomWakeLaunchTest` | | A1 | Who may write a synced row is one rule per type in sail-core; the same write gets the same refusal kind and code at every door and on main. | the rules: `SpecAuthorityTest`, `RoomAuthorityTest`, `ReviewAuthorityTest`, `RunAuthorityTest` (a run is created only for the owner of the spec or room it names, on every lane), `MessageAuthorityTest`, `WriterAuthorityTest`, `ProjectAuthorityTest` (a rename is an admin's); the doors: `NodeWritesTest` (one refusal — kind, code, message and fix — through an API token, a passkey session, a gateway session, the box credential, a run and a room run; the terminal keeps its own text), `AuthorityAuditTest` (a viewer's edit over HTTP as on the host CLI; a room's delete and its membership; a spec born over its owner's room at the door as on main; a merged resolve), `SailOperationsSeamTest` (a resolve: adopting main's side decides nothing, keeping or merging is an edit, on the socket and both web lanes), `CatalogAuthorityAuditTest` (the host CLI's catalog edit, project files and rename), `AdhocDispatchTest` (ad-hoc runs), `OneDecisionTest` (spec edits, posts); main: `PushAuthoritySyncTest` (a pushed run on another FDE's spec is denied, and so is one naming its own spec and another member's room; one ahead of a spec its box made lands), `DeniedSyncTest` | | A2 | No code path writes a synced row without its rule deciding it: a pushed revision at main's commit, every other write at the journal, each exactly once. | `JournalDecidesEveryWriteTest` (every public store method that reaches the journal, read from the stores' bytecode, is refused on the CLI, API, agent and room lanes and leaves nothing; a revision never admits itself; a refusal swallowed inside a transaction still rolls it back; a pushed revision is not decided twice), `OneDecisionPointTest` (in every module's classes only the journal and the message store append a revision, only they ask a write rule, and only main's commit takes a store's rule), `RevisionJournalTest`, `SqliteTest` (a doomed transaction); an operation's rows commit together or not at all: `GlobalSpecOperationsTest` (an update with its wake; the move to done; a delete), `ReviewOperationsTest` (a follow-up with its links), `BuildDispatchCoverageTest` (a dispatch and a restart: reservation, reset, claim, supersede), `SpecPruneTest` (a restore with its room) | @@ -492,7 +506,9 @@ A session opens with `hello` (protocol, build, fleet floor, box id) and is `welc `refuse`d once; floors compare as versions. The `welcome` names the handle main authenticated the session as and main's `limits.file_max`, so the node enforces the lower of its own limit and main's before a file ever reaches a sync; a node whose configured sync handle is blank or another -does nothing that round (`NodeRound.begin`); an older main names neither and is not asked. Main records the first box that +does nothing that round (`NodeRound.begin`); an older main names neither and is not asked. It +names main's database catalog too, which the node checks before anything else +(`NodeRound.admit`, see Topology). Main records the first box that syncs as each FDE (`fde_boxes`, never synced; main's own FDE's box is main) and refuses a session from any other until an admin runs `sail fde release-box`; removing an FDE releases its box. A node's answer to an offer main took can be lost on the way back. Main records the box each @@ -741,8 +757,10 @@ join` generated, so a missing key fails fast. Host keys are checked in batch mod account's `known_hosts`, where join pinned main's ed25519 key (see Onboarding); a round that fails on it quotes ssh's own last stderr line ("ssh said: Host key verification failed.") and names the whole recovery in `MainHostKey.rebuiltRecovery`'s words, the same sentence join's -refusal of a changed key carries: remove the old pin with `sudo ssh-keygen -R `, then -re-run `sudo sail init --main `. On main, that key sits on the locked `sail` +refusal of a changed key carries: if main was rebuilt from a copy of its database, remove the +old pin with `sudo ssh-keygen -R `, re-run `sudo sail init --main ` and compare +the fingerprint it prints with main's (`sudo ssh-keygen -lf +/etc/ssh/ssh_host_ed25519_key.pub` on main). On main, that key sits on the locked `sail` user's `authorized_keys` as a forced command, `command="…/sail _gateway --fde ",restrict …`. The gateway resolves the FDE and admits the `_sync` session. The write gate lives next to the write: every commit of a `_sync` @@ -866,12 +884,14 @@ handle is refused before any step): `host sync --as-main --handle` writes the ro sync handle in one identity change and adds that FDE as admin when the roster lacks it. On a node it means `join`, which first pins main's ed25519 host key into the invoking account's `known_hosts` (`MainHostKey`: `ssh-keyscan -H` on port 22, appended when absent, kept when equal, refused when it differs or -cannot be read, all before the host config is written), then generates the box's sync key and -prints the `sudo sail fde add … --key … --main-host-key SHA256:…` line for the operator to run -on main. A re-join with the sync key the box already had opens one sync session to main as the +cannot be read, all before the host config is written), prints "Pinned main's host key +SHA256:…. Check it on main: sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub", then +generates the box's sync key and prints the `sudo sail fde add … --key … --main-host-key +SHA256:…` line for the operator to run on main. A re-join with the sync key the box already had opens one sync session to main as the joined handle (`SyncOperations.welcomedAs`): a welcome as that handle means main holds the key, and the join says so instead of printing the line (`--json` still carries it); anything else, -main unreachable included, is no evidence, and the line is printed. +main unreachable included, is no evidence, and the line is printed. The probe prints nothing of +its own. Each step runs through `Main.report`, so a refusal inside init is printed once with no trace, then "Setup stopped" once. Main's `fde add` compares that fingerprint to its own host key before its transaction and refuses a mismatch, so a join that something else answered is never diff --git a/CHANGELOG.md b/CHANGELOG.md index e3afc43b2..fc2bc5bb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 0.54.0 +- **A node never says "in sync" with a main that isn't the one it synced with.** Field rehearsal round 4 deleted main's VM and set it up again empty at the same IP; the node re-pinned, main ran the `fde add` line, and `sudo sail sync` printed "✓ Already in sync with main." while the new main held only `demo` and the node `gamma`, so a spec the node then filed under `gamma` was pushed into a main that holds no such project. Every database now carries a catalog id (`DatabaseCatalog`), random, minted with the database (an existing one at its upgrade) and kept by every upgrade and every copy of the file, and main's welcome names it. A node records the catalog of the first main database that welcomes it (a node from before this release on its next round) and refuses any other before anything is pushed, adopted or recorded, with exit 1: "Main at is not the database this box synced with (catalog , now ). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again." It is decided on the catalog, not the box id, so a new database under the old `host.yaml` is refused too instead of silently skipped below the node's old checkpoint. Main set up again from a copy of its database is the same main: its nodes push what they wrote since the copy, and "Already in sync with main." follows. The README and ARCHITECTURE.md say to back up `/var/lib/sail/sail.db` and how to restore it; a copy taken before this release is minted a new catalog when restored, so take one after upgrading. Every join now prints the key it pinned, "Pinned main's host key SHA256:…. Check it on main: sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub", including a re-join after a host-key rotation that then says "Main already has this box's key"; the changed-key recovery reads "If main was rebuilt from a copy of its database, remove the old key with '…', re-run '…' and compare the fingerprint it prints with main's (…); otherwise something between this box and main is answering for it."; and the re-join's probe of main prints nothing of its own (the stray "⚠ hello: Sync channel closed before main replied." line is gone). - **Rows written together order as minted on a clock of any resolution.** Cutting v0.53.0, the macOS release job failed `ReviewLoopEveryStateTest` once, as v0.48.0 did. An instant is stored as its text, which drops trailing zeros, so `…51.123Z` sorts after the later `…51.123400Z` and `…51Z` after `…51.001Z`; a microsecond clock lands on such a value one time in a thousand. The stores ordered runs and reviews by that text, so the review loop could read a fix run as newer than the reviewer launched after it and ignore that reviewer's stop. Runs and reviews now order by their UUIDv7 id, as each box minted them; stored instants are unchanged and nothing migrates. The review loop fixture's fixed-width clock is gone. - **A box that stops being a node keeps nothing of the node it was.** The delta review of `sail-sync-is-exact` found two ways a box still carried node state after it stopped being one. Re-running `sudo sail host init` rewrote `host.yaml` from scratch, dropping the sync role, main, handle, box id and passkey settings, so a node or main turned standalone and orphaned every checkpoint its peers held for it; a re-run now refreshes only the fields it provisions and keeps the rest of `host.yaml`, read again under the box's round lock just before it writes, so a `host sync`, `join` or handle change that finished while the packages installed is kept, not undone. And a main promoted on v0.49.0 to v0.52.0, or one whose role change died between writing `host.yaml` and clearing its scope, kept the scope it last heard as a node, refusing its own `spec create` and its nodes' pushes into every other project and skipping its own descriptors at upgrade; `sail migrate` and every `sail server start` now clear that scope on any box that is not a node, and only a whole `host.yaml` decides it: one absent, empty or unreadable at that moment (truncated by a writer, or being moved into `/var/lib/sail`) keeps the scope for the next start to decide, so a node never loses its grant to a file in motion. `sudo sail host update` and `sail migrate`'s box-id and `host.yaml` relocation steps now read and write `host.yaml` under that lock too, and `host update` records the Incus version on the file as it is then, not as it was before the packages upgraded, so an identity change made meanwhile is kept. - **A scope can change after an FDE exists, and every box follows it whole.** Field rehearsal 2026-10-15 found no way to scope an FDE that joined unscoped short of `fde rm` and a re-add (which revoked its tokens, keys, sessions and passkeys), a scoped node still holding the names and emails of FDEs outside its projects, and a revoked project's container running on the node with its clone and credentials, still listed by `sail project list`. **`sudo sail fde update --scoped [--grant ]...`** narrows an FDE in place, in one transaction with the grants it names (`--grant` is repeatable and takes `a,b`), and **`--unscoped`** lets it see every project again and forgets its grants, so a later narrowing grants only what it names; the FDE keeps its id and every credential. Both take `--dry-run` (the real run, rolled back) and `--json` (`handle`, `scope`, `projects`, `changed`, `dry_run`), are refused on a node with the roster sentence, and refuse this box's own FDE and an admin as `fde add --scoped` does. A narrowing says what the FDE's box drops ("Scoped eng to acme. Its box drops beta, demo with their containers and unpushed work on its next sync."), a widening that its box holds every project after its next sync. `fde grant` on an unscoped FDE now names `sudo sail fde update --scoped`. A narrowing is a revocation on the node and a widening a grant, so the node drops and re-adopts as before. A scoped node now holds exactly the roster main lists to it: an FDE main stops listing is removed with its box-local credentials ("roster: removed 1 FDE(s) main no longer lists to this box"), the box's own FDE never; an unscoped node still disables such an FDE and keeps its row. Every round of a scoped node then takes off the box each project the scope does not admit and the box holds nothing of: its `~/.sail/projects/` directory, and its container when Sail provisioned it (the `sail-api-sock` device, the trust gate `project apply --all` uses), as `sudo sail project destroy` without `--purge` would ("project beta: its container deleted, workspace and uncommitted work included — main no longer grants this box the project"). A container Sail did not provision is left alone and named once; when the round cannot reach Incus (a non-root `sail sync`) nothing is deleted and the round that dropped the project says the next round that can (sail-api within a minute, or `sudo sail sync`) will; a project kept for a run under way keeps both until the round after the run ends. It is decided from what the box holds, so a box an earlier release left with such a directory, container or roster row is brought in line by its first round on this release. An unchanged scoped round asks `incus list` once, and an unscoped one asks Incus nothing. `sail migrate` no longer imports the descriptor or files of a project the box's scope does not admit, so an upgrade no longer imports a dropped project only for the next round to drop it again. An open event stream (`/v1/events/stream`) or run-log stream (`/v1/runs/{id}/stream`) now asks its caller's authentication again, at most once a second, before an event or a heartbeat: a grant, a revoke or a demotion holds from that event on, and a caller removed, disabled or with its token or session revoked gets a last `event: error` naming why, then the stream ends. **Fleet note:** upgrade a scoped node before narrowing or revoking: a node of an earlier release drops the rows but keeps the roster rows and the container. diff --git a/README.md b/README.md index d971b322d..a86831ad6 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,30 @@ keeps everything, **delete** keeps history so `sail spec restore` brings it back erases a spec everywhere, leaving one audit row. Retention is opt-in through a `retention` block in main's `host.yaml`. +Main's database is the org. Back it up on main, and keep the copy off the box: + +```bash +sudo apt-get install -y sqlite3 +sudo sqlite3 /var/lib/sail/sail.db ".backup /root/sail.db.copy" +``` + +A main set up again from that copy is the same main to every node. On the new main, run `sudo +sail init --as-main`, then put the copy in place: + +```bash +sudo systemctl stop sail-api +sudo install -m 660 -g sail /root/sail.db.copy /var/lib/sail/sail.db +sudo rm -f /var/lib/sail/sail.db-wal /var/lib/sail/sail.db-shm +sudo sail migrate +sudo systemctl start sail-api +``` + +Each node then re-pins main's new host key (`sudo ssh-keygen -R `, then `sudo sail init +--main `; compare the fingerprint it prints with `sudo ssh-keygen -lf +/etc/ssh/ssh_host_ed25519_key.pub` on main), and its next `sudo sail sync` pushes whatever it wrote +since the copy. A main set up empty is a new org: every node refuses to sync with it. A copy taken +before 0.54.0 counts as a new org too, so take a fresh one after upgrading. + Mast, the desktop client, is a thin client of a box: specs, rooms and reviews over its API and event stream, terminals and files over its pty host. diff --git a/sail-core/src/main/java/ai/singlr/sail/store/DatabaseCatalog.java b/sail-core/src/main/java/ai/singlr/sail/store/DatabaseCatalog.java new file mode 100644 index 000000000..4b25533cc --- /dev/null +++ b/sail-core/src/main/java/ai/singlr/sail/store/DatabaseCatalog.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 Standard Applied Intelligence Labs + * SPDX-License-Identifier: MIT + */ + +package ai.singlr.sail.store; + +import java.util.Objects; +import java.util.Optional; + +/** + * Which database this is, and on a node which of main's databases it syncs with. The id is random, + * minted when the database is created, kept by every upgrade and by every copy of the file: main's + * database is the org, so main set up again from a copy of it is the same main to every node, and + * main set up empty is another org. A node records main's catalog the first time a main that names + * one welcomes it, and never replaces it. + */ +public final class DatabaseCatalog { + + private final Sqlite db; + + public DatabaseCatalog(Sqlite db) { + this.db = Objects.requireNonNull(db, "db"); + } + + /** This database's own catalog id. */ + public String id() { + return db.queryOne("SELECT id FROM catalog", row -> row.text(0)).orElseThrow(); + } + + /** The catalog of the main database this node syncs with; empty until a round records one. */ + public Optional main() { + return db.queryOne("SELECT COALESCE(main, '') FROM catalog", row -> row.text(0)) + .filter(main -> !main.isEmpty()); + } + + /** + * Records {@code offered} as main's catalog unless one is recorded already, and returns the one + * recorded: {@code offered} itself only when it is the database this node synced with. + */ + public String recordMain(String offered) { + Objects.requireNonNull(offered, "offered"); + db.execute("UPDATE catalog SET main = ? WHERE main IS NULL", offered); + return main().orElseThrow(); + } +} diff --git a/sail-core/src/main/java/ai/singlr/sail/store/SchemaManager.java b/sail-core/src/main/java/ai/singlr/sail/store/SchemaManager.java index 5463acdd4..6436f2b95 100644 --- a/sail-core/src/main/java/ai/singlr/sail/store/SchemaManager.java +++ b/sail-core/src/main/java/ai/singlr/sail/store/SchemaManager.java @@ -91,8 +91,9 @@ INSERT INTO runs_v4 (id, project, spec_id, agent, branch, task, pid, status, * record of the one box each FDE syncs from), {@code materialized_files} (the versions of each * shared file this box wrote to disk or published), {@code fde_pairings} (the key and token this * box made for each FDE's Mast), {@code project_grants} (the projects main granted each scoped - * FDE) and {@code sync_scope} (the scope main last named for this node) are local-only as well — - * none of the nine ever joins a sync snapshot. + * FDE), {@code sync_scope} (the scope main last named for this node) and {@code catalog} (which + * database this is, and which of main's a node syncs with) are local-only as well — none of the + * ten ever joins a sync snapshot. */ /** * One-time sweep of the review-loop convergence gap: findings a spec shipped below the gate were @@ -251,6 +252,14 @@ fde_id TEXT NOT NULL REFERENCES fdes(id) ON DELETE CASCADE, PRIMARY KEY (fde_id, project) )"""; + /** + * Which database this is: a random id minted with the database and kept by every upgrade and + * every copy of its file, and, on a node, the catalog of the main database it syncs with, + * recorded by its first round ({@link DatabaseCatalog}). + */ + static final String CATALOG = + "CREATE TABLE catalog AS SELECT lower(hex(randomblob(16))) AS id, NULL AS main"; + /** * The project each change head belongs to, null when unknown, so a page is scoped from the head * alone: a pruned row has no row left to join, yet its erasure must still reach the nodes that @@ -797,7 +806,8 @@ fde_id TEXT PRIMARY KEY REFERENCES fdes(id) ON DELETE CASCADE, "DROP TRIGGER specs_status_since_insert", STATUS_SINCE_REVISION_ON_INSERT, "DROP TRIGGER specs_status_since_update", - STATUS_SINCE_REVISION_ON_UPDATE); + STATUS_SINCE_REVISION_ON_UPDATE, + CATALOG); /** The schema version this binary converges every database to. */ static final int CURRENT_VERSION = V1_VERSION + MIGRATIONS.size(); diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java b/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java index 56a254a0b..eed67786d 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java @@ -10,6 +10,7 @@ import ai.singlr.sail.identity.Actor; import ai.singlr.sail.identity.ProjectScope; import ai.singlr.sail.store.ChangeLog; +import ai.singlr.sail.store.DatabaseCatalog; import ai.singlr.sail.store.Decidability; import ai.singlr.sail.store.Erasure; import ai.singlr.sail.store.FileStore; @@ -34,8 +35,40 @@ */ public final class NodeRound { + /** Where main keeps its database: the org, which every node's catalog names. */ + private static final String MAIN_DATABASE = "/var/lib/sail/sail.db"; + private NodeRound() {} + /** + * Refuses {@code session}, before anything is offered, adopted or recorded, unless main at {@code + * target} serves the database this node, whose database is {@code db}, synced with: main set up + * empty at the same address welcomes the node as readily as the main it replaced. The first + * welcome that names a catalog is recorded as main's ({@link DatabaseCatalog#recordMain}). A main + * that names none predates catalogs and is not asked, unless this node recorded one: a database + * that has named its catalog always does. + */ + public static void admit(SyncSession session, Sqlite db, String target) { + var catalog = new DatabaseCatalog(db); + var offered = session.catalog(); + var synced = offered.map(catalog::recordMain).or(catalog::main); + if (synced.isEmpty() || synced.equals(offered)) { + return; + } + throw new SyncTransportException( + "refused", + "Main at " + + target + + " is not the database this box synced with (catalog " + + synced.get() + + ", now " + + offered.orElse("none") + + "). Restore main's database (" + + MAIN_DATABASE + + ") from a copy, then sync again.", + null); + } + /** * Begins a round over {@code session} for the node whose database is {@code db} and whose * configured sync handle is {@code handle}. It fails, before anything is offered or adopted, diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/PagedSyncSession.java b/sail-core/src/main/java/ai/singlr/sail/sync/PagedSyncSession.java index 5dcba9841..819b19ce6 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/PagedSyncSession.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/PagedSyncSession.java @@ -93,6 +93,7 @@ public final class PagedSyncSession implements SyncSession { private Consumer onDenial = ignored -> {}; private String broken; private String handle; + private String catalog; private long mainFileMax; private ProjectScope scope = ProjectScope.all(); private Set kept = Set.of(); @@ -130,6 +131,7 @@ PagedSyncSession frame(int frame) { copy.notice = notice; copy.onDenial = onDenial; copy.handle = handle; + copy.catalog = catalog; copy.mainFileMax = mainFileMax; copy.scope = scope; copy.kept = kept; @@ -174,6 +176,7 @@ static PagedSyncSession open( session.notice = notice; session.onDenial = onDenial; session.handle = welcome.handle(); + session.catalog = welcome.catalog(); session.mainFileMax = welcome.fileMax(); session.scope = welcome.scope(); return session; @@ -799,6 +802,11 @@ public long mainFileMax() { return mainFileMax; } + @Override + public Optional catalog() { + return Optional.ofNullable(catalog); + } + @Override public ProjectScope scope() { return scope; diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/SyncRpcServer.java b/sail-core/src/main/java/ai/singlr/sail/sync/SyncRpcServer.java index 2702a29c3..7374f0e26 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/SyncRpcServer.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/SyncRpcServer.java @@ -15,6 +15,7 @@ import ai.singlr.sail.identity.Role; import ai.singlr.sail.store.BlobStore; import ai.singlr.sail.store.ChangeLog; +import ai.singlr.sail.store.DatabaseCatalog; import ai.singlr.sail.store.Erasure; import ai.singlr.sail.store.FdeBoxes; import ai.singlr.sail.store.Sqlite; @@ -545,7 +546,8 @@ private SyncWire.Response onHello(SyncWire.Hello hello) { mainId(), Objects.toString(principal.handle(), ""), limits.fileMax(), - principal.scope()); + principal.scope(), + db == null ? null : new DatabaseCatalog(db).id()); } /** diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/SyncSession.java b/sail-core/src/main/java/ai/singlr/sail/sync/SyncSession.java index 7460a0140..7b06ee280 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/SyncSession.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/SyncSession.java @@ -202,6 +202,9 @@ public static TypeReport failed(String type, String failure) { */ void keep(Set projects); + /** The catalog of main's database as its welcome named it; empty from a main that never says. */ + Optional catalog(); + /** The box id main welcomed this session as: the peer this node's checkpoints are kept for. */ String mainId(); diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/SyncWire.java b/sail-core/src/main/java/ai/singlr/sail/sync/SyncWire.java index 94b1a69ee..d9a3ac91d 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/SyncWire.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/SyncWire.java @@ -99,6 +99,7 @@ public final class SyncWire { private static final String HANDLE = "handle"; private static final String FILE_MAX = "file_max"; private static final String SCOPE = "scope"; + private static final String CATALOG = "catalog"; private static final String TIPS = "tips"; private static final String FDES = "fdes"; private static final String MESSAGE = "message"; @@ -361,15 +362,33 @@ private static Map contentMap(Content content) { * Main accepted the hello: its protocol, build, the box id the node checkpoints against, and the * {@code handle} it authenticated the session as — blank for a session that names no FDE, and * null from an older main, which never says — and the {@code scope} of projects that FDE may see: - * every project unless main scoped it, which is also what an older main, naming none, means. + * every project unless main scoped it, which is also what an older main, naming none, means. The + * {@code catalog} names main's database ({@code DatabaseCatalog}), null from a main that never + * says. */ public record Welcome( - int protocol, String version, String mainId, String handle, long fileMax, ProjectScope scope) + int protocol, + String version, + String mainId, + String handle, + long fileMax, + ProjectScope scope, + String catalog) implements Response { public Welcome { Objects.requireNonNull(scope, "scope"); } + public Welcome( + int protocol, + String version, + String mainId, + String handle, + long fileMax, + ProjectScope scope) { + this(protocol, version, mainId, handle, fileMax, scope, null); + } + public Welcome(int protocol, String version, String mainId, String handle, long fileMax) { this(protocol, version, mainId, handle, fileMax, ProjectScope.all()); } @@ -701,6 +720,9 @@ public static String encode(Response response) { if (!welcome.scope().unrestricted()) { map.put(SCOPE, List.copyOf(welcome.scope().projects())); } + if (welcome.catalog() != null) { + map.put(CATALOG, welcome.catalog()); + } } case Refuse refuse -> { map.put(OP, OP_REFUSE); @@ -790,7 +812,8 @@ public static Response decodeResponse(String line) { string(map, MAIN_ID), string(map, HANDLE), longValue(map, FILE_MAX), - map.containsKey(SCOPE) ? ProjectScope.of(strings(map, SCOPE)) : ProjectScope.all()); + map.containsKey(SCOPE) ? ProjectScope.of(strings(map, SCOPE)) : ProjectScope.all(), + string(map, CATALOG)); case OP_REFUSE -> new Refuse(string(map, REASON)); case OP_TIPS -> new Tips(tips(map)); case OP_PAGE -> diff --git a/sail-core/src/test/java/ai/singlr/sail/store/DatabaseCatalogTest.java b/sail-core/src/test/java/ai/singlr/sail/store/DatabaseCatalogTest.java new file mode 100644 index 000000000..44c85b546 --- /dev/null +++ b/sail-core/src/test/java/ai/singlr/sail/store/DatabaseCatalogTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2026 Standard Applied Intelligence Labs + * SPDX-License-Identifier: MIT + */ + +package ai.singlr.sail.store; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** Which database this is: minted with it, kept by upgrades and copies, never by another one. */ +class DatabaseCatalogTest { + + @TempDir Path dir; + + private static Sqlite migrated(Path file) { + var db = Sqlite.open(file); + new SchemaManager(db).migrate(); + return db; + } + + @Test + void everyDatabaseIsMintedItsOwnRandomCatalogIdThatUpgradesAndCopiesKeep() { + try (var db = migrated(dir.resolve("a.db")); + var other = migrated(dir.resolve("b.db"))) { + var id = new DatabaseCatalog(db).id(); + assertTrue(id.matches("[0-9a-f]{32}"), id); + assertNotEquals(id, new DatabaseCatalog(other).id()); + + new SchemaManager(db).migrate(); + db.execute("VACUUM INTO ?", dir.resolve("copy.db").toString()); + + assertEquals(id, new DatabaseCatalog(db).id()); + try (var copy = migrated(dir.resolve("copy.db"))) { + assertEquals(id, new DatabaseCatalog(copy).id()); + } + } + } + + @Test + void aNodeRecordsTheFirstMainCatalogItIsOfferedAndKeepsItWhateverItIsOfferedNext() { + try (var db = migrated(dir.resolve("node.db"))) { + var catalog = new DatabaseCatalog(db); + assertEquals(Optional.empty(), catalog.main()); + + assertEquals("aaaa", catalog.recordMain("aaaa")); + assertEquals("aaaa", catalog.recordMain("bbbb")); + + assertEquals(Optional.of("aaaa"), catalog.main()); + assertEquals(List.of(1L), db.query("SELECT count(*) FROM catalog", row -> row.integer(0))); + } + } +} diff --git a/sail-core/src/test/java/ai/singlr/sail/store/SchemaManagerTest.java b/sail-core/src/test/java/ai/singlr/sail/store/SchemaManagerTest.java index 16757f825..799c861dc 100644 --- a/sail-core/src/test/java/ai/singlr/sail/store/SchemaManagerTest.java +++ b/sail-core/src/test/java/ai/singlr/sail/store/SchemaManagerTest.java @@ -1384,6 +1384,26 @@ void fromTheUnreleased0_46_4ShapeABoxGetsATableForWhatItWroteOfEachSharedFile() assertEquals(SchemaManager.CURRENT_VERSION, new SchemaManager(db).currentVersion()); } + @Test + void aDatabaseFromTheReleaseBeforeCatalogsIsMintedOneOnUpgradeAndNamesNoMainYet() { + stageAtBaseline(); + var prior = migrationIndex("CREATE TABLE catalog"); + db.execute("PRAGMA foreign_keys = OFF"); + SchemaManager.MIGRATIONS.subList(0, prior).forEach(db::execute); + db.execute("PRAGMA foreign_keys = ON"); + db.execute( + "INSERT INTO schema_version (version, applied_at) VALUES (?, 'staged')", + SchemaManager.V1_VERSION + prior); + assertFalse(tables().contains("catalog")); + + new SchemaManager(db).migrate(); + + var catalog = new DatabaseCatalog(db); + assertTrue(catalog.id().matches("[0-9a-f]{32}"), catalog.id()); + assertEquals(Optional.empty(), catalog.main()); + assertEquals(SchemaManager.CURRENT_VERSION, new SchemaManager(db).currentVersion()); + } + private List tables() { return db.query("SELECT name FROM sqlite_master WHERE type = 'table'", row -> row.text(0)); } diff --git a/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java b/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java new file mode 100644 index 000000000..55bc92e35 --- /dev/null +++ b/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2026 Standard Applied Intelligence Labs + * SPDX-License-Identifier: MIT + */ + +package ai.singlr.sail.sync; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import ai.singlr.sail.engine.OperatorCommand; +import ai.singlr.sail.identity.ProjectScope; +import ai.singlr.sail.store.DatabaseCatalog; +import ai.singlr.sail.store.SchemaManager; +import ai.singlr.sail.store.Sqlite; +import java.util.Optional; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** M1 and M2 at the node's door: a node admits only the main database it synced with. */ +class NodeRoundAdmitTest { + + private Sqlite db; + + @BeforeEach + void setUp() { + db = Sqlite.openMemory(); + new SchemaManager(db).migrate(); + } + + @AfterEach + void tearDown() { + db.close(); + } + + private static SyncSession welcomedBy(String catalog) { + var welcome = + new SyncWire.Welcome( + SyncWire.PROTOCOL, "0.53.0", "main", "nia", 0, ProjectScope.all(), catalog); + return SyncSession.open( + new ByteStreams.Input(SyncWire.encode(welcome) + "\n"), + new ByteStreams.Output(), + SyncWire.Hello.of("0.53.0", "node-box"), + ignored -> {}, + ignored -> {}, + OperatorCommand.PLAIN); + } + + private void admit(String catalog) { + try (var session = welcomedBy(catalog)) { + NodeRound.admit(session, db, "sail@main"); + } + } + + private static String refusal(String now) { + return "Main at sail@main is not the database this box synced with (catalog aaaa, now " + + now + + "). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again."; + } + + @Test + void aMainThatNamesNoCatalogIsNotAskedUntilTheNodeHasRecordedOne() { + admit(null); + + assertEquals(Optional.empty(), new DatabaseCatalog(db).main()); + } + + @Test + void theFirstCatalogIsRecordedAndTheSameOneIsAdmittedAgain() { + admit("aaaa"); + + assertDoesNotThrow(() -> admit("aaaa")); + assertEquals(Optional.of("aaaa"), new DatabaseCatalog(db).main()); + } + + @Test + void anotherCatalogIsRefusedAndNeverRecorded() { + admit("aaaa"); + + var refused = assertThrows(SyncTransportException.class, () -> admit("bbbb")); + + assertEquals("refused", refused.kind()); + assertEquals(refusal("bbbb"), refused.getMessage()); + assertEquals(Optional.of("aaaa"), new DatabaseCatalog(db).main()); + } + + @Test + void aMainThatNamesNoneOnceTheNodeRecordedOneIsAnotherDatabase() { + admit("aaaa"); + + var refused = assertThrows(SyncTransportException.class, () -> admit(null)); + + assertEquals(refusal("none"), refused.getMessage()); + } +} diff --git a/sail-core/src/test/java/ai/singlr/sail/sync/SyncWireTest.java b/sail-core/src/test/java/ai/singlr/sail/sync/SyncWireTest.java index 133641782..e6272ab5c 100644 --- a/sail-core/src/test/java/ai/singlr/sail/sync/SyncWireTest.java +++ b/sail-core/src/test/java/ai/singlr/sail/sync/SyncWireTest.java @@ -115,6 +115,21 @@ void aWelcomeNamesTheHandleOnlyWhenMainSaysOne() { .handle()); } + @Test + void aWelcomeNamesMainsCatalogOnlyWhenMainHasOne() { + var catalogued = + new SyncWire.Welcome(4, "0.53.0", "main-box", "ada", 0, ProjectScope.all(), "c0ffee"); + var older = new SyncWire.Welcome(4, "0.52.0", "main-box", "ada", 0); + + assertEquals( + "{\"op\": \"welcome\", \"protocol\": 4, \"version\": \"0.53.0\", \"mainId\":" + + " \"main-box\", \"handle\": \"ada\", \"catalog\": \"c0ffee\"}", + SyncWire.encode(catalogued)); + assertEquals(catalogued, SyncWire.decodeResponse(SyncWire.encode(catalogued))); + assertFalse(SyncWire.encode(older).contains("catalog")); + assertNull(((SyncWire.Welcome) SyncWire.decodeResponse(SyncWire.encode(older))).catalog()); + } + @Test void aWelcomeNamesAScopeOnlyForAScopedFdeAndAMissingKeyIsNeverAnEmptyGrant() { var unscoped = new SyncWire.Welcome(4, "0.48.0", "main-box", "ada", 0); diff --git a/sail-core/src/test/java/ai/singlr/sail/sync/UploadFailureRepliesTest.java b/sail-core/src/test/java/ai/singlr/sail/sync/UploadFailureRepliesTest.java index 29c0476e1..123ee0882 100644 --- a/sail-core/src/test/java/ai/singlr/sail/sync/UploadFailureRepliesTest.java +++ b/sail-core/src/test/java/ai/singlr/sail/sync/UploadFailureRepliesTest.java @@ -8,8 +8,10 @@ import ai.singlr.sail.config.FileLimits; import ai.singlr.sail.identity.Actor; +import ai.singlr.sail.identity.ProjectScope; import ai.singlr.sail.identity.Role; import ai.singlr.sail.store.BlobStore; +import ai.singlr.sail.store.DatabaseCatalog; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; @@ -147,15 +149,21 @@ private static void assertReplies( request.writeBytes(line(SyncWire.encode(new SyncWire.Announce(List.of(HASH))))); for (var content : afterAnnounce) request.writeBytes(line(SyncWire.encode(content))); request.writeBytes(trailing); - var expected = new StringBuilder(); - expected.append( - SyncWire.encode( - new SyncWire.Welcome( - SyncWire.PROTOCOL, SyncWire.UPGRADE_FLOOR, "main", "node", limits.fileMax()))); - expected.append('\n'); - expected.append(SyncWire.encode(new SyncWire.Lack(List.of(HASH)))).append('\n'); - for (var reply : replies) expected.append(SyncWire.encode(reply)).append('\n'); try (var main = new SyncBox("main")) { + var expected = new StringBuilder(); + expected.append( + SyncWire.encode( + new SyncWire.Welcome( + SyncWire.PROTOCOL, + SyncWire.UPGRADE_FLOOR, + "main", + "node", + limits.fileMax(), + ProjectScope.all(), + new DatabaseCatalog(main.db).id()))); + expected.append('\n'); + expected.append(SyncWire.encode(new SyncWire.Lack(List.of(HASH)))).append('\n'); + for (var reply : replies) expected.append(SyncWire.encode(reply)).append('\n'); var output = new ByteStreams.Output(); main.server(Actor.sync("node", Role.MEMBER)) .content(main.db, limits) diff --git a/sail-infra/src/main/java/ai/singlr/sail/commands/JoinCommand.java b/sail-infra/src/main/java/ai/singlr/sail/commands/JoinCommand.java index 811e18dd1..3ea9553c1 100644 --- a/sail-infra/src/main/java/ai/singlr/sail/commands/JoinCommand.java +++ b/sail-infra/src/main/java/ai/singlr/sail/commands/JoinCommand.java @@ -348,22 +348,31 @@ static String render(Plan plan, boolean json, OperatorCommand operator) { @|bold,green ✓|@ This box is now a @|bold node|@ syncing to @|bold %s|@. + %s + Main already has this box's key. Finish with @|bold %s|@.\ """ - .formatted(plan.target(), sync)); + .formatted(plan.target(), MainHostKey.pinnedLine(plan.mainHostKey()), sync)); } return Ansi.AUTO.string( """ @|bold,green ✓|@ This box is now a @|bold node|@ syncing to @|bold %s|@. + %s + Send this to the operator of %s — they run it once: @|cyan %s|@ Once they have, finish here with @|bold %s|@. Only a public key leaves this box — \ never a secret.""" - .formatted(plan.target(), plan.target(), authorize, sync)); + .formatted( + plan.target(), + MainHostKey.pinnedLine(plan.mainHostKey()), + plan.target(), + authorize, + sync)); } /** diff --git a/sail-infra/src/main/java/ai/singlr/sail/engine/MainHostKey.java b/sail-infra/src/main/java/ai/singlr/sail/engine/MainHostKey.java index 4f90a464f..ea2b54e0d 100644 --- a/sail-infra/src/main/java/ai/singlr/sail/engine/MainHostKey.java +++ b/sail-infra/src/main/java/ai/singlr/sail/engine/MainHostKey.java @@ -27,6 +27,10 @@ public final class MainHostKey { private static final String TYPE = "ssh-ed25519"; private static final String REVOKED = "@revoked"; + /** How main's operator reads the fingerprint of the host key a node pins. */ + private static final String FINGERPRINT_ON_MAIN = + "sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub"; + private final ShellExec shell; private final Path knownHosts; private final OperatorCommand operator; @@ -50,14 +54,21 @@ public static MainHostKey here(OperatorCommand operator) { /** * How a node recovers from main's host key no longer being the one it pinned for {@code target}: - * drop the old pin, then join again, which pins the new one. + * drop the old pin, then join again, which pins the new one and prints it to check against main. */ public static String rebuiltRecovery(String target, OperatorCommand operator) { - return "If main was rebuilt, remove the old key with '" + return "If main was rebuilt from a copy of its database, remove the old key with '" + operator.asThisProcess("ssh-keygen -R " + hostOf(target)) - + "' and re-run '" + + "', re-run '" + operator.sail("init --main " + target) - + "'; otherwise something between this box and main is answering for it."; + + "' and compare the fingerprint it prints with main's ('" + + FINGERPRINT_ON_MAIN + + "' on main); otherwise something between this box and main is answering for it."; + } + + /** The line a join prints for the key it pinned, {@code fingerprint}, to check against main. */ + public static String pinnedLine(String fingerprint) { + return "Pinned main's host key " + fingerprint + ". Check it on main: " + FINGERPRINT_ON_MAIN; } /** The bare host of a sync target, whose SSH host key is pinned: no {@code user@}, no port. */ diff --git a/sail-infra/src/main/java/ai/singlr/sail/engine/SyncOperations.java b/sail-infra/src/main/java/ai/singlr/sail/engine/SyncOperations.java index 16b7acafc..1ab0f2ade 100644 --- a/sail-infra/src/main/java/ai/singlr/sail/engine/SyncOperations.java +++ b/sail-infra/src/main/java/ai/singlr/sail/engine/SyncOperations.java @@ -249,7 +249,8 @@ public static Set acknowledgeHeld( /** * Whether main holds this box's sync key as {@code handle}: one session opened through {@code * channels} as the box {@code config} names is welcomed as that handle. Anything short of that, - * main unreachable included, is no evidence that it does. + * main unreachable included, is no evidence that it does. The probe says nothing of its own: the + * answer it gives is what the caller shows. */ public static boolean welcomedAs( SyncConfig config, Channels channels, String handle, OperatorCommand operator) { @@ -259,21 +260,38 @@ public static boolean welcomedAs( channel.reader(), channel.writer(), hello(config, operator), - SyncOperations::notice, - denial -> notice(denial.describe(operator)), + unsaid -> {}, + unsaid -> {}, operator)) { return session.handle().filter(handle::equals).isPresent(); } catch (Exception notWelcomed) { - notice(Objects.toString(notWelcomed.getMessage(), notWelcomed.getClass().getSimpleName())); return false; } } /** - * Opens a sync session to main over {@code channel} as the box whose config is {@code config}. + * Opens a sync session to main at {@code target} over {@code channel} as the box whose config is + * {@code config}, refused unless main serves the database this box synced with ({@link + * NodeRound#admit}). */ private static SyncSession open( Channel channel, SyncConfig config, Sqlite db, String target, OperatorCommand operator) { + var session = welcome(channel, config, db, target, operator); + try { + NodeRound.admit(session, db, target); + return session; + } catch (RuntimeException refused) { + try { + session.close(); + } catch (RuntimeException closing) { + refused.addSuppressed(closing); + } + throw refused; + } + } + + private static SyncSession welcome( + Channel channel, SyncConfig config, Sqlite db, String target, OperatorCommand operator) { try { return SyncSession.open( channel.reader(), diff --git a/sail-infra/src/test/java/ai/singlr/sail/commands/JoinCommandTest.java b/sail-infra/src/test/java/ai/singlr/sail/commands/JoinCommandTest.java index 695fe9b02..1895df09c 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/commands/JoinCommandTest.java +++ b/sail-infra/src/test/java/ai/singlr/sail/commands/JoinCommandTest.java @@ -34,7 +34,10 @@ import ai.singlr.sail.store.SchemaManager; import ai.singlr.sail.store.Sqlite; import ai.singlr.sail.sync.SyncBox; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermissions; @@ -157,9 +160,11 @@ void aMainAnsweringWithAnotherKeyIsRefusedBeforeTheHostConfigIsWritten() throws + IMPOSTOR.fingerprint() + ", but this box pinned " + MAIN.fingerprint() - + ". If main was rebuilt, remove the old key with 'sudo ssh-keygen -R maindevbox' and" - + " re-run 'sudo sail init --main sail@maindevbox'; otherwise something between this" - + " box and main is answering for it.", + + ". If main was rebuilt from a copy of its database, remove the old key with 'sudo" + + " ssh-keygen -R maindevbox', re-run 'sudo sail init --main sail@maindevbox' and" + + " compare the fingerprint it prints with main's ('sudo ssh-keygen -lf" + + " /etc/ssh/ssh_host_ed25519_key.pub' on main); otherwise something between this box" + + " and main is answering for it.", refused.getMessage()); assertEquals(before, Files.readString(hostConfig)); assertEquals(hashed(MAIN) + "\n", Files.readString(knownHosts())); @@ -527,13 +532,17 @@ void aFirstJoinHandsMainsOperatorTheLineThatAuthorizesThisBox() throws Exception ✓ This box is now a node syncing to sail@maindevbox. + Pinned main's host key %s. Check it on main: sudo ssh-keygen -lf \ + /etc/ssh/ssh_host_ed25519_key.pub + Send this to the operator of sail@maindevbox — they run it once: sudo sail fde add mady --role member --key "%s" --main-host-key %s Once they have, finish here with sudo sail sync. Only a public key leaves this box — \ - never a secret.""" - .formatted(PUB, MAIN.fingerprint()), + never a secret.\ + """ + .formatted(MAIN.fingerprint(), PUB, MAIN.fingerprint()), human(plan)); } @@ -570,8 +579,12 @@ void reJoiningTheMainThatWelcomesThisBoxsKeyAsksItsOperatorForNothing() throws E ✓ This box is now a node syncing to sail@maindevbox. + Pinned main's host key %s. Check it on main: sudo ssh-keygen -lf \ + /etc/ssh/ssh_host_ed25519_key.pub + Main already has this box's key. Finish with sudo sail sync.\ - """, + """ + .formatted(MAIN.fingerprint()), human(plan)); assertEquals( "sudo sail fde add mady --role member --key \"" @@ -581,6 +594,50 @@ void reJoiningTheMainThatWelcomesThisBoxsKeyAsksItsOperatorForNothing() throws E YamlUtil.parseMap(json(plan)).get("authorize_command")); } + @Test + void aReJoinAfterMainsHostKeyRotatedShowsTheNewKeyToCheckBeforeSayingMainHasThisBox() + throws Exception { + var hostConfig = writeHostConfig(); + answering = IMPOSTOR; + join(hostConfig); + Files.delete(knownHosts()); + answering = MAIN; + + var plan = join(hostConfig, "mady", mainServing(MADY)); + + assertEquals( + """ + + ✓ This box is now a node syncing to sail@maindevbox. + + Pinned main's host key %s. Check it on main: sudo ssh-keygen -lf \ + /etc/ssh/ssh_host_ed25519_key.pub + + Main already has this box's key. Finish with sudo sail sync.\ + """ + .formatted(MAIN.fingerprint()), + human(plan)); + } + + @Test + void aReJoinWhoseProbeMainNeverAnswersPrintsNothingOfItsOwn() throws Exception { + var hostConfig = writeHostConfig(); + join(hostConfig); + var err = System.err; + var said = new ByteArrayOutputStream(); + System.setErr(new PrintStream(said, true, StandardCharsets.UTF_8)); + JoinCommand.Plan plan; + try { + plan = join(hostConfig, "mady", target -> PipedSyncChannel.to((in, out) -> {})); + } finally { + System.setErr(err); + } + + assertFalse(plan.mainHasKey()); + assertEquals("", said.toString(StandardCharsets.UTF_8)); + assertTrue(human(plan).contains("sudo sail fde add mady"), human(plan)); + } + @Test void aFirstJoinNeverAsksMainAboutAKeyItJustMade() throws Exception { assertFalse(join(writeHostConfig(), "mady", NEVER_ASKED).mainHasKey()); diff --git a/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java b/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java new file mode 100644 index 000000000..22c4e60cb --- /dev/null +++ b/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2026 Standard Applied Intelligence Labs + * SPDX-License-Identifier: MIT + */ + +package ai.singlr.sail.commands; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import ai.singlr.sail.api.SyncReport; +import ai.singlr.sail.api.SyncRequest; +import ai.singlr.sail.config.SpecStatus; +import ai.singlr.sail.config.SyncConfig; +import ai.singlr.sail.engine.OperatorCommand; +import ai.singlr.sail.engine.PipedSyncChannel; +import ai.singlr.sail.engine.ShellExec; +import ai.singlr.sail.engine.SyncOperations; +import ai.singlr.sail.identity.Acting; +import ai.singlr.sail.identity.Actor; +import ai.singlr.sail.identity.ProjectScope; +import ai.singlr.sail.identity.Role; +import ai.singlr.sail.store.AuthSessionStore; +import ai.singlr.sail.store.DataMigration; +import ai.singlr.sail.store.DataMigrations; +import ai.singlr.sail.store.DatabaseCatalog; +import ai.singlr.sail.store.FdeStore; +import ai.singlr.sail.store.MigrationRunner; +import ai.singlr.sail.store.ProjectStore; +import ai.singlr.sail.store.SchemaManager; +import ai.singlr.sail.store.SpecStore; +import ai.singlr.sail.store.Sqlite; +import ai.singlr.sail.sync.SyncDatabase; +import ai.singlr.sail.sync.SyncTransitionSink; +import ai.singlr.sail.sync.SyncTransportException; +import ai.singlr.sail.sync.SyncWire; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * A node's round after the main at its address is set up again: main's {@code _sync} server answers + * a real session of {@code nia}, from the database the node synced with, from a new empty one, or + * from a copy of the first. A node never says it is in sync with a database it did not sync with, + * and moves nothing to or from one. + */ +class MainRebuiltRoundTest { + + private static final String TARGET = "sail@main"; + + @TempDir Path tempDir; + private final List mains = new ArrayList<>(); + private Main serving; + private PipedSyncChannel.Main answering; + private Sqlite nodeDb; + private SyncOperations node; + + /** One main database answering at {@link #TARGET} as the box {@code id}. */ + private record Main(SyncDatabase database, String id, String token) { + Sqlite db() { + return database.db(); + } + } + + @BeforeEach + void setUp() { + serving = main("a.db", "main", "gamma"); + nodeDb = Sqlite.open(tempDir.resolve("node.db")); + new SchemaManager(nodeDb).migrate(); + node = + new SyncOperations( + nodeDb, + "node", + tempDir.resolve("projects"), + () -> new SyncConfig(SyncConfig.ROLE_NODE, TARGET, "nia", "node-box"), + target -> { + if (answering != null) { + return PipedSyncChannel.to(answering); + } + var main = serving; + return PipedSyncChannel.to( + (in, out) -> + SyncServerCommand.serve( + main.database(), + main.id(), + main.token(), + in, + out, + SyncTransitionSink.NONE, + () -> new SyncConfig(SyncConfig.ROLE_MAIN, null, "ada", main.id()))); + }, + event -> {}, + OperatorCommand.PLAIN, + new NoContainers()); + node.prepare(); + } + + @AfterEach + void tearDown() { + nodeDb.close(); + mains.forEach(SyncDatabase::close); + } + + /** A main set up at {@code file} as the box {@code id}, holding {@code project}, nia added. */ + private Main main(String file, String id, String project) { + var path = tempDir.resolve(file); + try (var db = Sqlite.open(path)) { + MigrationRunner.applyAll( + db, DataMigrations.all(OperatorCommand.PLAIN), DataMigration.Prompter.NON_INTERACTIVE); + } + var database = SyncDatabase.converge(path, id); + mains.add(database); + var fdes = new FdeStore(database.db()); + fdes.add("ada", "Ada", "ada@x.dev", "admin"); + fdes.add("nia", "Nia", "nia@x.dev", "member"); + Acting.system(() -> new ProjectStore(database.db()).upsert(project, "name: " + project + "\n")); + return new Main(database, id, token(database.db())); + } + + /** A main serving a copy of {@code source}'s database file, as the box {@code id}. */ + private Main copyOf(Main source, String file, String id) { + var path = tempDir.resolve(file); + source.db().execute("VACUUM INTO ?", path.toString()); + var database = SyncDatabase.converge(path, id); + mains.add(database); + return new Main(database, id, token(database.db())); + } + + private static String token(Sqlite db) { + var nia = new FdeStore(db).byHandle("nia").orElseThrow(); + return new AuthSessionStore(db).create(nia.id(), Duration.ofHours(1)).token(); + } + + private SyncReport round() throws Exception { + var err = System.err; + System.setErr(new PrintStream(new ByteArrayOutputStream(), true, StandardCharsets.UTF_8)); + try { + return Actor.call( + new Actor("nia", Role.MEMBER, Actor.Lane.CLI), () -> node.sync(new SyncRequest(null))); + } finally { + System.setErr(err); + } + } + + private static String rendered(SyncReport round) { + return SyncCommand.render(round, false, OperatorCommand.PLAIN); + } + + private void write(String id) { + Acting.as("nia", () -> new SpecStore(nodeDb).create(spec(id))); + } + + private static List held(Sqlite db) { + var held = new ArrayList(); + held.addAll(db.query("SELECT 'project ' || name FROM projects ORDER BY name", r -> r.text(0))); + held.addAll( + db.query( + "SELECT 'spec ' || id || ' ' || rev || ' ' || COALESCE(base_rev, '-') FROM specs" + + " ORDER BY id", + r -> r.text(0))); + held.addAll( + db.query( + "SELECT 'log ' || entity_type || ' ' || entity_id || ' ' || rev FROM change_log" + + " ORDER BY seq", + r -> r.text(0))); + held.addAll( + db.query( + "SELECT 'checkpoint ' || peer || ' ' || entity_type || ' ' || checkpoint" + + " FROM sync_state ORDER BY peer, entity_type", + r -> r.text(0))); + held.addAll(db.query("SELECT 'catalog ' || COALESCE(main, '-') FROM catalog", r -> r.text(0))); + return held; + } + + private String refusal(Main now) { + return "Main at " + + TARGET + + " is not the database this box synced with (catalog " + + new DatabaseCatalog(mains.getFirst().db()).id() + + ", now " + + new DatabaseCatalog(now.db()).id() + + "). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again."; + } + + @Test + void aNodeRecordsTheCatalogOfTheMainDatabaseItFirstSyncsWith() throws Exception { + assertEquals(List.of(), new DatabaseCatalog(nodeDb).main().stream().toList()); + + round(); + + var catalog = new DatabaseCatalog(serving.db()).id(); + assertEquals(List.of(catalog), new DatabaseCatalog(nodeDb).main().stream().toList()); + assertNotEquals(catalog, new DatabaseCatalog(nodeDb).id()); + } + + @Test + void twoRoundsWithTheSameMainAndNothingChangedAreAlreadyInSync() throws Exception { + round(); + + assertEquals(" ✓ Already in sync with main.", rendered(round())); + } + + @Test + void anEmptyMainAtTheSameAddressIsRefusedBeforeAnythingMoves() throws Exception { + round(); + write("g2"); + var before = held(nodeDb); + var rebuilt = main("b.db", "main-b", "demo"); + var mainBefore = held(rebuilt.db()); + serving = rebuilt; + + var refused = assertThrows(SyncTransportException.class, this::round); + + assertEquals(refusal(rebuilt), refused.getMessage()); + assertEquals("refused", refused.kind()); + assertEquals(before, held(nodeDb)); + assertEquals(mainBefore, held(rebuilt.db())); + } + + @Test + void anEmptyMainUnderTheBoxIdTheNodeSyncedWithIsRefusedOnItsCatalog() throws Exception { + round(); + write("g2"); + var before = held(nodeDb); + var rebuilt = main("b.db", "main", "demo"); + var mainBefore = held(rebuilt.db()); + serving = rebuilt; + + var refused = assertThrows(SyncTransportException.class, this::round); + + assertEquals(refusal(rebuilt), refused.getMessage()); + assertEquals(before, held(nodeDb)); + assertEquals(mainBefore, held(rebuilt.db())); + } + + @Test + void aMainRebuiltFromACopyOfItsDatabaseTakesTheNodesNewerWork() throws Exception { + round(); + var restored = copyOf(serving, "copy.db", "main-restored"); + write("g2"); + serving = restored; + + var round = round(); + + assertEquals(1, round.report().pushed()); + assertEquals( + List.of("g2"), restored.db().query("SELECT id FROM specs ORDER BY id", r -> r.text(0))); + assertEquals(" ✓ Already in sync with main.", rendered(round())); + } + + @Test + void aNodeThatSyncedBeforeCatalogsRecordsMainsOnItsNextRound() throws Exception { + round(); + nodeDb.execute("UPDATE catalog SET main = NULL"); + + assertEquals(" ✓ Already in sync with main.", rendered(round())); + assertEquals( + List.of(new DatabaseCatalog(serving.db()).id()), + new DatabaseCatalog(nodeDb).main().stream().toList()); + } + + @Test + void aRefusalIsWhatTheNodeSaysEvenWhenMainHangsUpBeforeItsGoodbye() throws Exception { + round(); + var other = + new SyncWire.Welcome( + SyncWire.PROTOCOL, "0.54.0", "main", "nia", 0, ProjectScope.all(), "another"); + answering = + (in, out) -> { + SyncWire.readFramed(in); + in.close(); + out.write((SyncWire.encode(other) + "\n").getBytes(StandardCharsets.UTF_8)); + out.flush(); + }; + + var refused = assertThrows(SyncTransportException.class, this::round); + + assertEquals( + "Main at sail@main is not the database this box synced with (catalog " + + new DatabaseCatalog(serving.db()).id() + + ", now another). Restore main's database (/var/lib/sail/sail.db) from a copy, then" + + " sync again.", + refused.getMessage()); + assertEquals(1, refused.getSuppressed().length); + } + + private static SpecStore.SpecRow spec(String id) { + return new SpecStore.SpecRow( + id, + "gamma", + "the " + id + " spec", + SpecStatus.PENDING, + null, + null, + null, + null, + null, + 0, + "nia", + "", + "", + "nia", + List.of(), + List.of()); + } + + /** An Incus that holds no container. */ + private static final class NoContainers implements ShellExec { + @Override + public Result exec(List command) { + return new Result(0, "[]", ""); + } + + @Override + public Result exec(List command, Path workDir, Duration timeout) { + return exec(command); + } + + @Override + public boolean isDryRun() { + return false; + } + } +} diff --git a/sail-infra/src/test/java/ai/singlr/sail/commands/NativeFleet.java b/sail-infra/src/test/java/ai/singlr/sail/commands/NativeFleet.java index f849f0021..42dab193d 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/commands/NativeFleet.java +++ b/sail-infra/src/test/java/ai/singlr/sail/commands/NativeFleet.java @@ -68,6 +68,16 @@ public final class NativeFleet implements AutoCloseable { chmod 640 /var/lib/sail/host.yaml ssh-keygen -A >/dev/null """; + private static final String REBUILD_MAIN = + """ + set -e + rm -rf /var/lib/sail/* /etc/ssh/ssh_host_* + install -m 600 -o sail -g sail /dev/null /home/sail/.ssh/authorized_keys + echo 'storage_backend: dir' > /var/lib/sail/host.yaml + chgrp sail /var/lib/sail/host.yaml + chmod 640 /var/lib/sail/host.yaml + ssh-keygen -A >/dev/null + """; private static final String SHARE_DATABASE_WITH_GATEWAY = "cd /var/lib/sail && chgrp sail sail.db* && chmod 660 sail.db*"; private static final String PROVISION_NODE = @@ -133,16 +143,46 @@ public Box main(String handle, Path binary) throws Exception { var box = launch(handle, true, "/var/lib/sail/sail.db"); box.shOk(PROVISION_MAIN); box.install(binary); - box.shOk(SHARE_DATABASE_WITH_GATEWAY); - box.sailOk("host", "sync", "--as-main"); - box.sailOk("host", "config", "set", "sync-handle", handle); - box.sailOk("server", "init"); - box.sailOk("fde", "add", handle, "--role", "admin", "--email", handle + "@example.dev"); + setUpMain(box); main = box; sshd = box.sshd(); return box; } + private static void setUpMain(Box box) throws Exception { + box.shOk(SHARE_DATABASE_WITH_GATEWAY); + box.sailOk("host", "sync", "--as-main"); + box.sailOk("host", "config", "set", "sync-handle", box.name); + box.sailOk("server", "init"); + box.sailOk("fde", "add", box.name, "--role", "admin", "--email", box.name + "@example.dev"); + } + + /** + * Sets main up again at its address, as a VM deleted and set up with {@code sudo sail init + * --as-main} comes back: a new host key, {@code host.yaml}, {@code authorized_keys} and database. + * Given {@code copy}, the path in main's box of a copy of its database, that copy is then put in + * place of the new database and migrated, which restores the sail user's keys from it: the + * README's restore, but for stopping and starting sail-api, which no box here runs. + */ + public void rebuildMain(String copy) throws Exception { + sshd.close(); + try { + main.shOk(REBUILD_MAIN); + main.sailOk("migrate"); + setUpMain(main); + if (copy != null) { + main.shOk( + "install -m 660 -g sail " + + copy + + " /var/lib/sail/sail.db && rm -f /var/lib/sail/sail.db-wal" + + " /var/lib/sail/sail.db-shm"); + main.sailOk("migrate"); + } + } finally { + sshd = main.sshd(); + } + } + public Box node(String handle, Path binary) throws Exception { return node(handle, binary, new String[0]); } diff --git a/sail-infra/src/test/java/ai/singlr/sail/commands/NativeFleetIT.java b/sail-infra/src/test/java/ai/singlr/sail/commands/NativeFleetIT.java index 578bb91a8..7198f6f6a 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/commands/NativeFleetIT.java +++ b/sail-infra/src/test/java/ai/singlr/sail/commands/NativeFleetIT.java @@ -38,7 +38,8 @@ void aFleetUpgradedMainFirstKeepsEveryWriteAndSurfacesTheConflict() throws Excep () -> { main.createSpecs("seed", 5, BODY_BYTES); main.shOk( - "dd if=/dev/urandom of=/tmp/shared.bin bs=1M count=3 status=none && chmod 644 /tmp/shared.bin"); + "dd if=/dev/urandom of=/tmp/shared.bin bs=1M count=3 status=none && chmod 644" + + " /tmp/shared.bin"); main.sailOk( "project", "files", "add", "-p", "demo", "/tmp/shared.bin", "--as", "shared.bin"); main.apiOk("spec", "update", "seed-1", "--assignee", "mady"); @@ -129,7 +130,8 @@ void aFreshNodePullsPastOneFrameAndACutTransportLosesNothing() throws Exception main.serving( () -> { main.shOk( - "dd if=/dev/urandom of=/tmp/large.bin bs=1M count=40 status=none && chmod 750 /tmp/large.bin"); + "dd if=/dev/urandom of=/tmp/large.bin bs=1M count=40 status=none && chmod 750" + + " /tmp/large.bin"); main.sailOk( "project", "files", "add", "-p", "demo", "/tmp/large.bin", "--as", "large.bin"); }); @@ -194,8 +196,10 @@ void aNodeRecoversFromARekeyedMainByFollowingWhatItsSyncPrinted() throws Excepti .output() .contains( "hello: Sync channel closed before main replied. ssh said: Host key verification" - + " failed. If main was rebuilt, remove the old key with 'ssh-keygen -R" - + " localhost' and re-run 'sail init --main sail@localhost'; otherwise" + + " failed. If main was rebuilt from a copy of its database, remove the old" + + " key with 'ssh-keygen -R localhost', re-run 'sail init --main" + + " sail@localhost' and compare the fingerprint it prints with main's ('sudo" + + " ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub' on main); otherwise" + " something between this box and main is answering for it."), refused::output); @@ -210,6 +214,86 @@ void aNodeRecoversFromARekeyedMainByFollowingWhatItsSyncPrinted() throws Excepti } } + @Test + void aNodeRefusesAnEmptyMainSetUpAgainAtItsAddressAndMovesNothing() throws Exception { + try (var fleet = NativeFleet.openOrSkip()) { + var main = fleet.main("uday", fleet.candidate()); + main.serving(() -> main.createSpecs("seed", 2, 1_000)); + var mady = fleet.node("mady", fleet.candidate()); + mady.sailOk("sync"); + fleet.assertConverged(mady); + fleet.offline(mady, () -> mady.createSpecs("node-new", 1, 1_000)); + var synced = main.query("SELECT id FROM catalog").strip(); + var nodeBefore = mady.replicated(); + + fleet.rebuildMain(null); + mady.shOk("ssh-keygen -R localhost"); + var joined = NativeFleet.json(mady.sailOk("join", "localhost", "--handle", "mady", "--json")); + main.sailOk( + "fde", + "add", + "mady", + "--role", + "member", + "--email", + "mady@example.dev", + "--key", + (String) joined.get("public_key"), + "--main-host-key", + (String) joined.get("main_host_key")); + var mainBefore = main.replicated(); + var refused = mady.sail("sync"); + + assertEquals(1, refused.exit(), refused::output); + assertTrue( + refused + .output() + .contains( + "Main at sail@localhost is not the database this box synced with (catalog " + + synced + + ", now " + + main.query("SELECT id FROM catalog").strip() + + "). Restore main's database (/var/lib/sail/sail.db) from a copy, then" + + " sync again."), + refused::output); + assertFalse(refused.output().contains("Already in sync"), refused::output); + assertEquals(mainBefore, main.replicated()); + assertEquals(nodeBefore, mady.replicated()); + } + } + + @Test + void aMainSetUpAgainFromACopyOfItsDatabaseIsTheSameMainAndTakesTheNodesNewerWork() + throws Exception { + try (var fleet = NativeFleet.openOrSkip()) { + var main = fleet.main("uday", fleet.candidate()); + main.serving(() -> main.createSpecs("seed", 2, 1_000)); + var mady = fleet.node("mady", fleet.candidate()); + mady.sailOk("sync"); + fleet.assertConverged(mady); + main.shOk("sqlite3 /var/lib/sail/sail.db \".backup /root/sail.db.copy\""); + fleet.offline(mady, () -> mady.createSpecs("node-new", 1, 1_000)); + + fleet.rebuildMain("/root/sail.db.copy"); + mady.shOk("ssh-keygen -R localhost"); + var rejoined = mady.sailOk("join", "localhost", "--handle", "mady"); + var fingerprint = main.shOk("ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub").split(" ")[1]; + + assertTrue( + rejoined.contains( + "Pinned main's host key " + + fingerprint + + ". Check it on main: sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub"), + rejoined); + assertTrue(rejoined.contains("Main already has this box's key."), rejoined); + mady.sailOk("sync"); + assertEquals("1", main.query("SELECT count(*) FROM specs WHERE id = 'node-new-1'").strip()); + fleet.assertConverged(mady); + var again = mady.sailOk("sync"); + assertTrue(again.contains("Already in sync with main."), again); + } + } + @Test void aNodeUpgradedAheadOfMainRecoversOnceMainFollows() throws Exception { try (var fleet = NativeFleet.openOrSkip()) { @@ -282,7 +366,9 @@ void aPruneOnAnUpgradedMainLeavesTheNodeNoRowHistoryOrBlobOfItAfterOneRound() th main.apiOk("spec", "prune", "doomed-1", "--apply"); }); main.shOk( - "printf '#!/bin/sh\\necho \"[]\"\\n' > /usr/local/bin/incus && chmod 755 /usr/local/bin/incus"); + "printf '#!/bin/sh\\n" + + "echo \"[]\"\\n" + + "' > /usr/local/bin/incus && chmod 755 /usr/local/bin/incus"); main.sailOk("project", "destroy", "scratch", "--purge", "--yes", "--json"); mady.sailOk("sync"); @@ -371,7 +457,8 @@ void aScopedNodeHoldsExactlyItsGrantedProjectsThroughGrantWriteRevokeAndPrune() var listed = nia.shOk( "ssh -o BatchMode=yes -o IdentitiesOnly=yes -i /root/.sail/sync_ed25519" - + " sail@localhost spec list --all-projects --json --server http://127.0.0.1:7171"); + + " sail@localhost spec list --all-projects --json --server" + + " http://127.0.0.1:7171"); assertTrue(listed.contains("\"a-1\"") && listed.contains("\"a-2\""), listed); assertFalse(listed.contains("b-1") || listed.contains("beta"), listed); }); @@ -575,7 +662,8 @@ INSERT INTO change_log (entity_type, entity_id, rev, actor, recorded_at, origin, "role": "build", "agent": "claude", "status": "completed", "started_at": "2026-09-01T00:00:00Z", "repos": [], "principals": []}'); INSERT INTO change_heads (entity_type, entity_id, seq) - VALUES ('run', '%1$s', last_insert_rowid());""" + VALUES ('run', '%1$s', last_insert_rowid());\ + """ .formatted(DOOMED_RUN); /** diff --git a/sail-infra/src/test/java/ai/singlr/sail/commands/OperatorHintsTest.java b/sail-infra/src/test/java/ai/singlr/sail/commands/OperatorHintsTest.java index 71763e5d7..aa0f61018 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/commands/OperatorHintsTest.java +++ b/sail-infra/src/test/java/ai/singlr/sail/commands/OperatorHintsTest.java @@ -96,9 +96,11 @@ void aSudoCallerOnARootBoxIsToldEveryCommandWithSudo() { + " connect with, then apply the project to put it in the container:\n" + " sudo sail host config set ssh-public-key\n" + " sudo sail project apply gamma", - "If main was rebuilt, remove the old key with 'sudo ssh-keygen -R main' and re-run" - + " 'sudo sail init --main sail@main'; otherwise something between this box and" - + " main is answering for it."), + "If main was rebuilt from a copy of its database, remove the old key with 'sudo" + + " ssh-keygen -R main', re-run 'sudo sail init --main sail@main' and compare the" + + " fingerprint it prints with main's ('sudo ssh-keygen -lf" + + " /etc/ssh/ssh_host_ed25519_key.pub' on main); otherwise something between this" + + " box and main is answering for it."), hints(SUDO)); } @@ -133,9 +135,10 @@ void rootOrABoxsOwnAccountIsToldEveryCommandAsToday() { + " connect with, then apply the project to put it in the container:\n" + " sail host config set ssh-public-key\n" + " sail project apply gamma", - "If main was rebuilt, remove the old key with 'ssh-keygen -R main' and re-run 'sail" - + " init --main sail@main'; otherwise something between this box and main is" - + " answering for it."), + "If main was rebuilt from a copy of its database, remove the old key with 'ssh-keygen" + + " -R main', re-run 'sail init --main sail@main' and compare the fingerprint it" + + " prints with main's ('sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub' on" + + " main); otherwise something between this box and main is answering for it."), hints(PLAIN)); } diff --git a/sail-infra/src/test/java/ai/singlr/sail/commands/SyncCliTest.java b/sail-infra/src/test/java/ai/singlr/sail/commands/SyncCliTest.java index aeee92563..663844b78 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/commands/SyncCliTest.java +++ b/sail-infra/src/test/java/ai/singlr/sail/commands/SyncCliTest.java @@ -396,6 +396,7 @@ private static void verifyScheduledRoundWording() throws Exception { private static void verifyPreHealthUpgrade(String scenario, Sqlite db, HostOperations operations) throws Exception { + db.execute("DROP TABLE catalog"); db.execute("DROP TABLE sync_scope"); db.execute("ALTER TABLE change_heads DROP COLUMN project"); db.execute("DROP TABLE project_grants"); diff --git a/sail-infra/src/test/java/ai/singlr/sail/engine/MainHostKeyTest.java b/sail-infra/src/test/java/ai/singlr/sail/engine/MainHostKeyTest.java index d81f8fb3c..f50293138 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/engine/MainHostKeyTest.java +++ b/sail-infra/src/test/java/ai/singlr/sail/engine/MainHostKeyTest.java @@ -94,9 +94,11 @@ void aDifferentKeyIsRefusedNamingBothAndThePinIsKept() throws Exception { + IMPOSTOR.fingerprint() + ", but this box pinned " + MAIN.fingerprint() - + ". If main was rebuilt, remove the old key with 'sudo ssh-keygen -R maindevbox' and" - + " re-run 'sudo sail init --main sail@maindevbox'; otherwise something between this" - + " box and main is answering for it.", + + ". If main was rebuilt from a copy of its database, remove the old key with 'sudo" + + " ssh-keygen -R maindevbox', re-run 'sudo sail init --main sail@maindevbox' and" + + " compare the fingerprint it prints with main's ('sudo ssh-keygen -lf" + + " /etc/ssh/ssh_host_ed25519_key.pub' on main); otherwise something between this box" + + " and main is answering for it.", refused.getMessage()); assertEquals(pinned, Files.readString(knownHosts())); } @@ -104,8 +106,10 @@ void aDifferentKeyIsRefusedNamingBothAndThePinIsKept() throws Exception { @Test void theRecoveryIsOneSentenceWhereverAChangedKeyIsMet() { assertEquals( - "If main was rebuilt, remove the old key with 'sudo ssh-keygen -R 10.128.0.7' and re-run" - + " 'sudo sail init --main sail@10.128.0.7:2222'; otherwise something between this box" + "If main was rebuilt from a copy of its database, remove the old key with 'sudo ssh-keygen" + + " -R 10.128.0.7', re-run 'sudo sail init --main sail@10.128.0.7:2222' and compare the" + + " fingerprint it prints with main's ('sudo ssh-keygen -lf" + + " /etc/ssh/ssh_host_ed25519_key.pub' on main); otherwise something between this box" + " and main is answering for it.", MainHostKey.rebuiltRecovery("sail@10.128.0.7:2222", OperatorCommand.SUDO)); } @@ -124,23 +128,27 @@ void aFreshBoxsSudoCallerRemovesRootsPinWithSudo() throws Exception { + IMPOSTOR.fingerprint() + ", but this box pinned " + MAIN.fingerprint() - + ". If main was rebuilt, remove the old key with 'sudo ssh-keygen -R maindevbox' and" - + " re-run 'sudo sail init --main sail@maindevbox'; otherwise something between this" - + " box and main is answering for it.", + + ". If main was rebuilt from a copy of its database, remove the old key with 'sudo" + + " ssh-keygen -R maindevbox', re-run 'sudo sail init --main sail@maindevbox' and" + + " compare the fingerprint it prints with main's ('sudo ssh-keygen -lf" + + " /etc/ssh/ssh_host_ed25519_key.pub' on main); otherwise something between this box" + + " and main is answering for it.", refused.getMessage()); } @Test void anAccountsOwnPinIsRemovedWithoutSudo() { assertEquals( - "If main was rebuilt, remove the old key with 'ssh-keygen -R main' and re-run 'sudo sail" - + " init --main sail@main'; otherwise something between this box and main is answering" - + " for it.", + "If main was rebuilt from a copy of its database, remove the old key with 'ssh-keygen -R" + + " main', re-run 'sudo sail init --main sail@main' and compare the fingerprint it" + + " prints with main's ('sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub' on" + + " main); otherwise something between this box and main is answering for it.", MainHostKey.rebuiltRecovery("sail@main", OperatorCommand.UNPRIVILEGED)); assertEquals( - "If main was rebuilt, remove the old key with 'ssh-keygen -R main' and re-run 'sail init" - + " --main sail@main'; otherwise something between this box and main is answering for" - + " it.", + "If main was rebuilt from a copy of its database, remove the old key with 'ssh-keygen -R" + + " main', re-run 'sail init --main sail@main' and compare the fingerprint it prints" + + " with main's ('sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub' on main);" + + " otherwise something between this box and main is answering for it.", MainHostKey.rebuiltRecovery("sail@main", OperatorCommand.PLAIN)); } diff --git a/sail-infra/src/test/java/ai/singlr/sail/engine/SyncDiagnosticTest.java b/sail-infra/src/test/java/ai/singlr/sail/engine/SyncDiagnosticTest.java index 32f1511f3..f24194dcb 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/engine/SyncDiagnosticTest.java +++ b/sail-infra/src/test/java/ai/singlr/sail/engine/SyncDiagnosticTest.java @@ -65,10 +65,12 @@ public void close() {} @Test void anUnpinnedOrChangedHostKeyNamesTheRemedy() throws Exception { assertEquals( - "hello: Sync channel closed before main replied. ssh said: Host key verification failed." - + " If main was rebuilt, remove the old key with 'sudo ssh-keygen -R main' and re-run" - + " 'sudo sail init --main sail@main'; otherwise something between this box and main" - + " is answering for it.", + "hello: Sync channel closed before main replied. ssh said: Host key verification failed. If" + + " main was rebuilt from a copy of its database, remove the old key with 'sudo" + + " ssh-keygen -R main', re-run 'sudo sail init --main sail@main' and compare the" + + " fingerprint it prints with main's ('sudo ssh-keygen -lf" + + " /etc/ssh/ssh_host_ed25519_key.pub' on main); otherwise something between this box" + + " and main is answering for it.", syncFailure("Host key verification failed.")); } From 56b240ddbd411f4ed867b6dc993b973c76a21c4b Mon Sep 17 00:00:00 2001 From: Uday Chandra Date: Tue, 20 Oct 2026 23:07:53 +0000 Subject: [PATCH 2/4] Refuse a main that names no database catalog, recorded or not --- ARCHITECTURE.md | 7 +++-- CHANGELOG.md | 2 +- .../java/ai/singlr/sail/sync/NodeRound.java | 15 ++++++++-- .../singlr/sail/sync/NodeRoundAdmitTest.java | 9 ++++-- .../sail/commands/MainRebuiltRoundTest.java | 28 +++++++++++++++++++ 5 files changed, 53 insertions(+), 8 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index df0207fef..c56917922 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -81,7 +81,10 @@ Main's database is the org. Back up `/var/lib/sail/sail.db` (`sqlite3 /var/lib/s the file, and main's welcome names it. A node records the catalog of the first main database it syncs with and refuses any other before anything moves (`NodeRound.admit`): "Main at is not the database this box synced with (catalog , now ). Restore main's database -(/var/lib/sail/sail.db) from a copy, then sync again." So a main set up again from a copy (`sudo +(/var/lib/sail/sail.db) from a copy, then sync again." A main older than catalogs names none and +could be any database, so it is refused too, before anything moves: "Main at names no +database catalog, so this box cannot tell whether it is the database it synced with. Upgrade main +first, then sync again." So a main set up again from a copy (`sudo sail init --as-main`, the copy put in place of its new database, then `sudo sail migrate`, which also restores the sail user's keys from it) is the same main to every node, which then pushes whatever it wrote since the copy; a main set up empty is a new org, and its nodes refuse it. A @@ -200,7 +203,7 @@ and the spec that closes it. | I1 | Every write on every box is made by exactly one bound `Actor`, whichever door it came through. | `ActorTest`, `OperationsTakeNoActorTest` | | I2 | On a node only the box's FDE, its runs and FDE-less credentials write; any other credential reads. | `RoleRuleTest`, `NodeWritesTest`, `DeniedSyncTest` | | I3 | Main and the node agree who the node is, and one box syncs as each FDE. | `NodeIdentitySyncTest`, `SyncServerCommandTest` (main's own FDE), `FdeCommandTest` (`release-box`) | -| I5 | A node syncs only with the main database it first synced with: a welcome naming another catalog, or none once one is recorded, ends the round before any row moves, and "Already in sync" is said only after a round with the recorded one. | `DatabaseCatalogTest` (minted per database, kept by upgrades and copies, the first main recorded), `SchemaManagerTest` (a database from before catalogs), `SyncWireTest`, `NodeRoundAdmitTest`, `MainRebuiltRoundTest` (an empty main under a new or the same box id refused with nothing moved either way; a main restored from a copy takes the node's newer work; a node from before catalogs records main's), `NativeFleetIT` (an empty main set up again at the node's address; one restored from a copy) | +| I5 | A node syncs only with the main database it first synced with: a welcome naming another catalog, or none at all, ends the round before any row moves, and "Already in sync" is said only after a round with the recorded one. | `DatabaseCatalogTest` (minted per database, kept by upgrades and copies, the first main recorded), `SchemaManagerTest` (a database from before catalogs), `SyncWireTest`, `NodeRoundAdmitTest`, `MainRebuiltRoundTest` (an empty main under a new or the same box id refused with nothing moved either way; a main that names no catalog refused with nothing moved before the node recorded one; a main restored from a copy takes the node's newer work; a node from before catalogs records main's), `NativeFleetIT` (an empty main set up again at the node's address; one restored from a copy) | | I4 | Every run a box executes carries that box's handle as `node` and `owner`. | `BoxRunsSyncTest`, `RunAuthorityTest`, `RunStoreTest` (stamps; never another box's run), `SyncConfigTest` (one handle), `HandleChangeTest`, `JoinCommandTest`, `HostConfigSetCommandTest`, `HostSyncCommandTest`, `RoomWakeLaunchTest` | | A1 | Who may write a synced row is one rule per type in sail-core; the same write gets the same refusal kind and code at every door and on main. | the rules: `SpecAuthorityTest`, `RoomAuthorityTest`, `ReviewAuthorityTest`, `RunAuthorityTest` (a run is created only for the owner of the spec or room it names, on every lane), `MessageAuthorityTest`, `WriterAuthorityTest`, `ProjectAuthorityTest` (a rename is an admin's); the doors: `NodeWritesTest` (one refusal — kind, code, message and fix — through an API token, a passkey session, a gateway session, the box credential, a run and a room run; the terminal keeps its own text), `AuthorityAuditTest` (a viewer's edit over HTTP as on the host CLI; a room's delete and its membership; a spec born over its owner's room at the door as on main; a merged resolve), `SailOperationsSeamTest` (a resolve: adopting main's side decides nothing, keeping or merging is an edit, on the socket and both web lanes), `CatalogAuthorityAuditTest` (the host CLI's catalog edit, project files and rename), `AdhocDispatchTest` (ad-hoc runs), `OneDecisionTest` (spec edits, posts); main: `PushAuthoritySyncTest` (a pushed run on another FDE's spec is denied, and so is one naming its own spec and another member's room; one ahead of a spec its box made lands), `DeniedSyncTest` | | A2 | No code path writes a synced row without its rule deciding it: a pushed revision at main's commit, every other write at the journal, each exactly once. | `JournalDecidesEveryWriteTest` (every public store method that reaches the journal, read from the stores' bytecode, is refused on the CLI, API, agent and room lanes and leaves nothing; a revision never admits itself; a refusal swallowed inside a transaction still rolls it back; a pushed revision is not decided twice), `OneDecisionPointTest` (in every module's classes only the journal and the message store append a revision, only they ask a write rule, and only main's commit takes a store's rule), `RevisionJournalTest`, `SqliteTest` (a doomed transaction); an operation's rows commit together or not at all: `GlobalSpecOperationsTest` (an update with its wake; the move to done; a delete), `ReviewOperationsTest` (a follow-up with its links), `BuildDispatchCoverageTest` (a dispatch and a restart: reservation, reset, claim, supersede), `SpecPruneTest` (a restore with its room) | diff --git a/CHANGELOG.md b/CHANGELOG.md index fc2bc5bb5..ddb2c6dd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.54.0 -- **A node never says "in sync" with a main that isn't the one it synced with.** Field rehearsal round 4 deleted main's VM and set it up again empty at the same IP; the node re-pinned, main ran the `fde add` line, and `sudo sail sync` printed "✓ Already in sync with main." while the new main held only `demo` and the node `gamma`, so a spec the node then filed under `gamma` was pushed into a main that holds no such project. Every database now carries a catalog id (`DatabaseCatalog`), random, minted with the database (an existing one at its upgrade) and kept by every upgrade and every copy of the file, and main's welcome names it. A node records the catalog of the first main database that welcomes it (a node from before this release on its next round) and refuses any other before anything is pushed, adopted or recorded, with exit 1: "Main at is not the database this box synced with (catalog , now ). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again." It is decided on the catalog, not the box id, so a new database under the old `host.yaml` is refused too instead of silently skipped below the node's old checkpoint. Main set up again from a copy of its database is the same main: its nodes push what they wrote since the copy, and "Already in sync with main." follows. The README and ARCHITECTURE.md say to back up `/var/lib/sail/sail.db` and how to restore it; a copy taken before this release is minted a new catalog when restored, so take one after upgrading. Every join now prints the key it pinned, "Pinned main's host key SHA256:…. Check it on main: sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub", including a re-join after a host-key rotation that then says "Main already has this box's key"; the changed-key recovery reads "If main was rebuilt from a copy of its database, remove the old key with '…', re-run '…' and compare the fingerprint it prints with main's (…); otherwise something between this box and main is answering for it."; and the re-join's probe of main prints nothing of its own (the stray "⚠ hello: Sync channel closed before main replied." line is gone). +- **A node never says "in sync" with a main that isn't the one it synced with.** Field rehearsal round 4 deleted main's VM and set it up again empty at the same IP; the node re-pinned, main ran the `fde add` line, and `sudo sail sync` printed "✓ Already in sync with main." while the new main held only `demo` and the node `gamma`, so a spec the node then filed under `gamma` was pushed into a main that holds no such project. Every database now carries a catalog id (`DatabaseCatalog`), random, minted with the database (an existing one at its upgrade) and kept by every upgrade and every copy of the file, and main's welcome names it. A node records the catalog of the first main database that welcomes it (a node from before this release on its next round) and refuses any other before anything is pushed, adopted or recorded, with exit 1: "Main at is not the database this box synced with (catalog , now ). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again." A main older than this release names no catalog and could be any database, so it is refused too, with exit 1 and before anything moves: "Main at names no database catalog, so this box cannot tell whether it is the database it synced with. Upgrade main first, then sync again." **Fleet note:** upgrade main before its nodes; an upgraded node does not sync with a main on an older release. It is decided on the catalog, not the box id, so a new database under the old `host.yaml` is refused too instead of silently skipped below the node's old checkpoint. Main set up again from a copy of its database is the same main: its nodes push what they wrote since the copy, and "Already in sync with main." follows. The README and ARCHITECTURE.md say to back up `/var/lib/sail/sail.db` and how to restore it; a copy taken before this release is minted a new catalog when restored, so take one after upgrading. Every join now prints the key it pinned, "Pinned main's host key SHA256:…. Check it on main: sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub", including a re-join after a host-key rotation that then says "Main already has this box's key"; the changed-key recovery reads "If main was rebuilt from a copy of its database, remove the old key with '…', re-run '…' and compare the fingerprint it prints with main's (…); otherwise something between this box and main is answering for it."; and the re-join's probe of main prints nothing of its own (the stray "⚠ hello: Sync channel closed before main replied." line is gone). - **Rows written together order as minted on a clock of any resolution.** Cutting v0.53.0, the macOS release job failed `ReviewLoopEveryStateTest` once, as v0.48.0 did. An instant is stored as its text, which drops trailing zeros, so `…51.123Z` sorts after the later `…51.123400Z` and `…51Z` after `…51.001Z`; a microsecond clock lands on such a value one time in a thousand. The stores ordered runs and reviews by that text, so the review loop could read a fix run as newer than the reviewer launched after it and ignore that reviewer's stop. Runs and reviews now order by their UUIDv7 id, as each box minted them; stored instants are unchanged and nothing migrates. The review loop fixture's fixed-width clock is gone. - **A box that stops being a node keeps nothing of the node it was.** The delta review of `sail-sync-is-exact` found two ways a box still carried node state after it stopped being one. Re-running `sudo sail host init` rewrote `host.yaml` from scratch, dropping the sync role, main, handle, box id and passkey settings, so a node or main turned standalone and orphaned every checkpoint its peers held for it; a re-run now refreshes only the fields it provisions and keeps the rest of `host.yaml`, read again under the box's round lock just before it writes, so a `host sync`, `join` or handle change that finished while the packages installed is kept, not undone. And a main promoted on v0.49.0 to v0.52.0, or one whose role change died between writing `host.yaml` and clearing its scope, kept the scope it last heard as a node, refusing its own `spec create` and its nodes' pushes into every other project and skipping its own descriptors at upgrade; `sail migrate` and every `sail server start` now clear that scope on any box that is not a node, and only a whole `host.yaml` decides it: one absent, empty or unreadable at that moment (truncated by a writer, or being moved into `/var/lib/sail`) keeps the scope for the next start to decide, so a node never loses its grant to a file in motion. `sudo sail host update` and `sail migrate`'s box-id and `host.yaml` relocation steps now read and write `host.yaml` under that lock too, and `host update` records the Incus version on the file as it is then, not as it was before the packages upgraded, so an identity change made meanwhile is kept. - **A scope can change after an FDE exists, and every box follows it whole.** Field rehearsal 2026-10-15 found no way to scope an FDE that joined unscoped short of `fde rm` and a re-add (which revoked its tokens, keys, sessions and passkeys), a scoped node still holding the names and emails of FDEs outside its projects, and a revoked project's container running on the node with its clone and credentials, still listed by `sail project list`. **`sudo sail fde update --scoped [--grant ]...`** narrows an FDE in place, in one transaction with the grants it names (`--grant` is repeatable and takes `a,b`), and **`--unscoped`** lets it see every project again and forgets its grants, so a later narrowing grants only what it names; the FDE keeps its id and every credential. Both take `--dry-run` (the real run, rolled back) and `--json` (`handle`, `scope`, `projects`, `changed`, `dry_run`), are refused on a node with the roster sentence, and refuse this box's own FDE and an admin as `fde add --scoped` does. A narrowing says what the FDE's box drops ("Scoped eng to acme. Its box drops beta, demo with their containers and unpushed work on its next sync."), a widening that its box holds every project after its next sync. `fde grant` on an unscoped FDE now names `sudo sail fde update --scoped`. A narrowing is a revocation on the node and a widening a grant, so the node drops and re-adopts as before. A scoped node now holds exactly the roster main lists to it: an FDE main stops listing is removed with its box-local credentials ("roster: removed 1 FDE(s) main no longer lists to this box"), the box's own FDE never; an unscoped node still disables such an FDE and keeps its row. Every round of a scoped node then takes off the box each project the scope does not admit and the box holds nothing of: its `~/.sail/projects/` directory, and its container when Sail provisioned it (the `sail-api-sock` device, the trust gate `project apply --all` uses), as `sudo sail project destroy` without `--purge` would ("project beta: its container deleted, workspace and uncommitted work included — main no longer grants this box the project"). A container Sail did not provision is left alone and named once; when the round cannot reach Incus (a non-root `sail sync`) nothing is deleted and the round that dropped the project says the next round that can (sail-api within a minute, or `sudo sail sync`) will; a project kept for a run under way keeps both until the round after the run ends. It is decided from what the box holds, so a box an earlier release left with such a directory, container or roster row is brought in line by its first round on this release. An unchanged scoped round asks `incus list` once, and an unscoped one asks Incus nothing. `sail migrate` no longer imports the descriptor or files of a project the box's scope does not admit, so an upgrade no longer imports a dropped project only for the next round to drop it again. An open event stream (`/v1/events/stream`) or run-log stream (`/v1/runs/{id}/stream`) now asks its caller's authentication again, at most once a second, before an event or a heartbeat: a grant, a revoke or a demotion holds from that event on, and a caller removed, disabled or with its token or session revoked gets a last `event: error` naming why, then the stream ends. **Fleet note:** upgrade a scoped node before narrowing or revoking: a node of an earlier release drops the rows but keeps the roster rows and the container. diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java b/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java index eed67786d..551029ec0 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java @@ -45,14 +45,23 @@ private NodeRound() {} * target} serves the database this node, whose database is {@code db}, synced with: main set up * empty at the same address welcomes the node as readily as the main it replaced. The first * welcome that names a catalog is recorded as main's ({@link DatabaseCatalog#recordMain}). A main - * that names none predates catalogs and is not asked, unless this node recorded one: a database - * that has named its catalog always does. + * that names none predates catalogs, and could be any database at all, so it is refused too: + * asked to upgrade before this node has recorded a catalog, told it is another database after. */ public static void admit(SyncSession session, Sqlite db, String target) { var catalog = new DatabaseCatalog(db); var offered = session.catalog(); var synced = offered.map(catalog::recordMain).or(catalog::main); - if (synced.isEmpty() || synced.equals(offered)) { + if (synced.isEmpty()) { + throw new SyncTransportException( + "refused", + "Main at " + + target + + " names no database catalog, so this box cannot tell whether it is the database" + + " it synced with. Upgrade main first, then sync again.", + null); + } + if (synced.equals(offered)) { return; } throw new SyncTransportException( diff --git a/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java b/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java index 55bc92e35..c87f4410a 100644 --- a/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java +++ b/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java @@ -61,9 +61,14 @@ private static String refusal(String now) { } @Test - void aMainThatNamesNoCatalogIsNotAskedUntilTheNodeHasRecordedOne() { - admit(null); + void aMainThatNamesNoCatalogIsAskedToUpgradeBeforeTheNodeHasRecordedOne() { + var refused = assertThrows(SyncTransportException.class, () -> admit(null)); + assertEquals("refused", refused.kind()); + assertEquals( + "Main at sail@main names no database catalog, so this box cannot tell whether it is the" + + " database it synced with. Upgrade main first, then sync again.", + refused.getMessage()); assertEquals(Optional.empty(), new DatabaseCatalog(db).main()); } diff --git a/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java b/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java index 22c4e60cb..9fb6a53b7 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java +++ b/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java @@ -293,6 +293,34 @@ void aRefusalIsWhatTheNodeSaysEvenWhenMainHangsUpBeforeItsGoodbye() throws Excep assertEquals(1, refused.getSuppressed().length); } + @Test + void aMainThatNamesNoCatalogIsRefusedBeforeAnythingMovesEvenBeforeOneIsRecorded() + throws Exception { + round(); + nodeDb.execute("UPDATE catalog SET main = NULL"); + write("g2"); + var before = held(nodeDb); + var older = + new SyncWire.Welcome( + SyncWire.PROTOCOL, "0.53.0", "main", "nia", 0, ProjectScope.all(), null); + answering = + (in, out) -> { + SyncWire.readFramed(in); + in.close(); + out.write((SyncWire.encode(older) + "\n").getBytes(StandardCharsets.UTF_8)); + out.flush(); + }; + + var refused = assertThrows(SyncTransportException.class, this::round); + + assertEquals("refused", refused.kind()); + assertEquals( + "Main at sail@main names no database catalog, so this box cannot tell whether it is the" + + " database it synced with. Upgrade main first, then sync again.", + refused.getMessage()); + assertEquals(before, held(nodeDb)); + } + private static SpecStore.SpecRow spec(String id) { return new SpecStore.SpecRow( id, From 3f6e33636cec694c5fa08d55f8a6441d09313217 Mon Sep 17 00:00:00 2001 From: Uday Chandra Date: Tue, 20 Oct 2026 23:29:29 +0000 Subject: [PATCH 3/4] Word the catalog-less refusal as every upgrade-main remedy is --- ARCHITECTURE.md | 2 +- CHANGELOG.md | 2 +- sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java | 2 +- .../src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java | 2 +- .../test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c56917922..1f268a260 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -83,7 +83,7 @@ syncs with and refuses any other before anything moves (`NodeRound.admit`): "Mai not the database this box synced with (catalog , now ). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again." A main older than catalogs names none and could be any database, so it is refused too, before anything moves: "Main at names no -database catalog, so this box cannot tell whether it is the database it synced with. Upgrade main +database catalog, so this box cannot tell whether it is the database it synced with: upgrade main first, then sync again." So a main set up again from a copy (`sudo sail init --as-main`, the copy put in place of its new database, then `sudo sail migrate`, which also restores the sail user's keys from it) is the same main to every node, which then pushes diff --git a/CHANGELOG.md b/CHANGELOG.md index ddb2c6dd1..b8cb6c195 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.54.0 -- **A node never says "in sync" with a main that isn't the one it synced with.** Field rehearsal round 4 deleted main's VM and set it up again empty at the same IP; the node re-pinned, main ran the `fde add` line, and `sudo sail sync` printed "✓ Already in sync with main." while the new main held only `demo` and the node `gamma`, so a spec the node then filed under `gamma` was pushed into a main that holds no such project. Every database now carries a catalog id (`DatabaseCatalog`), random, minted with the database (an existing one at its upgrade) and kept by every upgrade and every copy of the file, and main's welcome names it. A node records the catalog of the first main database that welcomes it (a node from before this release on its next round) and refuses any other before anything is pushed, adopted or recorded, with exit 1: "Main at is not the database this box synced with (catalog , now ). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again." A main older than this release names no catalog and could be any database, so it is refused too, with exit 1 and before anything moves: "Main at names no database catalog, so this box cannot tell whether it is the database it synced with. Upgrade main first, then sync again." **Fleet note:** upgrade main before its nodes; an upgraded node does not sync with a main on an older release. It is decided on the catalog, not the box id, so a new database under the old `host.yaml` is refused too instead of silently skipped below the node's old checkpoint. Main set up again from a copy of its database is the same main: its nodes push what they wrote since the copy, and "Already in sync with main." follows. The README and ARCHITECTURE.md say to back up `/var/lib/sail/sail.db` and how to restore it; a copy taken before this release is minted a new catalog when restored, so take one after upgrading. Every join now prints the key it pinned, "Pinned main's host key SHA256:…. Check it on main: sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub", including a re-join after a host-key rotation that then says "Main already has this box's key"; the changed-key recovery reads "If main was rebuilt from a copy of its database, remove the old key with '…', re-run '…' and compare the fingerprint it prints with main's (…); otherwise something between this box and main is answering for it."; and the re-join's probe of main prints nothing of its own (the stray "⚠ hello: Sync channel closed before main replied." line is gone). +- **A node never says "in sync" with a main that isn't the one it synced with.** Field rehearsal round 4 deleted main's VM and set it up again empty at the same IP; the node re-pinned, main ran the `fde add` line, and `sudo sail sync` printed "✓ Already in sync with main." while the new main held only `demo` and the node `gamma`, so a spec the node then filed under `gamma` was pushed into a main that holds no such project. Every database now carries a catalog id (`DatabaseCatalog`), random, minted with the database (an existing one at its upgrade) and kept by every upgrade and every copy of the file, and main's welcome names it. A node records the catalog of the first main database that welcomes it (a node from before this release on its next round) and refuses any other before anything is pushed, adopted or recorded, with exit 1: "Main at is not the database this box synced with (catalog , now ). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again." A main older than this release names no catalog and could be any database, so it is refused too, with exit 1 and before anything moves: "Main at names no database catalog, so this box cannot tell whether it is the database it synced with: upgrade main first, then sync again." **Fleet note:** upgrade main before its nodes; an upgraded node does not sync with a main on an older release. It is decided on the catalog, not the box id, so a new database under the old `host.yaml` is refused too instead of silently skipped below the node's old checkpoint. Main set up again from a copy of its database is the same main: its nodes push what they wrote since the copy, and "Already in sync with main." follows. The README and ARCHITECTURE.md say to back up `/var/lib/sail/sail.db` and how to restore it; a copy taken before this release is minted a new catalog when restored, so take one after upgrading. Every join now prints the key it pinned, "Pinned main's host key SHA256:…. Check it on main: sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub", including a re-join after a host-key rotation that then says "Main already has this box's key"; the changed-key recovery reads "If main was rebuilt from a copy of its database, remove the old key with '…', re-run '…' and compare the fingerprint it prints with main's (…); otherwise something between this box and main is answering for it."; and the re-join's probe of main prints nothing of its own (the stray "⚠ hello: Sync channel closed before main replied." line is gone). - **Rows written together order as minted on a clock of any resolution.** Cutting v0.53.0, the macOS release job failed `ReviewLoopEveryStateTest` once, as v0.48.0 did. An instant is stored as its text, which drops trailing zeros, so `…51.123Z` sorts after the later `…51.123400Z` and `…51Z` after `…51.001Z`; a microsecond clock lands on such a value one time in a thousand. The stores ordered runs and reviews by that text, so the review loop could read a fix run as newer than the reviewer launched after it and ignore that reviewer's stop. Runs and reviews now order by their UUIDv7 id, as each box minted them; stored instants are unchanged and nothing migrates. The review loop fixture's fixed-width clock is gone. - **A box that stops being a node keeps nothing of the node it was.** The delta review of `sail-sync-is-exact` found two ways a box still carried node state after it stopped being one. Re-running `sudo sail host init` rewrote `host.yaml` from scratch, dropping the sync role, main, handle, box id and passkey settings, so a node or main turned standalone and orphaned every checkpoint its peers held for it; a re-run now refreshes only the fields it provisions and keeps the rest of `host.yaml`, read again under the box's round lock just before it writes, so a `host sync`, `join` or handle change that finished while the packages installed is kept, not undone. And a main promoted on v0.49.0 to v0.52.0, or one whose role change died between writing `host.yaml` and clearing its scope, kept the scope it last heard as a node, refusing its own `spec create` and its nodes' pushes into every other project and skipping its own descriptors at upgrade; `sail migrate` and every `sail server start` now clear that scope on any box that is not a node, and only a whole `host.yaml` decides it: one absent, empty or unreadable at that moment (truncated by a writer, or being moved into `/var/lib/sail`) keeps the scope for the next start to decide, so a node never loses its grant to a file in motion. `sudo sail host update` and `sail migrate`'s box-id and `host.yaml` relocation steps now read and write `host.yaml` under that lock too, and `host update` records the Incus version on the file as it is then, not as it was before the packages upgraded, so an identity change made meanwhile is kept. - **A scope can change after an FDE exists, and every box follows it whole.** Field rehearsal 2026-10-15 found no way to scope an FDE that joined unscoped short of `fde rm` and a re-add (which revoked its tokens, keys, sessions and passkeys), a scoped node still holding the names and emails of FDEs outside its projects, and a revoked project's container running on the node with its clone and credentials, still listed by `sail project list`. **`sudo sail fde update --scoped [--grant ]...`** narrows an FDE in place, in one transaction with the grants it names (`--grant` is repeatable and takes `a,b`), and **`--unscoped`** lets it see every project again and forgets its grants, so a later narrowing grants only what it names; the FDE keeps its id and every credential. Both take `--dry-run` (the real run, rolled back) and `--json` (`handle`, `scope`, `projects`, `changed`, `dry_run`), are refused on a node with the roster sentence, and refuse this box's own FDE and an admin as `fde add --scoped` does. A narrowing says what the FDE's box drops ("Scoped eng to acme. Its box drops beta, demo with their containers and unpushed work on its next sync."), a widening that its box holds every project after its next sync. `fde grant` on an unscoped FDE now names `sudo sail fde update --scoped`. A narrowing is a revocation on the node and a widening a grant, so the node drops and re-adopts as before. A scoped node now holds exactly the roster main lists to it: an FDE main stops listing is removed with its box-local credentials ("roster: removed 1 FDE(s) main no longer lists to this box"), the box's own FDE never; an unscoped node still disables such an FDE and keeps its row. Every round of a scoped node then takes off the box each project the scope does not admit and the box holds nothing of: its `~/.sail/projects/` directory, and its container when Sail provisioned it (the `sail-api-sock` device, the trust gate `project apply --all` uses), as `sudo sail project destroy` without `--purge` would ("project beta: its container deleted, workspace and uncommitted work included — main no longer grants this box the project"). A container Sail did not provision is left alone and named once; when the round cannot reach Incus (a non-root `sail sync`) nothing is deleted and the round that dropped the project says the next round that can (sail-api within a minute, or `sudo sail sync`) will; a project kept for a run under way keeps both until the round after the run ends. It is decided from what the box holds, so a box an earlier release left with such a directory, container or roster row is brought in line by its first round on this release. An unchanged scoped round asks `incus list` once, and an unscoped one asks Incus nothing. `sail migrate` no longer imports the descriptor or files of a project the box's scope does not admit, so an upgrade no longer imports a dropped project only for the next round to drop it again. An open event stream (`/v1/events/stream`) or run-log stream (`/v1/runs/{id}/stream`) now asks its caller's authentication again, at most once a second, before an event or a heartbeat: a grant, a revoke or a demotion holds from that event on, and a caller removed, disabled or with its token or session revoked gets a last `event: error` naming why, then the stream ends. **Fleet note:** upgrade a scoped node before narrowing or revoking: a node of an earlier release drops the rows but keeps the roster rows and the container. diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java b/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java index 551029ec0..93cc51ed4 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java @@ -58,7 +58,7 @@ public static void admit(SyncSession session, Sqlite db, String target) { "Main at " + target + " names no database catalog, so this box cannot tell whether it is the database" - + " it synced with. Upgrade main first, then sync again.", + + " it synced with: upgrade main first, then sync again.", null); } if (synced.equals(offered)) { diff --git a/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java b/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java index c87f4410a..07d46d617 100644 --- a/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java +++ b/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java @@ -67,7 +67,7 @@ void aMainThatNamesNoCatalogIsAskedToUpgradeBeforeTheNodeHasRecordedOne() { assertEquals("refused", refused.kind()); assertEquals( "Main at sail@main names no database catalog, so this box cannot tell whether it is the" - + " database it synced with. Upgrade main first, then sync again.", + + " database it synced with: upgrade main first, then sync again.", refused.getMessage()); assertEquals(Optional.empty(), new DatabaseCatalog(db).main()); } diff --git a/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java b/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java index 9fb6a53b7..01c251a39 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java +++ b/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java @@ -316,7 +316,7 @@ void aMainThatNamesNoCatalogIsRefusedBeforeAnythingMovesEvenBeforeOneIsRecorded( assertEquals("refused", refused.kind()); assertEquals( "Main at sail@main names no database catalog, so this box cannot tell whether it is the" - + " database it synced with. Upgrade main first, then sync again.", + + " database it synced with: upgrade main first, then sync again.", refused.getMessage()); assertEquals(before, held(nodeDb)); } From bf37dffcd889ca2e90bed90a8efbc48153374eea Mon Sep 17 00:00:00 2001 From: Uday Chandra Date: Wed, 21 Oct 2026 00:54:14 +0000 Subject: [PATCH 4/4] A main restored in place from an older copy is paged from the copy, and a box leaving the node role forgets main's catalog The delta review found two gaps. A main restored in place keeps its box id, so its log sat below the node's checkpoint: the node paged nothing and said it was in sync while main held rows written after the restore. A log below the checkpoint now forgets the checkpoints and walks every head again. A box promoted to main, or migrated as main or standalone, kept the catalog it recorded as a node and would refuse any other main forever with the wrong remedy; it forgets it with its scope. A blank catalog is refused at the record, the catalog-less refusal names the upgrade command as the protocol refusal does, main's database path comes from SailPaths, and the docs no longer promise that a restore recovers work main took and lost. --- ARCHITECTURE.md | 25 +++++++------- CHANGELOG.md | 2 +- README.md | 7 ++-- .../ai/singlr/sail/store/DatabaseCatalog.java | 8 ++++- .../ai/singlr/sail/store/MigrationRunner.java | 1 + .../ai/singlr/sail/store/SchemaManager.java | 2 +- .../ai/singlr/sail/sync/LocalReplica.java | 8 +++++ .../java/ai/singlr/sail/sync/NodeRound.java | 8 +++-- .../ai/singlr/sail/sync/PagedSyncSession.java | 4 +++ .../singlr/sail/sync/ScopedLocalReplica.java | 5 +++ .../ai/singlr/sail/sync/StoreReplica.java | 5 +++ .../sail/store/DatabaseCatalogTest.java | 25 ++++++++++++++ .../singlr/sail/sync/NodeRoundAdmitTest.java | 3 +- .../singlr/sail/sync/SyncTransportTest.java | 5 +++ .../ai/singlr/sail/engine/HandleChange.java | 2 ++ .../sail/commands/MainRebuiltRoundTest.java | 33 ++++++++++++++++++- .../ai/singlr/sail/commands/SyncCliTest.java | 4 +++ .../singlr/sail/engine/HandleChangeTest.java | 9 +++-- 18 files changed, 133 insertions(+), 23 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1f268a260..e9512a12e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -75,21 +75,24 @@ node` with `main: `, or unset for standalone. `sail host sync --as-m `sail host sync --main ` set it. Main never initiates. Nodes pull from and push to main. Main failover is manual and out of scope for v1. -Main's database is the org. Back up `/var/lib/sail/sail.db` (`sqlite3 /var/lib/sail/sail.db -".backup "`, or the file with sail-api stopped). Every database carries a catalog id +Main's database is the org. Back up `/var/lib/sail/sail.db` (`sqlite3 /var/lib/sail/sail.db ".backup +"`, or the file with sail-api stopped). Every database carries a catalog id (`DatabaseCatalog`), random, minted with the database and kept by every upgrade and every copy of -the file, and main's welcome names it. A node records the catalog of the first main database it -syncs with and refuses any other before anything moves (`NodeRound.admit`): "Main at is +the file, and main's welcome names it. A node records the catalog of the first main database that +welcomes it and refuses any other before anything moves (`NodeRound.admit`): "Main at is not the database this box synced with (catalog , now ). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again." A main older than catalogs names none and could be any database, so it is refused too, before anything moves: "Main at names no database catalog, so this box cannot tell whether it is the database it synced with: upgrade main -first, then sync again." So a main set up again from a copy (`sudo -sail init --as-main`, the copy put in place of its new database, then `sudo sail migrate`, which -also restores the sail user's keys from it) is the same main to every node, which then pushes -whatever it wrote since the copy; a main set up empty is a new org, and its nodes refuse it. A -copy taken before the release that mints catalog ids is minted a new one when restored, so take a -fresh copy after upgrading. Main is never rebuilt from its nodes. +('sudo sail upgrade' on main), then sync again." So a main set up again from a copy (`sudo sail init +--as-main`, the copy put in place of its new database, then `sudo sail migrate`, which also restores +the sail user's keys from it) is the same main to every node, which then pushes whatever it has not +pushed yet (work main took after the copy is lost with it); a main set up empty is a new org, and +its nodes refuse it. A main restored in place from an older copy keeps its box id, so its log sits +below the node's checkpoint: the node then walks every head again from the copy instead of reading +nothing and saying it is in sync. A box that stops being a node forgets main's catalog as it forgets +its scope. A copy taken before the release that mints catalog ids is minted a new one when restored, +so take a fresh copy after upgrading. Main is never rebuilt from its nodes. ## Module layout @@ -203,8 +206,8 @@ and the spec that closes it. | I1 | Every write on every box is made by exactly one bound `Actor`, whichever door it came through. | `ActorTest`, `OperationsTakeNoActorTest` | | I2 | On a node only the box's FDE, its runs and FDE-less credentials write; any other credential reads. | `RoleRuleTest`, `NodeWritesTest`, `DeniedSyncTest` | | I3 | Main and the node agree who the node is, and one box syncs as each FDE. | `NodeIdentitySyncTest`, `SyncServerCommandTest` (main's own FDE), `FdeCommandTest` (`release-box`) | -| I5 | A node syncs only with the main database it first synced with: a welcome naming another catalog, or none at all, ends the round before any row moves, and "Already in sync" is said only after a round with the recorded one. | `DatabaseCatalogTest` (minted per database, kept by upgrades and copies, the first main recorded), `SchemaManagerTest` (a database from before catalogs), `SyncWireTest`, `NodeRoundAdmitTest`, `MainRebuiltRoundTest` (an empty main under a new or the same box id refused with nothing moved either way; a main that names no catalog refused with nothing moved before the node recorded one; a main restored from a copy takes the node's newer work; a node from before catalogs records main's), `NativeFleetIT` (an empty main set up again at the node's address; one restored from a copy) | | I4 | Every run a box executes carries that box's handle as `node` and `owner`. | `BoxRunsSyncTest`, `RunAuthorityTest`, `RunStoreTest` (stamps; never another box's run), `SyncConfigTest` (one handle), `HandleChangeTest`, `JoinCommandTest`, `HostConfigSetCommandTest`, `HostSyncCommandTest`, `RoomWakeLaunchTest` | +| I5 | A node syncs only with the main database it first synced with: a welcome naming another catalog, or none at all, ends the round before any row moves, "Already in sync" is said only after a round with the recorded one, and a log below the node's checkpoint is walked again from its first head. | `DatabaseCatalogTest` (minted per database, kept by upgrades and copies, the first main recorded), `SchemaManagerTest` (a database from before catalogs), `SyncWireTest`, `NodeRoundAdmitTest`, `MainRebuiltRoundTest` (an empty main under a new or the same box id refused with nothing moved either way; a main that names no catalog refused with nothing moved before the node recorded one; a main restored from a copy takes the node's newer work; one restored in place from an older copy is paged from the copy; a node from before catalogs records main's), `HandleChangeTest` and `SyncCliTest` (a box that stops being a node forgets main's catalog), `NativeFleetIT` (an empty main set up again at the node's address; one restored from a copy) | | A1 | Who may write a synced row is one rule per type in sail-core; the same write gets the same refusal kind and code at every door and on main. | the rules: `SpecAuthorityTest`, `RoomAuthorityTest`, `ReviewAuthorityTest`, `RunAuthorityTest` (a run is created only for the owner of the spec or room it names, on every lane), `MessageAuthorityTest`, `WriterAuthorityTest`, `ProjectAuthorityTest` (a rename is an admin's); the doors: `NodeWritesTest` (one refusal — kind, code, message and fix — through an API token, a passkey session, a gateway session, the box credential, a run and a room run; the terminal keeps its own text), `AuthorityAuditTest` (a viewer's edit over HTTP as on the host CLI; a room's delete and its membership; a spec born over its owner's room at the door as on main; a merged resolve), `SailOperationsSeamTest` (a resolve: adopting main's side decides nothing, keeping or merging is an edit, on the socket and both web lanes), `CatalogAuthorityAuditTest` (the host CLI's catalog edit, project files and rename), `AdhocDispatchTest` (ad-hoc runs), `OneDecisionTest` (spec edits, posts); main: `PushAuthoritySyncTest` (a pushed run on another FDE's spec is denied, and so is one naming its own spec and another member's room; one ahead of a spec its box made lands), `DeniedSyncTest` | | A2 | No code path writes a synced row without its rule deciding it: a pushed revision at main's commit, every other write at the journal, each exactly once. | `JournalDecidesEveryWriteTest` (every public store method that reaches the journal, read from the stores' bytecode, is refused on the CLI, API, agent and room lanes and leaves nothing; a revision never admits itself; a refusal swallowed inside a transaction still rolls it back; a pushed revision is not decided twice), `OneDecisionPointTest` (in every module's classes only the journal and the message store append a revision, only they ask a write rule, and only main's commit takes a store's rule), `RevisionJournalTest`, `SqliteTest` (a doomed transaction); an operation's rows commit together or not at all: `GlobalSpecOperationsTest` (an update with its wake; the move to done; a delete), `ReviewOperationsTest` (a follow-up with its links), `BuildDispatchCoverageTest` (a dispatch and a restart: reservation, reset, claim, supersede), `SpecPruneTest` (a restore with its room) | | A3 | An event, hook or reactor never makes the machinery do what its sender could not. | the rule: `EventAuthorityTest` (every listed type and rule; every unlisted type refused; a run decides what its event is about and its spec whose it is; only this box's FDE reports what this box observed); both doors with the real subscribers behind them: `EventDoorTest` (a member's stop, completion, failure and review evidence for another member's spec, run or room never reach the bus, so the spec stays, the run stays and the reconciler still rescues; a run a member pushed is no key to another member's spec; an owner is not taken at their word for what this box observed; an owner's retelling and an admin's report land; a run publishes only its hooks; the server's clock, publisher and stored message replace the sender's); every real publisher through its own code path: `EventPublishersTest` (the host CLI's dispatch, restart, ad-hoc run and stop, the watcher's stop, a node's sync announcement, main's sync bridge relaying as the FDE who pushed, and refusing a pushed run on another member's spec); the socket's scoping to the credential's run: `LocalApiRouterTest`, `AgentPrincipalLifecycleTest` | diff --git a/CHANGELOG.md b/CHANGELOG.md index b8cb6c195..39ef7ec98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.54.0 -- **A node never says "in sync" with a main that isn't the one it synced with.** Field rehearsal round 4 deleted main's VM and set it up again empty at the same IP; the node re-pinned, main ran the `fde add` line, and `sudo sail sync` printed "✓ Already in sync with main." while the new main held only `demo` and the node `gamma`, so a spec the node then filed under `gamma` was pushed into a main that holds no such project. Every database now carries a catalog id (`DatabaseCatalog`), random, minted with the database (an existing one at its upgrade) and kept by every upgrade and every copy of the file, and main's welcome names it. A node records the catalog of the first main database that welcomes it (a node from before this release on its next round) and refuses any other before anything is pushed, adopted or recorded, with exit 1: "Main at is not the database this box synced with (catalog , now ). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again." A main older than this release names no catalog and could be any database, so it is refused too, with exit 1 and before anything moves: "Main at names no database catalog, so this box cannot tell whether it is the database it synced with: upgrade main first, then sync again." **Fleet note:** upgrade main before its nodes; an upgraded node does not sync with a main on an older release. It is decided on the catalog, not the box id, so a new database under the old `host.yaml` is refused too instead of silently skipped below the node's old checkpoint. Main set up again from a copy of its database is the same main: its nodes push what they wrote since the copy, and "Already in sync with main." follows. The README and ARCHITECTURE.md say to back up `/var/lib/sail/sail.db` and how to restore it; a copy taken before this release is minted a new catalog when restored, so take one after upgrading. Every join now prints the key it pinned, "Pinned main's host key SHA256:…. Check it on main: sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub", including a re-join after a host-key rotation that then says "Main already has this box's key"; the changed-key recovery reads "If main was rebuilt from a copy of its database, remove the old key with '…', re-run '…' and compare the fingerprint it prints with main's (…); otherwise something between this box and main is answering for it."; and the re-join's probe of main prints nothing of its own (the stray "⚠ hello: Sync channel closed before main replied." line is gone). +- **A node never says "in sync" with a main that isn't the one it synced with.** Field rehearsal round 4 deleted main's VM and set it up again empty at the same IP; the node re-pinned, main ran the `fde add` line, and `sudo sail sync` printed "✓ Already in sync with main." while the new main held only `demo` and the node `gamma`, so a spec the node then filed under `gamma` was pushed into a main that holds no such project. Every database now carries a catalog id (`DatabaseCatalog`), random, minted with the database (an existing one at its upgrade) and kept by every upgrade and every copy of the file, and main's welcome names it. A node records the catalog of the first main database that welcomes it (a node from before this release on its next round) and refuses any other before anything is pushed, adopted or recorded, with exit 1: "Main at is not the database this box synced with (catalog , now ). Restore main's database (/var/lib/sail/sail.db) from a copy, then sync again." A main older than this release names no catalog and could be any database, so it is refused too, with exit 1 and before anything moves: "Main at names no database catalog, so this box cannot tell whether it is the database it synced with: upgrade main ('sudo sail upgrade' on main), then sync again." **Fleet note:** upgrade main before its nodes; an upgraded node does not sync with a main on an older release. It is decided on the catalog, not the box id, so a new database under the old `host.yaml` is refused too instead of silently skipped below the node's old checkpoint. Main set up again from a copy of its database is the same main: its nodes push what they wrote since the copy, and "Already in sync with main." follows. The README and ARCHITECTURE.md say to back up `/var/lib/sail/sail.db` and how to restore it; a copy taken before this release is minted a new catalog when restored, so take one after upgrading. Every join now prints the key it pinned, "Pinned main's host key SHA256:…. Check it on main: sudo ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub", including a re-join after a host-key rotation that then says "Main already has this box's key"; the changed-key recovery reads "If main was rebuilt from a copy of its database, remove the old key with '…', re-run '…' and compare the fingerprint it prints with main's (…); otherwise something between this box and main is answering for it."; and the re-join's probe of main prints nothing of its own (the stray "⚠ hello: Sync channel closed before main replied." line is gone). A main restored in place from an older copy of its database keeps its box id, so its log sits below the node's checkpoint; the node then walks every head again from the copy instead of paging nothing and saying it is in sync. A box that stops being a node forgets main's catalog as it forgets its scope. - **Rows written together order as minted on a clock of any resolution.** Cutting v0.53.0, the macOS release job failed `ReviewLoopEveryStateTest` once, as v0.48.0 did. An instant is stored as its text, which drops trailing zeros, so `…51.123Z` sorts after the later `…51.123400Z` and `…51Z` after `…51.001Z`; a microsecond clock lands on such a value one time in a thousand. The stores ordered runs and reviews by that text, so the review loop could read a fix run as newer than the reviewer launched after it and ignore that reviewer's stop. Runs and reviews now order by their UUIDv7 id, as each box minted them; stored instants are unchanged and nothing migrates. The review loop fixture's fixed-width clock is gone. - **A box that stops being a node keeps nothing of the node it was.** The delta review of `sail-sync-is-exact` found two ways a box still carried node state after it stopped being one. Re-running `sudo sail host init` rewrote `host.yaml` from scratch, dropping the sync role, main, handle, box id and passkey settings, so a node or main turned standalone and orphaned every checkpoint its peers held for it; a re-run now refreshes only the fields it provisions and keeps the rest of `host.yaml`, read again under the box's round lock just before it writes, so a `host sync`, `join` or handle change that finished while the packages installed is kept, not undone. And a main promoted on v0.49.0 to v0.52.0, or one whose role change died between writing `host.yaml` and clearing its scope, kept the scope it last heard as a node, refusing its own `spec create` and its nodes' pushes into every other project and skipping its own descriptors at upgrade; `sail migrate` and every `sail server start` now clear that scope on any box that is not a node, and only a whole `host.yaml` decides it: one absent, empty or unreadable at that moment (truncated by a writer, or being moved into `/var/lib/sail`) keeps the scope for the next start to decide, so a node never loses its grant to a file in motion. `sudo sail host update` and `sail migrate`'s box-id and `host.yaml` relocation steps now read and write `host.yaml` under that lock too, and `host update` records the Incus version on the file as it is then, not as it was before the packages upgraded, so an identity change made meanwhile is kept. - **A scope can change after an FDE exists, and every box follows it whole.** Field rehearsal 2026-10-15 found no way to scope an FDE that joined unscoped short of `fde rm` and a re-add (which revoked its tokens, keys, sessions and passkeys), a scoped node still holding the names and emails of FDEs outside its projects, and a revoked project's container running on the node with its clone and credentials, still listed by `sail project list`. **`sudo sail fde update --scoped [--grant ]...`** narrows an FDE in place, in one transaction with the grants it names (`--grant` is repeatable and takes `a,b`), and **`--unscoped`** lets it see every project again and forgets its grants, so a later narrowing grants only what it names; the FDE keeps its id and every credential. Both take `--dry-run` (the real run, rolled back) and `--json` (`handle`, `scope`, `projects`, `changed`, `dry_run`), are refused on a node with the roster sentence, and refuse this box's own FDE and an admin as `fde add --scoped` does. A narrowing says what the FDE's box drops ("Scoped eng to acme. Its box drops beta, demo with their containers and unpushed work on its next sync."), a widening that its box holds every project after its next sync. `fde grant` on an unscoped FDE now names `sudo sail fde update --scoped`. A narrowing is a revocation on the node and a widening a grant, so the node drops and re-adopts as before. A scoped node now holds exactly the roster main lists to it: an FDE main stops listing is removed with its box-local credentials ("roster: removed 1 FDE(s) main no longer lists to this box"), the box's own FDE never; an unscoped node still disables such an FDE and keeps its row. Every round of a scoped node then takes off the box each project the scope does not admit and the box holds nothing of: its `~/.sail/projects/` directory, and its container when Sail provisioned it (the `sail-api-sock` device, the trust gate `project apply --all` uses), as `sudo sail project destroy` without `--purge` would ("project beta: its container deleted, workspace and uncommitted work included — main no longer grants this box the project"). A container Sail did not provision is left alone and named once; when the round cannot reach Incus (a non-root `sail sync`) nothing is deleted and the round that dropped the project says the next round that can (sail-api within a minute, or `sudo sail sync`) will; a project kept for a run under way keeps both until the round after the run ends. It is decided from what the box holds, so a box an earlier release left with such a directory, container or roster row is brought in line by its first round on this release. An unchanged scoped round asks `incus list` once, and an unscoped one asks Incus nothing. `sail migrate` no longer imports the descriptor or files of a project the box's scope does not admit, so an upgrade no longer imports a dropped project only for the next round to drop it again. An open event stream (`/v1/events/stream`) or run-log stream (`/v1/runs/{id}/stream`) now asks its caller's authentication again, at most once a second, before an event or a heartbeat: a grant, a revoke or a demotion holds from that event on, and a caller removed, disabled or with its token or session revoked gets a last `event: error` naming why, then the stream ends. **Fleet note:** upgrade a scoped node before narrowing or revoking: a node of an earlier release drops the rows but keeps the roster rows and the container. diff --git a/README.md b/README.md index a86831ad6..de60a36fa 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,10 @@ sudo systemctl start sail-api Each node then re-pins main's new host key (`sudo ssh-keygen -R `, then `sudo sail init --main `; compare the fingerprint it prints with `sudo ssh-keygen -lf -/etc/ssh/ssh_host_ed25519_key.pub` on main), and its next `sudo sail sync` pushes whatever it wrote -since the copy. A main set up empty is a new org: every node refuses to sync with it. A copy taken -before 0.54.0 counts as a new org too, so take a fresh one after upgrading. +/etc/ssh/ssh_host_ed25519_key.pub` on main), and its next `sudo sail sync` pushes whatever it has +not pushed yet; work main took after the copy and lost with it is not rebuilt from the node. A main +set up empty is a new org: every node refuses to sync with it. A copy taken before 0.54.0 counts as +a new org too, so take a fresh one after upgrading. Mast, the desktop client, is a thin client of a box: specs, rooms and reviews over its API and event stream, terminals and files over its pty host. diff --git a/sail-core/src/main/java/ai/singlr/sail/store/DatabaseCatalog.java b/sail-core/src/main/java/ai/singlr/sail/store/DatabaseCatalog.java index 4b25533cc..28a0013d3 100644 --- a/sail-core/src/main/java/ai/singlr/sail/store/DatabaseCatalog.java +++ b/sail-core/src/main/java/ai/singlr/sail/store/DatabaseCatalog.java @@ -5,6 +5,7 @@ package ai.singlr.sail.store; +import ai.singlr.sail.common.Strings; import java.util.Objects; import java.util.Optional; @@ -39,8 +40,13 @@ public Optional main() { * recorded: {@code offered} itself only when it is the database this node synced with. */ public String recordMain(String offered) { - Objects.requireNonNull(offered, "offered"); + Strings.requireNonBlank(offered, "catalog"); db.execute("UPDATE catalog SET main = ? WHERE main IS NULL", offered); return main().orElseThrow(); } + + /** Forgets main's catalog: a box that is no longer a node synced with no main database. */ + public void forgetMain() { + db.execute("UPDATE catalog SET main = NULL"); + } } diff --git a/sail-core/src/main/java/ai/singlr/sail/store/MigrationRunner.java b/sail-core/src/main/java/ai/singlr/sail/store/MigrationRunner.java index 60684dae6..4dcc5c81c 100644 --- a/sail-core/src/main/java/ai/singlr/sail/store/MigrationRunner.java +++ b/sail-core/src/main/java/ai/singlr/sail/store/MigrationRunner.java @@ -49,6 +49,7 @@ public static Result applyAll( private static void forgetNodeScope(Sqlite db) { if (NodeIdentity.declaresNotNode()) { new SyncScope(db).record(ProjectScope.all()); + new DatabaseCatalog(db).forgetMain(); } } diff --git a/sail-core/src/main/java/ai/singlr/sail/store/SchemaManager.java b/sail-core/src/main/java/ai/singlr/sail/store/SchemaManager.java index 6436f2b95..c8243695b 100644 --- a/sail-core/src/main/java/ai/singlr/sail/store/SchemaManager.java +++ b/sail-core/src/main/java/ai/singlr/sail/store/SchemaManager.java @@ -255,7 +255,7 @@ PRIMARY KEY (fde_id, project) /** * Which database this is: a random id minted with the database and kept by every upgrade and * every copy of its file, and, on a node, the catalog of the main database it syncs with, - * recorded by its first round ({@link DatabaseCatalog}). + * recorded from the first welcome that names one ({@link DatabaseCatalog}). */ static final String CATALOG = "CREATE TABLE catalog AS SELECT lower(hex(randomblob(16))) AS id, NULL AS main"; diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/LocalReplica.java b/sail-core/src/main/java/ai/singlr/sail/sync/LocalReplica.java index cd8e2206b..ba32d89b5 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/LocalReplica.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/LocalReplica.java @@ -157,6 +157,14 @@ void recordConflict( /** Advances the checkpoint for {@code peerId} to main's high-water sequence. */ void advanceCheckpoint(String peerId, long seq); + /** + * Forgets every checkpoint this node keeps for {@code peerId}, so the round walks main's heads + * from the first again: main's log sits below what this node read, which is main restored in + * place from an older copy of its database, and what it wrote since the copy would otherwise stay + * unread below the checkpoint. What the node already holds re-adopts at the same rev. + */ + void forgetCheckpoints(String peerId); + /** * This replica narrowed to {@code ids}: {@link #entityIds} answers exactly them and every other * operation reaches through to this replica. How one page of main's change log is reconciled as diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java b/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java index 93cc51ed4..2bebd7e95 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/NodeRound.java @@ -7,6 +7,7 @@ import ai.singlr.sail.common.Strings; import ai.singlr.sail.engine.OperatorCommand; +import ai.singlr.sail.engine.SailPaths; import ai.singlr.sail.identity.Actor; import ai.singlr.sail.identity.ProjectScope; import ai.singlr.sail.store.ChangeLog; @@ -19,6 +20,7 @@ import ai.singlr.sail.store.SyncLimits; import ai.singlr.sail.store.SyncScope; import ai.singlr.sail.store.SyncState; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -36,7 +38,7 @@ public final class NodeRound { /** Where main keeps its database: the org, which every node's catalog names. */ - private static final String MAIN_DATABASE = "/var/lib/sail/sail.db"; + private static final Path MAIN_DATABASE = SailPaths.systemDataDir().resolve("sail.db"); private NodeRound() {} @@ -58,7 +60,9 @@ public static void admit(SyncSession session, Sqlite db, String target) { "Main at " + target + " names no database catalog, so this box cannot tell whether it is the database" - + " it synced with: upgrade main first, then sync again.", + + " it synced with: upgrade main ('" + + OperatorCommand.elsewhere("upgrade") + + "' on main), then sync again.", null); } if (synced.equals(offered)) { diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/PagedSyncSession.java b/sail-core/src/main/java/ai/singlr/sail/sync/PagedSyncSession.java index 819b19ce6..9283d38be 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/PagedSyncSession.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/PagedSyncSession.java @@ -280,6 +280,10 @@ private TypeReport reconcileType(String type, LocalReplica local) { } acknowledgeLostAnswers(type, local); var since = local.checkpoint(mainId); + if (tip < since) { + local.forgetCheckpoints(mainId); + since = 0; + } var report = SyncEngine.Report.NONE; var pages = 0; var entries = 0; diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/ScopedLocalReplica.java b/sail-core/src/main/java/ai/singlr/sail/sync/ScopedLocalReplica.java index ee50c1625..dcd4938d5 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/ScopedLocalReplica.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/ScopedLocalReplica.java @@ -138,4 +138,9 @@ public long checkpoint(String peerId) { public void advanceCheckpoint(String peerId, long seq) { inner.advanceCheckpoint(peerId, seq); } + + @Override + public void forgetCheckpoints(String peerId) { + inner.forgetCheckpoints(peerId); + } } diff --git a/sail-core/src/main/java/ai/singlr/sail/sync/StoreReplica.java b/sail-core/src/main/java/ai/singlr/sail/sync/StoreReplica.java index 471a6b2bb..455da5978 100644 --- a/sail-core/src/main/java/ai/singlr/sail/sync/StoreReplica.java +++ b/sail-core/src/main/java/ai/singlr/sail/sync/StoreReplica.java @@ -320,6 +320,11 @@ public void advanceCheckpoint(String peerId, long seq) { syncState.advance(peerId, store.entityType(), seq); } + @Override + public void forgetCheckpoints(String peerId) { + syncState.reset(peerId); + } + private Optional erasure(String entityId) { return changeLog.erasure(store.entityType(), entityId); } diff --git a/sail-core/src/test/java/ai/singlr/sail/store/DatabaseCatalogTest.java b/sail-core/src/test/java/ai/singlr/sail/store/DatabaseCatalogTest.java index 44c85b546..8f104ed5c 100644 --- a/sail-core/src/test/java/ai/singlr/sail/store/DatabaseCatalogTest.java +++ b/sail-core/src/test/java/ai/singlr/sail/store/DatabaseCatalogTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.file.Path; @@ -57,4 +58,28 @@ void aNodeRecordsTheFirstMainCatalogItIsOfferedAndKeepsItWhateverItIsOfferedNext assertEquals(List.of(1L), db.query("SELECT count(*) FROM catalog", row -> row.integer(0))); } } + + @Test + void aBoxThatIsNoLongerANodeForgetsMainsCatalogAndRecordsTheNextItIsOffered() { + try (var db = migrated(dir.resolve("node.db"))) { + var catalog = new DatabaseCatalog(db); + catalog.recordMain("aaaa"); + + catalog.forgetMain(); + + assertEquals(Optional.empty(), catalog.main()); + assertEquals("bbbb", catalog.recordMain("bbbb")); + } + } + + @Test + void aBlankCatalogIsNeverRecorded() { + try (var db = migrated(dir.resolve("node.db"))) { + var catalog = new DatabaseCatalog(db); + + assertThrows(IllegalArgumentException.class, () -> catalog.recordMain(" ")); + + assertEquals(Optional.empty(), catalog.main()); + } + } } diff --git a/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java b/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java index 07d46d617..ccea37efb 100644 --- a/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java +++ b/sail-core/src/test/java/ai/singlr/sail/sync/NodeRoundAdmitTest.java @@ -67,7 +67,8 @@ void aMainThatNamesNoCatalogIsAskedToUpgradeBeforeTheNodeHasRecordedOne() { assertEquals("refused", refused.kind()); assertEquals( "Main at sail@main names no database catalog, so this box cannot tell whether it is the" - + " database it synced with: upgrade main first, then sync again.", + + " database it synced with: upgrade main ('sudo sail upgrade' on main), then sync" + + " again.", refused.getMessage()); assertEquals(Optional.empty(), new DatabaseCatalog(db).main()); } diff --git a/sail-core/src/test/java/ai/singlr/sail/sync/SyncTransportTest.java b/sail-core/src/test/java/ai/singlr/sail/sync/SyncTransportTest.java index 73854cb1b..54d80611a 100644 --- a/sail-core/src/test/java/ai/singlr/sail/sync/SyncTransportTest.java +++ b/sail-core/src/test/java/ai/singlr/sail/sync/SyncTransportTest.java @@ -381,6 +381,11 @@ public long checkpoint(String peer) { public void advanceCheckpoint(String peer, long seq) { inner.advanceCheckpoint(peer, seq); } + + @Override + public void forgetCheckpoints(String peer) { + inner.forgetCheckpoints(peer); + } } @Test diff --git a/sail-infra/src/main/java/ai/singlr/sail/engine/HandleChange.java b/sail-infra/src/main/java/ai/singlr/sail/engine/HandleChange.java index 52ab34438..902ec3ca0 100644 --- a/sail-infra/src/main/java/ai/singlr/sail/engine/HandleChange.java +++ b/sail-infra/src/main/java/ai/singlr/sail/engine/HandleChange.java @@ -11,6 +11,7 @@ import ai.singlr.sail.config.YamlUtil; import ai.singlr.sail.identity.ProjectScope; import ai.singlr.sail.store.BoxCredentialStore; +import ai.singlr.sail.store.DatabaseCatalog; import ai.singlr.sail.store.MigrationRunner; import ai.singlr.sail.store.RunStore; import ai.singlr.sail.store.Sqlite; @@ -175,6 +176,7 @@ private static List change( write.run(); if (leftNode) { new SyncScope(db).record(ProjectScope.all()); + new DatabaseCatalog(db).forgetMain(); } return after.isMain() ? runs.stampUnstamped(after.handle()) diff --git a/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java b/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java index 01c251a39..b1f237153 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java +++ b/sail-infra/src/test/java/ai/singlr/sail/commands/MainRebuiltRoundTest.java @@ -159,6 +159,14 @@ private void write(String id) { Acting.as("nia", () -> new SpecStore(nodeDb).create(spec(id))); } + private static void mainWrite(Main main, String id) { + Acting.as("nia", () -> new SpecStore(main.db()).create(spec(id))); + } + + /** + * The synced rows a database holds, its log and its checkpoints. Main's own record of the box + * that said hello ({@code fde_boxes}) is local to main and never synced, so it is not here. + */ private static List held(Sqlite db) { var held = new ArrayList(); held.addAll(db.query("SELECT 'project ' || name FROM projects ORDER BY name", r -> r.text(0))); @@ -257,6 +265,28 @@ void aMainRebuiltFromACopyOfItsDatabaseTakesTheNodesNewerWork() throws Exception assertEquals(" ✓ Already in sync with main.", rendered(round())); } + @Test + void aMainRestoredInPlaceFromAnOlderCopyIsPagedFromTheCopyNotFromTheNodesOldCheckpoint() + throws Exception { + round(); + var restored = copyOf(serving, "copy.db", serving.id()); + for (var id : List.of("m1", "m2", "m3")) { + mainWrite(serving, id); + } + round(); + serving = restored; + mainWrite(restored, "m4"); + + var round = round(); + + assertNotEquals(" ✓ Already in sync with main.", rendered(round)); + assertEquals( + List.of("m4"), + nodeDb.query("SELECT id FROM specs WHERE id = 'm4'", r -> r.text(0)), + "main's write after the restore reaches the node"); + assertEquals(" ✓ Already in sync with main.", rendered(round())); + } + @Test void aNodeThatSyncedBeforeCatalogsRecordsMainsOnItsNextRound() throws Exception { round(); @@ -316,7 +346,8 @@ void aMainThatNamesNoCatalogIsRefusedBeforeAnythingMovesEvenBeforeOneIsRecorded( assertEquals("refused", refused.kind()); assertEquals( "Main at sail@main names no database catalog, so this box cannot tell whether it is the" - + " database it synced with: upgrade main first, then sync again.", + + " database it synced with: upgrade main ('sudo sail upgrade' on main), then sync" + + " again.", refused.getMessage()); assertEquals(before, held(nodeDb)); } diff --git a/sail-infra/src/test/java/ai/singlr/sail/commands/SyncCliTest.java b/sail-infra/src/test/java/ai/singlr/sail/commands/SyncCliTest.java index 663844b78..f23d3117a 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/commands/SyncCliTest.java +++ b/sail-infra/src/test/java/ai/singlr/sail/commands/SyncCliTest.java @@ -31,6 +31,7 @@ import ai.singlr.sail.store.ContentMigration; import ai.singlr.sail.store.DataMigration; import ai.singlr.sail.store.DataMigrations; +import ai.singlr.sail.store.DatabaseCatalog; import ai.singlr.sail.store.EventStore; import ai.singlr.sail.store.HeldProjects; import ai.singlr.sail.store.MaterializedFilesMigration; @@ -288,6 +289,7 @@ private static void verifyScopeOffNode(String scenario, Sqlite db) throws IOExce HeldProjects.hold(db, "acme", "beta"); var granted = ProjectScope.of(List.of("acme")); new SyncScope(db).record(granted); + new DatabaseCatalog(db).recordMain("cat-a"); switch (scenario) { case "scope-start-gone" -> Files.delete(SailPaths.hostConfigPath()); case "scope-start-torn" -> Files.writeString(SailPaths.hostConfigPath(), ""); @@ -305,12 +307,14 @@ private static void verifyScopeOffNode(String scenario, Sqlite db) throws IOExce var push = projects.refusesInto(Actor.sync("nia", Role.MEMBER), null, "beta", true); if (scenario.endsWith("-main") || scenario.endsWith("-standalone")) { assertEquals(ProjectScope.all(), new SyncScope(db).held()); + assertEquals(Optional.empty(), new DatabaseCatalog(db).main()); assertEquals(Optional.empty(), push); Acting.as("ada", () -> new SpecStore(db).create(spec("b1", "beta"))); assertEquals("beta", new SpecStore(db).findById("b1").orElseThrow().project()); return; } assertEquals(granted, new SyncScope(db).held()); + assertEquals(Optional.of("cat-a"), new DatabaseCatalog(db).main()); assertTrue(push.isPresent()); } diff --git a/sail-infra/src/test/java/ai/singlr/sail/engine/HandleChangeTest.java b/sail-infra/src/test/java/ai/singlr/sail/engine/HandleChangeTest.java index 5bd755652..ec2d1b553 100644 --- a/sail-infra/src/test/java/ai/singlr/sail/engine/HandleChangeTest.java +++ b/sail-infra/src/test/java/ai/singlr/sail/engine/HandleChangeTest.java @@ -24,6 +24,7 @@ import ai.singlr.sail.identity.ProjectScope; import ai.singlr.sail.identity.Role; import ai.singlr.sail.store.BoxCredentialStore; +import ai.singlr.sail.store.DatabaseCatalog; import ai.singlr.sail.store.RunStore; import ai.singlr.sail.store.SchemaManager; import ai.singlr.sail.store.Sqlite; @@ -477,23 +478,27 @@ void aMainBecomingANodeStampsOnlyTheRunsItMadeNeverOneAnotherBoxExecuted() throw } @Test - void aBoxThatStopsBeingANodeForgetsTheScopeMainNamedIt() throws Exception { + void aBoxThatStopsBeingANodeForgetsTheScopeAndTheCatalogMainNamedIt() throws Exception { new SyncScope(db).record(ProjectScope.of(List.of("acme"))); + new DatabaseCatalog(db).recordMain("cat-a"); apply(node("ada"), main("ada")); assertTrue(new SyncScope(db).held().unrestricted()); + assertEquals(Optional.empty(), new DatabaseCatalog(db).main()); assertTrue(written.get()); } @Test - void aNodeRenamedStaysANodeAndKeepsTheScopeMainNamedIt() throws Exception { + void aNodeRenamedStaysANodeAndKeepsTheScopeAndTheCatalogMainNamedIt() throws Exception { var granted = ProjectScope.of(List.of("acme")); new SyncScope(db).record(granted); + new DatabaseCatalog(db).recordMain("cat-a"); apply(node("ada"), node("uday")); assertEquals(granted, new SyncScope(db).held()); + assertEquals(Optional.of("cat-a"), new DatabaseCatalog(db).main()); assertTrue(written.get()); }