From 786a4555867ae159f8939434df563a6e24cdd05f Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:23:48 +0300 Subject: [PATCH 1/2] feat(core-ts): restoreBundle, a site bundle of one snapshot restoreBundle(from, snapshot) rebuilds a site bundle from a snapshot's stored node bodies, for importSite in replace mode, so a site's working copy and published pointer can become that snapshot's again (the undo of `cloudhal import --replace`, RFC-0039). Revs only move forward: a node the site holds at the snapshot's rev or later, with other content, comes back at the rev after its current one, so a stale editor gets a conflict. It lists the media the nodes need and carries the store's takedowns. A missing or mismatched body is refused before anything is written. exportSite's output is unchanged; its takedown query and media rule are shared. Refs #47 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/restore-bundle.md | 5 + packages/core-ts/src/sync-published.ts | 11 +- packages/core-ts/src/sync-restore.ts | 138 ++++++++++++++ packages/core-ts/src/sync.ts | 37 +--- packages/core-ts/test/sync-restore.test.ts | 202 +++++++++++++++++++++ 5 files changed, 362 insertions(+), 31 deletions(-) create mode 100644 .changeset/restore-bundle.md create mode 100644 packages/core-ts/src/sync-restore.ts create mode 100644 packages/core-ts/test/sync-restore.test.ts diff --git a/.changeset/restore-bundle.md b/.changeset/restore-bundle.md new file mode 100644 index 00000000..28b4b028 --- /dev/null +++ b/.changeset/restore-bundle.md @@ -0,0 +1,5 @@ +--- +"@cloudhal/core-ts": patch +--- + +New `restoreBundle(from, snapshot)`: a site bundle of one snapshot's nodes, rebuilt from the stored node bodies, for `importSite(…, { mode: "replace" })`, which checks it like any bundle. The site's working copy and published pointer become that snapshot's again (`cloudhal import --revert`, RFC-0039). Revs only move forward: a node the site holds at the snapshot's rev or a later one, with other content, comes back at the rev after its current one, so a Studio tab or agent still holding a rev that node had gets a conflict instead of saving over it; a node whose content already is the snapshot's is left as it is. It lists the media those nodes need and carries the store's takedowns among them, so taken-down content stays down; a missing node body, or one that is not the node its snapshot ref names, is refused before anything is written. `exportSite`'s output is unchanged. diff --git a/packages/core-ts/src/sync-published.ts b/packages/core-ts/src/sync-published.ts index 73aefe3f..e17060ba 100644 --- a/packages/core-ts/src/sync-published.ts +++ b/packages/core-ts/src/sync-published.ts @@ -10,10 +10,13 @@ import type { NodeRow } from "./sync-bundle.ts"; const VOLATILE = new Set(["rev", "updatedAt"]); -/** The body without the fields a write always changes, for "same content" comparisons. */ -function stable(site: Site): string { +/** + * A node body without the fields a write always changes, for "same content" comparisons (also + * `restoreBundle`'s). + */ +export function stableJson(node: object): string { return canonicalJson( - Object.fromEntries(Object.entries(site).filter(([key]) => !VOLATILE.has(key))), + Object.fromEntries(Object.entries(node).filter(([key]) => !VOLATILE.has(key))), ); } @@ -38,7 +41,7 @@ export async function keepTargetPublished( return [...rows]; const merged: Site = { ...srcSite, publishedSnapshotId: dstSite.publishedSnapshotId }; let row: NodeRow = dst; - if (stable(merged) !== stable(dstSite)) { + if (stableJson(merged) !== stableJson(dstSite)) { const next: Site = { ...merged, rev: Math.max(src.rev, dst.rev) + 1 }; row = { ...src, rev: next.rev, hash: await nodeHash(next), body: canonicalJson(next) }; } diff --git a/packages/core-ts/src/sync-restore.ts b/packages/core-ts/src/sync-restore.ts new file mode 100644 index 00000000..1bd4db2d --- /dev/null +++ b/packages/core-ts/src/sync-restore.ts @@ -0,0 +1,138 @@ +/** + * A site bundle that makes a site's working copy one of its snapshots again (`cloudhal import + * --revert`, RFC-0039 §2 as amended): the snapshot's nodes rebuilt from their stored bodies, for + * `importSite(…, { mode: "replace" })`, which checks them like any bundle's (precompiled + * validators, keys, hosts) and writes them in one atomic batch, the site node and so its publish + * pointer included. Revs only move forward, as every other revert's do: a node the site holds at + * the snapshot's rev or a later one, with other content, comes back at the rev after its current + * one (its hash recomputed), so an editor holding a rev the node had (a Studio tab, an agent) + * fails its compare-and-swap instead of saving over it; a node whose content already is the + * snapshot's stays as it is. No snapshot or node body travels: the store holds them. The media the + * blob assets need are listed; the store's takedowns among them travel as takedowns, so taken-down + * content stays down (invariant 5). Worker-safe. + */ +import type { Asset, NodeRef, Snapshot } from "@cloudhal/core-contracts"; +import type { SqlDriver } from "@cloudhal/storage"; +import { canonicalJson, nodeHash } from "./hash.ts"; +import { type NodeKind, type NodeTypes, siteIdOf, uniquenessKey } from "./nodes.ts"; +import { + type BundleMedia, + type BundlePurge, + type NodeRow, + SITE_BUNDLE_FORMAT, + type SiteBundle, + type StoragePair, +} from "./sync-bundle.ts"; +import { assertNodeBody } from "./sync-checks.ts"; +import { stableJson } from "./sync-published.ts"; + +/** The site's takedowns in the media ledger, by content (what a bundle's `purged` holds). */ +export async function siteTakedowns(sql: SqlDriver, siteId: string): Promise { + const rows = await sql.all<{ + sha256: string; + purge_reason: string | null; + purge_legal: number | null; + purged_by: string | null; + purged_at: string | null; + }>( + "SELECT sha256, purge_reason, purge_legal, purged_by, purged_at FROM media_blobs WHERE site_id = ? AND state = 'purged' ORDER BY sha256", + [siteId], + ); + return rows.map((r) => ({ + sha256: r.sha256, + reason: r.purge_reason ?? "", + legal: r.purge_legal === 1, + by: r.purged_by ?? "", + at: r.purged_at ?? "", + })); +} + +/** A blob asset's media entry (`exportSite` and `restoreBundle`); none for other storage. */ +export function blobMedia(asset: Asset): BundleMedia | undefined { + return asset.storage?.kind === "blob" + ? { sha256: asset.sha256, bytes: asset.bytes, mediaType: asset.mediaType } + : undefined; +} + +/** A node as the store writes its row. */ +async function rowOf(kind: NodeKind, node: NodeTypes[NodeKind], hash?: string): Promise { + return { + id: node.id, + site_id: siteIdOf(kind, node), + kind, + key: uniquenessKey(kind, node), + rev: node.rev, + hash: hash ?? (await nodeHash(node)), + body: canonicalJson(node), + created_at: node.createdAt, + updated_at: node.updatedAt, + }; +} + +/** + * The row that brings `ref` back (see the module doc): the current row when its content is the + * snapshot's already, else the snapshot's body, at the rev after the current one when the site + * holds the node at the snapshot's rev or a later one. + */ +async function restoredRow( + snapshot: Snapshot, + ref: NodeRef, + text: string, + now: NodeRow | undefined, +): Promise { + let body: unknown; + try { + body = JSON.parse(text); + } catch { + // Never the parser's message: it can quote the body. + throw new Error(`blob ${ref.hash} is not JSON`); + } + assertNodeBody(snapshot.siteId, ref.hash, body, [ref]); + const node = body as NodeTypes[NodeKind]; + if (now && (now.hash === ref.hash || stableJson(JSON.parse(now.body)) === stableJson(node))) + return now; + if (!now || now.rev < ref.rev) return rowOf(ref.kind, node, ref.hash); + return rowOf(ref.kind, { ...node, rev: now.rev + 1 }); +} + +/** + * The bundle that brings `snapshot`'s site back to the snapshot's nodes, revs moved forward (see + * the module doc). A node body missing from the blob store, or one that is not the node its ref + * names, is refused before anything is written; every other check is `importSite`'s. + */ +export async function restoreBundle(from: StoragePair, snapshot: Snapshot): Promise { + const { siteId } = snapshot; + const current = new Map( + (await from.sql.all("SELECT * FROM nodes WHERE site_id = ?", [siteId])).map( + (r) => [r.id, r] as const, + ), + ); + const decoder = new TextDecoder(); + const refs = [...snapshot.nodes].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + const nodes: NodeRow[] = []; + for (const ref of refs) { + const data = await from.blobs.get(`nodes/${ref.hash}.json`); + if (!data) throw new Error(`snapshot ${snapshot.id} references a missing blob ${ref.hash}`); + nodes.push(await restoredRow(snapshot, ref, decoder.decode(data), current.get(ref.id))); + } + const down = new Map((await siteTakedowns(from.sql, siteId)).map((p) => [p.sha256, p] as const)); + const media = new Map(); + const purged = new Map(); + for (const n of nodes) { + const m = n.kind === "asset" ? blobMedia(JSON.parse(n.body) as Asset) : undefined; + if (!m) continue; + const takedown = down.get(m.sha256); + if (takedown) purged.set(m.sha256, takedown); + else media.set(m.sha256, m); + } + const bySha = (a: { sha256: string }, b: { sha256: string }) => a.sha256.localeCompare(b.sha256); + return { + format: SITE_BUNDLE_FORMAT, + siteId, + nodes, + snapshots: [], + blobs: {}, + ...(media.size > 0 ? { media: [...media.values()].sort(bySha) } : {}), + ...(purged.size > 0 ? { purged: [...purged.values()].sort(bySha) } : {}), + }; +} diff --git a/packages/core-ts/src/sync.ts b/packages/core-ts/src/sync.ts index fcb93c14..a163ce92 100644 --- a/packages/core-ts/src/sync.ts +++ b/packages/core-ts/src/sync.ts @@ -11,8 +11,9 @@ * the first write (`sync-plan.ts`), so a refused bundle leaves the target as it was. * * The bundle format lives in `sync-bundle.ts`, its checks in `sync-checks.ts`, the plan of an - * import in `sync-plan.ts` (its media ledger in `sync-ledger.ts`); this module re-exports the - * public parts. + * import in `sync-plan.ts` (its media ledger in `sync-ledger.ts`), and `restoreBundle` (a + * snapshot's nodes as a bundle, for `cloudhal import --revert`) in `sync-restore.ts`; this module + * re-exports the public parts. */ import type { Asset } from "@cloudhal/core-contracts"; @@ -31,9 +32,11 @@ import { type StoragePair, } from "./sync-bundle.ts"; import { prepareImport } from "./sync-plan.ts"; +import { blobMedia, siteTakedowns } from "./sync-restore.ts"; export * from "./sync-bundle.ts"; export { type ImportPlan, type ImportPlanOptions, planImport } from "./sync-plan.ts"; +export { restoreBundle } from "./sync-restore.ts"; /** Splits statements into batches that respect D1's per-batch limit. */ async function runBatches(sql: SqlDriver, statements: SqlStatement[], size = 500): Promise { @@ -73,24 +76,14 @@ export async function exportSite( blobs[hash] = decoder.decode(data); } // Media: blob assets of the current nodes and of every snapshot, minus taken-down content. - const purgedRows = await from.sql.all<{ - sha256: string; - purge_reason: string | null; - purge_legal: number | null; - purged_by: string | null; - purged_at: string | null; - }>( - "SELECT sha256, purge_reason, purge_legal, purged_by, purged_at FROM media_blobs WHERE site_id = ? AND state = 'purged' ORDER BY sha256", - [siteId], - ); + const purgedRows = await siteTakedowns(from.sql, siteId); const purged = new Set(purgedRows.map((r) => r.sha256)); // packages/cli/src/commands/archive-bundle.ts bundleMedia rebuilds this from an archive; keep - // the two in step (export-cli.test.ts compares them). + // the two in step (export-cli.test.ts compares them). `restoreBundle` shares `blobMedia`. const media = new Map(); const addAsset = (body: string) => { - const a = JSON.parse(body) as Asset; - if (a.storage?.kind === "blob" && !purged.has(a.sha256)) - media.set(a.sha256, { sha256: a.sha256, bytes: a.bytes, mediaType: a.mediaType }); + const m = blobMedia(JSON.parse(body) as Asset); + if (m && !purged.has(m.sha256)) media.set(m.sha256, m); }; for (const n of nodes) if (n.kind === "asset") addAsset(n.body); for (const snap of snapshots) @@ -111,17 +104,7 @@ export async function exportSite( ...(media.size > 0 ? { media: [...media.values()].sort((a, b) => a.sha256.localeCompare(b.sha256)) } : {}), - ...(purgedRows.length > 0 - ? { - purged: purgedRows.map((r) => ({ - sha256: r.sha256, - reason: r.purge_reason ?? "", - legal: r.purge_legal === 1, - by: r.purged_by ?? "", - at: r.purged_at ?? "", - })), - } - : {}), + ...(purgedRows.length > 0 ? { purged: purgedRows } : {}), }; } diff --git a/packages/core-ts/test/sync-restore.test.ts b/packages/core-ts/test/sync-restore.test.ts new file mode 100644 index 00000000..a3568694 --- /dev/null +++ b/packages/core-ts/test/sync-restore.test.ts @@ -0,0 +1,202 @@ +/** + * `restoreBundle` (RFC-0039 §2, `cloudhal import --revert`): a snapshot's nodes as a bundle that + * `importSite`'s replace writes back, the site node's publish pointer and hostnames with them; + * revs only move forward, so an editor holding a rev a node had gets a conflict; content taken + * down since stays down (invariant 5), and a missing node body is refused before any write. + */ +import type { ContentEntry, Page, Site } from "@cloudhal/core-contracts"; +import { MemoryBlobStore } from "@cloudhal/storage"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { describe, expect, it } from "vitest"; +import { mediaBlobKey } from "../src/assets.ts"; +import { ConflictError } from "../src/errors.ts"; +import { PNG_1X1 } from "../src/media/fixtures.ts"; +import { seedStarterSite } from "../src/seed.ts"; +import { SiteGraphStore } from "../src/store.ts"; +import { exportSite, importSite, planImport, restoreBundle } from "../src/sync.ts"; + +const actor = { kind: "human" as const, id: "user_restore0001" }; + +async function pair() { + const sql = await BetterSqlite3Driver.open(":memory:"); + const blobs = new MemoryBlobStore(); + const store = new SiteGraphStore({ sql, blobs }); + await store.init(); + return { sql, blobs, store }; +} + +type Pair = Awaited>; + +type Row = { id: string; rev: number; hash: string; body: string }; + +/** The node rows as the store holds them, without their timestamps. */ +const rowsOf = (p: Pair, siteId: string) => + p.sql.all( + "SELECT id, site_id, kind, key, rev, hash, body FROM nodes WHERE site_id = ? ORDER BY id", + [siteId], + ); + +/** The rows' content: without the rev a restore moves forward, and the hash that follows it. */ +const contentOf = (rows: readonly Row[]) => + rows.map(({ rev: _rev, hash: _hash, body, ...row }) => ({ + ...row, + body: { ...(JSON.parse(body) as object), rev: undefined }, + })); + +const revsOf = (rows: readonly Row[]) => new Map(rows.map((r) => [r.id, r.rev])); + +const hostsOf = (p: Pair) => p.sql.all("SELECT * FROM site_hosts ORDER BY host"); + +async function counts(p: Pair) { + const n = async (table: string) => + (await p.sql.get<{ n: number }>(`SELECT count(*) AS n FROM ${table}`))?.n; + return { nodes: await n("nodes"), snapshots: await n("snapshots"), blobs: await p.blobs.list() }; +} + +async function publishNow(p: Pair, siteId: string) { + const snap = await p.store.createSnapshot(siteId, { actor, reason: "publish" }); + await p.store.publish((await p.store.getSite(siteId)) as Site, snap.id); + return snap; +} + +describe("restoreBundle", () => { + it("brings back a snapshot's nodes, the publish pointer and the hostnames", async () => { + const p = await pair(); + const seeded = await seedStarterSite(p.store, { name: "Restore", slug: "restore" }); + const site = (await p.store.getSite(seeded.id)) as Site; + await p.store.updateSite({ + ...site, + domains: [{ hostname: "restore.example.com", primary: true }], + }); + const live = await publishNow(p, site.id); + const b = await p.store.createSnapshot(site.id, { actor, reason: "manual" }); + const rows = await rowsOf(p, site.id); + const hosts = await hostsOf(p); + expect(hosts).toHaveLength(1); + + // Edits after it: a retitled page, a deleted entry, a new page, no domain, another publish. + const about = (await p.store.listPages(site.id)).find((x) => x.path === "/about") as Page; + await p.store.updatePage({ ...about, title: "Changed" }); + const entry = (await p.store.listContentEntries(site.id))[0] as ContentEntry; + await p.store.deleteContentEntry(entry); + await p.store.createPage((await p.store.getSite(site.id)) as Site, { + siteId: site.id, + path: "/new", + title: "New", + locale: "en", + status: "draft", + sections: [], + }); + await p.store.updateSite({ ...((await p.store.getSite(site.id)) as Site), domains: [] }); + const later = await publishNow(p, site.id); + expect(await hostsOf(p)).toEqual([]); + const edited = revsOf(await rowsOf(p, site.id)); + + const bundle = await restoreBundle(p, b); + expect(bundle).toMatchObject({ siteId: site.id, snapshots: [], blobs: {} }); + expect(bundle.project).toBeUndefined(); + const plan = await planImport(p, bundle, { mode: "replace" }); + expect(plan).toMatchObject({ snapshots: [], blobs: 0, unchanged: false, fresh: false }); + expect(plan.nodes.added.map((n) => n.id)).toContain(entry.id); + await importSite(p, bundle, { mode: "replace" }); + const now = await rowsOf(p, site.id); + expect(contentOf(now)).toEqual(contentOf(rows)); + // A node edited since comes back at the rev after its current one; the rest as they were. + const atB = revsOf(rows); + const moved = (id: string) => edited.has(id) && edited.get(id) !== atB.get(id); + expect(revsOf(now)).toEqual( + new Map(rows.map((r) => [r.id, moved(r.id) ? (edited.get(r.id) as number) + 1 : r.rev])), + ); + expect([about.id, site.id].every(moved)).toBe(true); + expect((await planImport(p, await restoreBundle(p, b), { mode: "replace" })).unchanged).toBe( + true, + ); + const restored = (await p.store.getSite(site.id)) as Site; + expect(restored.publishedSnapshotId).toBe(live.id); + expect(restored.publishedSnapshotId).not.toBe(later.id); + expect(await hostsOf(p)).toEqual(hosts); + }); + + it("moves revs forward: an editor holding a rev the node had gets a conflict", async () => { + const p = await pair(); + const site = await seedStarterSite(p.store, { name: "Revs", slug: "revs" }); + const start = await exportSite(p, site.id); + const aboutOf = async () => + (await p.store.listPages(site.id)).find((x) => x.path === "/about") as Page; + const first = await aboutOf(); + await p.store.updatePage({ ...first, title: "Before" }); + const b = await p.store.createSnapshot(site.id, { actor, reason: "manual" }); + // A replace keeps the bundle's revs: the page is back at the rev it had first, and one edit + // later at the rev the snapshot holds, with other content (a Studio tab now holds that rev). + await importSite(p, start, { mode: "replace" }); + const tab = await p.store.updatePage({ ...(await aboutOf()), title: "After" }); + expect(tab.rev).toBe(b.nodes.find((n) => n.id === first.id)?.rev); + + await importSite(p, await restoreBundle(p, b), { mode: "replace" }); + expect(await aboutOf()).toMatchObject({ title: "Before", rev: tab.rev + 1 }); + await expect(p.store.updatePage({ ...tab, title: "Stale" })).rejects.toThrow(ConflictError); + expect((await aboutOf()).title).toBe("Before"); + expect((await planImport(p, await restoreBundle(p, b), { mode: "replace" })).unchanged).toBe( + true, + ); + }); + + it("keeps content taken down after the snapshot down: the asset comes back, its bytes do not", async () => { + const p = await pair(); + const site = await seedStarterSite(p.store, { name: "Media", slug: "media" }); + const { asset } = await p.store.ingestAsset({ + siteId: site.id, + bytes: PNG_1X1, + filename: "a.png", + uploadedBy: actor, + source: { kind: "upload" }, + }); + const b = await p.store.createSnapshot(site.id, { actor, reason: "manual" }); + const rows = await rowsOf(p, site.id); + await p.store.purgeBlob(site.id, asset.sha256, actor, { text: "notice", legal: false }); + const bundle = await restoreBundle(p, b); + expect(bundle.purged?.map((x) => x.sha256)).toEqual([asset.sha256]); + expect(bundle.media).toBeUndefined(); + await importSite(p, bundle, { mode: "replace" }); + expect(contentOf(await rowsOf(p, site.id))).toEqual(contentOf(rows)); + expect(await p.blobs.head(mediaBlobKey(site.id, asset.sha256))).toBeNull(); + const ledger = await p.sql.get( + "SELECT state FROM media_blobs WHERE site_id = ? AND sha256 = ?", + [site.id, asset.sha256], + ); + expect(ledger).toEqual({ state: "purged" }); + }); + + it("lists the media the snapshot's assets need", async () => { + const p = await pair(); + const site = await seedStarterSite(p.store, { name: "Listed", slug: "listed" }); + const { asset } = await p.store.ingestAsset({ + siteId: site.id, + bytes: PNG_1X1, + filename: "a.png", + uploadedBy: actor, + source: { kind: "upload" }, + }); + const b = await p.store.createSnapshot(site.id, { actor, reason: "manual" }); + const bundle = await restoreBundle(p, b); + expect(bundle.media).toEqual([ + { sha256: asset.sha256, bytes: asset.bytes, mediaType: asset.mediaType }, + ]); + expect(bundle.purged).toBeUndefined(); + expect((await planImport(p, bundle, { mode: "replace" })).unchanged).toBe(true); + }); + + it("refuses a snapshot whose node body is missing, before anything is written", async () => { + const p = await pair(); + const site = await seedStarterSite(p.store, { name: "Gone", slug: "gone" }); + const b = await p.store.createSnapshot(site.id, { actor, reason: "manual" }); + const ref = b.nodes.find((n) => n.kind === "page"); + if (!ref) throw new Error("no page ref"); + await p.blobs.delete(`nodes/${ref.hash}.json`); + const before = await counts(p); + await expect(restoreBundle(p, b)).rejects.toThrow( + `snapshot ${b.id} references a missing blob ${ref.hash}`, + ); + expect(await counts(p)).toEqual(before); + }); +}); From 33ae9111298d3a426222877a0be80e80e10e3f0a Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:23:48 +0300 Subject: [PATCH 2/2] feat(cli): cloudhal import --replace and --revert (#47) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC-0039 rollout step 4. - `import --replace` makes the local site's nodes the archive's, its published snapshot included (as pull --replace; push still keeps a deployment's own). When it changes a site the project holds, the current graph is saved first as `before import ` and the result as `import `; the summary prints the undo. - `import --revert ` brings the site back to such a snapshot through restoreBundle and importSite, saving the current graph first, so a revert can itself be reverted. Studio's rollback only moves the published pointer to an ancestor, which cannot undo a replace. - An archive of a site cloudhal.json does not list switches siteId (and the old id's place in sites), editing only those values once the import has landed; the old site stays in the store, untouched. - Refused before anything is written: open releases, an unfinished field operation, a hostname another local site serves. - --cancel-schedules is now refused by every command but push. RFC-0039 §2 and invariant 6 and ADR-0041 decision 4 are amended to name the undo and the published-pointer choice. Refs #47 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/site-archive-import.md | 2 +- .changeset/site-archive-replace.md | 13 + docs/adr/0041-site-archives.md | 3 +- docs/rfcs/0039-site-archive-export-import.md | 13 +- packages/cli/src/cli.ts | 1 + packages/cli/src/cli/commands-content.ts | 33 +- packages/cli/src/cli/help.ts | 22 +- packages/cli/src/commands/archive-bundle.ts | 24 +- packages/cli/src/commands/import-apply.ts | 145 +++++ packages/cli/src/commands/import-checks.ts | 73 ++- .../src/commands/import-describe-replace.ts | 105 ++++ packages/cli/src/commands/import-describe.ts | 65 ++- packages/cli/src/commands/import-read.ts | 93 +++ packages/cli/src/commands/import-replace.ts | 226 ++++++++ packages/cli/src/commands/import-revert.ts | 196 +++++++ packages/cli/src/commands/import.ts | 352 ++++++------ packages/cli/src/config-site.ts | 129 +++++ packages/cli/src/config.ts | 5 + packages/cli/test/import-cli.test.ts | 59 +- packages/cli/test/import-replace.test.ts | 543 ++++++++++++++++++ packages/cli/test/import-switch.test.ts | 327 +++++++++++ packages/cli/test/site-fixture.ts | 41 +- 22 files changed, 2195 insertions(+), 275 deletions(-) create mode 100644 .changeset/site-archive-replace.md create mode 100644 packages/cli/src/commands/import-apply.ts create mode 100644 packages/cli/src/commands/import-describe-replace.ts create mode 100644 packages/cli/src/commands/import-read.ts create mode 100644 packages/cli/src/commands/import-replace.ts create mode 100644 packages/cli/src/commands/import-revert.ts create mode 100644 packages/cli/src/config-site.ts create mode 100644 packages/cli/test/import-replace.test.ts create mode 100644 packages/cli/test/import-switch.test.ts diff --git a/.changeset/site-archive-import.md b/.changeset/site-archive-import.md index 37b281f3..b9f058a1 100644 --- a/.changeset/site-archive-import.md +++ b/.changeset/site-archive-import.md @@ -5,7 +5,7 @@ New `cloudhal import [--dry-run]` (RFC-0039, #47) reads a `.cloudhal` file written by `cloudhal export` into the local project, keeping the site's id. - It reads the file twice. First it verifies every file's size and SHA-256, every row and every node body (each must be the node its snapshot refs name), and checks the archive against the project. A damaged or edited file, invalid content, an archive of another site, a site whose nodes differ, or missing media is therefore refused before anything is written. Then it reads the file again, refusing it if it changed in between, copies the media the project lacks and imports the graph, keeping the archive's published snapshot as `cloudhal pull` does. -- Only bootstrap imports for now. The archive's site must be `cloudhal.json`'s `siteId` or listed in `sites`, and the project must not hold it yet or hold exactly its nodes; missing snapshots, node bodies and media are added. A site whose nodes differ is refused; `--replace` is not available yet. +- By default a bootstrap import. The archive's site must be `cloudhal.json`'s `siteId` or listed in `sites`, and the project must not hold it yet or hold exactly its nodes; missing snapshots, node bodies and media are added. A site whose nodes differ is refused; `--replace` replaces them (see its own note). - An archive written with `--no-media` imports only when the project already has its media. - Taken-down content stays down: the archive's takedowns are recorded and their bytes never written, a local copy of content the archive's ledger took down is deleted (the summary says how many), and an archive carrying content this project has taken down is refused. - `--dry-run` prints the manifest and what the import would add, change or remove (nodes by kind and title, snapshots, media to copy), writes nothing, and exits 1 when the import would be refused. diff --git a/.changeset/site-archive-replace.md b/.changeset/site-archive-replace.md new file mode 100644 index 00000000..0e9231bb --- /dev/null +++ b/.changeset/site-archive-replace.md @@ -0,0 +1,13 @@ +--- +"@cloudhal/cli": patch +--- + +`cloudhal import --replace [--dry-run]` (RFC-0039 rollout step 4, #47) makes the local site's nodes the archive's, its published snapshot included, as `cloudhal pull --replace` does. `cloudhal push` still keeps a deployment's own published snapshot, so a local replace never moves production. Snapshots are only ever added. + +- When it changes a site the project already holds, the current graph is saved first as a snapshot labelled `before import ` and the result as `import `; the summary prints `(undo: cloudhal import --revert )`. A new or unchanged site gets no snapshot; a site the plan found unchanged is checked again just before the write, and refused if another process changed it meanwhile. A run that stopped after the first snapshot reuses it the next time. +- The archive's nodes keep their revs, as with `pull --replace`. A Studio tab or agent still holding a node at the rev the archive gives it can save over it without a conflict, so the summary counts those nodes and says to reload open Studio tabs. +- New `cloudhal import --revert [--dry-run]` brings the site back to a `before import` snapshot, its published snapshot included, saving the current graph first as `before revert import `, so a revert can itself be reverted. Its revs only move forward: a node edited since comes back at the rev after its current one, so a stale editor gets a conflict. Studio's rollback only moves the published pointer to an earlier snapshot, which cannot undo a replace. +- An archive of a site `cloudhal.json` does not list switches its `siteId`, and the old id's place in `sites`, to the archive's, editing only those values in the file (layout and other keys stay as written), once the import has landed. The old site stays in the local store, unlisted and untouched; writing the old id back by hand undoes the switch (`--revert` restores nodes only), and the summary names `cloudhal site delete` to remove the old site for good. A deployed project is warned that its deployment serves the old site, and that a deployment of the imported site is updated from the project that deployed it. +- Refused before anything is written: open releases of the site (cancel them in Studio), an unfinished field operation, a hostname another local site serves (now a refusal `--dry-run` reports, for bootstrap imports too), and an archive site id `cloudhal.json` cannot name. +- `--dry-run` shows the snapshots it would take, the published-snapshot change and the `cloudhal.json` change; it exits 0, or 1 when the import would be refused. +- `--cancel-schedules` is now refused by every command but `cloudhal push`. diff --git a/docs/adr/0041-site-archives.md b/docs/adr/0041-site-archives.md index 8b22c2e5..3e15824a 100644 --- a/docs/adr/0041-site-archives.md +++ b/docs/adr/0041-site-archives.md @@ -2,6 +2,7 @@ - Status: Accepted - Date: 2026-10-22 (decided by Halil) - Amendment (2026-10-25, RFC-0039 rollout step 1): deterministic bytes hold for the same Node/zlib build; the tar stream and the manifest's SHA-256s on every platform. See rule 6. +- Amendment (2026-10-26, RFC-0039 rollout step 4): `--replace` takes the archive's published snapshot (a deployment keeps its own on `push`) and is undone with `cloudhal import --revert `, which restores the `before import` snapshot. See decision 4. - Related: RFC-0039, RFC-0004, RFC-0010, ADR-0002, #47 - Sources: [Sanity datasets](https://www.sanity.io/docs/cli-reference/dataset) · [Strapi data management](https://docs.strapi.io/cms/data-management/export) · [Contentful CLI export](https://github.com/contentful/contentful-cli/tree/main/docs/space/export) · [Ghost migration](https://docs.ghost.org/migration/ghost) · [WordPress export](https://wordpress.org/documentation/article/tools-export-screen/) @@ -14,7 +15,7 @@ A site's content can only leave a project through a deployment (`push`, `pull`), 3. **Ids are kept.** An imported site is the same site. 4. **Import modes:** - bootstrap (default): adds what is missing and refuses differing nodes; - - `--replace`: first saves the current graph as a `before import` snapshot, so the import can be undone; + - `--replace`: first saves the current graph as a `before import` snapshot, so the import can be undone (amended: with `cloudhal import --revert `, a site id switch by writing the old `siteId` back; the published snapshot becomes the archive's); - `--dry-run`: shows the difference and writes nothing. Never erase-then-import. diff --git a/docs/rfcs/0039-site-archive-export-import.md b/docs/rfcs/0039-site-archive-export-import.md index 8778eefb..896aa70d 100644 --- a/docs/rfcs/0039-site-archive-export-import.md +++ b/docs/rfcs/0039-site-archive-export-import.md @@ -2,6 +2,7 @@ - Status: Accepted (2026-10-22, by Halil; ADR-0041). Open questions decided as recommended. Revised before acceptance after a comparison with Sanity, Strapi, Contentful, Ghost, WordPress, Directus and Webflow (see Prior art): NDJSON parts instead of one JSON file, per-file checksums, contract versions in the manifest, and a snapshot before `--replace`. - Amendment (2026-10-25, rollout step 1): identical `.cloudhal` bytes need the same Node/zlib build; the tar stream inside, and so every SHA-256 in the manifest, is identical on every platform. See invariant 7. +- Amendment (2026-10-26, rollout step 4): `--replace` takes the archive's published snapshot (the site node carries it, as `pull --replace`; `push` keeps a deployment's own); it saves `before import ` and the result as `import `; the "rollback" that undoes it is `cloudhal import --revert `, since Studio's rollback moves only the published pointer, to an ancestor; and `cloudhal site delete` already deletes a local site. See §2 and invariant 6. - Author: Halil (Madraka), drafted with Claude · Date: 2026-10-21 - Related: - #47 (found while building cloudhal.com: no way to keep or move a site's content without a deployment, so it was written as code). @@ -69,6 +70,7 @@ interface SiteArchiveManifest { ```text cloudhal export [--out ] [--site ] [--no-media] [--latest-only] cloudhal import [--replace] [--dry-run] +cloudhal import --revert [--dry-run] (amended 2026-10-26) ``` - **export:** @@ -79,6 +81,13 @@ cloudhal import [--replace] [--dry-run] - **import:** checks the manifest and every file's SHA-256 as it streams (media files are named by their hash as well), then calls `importSite`. Three modes: - **bootstrap** (default), into a project with no site or one whose site is the archive's (same `siteId`). Like `pull`, it adds what is missing and refuses nodes that differ: never Strapi's "erase the target first". - **`--replace`:** the local nodes become the archive's. Snapshots are only ever added. First, the current local graph is saved as a snapshot labelled `before import `, so the import can be undone with a rollback. Into a project whose site has another id, `--replace` switches `cloudhal.json`'s `siteId` to the archive's. The old site stays in the local store, unlisted; there is no command to delete a local site yet. + *Amended 2026-10-26 (rollout step 4):* + - The published snapshot is the archive's: the site node carries it, so `--replace` takes it as `pull --replace` does. A deployment keeps its own, since `push` applies with the deployment's pointer. + - The snapshot is taken only when the project holds the site and its nodes would change (none for a new or unchanged site), with the file's base name; the result is saved as `import ` right after, so the latest snapshot is the imported graph. + - The undo is `cloudhal import --revert `: it restores that snapshot's nodes, the site node and so its published snapshot included, through `importSite`'s replace, saving the current graph first as `before revert import `. Studio's rollback only moves the published pointer to an ancestor of the published snapshot, which a `before import` snapshot is not. + - Revs: the store's compare-and-swap checks only a node's rev. `--revert` moves revs forward (a node edited since comes back at the rev after its current one), so a Studio tab or agent holding a rev the node had gets a conflict. `--replace` keeps the archive's revs, as `pull --replace` does, so the local site equals the archive; its summary counts the nodes given a rev no higher than they had, which such an editor could save over, and says to reload. + - The switch edits only `siteId` and the old id's entry in `sites`, after the import has landed. It is undone by writing the old id back by hand: the old site is untouched, and `--revert` restores nodes only. `cloudhal site delete --confirm ` deletes the old site for good. + - Refused before anything is written: open releases (their pinned versions point into the replaced graph; cancelled in Studio, no `--cancel-schedules` here), an unfinished field operation (RFC-0041; its marker is on the site node), and a hostname another local site serves (never released: that would write the old site). - **`--dry-run`:** prints the manifest and the difference the import would make (nodes added, changed and removed by kind and title; snapshots added; media to copy) and writes nothing. ### 3. Invariants @@ -88,7 +97,7 @@ cloudhal import [--replace] [--dry-run] 3. Every file is verified against the manifest (SHA-256 and size), and every media file against its own name, before it is used. A missing or wrong file fails the import before any node is written (the bundle's `missing_media` rule). 4. An archive written by a newer format or a newer contracts `schemaVersion` is refused with the CLI version needed. Older ones are read, and migrated forward if a later schema version requires it. 5. Taken-down content stays taken down: the `purged` ledger travels and the import never restores those bytes. -6. Import writes the graph in atomic batches (no interactive transactions, AGENTS.md rule 4), guarded like `importSite` today. A failure leaves the project as it was, and `--replace` can always be undone through its `before import` snapshot. +6. Import writes the graph in atomic batches (no interactive transactions, AGENTS.md rule 4), guarded like `importSite` today. A failure leaves the project as it was, and `--replace` can always be undone through its `before import` snapshot. *Amended 2026-10-26:* with `cloudhal import --revert ` (§2), and a site id switch by writing the old `siteId` back (the old site is untouched); a failure after that snapshot was taken leaves it, holding the graph as it was, and the message names it. 7. Same content, same file: rows and files are written in a fixed order, with zero timestamps and fixed modes in the tar headers, so exporting an unchanged site twice gives identical bytes (good for git and for diffing). *Amended 2026-10-25:* identical bytes hold for the same Node/zlib build. gzip output differs between zlib builds (Homebrew's zlib and the Chromium zlib that official Node binaries ship compress the same input differently), and only a dependency or uncompressed output would avoid that. The tar stream inside the gzip, and so every SHA-256 in the manifest, is identical on every platform. A committed archive (step 5's demo) is regenerated on one pinned Node in CI, or compared through its decompressed tar. ### 4. What is not in an archive @@ -125,7 +134,7 @@ The manifest says so, and `export` prints it. People and personal data never lea 1. The streaming tar writer and reader, with tests (round trip, damaged file, wrong hash, path traversal, deterministic bytes). 2. `export`. 3. `import` with `--dry-run`. - 4. `--replace`, the `before import` snapshot, and the site id switch. + 4. `--replace`, the `before import` snapshot, and the site id switch. *Amended 2026-10-26:* with `import --revert` as the undo. 5. Docs and a starter demo archive (regenerated on one pinned Node, or compared through its decompressed tar: invariant 7). ## Security & policy impact diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index a667fd9c..d4d464ef 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -83,6 +83,7 @@ const ONLY_FOR: Readonly< site: { commands: ["export"], note: "; every other command acts on cloudhal.json's siteId" }, "no-media": { commands: ["export"] }, "latest-only": { commands: ["export"] }, + "cancel-schedules": { commands: ["push"] }, }; function refuseForeignFlags(command: string, values: Record): void { diff --git a/packages/cli/src/cli/commands-content.ts b/packages/cli/src/cli/commands-content.ts index 85f1b3fb..915d14f2 100644 --- a/packages/cli/src/cli/commands-content.ts +++ b/packages/cli/src/cli/commands-content.ts @@ -7,7 +7,8 @@ import { import { describeExport, exportArchive } from "../commands/export.ts"; import { fieldsCommand } from "../commands/fields.ts"; import { ImportInterruptedError, importArchive } from "../commands/import.ts"; -import { describeImport } from "../commands/import-describe.ts"; +import { describeImport, describeRevert } from "../commands/import-describe.ts"; +import { revertImport } from "../commands/import-revert.ts"; import { describeMediaImport, mediaGc, mediaImportUrls, mediaVerify } from "../commands/media.ts"; import { describePlaceholders, mediaPlaceholders } from "../commands/media-placeholders.ts"; import { describeMigration, migrateTags } from "../commands/migrate.ts"; @@ -171,25 +172,37 @@ const exportCommand: CliCommand = async (rest, values, { projectDir }) => { } }; +const IMPORT_USAGE = + "usage: cloudhal import [--replace] [--dry-run] | --revert [--dry-run]"; + +/** `cloudhal import --revert [--dry-run]`; a dry run that would be refused exits 1. */ +const revertCommand: CliCommand = async (rest, values, { projectDir }) => { + if (rest.length > 0 || values.replace || !values.revert) throw new Error(IMPORT_USAGE); + const result = await revertImport({ + projectDir, + snapshotId: values.revert, + dryRun: values["dry-run"] === true, + }); + for (const line of describeRevert(result)) console.log(line); + return result.refusals.length > 0 ? 1 : 0; +}; + /** - * `cloudhal import [--dry-run]` (bootstrap only; `--replace` is RFC-0039 step 4). Ctrl-C or + * `cloudhal import [--replace] [--dry-run]` (RFC-0039 §2), or `--revert `. Ctrl-C or * SIGTERM before the graph is written stops it (exit 130/143); a dry run that would be refused * exits 1. */ -const importCommand: CliCommand = async (rest, values, { projectDir }) => { +const importCommand: CliCommand = async (rest, values, context) => { + if (values.revert !== undefined) return revertCommand(rest, values, context); const [file] = rest; - if (file === undefined || rest.length !== 1) - throw new Error("usage: cloudhal import [--dry-run]"); - if (values.replace) - throw new Error( - "cloudhal import --replace is not available yet; this version imports in bootstrap mode only", - ); + if (file === undefined || rest.length !== 1) throw new Error(IMPORT_USAGE); const interrupt = abortOnSignal(); try { const result = await importArchive({ - projectDir, + projectDir: context.projectDir, file, dryRun: values["dry-run"] === true, + replace: values.replace === true, signal: interrupt.signal, log: (l) => process.stderr.write(`${l}\n`), }); diff --git a/packages/cli/src/cli/help.ts b/packages/cli/src/cli/help.ts index 24a4ae36..6439aa76 100644 --- a/packages/cli/src/cli/help.ts +++ b/packages/cli/src/cli/help.ts @@ -38,13 +38,21 @@ Usage: --site: a site listed in cloudhal.json sites; --no-media: the graph only; --latest-only: only the published and the latest snapshot - cloudhal import [--dry-run] a .cloudhal file into this project (RFC-0039), its - site id kept: the archive's site must be cloudhal.json's - siteId or listed in sites, and absent here or holding - exactly the archive's nodes (missing snapshots and media - are added); every file is verified before anything is - written; --dry-run prints the manifest and the difference - and writes nothing + cloudhal import [--replace] [--dry-run] | --revert [--dry-run] + a .cloudhal file into this project (RFC-0039), its site + id kept; every file is verified before anything is + written. By default the archive's site must be + cloudhal.json's siteId or listed in sites, and absent + here or holding exactly the archive's nodes (missing + snapshots and media are added); --replace: the local + nodes become the archive's, its published snapshot + included, after a "before import " snapshot of + them; an archive of another site switches cloudhal.json + siteId to it (the old site stays in the local store, + unlisted; write its id back to undo); --revert : + back to such a snapshot, revs moved forward, saving the + current graph first; --dry-run prints the manifest and + the difference and writes nothing cloudhal owner transfer --to [--admin-url ] make a member the deployment's Owner (lockout recovery; uses the deployment token) diff --git a/packages/cli/src/commands/archive-bundle.ts b/packages/cli/src/commands/archive-bundle.ts index 4a66c099..d26961d0 100644 --- a/packages/cli/src/commands/archive-bundle.ts +++ b/packages/cli/src/commands/archive-bundle.ts @@ -187,21 +187,29 @@ export function contentCreatedAt(bundle: SiteBundle): string { * RFC-0041 §6: a field operation whose marker (`migration`) is still on a field of the site node * needs its `before` snapshot (the marker's id) to be continued or reverted, and blocks any other * operation on its collection until then. Dropping that snapshot would leave the collection stuck - * after an import, so `--latest-only` is refused until the operation is finished or reverted. + * after an import, so `--latest-only` is refused until the operation is finished or reverted; + * `import --replace` and `--revert`, which would drop the marker itself, are refused too. This is + * the first such operation of a site node's body, if any. */ -function refuseUnfinishedOperations(bundle: SiteBundle): void { - const row = bundle.nodes.find((n) => n.kind === "site" && n.id === bundle.siteId); - const site: unknown = row ? JSON.parse(row.body) : undefined; +export function unfinishedOperation( + site: unknown, +): { readonly collection: string; readonly id: string } | undefined { const collections = isRecord(site) && isRecord(site.collections) ? site.collections : {}; for (const [collection, c] of Object.entries(collections)) { const fields: unknown[] = isRecord(c) && Array.isArray(c.fields) ? c.fields : []; const migration = fields.map((f) => (isRecord(f) ? f.migration : undefined)).find(isRecord); - if (!migration) continue; - const id = String(migration.id); + if (migration) return { collection, id: String(migration.id) }; + } + return undefined; +} + +function refuseUnfinishedOperations(bundle: SiteBundle): void { + const row = bundle.nodes.find((n) => n.kind === "site" && n.id === bundle.siteId); + const op = unfinishedOperation(row ? JSON.parse(row.body) : undefined); + if (op) throw new Error( - `field operation ${id} on ${collection} has not finished, and --latest-only would drop the snapshot it needs; finish or revert it first (\`cloudhal fields continue|revert ${collection} ${id}\`), or export the full history`, + `field operation ${op.id} on ${op.collection} has not finished, and --latest-only would drop the snapshot it needs; finish or revert it first (\`cloudhal fields continue|revert ${op.collection} ${op.id}\`), or export the full history`, ); - } } /** diff --git a/packages/cli/src/commands/import-apply.ts b/packages/cli/src/commands/import-apply.ts new file mode 100644 index 00000000..cc0fb565 --- /dev/null +++ b/packages/cli/src/commands/import-apply.ts @@ -0,0 +1,145 @@ +/** + * `cloudhal import`'s writes, once the archive (or, for `--revert`, the snapshot) was verified and + * planned (RFC-0039 §2). For a `--replace` or `--revert` of a site this project holds: + * + * 1. a `before` snapshot of the current graph (invariant 6: what `cloudhal import --revert` brings + * back). A run that stopped after taking it left it as the latest snapshot: it is reused when it + * still holds the current graph, so a second run takes no duplicate. A site the plan found + * unchanged gets none, so its nodes are checked again here: the plan can be minutes old (pass 2 + * copies media), and nodes another process wrote since are refused, never replaced unsaved; + * 2. `importSite`, keeping the bundle's published snapshot (`published: "source"`, as `cloudhal + * pull` does), its node set one atomic batch; + * 3. an `after` snapshot of the result, so the latest snapshot (what `cloudhal build` renders for a + * site never published and `export --latest-only` keeps) is the imported graph, not the one it + * replaced. A run that stopped after the node batch took it next time: the nodes are then + * unchanged, and the latest snapshot is a `before` of the same import that no longer holds them. + * + * Then the search index is brought to the live snapshot. No signal is honoured from here. + */ +import type { Actor, NodeRef, Snapshot } from "@cloudhal/core-contracts"; +import { type CopySiteResult, importSite, type SiteBundle } from "@cloudhal/core-ts"; +import type { LocalStore } from "@cloudhal/core-ts/local"; +import { ImportRefusal } from "./import-checks.ts"; + +/** Who the CLI's snapshots name, as `cloudhal snapshot`'s do. */ +export const CLI_ACTOR: Actor = { kind: "human", id: "user_localcli0001", name: "cloudhal CLI" }; + +/** A snapshot's label and metadata (`cloudhal.import.*` keys). */ +export interface SnapshotNote { + readonly label: string; + readonly metadata: Readonly>; +} + +/** The snapshots taken around one import's write. */ +export interface ImportBracket { + readonly before: SnapshotNote; + after(beforeId: string): SnapshotNote; +} + +export interface ApplyImportOptions { + readonly mode: "bootstrap" | "replace"; + /** A replace of a site this project holds: the snapshots around the write. */ + readonly bracket?: ImportBracket | undefined; + /** The plan found the site's nodes as the bundle has them (checked again before the write). */ + readonly unchanged?: boolean | undefined; + /** The `before` snapshot exists (taken or reused): failure messages name it from here. */ + readonly onBefore?: ((snapshot: Snapshot) => void) | undefined; +} + +export interface AppliedImport { + readonly result: CopySiteResult; + readonly before?: Snapshot | undefined; + readonly after?: Snapshot | undefined; + readonly searchRefreshed: boolean; +} + +const sig = (refs: readonly Pick[]) => + refs + .map((r) => `${r.id}@${r.rev}:${r.hash}`) + .sort() + .join("\n"); + +/** `snapshot` is a `before` snapshot of this very import (its label and every metadata value). */ +function isBefore(snapshot: Snapshot, note: SnapshotNote): boolean { + const meta = snapshot.metadata ?? {}; + return ( + snapshot.label === note.label && + Object.entries(note.metadata).every(([k, v]) => Object.hasOwn(meta, k) && meta[k] === v) + ); +} + +function take(local: LocalStore, siteId: string, note: SnapshotNote): Promise { + return local.store.createSnapshot(siteId, { + actor: CLI_ACTOR, + reason: "import", + label: note.label, + metadata: { ...note.metadata }, + }); +} + +/** + * The `before` snapshot this write needs: a new one, the one a stopped run took (it still holds + * the current graph), or, when the nodes are still the bundle's, the stopped run's (its `after` + * is missing), else none. Nodes the plan found unchanged that differ now are refused. + */ +async function beforeOf( + local: LocalStore, + bundle: SiteBundle, + options: ApplyImportOptions, +): Promise<{ snapshot?: Snapshot; take: boolean }> { + const { bracket } = options; + if (!bracket) return { take: false }; + const { siteId } = bundle; + const latest = await local.store.latestSnapshot(siteId); + const stopped = latest && isBefore(latest, bracket.before) ? latest : undefined; + const current = + stopped || options.unchanged ? sig(await local.store.currentNodeRefs(siteId)) : undefined; + const holdsCurrent = stopped !== undefined && sig(stopped.nodes) === current; + if (!options.unchanged) return holdsCurrent ? { snapshot: stopped, take: false } : { take: true }; + if (current !== sig(bundle.nodes)) + throw new ImportRefusal( + `site ${siteId} changed after it was planned (another process wrote it); run it again`, + ); + return stopped && !holdsCurrent ? { snapshot: stopped, take: false } : { take: false }; +} + +/** + * The search index is behind the site's live snapshot (an import that stopped after the node set + * landed never refreshed it). + */ +async function searchBehind(local: LocalStore, siteId: string): Promise { + try { + const live = (await local.store.getSite(siteId))?.publishedSnapshotId; + return live !== undefined && (await local.store.searchStatus(siteId))?.snapshotId !== live; + } catch { + return true; + } +} + +/** The snapshots, `importSite` and the search index, in that order (see the module doc). */ +export async function applyImport( + local: LocalStore, + bundle: SiteBundle, + options: ApplyImportOptions, +): Promise { + const { siteId } = bundle; + const found = await beforeOf(local, bundle, options); + const before = + found.take && options.bracket + ? await take(local, siteId, options.bracket.before) + : found.snapshot; + if (before) options.onBefore?.(before); + const result = await importSite(local, bundle, { mode: options.mode }); + const after = + before && options.bracket + ? await take(local, siteId, options.bracket.after(before.id)) + : undefined; + let searchRefreshed = true; + if (!result.nodes.unchanged || result.snapshots > 0 || (await searchBehind(local, siteId))) + // RFC-0019 §3: an import leaves the index as it was; bring it to the live snapshot. + searchRefreshed = await local.store.refreshSearch(siteId, { full: true }).then( + () => true, + () => false, + ); + return { result, before, after, searchRefreshed }; +} diff --git a/packages/cli/src/commands/import-checks.ts b/packages/cli/src/commands/import-checks.ts index 442721c2..aca8193d 100644 --- a/packages/cli/src/commands/import-checks.ts +++ b/packages/cli/src/commands/import-checks.ts @@ -1,13 +1,14 @@ /** - * `cloudhal import`'s rules and words (RFC-0039 §2, rollout step 3), with no I/O: which site an - * archive may bring into this project, why a bootstrap import is refused, and every failure as - * one line. Text that comes from the archive or the project (titles, names, file names, the row + * `cloudhal import`'s rules and words (RFC-0039 §2, rollout steps 3 and 4), with no I/O: which + * site a bootstrap import may bring into this project, why an import is refused, and every failure + * as one line (`import-replace.ts` has `--replace`'s own). Text that comes from the archive or the project (titles, names, file names, the row * ids core-ts messages quote) goes through `displayText`, so it cannot carry terminal escapes, * bidi overrides or megabytes into the terminal. */ import { type ImportPlan, InvariantError, SyncConflictError } from "@cloudhal/core-ts"; import { ArchiveError, type ArchiveErrorCode, type SiteArchiveManifest } from "../archive/index.ts"; import type { ProjectConfig } from "../config.ts"; +import { SiteSwitchError } from "../config-site.ts"; /** * Controls (C0, C1, DEL), format characters (bidi overrides, zero-width), lone surrogates and the @@ -40,10 +41,24 @@ export function copiedNote(copied: number, mediaDir: string): string { return `${copied} media file(s) already copied stay in ${mediaDir}, unreferenced, and the next import reuses them`; } +/** Ctrl-C or SIGTERM before the graph was written. */ +export class ImportInterruptedError extends Error { + override readonly name = "ImportInterruptedError"; + readonly mediaCopied: number; + readonly mediaDir: string; + constructor(mediaCopied: number, mediaDir: string) { + super( + `import interrupted; nothing was imported${mediaCopied > 0 ? `; ${copiedNote(mediaCopied, mediaDir)}` : ""}`, + ); + this.mediaCopied = mediaCopied; + this.mediaDir = mediaDir; + } +} + /** - * The archive's site must be this project's: `cloudhal.json`'s `siteId`, or listed in `sites`. - * Any other id is refused (switching `siteId` is `--replace`'s, RFC-0039 §2); without a local store - * the user may switch it by hand, knowing that push and deploy follow it. + * Bootstrap: the archive's site must be this project's, `cloudhal.json`'s `siteId` or listed in + * `sites`. Any other id is refused, pointing to `--replace`, which imports it and switches + * `siteId` (RFC-0039 §2). */ export function siteRefusal( cfg: ProjectConfig, @@ -56,30 +71,32 @@ export function siteRefusal( ? `sites are ${allowed.map((id) => displayText(id, 80)).join(", ")} (cloudhal.json siteId and sites)` : `site is ${displayText(cfg.siteId, 80)} (cloudhal.json siteId)`; const a = manifest.siteId; - const hint = storeExists - ? "" - : `; this project has no local store yet, so setting cloudhal.json siteId to ${a} also works (push and deploy then use ${a})`; - return `it holds site ${a}, and this project's ${ours}; importing another site into a project comes with \`cloudhal import --replace\`, not available yet${hint}`; + const old = storeExists + ? ` (${displayText(cfg.siteId, 80)} stays in the local store, unlisted)` + : ""; + return `it holds site ${a}, and this project's ${ours}; \`cloudhal import --replace\` imports it and switches cloudhal.json siteId to ${a}${old}`; } /** A bootstrap conflict, from the plan or from `SyncConflictError.differing` (never its message). */ export function conflictText(siteId: string, differing: readonly string[]): string { const shown = differing.slice(0, 5).map((id) => displayText(id, 80)); - return `this project's site ${displayText(siteId, 80)} differs from the archive in ${differing.length} node(s) (${shown.join(", ")}${differing.length > 5 ? ", …" : ""}); bootstrap imports only into a project without the site or holding exactly its nodes, and --replace is not available yet`; + return `this project's site ${displayText(siteId, 80)} differs from the archive in ${differing.length} node(s) (${shown.join(", ")}${differing.length > 5 ? ", …" : ""}); bootstrap imports only into a project without the site or holding exactly its nodes; --replace makes them the archive's, saving the local ones as a snapshot first`; } /** - * Why a bootstrap import of this plan is refused, in order: nodes that differ, media this project - * has taken down (invariant 5: an import never writes them back), and media a `--no-media` archive - * cannot bring that the project lacks or holds at another size. Empty: the import may proceed. + * Why an import of this plan is refused, in order: nodes that differ (bootstrap only), media this + * project has taken down (invariant 5: an import never writes them back), and media a + * `--no-media` archive cannot bring that the project lacks or holds at another size. Empty: the + * import may proceed. */ -export function bootstrapRefusals( +export function importRefusals( plan: ImportPlan, manifest: SiteArchiveManifest, dataDir: string, ): string[] { const out: string[] = []; - if (plan.conflict) out.push(conflictText(plan.siteId, plan.conflict)); + if (plan.mode === "bootstrap" && plan.conflict) + out.push(conflictText(plan.siteId, plan.conflict)); const down = plan.media.takenDownHere; if (down[0]) out.push( @@ -96,7 +113,7 @@ export function bootstrapRefusals( return out; } -/** A refusal of this module's own (`siteRefusal`, `bootstrapRefusals`): its parts are cleaned. */ +/** A refusal of the import's own (`siteRefusal`, `importRefusals`, `import-replace.ts`): its parts are cleaned. */ export class ImportRefusal extends Error { override readonly name = "ImportRefusal"; } @@ -130,6 +147,8 @@ export interface ImportFailureContext { readonly siteId?: string | undefined; /** The graph was being written (`importSite`, and opening the store for it). */ readonly applying?: boolean | undefined; + /** `--replace`: the snapshot of the local graph taken before `importSite`. */ + readonly snapshotId?: string | undefined; } /** @@ -145,24 +164,34 @@ function refusedBeforeWriting(error: unknown): boolean { } /** - * Any failure as one line: `cannot import : ; nothing was imported`, and what - * happened to the media already copied. A failure while the graph was being written (a storage + * Any failure as one line: `cannot import : ; nothing was imported`, what happened + * to the media already copied, and the snapshot `--replace` took first. A failure while the graph was being written (a storage * error, or a store another process changed since the plan) may have left part of it, so it says * so and to run the import again, which completes it. Every reason is cleaned and cut * (`displayText`; 400 code points, or 1000 for this module's own refusals), since row ids, kinds - * and validator paths in core-ts messages come from the archive. + * and validator paths in core-ts messages come from the archive. A `cloudhal.json` that could not + * be switched after the import landed says so (`SiteSwitchError`). */ export function importFailure(error: unknown, at: ImportFailureContext): Error { + if (error instanceof SiteSwitchError) + return new Error(`imported ${displayText(at.file)}, but ${displayText(error.message, 600)}`, { + cause: error, + }); const max = error instanceof ImportRefusal ? 1000 : 400; const head = `cannot import ${displayText(at.file)}: ${displayText(reason(error, at.siteId), max)}`; + const snap = at.snapshotId; if (at.applying && !refusedBeforeWriting(error)) { const reused = at.mediaCopied > 0 ? ` (the ${at.mediaCopied} media file(s) copied are reused)` : ""; + const saved = snap + ? `; snapshot ${snap} taken first holds the site as it was (undo: cloudhal import --revert ${snap})` + : ""; return new Error( - `${head}; the import stopped after it began writing, so this project may hold part of it; run it again to finish${reused}`, + `${head}; the import stopped after it began writing, so this project may hold part of it; run it again to finish${reused}${saved}`, { cause: error }, ); } const copied = at.mediaCopied > 0 ? `; ${copiedNote(at.mediaCopied, at.mediaDir)}` : ""; - return new Error(`${head}; nothing was imported${copied}`, { cause: error }); + const saved = snap ? `; snapshot ${snap} taken first holds the current graph` : ""; + return new Error(`${head}; nothing was imported${copied}${saved}`, { cause: error }); } diff --git a/packages/cli/src/commands/import-describe-replace.ts b/packages/cli/src/commands/import-describe-replace.ts new file mode 100644 index 00000000..0b7e6d9e --- /dev/null +++ b/packages/cli/src/commands/import-describe-replace.ts @@ -0,0 +1,105 @@ +/** + * What `cloudhal import --replace` and `--revert` add to the dry run and the summary (RFC-0039 §2): + * the published snapshot when the write moves it, the snapshot saved first and how to undo with + * it, the nodes whose rev a stale editor could still save over, the `cloudhal.json` switch with + * how to undo it and the old site left in the store, and warnings for a deployed project and the + * old site's open releases. Pure. + */ +import type { ImportPlan } from "@cloudhal/core-ts"; +import { displayText } from "./import-checks.ts"; +import type { ReplaceOutcome } from "./import-replace.ts"; + +export interface ReplaceLinesOptions { + readonly dryRun: boolean; + readonly plan?: ImportPlan | undefined; + /** Where the incoming nodes come from: `the archive`, `the snapshot`. */ + readonly from: string; +} + +const quoted = (label: string) => `"${displayText(label, 200)}"`; + +function publishedLines(outcome: ReplaceOutcome, options: ReplaceLinesOptions): string[] { + const p = outcome.published; + if (!p) return []; + const incoming = p.incoming ? `snapshot ${p.incoming}` : "none"; + return [ + `published: ${incoming}, as ${options.from} has it (${options.dryRun ? "here" : "was"}: ${p.here ?? "none"})`, + ]; +} + +function savedLines(outcome: ReplaceOutcome, options: ReplaceLinesOptions): string[] { + const s = outcome.snapshot; + if (s?.beforeId) + return [ + `saved first: snapshot ${s.beforeId} ${quoted(s.label)} holds the site as it was (undo: cloudhal import --revert ${s.beforeId})${s.afterId ? `; ${s.afterId} ${quoted(s.afterLabel)} holds the result` : ""}`, + ]; + if (!options.dryRun) return []; + if (s) + return [ + `saved first: this project's site (${s.nodes} nodes) as snapshot ${quoted(s.label)}, the result as ${quoted(s.afterLabel)} (undo: cloudhal import --revert )`, + ]; + const plan = options.plan; + if (!plan) return []; + return [ + `saved first: nothing (${plan.fresh ? "the site is new to this project" : "its nodes are unchanged"})`, + ]; +} + +function revLines(outcome: ReplaceOutcome, options: ReplaceLinesOptions): string[] { + const n = outcome.revsReused; + if (!n) return []; + const [verb, still] = options.dryRun ? ["would take", "holds"] : ["took", "still holds"]; + return [ + `note: ${n} node(s) ${verb} ${options.from}'s rev, no higher than theirs here; reload Studio tabs open on this site: an edit a tab or agent ${still} at that rev saves over them without a conflict`, + ]; +} + +function switchLines(outcome: ReplaceOutcome, dryRun: boolean): string[] { + const c = outcome.switch; + if (!c || (!dryRun && !c.written)) return []; + const sites = c.sites ? `, sites ${c.sites.before.join(", ")} → ${c.sites.after.join(", ")}` : ""; + // The switch only put `to` in place of `from`: undone by hand (--revert restores nodes only). + const undo = `undo: write ${c.from} back in place of ${c.to} by hand; --revert does not`; + const old = outcome.oldSite; + const stays = !old + ? "" + : dryRun + ? `; site ${c.from} would stay in the local store, unlisted` + : `; site ${c.from} stays in the local store, unlisted and untouched (\`cloudhal site delete ${c.from} --confirm ${displayText(old.slug, 80)}\` deletes it for good)`; + const lines = [`cloudhal.json: siteId ${c.from} → ${c.to}${sites} (${undo})${stays}`]; + if (old && old.openReleases > 0) + lines.push( + `warning: site ${c.from} has ${old.openReleases} open release(s); the scheduler of cloudhal dev, studio and mcp still runs them until they are cancelled in Studio`, + ); + return lines; +} + +function deployedLines(outcome: ReplaceOutcome, dryRun: boolean): string[] { + const name = outcome.deployed; + if (name === undefined) return []; + const c = outcome.switch; + // Never `--admin-url`: a push sends this project's deploy token, which no other deployment takes. + if (c) + return [ + `warning: this project is deployed (${name}) and its deployment serves site ${c.from}: \`cloudhal push\` ${dryRun ? "would send" : "now sends"} site ${c.to} there as another site, and the next \`cloudhal deploy\` serves it instead; a deployment of site ${c.to} itself is updated from the project that deployed it, not from here`, + ]; + if (!outcome.snapshot) return []; + return [ + `note: this project is deployed (${name}); \`cloudhal push --replace\` sends this graph there, and the deployment keeps its own published snapshot`, + ]; +} + +/** The lines a replace (or revert) adds; none without an outcome. */ +export function replaceLines( + outcome: ReplaceOutcome | undefined, + options: ReplaceLinesOptions, +): string[] { + if (!outcome) return []; + return [ + ...publishedLines(outcome, options), + ...savedLines(outcome, options), + ...revLines(outcome, options), + ...switchLines(outcome, options.dryRun), + ...deployedLines(outcome, options.dryRun), + ]; +} diff --git a/packages/cli/src/commands/import-describe.ts b/packages/cli/src/commands/import-describe.ts index a72e2873..b4d529fb 100644 --- a/packages/cli/src/commands/import-describe.ts +++ b/packages/cli/src/commands/import-describe.ts @@ -1,7 +1,8 @@ /** * What `cloudhal import` prints (RFC-0039 §2): the manifest, the difference a `--dry-run` finds - * (nodes added, changed and removed by kind and title; snapshots; media to copy) and the summary of - * an import. Pure. A title is read from its row's body, with the row id as the fallback, and goes + * (nodes added, changed and removed by kind and title; snapshots; media to copy), the summary of + * an import, and the same for `--revert` (`--replace`'s own lines: `import-describe-replace.ts`). + * Pure. A title is read from its row's body, with the row id as the fallback, and goes * through `displayText`; no body is ever printed. */ import type { NodeRef } from "@cloudhal/core-contracts"; @@ -10,6 +11,8 @@ import { isRecord } from "./archive-bundle.ts"; import { formatBytes } from "./export.ts"; import type { ImportArchiveResult } from "./import.ts"; import { displayText } from "./import-checks.ts"; +import { replaceLines } from "./import-describe-replace.ts"; +import type { RevertImportResult } from "./import-revert.ts"; type Kind = NodeRef["kind"]; @@ -153,7 +156,15 @@ function capped(lines: string[]): string[] { return [...lines.slice(0, MAX_LINES), ` … and ${lines.length - MAX_LINES} more`]; } -function nodeLines(plan: ImportPlan, locale: string | undefined): string[] { +/** + * The plan's nodes by kind and title; `source` names where the incoming revs are (a revert's are + * its own: they move forward). + */ +export function nodeLines( + plan: ImportPlan, + locale: string | undefined, + source = "in the archive", +): string[] { const { added, changed, removed } = plan.nodes; if (plan.unchanged) return ["nodes: unchanged"]; const line = (sign: string, r: NodeRow) => ` ${sign} ${label(r.kind)} ${nodeTitle(r, locale)}`; @@ -162,7 +173,7 @@ function nodeLines(plan: ImportPlan, locale: string | undefined): string[] { ...capped(sorted(added, (r) => r).map((r) => line("+", r))), ...capped( sorted(changed, (c) => c.to).map( - (c) => `${line("~", c.to)}, rev ${c.from.rev} here, ${c.to.rev} in the archive`, + (c) => `${line("~", c.to)}, rev ${c.from.rev} here, ${c.to.rev} ${source}`, ), ), ...capped(sorted(removed, (r) => r).map((r) => line("-", r))), @@ -191,7 +202,7 @@ function planMediaLine(result: ImportArchiveResult, plan: ImportPlan): string { function planLines(result: ImportArchiveResult, plan: ImportPlan): string[] { const lines = [ - `dry run: bootstrap import into site ${result.site.id}, ${plan.fresh ? "which is new to this project" : "which this project already holds"}`, + `dry run: ${plan.mode} import into site ${result.site.id}, ${plan.fresh ? "which is new to this project" : "which this project already holds"}`, ]; if (plan.projectAdded) lines.push(`project: ${plain(result.site.projectId ?? "its project")} is new to this project`); @@ -236,8 +247,12 @@ function importedLines( ): string[] { const file = displayText(result.file); const { id, slug } = result.site; - // Recording a takedown or deleting a local copy of taken-down content is a change too. + const replaced = result.replace; + // Recording a takedown or deleting a local copy of taken-down content is a change too, and so + // are a snapshot a stopped replace still owed and a cloudhal.json switch. const nothing = + replaced?.snapshot === undefined && + replaced?.switch?.written !== true && done.nodes.unchanged && done.snapshots === 0 && done.blobs === 0 && @@ -249,9 +264,10 @@ function importedLines( return [ `nothing to import: this project already holds ${file} (site ${id}${slug ? `, ${plain(slug)}` : ""})`, ]; + const overwrote = result.mode === "replace" && done.nodes.deleted > 0; const lines = [ - `imported ${file}: site ${id}${slug ? ` (${plain(slug)})` : ""} is in this project`, - `nodes: ${done.nodes.unchanged ? "unchanged" : `${done.nodes.inserted} written`}; snapshots: ${done.snapshots} added; node bodies: ${done.blobs} added`, + `imported ${file}: site ${id}${slug ? ` (${plain(slug)})` : ""} is in this project${overwrote ? ", its nodes replaced by the archive's" : ""}`, + `nodes: ${done.nodes.unchanged ? "unchanged" : `${done.nodes.inserted} written${overwrote ? ` (${done.nodes.deleted} replaced)` : ""}`}; snapshots: ${done.snapshots} added; node bodies: ${done.blobs} added`, ]; const media = [ ...(result.mediaNeeded.files > 0 @@ -266,11 +282,13 @@ function importedLines( : []), ]; if (media.length > 0) lines.push(`media: ${media.join("; ")}`); - if (!done.searchRefreshed) - lines.push("search index not refreshed; run `cloudhal search reindex`"); + lines.push(...replaceLines(replaced, { dryRun: false, plan, from: "the archive" })); + if (!done.searchRefreshed) lines.push(SEARCH_BEHIND); return lines; } +const SEARCH_BEHIND = "search index not refreshed; run `cloudhal search reindex`"; + /** The lines `cloudhal import` prints to stdout: the dry run's report, or the import's summary. */ export function describeImport(result: ImportArchiveResult): string[] { const { plan, imported } = result; @@ -278,6 +296,33 @@ export function describeImport(result: ImportArchiveResult): string[] { return [ ...manifestLines(result), ...(plan ? planLines(result, plan) : []), + ...replaceLines(result.replace, { dryRun: true, plan, from: "the archive" }), + ...result.refusals.map((r) => `would refuse: ${r}`), + "nothing written (dry run)", + ]; +} + +/** The lines `cloudhal import --revert` prints: the dry run's report, or the revert's summary. */ +export function describeRevert(result: RevertImportResult): string[] { + const { snapshot: s, plan, outcome, reverted } = result; + const what = `snapshot ${s.id} "${displayText(s.label ?? "", 200)}" (site ${s.siteId}, ${s.createdAt})`; + const options = { plan, from: "the snapshot" }; + if (reverted) { + if (reverted.nodes.unchanged && outcome?.snapshot === undefined) + return [`nothing to revert: the site already holds the nodes of ${what}`]; + return [ + `reverted to ${what}`, + `nodes: ${reverted.nodes.unchanged ? "unchanged" : `${reverted.nodes.inserted} written (${reverted.nodes.deleted} replaced)`}`, + ...replaceLines(outcome, { ...options, dryRun: false }), + ...(reverted.searchRefreshed ? [] : [SEARCH_BEHIND]), + ]; + } + return [ + `revert to ${what}`, + ...(plan + ? [`dry run: replace the site's nodes`, ...nodeLines(plan, undefined, "after the revert")] + : []), + ...replaceLines(outcome, { ...options, dryRun: true }), ...result.refusals.map((r) => `would refuse: ${r}`), "nothing written (dry run)", ]; diff --git a/packages/cli/src/commands/import-read.ts b/packages/cli/src/commands/import-read.ts new file mode 100644 index 00000000..d4ab478f --- /dev/null +++ b/packages/cli/src/commands/import-read.ts @@ -0,0 +1,93 @@ +/** + * `cloudhal import`'s reads of the file (RFC-0039 §2): its size (a missing file or a folder is + * refused before anything is read), one pass over the whole archive, verified and reassembled into + * a bundle with its manifest checked before the rest is read, and what the result says about the + * archive's site, read from that bundle (untrusted until planned: printed cleaned). + */ +import { createReadStream } from "node:fs"; +import { stat } from "node:fs/promises"; +import { addAbortSignal } from "node:stream"; +import type { SiteBundle } from "@cloudhal/core-ts"; +import { type ArchiveItem, readArchive, type SiteArchiveManifest } from "../archive/index.ts"; +import type { ProjectConfig } from "../config.ts"; +import { isRecord } from "./archive-bundle.ts"; +import { type ArchiveMediaItem, bundleFromArchive } from "./archive-reassemble.ts"; + +export async function statOrUndefined(file: string) { + return stat(file).catch((e: NodeJS.ErrnoException) => { + if (e.code === "ENOENT" || e.code === "ENOTDIR") return undefined; + throw e; + }); +} + +/** The archive's size; a missing file or a folder is refused before anything is read. */ +export async function archiveSize(file: string): Promise { + const s = await statOrUndefined(file); + if (!s) throw new Error("no such file"); + if (s.isDirectory()) throw new Error("it is a directory; name a .cloudhal file"); + return s.size; +} + +/** The first item (the manifest) checked by `check` before the rest is read. */ +async function* checkManifest( + items: AsyncIterable, + check: (manifest: SiteArchiveManifest) => void, +): AsyncGenerator { + let first = true; + for await (const item of items) { + if (first && item.type === "manifest") check(item.manifest); + first = false; + yield item; + } +} + +export interface ReadPassOptions { + readonly file: string; + readonly signal?: AbortSignal | undefined; + /** Throws to refuse the archive once its manifest is read. */ + readonly check: (manifest: SiteArchiveManifest) => void; + readonly onMedia?: ((item: ArchiveMediaItem) => Promise) | undefined; +} + +/** One read of the whole file: verified, reassembled, its media handed to `onMedia`. */ +export async function readPass( + options: ReadPassOptions, +): Promise<{ manifest: SiteArchiveManifest; bundle: SiteBundle }> { + const stream = createReadStream(options.file); + if (options.signal) addAbortSignal(options.signal, stream); + try { + const { onMedia } = options; + const read = await bundleFromArchive( + checkManifest(readArchive(stream), options.check), + onMedia ? { onMedia } : {}, + ); + return { manifest: read.manifest, bundle: read.bundle }; + } finally { + stream.destroy(); + } +} + +/** What the result says about the archive's site and media, read from pass 1's bundle. */ +export function siteSummary(cfg: ProjectConfig, manifest: SiteArchiveManifest, bundle: SiteBundle) { + const row = bundle.nodes.find((n) => n.kind === "site" && n.id === manifest.siteId); + let site: unknown; + try { + site = row ? JSON.parse(row.body) : undefined; + } catch { + site = undefined; + } + const field = (key: string) => + isRecord(site) && typeof site[key] === "string" ? (site[key] as string) : undefined; + const media = bundle.media ?? []; + return { + site: { + id: manifest.siteId, + slug: field("slug"), + locale: field("defaultLocale"), + projectId: field("projectId"), + published: field("publishedSnapshotId"), + listed: (cfg.sites ?? [cfg.siteId]).includes(manifest.siteId), + }, + mediaNeeded: { files: media.length, bytes: media.reduce((n, m) => n + m.bytes, 0) }, + }; +} diff --git a/packages/cli/src/commands/import-replace.ts b/packages/cli/src/commands/import-replace.ts new file mode 100644 index 00000000..bfe565e7 --- /dev/null +++ b/packages/cli/src/commands/import-replace.ts @@ -0,0 +1,226 @@ +/** + * The rules and words of `cloudhal import --replace` and `--revert` (RFC-0039 §2, rollout step 4), + * with reads only: + * + * - **Snapshots** around a write that changes a site this project holds: `before import ` + * and `import ` (`before revert import ` and `revert import `), reason + * `import`, with `cloudhal.import.*` metadata: the phase, the file's name (its base name only, + * cleaned and cut, since history travels in archives), the `before` one an `after` follows, and + * the snapshot a revert went back to. A `before` snapshot is what `--revert` accepts. + * - **Refusals** of a replace that would change the local site's nodes: open releases, whose + * pinned versions it would replace (RFC-0008; `cloudhal push` refuses them the same way), and an + * unfinished field operation, whose marker on the site node it would drop (RFC-0041). There is + * no `--cancel-schedules` here: releases are cancelled in Studio or through MCP. + * - **Hostnames** another site of the local store serves are refused, never released: releasing + * them would write the other site, which a replace leaves as it is. + * - **The site id switch:** an archive of a site `cloudhal.json` does not list switches `siteId` to + * it (`config-site.ts`); the old site stays in the store, unlisted and untouched, so setting + * `siteId` back by hand undoes the switch. + * - **Revs:** a replace writes the archive's revs, as `pull --replace` does, and the store's + * compare-and-swap checks only the rev, so the nodes it gives a rev no higher than they had are + * counted: an editor still holding that rev saves over them without a conflict. A revert's + * revs only move forward (`restoreBundle`), so it has none. + * - **What is shown:** the published snapshot before and after, the snapshots, those nodes, the + * switch, the old site and the deployment (`state.json` only; never the deploy secrets). + */ +import path from "node:path"; +import type { Snapshot } from "@cloudhal/core-contracts"; +import { type ImportPlan, InvariantError } from "@cloudhal/core-ts"; +import type { LocalStore } from "@cloudhal/core-ts/local"; +import { ReleaseService } from "@cloudhal/core-ts/releases"; +import { isSiteId, type ProjectConfig } from "../config.ts"; +import { planSiteIdSwitch, type SiteIdSwitch } from "../config-site.ts"; +import { unfinishedOperation } from "./archive-bundle.ts"; +import { readDeployState } from "./deploy/state.ts"; +import type { AppliedImport, ImportBracket } from "./import-apply.ts"; +import { displayText } from "./import-checks.ts"; + +/** The metadata keys of the snapshots an import takes. */ +export const IMPORT_META = { + phase: "cloudhal.import.phase", + file: "cloudhal.import.file", + of: "cloudhal.import.of", + revert: "cloudhal.import.revert", +} as const; + +/** + * The file as labels name it: its base name, cleaned, at most 160 code points (the longest label, + * `before revert import `, stays within the snapshot label's 200). + */ +export function importFileName(file: string): string { + return displayText(path.basename(file), 160); +} + +function bracket(prefix: string, file: string, extra: Record): ImportBracket { + const { phase, of } = IMPORT_META; + const meta = { [IMPORT_META.file]: file, ...extra }; + return { + before: { label: `before ${prefix}${file}`, metadata: { [phase]: "before", ...meta } }, + after: (id) => ({ + label: `${prefix}${file}`, + metadata: { [phase]: "after", ...meta, [of]: id }, + }), + }; +} + +/** `before import ` and `import `. */ +export function importBracket(file: string): ImportBracket { + return bracket("import ", importFileName(file), {}); +} + +/** `before revert import ` and `revert import `, for a revert to `target`. */ +export function revertBracket(target: Snapshot): ImportBracket { + const named = displayText(target.metadata?.[IMPORT_META.file] ?? "", 160) || target.id; + return bracket("revert import ", named, { [IMPORT_META.revert]: target.id }); +} + +/** + * Why a replace (or revert) of `plan` is refused, for a site this project holds whose nodes it + * would change: open releases, an unfinished field operation. Empty otherwise. + */ +export async function replaceRefusals( + local: LocalStore | undefined, + plan: ImportPlan, + flag: "--replace" | "--revert", +): Promise { + if (!local || plan.fresh || plan.unchanged) return []; + const again = flag === "--replace" ? "import again" : "revert again"; + const out: string[] = []; + const open = await new ReleaseService({ graph: local.store }).openReleases(plan.siteId); + if (open.length > 0) { + const shown = open + .slice(0, 5) + .map((r) => `${r.id} (${r.state}${r.title ? `, "${displayText(r.title, 60)}"` : ""})`); + out.push( + `site ${plan.siteId} has ${open.length} open release(s) (${shown.join(", ")}${open.length > 5 ? ", …" : ""}), and ${flag} replaces the graph their pinned versions point into; cancel them in Studio (\`cloudhal studio\`) or with the MCP tool cancel_schedule, then ${again}`, + ); + } + const op = unfinishedOperation(await local.store.getSite(plan.siteId)); + if (op) { + const c = displayText(op.collection, 80); + const id = displayText(op.id, 80); + out.push( + `field operation ${id} on ${c} has not finished, and ${flag} would drop its marker; finish or revert it first (\`cloudhal fields continue|revert ${c} ${id}\`), then ${again}`, + ); + } + return out; +} + +/** `planImport`'s `host_taken` as a refusal; undefined for any other error. */ +export function hostRefusal(error: unknown, whose: string): string | undefined { + if (!(error instanceof InvariantError) || error.code !== "host_taken") return undefined; + const host = displayText(String(error.details.host ?? ""), 120); + const site = displayText(String(error.details.siteId ?? ""), 80); + return `${whose} site serves ${host}, which site ${site} of this project's store already serves; remove it from that site's domains in Studio, or delete that site when it is a leftover (\`cloudhal site delete ${site} --confirm \`)`; +} + +/** + * Whether `--replace` of an archive of `siteId` switches `cloudhal.json`: when it names neither as + * `siteId` nor in `sites`. Refused for an id `sites` would not accept, or an edit that cannot be + * made in place. + */ +export function switchFor( + cfg: ProjectConfig, + raw: string, + siteId: string, +): { readonly change?: SiteIdSwitch; readonly refusal?: string } { + if ((cfg.sites ?? [cfg.siteId]).includes(siteId)) return {}; + if (!isSiteId(siteId)) + return { + refusal: `it holds site ${displayText(siteId, 80)}, which cloudhal.json cannot name (site_ and 6 to 32 letters or digits)`, + }; + const planned = planSiteIdSwitch(raw, siteId); + return "refusal" in planned ? { refusal: planned.refusal } : { change: planned }; +} + +/** What a replace (or revert) did or would do, for the dry run and the summary. */ +export interface ReplaceOutcome { + /** The snapshots around the write: planned (labels, nodes saved), then taken (ids). */ + readonly snapshot?: + | { + readonly label: string; + readonly afterLabel: string; + readonly nodes: number; + readonly beforeId?: string | undefined; + readonly afterId?: string | undefined; + } + | undefined; + /** The site's published snapshot here and in the bundle, when the write moves it. */ + readonly published?: { readonly here?: string; readonly incoming?: string } | undefined; + /** Nodes the write gives a rev no higher than they have here (see the module doc). */ + readonly revsReused?: number | undefined; + readonly switch?: (SiteIdSwitch & { readonly written?: boolean }) | undefined; + /** After a switch, the old site in the local store. */ + readonly oldSite?: { readonly slug: string; readonly openReleases: number } | undefined; + /** The deployment's name when this project is deployed. */ + readonly deployed?: string | undefined; +} + +export interface OutcomeInput { + readonly projectDir: string; + readonly cfg: ProjectConfig; + readonly plan: ImportPlan; + readonly local?: LocalStore | undefined; + readonly change?: SiteIdSwitch | undefined; + readonly bracket: ImportBracket; + /** The bundle's site node's `publishedSnapshotId`. */ + readonly incoming?: string | undefined; +} + +/** The old site after a switch: its slug (for `cloudhal site delete`) and open releases. */ +async function oldSiteOf(local: LocalStore | undefined, change: SiteIdSwitch | undefined) { + const old = change && local ? await local.store.getSite(change.from) : undefined; + if (!old || !local) return undefined; + const open = await new ReleaseService({ graph: local.store }).openReleases(old.id); + return { slug: old.slug, openReleases: open.length }; +} + +/** The outcome as planned, before anything is written. */ +export async function replaceOutcome(input: OutcomeInput): Promise { + const { plan, local, change, incoming } = input; + const holds = local !== undefined && !plan.fresh; + const here = holds ? (await local.store.getSite(plan.siteId))?.publishedSnapshotId : undefined; + const snapshot = + holds && !plan.unchanged + ? { + label: input.bracket.before.label, + afterLabel: input.bracket.after("").label, + nodes: (await local.store.currentNodeRefs(plan.siteId)).length, + } + : undefined; + const oldSite = await oldSiteOf(local, change); + const deploy = await readDeployState(input.projectDir, input.cfg.dataDir); + const reused = plan.nodes.changed.filter((c) => c.to.rev <= c.from.rev).length; + return { + ...(snapshot ? { snapshot } : {}), + ...(holds && here !== incoming + ? { published: { ...(here ? { here } : {}), ...(incoming ? { incoming } : {}) } } + : {}), + ...(reused > 0 ? { revsReused: reused } : {}), + ...(change ? { switch: change } : {}), + ...(oldSite ? { oldSite } : {}), + ...(deploy ? { deployed: displayText(String(deploy.name), 80) } : {}), + }; +} + +/** The outcome once written: the snapshots taken, and whether `cloudhal.json` was switched. */ +export function settledOutcome( + outcome: ReplaceOutcome, + applied: AppliedImport, + switched: boolean, +): ReplaceOutcome { + const { before, after } = applied; + return { + ...outcome, + snapshot: before + ? { + label: before.label ?? "", + afterLabel: after?.label ?? "", + nodes: before.nodes.length, + beforeId: before.id, + ...(after ? { afterId: after.id } : {}), + } + : undefined, + ...(outcome.switch ? { switch: { ...outcome.switch, written: switched } } : {}), + }; +} diff --git a/packages/cli/src/commands/import-revert.ts b/packages/cli/src/commands/import-revert.ts new file mode 100644 index 00000000..07cd844a --- /dev/null +++ b/packages/cli/src/commands/import-revert.ts @@ -0,0 +1,196 @@ +/** + * `cloudhal import --revert [--dry-run]` (RFC-0039 §2 and invariant 6, as amended + * for rollout step 4): a site back to a `before import` snapshot that `--replace` took. Studio's + * rollback moves only the published pointer, and only to an ancestor of the published snapshot, + * so it cannot undo a replace. This restores the snapshot's nodes, the site node and so its + * published snapshot included, through `importSite`'s replace (`restoreBundle`): validated, one + * atomic batch, hostnames checked. Revs only move forward: a node edited since comes back at the + * rev after its current one, never at a rev it had, so a Studio tab or agent still holding one + * gets a conflict instead of saving over it (the store's compare-and-swap checks only the rev). + * The current graph is saved first as `before revert import `, itself a snapshot `--revert` + * accepts, so edits made after the import are never lost, and the result as `revert import `. + * It restores nodes only: a `cloudhal.json` switch is undone by hand. + * + * Refused before anything is written, as a replace is: a snapshot that is not a `before` one of a + * site `cloudhal.json` lists, open releases, an unfinished field operation, a hostname another + * local site serves, media this project no longer holds. Taken-down content stays down + * (invariant 5): its asset comes back, its bytes do not. + */ +import path from "node:path"; +import type { Snapshot } from "@cloudhal/core-contracts"; +import { + type CopySiteResult, + type ImportPlan, + InvariantError, + planImport, + restoreBundle, + type SiteBundle, +} from "@cloudhal/core-ts"; +import { type LocalStore, openLocalStore } from "@cloudhal/core-ts/local"; +import { type ProjectConfig, readProjectConfig } from "../config.ts"; +import { applyImport } from "./import-apply.ts"; +import { displayText, ImportRefusal } from "./import-checks.ts"; +import { statOrUndefined } from "./import-read.ts"; +import { + hostRefusal, + IMPORT_META, + type ReplaceOutcome, + replaceOutcome, + replaceRefusals, + revertBracket, + settledOutcome, +} from "./import-replace.ts"; + +export interface RevertImportOptions { + readonly projectDir: string; + readonly snapshotId: string; + /** Plan only; write nothing. */ + readonly dryRun?: boolean | undefined; +} + +export interface RevertImportResult { + /** The snapshot gone back to. */ + readonly snapshot: Snapshot; + readonly dryRun: boolean; + /** Absent when refused before planning (a hostname). */ + readonly plan?: ImportPlan | undefined; + /** Why the revert is refused (a dry run reports them; a revert throws the first). */ + readonly refusals: readonly string[]; + readonly outcome?: ReplaceOutcome | undefined; + readonly reverted?: + | { readonly nodes: CopySiteResult["nodes"]; readonly searchRefreshed: boolean } + | undefined; +} + +/** What one revert has done so far, for its failure message. */ +interface State { + applying: boolean; + snapshotId?: string | undefined; +} + +/** The snapshot to go back to: a `before` one of a site `cloudhal.json` lists. */ +async function target(local: LocalStore, cfg: ProjectConfig, id: string): Promise { + const snapshot = await local.store.getSnapshot(id); + if (snapshot?.metadata?.[IMPORT_META.phase] !== "before") + throw new ImportRefusal( + 'it is not a "before import" snapshot of this project (`cloudhal import --replace` names the one it takes)', + ); + if (!(cfg.sites ?? [cfg.siteId]).includes(snapshot.siteId)) + throw new ImportRefusal( + `it is a snapshot of site ${snapshot.siteId}, which cloudhal.json does not list (siteId and sites)`, + ); + return snapshot; +} + +/** Media the snapshot's assets need that this project lacks or holds at another size. */ +function mediaRefusals(plan: ImportPlan): string[] { + const lacking = [...plan.media.missing, ...plan.media.wrongSize]; + const [first] = lacking; + if (!first) return []; + return [ + `this project lacks or has a damaged copy of ${lacking.length} media file(s) the snapshot needs (first ${first.sha256})`, + ]; +} + +/** The restored site node's `publishedSnapshotId`. */ +function publishedOf(bundle: SiteBundle): string | undefined { + const row = bundle.nodes.find((n) => n.kind === "site" && n.id === bundle.siteId); + const body = row ? (JSON.parse(row.body) as { publishedSnapshotId?: unknown }) : undefined; + return typeof body?.publishedSnapshotId === "string" ? body.publishedSnapshotId : undefined; +} + +/** The plan, or a hostname another local site serves as a refusal. */ +async function planRevert( + local: LocalStore, + bundle: SiteBundle, +): Promise<{ plan: ImportPlan } | { refusal: string }> { + try { + return { plan: await planImport(local, bundle, { mode: "replace", report: true }) }; + } catch (e) { + const refusal = hostRefusal(e, "the snapshot's"); + if (refusal === undefined) throw e; + return { refusal }; + } +} + +async function runRevert( + local: LocalStore, + options: RevertImportOptions & { cfg: ProjectConfig }, + state: State, +): Promise { + const snapshot = await target(local, options.cfg, options.snapshotId); + const bundle = await restoreBundle(local, snapshot); + const dryRun = options.dryRun === true; + const planned = await planRevert(local, bundle); + if ("refusal" in planned) { + if (!dryRun) throw new ImportRefusal(planned.refusal); + return { snapshot, dryRun, refusals: [planned.refusal] }; + } + const { plan } = planned; + const refusals = [...mediaRefusals(plan), ...(await replaceRefusals(local, plan, "--revert"))]; + const bracket = revertBracket(snapshot); + const outcome = await replaceOutcome({ + projectDir: options.projectDir, + cfg: options.cfg, + plan, + local, + bracket, + incoming: publishedOf(bundle), + }); + if (dryRun) return { snapshot, dryRun, plan, refusals, outcome }; + if (refusals[0] !== undefined) throw new ImportRefusal(refusals[0]); + state.applying = true; + const applied = await applyImport(local, bundle, { + mode: "replace", + bracket, + unchanged: plan.unchanged, + onBefore: (s) => { + state.snapshotId = s.id; + }, + }); + const { nodes } = applied.result; + return { + snapshot, + dryRun, + plan, + refusals: [], + outcome: settledOutcome(outcome, applied, false), + reverted: { nodes, searchRefreshed: applied.searchRefreshed }, + }; +} + +/** + * Any failure as one line: `cannot revert to : ; nothing was reverted`, or, once the + * graph was being written, that it may hold part of it and the snapshot taken first. + */ +function revertFailure(error: unknown, id: string, state: State): Error { + const refused = error instanceof ImportRefusal || error instanceof InvariantError; + const reason = error instanceof Error ? error.message : String(error); + const head = `cannot revert to ${displayText(id, 80)}: ${displayText(reason, refused ? 1000 : 400)}`; + const snap = state.snapshotId; + if (state.applying && !refused) + return new Error( + `${head}; the revert stopped after it began writing, so the site may hold part of it; run it again to finish${snap ? `; snapshot ${snap} taken first holds the site as it was (undo: cloudhal import --revert ${snap})` : ""}`, + { cause: error }, + ); + const saved = snap ? `; snapshot ${snap} taken first holds the current graph` : ""; + return new Error(`${head}; nothing was reverted${saved}`, { cause: error }); +} + +/** Reverts (or, with `dryRun`, plans) the site to the snapshot; see the module doc. */ +export async function revertImport(options: RevertImportOptions): Promise { + const state: State = { applying: false }; + let local: LocalStore | undefined; + try { + const cfg = await readProjectConfig(options.projectDir); + const dataDir = path.resolve(options.projectDir, cfg.dataDir); + if ((await statOrUndefined(path.join(dataDir, "data.db")))?.isFile() !== true) + throw new ImportRefusal("this project has no local store"); + local = await openLocalStore(dataDir); + return await runRevert(local, { ...options, cfg }, state); + } catch (e) { + throw revertFailure(e, options.snapshotId, state); + } finally { + await local?.close(); + } +} diff --git a/packages/cli/src/commands/import.ts b/packages/cli/src/commands/import.ts index a6e58bd0..4575ffb1 100644 --- a/packages/cli/src/commands/import.ts +++ b/packages/cli/src/commands/import.ts @@ -1,12 +1,23 @@ /** - * `cloudhal import [--dry-run]` (RFC-0039 §2, rollout step 3): a `.cloudhal` file written by - * `cloudhal export` into this project, in bootstrap mode, the site's id kept (invariant 1). + * `cloudhal import [--replace] [--dry-run]` (RFC-0039 §2, rollout steps 3 and 4): a file + * written by `cloudhal export` into this project, the site's id kept (invariant 1). * - * - **Site:** the archive's must be `cloudhal.json`'s `siteId` or listed in `sites`, checked as soon - * as the manifest is read. Another id is refused, naming both: importing another site (and - * switching `siteId`) belongs to `--replace`, not available yet. - * - **Bootstrap:** the project must not hold the site yet, or hold exactly the archive's nodes; the + * - **Bootstrap** (default): the archive's site must be `cloudhal.json`'s `siteId` or listed in + * `sites`, checked as soon as the manifest is read; another id is refused, naming both and + * `--replace`. The project must not hold the site yet, or hold exactly the archive's nodes; the * snapshots, node bodies and media it lacks are added. A site whose nodes differ is refused. + * - **`--replace`:** the local nodes become the archive's, its published snapshot included (the + * site node carries it; `published: "source"`, as `cloudhal pull --replace`). A deployment keeps + * its own: `cloudhal push` applies with `published: "target"`, so a local replace never moves + * production. Snapshots are only ever added. When the project holds the site and its nodes would + * change, the current graph is saved first as `before import ` (`cloudhal import --revert + * ` brings it back) and the result as `import ` (`import-apply.ts`). Refused, before + * anything is written, while the site has open releases or an unfinished field operation, or + * when another local site serves one of its hostnames (`import-replace.ts`). + * - **Another site:** `--replace` of an archive whose site `cloudhal.json` does not list imports it + * and switches `siteId` (and the old id in `sites`) to it, editing only those values, once the + * import has landed (`config-site.ts`). The old site stays in the local store, unlisted and + * untouched: setting `siteId` back by hand undoes the switch (`--revert` restores nodes only). * - **Two passes over the file** (invariants 3 and 6). The first reads and verifies the whole * archive (every file's size and SHA-256, every row; media bodies are hashed, not kept) and plans * the import against the local store (`planImport`), so every refusal comes before any write: a @@ -14,16 +25,16 @@ * conflict, media this project has taken down (invariant 5), media a `--no-media` archive lacks. * The second runs only when media must be copied: it reads the file again, requires the same * manifest (which pins every file's SHA-256, so the bundle is the one planned) and writes only - * the planned media into the blob store, which verifies them again. Then `importSite` applies the - * graph, keeping the archive's published snapshot as `cloudhal pull` does, and the search index - * is refreshed. A local media file of another size counts as damaged: a full archive replaces - * it, a `--no-media` one is refused. Local copies of content the archive's ledger took down are - * deleted, and the summary says so. + * the planned media into the blob store, which verifies them again. Then the graph is written + * (`import-apply.ts`). A local media file of another size counts as damaged: a full archive + * replaces it, a `--no-media` one is refused. Local copies of content the archive's ledger took + * down are deleted, and the summary says so. * - **No store yet:** planning runs on an empty store in memory, so a refusal or a dry run creates * no `.cloudhal/`; media are copied before `data.db` exists. - * - **`--dry-run`:** prints the manifest and the plan and writes nothing: no blobs, rows, config or - * search index. Opening an existing store applies pending migrations, as every command does. - * - **Ctrl-C, SIGTERM** stop the import before `importSite` runs (between the passes or between + * - **`--dry-run`:** prints the manifest and the plan (with `--replace`, the snapshots, the + * published snapshot and the switch too) and writes nothing: no blobs, rows, config or search + * index. Opening an existing store applies pending migrations, as every command does. + * - **Ctrl-C, SIGTERM** stop the import before its writes start (between the passes or between * media files); from there it finishes, its node set being one atomic batch. Media already * copied stay in the blob store, unreferenced: content-addressed, the next import reuses them * (`cloudhal media gc` only sees media with a ledger row). A hard kill can leave a temporary @@ -32,17 +43,17 @@ * Known limitations: the graph rows are held in memory (one bundle at a time, as for export); a * disk error while `importSite` writes can leave node bodies, snapshots or the media ledger * without the rest, as for `pull`: the message then says the import stopped partway, and running - * it again completes it (and refreshes a search index left behind the live snapshot). + * it again completes it (and refreshes a search index left behind the live snapshot). As for + * `pull --replace`, an edit another process makes between the `before` snapshot (or the recheck + * of an unchanged site) and the node batch is replaced and not in that snapshot, and the archive's + * nodes keep their revs: a Studio tab or agent still holding a node at the rev the archive gives + * it saves over it without a conflict, so the summary counts those nodes and says to reload. */ -import { createReadStream } from "node:fs"; -import { stat } from "node:fs/promises"; import path from "node:path"; -import { addAbortSignal } from "node:stream"; import { isDeepStrictEqual } from "node:util"; import { type CopySiteResult, type ImportPlan, - importSite, planImport, type SiteBundle, } from "@cloudhal/core-ts"; @@ -50,21 +61,33 @@ import { type LocalStore, openLocalStore } from "@cloudhal/core-ts/local"; import { migrate, migrations } from "@cloudhal/storage"; import { FsBlobStore } from "@cloudhal/storage/fs"; import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; -import { type ArchiveItem, readArchive, type SiteArchiveManifest } from "../archive/index.ts"; +import type { SiteArchiveManifest } from "../archive/index.ts"; import { type ProjectConfig, readProjectConfig } from "../config.ts"; -import { isRecord } from "./archive-bundle.ts"; -import { type ArchiveMediaItem, bundleFromArchive } from "./archive-reassemble.ts"; +import { readConfigText, type SiteIdSwitch, writeSiteIdSwitch } from "../config-site.ts"; import { formatBytes } from "./export.ts"; +import { type AppliedImport, applyImport } from "./import-apply.ts"; import { - bootstrapRefusals, - copiedNote, displayText, + ImportInterruptedError, ImportRefusal, importFailure, + importRefusals, mediaFolder, siteRefusal, } from "./import-checks.ts"; import { writeMedia } from "./import-media.ts"; +import { archiveSize, readPass, siteSummary, statOrUndefined } from "./import-read.ts"; +import { + hostRefusal, + importBracket, + type ReplaceOutcome, + replaceOutcome, + replaceRefusals, + settledOutcome, + switchFor, +} from "./import-replace.ts"; + +export { ImportInterruptedError } from "./import-checks.ts"; export interface ImportArchiveOptions { readonly projectDir: string; @@ -72,6 +95,8 @@ export interface ImportArchiveOptions { readonly file: string; /** Verify and plan only; write nothing. */ readonly dryRun?: boolean | undefined; + /** `--replace`: the local nodes become the archive's (see the module doc). */ + readonly replace?: boolean | undefined; /** Stops the import before the graph is written (the command's Ctrl-C and SIGTERM). */ readonly signal?: AbortSignal | undefined; /** Progress lines (the command writes them to stderr). */ @@ -83,12 +108,15 @@ export interface ImportArchiveResult { readonly file: string; readonly manifest: SiteArchiveManifest; readonly dryRun: boolean; + readonly mode: ImportPlan["mode"]; /** The archive's site, as its site node says (untrusted until planned: printed cleaned). */ readonly site: { readonly id: string; readonly slug?: string | undefined; readonly locale?: string | undefined; readonly projectId?: string | undefined; + /** Its `publishedSnapshotId`. */ + readonly published?: string | undefined; /** It is `cloudhal.json`'s `siteId` or listed in `sites`. */ readonly listed: boolean; }; @@ -117,20 +145,8 @@ export interface ImportArchiveResult { readonly searchRefreshed: boolean; } | undefined; -} - -/** Ctrl-C or SIGTERM before the graph was written. */ -export class ImportInterruptedError extends Error { - override readonly name = "ImportInterruptedError"; - readonly mediaCopied: number; - readonly mediaDir: string; - constructor(mediaCopied: number, mediaDir: string) { - super( - `import interrupted; nothing was imported${mediaCopied > 0 ? `; ${copiedNote(mediaCopied, mediaDir)}` : ""}`, - ); - this.mediaCopied = mediaCopied; - this.mediaDir = mediaDir; - } + /** `--replace`: the snapshots, the published snapshot, the switch (planned, then done). */ + readonly replace?: ReplaceOutcome | undefined; } /** What one import has done so far, for its messages and its cleanup. */ @@ -139,9 +155,11 @@ interface State { mediaDir: string; copied: number; copiedBytes: number; - /** `importSite` has started: the signal is no longer honoured. */ + /** The writes have started: the signal is no longer honoured. */ applying: boolean; local?: LocalStore | undefined; + /** `--replace`: the snapshot of the local graph taken before `importSite`. */ + snapshotId?: string | undefined; } /** One import: the project, its paths and the options. */ @@ -151,103 +169,48 @@ interface Run { readonly cfg: ProjectConfig; readonly dataDir: string; readonly storeExists: boolean; + readonly mode: ImportPlan["mode"]; + /** `--replace`: `cloudhal.json` as text, read before the archive, for the site id switch. */ + readonly raw?: string | undefined; readonly state: State; } +/** How pass 1 judged the archive's site: refused, or (`--replace`) a `siteId` switch. */ +type SiteRule = { refusal?: string | undefined; change?: SiteIdSwitch | undefined }; + /** Pass 1's result; its bundle is dropped before pass 2 reads its own. */ interface FirstPass { readonly manifest: SiteArchiveManifest; bundle: SiteBundle | undefined; } -async function statOrUndefined(file: string) { - return stat(file).catch((e: NodeJS.ErrnoException) => { - if (e.code === "ENOENT" || e.code === "ENOTDIR") return undefined; - throw e; - }); -} - -/** The archive's size; a missing file or a folder is refused before anything is read. */ -async function archiveSize(file: string): Promise { - const s = await statOrUndefined(file); - if (!s) throw new Error("no such file"); - if (s.isDirectory()) throw new Error("it is a directory; name a .cloudhal file"); - return s.size; -} - -/** The first item (the manifest) checked by `check` before the rest is read. */ -async function* checkManifest( - items: AsyncIterable, - check: (manifest: SiteArchiveManifest) => void, -): AsyncGenerator { - let first = true; - for await (const item of items) { - if (first && item.type === "manifest") check(item.manifest); - first = false; - yield item; - } -} - -/** One read of the whole file: verified, reassembled, its media handed to `onMedia`. */ -async function readPass( - run: Run, - check: (manifest: SiteArchiveManifest) => void, - onMedia?: (item: ArchiveMediaItem) => Promise, -): Promise<{ manifest: SiteArchiveManifest; bundle: SiteBundle }> { - const stream = createReadStream(run.file); - if (run.options.signal) addAbortSignal(run.options.signal, stream); - try { - const read = await bundleFromArchive( - checkManifest(readArchive(stream), check), - onMedia ? { onMedia } : {}, - ); - return { manifest: read.manifest, bundle: read.bundle }; - } finally { - stream.destroy(); - } -} - -/** What the result says about the archive's site and media, read from pass 1's bundle. */ -function siteSummary(run: Run, manifest: SiteArchiveManifest, bundle: SiteBundle) { - const row = bundle.nodes.find((n) => n.kind === "site" && n.id === manifest.siteId); - let site: unknown; - try { - site = row ? JSON.parse(row.body) : undefined; - } catch { - site = undefined; - } - const field = (key: string) => - isRecord(site) && typeof site[key] === "string" ? (site[key] as string) : undefined; - const media = bundle.media ?? []; - return { - site: { - id: manifest.siteId, - slug: field("slug"), - locale: field("defaultLocale"), - projectId: field("projectId"), - listed: (run.cfg.sites ?? [run.cfg.siteId]).includes(manifest.siteId), - }, - mediaNeeded: { files: media.length, bytes: media.reduce((n, m) => n + m.bytes, 0) }, - }; -} - /** * The plan against the local store, or, when the project has none yet, against an empty store in * memory (with the project's blob folder, which may already hold the media), so nothing is created. + * A hostname another local site serves is a refusal, so a dry run reports it. */ -async function planLocally(run: Run, bundle: SiteBundle): Promise { - const options = { mode: "bootstrap", report: true } as const; - if (run.storeExists) { - run.state.local = await openLocalStore(run.dataDir); - return planImport(run.state.local, bundle, options); - } - const sql = await BetterSqlite3Driver.open(":memory:"); +async function planLocally( + run: Run, + bundle: SiteBundle, +): Promise<{ plan: ImportPlan } | { refusal: string }> { + const options = { mode: run.mode, report: true } as const; try { - await migrate(sql, migrations); - const blobs = new FsBlobStore(path.join(run.dataDir, "blobs")); - return await planImport({ sql, blobs }, bundle, options); - } finally { - await sql.close(); + if (run.storeExists) { + run.state.local = await openLocalStore(run.dataDir); + return { plan: await planImport(run.state.local, bundle, options) }; + } + const sql = await BetterSqlite3Driver.open(":memory:"); + try { + await migrate(sql, migrations); + const blobs = new FsBlobStore(path.join(run.dataDir, "blobs")); + return { plan: await planImport({ sql, blobs }, bundle, options) }; + } finally { + await sql.close(); + } + } catch (e) { + const refusal = hostRefusal(e, "the archive's"); + if (refusal === undefined) throw e; + return { refusal }; } } @@ -269,44 +232,42 @@ async function copyMedia(run: Run, first: FirstPass, plan: ImportPlan): Promise< if (!isDeepStrictEqual(plain(m), plain(first.manifest))) throw new Error("the file changed while it was being imported"); }; - const again = await readPass(run, same, async (item) => { - const m = copy.get(item.sha256); - if (!m) return; - run.options.signal?.throwIfAborted(); - if (!(await writeMedia(blobs, plan.siteId, item, m))) return; - state.copied++; - state.copiedBytes += m.bytes; + const { signal } = run.options; + const again = await readPass({ + file: run.file, + signal, + check: same, + onMedia: async (item) => { + const m = copy.get(item.sha256); + if (!m) return; + signal?.throwIfAborted(); + if (!(await writeMedia(blobs, plan.siteId, item, m))) return; + state.copied++; + state.copiedBytes += m.bytes; + }, }); return again.bundle; } -/** - * The search index is behind the site's live snapshot (an import that stopped after the node set - * landed never refreshed it). - */ -async function searchBehind(local: LocalStore, siteId: string): Promise { - try { - const live = (await local.store.getSite(siteId))?.publishedSnapshotId; - return live !== undefined && (await local.store.searchStatus(siteId))?.snapshotId !== live; - } catch { - return true; - } -} - -/** `importSite` and the search index; the signal is not honoured from here. */ -async function apply(run: Run, bundle: SiteBundle, plan: ImportPlan) { +/** The writes (`import-apply.ts`): the signal is not honoured from here. */ +async function apply(run: Run, bundle: SiteBundle, plan: ImportPlan): Promise { const { state } = run; state.applying = true; state.local ??= await openLocalStore(run.dataDir); - const local = state.local; - const result = await importSite(local, bundle, { mode: "bootstrap" }); - let searchRefreshed = true; - if (!result.nodes.unchanged || result.snapshots > 0 || (await searchBehind(local, plan.siteId))) - // RFC-0019 §3: an import leaves the index as it was; bring it to the live snapshot. - searchRefreshed = await local.store.refreshSearch(plan.siteId, { full: true }).then( - () => true, - () => false, - ); + const holds = run.mode === "replace" && !plan.fresh; + return applyImport(state.local, bundle, { + mode: run.mode, + ...(holds ? { bracket: importBracket(run.options.file), unchanged: plan.unchanged } : {}), + onBefore: (snapshot) => { + state.snapshotId = snapshot.id; + }, + }); +} + +/** The summary of a written import. */ +function importedOf(run: Run, plan: ImportPlan, applied: AppliedImport) { + const { result } = applied; + const { state } = run; return { nodes: result.nodes, snapshots: result.snapshots, @@ -314,25 +275,44 @@ async function apply(run: Run, bundle: SiteBundle, plan: ImportPlan) { media: { copied: state.copied, bytes: state.copiedBytes, present: plan.media.present.length }, takedowns: plan.takedowns.listed, takedownsDeleted: plan.takedowns.bytesHere.length, - searchRefreshed, + searchRefreshed: applied.searchRefreshed, }; } -/** The project, the file and pass 1, then the plan; on to the writes unless refused or a dry run. */ -async function runImport(options: ImportArchiveOptions, state: State) { +/** The project and its paths; `cloudhal.json`'s text too for `--replace`. */ +async function startRun(options: ImportArchiveOptions, state: State) { const file = path.resolve(options.projectDir, options.file); const size = await archiveSize(file); const cfg = await readProjectConfig(options.projectDir); + const raw = options.replace ? await readConfigText(options.projectDir) : undefined; options.signal?.throwIfAborted(); const dataDir = path.resolve(options.projectDir, cfg.dataDir); const storeExists = (await statOrUndefined(path.join(dataDir, "data.db")))?.isFile() === true; - const run: Run = { options, file, cfg, dataDir, storeExists, state }; - const rule: { refusal?: string | undefined } = {}; - const first: FirstPass = await readPass(run, (manifest) => { - state.siteId = manifest.siteId; - state.mediaDir = mediaFolder(cfg.dataDir, manifest.siteId); - rule.refusal = siteRefusal(cfg, manifest, storeExists); - if (rule.refusal && !options.dryRun) throw new ImportRefusal(rule.refusal); + const mode: ImportPlan["mode"] = options.replace ? "replace" : "bootstrap"; + const run: Run = { options, file, cfg, dataDir, storeExists, mode, raw, state }; + return { run, size }; +} + +/** Pass 1's judgement of the archive's site (`siteRefusal`, or `switchFor` with `--replace`). */ +function siteRule(run: Run, manifest: SiteArchiveManifest): SiteRule { + if (run.mode === "replace") return switchFor(run.cfg, run.raw ?? "", manifest.siteId); + return { refusal: siteRefusal(run.cfg, manifest, run.storeExists) }; +} + +/** The project, the file and pass 1, then the plan; on to the writes unless refused or a dry run. */ +async function runImport(options: ImportArchiveOptions, state: State) { + const { run, size } = await startRun(options, state); + const { cfg } = run; + const rule: SiteRule = {}; + const first: FirstPass = await readPass({ + file: run.file, + signal: options.signal, + check: (manifest) => { + state.siteId = manifest.siteId; + state.mediaDir = mediaFolder(cfg.dataDir, manifest.siteId); + Object.assign(rule, siteRule(run, manifest)); + if (rule.refusal && !options.dryRun) throw new ImportRefusal(rule.refusal); + }, }); const { manifest } = first; options.log?.( @@ -342,18 +322,49 @@ async function runImport(options: ImportArchiveOptions, state: State) { file: options.file, manifest, dryRun: options.dryRun === true, - newStore: !storeExists, - ...siteSummary(run, manifest, first.bundle as SiteBundle), + mode: run.mode, + newStore: !run.storeExists, + ...siteSummary(cfg, manifest, first.bundle as SiteBundle), }; - if (rule.refusal) return { ...base, refusals: [rule.refusal] }; - const plan = await planLocally(run, first.bundle as SiteBundle); - const refusals = bootstrapRefusals(plan, manifest, cfg.dataDir); - if (options.dryRun) return { ...base, plan, refusals }; + const planned = rule.refusal + ? { refusal: rule.refusal } + : await planLocally(run, first.bundle as SiteBundle); + if ("refusal" in planned) { + if (!options.dryRun) throw new ImportRefusal(planned.refusal); + return { ...base, refusals: [planned.refusal] }; + } + const { plan } = planned; + const replacing = run.mode === "replace"; + const refusals = [ + ...importRefusals(plan, manifest, cfg.dataDir), + ...(replacing ? await replaceRefusals(state.local, plan, "--replace") : []), + ]; + const outcome = replacing + ? await replaceOutcome({ + projectDir: options.projectDir, + cfg, + plan, + local: state.local, + change: rule.change, + bracket: importBracket(options.file), + incoming: base.site.published, + }) + : undefined; + if (options.dryRun) return { ...base, plan, refusals, replace: outcome }; if (refusals[0] !== undefined) throw new ImportRefusal(refusals[0]); options.signal?.throwIfAborted(); const bundle = await copyMedia(run, first, plan); options.signal?.throwIfAborted(); - return { ...base, plan, refusals: [], imported: await apply(run, bundle, plan) }; + const applied = await apply(run, bundle, plan); + // Last: a failure before this leaves cloudhal.json naming the old site, which is intact. + if (rule.change) await writeSiteIdSwitch(options.projectDir, rule.change); + return { + ...base, + plan, + refusals: [], + imported: importedOf(run, plan, applied), + replace: outcome && settledOutcome(outcome, applied, rule.change !== undefined), + }; } /** Imports (or, with `dryRun`, plans) the archive; see the module doc for every rule. */ @@ -370,6 +381,7 @@ export async function importArchive(options: ImportArchiveOptions): Promise s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +const isObject = (v: unknown): v is Record => + typeof v === "object" && v !== null && !Array.isArray(v); + +/** `cloudhal.json` as text, for `planSiteIdSwitch` (read before the archive is). */ +export function readConfigText(projectDir: string): Promise { + return readFile(path.join(projectDir, CONFIG_FILE), "utf8"); +} + +/** The edit that switches `siteId` to `to`, or why it cannot be made in place. */ +export function planSiteIdSwitch( + raw: string, + to: string, +): SiteIdSwitch | { readonly refusal: string } { + const byHand = `; set cloudhal.json siteId to ${to} by hand (push and deploy then use it), then import again`; + let cfg: unknown; + try { + cfg = JSON.parse(raw); + } catch { + cfg = undefined; + } + if (!isObject(cfg) || typeof cfg.siteId !== "string") + return { refusal: `cloudhal.json has no siteId to switch${byHand}` }; + const from = cfg.siteId; + const quotedFrom = JSON.stringify(from); + const quotedTo = JSON.stringify(to); + const text = raw + .replace( + new RegExp(`("siteId"\\s*:\\s*)${escapeRegExp(quotedFrom)}`, "g"), + (_m, key: string) => `${key}${quotedTo}`, + ) + .replace( + /("sites"\s*:\s*\[)([^\]]*)\]/g, + (_m, head: string, items: string) => `${head}${items.replaceAll(quotedFrom, quotedTo)}]`, + ); + const sites = Array.isArray(cfg.sites) + ? cfg.sites.map((s: unknown) => (s === from ? to : s)) + : undefined; + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + parsed = undefined; + } + if (!isDeepStrictEqual(parsed, { ...cfg, siteId: to, ...(sites ? { sites } : {}) })) + return { + refusal: `cloudhal.json cannot be switched in place: it names ${from} in more than its siteId and sites values, or writes it escaped${byHand}`, + }; + return { + from, + to, + ...(sites + ? { + sites: { + before: parseSites(from, cfg.sites) ?? [from], + after: parseSites(to, sites) ?? [to], + }, + } + : {}), + raw, + text, + }; +} + +/** + * Writes the switch: refused when the file changed since it was read; a temporary file next to it + * (with its mode) is renamed over it, so a failure leaves the file as it was. + */ +export async function writeSiteIdSwitch(projectDir: string, change: SiteIdSwitch): Promise { + const file = path.join(projectDir, CONFIG_FILE); + const tail = `; run the import again, which only switches siteId to ${change.to}, or set it by hand`; + const now = await readFile(file, "utf8").catch(() => undefined); + if (now !== change.raw) + throw new SiteSwitchError( + `cloudhal.json changed while importing, so it still names site ${change.from}${tail}`, + ); + const temp = path.join( + projectDir, + `.${CONFIG_FILE}.partial-${process.pid}-${randomUUID().slice(0, 8)}`, + ); + try { + const { mode } = await stat(file); + await writeFile(temp, change.text, { flag: "wx" }); + await chmod(temp, mode & 0o777); + await rename(temp, file); + } catch (e) { + await rm(temp, { force: true }); + const reason = e instanceof Error ? e.message : String(e); + throw new SiteSwitchError( + `cloudhal.json could not be written (${reason}), so it still names site ${change.from}${tail}`, + { cause: e }, + ); + } +} diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts index eee11815..0d3b72f6 100644 --- a/packages/cli/src/config.ts +++ b/packages/cli/src/config.ts @@ -298,6 +298,11 @@ export function analyticsDefaultsOf(cfg: Pick): { const SITE_ID = /^site_[A-Za-z0-9]{6,32}$/; +/** A site id `sites` accepts (`site_` and 6 to 32 letters or digits). */ +export function isSiteId(id: string): boolean { + return SITE_ID.test(id); +} + /** `sites`: valid ids, each once, the project's site first; undefined unless it adds a site. */ export function parseSites(siteId: string, input: unknown): string[] | undefined { if (input === undefined) return undefined; diff --git a/packages/cli/test/import-cli.test.ts b/packages/cli/test/import-cli.test.ts index 712e3ca5..750c6d3f 100644 --- a/packages/cli/test/import-cli.test.ts +++ b/packages/cli/test/import-cli.test.ts @@ -8,7 +8,7 @@ * so; `--dry-run` shows the difference by kind and title and writes nothing. */ import { writeFileSync } from "node:fs"; -import { cp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { cp, readFile, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import type { Page } from "@cloudhal/core-contracts"; import { type NodeRow, type SiteBundle, SyncConflictError } from "@cloudhal/core-ts"; @@ -33,11 +33,14 @@ import { blobListing, bundleOf, copyProject, + emptyProject as emptyProjectOf, + exists, type Fixture, mediaFile, person, removeTempDirs, seedFixture, + stateOf, tempDir, withStore, } from "./site-fixture.ts"; @@ -63,40 +66,12 @@ afterAll(removeTempDirs); afterEach(() => vi.restoreAllMocks()); /** A project with only its cloudhal.json (no .cloudhal): the fixture's sites unless `cfg` says. */ -async function emptyProject(cfg?: Pick): Promise { - const dir = await tempDir("cloudhal-import-"); - await writeProjectConfig(dir, { - dataDir: ".cloudhal", - ...(cfg ?? { siteId: fx.siteId, sites: [fx.siteId, fx.trId] }), - }); - return dir; -} - -const exists = (file: string) => - stat(file).then( - () => true, - () => false, - ); +const emptyProject = (cfg?: Pick) => emptyProjectOf(fx, cfg); async function expectNoStore(dir: string): Promise { expect(await exists(path.join(dir, ".cloudhal"))).toBe(false); } -/** What a refused import must leave as it was (opens the store: call it only where one exists). */ -async function stateOf(dir: string) { - const rows = await withStore(dir, async ({ sql }) => ({ - nodes: await sql.all("SELECT id, site_id, rev, hash FROM nodes ORDER BY id"), - snapshots: await sql.all("SELECT id FROM snapshots ORDER BY id"), - projects: await sql.all("SELECT id, rev FROM projects ORDER BY id"), - media: await sql.all("SELECT site_id, sha256, state FROM media_blobs ORDER BY site_id, sha256"), - })); - return { - rows, - blobs: await blobListing(dir), - config: await readFile(path.join(dir, "cloudhal.json"), "utf8"), - }; -} - const sourceBytes = (sha: string) => readFile(mediaFile(fx.projectDir, fx.siteId, sha)); /** An archive in its own folder. */ @@ -508,7 +483,7 @@ describe("cloudhal import: a refused import leaves the project as it was (invari )) as Error; expect(error.message).toMatch( new RegExp( - `^cannot import .*moved\\.cloudhal: this project's site ${fx.siteId} differs from the archive in 2 node\\(s\\) \\(.*\\); bootstrap imports only into a project without the site or holding exactly its nodes, and --replace is not available yet; nothing was imported$`, + `^cannot import .*moved\\.cloudhal: this project's site ${fx.siteId} differs from the archive in 2 node\\(s\\) \\(.*\\); bootstrap imports only into a project without the site or holding exactly its nodes; --replace makes them the archive's, saving the local ones as a snapshot first; nothing was imported$`, ), ); expect(error.message).toContain(page.id); @@ -626,11 +601,11 @@ describe("cloudhal import: a refused import leaves the project as it was (invari }); describe("cloudhal import: which site (the archive's id is kept)", () => { - it("refuses another site's archive, naming both ids; the hint works without a store", async () => { + it("refuses another site's archive, naming both ids and --replace", async () => { const dir = await emptyProject({ siteId: "site_other00001" }); const config = await readFile(path.join(dir, "cloudhal.json")); await expect(importArchive({ projectDir: dir, file: archives.full })).rejects.toThrow( - `it holds site ${fx.siteId}, and this project's site is site_other00001 (cloudhal.json siteId); importing another site into a project comes with \`cloudhal import --replace\`, not available yet; this project has no local store yet, so setting cloudhal.json siteId to ${fx.siteId} also works (push and deploy then use ${fx.siteId}); nothing was imported`, + `it holds site ${fx.siteId}, and this project's site is site_other00001 (cloudhal.json siteId); \`cloudhal import --replace\` imports it and switches cloudhal.json siteId to ${fx.siteId}; nothing was imported`, ); await expectNoStore(dir); expect(Buffer.compare(await readFile(path.join(dir, "cloudhal.json")), config)).toBe(0); @@ -638,13 +613,13 @@ describe("cloudhal import: which site (the archive's id is kept)", () => { const dry = await importArchive({ projectDir: dir, file: archives.full, dryRun: true }); expect(dry.plan).toBeUndefined(); expect(describeImport(dry).at(-2)).toMatch(/^would refuse: it holds site /); - // As the hint says. + // Without a store, setting siteId by hand works too. await writeProjectConfig(dir, { siteId: fx.siteId, dataDir: ".cloudhal" }); await importArchive({ projectDir: dir, file: archives.full }); expect(await bundleOf(dir, fx.siteId)).toStrictEqual(await bundleOf(fx.projectDir, fx.siteId)); }); - it("refuses another site's archive into a project with a store, without the hint", async () => { + it("refuses another site's archive into a project with a store, which keeps the old site", async () => { const dir = await copyProject(fx); await writeProjectConfig(dir, { siteId: fx.trId, dataDir: ".cloudhal" }); const before = await stateOf(dir); @@ -652,7 +627,7 @@ describe("cloudhal import: which site (the archive's id is kept)", () => { (e: unknown) => e, )) as Error; expect(error.message).toContain( - `it holds site ${fx.siteId}, and this project's site is ${fx.trId} (cloudhal.json siteId); importing another site into a project comes with \`cloudhal import --replace\`, not available yet; nothing was imported`, + `it holds site ${fx.siteId}, and this project's site is ${fx.trId} (cloudhal.json siteId); \`cloudhal import --replace\` imports it and switches cloudhal.json siteId to ${fx.siteId} (${fx.trId} stays in the local store, unlisted); nothing was imported`, ); expect(await stateOf(dir)).toEqual(before); }); @@ -746,12 +721,14 @@ describe("cloudhal import --dry-run", () => { describe("cloudhal import: the command", () => { it("is in --help and checks its arguments and flags", async () => { - expect(HELP).toMatch(/^ {2}cloudhal import \[--dry-run\] +a \.cloudhal file/m); - for (const argv of [["import"], ["import", "a", "b"]]) - await expect(main(argv)).rejects.toThrow("usage: cloudhal import [--dry-run]"); - await expect(main(["import", "x.cloudhal", "--replace"])).rejects.toThrow( - "cloudhal import --replace is not available yet; this version imports in bootstrap mode only", + expect(HELP).toMatch( + /^ {2}cloudhal import \[--replace\] \[--dry-run\] \| --revert \[--dry-run\]\n {58}a \.cloudhal file/m, ); + expect(HELP).toContain("--replace: the local"); + for (const argv of [["import"], ["import", "a", "b"]]) + await expect(main(argv)).rejects.toThrow( + "usage: cloudhal import [--replace] [--dry-run] | --revert [--dry-run]", + ); await expect(main(["import", "x.cloudhal", "--site", "site_other00001"])).rejects.toThrow( "cloudhal import does not take --site (only cloudhal export does)", ); diff --git a/packages/cli/test/import-replace.test.ts b/packages/cli/test/import-replace.test.ts new file mode 100644 index 00000000..302f5816 --- /dev/null +++ b/packages/cli/test/import-replace.test.ts @@ -0,0 +1,543 @@ +/** + * `cloudhal import --replace` and `--revert` (RFC-0039 §2, rollout step 4) on real local projects: + * the local nodes become the archive's, its published snapshot included, after a `before import` + * snapshot of them that `--revert` brings back (invariant 6), itself undone the same way; the + * nodes a replace gives a rev no higher than they had are counted, and a revert moves revs forward + * so a stale editor gets a conflict; an unchanged site gets no snapshot, and is refused when + * another process changes it before the write; `--dry-run` shows the snapshots and the published + * snapshot and writes nothing; open releases (RFC-0008) and an unfinished field operation + * (RFC-0041) refuse it before anything is written; a storage failure after the snapshot names it, + * and a second run reuses it. The site id switch is in `import-switch.test.ts`. + */ +import { rm } from "node:fs/promises"; +import path from "node:path"; +import type { ContentEntry, NodeRef, Page, Site, Snapshot } from "@cloudhal/core-contracts"; +import { ConflictError, runFieldOperation } from "@cloudhal/core-ts"; +import { ReleaseService } from "@cloudhal/core-ts/releases"; +import { FsBlobStore } from "@cloudhal/storage/fs"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { main } from "../src/cli.ts"; +import { exportArchive } from "../src/commands/export.ts"; +import { importArchive } from "../src/commands/import.ts"; +import { describeImport, describeRevert } from "../src/commands/import-describe.ts"; +import { revertImport } from "../src/commands/import-revert.ts"; +import { writeProjectConfig } from "../src/config.ts"; +import { + bundleOf, + copyProject, + emptyProject, + type Fixture, + mediaFile, + person, + removeTempDirs, + seedFixture, + stateOf, + tempDir, + withStore, +} from "./site-fixture.ts"; + +let fx: Fixture; +let archive: string; + +beforeAll(async () => { + fx = await seedFixture("cloudhal-replace-src-"); + const out = path.join(await tempDir("cloudhal-replace-archives-"), "english.cloudhal"); + archive = (await exportArchive({ projectDir: fx.projectDir, out })).file; +}); +afterAll(removeTempDirs); +afterEach(() => vi.restoreAllMocks()); + +const BEFORE = "before import english.cloudhal"; +const AFTER = "import english.cloudhal"; + +type Row = { id: string; rev: number; hash: string; body: string }; + +/** The node rows without their timestamps (a restore rebuilds those from the bodies). */ +const rowsOf = (dir: string, siteId = fx.siteId) => + withStore(dir, ({ sql }) => + sql.all("SELECT id, kind, key, rev, hash, body FROM nodes WHERE site_id = ? ORDER BY id", [ + siteId, + ]), + ); + +/** The rows' content: without the rev a revert moves forward, and the hash that follows it. */ +const contentOf = (rows: readonly Row[]) => + rows.map(({ rev: _rev, hash: _hash, body, ...row }) => ({ + ...row, + body: { ...(JSON.parse(body) as object), rev: undefined }, + })); + +const REVS_TOOK = (n: number) => + `note: ${n} node(s) took the archive's rev, no higher than theirs here; reload Studio tabs open on this site: an edit a tab or agent still holds at that rev saves over them without a conflict`; + +const snapshotsOf = (dir: string, siteId = fx.siteId): Promise => + withStore(dir, ({ store }) => store.listSnapshots(siteId, 200)); + +async function labelled(dir: string, label: string): Promise { + const found = (await snapshotsOf(dir)).filter((s) => s.label === label); + expect(found).toHaveLength(1); + return found[0] as Snapshot; +} + +const aboutOf = async (store: Parameters[1]>[0]["store"]) => + (await store.listPages(fx.siteId)).find((p) => p.path === "/about") as Page; + +/** A copy of the fixture with /about retitled here and published as `s4`. */ +async function editedCopy(): Promise<{ dir: string; s4: string }> { + const dir = await copyProject(fx); + const s4 = await withStore(dir, async ({ store }) => { + await store.updatePage({ ...(await aboutOf(store)), title: "Edited here" }); + const snap = await store.createSnapshot(fx.siteId, { actor: person, reason: "publish" }); + await store.publish((await store.getSite(fx.siteId)) as Site, snap.id); + return snap.id; + }); + return { dir, s4 }; +} + +const replace = (dir: string, dryRun = false) => + importArchive({ projectDir: dir, file: archive, replace: true, dryRun }); + +describe("cloudhal import --replace: the local nodes become the archive's", () => { + it("replaces an edited site, published snapshot included, saving it first", async () => { + const { dir, s4 } = await editedCopy(); + const refs = await withStore(dir, ({ store }) => store.currentNodeRefs(fx.siteId)); + const result = await replace(dir); + const source = await bundleOf(fx.projectDir, fx.siteId); + expect((await bundleOf(dir, fx.siteId)).nodes).toStrictEqual(source.nodes); + const after = await withStore(dir, async ({ store }) => ({ + published: (await store.getSite(fx.siteId))?.publishedSnapshotId, + latest: await store.latestSnapshot(fx.siteId), + refs: await store.currentNodeRefs(fx.siteId), + search: await store.searchStatus(fx.siteId), + })); + expect(after.published).toBe(fx.snapshots.s2); + expect(after.search?.snapshotId).toBe(fx.snapshots.s2); + const b = await labelled(dir, BEFORE); + const a = await labelled(dir, AFTER); + const extra = (await snapshotsOf(dir)) + .map((s) => s.id) + .filter((id) => !source.snapshots.some((s) => s.id === id)); + expect(new Set(extra)).toEqual(new Set([s4, b.id, a.id])); + expect(b).toMatchObject({ + reason: "import", + metadata: { "cloudhal.import.phase": "before", "cloudhal.import.file": "english.cloudhal" }, + }); + expect(b.nodes).toEqual(refs); + expect(a).toMatchObject({ + reason: "import", + parentId: b.id, + metadata: { "cloudhal.import.phase": "after", "cloudhal.import.of": b.id }, + }); + expect(a.nodes).toEqual(after.refs); + expect(after.latest?.id).toBe(a.id); + const printed = describeImport(result); + expect(printed[0]).toBe( + `imported ${archive}: site ${fx.siteId} (english) is in this project, its nodes replaced by the archive's`, + ); + expect(printed).toContain( + `published: snapshot ${fx.snapshots.s2}, as the archive has it (was: ${s4})`, + ); + expect(printed).toContain( + `saved first: snapshot ${b.id} "${BEFORE}" holds the site as it was (undo: cloudhal import --revert ${b.id}); ${a.id} "${AFTER}" holds the result`, + ); + // The site node and /about went back to the archive's lower revs: a stale editor can save. + expect(printed).toContain(REVS_TOOK(2)); + }); + + it("is undone by --revert, edit and published snapshot included, and a revert is undone too", async () => { + const { dir, s4 } = await editedCopy(); + const rows = await rowsOf(dir); + await replace(dir); + const imported = await rowsOf(dir); + const b = await labelled(dir, BEFORE); + const dry = await revertImport({ projectDir: dir, snapshotId: b.id, dryRun: true }); + expect(describeRevert(dry)).toContainEqual( + expect.stringMatching( + /^ {2}~ page "Edited here" \(\/about\), rev \d+ here, \d+ after the revert$/, + ), + ); + expect(await rowsOf(dir)).toEqual(imported); + + const reverted = await revertImport({ projectDir: dir, snapshotId: b.id }); + expect(await rowsOf(dir)).toEqual(rows); + const live = await withStore(dir, async ({ store }) => ({ + title: (await aboutOf(store)).title, + published: (await store.getSite(fx.siteId))?.publishedSnapshotId, + search: (await store.searchStatus(fx.siteId))?.snapshotId, + latest: (await store.latestSnapshot(fx.siteId))?.label, + })); + expect(live).toEqual({ + title: "Edited here", + published: s4, + search: s4, + latest: `revert ${AFTER}`, + }); + const r = await labelled(dir, `before revert ${AFTER}`); + expect(r.metadata).toMatchObject({ + "cloudhal.import.phase": "before", + "cloudhal.import.revert": b.id, + }); + const printed = describeRevert(reverted); + expect(printed[0]).toMatch(new RegExp(`^reverted to snapshot ${b.id} "${BEFORE}" \\(site `)); + expect(printed).toContain( + `published: snapshot ${s4}, as the snapshot has it (was: ${fx.snapshots.s2})`, + ); + expect(printed).toContainEqual( + expect.stringContaining(`saved first: snapshot ${r.id} "before revert ${AFTER}"`), + ); + expect(printed).not.toContainEqual(expect.stringMatching(/^note: \d+ node\(s\)/)); + + // Again: nothing to do, no snapshot. + const count = (await snapshotsOf(dir)).length; + const again = await revertImport({ projectDir: dir, snapshotId: b.id }); + expect(describeRevert(again)).toEqual([ + expect.stringMatching( + new RegExp(`^nothing to revert: the site already holds the nodes of snapshot ${b.id} `), + ), + ]); + expect((await snapshotsOf(dir)).length).toBe(count); + // The revert's own `before` snapshot brings the archive's content back, at revs past these. + const revertedRows = await rowsOf(dir); + await revertImport({ projectDir: dir, snapshotId: r.id }); + const back = await rowsOf(dir); + expect(contentOf(back)).toEqual(contentOf(imported)); + // The site node and /about: the archive's content at the rev after the reverted one. + const was = new Map(revertedRows.map((x) => [x.id, x])); + const moved = back.filter((row) => was.get(row.id)?.hash !== row.hash); + expect(moved.map((row) => row.rev)).toEqual( + moved.map((row) => (was.get(row.id)?.rev ?? 0) + 1), + ); + expect(moved).toHaveLength(2); + }); + + it("--revert moves revs forward: an editor holding a rev a node had gets a conflict", async () => { + const { dir } = await editedCopy(); + await replace(dir); + const b = await labelled(dir, BEFORE); + // One edit after the import brings /about back to the rev the snapshot holds, with other + // content; a Studio tab now holds that rev. + const tab = await withStore(dir, async ({ store }) => + store.updatePage({ ...(await aboutOf(store)), title: "After the import" }), + ); + expect(tab.rev).toBe(b.nodes.find((n) => n.id === tab.id)?.rev); + const result = await revertImport({ projectDir: dir, snapshotId: b.id }); + await withStore(dir, async ({ store }) => { + expect(await aboutOf(store)).toMatchObject({ title: "Edited here", rev: tab.rev + 1 }); + await expect(store.updatePage({ ...tab, title: "Stale tab" })).rejects.toThrow(ConflictError); + expect((await aboutOf(store)).title).toBe("Edited here"); + }); + expect(result.outcome?.revsReused).toBeUndefined(); + // Again: the content is the snapshot's already, so nothing moves. + const again = await revertImport({ projectDir: dir, snapshotId: b.id }); + expect(again.reverted?.nodes.unchanged).toBe(true); + }); + + it("takes no snapshot and changes nothing for a site that holds the archive's nodes", async () => { + const dir = await copyProject(fx); + const before = await stateOf(dir); + const result = await replace(dir); + expect(await stateOf(dir)).toEqual(before); + expect(describeImport(result)).toEqual([ + `nothing to import: this project already holds ${archive} (site ${fx.siteId}, english)`, + ]); + expect((await snapshotsOf(dir)).some((s) => s.metadata?.["cloudhal.import.phase"])).toBe(false); + }); + + it("refuses a site found unchanged that another process changes before the write", async () => { + const dir = await copyProject(fx); + const [m] = (await bundleOf(dir, fx.siteId)).media ?? []; + if (!m) throw new Error("the fixture has no media"); + // A missing media file makes the import read the archive again before it writes. + await rm(mediaFile(dir, fx.siteId, m.sha256)); + const putStream = FsBlobStore.prototype.putStream; + let edited = false; + vi.spyOn(FsBlobStore.prototype, "putStream").mockImplementation(async function ( + this: FsBlobStore, + ...args: Parameters + ) { + if (!edited) { + edited = true; + await withStore(dir, async ({ store }) => { + await store.updatePage({ ...(await aboutOf(store)), title: "Edited meanwhile" }); + }); + } + return putStream.apply(this, args); + }); + await expect(replace(dir)).rejects.toThrow( + `cannot import ${archive}: site ${fx.siteId} changed after it was planned (another process wrote it); run it again; nothing was imported`, + ); + vi.restoreAllMocks(); + expect(edited).toBe(true); + const about = await withStore(dir, ({ store }) => aboutOf(store)); + expect(about.title).toBe("Edited meanwhile"); + expect((await snapshotsOf(dir)).some((s) => s.metadata?.["cloudhal.import.phase"])).toBe(false); + // Again: the site now differs, so the edit is saved first, then replaced. + await replace(dir); + const b = await labelled(dir, BEFORE); + const saved = await withStore(dir, async ({ store }) => ({ + page: await store.readSnapshotNode(b.nodes.find((n) => n.id === about.id) as NodeRef), + now: (await aboutOf(store)).title, + })); + expect(saved).toEqual({ + page: expect.objectContaining({ title: "Edited meanwhile" }), + now: "About us", + }); + }); + + it("acts as a bootstrap into a project without the site, with no snapshot", async () => { + const dir = await emptyProject(fx); + const result = await replace(dir); + expect(result.replace?.snapshot).toBeUndefined(); + expect(await bundleOf(dir, fx.siteId)).toStrictEqual(await bundleOf(fx.projectDir, fx.siteId)); + expect((await snapshotsOf(dir)).some((s) => s.metadata?.["cloudhal.import.phase"])).toBe(false); + }); +}); + +describe("cloudhal import --replace --dry-run", () => { + it("shows the change, the published snapshot and the snapshot it would take, and writes nothing", async () => { + const { dir, s4 } = await editedCopy(); + const before = { ...(await stateOf(dir)), bundle: await bundleOf(dir, fx.siteId) }; + vi.spyOn(process, "cwd").mockReturnValue(dir); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + expect(await main(["import", archive, "--replace", "--dry-run"])).toBe(0); + const printed = log.mock.calls.map((c) => String(c[0])); + expect(printed).toContain( + `dry run: replace import into site ${fx.siteId}, which this project already holds`, + ); + expect(printed).toContain("nodes: 0 to add, 2 to change (1 site, 1 page), 0 to remove"); + expect(printed).toContainEqual( + expect.stringMatching( + /^ {2}~ page "About us" \(\/about\), rev \d+ here, \d+ in the archive$/, + ), + ); + expect(printed).toContain( + `published: snapshot ${fx.snapshots.s2}, as the archive has it (here: ${s4})`, + ); + expect(printed).toContainEqual( + expect.stringMatching( + /^saved first: this project's site \(\d+ nodes\) as snapshot "before import english\.cloudhal", the result as "import english\.cloudhal" \(undo: cloudhal import --revert \)$/, + ), + ); + expect(printed).toContain( + "note: 2 node(s) would take the archive's rev, no higher than theirs here; reload Studio tabs open on this site: an edit a tab or agent holds at that rev saves over them without a conflict", + ); + expect(printed.at(-1)).toBe("nothing written (dry run)"); + expect({ ...(await stateOf(dir)), bundle: await bundleOf(dir, fx.siteId) }).toEqual(before); + }); +}); + +describe("cloudhal import --replace: refused before anything is written (invariant 6)", () => { + it("refuses while the site has open releases (RFC-0008), and --revert too", async () => { + const { dir } = await editedCopy(); + const draft = async (title: string) => + withStore(dir, async ({ store }) => { + const about = await aboutOf(store); + await store.updatePage({ ...about, title }); + const svc = new ReleaseService({ graph: store }); + return (await svc.create({ siteId: fx.siteId, set: [about.id], createdBy: person, title })) + .release; + }); + const cancel = (id: string, rev: number) => + withStore(dir, ({ store }) => + new ReleaseService({ graph: store }).cancel(fx.siteId, id, { + expectedRev: rev, + by: person, + }), + ); + const release = await draft("Launch"); + const before = await stateOf(dir); + await expect(replace(dir)).rejects.toThrow( + `site ${fx.siteId} has 1 open release(s) (${release.id} (draft, "Launch")), and --replace replaces the graph their pinned versions point into; cancel them in Studio (\`cloudhal studio\`) or with the MCP tool cancel_schedule, then import again; nothing was imported`, + ); + expect(await stateOf(dir)).toEqual(before); + const dry = await replace(dir, true); + expect(dry.refusals).toHaveLength(1); + expect(describeImport(dry)).toContainEqual( + expect.stringMatching(/^would refuse: site .* has 1 open release/), + ); + expect(await stateOf(dir)).toEqual(before); + + await cancel(release.id, release.rev); + await replace(dir); + const b = await labelled(dir, BEFORE); + const second = await draft("Second"); + const count = (await snapshotsOf(dir)).length; + await expect(revertImport({ projectDir: dir, snapshotId: b.id })).rejects.toThrow( + new RegExp( + `^cannot revert to ${b.id}: site ${fx.siteId} has 1 open release\\(s\\) \\(${second.id} .*, and --revert replaces .*then revert again; nothing was reverted$`, + ), + ); + expect((await snapshotsOf(dir)).length).toBe(count); + }); + + it("refuses while a field operation is unfinished (RFC-0041)", async () => { + const dir = await copyProject(fx); + const operationId = await withStore(dir, async ({ store }) => { + const site = (await store.getSite(fx.siteId)) as Site; + const speaker = { key: "speaker", type: "text", labels: { en: { label: "Speaker" } } }; + await store.updateSite({ + ...site, + collections: { + ...site.collections, + events: { fields: [{ ...speaker, localized: false }] }, + }, + } as never); + for (const slug of ["one", "two", "three"]) + await store.createContentEntry({ + siteId: fx.siteId, + collection: "events", + slug, + locale: "en", + status: "draft", + title: slug, + body: { format: "markdown", value: "Body" }, + fields: { speaker: slug }, + tags: [], + author: person, + } as never); + const part = await runFieldOperation( + store, + fx.siteId, + { op: "rename_field", collection: "events", from: "speaker", to: "presenter" }, + { actor: person, budget: { entries: 1 } }, + ); + expect(part.done).toBe(false); + return part.operationId; + }); + const before = await stateOf(dir); + await expect(replace(dir)).rejects.toThrow( + `field operation ${operationId} on events has not finished, and --replace would drop its marker; finish or revert it first (\`cloudhal fields continue|revert events ${operationId}\`), then import again; nothing was imported`, + ); + expect(await stateOf(dir)).toEqual(before); + }); +}); + +describe("cloudhal import --replace: a failure after the snapshot", () => { + it("names the snapshot taken first; a second run reuses it and completes", async () => { + const { dir } = await editedCopy(); + const rows = await rowsOf(dir); + const batch = BetterSqlite3Driver.prototype.batch; + let failed = false; + vi.spyOn(BetterSqlite3Driver.prototype, "batch").mockImplementation(async function ( + this: BetterSqlite3Driver, + statements: Parameters[0], + ) { + if (!failed && statements.some((s) => s.sql.startsWith("DELETE FROM nodes WHERE site_id"))) { + failed = true; + throw new Error("disk on fire"); + } + return batch.call(this, statements); + }); + const error = (await replace(dir).catch((e: unknown) => e)) as Error; + vi.restoreAllMocks(); + const b = await labelled(dir, BEFORE); + expect(error.message).toBe( + `cannot import ${archive}: disk on fire; the import stopped after it began writing, so this project may hold part of it; run it again to finish; snapshot ${b.id} taken first holds the site as it was (undo: cloudhal import --revert ${b.id})`, + ); + expect(await rowsOf(dir)).toEqual(rows); + + await replace(dir); + expect((await labelled(dir, BEFORE)).id).toBe(b.id); + expect((await labelled(dir, AFTER)).parentId).toBe(b.id); + expect((await bundleOf(dir, fx.siteId)).nodes).toStrictEqual( + (await bundleOf(fx.projectDir, fx.siteId)).nodes, + ); + }); + + it("takes the snapshot of the result a run that stopped after the node batch still owed", async () => { + const { dir } = await editedCopy(); + const batch = BetterSqlite3Driver.prototype.batch; + let stop = false; + // The media ledger's batch comes right after the node batch. + vi.spyOn(BetterSqlite3Driver.prototype, "batch").mockImplementation(async function ( + this: BetterSqlite3Driver, + statements: Parameters[0], + ) { + const result = await batch.call(this, statements); + if (!stop && statements.some((s) => s.sql.startsWith("DELETE FROM nodes WHERE site_id"))) { + stop = true; + throw new Error("disk on fire"); + } + return result; + }); + await expect(replace(dir)).rejects.toThrow(/the import stopped after it began writing/); + vi.restoreAllMocks(); + const b = await labelled(dir, BEFORE); + expect((await snapshotsOf(dir)).some((s) => s.label === AFTER)).toBe(false); + const again = await replace(dir); + expect((await labelled(dir, AFTER)).parentId).toBe(b.id); + expect(describeImport(again)).toContainEqual( + expect.stringContaining(`saved first: snapshot ${b.id} "${BEFORE}"`), + ); + }); +}); + +describe("cloudhal import --replace and --revert: the command", () => { + it("checks its arguments and flags", async () => { + const usage = + "usage: cloudhal import [--replace] [--dry-run] | --revert [--dry-run]"; + for (const argv of [ + ["import", "a.cloudhal", "--revert", "snap_x"], + ["import", "--revert", "snap_x", "--replace"], + ]) + await expect(main(argv)).rejects.toThrow(usage); + await expect(main(["import", "x.cloudhal", "--replace", "--cancel-schedules"])).rejects.toThrow( + "cloudhal import does not take --cancel-schedules (only cloudhal push does)", + ); + const dir = await emptyProject(fx); + vi.spyOn(process, "cwd").mockReturnValue(dir); + await expect(main(["import", "x.cloudhal", "--replace"])).rejects.toThrow( + "cannot import x.cloudhal: no such file; nothing was imported", + ); + await expect(main(["import", "--revert", "snap_x"])).rejects.toThrow( + "cannot revert to snap_x: this project has no local store; nothing was reverted", + ); + }); + + it("reverts only a before-import snapshot of a site cloudhal.json lists", async () => { + const dir = await copyProject(fx); + const notBefore = + 'it is not a "before import" snapshot of this project (`cloudhal import --replace` names the one it takes); nothing was reverted'; + for (const id of ["snap_unknown01", fx.snapshots.s1]) + await expect(revertImport({ projectDir: dir, snapshotId: id })).rejects.toThrow( + `cannot revert to ${id}: ${notBefore}`, + ); + const tr = await withStore(dir, ({ store }) => + store.createSnapshot(fx.trId, { + actor: person, + reason: "import", + label: "before import tr.cloudhal", + metadata: { "cloudhal.import.phase": "before" }, + }), + ); + await writeProjectConfig(dir, { siteId: fx.siteId, dataDir: ".cloudhal" }); + await expect(revertImport({ projectDir: dir, snapshotId: tr.id })).rejects.toThrow( + `cannot revert to ${tr.id}: it is a snapshot of site ${fx.trId}, which cloudhal.json does not list (siteId and sites); nothing was reverted`, + ); + }); + + it("prints a revert's dry run and exits 1 when it would be refused", async () => { + const { dir } = await editedCopy(); + await replace(dir); + const b = await labelled(dir, BEFORE); + await withStore(dir, async ({ store }) => { + const entry = (await store.listContentEntries(fx.siteId))[0] as ContentEntry; + await store.updateContentEntry({ ...entry, title: "Due later" }); + await new ReleaseService({ graph: store }).create({ + siteId: fx.siteId, + set: [entry.id], + createdBy: person, + }); + }); + vi.spyOn(process, "cwd").mockReturnValue(dir); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + expect(await main(["import", "--revert", b.id, "--dry-run"])).toBe(1); + const printed = log.mock.calls.map((c) => String(c[0])); + expect(printed[0]).toMatch(new RegExp(`^revert to snapshot ${b.id} "${BEFORE}"`)); + expect(printed).toContain("dry run: replace the site's nodes"); + expect(printed).toContainEqual(expect.stringMatching(/^would refuse: site .* open release/)); + expect(printed.at(-1)).toBe("nothing written (dry run)"); + }); +}); diff --git a/packages/cli/test/import-switch.test.ts b/packages/cli/test/import-switch.test.ts new file mode 100644 index 00000000..31294212 --- /dev/null +++ b/packages/cli/test/import-switch.test.ts @@ -0,0 +1,327 @@ +/** + * `cloudhal import --replace` of another site's archive (RFC-0039 §2): `cloudhal.json`'s `siteId` + * (and the old id in `sites`) switches to the archive's, editing only those values in the file's + * text, last: a failed import leaves it as it was, and one that landed while the file changed says + * so and a second run only switches; the old site stays in the local store, unlisted and + * untouched, and writing its id back undoes the switch; `--dry-run` shows the switch and writes + * nothing; a deployed project is warned that its deployment serves the old site (invariant 1), + * from `state.json` alone; a hostname the old site serves refuses it before anything is written. + * Then the text edit on its own. + */ +import { writeFileSync } from "node:fs"; +import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { Page, Site } from "@cloudhal/core-contracts"; +import { seedStarterSite } from "@cloudhal/core-ts"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { main } from "../src/cli.ts"; +import { exportArchive } from "../src/commands/export.ts"; +import { importArchive } from "../src/commands/import.ts"; +import { describeImport } from "../src/commands/import-describe.ts"; +import { listSites } from "../src/commands/site.ts"; +import { readProjectConfig } from "../src/config.ts"; +import { planSiteIdSwitch, type SiteIdSwitch, writeSiteIdSwitch } from "../src/config-site.ts"; +import { + bundleOf, + copyProject, + exists, + type Fixture, + removeTempDirs, + seedFixture, + stateOf, + tempDir, + withStore, +} from "./site-fixture.ts"; + +let fx: Fixture; +let archive: string; + +beforeAll(async () => { + fx = await seedFixture("cloudhal-switch-src-"); + const out = path.join(await tempDir("cloudhal-switch-archives-"), "english.cloudhal"); + archive = (await exportArchive({ projectDir: fx.projectDir, out })).file; +}); +afterAll(removeTempDirs); +afterEach(() => vi.restoreAllMocks()); + +const LISTED = "site_Listed0001"; + +/** + * A project whose store holds another site (slug `other`, optionally serving `host`), with a + * cloudhal.json written by hand: four-space indents, a key the CLI does not own, a one-line + * `sites` and `dataDir` last. + */ +async function otherProject(host?: string) { + const dir = await tempDir("cloudhal-switch-"); + const other = await withStore(dir, async ({ store }) => { + const site = await seedStarterSite(store, { name: "Other", slug: "other" }); + if (!host) return site; + return store.updateSite({ ...site, domains: [{ hostname: host, primary: true }] }); + }); + const raw = `{\n "siteId": "${other.id}",\n "media": { "keep": true },\n "sites": ["${other.id}", "${LISTED}"],\n "dataDir": ".cloudhal"\n}\n`; + await writeFile(path.join(dir, "cloudhal.json"), raw); + return { dir, other, raw }; +} + +/** Every row of the old site, as the switch must leave them. */ +const rowsOfSite = (dir: string, siteId: string) => + withStore(dir, async ({ sql }) => ({ + nodes: await sql.all("SELECT * FROM nodes WHERE site_id = ? ORDER BY id", [siteId]), + snapshots: await sql.all("SELECT * FROM snapshots WHERE site_id = ? ORDER BY id", [siteId]), + hosts: await sql.all("SELECT * FROM site_hosts WHERE site_id = ? ORDER BY host", [siteId]), + })); + +const configOf = (dir: string) => readFile(path.join(dir, "cloudhal.json"), "utf8"); + +/** `state.json` only, as `cloudhal deploy` leaves it (no secrets file is written or read). */ +async function deployed(dir: string) { + await mkdir(path.join(dir, ".cloudhal", "deploy"), { recursive: true }); + await writeFile(path.join(dir, ".cloudhal", "deploy", "state.json"), '{ "name": "smoke" }\n'); +} + +const secretsFile = (dir: string) => path.join(dir, ".cloudhal", "deploy", "secrets.json"); + +/** The switch line, real or dry run (`stays` ends it). */ +const switchLine = (from: string, stays: string) => + `cloudhal.json: siteId ${from} → ${fx.siteId}, sites ${from}, ${LISTED} → ${fx.siteId}, ${LISTED} (undo: write ${from} back in place of ${fx.siteId} by hand; --revert does not); site ${from} ${stays}`; + +describe("cloudhal import --replace of another site: the site id switch", () => { + it("switches cloudhal.json in place and keeps the old site in the store, unlisted", async () => { + const { dir, other, raw } = await otherProject(); + const old = await rowsOfSite(dir, other.id); + const result = await importArchive({ projectDir: dir, file: archive, replace: true }); + expect(await configOf(dir)).toBe(raw.replaceAll(other.id, fx.siteId)); + const cfg = await readProjectConfig(dir); + expect(cfg.siteId).toBe(fx.siteId); + expect(cfg.sites).toEqual([fx.siteId, LISTED]); + expect(await rowsOfSite(dir, other.id)).toEqual(old); + expect((await bundleOf(dir, fx.siteId)).nodes).toStrictEqual( + (await bundleOf(fx.projectDir, fx.siteId)).nodes, + ); + const labels = await withStore(dir, async ({ store }) => + (await store.listSnapshots(fx.siteId, 200)).map((s) => s.label ?? ""), + ); + expect(labels.some((l) => l.includes("import"))).toBe(false); + await expect(exportArchive({ projectDir: dir, siteId: other.id })).rejects.toThrow( + `site ${other.id} is not listed in cloudhal.json \`sites\``, + ); + expect((await listSites({ projectDir: dir })).map((s) => s.id)).toContain(other.id); + expect(describeImport(result)).toContain( + switchLine( + other.id, + `stays in the local store, unlisted and untouched (\`cloudhal site delete ${other.id} --confirm other\` deletes it for good)`, + ), + ); + // Again: nothing changes, and cloudhal.json already names the site. + const again = await importArchive({ projectDir: dir, file: archive, replace: true }); + expect(again.replace?.switch).toBeUndefined(); + expect(describeImport(again)[0]).toMatch(/^nothing to import: /); + // The undo it names: the old id written back makes the old site the project's again. + await writeFile(path.join(dir, "cloudhal.json"), raw); + const out = path.join(await tempDir("cloudhal-switch-undo-"), "other.cloudhal"); + expect((await exportArchive({ projectDir: dir, out })).manifest.siteId).toBe(other.id); + expect(await rowsOfSite(dir, other.id)).toEqual(old); + }); + + it("--dry-run shows the switch and writes nothing", async () => { + const { dir, other } = await otherProject(); + const before = await stateOf(dir); + vi.spyOn(process, "cwd").mockReturnValue(dir); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + expect(await main(["import", archive, "--replace", "--dry-run"])).toBe(0); + const printed = log.mock.calls.map((c) => String(c[0])); + expect(printed).toContain( + `dry run: replace import into site ${fx.siteId}, which is new to this project`, + ); + expect(printed).toContain("saved first: nothing (the site is new to this project)"); + expect(printed).toContain(switchLine(other.id, "would stay in the local store, unlisted")); + expect(printed.at(-1)).toBe("nothing written (dry run)"); + expect(await stateOf(dir)).toEqual(before); + }); + + it("refuses a hostname the old site serves, before anything is written; a bootstrap dry run reports it too", async () => { + const host = "english.example.com"; + const source = await copyProject(fx); + await withStore(source, async ({ store }) => { + const site = (await store.getSite(fx.siteId)) as Site; + await store.updateSite({ ...site, domains: [{ hostname: host, primary: true }] }); + }); + const file = path.join(source, "hosted.cloudhal"); + await exportArchive({ projectDir: source, out: file }); + const { dir, other } = await otherProject(host); + const before = await stateOf(dir); + const refusal = `the archive's site serves ${host}, which site ${other.id} of this project's store already serves; remove it from that site's domains in Studio, or delete that site when it is a leftover (\`cloudhal site delete ${other.id} --confirm \`)`; + await expect(importArchive({ projectDir: dir, file, replace: true })).rejects.toThrow( + `cannot import ${file}: ${refusal}; nothing was imported`, + ); + expect(await stateOf(dir)).toEqual(before); + vi.spyOn(process, "cwd").mockReturnValue(dir); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + expect(await main(["import", file, "--replace", "--dry-run"])).toBe(1); + expect(log.mock.calls.map((c) => String(c[0]))).toContain(`would refuse: ${refusal}`); + expect(await stateOf(dir)).toEqual(before); + // Bootstrap, with the archive's site listed: the same refusal, reported by the dry run. + await writeFile( + path.join(dir, "cloudhal.json"), + `${JSON.stringify({ siteId: fx.siteId, sites: [fx.siteId, other.id], dataDir: ".cloudhal" })}\n`, + ); + const dry = await importArchive({ projectDir: dir, file, dryRun: true }); + expect(dry.refusals).toEqual([refusal]); + expect(dry.plan).toBeUndefined(); + }); +}); + +describe("cloudhal import --replace of another site: cloudhal.json is switched last", () => { + it("leaves cloudhal.json and the old site as they were when the graph write fails", async () => { + const { dir, other, raw } = await otherProject(); + const old = await rowsOfSite(dir, other.id); + const batch = BetterSqlite3Driver.prototype.batch; + vi.spyOn(BetterSqlite3Driver.prototype, "batch").mockImplementation(async function ( + this: BetterSqlite3Driver, + statements: Parameters[0], + ) { + if (statements.some((s) => s.sql.startsWith("DELETE FROM nodes WHERE site_id"))) + throw new Error("disk on fire"); + return batch.call(this, statements); + }); + await expect(importArchive({ projectDir: dir, file: archive, replace: true })).rejects.toThrow( + `cannot import ${archive}: disk on fire; the import stopped after it began writing`, + ); + vi.restoreAllMocks(); + expect(await configOf(dir)).toBe(raw); + expect(await rowsOfSite(dir, other.id)).toEqual(old); + // Again: the import completes, and the switch comes with it. + await importArchive({ projectDir: dir, file: archive, replace: true }); + expect(await configOf(dir)).toBe(raw.replaceAll(other.id, fx.siteId)); + }); + + it("says the import landed when cloudhal.json changed meanwhile; a second run only switches", async () => { + const { dir, other, raw } = await otherProject(); + const file = path.join(dir, "cloudhal.json"); + // `log` runs after the first pass, before anything is written. + const log = () => writeFileSync(file, `${raw} `); + await expect( + importArchive({ projectDir: dir, file: archive, replace: true, log }), + ).rejects.toThrow( + `imported ${archive}, but cloudhal.json changed while importing, so it still names site ${other.id}; run the import again, which only switches siteId to ${fx.siteId}, or set it by hand`, + ); + expect(await configOf(dir)).toBe(`${raw} `); + expect((await bundleOf(dir, fx.siteId)).nodes).toStrictEqual( + (await bundleOf(fx.projectDir, fx.siteId)).nodes, + ); + await writeFile(file, raw); + const again = await importArchive({ projectDir: dir, file: archive, replace: true }); + expect(await configOf(dir)).toBe(raw.replaceAll(other.id, fx.siteId)); + const printed = describeImport(again); + expect(printed[0]).toMatch(/^imported /); + expect(printed).toContain( + switchLine( + other.id, + `stays in the local store, unlisted and untouched (\`cloudhal site delete ${other.id} --confirm other\` deletes it for good)`, + ), + ); + }); +}); + +describe("cloudhal import --replace in a deployed project (invariant 1)", () => { + it("warns that the deployment serves the old site, from state.json alone", async () => { + const { dir, other } = await otherProject(); + await deployed(dir); + const warning = (verb: string) => + `warning: this project is deployed (smoke) and its deployment serves site ${other.id}: \`cloudhal push\` ${verb} site ${fx.siteId} there as another site, and the next \`cloudhal deploy\` serves it instead; a deployment of site ${fx.siteId} itself is updated from the project that deployed it, not from here`; + const dry = await importArchive({ + projectDir: dir, + file: archive, + replace: true, + dryRun: true, + }); + expect(describeImport(dry)).toContain(warning("would send")); + const done = await importArchive({ projectDir: dir, file: archive, replace: true }); + const printed = describeImport(done); + expect(printed).toContain(warning("now sends")); + // A push to another admin URL would send this project's deploy token there. + expect(printed.some((l) => l.includes("--admin-url"))).toBe(false); + expect(await exists(secretsFile(dir))).toBe(false); + }); + + it("notes that a push carries the replaced graph and the deployment keeps its published snapshot", async () => { + const dir = await copyProject(fx); + await withStore(dir, async ({ store }) => { + const about = (await store.listPages(fx.siteId)).find((p) => p.path === "/about") as Page; + await store.updatePage({ ...about, title: "Edited here" }); + }); + await deployed(dir); + const done = await importArchive({ projectDir: dir, file: archive, replace: true }); + expect(describeImport(done)).toContain( + "note: this project is deployed (smoke); `cloudhal push --replace` sends this graph there, and the deployment keeps its own published snapshot", + ); + expect(await exists(secretsFile(dir))).toBe(false); + }); +}); + +describe("the cloudhal.json switch on its own", () => { + const A = "site_Aaaaaa0001"; + const B = "site_Bbbbbb0002"; + const planned = (raw: string) => { + const p = planSiteIdSwitch(raw, B); + if ("refusal" in p) throw new Error(p.refusal); + return p; + }; + + it("edits only the siteId and sites values, in any layout", () => { + const layouts = [ + `{\n "siteId": "${A}",\n "dataDir": ".cloudhal"\n}\n`, + `{\n "siteId" : "${A}",\n "sites": [\n "${A}",\n "site_Tttttt0003"\n ]\n}\n`, + `{\n\t"sites": ["site_Tttttt0003"],\n\t"siteId": "${A}"\n}`, + `{\r\n "siteId": "${A}",\r\n "sites": ["${A}", "${A}"]\r\n}\r\n`, + `{"siteId":"${A}","sites":["${A}"],"media":{"keep":true}}`, + ]; + for (const raw of layouts) expect(planned(raw).text).toBe(raw.replaceAll(A, B)); + expect(planned(layouts[0] as string).sites).toBeUndefined(); + expect(planned(layouts[1] as string).sites).toEqual({ + before: [A, "site_Tttttt0003"], + after: [B, "site_Tttttt0003"], + }); + // A list without the old id keeps it as written; the project's site comes first on reading. + expect(planned(layouts[2] as string).sites).toEqual({ + before: [A, "site_Tttttt0003"], + after: [B, "site_Tttttt0003"], + }); + }); + + it("refuses what it cannot place exactly", () => { + const refusals = [ + `{"siteId": "${A}", "search": {"siteId": "${A}"}}`, + `{"siteId": "site\\u005fAaaaaa0001"}`, + `{"dataDir": ".cloudhal"}`, + "not json", + ]; + for (const raw of refusals) + expect(planSiteIdSwitch(raw, B)).toEqual({ + refusal: expect.stringMatching( + new RegExp( + `; set cloudhal\\.json siteId to ${B} by hand \\(push and deploy then use it\\), then import again$`, + ), + ), + }); + }); + + it("writes through a temporary file, and refuses a file changed since it was read", async () => { + const dir = await tempDir("cloudhal-switch-write-"); + const raw = `{\n "siteId": "${A}",\n "dataDir": ".cloudhal"\n}\n`; + await writeFile(path.join(dir, "cloudhal.json"), raw); + const change: SiteIdSwitch = planned(raw); + await writeFile(path.join(dir, "cloudhal.json"), `${raw} `); + await expect(writeSiteIdSwitch(dir, change)).rejects.toThrow( + `cloudhal.json changed while importing, so it still names site ${A}; run the import again, which only switches siteId to ${B}, or set it by hand`, + ); + expect(await configOf(dir)).toBe(`${raw} `); + await writeFile(path.join(dir, "cloudhal.json"), raw); + await writeSiteIdSwitch(dir, change); + expect(await configOf(dir)).toBe(raw.replaceAll(A, B)); + expect(await readdir(dir)).toEqual(["cloudhal.json"]); + }); +}); diff --git a/packages/cli/test/site-fixture.ts b/packages/cli/test/site-fixture.ts index 0645b185..ec5ce455 100644 --- a/packages/cli/test/site-fixture.ts +++ b/packages/cli/test/site-fixture.ts @@ -3,9 +3,10 @@ * English site with three uploads (A the entry's cover; B taken down; C deleted, kept by the first * snapshot only), three snapshots (the second published, the third after an edit of /about) and a * second listed site (`tr`), all written on `SEEDED_AT`. Helpers open its store, copy it and list - * its blobs; every folder they make is removed by `removeTempDirs`. + * its blobs, make an empty project and capture what a refused import must leave as it was; every + * folder they make is removed by `removeTempDirs`. */ -import { cp, mkdtemp, readdir, rm } from "node:fs/promises"; +import { cp, mkdtemp, readdir, readFile, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import type { Actor, ContentEntry, Page, Site } from "@cloudhal/core-contracts"; @@ -13,7 +14,7 @@ import { exportSite, type SiteBundle, seedStarterSite } from "@cloudhal/core-ts" import { type LocalStore, openLocalStore } from "@cloudhal/core-ts/local"; import { makePng, PNG_1X1 } from "@cloudhal/core-ts/media/fixtures"; import { vi } from "vitest"; -import { writeProjectConfig } from "../src/config.ts"; +import { type ProjectConfig, writeProjectConfig } from "../src/config.ts"; export const person: Actor = { kind: "human", id: "user_01HZX8K3Q9CD", name: "Halil" }; export const robot: Actor = { kind: "agent", id: "agent_01HZX8K3Q9YZ", name: "Newsroom Agent" }; @@ -146,3 +147,37 @@ export async function blobListing(projectDir: string): Promise { .filter((f) => f.split(path.sep)[0] !== ".tmp") .sort(); } + +/** A project with only its cloudhal.json (no .cloudhal): the fixture's sites unless `cfg` says. */ +export async function emptyProject( + fx: Fixture, + cfg?: Pick, +): Promise { + const dir = await tempDir("cloudhal-import-"); + await writeProjectConfig(dir, { + dataDir: ".cloudhal", + ...(cfg ?? { siteId: fx.siteId, sites: [fx.siteId, fx.trId] }), + }); + return dir; +} + +export const exists = (file: string) => + stat(file).then( + () => true, + () => false, + ); + +/** What a refused import must leave as it was (opens the store: call it only where one exists). */ +export async function stateOf(dir: string) { + const rows = await withStore(dir, async ({ sql }) => ({ + nodes: await sql.all("SELECT id, site_id, rev, hash FROM nodes ORDER BY id"), + snapshots: await sql.all("SELECT id FROM snapshots ORDER BY id"), + projects: await sql.all("SELECT id, rev FROM projects ORDER BY id"), + media: await sql.all("SELECT site_id, sha256, state FROM media_blobs ORDER BY site_id, sha256"), + })); + return { + rows, + blobs: await blobListing(dir), + config: await readFile(path.join(dir, "cloudhal.json"), "utf8"), + }; +}