diff --git a/.changeset/bundle-project-row.md b/.changeset/bundle-project-row.md new file mode 100644 index 00000000..4e720fc1 --- /dev/null +++ b/.changeset/bundle-project-row.md @@ -0,0 +1,5 @@ +--- +"@cloudhal/core-ts": patch +--- + +`importSite` (behind `cloudhal pull`, the admin bundle route `cloudhal push` uses, and `cloudhal import`) now checks a bundle's project row before inserting it: it must be the site node's project (`projectId`), a valid Project, with its `id` and `rev` columns matching its body. Before, a bundle could plant any projects row. diff --git a/.changeset/import-plan.md b/.changeset/import-plan.md new file mode 100644 index 00000000..7d7f3db1 --- /dev/null +++ b/.changeset/import-plan.md @@ -0,0 +1,20 @@ +--- +"@cloudhal/core-ts": patch +--- + +`importSite`, used by `cloudhal pull`, by the admin bundle route `cloudhal push` uses, and by `cloudhal import`, now runs every check before it writes anything, the bootstrap conflict and the 1000-statement limit included, and plans the media ledger with them. A refused bundle leaves the target as it was; before, node bodies could land ahead of a conflict and snapshots ahead of the size limit. Only a storage failure can stop it between its writes, and importing the bundle again completes it. New checks, each before the first write: + +- Every node body the target lacks must be a valid contract document of the kind of each snapshot ref that points to it, with that ref's id and rev, of the bundle's site (precompiled validators). A body no snapshot references is refused. Before, a body was only checked against its hash, so a crafted bundle could put invalid content behind the published snapshot. +- Node and snapshot ids may not belong to another site of the target, each appears once, and no two nodes share a kind and key or a translation group and locale (the node batch's unique keys). +- Every takedown entry has its fields with their types (`reason`, `legal`, `by`, `at`). +- The media ledger (`snapshot_blobs`) reads the target's own asset bodies, never the bundle's text for a body the target already holds. + +New `planImport(to, bundle, options)` returns what an import would change, without writing: + +- the nodes added, changed and removed; +- the snapshots and node bodies to add; +- whether the project row is new; +- the listed media that are present, missing, of another size or taken down on the target; +- the bundle's takedowns, those whose bytes the target holds (an import deletes them) and those its ledger does not hold yet. + +With `report: true` it returns a bootstrap conflict and missing media instead of throwing them. A node body that is not JSON is refused as `blob is not JSON` instead of with the parser's message. diff --git a/.changeset/site-archive-export.md b/.changeset/site-archive-export.md new file mode 100644 index 00000000..db6c7416 --- /dev/null +++ b/.changeset/site-archive-export.md @@ -0,0 +1,12 @@ +--- +"@cloudhal/cli": patch +--- + +New `cloudhal export` (RFC-0039, #47) writes one site of the local project to a single `.cloudhal` file. The file is a gzipped tar holding the site's content graph (nodes, snapshots and the node bodies they reference, as NDJSON parts), its media files and its takedown ledger, with every file's SHA-256 listed in the manifest. The flags: + +- `--out ` names the file, and replaces a file already there once the new one is complete. By default the file is `-.cloudhal` in the current folder, dated by the newest timestamp among the rows it holds (deleting content alone does not move that date; the next edit or snapshot does). A file already at the default name is replaced only by the same archive; any other (a later edit the same day, other flags, a deletion) is left alone and the export asks for `--out`. +- `--site ` exports another site listed in `cloudhal.json` `sites`. `--site`, `--no-media` and `--latest-only` are refused on every other command. +- `--no-media` writes the graph only. +- `--latest-only` keeps only the published and the latest snapshot, and only the media they and the current content need. It is refused while a field operation is unfinished (`cloudhal fields continue|revert`). + +A damaged or missing local media file stops the export (run `cloudhal media verify`); so do Ctrl-C and SIGTERM. Either way no file is written and an existing one is left as it was. Exporting an unchanged site again with the same CLI and Node version gives the same bytes. An archive never contains user accounts, sessions, roles, agent keys, comments, readers, analytics, the audit log, automations, schedule runs, deployment secrets or webhook credentials. Its history does name who made each change (the user or agent of each snapshot, upload, block edit, review and takedown). diff --git a/.changeset/site-archive-import.md b/.changeset/site-archive-import.md new file mode 100644 index 00000000..37b281f3 --- /dev/null +++ b/.changeset/site-archive-import.md @@ -0,0 +1,13 @@ +--- +"@cloudhal/cli": patch +--- + +New `cloudhal import [--dry-run]` (RFC-0039, #47) reads a `.cloudhal` file written by `cloudhal export` into the local project, keeping the site's id. + +- It reads the file twice. First it verifies every file's size and SHA-256, every row and every node body (each must be the node its snapshot refs name), and checks the archive against the project. A damaged or edited file, invalid content, an archive of another site, a site whose nodes differ, or missing media is therefore refused before anything is written. Then it reads the file again, refusing it if it changed in between, copies the media the project lacks and imports the graph, keeping the archive's published snapshot as `cloudhal pull` does. +- Only bootstrap imports for now. The archive's site must be `cloudhal.json`'s `siteId` or listed in `sites`, and the project must not hold it yet or hold exactly its nodes; missing snapshots, node bodies and media are added. A site whose nodes differ is refused; `--replace` is not available yet. +- An archive written with `--no-media` imports only when the project already has its media. +- Taken-down content stays down: the archive's takedowns are recorded and their bytes never written, a local copy of content the archive's ledger took down is deleted (the summary says how many), and an archive carrying content this project has taken down is refused. +- `--dry-run` prints the manifest and what the import would add, change or remove (nodes by kind and title, snapshots, media to copy), writes nothing, and exits 1 when the import would be refused. +- Ctrl-C and SIGTERM stop the import before the graph is written; media already copied stay in the blob store, unreferenced, and the next import reuses them. A storage failure while the graph is written says the import stopped partway; running it again completes it, search index included. +- Opening an existing local store applies pending migrations, as every command does; a project without one gets none from a dry run or a refused import. diff --git a/packages/cli/src/archive/index.ts b/packages/cli/src/archive/index.ts index 646e2970..51ffb3ad 100644 --- a/packages/cli/src/archive/index.ts +++ b/packages/cli/src/archive/index.ts @@ -14,4 +14,9 @@ export { export { encodeNdjson, type NdjsonSplitOptions, splitNdjson } from "./parts.ts"; export type { ArchiveContent, ArchiveMediaSource } from "./plan.ts"; export { type ArchiveItem, type ReadArchiveOptions, readArchive } from "./read.ts"; -export { archiveStream, type WriteArchiveOptions, writeArchiveFile } from "./write.ts"; +export { + archiveStream, + type WriteArchiveFileOptions, + type WriteArchiveOptions, + writeArchiveFile, +} from "./write.ts"; diff --git a/packages/cli/src/archive/write.ts b/packages/cli/src/archive/write.ts index a9786cf6..0ad7553c 100644 --- a/packages/cli/src/archive/write.ts +++ b/packages/cli/src/archive/write.ts @@ -33,6 +33,14 @@ export interface WriteArchiveOptions { limits?: Partial; } +export interface WriteArchiveFileOptions extends WriteArchiveOptions { + /** + * Stops the write (`cloudhal export` aborts it on Ctrl-C and SIGTERM): the temporary file is + * removed, an existing `file` stays as it was, and the promise rejects. + */ + signal?: AbortSignal | undefined; +} + /** A media body as it is written: no more than `bytes`, then the SHA-256. */ async function* verifiedMedia( file: string, @@ -96,13 +104,16 @@ export function archiveStream( /** * Writes the archive of `content` to `file`: to a temporary file next to it first, synced and then - * renamed over `file`, so a failure leaves no file behind and an existing `file` unchanged. + * renamed over `file`, so a failure or an abort leaves no file behind and an existing `file` + * unchanged. Only a killed process (SIGKILL, power loss) can leave the `..partial-…` file. */ export async function writeArchiveFile( file: string, content: ArchiveContent, - options: WriteArchiveOptions = {}, + options: WriteArchiveFileOptions = {}, ): Promise { + const { signal } = options; + signal?.throwIfAborted(); const { manifest, bytes } = archiveStream(content, options); const temp = path.join( path.dirname(file), @@ -112,7 +123,7 @@ export async function writeArchiveFile( try { handle = await open(temp, "wx"); // The stream owns the handle: it fsyncs (flush) and closes it, on errors too. - await pipeline(bytes, handle.createWriteStream({ flush: true })); + await pipeline(bytes, handle.createWriteStream({ flush: true }), signal ? { signal } : {}); await rename(temp, file); return manifest; } catch (e) { diff --git a/packages/cli/src/assets.ts b/packages/cli/src/assets.ts index 58ab6664..fc491c89 100644 --- a/packages/cli/src/assets.ts +++ b/packages/cli/src/assets.ts @@ -33,6 +33,14 @@ async function packageRoot(): Promise { throw new Error("could not locate the @cloudhal/cli package root"); } +/** The `@cloudhal/cli` version (its package.json's), from either layout. */ +export async function cliVersion(): Promise { + const pkg = path.join(await packageRoot(), "package.json"); + const version = (JSON.parse(await readFile(pkg, "utf8")) as { version?: unknown }).version; + if (typeof version !== "string") throw new Error(`${pkg} has no version`); + return version; +} + export type AssetKind = "studio" | "starters" | "skills"; const MONOREPO: Record = { diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index d7fd9047..a667fd9c 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -38,6 +38,9 @@ export const OPTIONS = { views: { type: "string" }, from: { type: "string" }, out: { type: "string" }, + site: { type: "string" }, + "no-media": { type: "boolean" }, + "latest-only": { type: "boolean" }, jurisdiction: { type: "string" }, in: { type: "string" }, file: { type: "string" }, @@ -70,6 +73,26 @@ export const OPTIONS = { clear: { type: "boolean" }, } as const; +/** + * Flags only some commands take. `OPTIONS` is one table for every command, so without this a + * `cloudhal push --replace --site ` would parse and act on cloudhal.json's `siteId`. + */ +const ONLY_FOR: Readonly< + Partial> +> = { + site: { commands: ["export"], note: "; every other command acts on cloudhal.json's siteId" }, + "no-media": { commands: ["export"] }, + "latest-only": { commands: ["export"] }, +}; + +function refuseForeignFlags(command: string, values: Record): void { + for (const [flag, only] of Object.entries(ONLY_FOR)) + if (values[flag] !== undefined && !only.commands.includes(command)) + throw new Error( + `cloudhal ${command} does not take --${flag} (only cloudhal ${only.commands.join(", ")} does)${only.note ?? ""}`, + ); +} + /** `cloudhal ` → its handler. */ export const COMMANDS: Record = { ...LOCAL_COMMANDS, @@ -95,5 +118,6 @@ export async function main(argv: string[]): Promise { process.stderr.write(`unknown command "${command}"\n\n${HELP}`); return 1; } + refuseForeignFlags(command, values); return run(rest, values, { projectDir: process.cwd() }); } diff --git a/packages/cli/src/cli/commands-content.ts b/packages/cli/src/cli/commands-content.ts index b062d873..85f1b3fb 100644 --- a/packages/cli/src/cli/commands-content.ts +++ b/packages/cli/src/cli/commands-content.ts @@ -4,14 +4,20 @@ import { commentsStatusCli, describeCommentsStatus, } from "../commands/comments.ts"; +import { describeExport, exportArchive } from "../commands/export.ts"; import { fieldsCommand } from "../commands/fields.ts"; +import { ImportInterruptedError, importArchive } from "../commands/import.ts"; +import { describeImport } from "../commands/import-describe.ts"; import { describeMediaImport, mediaGc, mediaImportUrls, mediaVerify } from "../commands/media.ts"; import { describePlaceholders, mediaPlaceholders } from "../commands/media-placeholders.ts"; import { describeMigration, migrateTags } from "../commands/migrate.ts"; import { describeReindex, searchReindex } from "../commands/search.ts"; -import { type CliCommand, remoteFlags, subcommands } from "./types.ts"; +import { abortOnSignal, type CliCommand, remoteFlags, subcommands } from "./types.ts"; -/** `cloudhal migrate`, `media`, `search`, `comments`: the site's content and its stores. */ +/** + * `cloudhal migrate`, `fields`, `media`, `search`, `comments`, `export`, `import`: the site's + * content and its stores. + */ const migrateCommand: CliCommand = async (rest, values, { projectDir }) => { const [what] = rest; @@ -138,10 +144,72 @@ const commentsCommand = subcommands( "usage: cloudhal comments status|export|import", ); +/** Ctrl-C or SIGTERM stops the export: no file is written, an existing one stays (exit 130/143). */ +const exportCommand: CliCommand = async (rest, values, { projectDir }) => { + if (rest.length > 0) + throw new Error( + "usage: cloudhal export [--out ] [--site ] [--no-media] [--latest-only]", + ); + const interrupt = abortOnSignal(); + try { + const result = await exportArchive({ + projectDir, + ...(values.out !== undefined ? { out: values.out } : {}), + ...(values.site !== undefined ? { siteId: values.site } : {}), + ...(values["no-media"] ? { noMedia: true } : {}), + ...(values["latest-only"] ? { latestOnly: true } : {}), + signal: interrupt.signal, + }); + for (const line of describeExport(result, projectDir)) console.log(line); + return 0; + } catch (e) { + if (!interrupt.signal.aborted) throw e; + console.error("cloudhal: export interrupted; no file written, an existing one left as it was"); + return interrupt.exitCode(); + } finally { + interrupt.dispose(); + } +}; + +/** + * `cloudhal import [--dry-run]` (bootstrap only; `--replace` is RFC-0039 step 4). Ctrl-C or + * SIGTERM before the graph is written stops it (exit 130/143); a dry run that would be refused + * exits 1. + */ +const importCommand: CliCommand = async (rest, values, { projectDir }) => { + const [file] = rest; + if (file === undefined || rest.length !== 1) + throw new Error("usage: cloudhal import [--dry-run]"); + if (values.replace) + throw new Error( + "cloudhal import --replace is not available yet; this version imports in bootstrap mode only", + ); + const interrupt = abortOnSignal(); + try { + const result = await importArchive({ + projectDir, + file, + dryRun: values["dry-run"] === true, + signal: interrupt.signal, + log: (l) => process.stderr.write(`${l}\n`), + }); + for (const line of describeImport(result)) console.log(line); + return result.refusals.length > 0 ? 1 : 0; + } catch (e) { + if (!(e instanceof ImportInterruptedError)) throw e; + console.error(`cloudhal: ${e.message}`); + return interrupt.exitCode(); + } finally { + interrupt.dispose(); + } +}; + export const CONTENT_COMMANDS = { migrate: migrateCommand, fields: fieldsCommand, media: mediaCommand, search: searchCommand, comments: commentsCommand, + export: exportCommand, + import: importCommand, } satisfies Record; diff --git a/packages/cli/src/cli/help.ts b/packages/cli/src/cli/help.ts index be7bb068..24a4ae36 100644 --- a/packages/cli/src/cli/help.ts +++ b/packages/cli/src/cli/help.ts @@ -27,6 +27,24 @@ Usage: local site → deployment (refused while releases are open unless --cancel-schedules cancels them) cloudhal pull [--replace] [--admin-url ] deployment → local site + cloudhal export [--out ] [--site ] [--no-media] [--latest-only] + one site as a .cloudhal file (RFC-0039): its graph, + snapshots and media; never user accounts, comments, + analytics or secrets (its history names who made each + change). Default -.cloudhal, dated + by the newest row it holds (a deletion alone does not + move it); a file there is replaced only by the same + archive, --out replaces any; + --site: a site listed in cloudhal.json sites; + --no-media: the graph only; --latest-only: only the + published and the latest snapshot + cloudhal import [--dry-run] a .cloudhal file into this project (RFC-0039), its + site id kept: the archive's site must be cloudhal.json's + siteId or listed in sites, and absent here or holding + exactly the archive's nodes (missing snapshots and media + are added); every file is verified before anything is + written; --dry-run prints the manifest and the difference + and writes nothing cloudhal owner transfer --to [--admin-url ] make a member the deployment's Owner (lockout recovery; uses the deployment token) diff --git a/packages/cli/src/cli/types.ts b/packages/cli/src/cli/types.ts index 9d6bda8e..f8462e75 100644 --- a/packages/cli/src/cli/types.ts +++ b/packages/cli/src/cli/types.ts @@ -20,6 +20,36 @@ export function untilSignal(also?: (resolve: () => void) => void): Promise }); } +/** + * An `AbortSignal` that Ctrl-C and SIGTERM abort while a command writes a file. Listening replaces + * Node's default exit, which runs no `finally`, so the writer can remove its temporary file; a + * second Ctrl-C exits at once. `exitCode` is the shell's code for the signal received (130, 143); + * `dispose` stops listening. + */ +export function abortOnSignal(): { + signal: AbortSignal; + exitCode(): number; + dispose(): void; +} { + const controller = new AbortController(); + let code = 1; + const handlers = (["SIGINT", "SIGTERM"] as const).map((sig) => { + const handler = () => { + code = sig === "SIGINT" ? 130 : 143; + controller.abort(); + }; + process.once(sig, handler); + return { sig, handler }; + }); + return { + signal: controller.signal, + exitCode: () => code, + dispose: () => { + for (const { sig, handler } of handlers) process.off(sig, handler); + }, + }; +} + /** * A command whose first positional picks an action (`cloudhal media gc`); an unknown or missing * action throws `usage`. The action gets the positionals after its name. diff --git a/packages/cli/src/commands/archive-bundle.ts b/packages/cli/src/commands/archive-bundle.ts new file mode 100644 index 00000000..4a66c099 --- /dev/null +++ b/packages/cli/src/commands/archive-bundle.ts @@ -0,0 +1,294 @@ +/** + * RFC-0039 rollout step 2: a `SiteBundle` (what `exportSite` returns and `importSite` takes) as the + * archive container's content, and the parts both directions share; `archive-reassemble.ts` is the + * way back. Pure (no I/O), so the container in `../archive/` stays generic and these run in tests + * without a store. + * + * - Rows are written with the keys of the field tables below, in their order, so the archive's + * bytes do not depend on the column order a driver returns; the reader checks every row against + * the same tables. + * - `bundleMedia` rebuilds `bundle.media` from the rows the way `exportSite` does: the archive + * holds the media files, not their media types. + * + * Known limitation: `exportSite` returns a whole bundle, so the graph rows and the blob texts are + * held in memory. `importSite`'s single atomic batch already bounds a site to about 1000 nodes; + * snapshots are not bounded that way. Only the media bytes stream. + */ +import type { Asset } from "@cloudhal/core-contracts"; +import type { BundleMedia, BundlePurge, SiteBundle } from "@cloudhal/core-ts"; +import { type ArchiveContent, encodeNdjson, splitNdjson } from "../archive/index.ts"; + +/** How a row field is checked on the way back. */ +export type FieldType = "string" | "string|null" | "integer" | "boolean"; + +type Fields = Readonly>; + +/** `NodeRow` (core-ts sync.ts), in the `nodes` table's column order. */ +export const NODE_FIELDS = { + id: "string", + site_id: "string", + kind: "string", + key: "string", + rev: "integer", + hash: "string", + body: "string", + created_at: "string", + updated_at: "string", +} as const satisfies Fields; + +/** `SnapshotRow`. */ +export const SNAPSHOT_FIELDS = { + id: "string", + site_id: "string", + parent_id: "string|null", + site_rev: "integer", + body: "string", + created_at: "string", +} as const satisfies Fields; + +/** `ProjectRow`. */ +export const PROJECT_FIELDS = { + id: "string", + rev: "integer", + body: "string", + created_at: "string", + updated_at: "string", +} as const satisfies Fields; + +/** A line of `blobs/NNNN.ndjson`: a node body the snapshots reference, by its content hash. */ +export const BLOB_FIELDS = { hash: "string", body: "string" } as const satisfies Fields; + +/** `BundlePurge`: a line of `ledger.ndjson`. */ +export const PURGE_FIELDS = { + sha256: "string", + reason: "string", + legal: "boolean", + by: "string", + at: "string", +} as const satisfies Fields; + +/** A node ref of a snapshot body, as far as the archive needs it. */ +export interface SnapshotRef { + kind: string; + hash: string; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** Code-unit order: the same on every machine (and, for lowercase hex, `localeCompare`'s). */ +export function byCodeUnit(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +/** `row` with exactly the keys of `fields`, in their order. */ +function ordered(row: object, fields: Fields): Record { + const out: Record = {}; + for (const key of Object.keys(fields)) out[key] = (row as Record)[key]; + return out; +} + +/** The node refs of a snapshot body, or undefined when it is not `{nodes: [{kind, hash}, …]}`. */ +export function snapshotRefs(body: string): SnapshotRef[] | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(body); + } catch { + return undefined; + } + const nodes = isRecord(parsed) ? parsed.nodes : undefined; + if (!Array.isArray(nodes)) return undefined; + const refs: SnapshotRef[] = []; + for (const n of nodes) { + if (!isRecord(n) || typeof n.kind !== "string" || typeof n.hash !== "string") return undefined; + refs.push({ kind: n.kind, hash: n.hash }); + } + return refs; +} + +/** What the archive and its file name need from the bundle's site node. */ +export function siteFacts(bundle: Pick): { + name: string; + slug: string; + publishedSnapshotId?: string | undefined; +} { + const row = bundle.nodes.find((n) => n.kind === "site" && n.id === bundle.siteId); + if (!row) throw new Error(`the bundle has no site node for ${bundle.siteId}`); + const site = JSON.parse(row.body) as { + name?: unknown; + slug?: unknown; + publishedSnapshotId?: unknown; + }; + const published = site.publishedSnapshotId; + return { + name: typeof site.name === "string" ? site.name : "", + slug: typeof site.slug === "string" ? site.slug : "", + ...(typeof published === "string" ? { publishedSnapshotId: published } : {}), + }; +} + +/** + * `bundle.media` as `exportSite` builds it (core-ts sync.ts): the blob assets of the current nodes + * (in id order), then of each snapshot (in bundle order, refs in body order, bodies from `blobs`), + * the last one seen for a hash winning; taken-down content left out; sorted by hash; undefined + * when there is none. Keep the two in step: export-cli.test.ts compares them. + */ +export function bundleMedia( + rows: Pick, + purged: ReadonlySet, +): BundleMedia[] | undefined { + const media = new Map(); + const add = (body: string) => { + const a = JSON.parse(body) as Asset; + if (a.storage?.kind === "blob" && !purged.has(a.sha256)) + media.set(a.sha256, { sha256: a.sha256, bytes: a.bytes, mediaType: a.mediaType }); + }; + for (const n of rows.nodes) if (n.kind === "asset") add(n.body); + for (const snap of rows.snapshots) + for (const ref of snapshotRefs(snap.body) ?? []) { + const text = Object.hasOwn(rows.blobs, ref.hash) ? rows.blobs[ref.hash] : undefined; + if (ref.kind === "asset" && text) add(text); + } + if (media.size === 0) return undefined; + return [...media.values()].sort((a, b) => byCodeUnit(a.sha256, b.sha256)); +} + +/** + * The archive's `createdAt` (invariant 7 as amended): the newest timestamp among the rows it holds + * (nodes, snapshots, the project and the takedowns, since a takedown alone changes the ledger), + * never the clock, so an unchanged site exports identically. Not "the newest change": a deletion + * stamps no remaining row, so it alone leaves the date as it was (or moves it back, when the + * newest row went); the next edit or snapshot moves it on. Values that do not parse (the ledger's + * `''` defaults) are skipped. + */ +export function contentCreatedAt(bundle: SiteBundle): string { + let newest = Number.NEGATIVE_INFINITY; + const see = (at: string) => { + const t = Date.parse(at); + if (t > newest) newest = t; + }; + for (const n of bundle.nodes) { + see(n.created_at); + see(n.updated_at); + } + for (const s of bundle.snapshots) see(s.created_at); + if (bundle.project) { + see(bundle.project.created_at); + see(bundle.project.updated_at); + } + for (const p of bundle.purged ?? []) see(p.at); + if (!Number.isFinite(newest)) + throw new Error(`site ${bundle.siteId} has no row with a timestamp to date the archive by`); + return new Date(newest).toISOString(); +} + +/** + * RFC-0041 §6: a field operation whose marker (`migration`) is still on a field of the site node + * needs its `before` snapshot (the marker's id) to be continued or reverted, and blocks any other + * operation on its collection until then. Dropping that snapshot would leave the collection stuck + * after an import, so `--latest-only` is refused until the operation is finished or reverted. + */ +function refuseUnfinishedOperations(bundle: SiteBundle): void { + const row = bundle.nodes.find((n) => n.kind === "site" && n.id === bundle.siteId); + const site: unknown = row ? JSON.parse(row.body) : undefined; + const collections = isRecord(site) && isRecord(site.collections) ? site.collections : {}; + for (const [collection, c] of Object.entries(collections)) { + const fields: unknown[] = isRecord(c) && Array.isArray(c.fields) ? c.fields : []; + const migration = fields.map((f) => (isRecord(f) ? f.migration : undefined)).find(isRecord); + if (!migration) continue; + const id = String(migration.id); + throw new Error( + `field operation ${id} on ${collection} has not finished, and --latest-only would drop the snapshot it needs; finish or revert it first (\`cloudhal fields continue|revert ${collection} ${id}\`), or export the full history`, + ); + } +} + +/** + * `cloudhal export --latest-only`: only the published snapshot (the site node's pointer) and the + * latest one (`latestId`, the store's `latestSnapshot`; the bundle's last row when not given), in + * bundle order, and only the blobs they reference. Media are what the current nodes and those + * snapshots need; nodes, the project and the takedowns stay. A kept snapshot's `parent_id` may + * name a dropped one: nothing follows that link. Refused while a field operation is unfinished + * (`refuseUnfinishedOperations`). + */ +export function latestOnly(bundle: SiteBundle, latestId?: string): SiteBundle { + refuseUnfinishedOperations(bundle); + const latest = latestId ?? bundle.snapshots.at(-1)?.id; + const keep = new Set([siteFacts(bundle).publishedSnapshotId, latest]); + const snapshots = bundle.snapshots.filter((s) => keep.has(s.id)); + const hashes = new Set(snapshots.flatMap((s) => (snapshotRefs(s.body) ?? []).map((r) => r.hash))); + const blobs: Record = {}; + for (const [hash, text] of Object.entries(bundle.blobs)) if (hashes.has(hash)) blobs[hash] = text; + const purged = new Set((bundle.purged ?? []).map((p) => p.sha256)); + const media = bundleMedia({ nodes: bundle.nodes, snapshots, blobs }, purged); + return { + format: bundle.format, + siteId: bundle.siteId, + ...(bundle.project ? { project: bundle.project } : {}), + nodes: bundle.nodes, + snapshots, + blobs, + ...(media ? { media } : {}), + ...(bundle.purged ? { purged: bundle.purged } : {}), + }; +} + +/** Archives are self-contained: every snapshot ref has its body in the bundle's blobs. */ +function assertSelfContained(bundle: SiteBundle): void { + for (const snap of bundle.snapshots) { + const refs = snapshotRefs(snap.body); + if (!refs) throw new Error(`snapshot ${snap.id}: its body is not a snapshot`); + for (const ref of refs) + if (!Object.hasOwn(bundle.blobs, ref.hash)) + throw new Error(`snapshot ${snap.id} references node body ${ref.hash}, not in the bundle`); + } +} + +export interface BundleArchiveOptions { + /** The `@cloudhal/cli` version. */ + createdBy: string; + /** The bytes of a media file, opened when the writer reaches it. */ + openMedia(m: BundleMedia): AsyncIterable; + /** `--no-media`: no media files, `mediaOmitted` in the manifest. */ + mediaOmitted?: boolean | undefined; + /** NDJSON part size (tests; default 8 MiB). */ + partBytes?: number | undefined; +} + +/** + * `bundle` as the container's content (RFC-0039 §1): `project.json`, the node, snapshot and blob + * parts in the bundle's order (blobs sorted by hash), one media source per `bundle.media` entry + * and `ledger.ndjson` when content was taken down. + */ +export function bundleToArchive(bundle: SiteBundle, options: BundleArchiveOptions): ArchiveContent { + assertSelfContained(bundle); + const split = (rows: unknown[]) => + splitNdjson(rows, options.partBytes === undefined ? {} : { partBytes: options.partBytes }); + const blobs = Object.keys(bundle.blobs) + .sort(byCodeUnit) + .map((hash) => ({ hash, body: bundle.blobs[hash] })); + const purged: readonly BundlePurge[] = bundle.purged ?? []; + return { + siteId: bundle.siteId, + siteName: siteFacts(bundle).name, + createdBy: options.createdBy, + createdAt: contentCreatedAt(bundle), + ...(bundle.project ? { project: encodeNdjson([ordered(bundle.project, PROJECT_FIELDS)]) } : {}), + nodes: split(bundle.nodes.map((n) => ordered(n, NODE_FIELDS))), + snapshots: split(bundle.snapshots.map((s) => ordered(s, SNAPSHOT_FIELDS))), + blobs: split(blobs), + ...(purged.length > 0 + ? { ledger: encodeNdjson(purged.map((p) => ordered(p, PURGE_FIELDS))) } + : {}), + ...(options.mediaOmitted + ? { mediaOmitted: true as const } + : { + media: (bundle.media ?? []).map((m) => ({ + sha256: m.sha256, + bytes: m.bytes, + open: () => options.openMedia(m), + })), + }), + }; +} diff --git a/packages/cli/src/commands/archive-reassemble.ts b/packages/cli/src/commands/archive-reassemble.ts new file mode 100644 index 00000000..7a354fc5 --- /dev/null +++ b/packages/cli/src/commands/archive-reassemble.ts @@ -0,0 +1,353 @@ +/** + * RFC-0039 §1: a site archive's items (from `readArchive`) back into exactly the `SiteBundle` that + * `exportSite` returned when it was written. `cloudhal import` (rollout step 3) hands the result + * to `importSite`; the tests compare it with `exportSite`. + * + * Every row is checked against the field tables of `archive-bundle.ts` (the exact keys, strings, + * non-negative integers), belongs to the manifest's site and appears once; every snapshot ref has + * its body in the archive. Every asset body `bundleMedia` reads (asset nodes, the bodies of + * snapshots' asset refs) is a JSON object with a `storage` kind and, for a blob asset, has a hex + * SHA-256, a size and a media type, so the media rule never throws or quotes; `importSite` then + * validates every body against its contract. The media files must match the graph: + * none that no asset needs, none taken down by the archive's own ledger (invariant 5), each with + * its asset's size, and, unless the manifest says `mediaOmitted`, none missing. + * + * The ledger comes after the media in the layout, so a media file for taken-down content is + * refused only at the end, after `onMedia` saw it. `cloudhal import` therefore reads the file + * twice: a first pass verifies everything (no `onMedia`), so the second writes only verified media; + * if a later step fails, media already written stay unreferenced (content-addressed). Nothing is + * returned before `readArchive` has verified the whole archive. + * + * Errors name the archive path, the line, our own field names and validated hex hashes; never ids, + * keys, bodies or JSON.parse messages, which can quote untrusted input. + */ +import { + type BundleMedia, + type BundlePurge, + type NodeRow, + type ProjectRow, + SITE_BUNDLE_FORMAT, + type SiteBundle, + type SnapshotRow, +} from "@cloudhal/core-ts"; +import type { ArchiveItem, PartKind, SiteArchiveManifest } from "../archive/index.ts"; +import { + BLOB_FIELDS, + bundleMedia, + type FieldType, + isRecord, + NODE_FIELDS, + PROJECT_FIELDS, + PURGE_FIELDS, + SNAPSHOT_FIELDS, + type SnapshotRef, + snapshotRefs, +} from "./archive-bundle.ts"; + +export type ArchiveMediaItem = Extract; +type FileItem = Extract; + +export interface ReassembleOptions { + /** + * Called for each media file, in archive order, after it was matched against the graph; reads + * (or ignores) its body before resolving. The body is verified as it is read. + */ + onMedia?: ((item: ArchiveMediaItem) => Promise) | undefined; +} + +export interface ReassembledArchive { + manifest: SiteArchiveManifest; + bundle: SiteBundle; + /** The media files the archive held, in archive order. */ + media: { sha256: string; size: number }[]; +} + +const SHA256 = /^[0-9a-f]{64}$/; +const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + +/** What has been read so far. */ +interface Draft { + /** Blob texts already checked as asset bodies. */ + assetBlobs: Set; + manifest: SiteArchiveManifest; + project?: ProjectRow | undefined; + nodes: NodeRow[]; + nodeIds: Set; + snapshots: SnapshotRow[]; + snapshotIds: Set; + /** Each snapshot's refs and where it was read, for the checks once the blobs are in. */ + refs: { at: string; refs: SnapshotRef[] }[]; + blobs: Record; + purged: BundlePurge[]; + purgedShas: Set; + media: { sha256: string; size: number }[]; + /** Set once the graph is complete (at the first media file, or at the end). */ + needed?: Map | undefined; +} + +type Row>> = { + -readonly [K in keyof F]: F[K] extends "integer" + ? number + : F[K] extends "boolean" + ? boolean + : F[K] extends "string|null" + ? string | null + : string; +}; + +const FIELD_CHECKS: Record boolean> = { + string: (v) => typeof v === "string", + "string|null": (v) => v === null || typeof v === "string", + integer: (v) => Number.isSafeInteger(v) && (v as number) >= 0, + boolean: (v) => typeof v === "boolean", +}; + +const FIELD_WORDS: Record = { + string: "a string", + "string|null": "a string or null", + integer: "a non-negative integer", + boolean: "true or false", +}; + +/** A new row with exactly `fields`' keys, in their order, each of its type; or the reason not. */ +function rowOf>>( + value: unknown, + fields: F, + at: string, +): Row { + if (!isRecord(value)) throw new Error(`${at}: not a JSON object`); + const keys = Object.keys(fields); + if (Object.keys(value).length !== keys.length || !keys.every((k) => Object.hasOwn(value, k))) + throw new Error(`${at}: needs exactly the fields ${keys.join(", ")}`); + const row: Record = {}; + for (const key of keys) { + const type = fields[key] as FieldType; + if (!FIELD_CHECKS[type](value[key])) + throw new Error(`${at}: ${key} must be ${FIELD_WORDS[type]}`); + row[key] = value[key]; + } + return row as Row; +} + +/** The JSON value of each line of an NDJSON file, with where it is (`path line n`). */ +function* lines(item: FileItem): Generator<{ value: unknown; at: string }> { + let text: string; + try { + text = decoder.decode(item.bytes); + } catch { + throw new Error(`${item.path}: not UTF-8`); + } + // The reader checks this for the parts and the ledger, not for project.json. + if (!text.endsWith("\n")) throw new Error(`${item.path}: not NDJSON (no final newline)`); + const all = text.split("\n"); + for (let i = 0; i < all.length - 1; i++) { + const at = `${item.path} line ${i + 1}`; + let value: unknown; + try { + value = JSON.parse(all[i] as string); + } catch { + throw new Error(`${at}: not JSON`); + } + yield { value, at }; + } +} + +function ownSite(draft: Draft, siteId: string, at: string): void { + if (siteId !== draft.manifest.siteId) throw new Error(`${at}: site_id is not the archive's site`); +} + +/** + * An asset body as `bundleMedia` and the media ledger read it: a JSON object with a `storage` + * object naming its kind and, when its bytes are in the blob store, a hex `sha256`, a non-negative + * `bytes` and a string `mediaType`. Never quotes the body. + */ +function checkAssetBody(body: string, at: string, what: string): void { + let asset: unknown; + try { + asset = JSON.parse(body); + } catch { + throw new Error(`${at}: ${what} is not JSON`); + } + if (!isRecord(asset)) throw new Error(`${at}: ${what} is not a JSON object`); + if (!isRecord(asset.storage) || typeof asset.storage.kind !== "string") + throw new Error(`${at}: ${what} needs storage, an object with a kind`); + if (asset.storage.kind !== "blob") return; + if (typeof asset.sha256 !== "string" || !SHA256.test(asset.sha256)) + throw new Error(`${at}: ${what} needs a sha256 of 64 lowercase hex digits`); + if (!FIELD_CHECKS.integer(asset.bytes)) + throw new Error(`${at}: ${what} needs bytes, a non-negative integer`); + if (typeof asset.mediaType !== "string") + throw new Error(`${at}: ${what} needs a mediaType string`); +} + +function addNode(draft: Draft, value: unknown, at: string): void { + const row = rowOf(value, NODE_FIELDS, at); + ownSite(draft, row.site_id, at); + if (row.kind === "asset") checkAssetBody(row.body, at, "the asset body"); + if (draft.nodeIds.has(row.id)) throw new Error(`${at}: a node id appears twice`); + draft.nodeIds.add(row.id); + draft.nodes.push(row); +} + +function addSnapshot(draft: Draft, value: unknown, at: string): void { + const row = rowOf(value, SNAPSHOT_FIELDS, at); + ownSite(draft, row.site_id, at); + if (draft.snapshotIds.has(row.id)) throw new Error(`${at}: a snapshot id appears twice`); + const refs = snapshotRefs(row.body); + if (!refs) throw new Error(`${at}: body is not a snapshot (no {nodes: [{kind, hash}]})`); + draft.snapshotIds.add(row.id); + draft.snapshots.push(row); + draft.refs.push({ at, refs }); +} + +function addBlob(draft: Draft, value: unknown, at: string): void { + const row = rowOf(value, BLOB_FIELDS, at); + if (!SHA256.test(row.hash)) throw new Error(`${at}: hash must be 64 lowercase hex digits`); + if (Object.hasOwn(draft.blobs, row.hash)) + throw new Error(`${at}: blob ${row.hash} appears twice`); + draft.blobs[row.hash] = row.body; +} + +function addPurge(draft: Draft, value: unknown, at: string): void { + const row = rowOf(value, PURGE_FIELDS, at); + if (!SHA256.test(row.sha256)) throw new Error(`${at}: sha256 must be 64 lowercase hex digits`); + if (draft.purgedShas.has(row.sha256)) throw new Error(`${at}: ${row.sha256} appears twice`); + draft.purgedShas.add(row.sha256); + draft.purged.push(row); +} + +const PART_ROWS: Record void> = { + nodes: addNode, + snapshots: addSnapshot, + blobs: addBlob, +}; + +/** One handler per kind of buffered file. */ +const FILES: Record void> = { + project: (draft, item) => { + for (const { value, at } of lines(item)) { + if (draft.project) throw new Error(`${at}: project.json holds more than one row`); + draft.project = rowOf(value, PROJECT_FIELDS, at); + } + }, + part: (draft, item) => { + const slot = item.slot; + if (slot.kind !== "part") return; + if (draft.needed) throw new Error(`${item.path}: comes after the media`); + const add = PART_ROWS[slot.part]; + for (const { value, at } of lines(item)) add(draft, value, at); + }, + ledger: (draft, item) => { + for (const { value, at } of lines(item)) addPurge(draft, value, at); + }, +}; + +/** + * The graph checks that need every row, run once: the site node, every snapshot ref's body. Then + * the media the graph needs, taken-down content not yet left out (the ledger comes last). + */ +function needed(draft: Draft): Map { + if (draft.needed) return draft.needed; + const siteId = draft.manifest.siteId; + if (!draft.nodes.some((n) => n.kind === "site" && n.id === siteId)) + throw new Error(`nodes: the archive has no site node for ${siteId}`); + for (const { at, refs } of draft.refs) + for (const ref of refs) { + if (!Object.hasOwn(draft.blobs, ref.hash)) + throw new Error(`${at}: references a node body that is not in the archive's blobs`); + const text = draft.blobs[ref.hash]; + // `bundleMedia` reads only non-empty bodies of asset refs. + if (ref.kind !== "asset" || !text || draft.assetBlobs.has(ref.hash)) continue; + checkAssetBody(text, at, "an asset body it references"); + draft.assetBlobs.add(ref.hash); + } + const media = bundleMedia(draft, new Set()) ?? []; + draft.needed = new Map(media.map((m) => [m.sha256, m])); + return draft.needed; +} + +async function addMedia( + draft: Draft, + item: ArchiveMediaItem, + options: ReassembleOptions, +): Promise { + const asset = needed(draft).get(item.sha256); + if (!asset) throw new Error(`${item.path}: no asset of the archive needs this file`); + if (asset.bytes !== item.size) + throw new Error(`${item.path}: ${item.size} bytes, its asset says ${asset.bytes}`); + await options.onMedia?.(item); + draft.media.push({ sha256: item.sha256, size: item.size }); +} + +/** The media files against the final media (the ledger applied); `bundle.media`. */ +function finalMedia(draft: Draft): BundleMedia[] | undefined { + needed(draft); + const media = bundleMedia(draft, draft.purgedShas); + for (const m of draft.media) + if (draft.purgedShas.has(m.sha256)) + throw new Error(`media/${m.sha256}: the archive's ledger has taken this content down`); + if (!draft.manifest.mediaOmitted) { + const present = new Set(draft.media.map((m) => m.sha256)); + for (const m of media ?? []) + if (!present.has(m.sha256)) + throw new Error(`media/${m.sha256}: an asset needs it, and the archive does not hold it`); + } + return media; +} + +async function firstManifest(iterator: AsyncIterator): Promise { + const first = await iterator.next(); + if (first.done || first.value.type !== "manifest") + throw new Error("the archive's items do not start with its manifest"); + return { + assetBlobs: new Set(), + manifest: first.value.manifest, + nodes: [], + nodeIds: new Set(), + snapshots: [], + snapshotIds: new Set(), + refs: [], + blobs: {}, + purged: [], + purgedShas: new Set(), + media: [], + }; +} + +function assemble(draft: Draft): ReassembledArchive { + const media = finalMedia(draft); + const bundle: SiteBundle = { + format: SITE_BUNDLE_FORMAT, + siteId: draft.manifest.siteId, + ...(draft.project ? { project: draft.project } : {}), + nodes: draft.nodes, + snapshots: draft.snapshots, + blobs: draft.blobs, + ...(media ? { media } : {}), + ...(draft.purged.length > 0 ? { purged: draft.purged } : {}), + }; + return { manifest: draft.manifest, bundle, media: draft.media }; +} + +/** + * The bundle of an archive: reads `items` to the end (the reader verifies the whole archive only + * then) and hands each media file to `onMedia` on the way. A refusal stops the reader. + */ +export async function bundleFromArchive( + items: AsyncIterable, + options: ReassembleOptions = {}, +): Promise { + const iterator = items[Symbol.asyncIterator](); + try { + const draft = await firstManifest(iterator); + for (let next = await iterator.next(); !next.done; next = await iterator.next()) { + const item = next.value; + if (item.type === "manifest") throw new Error("manifest.json: appears twice"); + if (item.type === "media") await addMedia(draft, item, options); + else FILES[item.slot.kind](draft, item); + } + return assemble(draft); + } finally { + await iterator.return?.(); + } +} diff --git a/packages/cli/src/commands/export.ts b/packages/cli/src/commands/export.ts new file mode 100644 index 00000000..f37012fa --- /dev/null +++ b/packages/cli/src/commands/export.ts @@ -0,0 +1,337 @@ +/** + * `cloudhal export [--out ] [--site ] [--no-media] [--latest-only]` (RFC-0039 §2, + * rollout step 2): one site of the local project as a `.cloudhal` file, its content graph, + * snapshots, the node bodies they reference, its media and its takedown ledger. + * + * - **Site:** the project's (`cloudhal.json` `siteId`), or with `--site` another one listed in + * `sites`; any other id is refused before the store is opened. + * - **File:** `-.cloudhal` in the current folder unless `--out` names one; + * the date is that of the archive's `createdAt`. The default name says only the site and a day, + * and the same day can hold other content (a later edit that day, another flag, or a deletion, + * which moves no timestamp), so a file already there is replaced only by the same archive (its + * manifest, which pins every file's SHA-256, is the new one's); any other is refused, and + * `--out` naming it replaces it. A file is replaced only once the new one is complete + * (`writeArchiveFile` writes a temporary file beside it and renames it over): a failed or + * interrupted (Ctrl-C, SIGTERM) export leaves no file and an existing one unchanged. + * - **createdAt** (invariant 7 as amended): the newest timestamp among the exported rows, never + * the clock, so an unchanged site exports to the same name and, with the same CLI and Node/zlib + * build, the same bytes. Deleting content stamps no remaining row, so it alone does not move the + * date; the next edit or snapshot does. + * - **`--latest-only`:** only the published and the latest snapshot (`latestSnapshot`, what + * `cloudhal build` falls back to), only the blobs they reference, and only the media the current + * nodes and those snapshots need; the takedown ledger still travels. Refused while an RFC-0041 + * field operation is unfinished: it needs its `before` snapshot. + * - **`--no-media`:** the graph only; the manifest says `mediaOmitted`. + * - **Never in an archive (§4):** user accounts, sessions, roles, agent keys, comments, readers, + * analytics, the audit log, automations, schedule runs, deployment secrets and webhook + * credentials. None of their tables or files are read. The graph does name who made each change, + * as its contracts require: snapshots, uploads, block edits, reviews, term proposals and + * takedowns carry user and agent ids (most with display names), entries their authors and the + * project its owner's id. + * + * A media file whose local bytes are missing, or differ from its asset's size and SHA-256, stops + * the export (`cloudhal media verify` finds them). Known limitation: the graph rows are held in + * memory (`exportSite` returns a whole bundle); media bytes stream from the blob store. + */ +import { createReadStream } from "node:fs"; +import { mkdir, stat } from "node:fs/promises"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import type { Site } from "@cloudhal/core-contracts"; +import { type BundleMedia, exportSite, mediaBlobKey, slugify } from "@cloudhal/core-ts"; +import { openLocalStore } from "@cloudhal/core-ts/local"; +import type { BlobStore } from "@cloudhal/storage"; +import { + type ArchiveContent, + ArchiveError, + archiveStream, + readArchive, + type SiteArchiveManifest, + writeArchiveFile, +} from "../archive/index.ts"; +import { cliVersion } from "../assets.ts"; +import { type ProjectConfig, readProjectConfig } from "../config.ts"; +import { bundleToArchive, latestOnly } from "./archive-bundle.ts"; + +export interface ExportArchiveOptions { + readonly projectDir: string; + /** The file to write, relative to `projectDir`; default `-.cloudhal`. */ + readonly out?: string | undefined; + /** A site listed in `cloudhal.json` `sites`; default the project's `siteId`. */ + readonly siteId?: string | undefined; + readonly noMedia?: boolean | undefined; + readonly latestOnly?: boolean | undefined; + /** Aborts the export (the command's Ctrl-C and SIGTERM): no file is written. */ + readonly signal?: AbortSignal | undefined; +} + +export interface ExportArchiveResult { + /** Absolute path of the archive. */ + readonly file: string; + readonly bytes: number; + /** A file was there before and has been replaced. */ + readonly replaced: boolean; + readonly manifest: SiteArchiveManifest; + readonly slug: string; + /** Takedown ledger rows. */ + readonly purged: number; + readonly latestOnly: boolean; +} + +/** RFC-0039 §4: what an archive never contains. */ +export const NEVER_IN_ARCHIVE: readonly string[] = [ + "user accounts", + "sessions", + "roles", + "agent keys", + "comments", + "readers", + "analytics", + "the audit log", + "automations", + "schedule runs", + "deployment secrets", + "webhook credentials", +]; + +/** What the graph does carry about people (the summary says so beside §4's list). */ +export const NAMED_IN_ARCHIVE = + "who made each change: snapshots, uploads, block edits, reviews and takedowns name their user or agent (by id, most also by display name)"; + +const SLUG = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const SAFE_ID = /^[A-Za-z0-9_-]{1,64}$/; + +/** + * `-.cloudhal`, dated by `createdAt` in UTC. The slug is the site's (always + * valid in the store); then the slug of its name; then its id. + */ +export function archiveFileName( + site: Pick, + createdAt: string, +): string { + const base = + [site.slug, slugify(site.name)].find((s) => SLUG.test(s) && s.length <= 128) ?? + (SAFE_ID.test(site.id) ? site.id : "site"); + const date = new Date(createdAt).toISOString().slice(0, 10).replaceAll("-", ""); + return `${base}-${date}.cloudhal`; +} + +function missingMedia(sha256: string): Error { + return new Error( + `local media ${sha256} is missing; run \`cloudhal media verify\` (or export with --no-media)`, + ); +} + +/** One media file's bytes: streamed when the store can, read whole otherwise. */ +async function* mediaBytes( + blobs: BlobStore, + siteId: string, + sha256: string, +): AsyncGenerator { + const key = mediaBlobKey(siteId, sha256); + if (!blobs.getStream) { + const bytes = await blobs.get(key); + if (!bytes) throw missingMedia(sha256); + yield bytes; + return; + } + const stream = await blobs.getStream(key); + if (!stream) throw missingMedia(sha256); + const reader = stream.body.getReader(); + try { + for (let read = await reader.read(); !read.done; read = await reader.read()) yield read.value; + } finally { + // Stopped early (the writer refused the file): closes the file it reads. + await reader.cancel().catch(() => {}); + } +} + +/** The media sources' bytes, from the site's local blob store. */ +export function localMediaReader( + blobs: BlobStore, + siteId: string, +): (m: BundleMedia) => AsyncIterable { + return (m) => mediaBytes(blobs, siteId, m.sha256); +} + +/** The site to export: the project's, or one listed in `sites`. Checked before any I/O. */ +function chooseSite(cfg: ProjectConfig, requested: string | undefined): string { + const siteId = requested ?? cfg.siteId; + const allowed = cfg.sites ?? [cfg.siteId]; + if (!allowed.includes(siteId)) + throw new Error( + `site ${siteId} is not listed in cloudhal.json \`sites\` (this project's sites: ${allowed.join(", ")})`, + ); + return siteId; +} + +/** Every media file the archive will hold must be in the local blob store; one message for all. */ +async function assertLocalMedia( + blobs: BlobStore, + siteId: string, + media: readonly BundleMedia[], +): Promise { + const missing: string[] = []; + for (const m of media) + if (!(await blobs.head(mediaBlobKey(siteId, m.sha256)))) missing.push(m.sha256); + const [first] = missing; + if (first === undefined) return; + if (missing.length === 1) throw missingMedia(first); + throw new Error( + `local media ${first} and ${missing.length - 1} more are missing; run \`cloudhal media verify\` (or export with --no-media)`, + ); +} + +/** The archive's path (its folder created), and whether a file is there already. */ +async function outputFile( + projectDir: string, + out: string | undefined, + name: string, +): Promise<{ file: string; replaced: boolean }> { + const shown = out ?? name; + if (shown === "") throw new Error("--out needs a file name"); + if (shown.endsWith("/") || shown.endsWith(path.sep)) + throw new Error(`--out ${shown} is a directory; name a file`); + const file = path.resolve(projectDir, shown); + const existing = await stat(file).catch((e: NodeJS.ErrnoException) => { + if (e.code === "ENOENT") return undefined; + throw e; + }); + if (existing?.isDirectory()) throw new Error(`--out ${shown} is a directory; name a file`); + await mkdir(path.dirname(file), { recursive: true }); + return { file, replaced: existing !== undefined }; +} + +/** The manifest of the archive at `file`; undefined when it is not one (or does not read as one). */ +async function manifestAt(file: string): Promise { + const stream = createReadStream(file); + const items = readArchive(stream); + try { + const first = await items.next(); + return !first.done && first.value.type === "manifest" ? first.value.manifest : undefined; + } catch { + return undefined; + } finally { + await items.return(undefined).catch(() => {}); + stream.destroy(); + } +} + +/** + * A file at the default name is replaced only by the same archive: its manifest lists the same + * files with the same SHA-256s, so its tar is byte for byte the new one's (only another zlib + * build's gzip bytes can differ). Anything else stays; `--out` naming it replaces it. + */ +async function keepOtherArchive( + file: string, + name: string, + content: ArchiveContent, +): Promise { + const existing = await manifestAt(file); + const plain = (m: SiteArchiveManifest) => JSON.parse(JSON.stringify(m)) as unknown; + if (existing && isDeepStrictEqual(plain(existing), plain(archiveStream(content).manifest))) + return; + throw new Error( + `${name} already exists and is not this archive (other content or flags, or another CLI version); pass --out ${name} to replace it, or --out to keep both`, + ); +} + +/** `writeArchiveFile`, with a damaged local media file named as such. */ +async function writeArchive( + file: string, + content: ArchiveContent, + signal: AbortSignal | undefined, +): Promise { + try { + return await writeArchiveFile(file, content, { signal }); + } catch (e) { + if (e instanceof ArchiveError && e.code === "damaged" && e.path?.startsWith("media/")) + throw new ArchiveError( + "damaged", + `${e.message}; the local copy is damaged: run \`cloudhal media verify\` (or export with --no-media)`, + { path: e.path, cause: e }, + ); + throw e; + } +} + +/** Writes the archive; see the module doc for every rule. */ +export async function exportArchive(options: ExportArchiveOptions): Promise { + const cfg = await readProjectConfig(options.projectDir); + const siteId = chooseSite(cfg, options.siteId); + const local = await openLocalStore(path.join(options.projectDir, cfg.dataDir)); + // Open until the archive is written: the media are opened as the writer reaches them. + try { + const site = await local.store.getSite(siteId); + if (!site) + throw new Error( + `site ${siteId} is listed in cloudhal.json but the local store (${cfg.dataDir}) has no such site`, + ); + const full = await exportSite(local, siteId); + const bundle = options.latestOnly + ? latestOnly(full, (await local.store.latestSnapshot(siteId))?.id) + : full; + if (!options.noMedia) await assertLocalMedia(local.blobs, siteId, bundle.media ?? []); + const content = bundleToArchive(bundle, { + createdBy: await cliVersion(), + mediaOmitted: options.noMedia === true, + openMedia: localMediaReader(local.blobs, siteId), + }); + const name = archiveFileName(site, content.createdAt); + const { file, replaced } = await outputFile(options.projectDir, options.out, name); + if (replaced && options.out === undefined) await keepOtherArchive(file, name, content); + const manifest = await writeArchive(file, content, options.signal); + return { + file, + bytes: (await stat(file)).size, + replaced, + manifest, + slug: site.slug, + purged: bundle.purged?.length ?? 0, + latestOnly: options.latestOnly === true, + }; + } finally { + await local.close(); + } +} + +/** 1536 → `1.5 KiB` (also `cloudhal import`'s sizes). */ +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + const units = ["KiB", "MiB", "GiB", "TiB"]; + let value = bytes / 1024; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit++; + } + return `${value.toFixed(1)} ${units[unit]}`; +} + +/** + * The summary `cloudhal export` prints: paths, sizes, counts, the site id and slug, and what an + * archive never holds. Never the site name or any row content. + */ +export function describeExport(result: ExportArchiveResult, projectDir: string): string[] { + const { manifest } = result; + const { counts } = manifest; + const relative = path.relative(projectDir, result.file); + const shown = + relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative) + ? relative + : result.file; + const media = manifest.mediaOmitted + ? "" + : `, ${counts.media} media file(s) (${formatBytes(counts.mediaBytes)})`; + const lines = [ + `wrote ${shown} (${formatBytes(result.bytes)}${result.replaced ? "; replaced the existing file" : ""})`, + `site ${manifest.siteId} (${result.slug}): ${counts.nodes} node(s), ${counts.snapshots} snapshot(s), ${counts.blobs} blob(s)${media}${result.purged > 0 ? `, ${result.purged} taken down` : ""}`, + ]; + if (manifest.mediaOmitted) + lines.push("media left out (--no-media): an import needs the media from elsewhere"); + if (result.latestOnly) + lines.push("history: the published and the latest snapshot only (--latest-only)"); + lines.push(`not in the archive (RFC-0039 §4): ${NEVER_IN_ARCHIVE.join(", ")}`); + lines.push(`in the archive: ${NAMED_IN_ARCHIVE}`); + return lines; +} diff --git a/packages/cli/src/commands/import-checks.ts b/packages/cli/src/commands/import-checks.ts new file mode 100644 index 00000000..442721c2 --- /dev/null +++ b/packages/cli/src/commands/import-checks.ts @@ -0,0 +1,168 @@ +/** + * `cloudhal import`'s rules and words (RFC-0039 §2, rollout step 3), with no I/O: which site an + * archive may bring into this project, why a bootstrap import is refused, and every failure as + * one line. Text that comes from the archive or the project (titles, names, file names, the row + * ids core-ts messages quote) goes through `displayText`, so it cannot carry terminal escapes, + * bidi overrides or megabytes into the terminal. + */ +import { type ImportPlan, InvariantError, SyncConflictError } from "@cloudhal/core-ts"; +import { ArchiveError, type ArchiveErrorCode, type SiteArchiveManifest } from "../archive/index.ts"; +import type { ProjectConfig } from "../config.ts"; + +/** + * Controls (C0, C1, DEL), format characters (bidi overrides, zero-width), lone surrogates and the + * line and paragraph separators. Property escapes, so the pattern holds no control character. + */ +const UNSAFE = /[\p{Cc}\p{Cf}\p{Cs}\p{Zl}\p{Zp}]/gu; + +/** + * Untrusted text as one printable line: unsafe characters become spaces, runs of white space one + * space, and it is cut at `max` code points with `…`. + */ +export function displayText(raw: string, max = 200): string { + const clean = raw.replace(UNSAFE, " ").replace(/\s+/g, " ").trim(); + const chars = [...clean]; + return chars.length > max + ? `${chars + .slice(0, max - 1) + .join("") + .trimEnd()}…` + : clean; +} + +/** Where a site's media files are, as the messages name it. */ +export function mediaFolder(dataDir: string, siteId: string): string { + return `${displayText(dataDir.replace(/\/+$/, ""))}/blobs/media/${siteId}/`; +} + +/** What an interrupted or refused import says about the media it had already copied. */ +export function copiedNote(copied: number, mediaDir: string): string { + return `${copied} media file(s) already copied stay in ${mediaDir}, unreferenced, and the next import reuses them`; +} + +/** + * The archive's site must be this project's: `cloudhal.json`'s `siteId`, or listed in `sites`. + * Any other id is refused (switching `siteId` is `--replace`'s, RFC-0039 §2); without a local store + * the user may switch it by hand, knowing that push and deploy follow it. + */ +export function siteRefusal( + cfg: ProjectConfig, + manifest: SiteArchiveManifest, + storeExists: boolean, +): string | undefined { + const allowed = cfg.sites ?? [cfg.siteId]; + if (allowed.includes(manifest.siteId)) return undefined; + const ours = cfg.sites + ? `sites are ${allowed.map((id) => displayText(id, 80)).join(", ")} (cloudhal.json siteId and sites)` + : `site is ${displayText(cfg.siteId, 80)} (cloudhal.json siteId)`; + const a = manifest.siteId; + const hint = storeExists + ? "" + : `; this project has no local store yet, so setting cloudhal.json siteId to ${a} also works (push and deploy then use ${a})`; + return `it holds site ${a}, and this project's ${ours}; importing another site into a project comes with \`cloudhal import --replace\`, not available yet${hint}`; +} + +/** A bootstrap conflict, from the plan or from `SyncConflictError.differing` (never its message). */ +export function conflictText(siteId: string, differing: readonly string[]): string { + const shown = differing.slice(0, 5).map((id) => displayText(id, 80)); + return `this project's site ${displayText(siteId, 80)} differs from the archive in ${differing.length} node(s) (${shown.join(", ")}${differing.length > 5 ? ", …" : ""}); bootstrap imports only into a project without the site or holding exactly its nodes, and --replace is not available yet`; +} + +/** + * Why a bootstrap import of this plan is refused, in order: nodes that differ, media this project + * has taken down (invariant 5: an import never writes them back), and media a `--no-media` archive + * cannot bring that the project lacks or holds at another size. Empty: the import may proceed. + */ +export function bootstrapRefusals( + plan: ImportPlan, + manifest: SiteArchiveManifest, + dataDir: string, +): string[] { + const out: string[] = []; + if (plan.conflict) out.push(conflictText(plan.siteId, plan.conflict)); + const down = plan.media.takenDownHere; + if (down[0]) + out.push( + `the archive carries ${down.length} media file(s) this project has taken down (first ${down[0].sha256}); an import never brings taken-down content back`, + ); + const lacking = [...plan.media.missing, ...plan.media.wrongSize]; + if (manifest.mediaOmitted && lacking[0]) { + const total = plan.media.present.length + lacking.length; + const damaged = plan.media.wrongSize.length > 0 ? " or has a damaged copy of" : ""; + out.push( + `the archive has no media (written with --no-media) and this project lacks${damaged} ${lacking.length} of the ${total} media file(s) it needs (first ${lacking[0].sha256}); copy them into ${mediaFolder(dataDir, plan.siteId)} or import an archive with media`, + ); + } + return out; +} + +/** A refusal of this module's own (`siteRefusal`, `bootstrapRefusals`): its parts are cleaned. */ +export class ImportRefusal extends Error { + override readonly name = "ImportRefusal"; +} + +const DAMAGED = "; the file is damaged or incomplete, export it again"; + +/** What an archive error adds to its own (safe) message. */ +const ARCHIVE_TAILS: Partial> = { + damaged: DAMAGED, + mismatch: DAMAGED, + truncated: DAMAGED, + unsafe_entry: "; this file was not written by cloudhal export", + too_large: "; it is over this CLI's archive limits", +}; + +function reason(error: unknown, siteId: string | undefined): string { + if (error instanceof ArchiveError) return `${error.message}${ARCHIVE_TAILS[error.code] ?? ""}`; + if (error instanceof SyncConflictError) return conflictText(siteId ?? "", error.differing); + if (error instanceof Error) return error.message; + return String(error); +} + +/** Where a failure happened, for its message. */ +export interface ImportFailureContext { + /** The file as the user named it. */ + readonly file: string; + /** Media files written before the failure (pass 2). */ + readonly mediaCopied: number; + readonly mediaDir: string; + /** The archive's site, once its manifest was read. */ + readonly siteId?: string | undefined; + /** The graph was being written (`importSite`, and opening the store for it). */ + readonly applying?: boolean | undefined; +} + +/** + * Refusals that come before anything of the graph is written even once `importSite` runs: its + * plan's conflict and invariants (`prepareImport`), and this module's own. + */ +function refusedBeforeWriting(error: unknown): boolean { + return ( + error instanceof SyncConflictError || + error instanceof InvariantError || + error instanceof ImportRefusal + ); +} + +/** + * Any failure as one line: `cannot import : ; nothing was imported`, and what + * happened to the media already copied. A failure while the graph was being written (a storage + * error, or a store another process changed since the plan) may have left part of it, so it says + * so and to run the import again, which completes it. Every reason is cleaned and cut + * (`displayText`; 400 code points, or 1000 for this module's own refusals), since row ids, kinds + * and validator paths in core-ts messages come from the archive. + */ +export function importFailure(error: unknown, at: ImportFailureContext): Error { + const max = error instanceof ImportRefusal ? 1000 : 400; + const head = `cannot import ${displayText(at.file)}: ${displayText(reason(error, at.siteId), max)}`; + if (at.applying && !refusedBeforeWriting(error)) { + const reused = + at.mediaCopied > 0 ? ` (the ${at.mediaCopied} media file(s) copied are reused)` : ""; + return new Error( + `${head}; the import stopped after it began writing, so this project may hold part of it; run it again to finish${reused}`, + { cause: error }, + ); + } + const copied = at.mediaCopied > 0 ? `; ${copiedNote(at.mediaCopied, at.mediaDir)}` : ""; + return new Error(`${head}; nothing was imported${copied}`, { cause: error }); +} diff --git a/packages/cli/src/commands/import-describe.ts b/packages/cli/src/commands/import-describe.ts new file mode 100644 index 00000000..a72e2873 --- /dev/null +++ b/packages/cli/src/commands/import-describe.ts @@ -0,0 +1,284 @@ +/** + * What `cloudhal import` prints (RFC-0039 §2): the manifest, the difference a `--dry-run` finds + * (nodes added, changed and removed by kind and title; snapshots; media to copy) and the summary of + * an import. Pure. A title is read from its row's body, with the row id as the fallback, and goes + * through `displayText`; no body is ever printed. + */ +import type { NodeRef } from "@cloudhal/core-contracts"; +import { type ImportPlan, type NodeRow, termName } from "@cloudhal/core-ts"; +import { isRecord } from "./archive-bundle.ts"; +import { formatBytes } from "./export.ts"; +import type { ImportArchiveResult } from "./import.ts"; +import { displayText } from "./import-checks.ts"; + +type Kind = NodeRef["kind"]; + +/** How a node kind is named in the output. */ +export const KIND_LABEL: Readonly> = { + site: "site", + page: "page", + contentEntry: "entry", + taxonomy: "taxonomy", + term: "term", + redirect: "redirect", + menu: "menu", + glossary: "glossary", + asset: "media", +}; + +const PLURAL: Readonly> = { + site: "sites", + page: "pages", + contentEntry: "entries", + taxonomy: "taxonomies", + term: "terms", + redirect: "redirects", + menu: "menus", + glossary: "glossaries", + asset: "media", +}; + +/** Studio's order (apps/studio/src/lib/diff.ts): assets before the pages that show them. */ +const KIND_ORDER: readonly Kind[] = [ + "site", + "menu", + "taxonomy", + "term", + "asset", + "page", + "contentEntry", + "redirect", + "glossary", +]; + +/** Node lines per group before `… and N more`. */ +const MAX_LINES = 20; + +type Body = Record; + +const text = (value: unknown): string | undefined => + typeof value === "string" && value.trim() !== "" ? value : undefined; +const quoted = (value: string) => `"${displayText(value, 80)}"`; +const plain = (value: string) => displayText(value, 80); + +/** A per-locale map's value in `locale`. */ +function inLocale(map: unknown, locale: string | undefined): unknown { + return locale && isRecord(map) && Object.hasOwn(map, locale) ? map[locale] : undefined; +} + +/** A per-locale map's first value. */ +function firstValue(map: unknown): unknown { + return isRecord(map) ? Object.values(map)[0] : undefined; +} + +function withSuffix(title: unknown, suffix: string | undefined): string | undefined { + const t = text(title); + if (!t) return undefined; + return suffix ? `${quoted(t)} (${plain(suffix)})` : quoted(t); +} + +const TITLES: Readonly string | undefined>> = + { + site: (b) => withSuffix(b.name, undefined), + page: (b) => withSuffix(b.title, text(b.path)), + contentEntry: (b) => + withSuffix(b.title, text(b.collection) && text(b.slug) && `${b.collection}/${b.slug}`), + taxonomy: (b, locale) => { + const label = inLocale(b.labels, locale) ?? firstValue(b.labels); + const name = isRecord(label) ? text(label.plural) : undefined; + return name ? quoted(name) : text(b.key) && plain(b.key as string); + }, + term: (b, locale) => + isRecord(b.labels) + ? quoted(termName(b as never, locale)) + : text(b.slug) && plain(b.slug as string), + redirect: (b) => text(b.from) && text(b.to) && plain(`${b.from} → ${b.to}`), + menu: (b) => text(b.key) && plain(b.key as string), + glossary: () => "glossary", + asset: (b, locale) => { + const named = + text(inLocale(b.title, locale)) ?? + text(inLocale(b.alt, locale)) ?? + text(firstValue(b.title)) ?? + (isRecord(b.source) ? text(b.source.filename) : undefined); + return named ? quoted(named) : text(b.slug) && plain(b.slug as string); + }, + }; + +/** A node's human title from its body (`"About us" (/about)`), else its id. Never the body. */ +export function nodeTitle(row: NodeRow, locale: string | undefined): string { + const title = Object.hasOwn(TITLES, row.kind) ? TITLES[row.kind as Kind] : undefined; + try { + const body: unknown = JSON.parse(row.body); + const shown = title && isRecord(body) ? title(body, locale) : undefined; + if (shown) return shown; + } catch { + // the id below + } + return plain(row.id); +} + +function label(kind: string): string { + return Object.hasOwn(KIND_LABEL, kind) ? KIND_LABEL[kind as Kind] : plain(kind); +} + +function rank(kind: string): number { + const i = KIND_ORDER.indexOf(kind as Kind); + return i < 0 ? KIND_ORDER.length : i; +} + +function sorted(items: readonly T[], row: (item: T) => NodeRow): T[] { + return [...items].sort( + (a, b) => rank(row(a).kind) - rank(row(b).kind) || row(a).id.localeCompare(row(b).id), + ); +} + +/** `(2 pages, 1 entry)`, or nothing for none. */ +function byKind(rows: readonly NodeRow[]): string { + if (rows.length === 0) return ""; + const counts = new Map(); + for (const r of rows) counts.set(r.kind, (counts.get(r.kind) ?? 0) + 1); + const parts = [...counts] + .sort(([a], [b]) => rank(a) - rank(b) || a.localeCompare(b)) + .map(([kind, n]) => { + const many = Object.hasOwn(PLURAL, kind) ? PLURAL[kind as Kind] : plain(kind); + return `${n} ${n === 1 ? label(kind) : many}`; + }); + return ` (${parts.join(", ")})`; +} + +/** At most `MAX_LINES` lines, then `… and N more`. */ +function capped(lines: string[]): string[] { + if (lines.length <= MAX_LINES) return lines; + return [...lines.slice(0, MAX_LINES), ` … and ${lines.length - MAX_LINES} more`]; +} + +function nodeLines(plan: ImportPlan, locale: string | undefined): string[] { + const { added, changed, removed } = plan.nodes; + if (plan.unchanged) return ["nodes: unchanged"]; + const line = (sign: string, r: NodeRow) => ` ${sign} ${label(r.kind)} ${nodeTitle(r, locale)}`; + return [ + `nodes: ${added.length} to add${byKind(added)}, ${changed.length} to change${byKind(changed.map((c) => c.to))}, ${removed.length} to remove${byKind(removed)}`, + ...capped(sorted(added, (r) => r).map((r) => line("+", r))), + ...capped( + sorted(changed, (c) => c.to).map( + (c) => `${line("~", c.to)}, rev ${c.from.rev} here, ${c.to.rev} in the archive`, + ), + ), + ...capped(sorted(removed, (r) => r).map((r) => line("-", r))), + ]; +} + +function bytesOf(media: readonly { bytes: number }[]): number { + return media.reduce((n, m) => n + m.bytes, 0); +} + +function planMediaLine(result: ImportArchiveResult, plan: ImportPlan): string { + const { present, missing, wrongSize } = plan.media; + const total = present.length + missing.length + wrongSize.length; + if (total === 0) return "media: none"; + if (result.manifest.mediaOmitted) { + const lacking = [ + ...(missing.length > 0 ? [`${missing.length} missing`] : []), + ...(wrongSize.length > 0 ? [`${wrongSize.length} of another size`] : []), + ]; + return `media: none in the archive (--no-media); ${present.length} of ${total} already here${lacking.length > 0 ? `, ${lacking.join(", ")}` : ""}`; + } + const copy = [...missing, ...wrongSize]; + const damaged = wrongSize.length > 0 ? `, ${wrongSize.length} of them over a damaged copy` : ""; + return `media: ${copy.length} file(s) to copy (${formatBytes(bytesOf(copy))})${damaged}, ${present.length} already here`; +} + +function planLines(result: ImportArchiveResult, plan: ImportPlan): string[] { + const lines = [ + `dry run: bootstrap import into site ${result.site.id}, ${plan.fresh ? "which is new to this project" : "which this project already holds"}`, + ]; + if (plan.projectAdded) + lines.push(`project: ${plain(result.site.projectId ?? "its project")} is new to this project`); + lines.push(...nodeLines(plan, result.site.locale)); + const snaps = plan.snapshots; + lines.push( + snaps.length === 0 + ? `snapshots: none to add${plan.blobs > 0 ? `, ${plan.blobs} node bodies` : ""}` + : `snapshots: ${snaps.length} to add (${snaps.slice(0, 5).join(", ")}${snaps.length > 5 ? ", …" : ""}), with ${plan.blobs} node bodies`, + ); + lines.push(planMediaLine(result, plan)); + const { listed, bytesHere } = plan.takedowns; + if (listed > 0) + lines.push( + `taken down: ${listed} file(s) in the archive's ledger, never imported${bytesHere.length > 0 ? `; ${bytesHere.length} of them are in this project and would be deleted` : ""}`, + ); + return lines; +} + +function manifestLines(result: ImportArchiveResult): string[] { + const { manifest, site, mediaNeeded } = result; + const { counts } = manifest; + const slug = site.slug ? ` (${plain(site.slug)})` : ""; + const media = manifest.mediaOmitted + ? "no media (--no-media)" + : `${counts.media} media file(s) (${formatBytes(counts.mediaBytes)})`; + const lines = [ + `archive ${displayText(result.file)}: ${manifest.format} (contracts ${manifest.contracts.schemaVersion}), written by @cloudhal/cli ${manifest.createdBy} at ${manifest.createdAt}`, + `site ${manifest.siteId} ${quoted(manifest.siteName)}${slug}: ${counts.nodes} node(s), ${counts.snapshots} snapshot(s), ${counts.blobs} node bodies, ${media}`, + ]; + if (manifest.mediaOmitted && mediaNeeded.files > 0) + lines.push( + `warning: the archive was written with --no-media; its ${mediaNeeded.files} media file(s) (${formatBytes(mediaNeeded.bytes)}) must already be in this project`, + ); + return lines; +} + +function importedLines( + result: ImportArchiveResult, + plan: ImportPlan, + done: NonNullable, +): string[] { + const file = displayText(result.file); + const { id, slug } = result.site; + // Recording a takedown or deleting a local copy of taken-down content is a change too. + const nothing = + done.nodes.unchanged && + done.snapshots === 0 && + done.blobs === 0 && + done.media.copied === 0 && + !plan.projectAdded && + done.takedownsDeleted === 0 && + plan.takedowns.recorded.length === 0; + if (nothing) + return [ + `nothing to import: this project already holds ${file} (site ${id}${slug ? `, ${plain(slug)}` : ""})`, + ]; + const lines = [ + `imported ${file}: site ${id}${slug ? ` (${plain(slug)})` : ""} is in this project`, + `nodes: ${done.nodes.unchanged ? "unchanged" : `${done.nodes.inserted} written`}; snapshots: ${done.snapshots} added; node bodies: ${done.blobs} added`, + ]; + const media = [ + ...(result.mediaNeeded.files > 0 + ? [ + `${done.media.copied} file(s) copied (${formatBytes(done.media.bytes)}), ${done.media.present} already here`, + ] + : []), + ...(done.takedowns > 0 + ? [ + `${done.takedowns} taken down, never imported${done.takedownsDeleted > 0 ? `; ${done.takedownsDeleted} of them deleted from this project` : ""}`, + ] + : []), + ]; + if (media.length > 0) lines.push(`media: ${media.join("; ")}`); + if (!done.searchRefreshed) + lines.push("search index not refreshed; run `cloudhal search reindex`"); + return lines; +} + +/** The lines `cloudhal import` prints to stdout: the dry run's report, or the import's summary. */ +export function describeImport(result: ImportArchiveResult): string[] { + const { plan, imported } = result; + if (imported && plan) return importedLines(result, plan, imported); + return [ + ...manifestLines(result), + ...(plan ? planLines(result, plan) : []), + ...result.refusals.map((r) => `would refuse: ${r}`), + "nothing written (dry run)", + ]; +} diff --git a/packages/cli/src/commands/import-media.ts b/packages/cli/src/commands/import-media.ts new file mode 100644 index 00000000..0ff4fcb8 --- /dev/null +++ b/packages/cli/src/commands/import-media.ts @@ -0,0 +1,43 @@ +/** + * `cloudhal import`'s media writes (RFC-0039 §2, pass 2): one media file of the archive into the + * project's blob store, streamed when the store can, verified by the store against its SHA-256 + * (`FsBlobStore` refuses a stream of another size or content before the file is in place). + */ +import { type BundleMedia, mediaBlobKey } from "@cloudhal/core-ts"; +import type { BlobStore } from "@cloudhal/storage"; +import type { ArchiveMediaItem } from "./archive-reassemble.ts"; + +/** A media body as the stream `putStream` takes. */ +function webStream(body: AsyncIterable): ReadableStream { + const iterator = body[Symbol.asyncIterator](); + return new ReadableStream({ + async pull(controller) { + const next = await iterator.next(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + async cancel() { + await iterator.return?.(); + }, + }); +} + +/** Writes one media file unless the store has it at its size; the store verifies its SHA-256. */ +export async function writeMedia( + blobs: BlobStore, + siteId: string, + item: ArchiveMediaItem, + m: BundleMedia, +): Promise { + const key = mediaBlobKey(siteId, m.sha256); + if ((await blobs.head(key))?.size === m.bytes) return false; + const meta = { sha256: m.sha256, contentType: m.mediaType }; + if (blobs.putStream) { + await blobs.putStream(key, webStream(item.body), { ...meta, size: m.bytes }); + return true; + } + const chunks: Uint8Array[] = []; + for await (const chunk of item.body) chunks.push(chunk); + await blobs.put(key, Buffer.concat(chunks), meta); + return true; +} diff --git a/packages/cli/src/commands/import.ts b/packages/cli/src/commands/import.ts new file mode 100644 index 00000000..a6e58bd0 --- /dev/null +++ b/packages/cli/src/commands/import.ts @@ -0,0 +1,377 @@ +/** + * `cloudhal import [--dry-run]` (RFC-0039 §2, rollout step 3): a `.cloudhal` file written by + * `cloudhal export` into this project, in bootstrap mode, the site's id kept (invariant 1). + * + * - **Site:** the archive's must be `cloudhal.json`'s `siteId` or listed in `sites`, checked as soon + * as the manifest is read. Another id is refused, naming both: importing another site (and + * switching `siteId`) belongs to `--replace`, not available yet. + * - **Bootstrap:** the project must not hold the site yet, or hold exactly the archive's nodes; the + * snapshots, node bodies and media it lacks are added. A site whose nodes differ is refused. + * - **Two passes over the file** (invariants 3 and 6). The first reads and verifies the whole + * archive (every file's size and SHA-256, every row; media bodies are hashed, not kept) and plans + * the import against the local store (`planImport`), so every refusal comes before any write: a + * damaged or edited file, invalid rows or node bodies (precompiled validators, invariant 2), a + * conflict, media this project has taken down (invariant 5), media a `--no-media` archive lacks. + * The second runs only when media must be copied: it reads the file again, requires the same + * manifest (which pins every file's SHA-256, so the bundle is the one planned) and writes only + * the planned media into the blob store, which verifies them again. Then `importSite` applies the + * graph, keeping the archive's published snapshot as `cloudhal pull` does, and the search index + * is refreshed. A local media file of another size counts as damaged: a full archive replaces + * it, a `--no-media` one is refused. Local copies of content the archive's ledger took down are + * deleted, and the summary says so. + * - **No store yet:** planning runs on an empty store in memory, so a refusal or a dry run creates + * no `.cloudhal/`; media are copied before `data.db` exists. + * - **`--dry-run`:** prints the manifest and the plan and writes nothing: no blobs, rows, config or + * search index. Opening an existing store applies pending migrations, as every command does. + * - **Ctrl-C, SIGTERM** stop the import before `importSite` runs (between the passes or between + * media files); from there it finishes, its node set being one atomic batch. Media already + * copied stay in the blob store, unreferenced: content-addressed, the next import reuses them + * (`cloudhal media gc` only sees media with a ledger row). A hard kill can leave a temporary + * file in `blobs/.tmp`. + * + * Known limitations: the graph rows are held in memory (one bundle at a time, as for export); a + * disk error while `importSite` writes can leave node bodies, snapshots or the media ledger + * without the rest, as for `pull`: the message then says the import stopped partway, and running + * it again completes it (and refreshes a search index left behind the live snapshot). + */ +import { createReadStream } from "node:fs"; +import { stat } from "node:fs/promises"; +import path from "node:path"; +import { addAbortSignal } from "node:stream"; +import { isDeepStrictEqual } from "node:util"; +import { + type CopySiteResult, + type ImportPlan, + importSite, + planImport, + type SiteBundle, +} from "@cloudhal/core-ts"; +import { type LocalStore, openLocalStore } from "@cloudhal/core-ts/local"; +import { migrate, migrations } from "@cloudhal/storage"; +import { FsBlobStore } from "@cloudhal/storage/fs"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { type ArchiveItem, readArchive, type SiteArchiveManifest } from "../archive/index.ts"; +import { type ProjectConfig, readProjectConfig } from "../config.ts"; +import { isRecord } from "./archive-bundle.ts"; +import { type ArchiveMediaItem, bundleFromArchive } from "./archive-reassemble.ts"; +import { formatBytes } from "./export.ts"; +import { + bootstrapRefusals, + copiedNote, + displayText, + ImportRefusal, + importFailure, + mediaFolder, + siteRefusal, +} from "./import-checks.ts"; +import { writeMedia } from "./import-media.ts"; + +export interface ImportArchiveOptions { + readonly projectDir: string; + /** The `.cloudhal` file, relative to `projectDir`. */ + readonly file: string; + /** Verify and plan only; write nothing. */ + readonly dryRun?: boolean | undefined; + /** Stops the import before the graph is written (the command's Ctrl-C and SIGTERM). */ + readonly signal?: AbortSignal | undefined; + /** Progress lines (the command writes them to stderr). */ + readonly log?: ((line: string) => void) | undefined; +} + +export interface ImportArchiveResult { + /** The file as the user named it. */ + readonly file: string; + readonly manifest: SiteArchiveManifest; + readonly dryRun: boolean; + /** The archive's site, as its site node says (untrusted until planned: printed cleaned). */ + readonly site: { + readonly id: string; + readonly slug?: string | undefined; + readonly locale?: string | undefined; + readonly projectId?: string | undefined; + /** It is `cloudhal.json`'s `siteId` or listed in `sites`. */ + readonly listed: boolean; + }; + /** The project had no local store (`data.db`) before. */ + readonly newStore: boolean; + /** The media the archive's graph needs, whether or not it holds them. */ + readonly mediaNeeded: { readonly files: number; readonly bytes: number }; + /** Absent when the site was refused before planning. */ + readonly plan?: ImportPlan | undefined; + /** Why the import is refused (a dry run reports them; an import throws the first). */ + readonly refusals: readonly string[]; + readonly imported?: + | { + readonly nodes: CopySiteResult["nodes"]; + readonly snapshots: number; + readonly blobs: number; + readonly media: { + readonly copied: number; + readonly bytes: number; + readonly present: number; + }; + /** The archive's takedowns. */ + readonly takedowns: number; + /** Local media files deleted because the archive's ledger took their content down. */ + readonly takedownsDeleted: number; + readonly searchRefreshed: boolean; + } + | undefined; +} + +/** Ctrl-C or SIGTERM before the graph was written. */ +export class ImportInterruptedError extends Error { + override readonly name = "ImportInterruptedError"; + readonly mediaCopied: number; + readonly mediaDir: string; + constructor(mediaCopied: number, mediaDir: string) { + super( + `import interrupted; nothing was imported${mediaCopied > 0 ? `; ${copiedNote(mediaCopied, mediaDir)}` : ""}`, + ); + this.mediaCopied = mediaCopied; + this.mediaDir = mediaDir; + } +} + +/** What one import has done so far, for its messages and its cleanup. */ +interface State { + siteId?: string | undefined; + mediaDir: string; + copied: number; + copiedBytes: number; + /** `importSite` has started: the signal is no longer honoured. */ + applying: boolean; + local?: LocalStore | undefined; +} + +/** One import: the project, its paths and the options. */ +interface Run { + readonly options: ImportArchiveOptions; + readonly file: string; + readonly cfg: ProjectConfig; + readonly dataDir: string; + readonly storeExists: boolean; + readonly state: State; +} + +/** Pass 1's result; its bundle is dropped before pass 2 reads its own. */ +interface FirstPass { + readonly manifest: SiteArchiveManifest; + bundle: SiteBundle | undefined; +} + +async function statOrUndefined(file: string) { + return stat(file).catch((e: NodeJS.ErrnoException) => { + if (e.code === "ENOENT" || e.code === "ENOTDIR") return undefined; + throw e; + }); +} + +/** The archive's size; a missing file or a folder is refused before anything is read. */ +async function archiveSize(file: string): Promise { + const s = await statOrUndefined(file); + if (!s) throw new Error("no such file"); + if (s.isDirectory()) throw new Error("it is a directory; name a .cloudhal file"); + return s.size; +} + +/** The first item (the manifest) checked by `check` before the rest is read. */ +async function* checkManifest( + items: AsyncIterable, + check: (manifest: SiteArchiveManifest) => void, +): AsyncGenerator { + let first = true; + for await (const item of items) { + if (first && item.type === "manifest") check(item.manifest); + first = false; + yield item; + } +} + +/** One read of the whole file: verified, reassembled, its media handed to `onMedia`. */ +async function readPass( + run: Run, + check: (manifest: SiteArchiveManifest) => void, + onMedia?: (item: ArchiveMediaItem) => Promise, +): Promise<{ manifest: SiteArchiveManifest; bundle: SiteBundle }> { + const stream = createReadStream(run.file); + if (run.options.signal) addAbortSignal(run.options.signal, stream); + try { + const read = await bundleFromArchive( + checkManifest(readArchive(stream), check), + onMedia ? { onMedia } : {}, + ); + return { manifest: read.manifest, bundle: read.bundle }; + } finally { + stream.destroy(); + } +} + +/** What the result says about the archive's site and media, read from pass 1's bundle. */ +function siteSummary(run: Run, manifest: SiteArchiveManifest, bundle: SiteBundle) { + const row = bundle.nodes.find((n) => n.kind === "site" && n.id === manifest.siteId); + let site: unknown; + try { + site = row ? JSON.parse(row.body) : undefined; + } catch { + site = undefined; + } + const field = (key: string) => + isRecord(site) && typeof site[key] === "string" ? (site[key] as string) : undefined; + const media = bundle.media ?? []; + return { + site: { + id: manifest.siteId, + slug: field("slug"), + locale: field("defaultLocale"), + projectId: field("projectId"), + listed: (run.cfg.sites ?? [run.cfg.siteId]).includes(manifest.siteId), + }, + mediaNeeded: { files: media.length, bytes: media.reduce((n, m) => n + m.bytes, 0) }, + }; +} + +/** + * The plan against the local store, or, when the project has none yet, against an empty store in + * memory (with the project's blob folder, which may already hold the media), so nothing is created. + */ +async function planLocally(run: Run, bundle: SiteBundle): Promise { + const options = { mode: "bootstrap", report: true } as const; + if (run.storeExists) { + run.state.local = await openLocalStore(run.dataDir); + return planImport(run.state.local, bundle, options); + } + const sql = await BetterSqlite3Driver.open(":memory:"); + try { + await migrate(sql, migrations); + const blobs = new FsBlobStore(path.join(run.dataDir, "blobs")); + return await planImport({ sql, blobs }, bundle, options); + } finally { + await sql.close(); + } +} + +/** + * Pass 2, only when media must be copied: the file again, with the same manifest, its planned + * media written. Returns the bundle to import (pass 2's, or pass 1's when nothing was copied). + */ +async function copyMedia(run: Run, first: FirstPass, plan: ImportPlan): Promise { + const copy = new Map([...plan.media.missing, ...plan.media.wrongSize].map((m) => [m.sha256, m])); + if (copy.size === 0 && first.bundle) return first.bundle; + const total = [...copy.values()].reduce((n, m) => n + m.bytes, 0); + run.options.log?.(`copying ${copy.size} media file(s) (${formatBytes(total)})`); + first.bundle = undefined; + const { state } = run; + // Without a store yet, a bare blob store: an interrupted copy creates no data.db. + const blobs = state.local?.blobs ?? new FsBlobStore(path.join(run.dataDir, "blobs")); + const plain = (m: SiteArchiveManifest) => JSON.parse(JSON.stringify(m)) as unknown; + const same = (m: SiteArchiveManifest) => { + if (!isDeepStrictEqual(plain(m), plain(first.manifest))) + throw new Error("the file changed while it was being imported"); + }; + const again = await readPass(run, same, async (item) => { + const m = copy.get(item.sha256); + if (!m) return; + run.options.signal?.throwIfAborted(); + if (!(await writeMedia(blobs, plan.siteId, item, m))) return; + state.copied++; + state.copiedBytes += m.bytes; + }); + return again.bundle; +} + +/** + * The search index is behind the site's live snapshot (an import that stopped after the node set + * landed never refreshed it). + */ +async function searchBehind(local: LocalStore, siteId: string): Promise { + try { + const live = (await local.store.getSite(siteId))?.publishedSnapshotId; + return live !== undefined && (await local.store.searchStatus(siteId))?.snapshotId !== live; + } catch { + return true; + } +} + +/** `importSite` and the search index; the signal is not honoured from here. */ +async function apply(run: Run, bundle: SiteBundle, plan: ImportPlan) { + const { state } = run; + state.applying = true; + state.local ??= await openLocalStore(run.dataDir); + const local = state.local; + const result = await importSite(local, bundle, { mode: "bootstrap" }); + let searchRefreshed = true; + if (!result.nodes.unchanged || result.snapshots > 0 || (await searchBehind(local, plan.siteId))) + // RFC-0019 §3: an import leaves the index as it was; bring it to the live snapshot. + searchRefreshed = await local.store.refreshSearch(plan.siteId, { full: true }).then( + () => true, + () => false, + ); + return { + nodes: result.nodes, + snapshots: result.snapshots, + blobs: result.blobs, + media: { copied: state.copied, bytes: state.copiedBytes, present: plan.media.present.length }, + takedowns: plan.takedowns.listed, + takedownsDeleted: plan.takedowns.bytesHere.length, + searchRefreshed, + }; +} + +/** The project, the file and pass 1, then the plan; on to the writes unless refused or a dry run. */ +async function runImport(options: ImportArchiveOptions, state: State) { + const file = path.resolve(options.projectDir, options.file); + const size = await archiveSize(file); + const cfg = await readProjectConfig(options.projectDir); + options.signal?.throwIfAborted(); + const dataDir = path.resolve(options.projectDir, cfg.dataDir); + const storeExists = (await statOrUndefined(path.join(dataDir, "data.db")))?.isFile() === true; + const run: Run = { options, file, cfg, dataDir, storeExists, state }; + const rule: { refusal?: string | undefined } = {}; + const first: FirstPass = await readPass(run, (manifest) => { + state.siteId = manifest.siteId; + state.mediaDir = mediaFolder(cfg.dataDir, manifest.siteId); + rule.refusal = siteRefusal(cfg, manifest, storeExists); + if (rule.refusal && !options.dryRun) throw new ImportRefusal(rule.refusal); + }); + const { manifest } = first; + options.log?.( + `verified ${displayText(options.file)}: ${manifest.files.length + 1} file(s), ${formatBytes(size)}`, + ); + const base = { + file: options.file, + manifest, + dryRun: options.dryRun === true, + newStore: !storeExists, + ...siteSummary(run, manifest, first.bundle as SiteBundle), + }; + if (rule.refusal) return { ...base, refusals: [rule.refusal] }; + const plan = await planLocally(run, first.bundle as SiteBundle); + const refusals = bootstrapRefusals(plan, manifest, cfg.dataDir); + if (options.dryRun) return { ...base, plan, refusals }; + if (refusals[0] !== undefined) throw new ImportRefusal(refusals[0]); + options.signal?.throwIfAborted(); + const bundle = await copyMedia(run, first, plan); + options.signal?.throwIfAborted(); + return { ...base, plan, refusals: [], imported: await apply(run, bundle, plan) }; +} + +/** Imports (or, with `dryRun`, plans) the archive; see the module doc for every rule. */ +export async function importArchive(options: ImportArchiveOptions): Promise { + const state: State = { mediaDir: "", copied: 0, copiedBytes: 0, applying: false }; + try { + return await runImport(options, state); + } catch (e) { + if (options.signal?.aborted && !state.applying) + throw new ImportInterruptedError(state.copied, state.mediaDir); + throw importFailure(e, { + file: options.file, + mediaCopied: state.copied, + mediaDir: state.mediaDir, + siteId: state.siteId, + applying: state.applying, + }); + } finally { + await state.local?.close(); + } +} diff --git a/packages/cli/test/archive-bundle.test.ts b/packages/cli/test/archive-bundle.test.ts new file mode 100644 index 00000000..262c48fd --- /dev/null +++ b/packages/cli/test/archive-bundle.test.ts @@ -0,0 +1,511 @@ +/** + * RFC-0039 rollout step 2, the pure half: a `SiteBundle` to the container's content and back + * (`archive-bundle.ts`, `archive-reassemble.ts`) on hand-built bundles, `exportSite`'s media rule, + * `--latest-only`, the archive's date and file name, the local media reader, and every refusal of + * the reassembly. The store-backed command is in export-cli.test.ts. + */ +import type { BundleMedia, BundlePurge, NodeRow, SiteBundle, SnapshotRow } from "@cloudhal/core-ts"; +import { MemoryBlobStore } from "@cloudhal/storage"; +import { describe, expect, it } from "vitest"; +import { + type ArchiveContent, + type ArchiveMediaSource, + archiveStream, + encodeNdjson, + readArchive, + splitNdjson, +} from "../src/archive/index.ts"; +import { + bundleMedia, + bundleToArchive, + contentCreatedAt, + latestOnly, +} from "../src/commands/archive-bundle.ts"; +import { bundleFromArchive } from "../src/commands/archive-reassemble.ts"; +import { archiveFileName, localMediaReader } from "../src/commands/export.ts"; +import { chunked, collect, pattern, sha256, utf8 } from "./archive-fixture.ts"; + +const SITE_ID = "site_demo0001"; +const BYTES = { a: pattern(300, 1), b: pattern(500, 2), extra: pattern(100, 7) }; +const SHA = { + a: sha256(BYTES.a), + b: sha256(BYTES.b), + extra: sha256(BYTES.extra), + gone: sha256(utf8("taken down")), + ext: sha256(utf8("external")), +}; +const BY_SHA = new Map([ + [SHA.a, BYTES.a], + [SHA.b, BYTES.b], + [SHA.extra, BYTES.extra], +]); + +const at = (minute: number) => `2026-10-20T12:${String(minute).padStart(2, "0")}:00.000Z`; + +function asset( + id: string, + sha: string, + o: { bytes?: number; type?: string; external?: boolean } = {}, +) { + return JSON.stringify({ + id, + kind: "image", + sha256: sha, + bytes: o.bytes ?? BY_SHA.get(sha)?.length ?? 10, + mediaType: o.type ?? "image/png", + storage: o.external ? { kind: "external", url: "https://example.com/x.png" } : { kind: "blob" }, + }); +} + +function node(id: string, kind: string, body: string, minute = 1): NodeRow { + const stamp = at(minute); + return { + id, + site_id: SITE_ID, + kind, + key: id, + rev: 1, + hash: sha256(utf8(body)), + body, + created_at: stamp, + updated_at: stamp, + }; +} + +function snapshot(id: string, refs: { kind: string; text: string }[], minute: number): SnapshotRow { + const nodes = refs.map((r) => ({ kind: r.kind, hash: sha256(utf8(r.text)) })); + return { + id, + site_id: SITE_ID, + parent_id: null, + site_rev: 1, + body: JSON.stringify({ id, nodes }), + created_at: at(minute), + }; +} + +function blobsOf(...texts: string[]): Record { + const out: Record = {}; + for (const text of texts + .map((t) => [sha256(utf8(t)), t] as const) + .sort(([x], [y]) => (x < y ? -1 : 1))) + out[text[0]] = text[1]; + return out; +} + +const SITE_BODY = (published?: string) => + JSON.stringify({ + id: SITE_ID, + name: "Demo site", + slug: "demo", + ...(published ? { publishedSnapshotId: published } : {}), + }); + +/** + * A bundle as `exportSite` returns it: blob asset A current and in both snapshots, B only in the + * second, an external asset, a taken-down blob asset, a project and a ledger row. + */ +function sampleBundle(o: { aBytes?: number } = {}): SiteBundle { + const aBody = asset("asset_aaaaaa01", SHA.a, o.aBytes === undefined ? {} : { bytes: o.aBytes }); + const bBody = asset("asset_bbbbbb01", SHA.b, { type: "image/webp" }); + const site = SITE_BODY("snap_00000001"); + const nodes = [ + node("asset_aaaaaa01", "asset", aBody), + node("asset_eeeeee01", "asset", asset("asset_eeeeee01", SHA.ext, { external: true })), + node("asset_gggggg01", "asset", asset("asset_gggggg01", SHA.gone)), + node(SITE_ID, "site", site, 3), + ]; + const snapshots = [ + snapshot( + "snap_00000001", + [ + { kind: "asset", text: aBody }, + { kind: "site", text: site }, + ], + 4, + ), + snapshot( + "snap_00000002", + [ + { kind: "asset", text: aBody }, + { kind: "asset", text: bBody }, + ], + 5, + ), + ]; + const blobs = blobsOf(aBody, bBody, site); + const purged = [ + { sha256: SHA.gone, reason: "notice", legal: false, by: "human:user_x", at: at(6) }, + ]; + const media = bundleMedia({ nodes, snapshots, blobs }, new Set([SHA.gone])); + return { + format: "cloudhal.site-bundle@1", + siteId: SITE_ID, + project: { id: "prj_demo0001", rev: 2, body: "{}", created_at: at(0), updated_at: at(2) }, + nodes, + snapshots, + blobs, + ...(media ? { media } : {}), + purged, + }; +} + +const openMedia = (m: BundleMedia) => chunked(BY_SHA.get(m.sha256) as Uint8Array, 64); + +async function roundTrip( + content: ArchiveContent, + onMedia?: (sha: string) => void, +): Promise { + const bytes = await collect(archiveStream(content).bytes); + const result = await bundleFromArchive(readArchive(chunked(bytes, 1000)), { + onMedia: async (item) => { + onMedia?.(item.sha256); + await collect(item.body); + }, + }); + return result.bundle; +} + +describe("bundle → archive → bundle", () => { + it("comes back exactly, over several parts, with media streamed", async () => { + const bundle = sampleBundle(); + expect(bundle.media?.map((m) => m.sha256)).toEqual([SHA.a, SHA.b].sort()); + const content = bundleToArchive(bundle, { createdBy: "0.6.1", partBytes: 200, openMedia }); + expect(content.nodes.length).toBeGreaterThan(1); + expect(content.createdAt).toBe(at(6)); + expect(content.siteName).toBe("Demo site"); + const seen: string[] = []; + expect(await roundTrip(content, (sha) => seen.push(sha))).toStrictEqual(bundle); + expect(seen).toEqual([SHA.a, SHA.b].sort()); + }); + + it("leaves out project, media and purged when the bundle has none", async () => { + const site = SITE_BODY(); + const bundle: SiteBundle = { + format: "cloudhal.site-bundle@1", + siteId: SITE_ID, + nodes: [ + node("asset_eeeeee01", "asset", asset("asset_eeeeee01", SHA.ext, { external: true })), + node(SITE_ID, "site", site), + ], + snapshots: [snapshot("snap_00000001", [{ kind: "site", text: site }], 2)], + blobs: blobsOf(site), + }; + const content = bundleToArchive(bundle, { createdBy: "0.6.1", openMedia }); + const { manifest } = archiveStream(content); + expect(manifest.files.map((f) => f.path)).toEqual([ + "nodes/0001.ndjson", + "snapshots/0001.ndjson", + "blobs/0001.ndjson", + ]); + const back = await roundTrip(content); + expect(back).toStrictEqual(bundle); + expect(Object.keys(back)).toEqual(["format", "siteId", "nodes", "snapshots", "blobs"]); + }); + + it("--no-media: no media files, and the media rebuilt from the rows", async () => { + const bundle = sampleBundle(); + const content = bundleToArchive(bundle, { createdBy: "0.6.1", mediaOmitted: true, openMedia }); + expect(content.media).toBeUndefined(); + expect(archiveStream(content).manifest.mediaOmitted).toBe(true); + expect(await roundTrip(content)).toStrictEqual(bundle); + }); + + it("refuses to write a snapshot whose node body the bundle lacks", () => { + const bundle = sampleBundle(); + const { [Object.keys(bundle.blobs)[0] as string]: _dropped, ...blobs } = bundle.blobs; + expect(() => bundleToArchive({ ...bundle, blobs }, { createdBy: "0.6.1", openMedia })).toThrow( + /not in the bundle/, + ); + }); +}); + +describe("bundleMedia (exportSite's rule)", () => { + it("lets the last body seen win, skips external and taken-down content, sorts by hash", () => { + const current = asset("asset_aaaaaa01", SHA.a, { type: "image/png" }); + const later = asset("asset_aaaaaa01", SHA.a, { type: "image/webp" }); + const rows = { + nodes: [ + node("asset_aaaaaa01", "asset", current), + node("asset_bbbbbb01", "asset", asset("asset_bbbbbb01", SHA.b)), + node("asset_eeeeee01", "asset", asset("asset_eeeeee01", SHA.ext, { external: true })), + node("asset_gggggg01", "asset", asset("asset_gggggg01", SHA.gone)), + ], + snapshots: [snapshot("snap_00000001", [{ kind: "asset", text: later }], 2)], + blobs: blobsOf(later), + }; + expect(bundleMedia(rows, new Set([SHA.gone]))).toEqual( + [ + { sha256: SHA.a, bytes: 300, mediaType: "image/webp" }, + { sha256: SHA.b, bytes: 500, mediaType: "image/png" }, + ].sort((x, y) => (x.sha256 < y.sha256 ? -1 : 1)), + ); + expect(bundleMedia({ nodes: [], snapshots: [], blobs: {} }, new Set())).toBeUndefined(); + }); +}); + +describe("latestOnly", () => { + function threeSnapshots(published: string) { + const site = SITE_BODY(published); + const a = asset("asset_aaaaaa01", SHA.a); + const b = asset("asset_bbbbbb01", SHA.b); + const s2 = { ...snapshot("snap_00000002", [{ kind: "asset", text: b }], 5) }; + const s3 = { + ...snapshot("snap_00000003", [{ kind: "site", text: site }], 6), + parent_id: s2.id, + }; + const bundle: SiteBundle = { + format: "cloudhal.site-bundle@1", + siteId: SITE_ID, + nodes: [node(SITE_ID, "site", site)], + snapshots: [snapshot("snap_00000001", [{ kind: "asset", text: a }], 4), s2, s3], + blobs: blobsOf(a, b, site), + media: [], + purged: [{ sha256: SHA.gone, reason: "", legal: true, by: "", at: "" }], + }; + return { bundle, a, b, site }; + } + + it("keeps the published and the latest snapshot, their blobs and media; purged stays", () => { + const { bundle, a, site } = threeSnapshots("snap_00000001"); + const kept = latestOnly(bundle, "snap_00000003"); + expect(kept.snapshots.map((s) => s.id)).toEqual(["snap_00000001", "snap_00000003"]); + expect(kept.snapshots[1]?.parent_id).toBe("snap_00000002"); + expect(kept.blobs).toEqual(blobsOf(a, site)); + expect(kept.media).toEqual([{ sha256: SHA.a, bytes: 300, mediaType: "image/png" }]); + expect(kept.purged).toEqual(bundle.purged); + expect(kept.nodes).toBe(bundle.nodes); + }); + + it("refuses a site whose field operation is unfinished (its before snapshot would go)", () => { + const { bundle } = threeSnapshots("snap_00000003"); + const marker = { id: "snap_00000002", fromKey: "speaker" }; + const field = { key: "presenter", type: "text", localized: false, migration: marker }; + const site = JSON.stringify({ + ...JSON.parse(SITE_BODY("snap_00000003")), + collections: { posts: { fields: [] }, events: { fields: [field] } }, + }); + expect(() => latestOnly({ ...bundle, nodes: [node(SITE_ID, "site", site)] })).toThrow( + "field operation snap_00000002 on events has not finished, and --latest-only would drop the snapshot it needs", + ); + }); + + it("keeps one when published is latest, and falls back to the last row without an id", () => { + const { bundle, site } = threeSnapshots("snap_00000003"); + const one = latestOnly(bundle); + expect(one.snapshots.map((s) => s.id)).toEqual(["snap_00000003"]); + expect(one.blobs).toEqual(blobsOf(site)); + expect(one).not.toHaveProperty("media"); + }); +}); + +describe("contentCreatedAt", () => { + it("is the newest row timestamp, the ledger's included, never the clock", () => { + const bundle = sampleBundle(); + expect(contentCreatedAt(bundle)).toBe(at(6)); + const purge = bundle.purged?.[0] as BundlePurge; + expect(contentCreatedAt({ ...bundle, purged: [{ ...purge, at: "" }] })).toBe(at(5)); + const offset = { ...purge, at: "2026-10-21T01:30:00+02:00" }; + expect(contentCreatedAt({ ...bundle, purged: [offset] })).toBe("2026-10-20T23:30:00.000Z"); + }); + + it("refuses a bundle without a single timestamp", () => { + const bare = { + format: "cloudhal.site-bundle@1" as const, + siteId: SITE_ID, + nodes: [{ ...node(SITE_ID, "site", SITE_BODY()), created_at: "", updated_at: "" }], + snapshots: [], + blobs: {}, + }; + expect(() => contentCreatedAt(bare)).toThrow(/no row with a timestamp/); + }); +}); + +describe("archiveFileName", () => { + it("is -.cloudhal, falling back to the name's slug, then the id", () => { + const site = { id: SITE_ID, slug: "english", name: "English" }; + expect(archiveFileName(site, "2026-10-25T09:00:00.000Z")).toBe("english-20261025.cloudhal"); + expect(archiveFileName(site, "2026-10-25T01:30:00+02:00")).toBe("english-20261024.cloudhal"); + expect(archiveFileName({ ...site, slug: "Not A Slug", name: "Türkçe Blog" }, at(0))).toBe( + "turkce-blog-20261020.cloudhal", + ); + expect(archiveFileName({ ...site, slug: "", name: "日本" }, at(0))).toBe( + `${SITE_ID}-20261020.cloudhal`, + ); + }); +}); + +describe("localMediaReader", () => { + const key = (sha: string) => `media/${SITE_ID}/${sha}`; + const m = { sha256: SHA.a, bytes: BYTES.a.length, mediaType: "image/png" }; + + it("streams through getStream, falls back to get, and names a missing file", async () => { + const store = new MemoryBlobStore(); + await store.put(key(SHA.a), BYTES.a); + expect(Buffer.compare(await collect(localMediaReader(store, SITE_ID)(m)), BYTES.a)).toBe(0); + const plain = { + name: "plain", + get: (k: string) => store.get(k), + head: (k: string) => store.head(k), + put: (k: string, d: Uint8Array) => store.put(k, d), + delete: (k: string) => store.delete(k), + list: (p?: string) => store.list(p), + }; + expect(Buffer.compare(await collect(localMediaReader(plain, SITE_ID)(m)), BYTES.a)).toBe(0); + const absent = { ...m, sha256: SHA.b }; + await expect(collect(localMediaReader(store, SITE_ID)(absent))).rejects.toThrow( + /local media [0-9a-f]{64} is missing/, + ); + await expect(collect(localMediaReader(plain, SITE_ID)(absent))).rejects.toThrow(/is missing/); + }); +}); + +/** Raw rows of the sample bundle, for crafted archives the writer does not judge. */ +function rawContent(edit: { + nodes?: (rows: Record[]) => unknown[]; + blobs?: (rows: { hash: string; body: string }[]) => unknown[]; + ledger?: unknown[]; + media?: (sources: ArchiveMediaSource[]) => ArchiveMediaSource[]; + bundle?: SiteBundle; +}): ArchiveContent { + const bundle = edit.bundle ?? sampleBundle(); + const content = bundleToArchive(bundle, { createdBy: "0.6.1", openMedia }); + const nodes = bundle.nodes.map((n) => ({ ...n }) as Record); + const blobs = Object.entries(bundle.blobs).map(([hash, body]) => ({ hash, body })); + return { + ...content, + nodes: splitNdjson(edit.nodes ? edit.nodes(nodes) : nodes), + blobs: splitNdjson(edit.blobs ? edit.blobs(blobs) : blobs), + ...(edit.ledger ? { ledger: encodeNdjson(edit.ledger) } : {}), + media: edit.media ? edit.media([...(content.media ?? [])]) : content.media, + }; +} + +async function refusal(content: ArchiveContent, seen: string[] = []): Promise { + const error = await roundTrip(content, (sha) => seen.push(sha)).then( + () => undefined, + (e: unknown) => e, + ); + expect(error).toBeInstanceOf(Error); + return (error as Error).message; +} + +describe("bundleFromArchive refuses", () => { + it("rows that are not exactly the bundle's", async () => { + expect(await refusal(rawContent({ nodes: (r) => [...r, r[0]] }))).toMatch( + /^nodes\/0001\.ndjson line 5: a node id appears twice$/, + ); + expect( + await refusal(rawContent({ nodes: (r) => [{ ...r[0], extra: 1 }, ...r.slice(1)] })), + ).toMatch(/^nodes\/0001\.ndjson line 1: needs exactly the fields id, site_id, kind/); + expect( + await refusal(rawContent({ nodes: (r) => [{ ...r[0], rev: "1" }, ...r.slice(1)] })), + ).toBe("nodes/0001.ndjson line 1: rev must be a non-negative integer"); + expect( + await refusal( + rawContent({ nodes: (r) => [...r.slice(0, 3), { ...r[3], site_id: "site_other001" }] }), + ), + ).toBe("nodes/0001.ndjson line 4: site_id is not the archive's site"); + expect(await refusal(rawContent({ nodes: (r) => r.slice(0, 3) }))).toBe( + `nodes: the archive has no site node for ${SITE_ID}`, + ); + }); + + it("lines that are not JSON or not UTF-8", async () => { + expect(await refusal({ ...rawContent({}), nodes: [utf8('{"id":\n')] })).toBe( + "nodes/0001.ndjson line 1: not JSON", + ); + expect( + await refusal({ ...rawContent({}), nodes: [Uint8Array.from([0x7b, 0xff, 0x7d, 0x0a])] }), + ).toBe("nodes/0001.ndjson: not UTF-8"); + }); + + it("a snapshot ref whose node body is not in the archive", async () => { + const message = await refusal(rawContent({ blobs: (rows) => rows.slice(1) })); + expect(message).toMatch(/^snapshots\/0001\.ndjson line \d: references a node body/); + }); + + it("media no asset needs, before onMedia sees them", async () => { + const seen: string[] = []; + const extra = { + sha256: SHA.extra, + bytes: BYTES.extra.length, + open: () => chunked(BYTES.extra, 64), + }; + const message = await refusal(rawContent({ media: (m) => [...m, extra] }), seen); + expect(message).toBe(`media/${SHA.extra}: no asset of the archive needs this file`); + expect(seen).not.toContain(SHA.extra); + }); + + it("media the ledger took down (invariant 5), missing media and a size the asset disagrees with", async () => { + const ledger = [ + { sha256: SHA.a, reason: "notice", legal: true, by: "human:user_x", at: at(7) }, + ]; + expect(await refusal(rawContent({ ledger }))).toBe( + `media/${SHA.a}: the archive's ledger has taken this content down`, + ); + expect(await refusal(rawContent({ media: (m) => m.filter((s) => s.sha256 !== SHA.b) }))).toBe( + `media/${SHA.b}: an asset needs it, and the archive does not hold it`, + ); + const bundle = sampleBundle({ aBytes: 301 }); + const sources = (m: ArchiveMediaSource[]) => + m.map((s) => (s.sha256 === SHA.a ? { ...s, bytes: BYTES.a.length } : s)); + expect(await refusal(rawContent({ bundle, media: sources }))).toBe( + `media/${SHA.a}: 300 bytes, its asset says 301`, + ); + }); + + it("asset bodies the media rule cannot trust, without quoting them", async () => { + const evil = "\u001b[31mEVIL"; + const blobAsset = (extra: Record) => + JSON.stringify({ ...JSON.parse(asset("asset_aaaaaa01", SHA.a)), ...extra }); + const withBody = (body: string) => + rawContent({ + nodes: (rows) => rows.map((n) => (n.id === "asset_aaaaaa01" ? { ...n, body } : n)), + }); + const line = "nodes/0001.ndjson line 1: the asset body"; + for (const [body, message] of [ + [evil, `${line} is not JSON`], + ["null", `${line} is not a JSON object`], + // The media ledger reads `storage.kind` of every asset body. + [blobAsset({ storage: undefined }), `${line} needs storage, an object with a kind`], + [blobAsset({ storage: { kind: 7 } }), `${line} needs storage, an object with a kind`], + [blobAsset({ sha256: evil }), `${line} needs a sha256 of 64 lowercase hex digits`], + [blobAsset({ bytes: evil }), `${line} needs bytes, a non-negative integer`], + [blobAsset({ bytes: -1 }), `${line} needs bytes, a non-negative integer`], + [blobAsset({ mediaType: 7 }), `${line} needs a mediaType string`], + ] as const) { + const got = await refusal(withBody(body)); + expect(got).toBe(message); + expect(got).not.toContain("\u001b"); + } + // The body a snapshot's asset ref points at. + const aHash = sha256(utf8(asset("asset_aaaaaa01", SHA.a))); + const got = await refusal( + rawContent({ + blobs: (rows) => rows.map((b) => (b.hash === aHash ? { ...b, body: evil } : b)), + }), + ); + expect(got).toBe("snapshots/0001.ndjson line 1: an asset body it references is not JSON"); + // An external asset's hash is not the media rule's business. + const external = asset("asset_eeeeee01", SHA.ext, { external: true }); + const odd = JSON.stringify({ ...JSON.parse(external), sha256: evil }); + const bundle = sampleBundle(); + const nodes = bundle.nodes.map((n) => (n.id === "asset_eeeeee01" ? { ...n, body: odd } : n)); + expect( + await roundTrip(bundleToArchive({ ...bundle, nodes }, { createdBy: "0.6.1", openMedia })), + ).toStrictEqual({ + ...bundle, + nodes, + }); + }); + + it("and never echoes untrusted ids", async () => { + const evil = "page_\u001b[31mred\u001b[0m"; + const message = await refusal( + rawContent({ nodes: (r) => [{ ...r[0], id: evil }, { ...r[0], id: evil }, ...r.slice(1)] }), + ); + expect(message).toBe("nodes/0001.ndjson line 2: a node id appears twice"); + expect(message).not.toContain("\u001b"); + }); +}); diff --git a/packages/cli/test/archive-fixture.ts b/packages/cli/test/archive-fixture.ts index b65c195a..86130127 100644 --- a/packages/cli/test/archive-fixture.ts +++ b/packages/cli/test/archive-fixture.ts @@ -224,6 +224,18 @@ export function archiveOf( return gzipSync(rawTar([head, ...entries], options)); } +/** The offset of the body of `path` in a tar stream. */ +export function bodyOffset(tar: Buffer, path: string): number { + for (let at = 0; at < tar.length; at += 512) + if ( + Buffer.from(tar.subarray(at, at + 100)) + .toString("latin1") + .split("\0")[0] === path + ) + return at + 512; + throw new Error(`no ${path}`); +} + /** A valid archive's tar, edited, compressed again. */ export function retar(archive: Uint8Array, edit: (tar: Buffer) => Buffer | undefined): Buffer { const tar = gunzipSync(archive); diff --git a/packages/cli/test/archive-hostile.test.ts b/packages/cli/test/archive-hostile.test.ts index fde6b52f..e2fec4b3 100644 --- a/packages/cli/test/archive-hostile.test.ts +++ b/packages/cli/test/archive-hostile.test.ts @@ -15,6 +15,7 @@ import { tarLength } from "../src/archive/tar.ts"; import { archiveOf, baseFiles, + bodyOffset, chunked, collect, expectArchiveError, @@ -42,18 +43,6 @@ afterAll(async () => { await rm(dir, { recursive: true, force: true }); }); -/** The offset of the body of `path` in a tar stream. */ -function bodyOffset(tar: Buffer, path: string): number { - for (let at = 0; at < tar.length; at += 512) - if ( - Buffer.from(tar.subarray(at, at + 100)) - .toString("latin1") - .split("\0")[0] === path - ) - return at + 512; - throw new Error(`no ${path}`); -} - /** Items read until the first error, which is returned with them. */ async function readUntilError(archive: Uint8Array, drainMedia = true) { const seen: ArchiveItem[] = []; diff --git a/packages/cli/test/archive-roundtrip.test.ts b/packages/cli/test/archive-roundtrip.test.ts index 6dfa7c26..ffe07857 100644 --- a/packages/cli/test/archive-roundtrip.test.ts +++ b/packages/cli/test/archive-roundtrip.test.ts @@ -283,6 +283,27 @@ describe("media verified while writing", () => { await expectArchiveError(readAll(cut), "truncated"); }); + it("stops on an abort, removing its temporary file and leaving an existing file as it was", async () => { + await writeFile(target(), "old"); + const m = MEDIA[0] as Uint8Array; + const controller = new AbortController(); + async function* interrupted() { + yield m.subarray(0, 100); + controller.abort(); + await new Promise((resolve) => setTimeout(resolve, 20)); + yield m.subarray(100); + } + const write = writeArchiveFile(target(), withMedia(interrupted), { signal: controller.signal }); + await expect(write).rejects.toMatchObject({ name: "AbortError" }); + expect(await readdir(dir)).toEqual(["site.cloudhal"]); + expect(await readFile(target(), "utf8")).toBe("old"); + await rm(target()); + // Aborted before it starts: no file is opened. + const early = writeArchiveFile(target(), sampleContent(), { signal: AbortSignal.abort() }); + await expect(early).rejects.toMatchObject({ name: "AbortError" }); + expect(await readdir(dir)).toEqual([]); + }); + it("writes a file that reads back from disk", async () => { const manifest = await writeArchiveFile(target(), sampleContent()); let read: unknown; diff --git a/packages/cli/test/export-cli.test.ts b/packages/cli/test/export-cli.test.ts new file mode 100644 index 00000000..e8d6507c --- /dev/null +++ b/packages/cli/test/export-cli.test.ts @@ -0,0 +1,524 @@ +/** + * `cloudhal export` (RFC-0039 §2, rollout step 2) on a real local project: what it writes reads + * back into exactly the bundle `exportSite` returns (§1, invariant 1), the same content gives the + * same bytes (invariant 7), the file is dated by its rows (seeded on a fixed day, not today), + * `--no-media`, `--latest-only` and `--site`, a file at the default name is replaced only by the + * same archive, a damaged local media file or Ctrl-C fails the export and leaves no file + * (invariant 3), taken-down content never travels as bytes (invariant 5), nothing from §4's list + * ends up in the file or the summary, and `importSite` refuses a crafted project row (step 3's + * path). + */ +import { createReadStream } from "node:fs"; +import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { gunzipSync } from "node:zlib"; +import type { ContentEntry, Site } from "@cloudhal/core-contracts"; +import { + continueFieldOperation, + importSite, + runFieldOperation, + type SiteBundle, +} from "@cloudhal/core-ts"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { ArchiveError, archiveStream, readArchive } from "../src/archive/index.ts"; +import { HELP } from "../src/cli/help.ts"; +import { main } from "../src/cli.ts"; +import { bundleToArchive, latestOnly, snapshotRefs } from "../src/commands/archive-bundle.ts"; +import { bundleFromArchive } from "../src/commands/archive-reassemble.ts"; +import { + describeExport, + exportArchive, + NAMED_IN_ARCHIVE, + NEVER_IN_ARCHIVE, +} from "../src/commands/export.ts"; +import { writeProjectConfig } from "../src/config.ts"; +import { + bundleOf, + copyProject, + type Fixture, + mediaFile, + person, + removeTempDirs, + SEEDED_DAY, + seedFixture, + tempDir, + withStore, +} from "./site-fixture.ts"; + +let fx: Fixture; +beforeAll(async () => { + fx = await seedFixture("cloudhal-export-"); +}); +afterAll(removeTempDirs); +afterEach(() => vi.restoreAllMocks()); + +const freshProject = (): Promise => copyProject(fx); + +const latestId = (projectDir: string, siteId: string): Promise => + withStore( + projectDir, + async (local) => ((await local.store.latestSnapshot(siteId)) as { id: string }).id, + ); + +const partials = async (dir: string) => (await readdir(dir)).filter((f) => f.includes(".partial-")); + +const readBack = (file: string) => bundleFromArchive(readArchive(createReadStream(file))); +const day = (createdAt: string) => createdAt.slice(0, 10).replaceAll("-", ""); + +/** The newest timestamp of the bundle's rows, computed apart from `contentCreatedAt`. */ +function newest(bundle: SiteBundle): string { + const stamps = [ + ...bundle.nodes.flatMap((n) => [n.created_at, n.updated_at]), + ...bundle.snapshots.map((s) => s.created_at), + ...(bundle.project ? [bundle.project.created_at, bundle.project.updated_at] : []), + ...(bundle.purged ?? []).map((p) => p.at), + ]; + return new Date( + Math.max(...stamps.map((s) => Date.parse(s)).filter((t) => !Number.isNaN(t))), + ).toISOString(); +} + +describe("cloudhal export", () => { + it("writes -.cloudhal that reads back into exactly exportSite's bundle", async () => { + const result = await exportArchive({ projectDir: fx.projectDir }); + const { manifest } = result; + // Dated by the rows (seeded on SEEDED_AT), not by today. + expect(result.file).toBe(path.join(fx.projectDir, `english-${SEEDED_DAY}.cloudhal`)); + expect(day(manifest.createdAt)).toBe(SEEDED_DAY); + expect(result.replaced).toBe(false); + expect((await stat(result.file)).size).toBe(result.bytes); + const pkg = JSON.parse( + await readFile(path.join(import.meta.dirname, "../package.json"), "utf8"), + ) as { + version: string; + }; + expect(manifest).toMatchObject({ + siteId: fx.siteId, + siteName: "English", + createdBy: pkg.version, + }); + const media = [fx.sha.a, fx.sha.c].sort(); + expect(manifest.files.map((f) => f.path)).toEqual([ + "project.json", + "nodes/0001.ndjson", + "snapshots/0001.ndjson", + "blobs/0001.ndjson", + ...media.map((sha) => `media/${sha}`), + "ledger.ndjson", + ]); + const back = await readBack(result.file); + const bundle = await bundleOf(fx.projectDir, fx.siteId); + expect(back.bundle).toStrictEqual(bundle); + expect(back.bundle.media?.map((m) => m.sha256)).toEqual(media); + expect(back.bundle.purged?.map((p) => p.sha256)).toEqual([fx.sha.b]); + expect(back.media.map((m) => m.sha256)).not.toContain(fx.sha.b); + expect(manifest.createdAt).toBe(newest(bundle)); + }); + + it("gives the same bytes for the same content, and replaces the default file", async () => { + const first = await exportArchive({ projectDir: fx.projectDir, out: "one.cloudhal" }); + await new Promise((resolve) => setTimeout(resolve, 20)); + const second = await exportArchive({ projectDir: fx.projectDir, out: "two.cloudhal" }); + const bytes = await readFile(first.file); + expect(Buffer.compare(bytes, await readFile(second.file))).toBe(0); + await exportArchive({ projectDir: fx.projectDir }); + const again = await exportArchive({ projectDir: fx.projectDir }); + expect(again.replaced).toBe(true); + expect(Buffer.compare(bytes, await readFile(again.file))).toBe(0); + expect(describeExport(again, fx.projectDir)[0]).toMatch(/; replaced the existing file\)$/); + }); + + it("--no-media writes the graph only; the media list is rebuilt from it", async () => { + const result = await exportArchive({ + projectDir: fx.projectDir, + out: "graph.cloudhal", + noMedia: true, + }); + expect(result.manifest.mediaOmitted).toBe(true); + expect(result.manifest.counts.media).toBe(0); + expect(result.manifest.files.some((f) => f.path.startsWith("media/"))).toBe(false); + const back = await readBack(result.file); + expect(back.media).toEqual([]); + expect(back.bundle).toStrictEqual(await bundleOf(fx.projectDir, fx.siteId)); + }); + + it("--latest-only keeps the published and the latest snapshot and what they need", async () => { + const result = await exportArchive({ + projectDir: fx.projectDir, + out: "latest.cloudhal", + latestOnly: true, + }); + const { bundle } = await readBack(result.file); + const latest = await latestId(fx.projectDir, fx.siteId); + expect(latest).toBe(fx.snapshots.s3); + expect(new Set(bundle.snapshots.map((s) => s.id))).toEqual( + new Set([fx.snapshots.s2, fx.snapshots.s3]), + ); + const refs = bundle.snapshots.flatMap((s) => (snapshotRefs(s.body) ?? []).map((r) => r.hash)); + expect(Object.keys(bundle.blobs).sort()).toEqual([...new Set(refs)].sort()); + expect(bundle.media?.map((m) => m.sha256)).toEqual([fx.sha.a]); + expect(bundle.purged?.map((p) => p.sha256)).toEqual([fx.sha.b]); + expect(bundle).toStrictEqual(latestOnly(await bundleOf(fx.projectDir, fx.siteId), latest)); + expect(describeExport(result, fx.projectDir)).toContain( + "history: the published and the latest snapshot only (--latest-only)", + ); + }); + + it("--site exports another listed site and refuses any other id", async () => { + const result = await exportArchive({ projectDir: fx.projectDir, siteId: fx.trId }); + expect(result.manifest.siteId).toBe(fx.trId); + expect(path.basename(result.file)).toBe(`turkce-${SEEDED_DAY}.cloudhal`); + expect((await readBack(result.file)).bundle).toStrictEqual( + await bundleOf(fx.projectDir, fx.trId), + ); + const before = await readdir(fx.projectDir); + await expect( + exportArchive({ projectDir: fx.projectDir, siteId: "site_notlisted01" }), + ).rejects.toThrow(/site site_notlisted01 is not listed in cloudhal\.json `sites`/); + expect(await readdir(fx.projectDir)).toEqual(before); + + const projectDir = await freshProject(); + await writeProjectConfig(projectDir, { + siteId: fx.siteId, + dataDir: ".cloudhal", + sites: [fx.siteId, fx.trId, "site_ghost00001"], + }); + await expect(exportArchive({ projectDir, siteId: "site_ghost00001" })).rejects.toThrow( + /the local store \(\.cloudhal\) has no such site/, + ); + }); +}); + +describe("cloudhal export and a file already at the default name", () => { + const name = `english-${SEEDED_DAY}.cloudhal`; + const refused = new RegExp( + `^${name.replaceAll(".", "\\.")} already exists and is not this archive .*; pass --out ${name.replaceAll(".", "\\.")} to replace it`, + ); + + it("is replaced only by the same archive: not by another flag, a deletion or another file", async () => { + const projectDir = await freshProject(); + const first = await exportArchive({ projectDir }); + expect(path.basename(first.file)).toBe(name); + const bytes = await readFile(first.file); + const unchanged = async () => { + expect(Buffer.compare(await readFile(first.file), bytes)).toBe(0); + expect(await partials(projectDir)).toEqual([]); + }; + // Same content, same day, other flags: the full backup stays. + for (const flags of [{ latestOnly: true }, { noMedia: true }]) { + await expect(exportArchive({ projectDir, ...flags })).rejects.toThrow(refused); + await unchanged(); + } + // A deletion stamps no remaining row, so the date stays and the name is the same. + await withStore(projectDir, async ({ store }) => { + const [entry] = await store.listContentEntries(fx.siteId); + await store.deleteContentEntry(entry as ContentEntry); + }); + await expect(exportArchive({ projectDir })).rejects.toThrow(refused); + await unchanged(); + // A file that is no archive at all stays too. + const other = path.join(projectDir, `turkce-${SEEDED_DAY}.cloudhal`); + await writeFile(other, "my notes"); + await expect(exportArchive({ projectDir, siteId: fx.trId })).rejects.toThrow( + /^turkce-\d{8}\.cloudhal already exists and is not this archive/, + ); + expect(await readFile(other, "utf8")).toBe("my notes"); + // --out naming it replaces it. + const replaced = await exportArchive({ projectDir, out: name }); + expect(replaced).toMatchObject({ file: first.file, replaced: true }); + expect((await readBack(first.file)).bundle).toStrictEqual( + await bundleOf(projectDir, fx.siteId), + ); + }); +}); + +describe("cloudhal export --latest-only and the store's history", () => { + it("keeps the snapshot latestSnapshot picks when two share a created_at", async () => { + const projectDir = await freshProject(); + // The newest snapshot (by insertion) ties with an earlier one whose id sorts after it, so the + // bundle's last row (created_at, id) is not the store's latest (created_at, rowid). The + // earlier one is the largest such id: other rows may already share newest's created_at (the + // seed's clock moves in 20 ms steps), and it must still sort last among them. + const tie = await withStore(projectDir, async ({ store, sql }) => { + for (let i = 0; i < 40; i++) { + const rows = await sql.all<{ id: string; created_at: string }>( + "SELECT id, created_at FROM snapshots WHERE site_id = ? ORDER BY rowid", + [fx.siteId], + ); + const newest = rows.at(-1) as { id: string; created_at: string }; + const earlier = rows + .slice(0, -1) + .filter((r) => r.id > newest.id) + // Code-unit order, as SQLite's BINARY collation sorts ids. + .reduce<{ id: string } | undefined>( + (max, r) => (max && max.id > r.id ? max : r), + undefined, + ); + if (earlier) { + await sql.run("UPDATE snapshots SET created_at = ? WHERE id = ?", [ + newest.created_at, + earlier.id, + ]); + return { newest: newest.id, earlier: earlier.id }; + } + await store.createSnapshot(fx.siteId, { actor: person, reason: "manual" }); + } + throw new Error("no two snapshots to tie"); + }); + expect(await latestId(projectDir, fx.siteId)).toBe(tie.newest); + expect((await bundleOf(projectDir, fx.siteId)).snapshots.at(-1)?.id).toBe(tie.earlier); + const result = await exportArchive({ projectDir, out: "tie.cloudhal", latestOnly: true }); + const { bundle } = await readBack(result.file); + expect(new Set(bundle.snapshots.map((s) => s.id))).toEqual( + new Set([fx.snapshots.s2, tie.newest]), + ); + }); + + it("is refused while a field operation is unfinished, and works once it is done", async () => { + const projectDir = await freshProject(); + const op = { + op: "rename_field" as const, + collection: "events", + from: "speaker", + to: "presenter", + }; + const operationId = await withStore(projectDir, async ({ store }) => { + const site = (await store.getSite(fx.siteId)) as Site; + const speaker = { key: "speaker", type: "text", labels: { en: { label: "Speaker" } } }; + await store.updateSite({ + ...site, + collections: { + ...site.collections, + events: { fields: [{ ...speaker, localized: false }] }, + } as never, + }); + for (const slug of ["one", "two", "three"]) + await store.createContentEntry({ + siteId: fx.siteId, + collection: "events", + slug, + locale: "en", + status: "draft", + title: slug, + body: { format: "markdown", value: "Body" }, + fields: { speaker: slug }, + tags: [], + author: person, + } as never); + const part = await runFieldOperation(store, fx.siteId, op, { + actor: person, + budget: { entries: 1 }, + }); + expect(part.done).toBe(false); + // A later snapshot: the operation's `before` is neither the published nor the latest one. + await store.createSnapshot(fx.siteId, { actor: person, reason: "manual" }); + return part.operationId; + }); + const out = path.join(projectDir, "template.cloudhal"); + await expect( + exportArchive({ projectDir, out: "template.cloudhal", latestOnly: true }), + ).rejects.toThrow( + `field operation ${operationId} on events has not finished, and --latest-only would drop the snapshot it needs; finish or revert it first (\`cloudhal fields continue|revert events ${operationId}\`)`, + ); + await expect(stat(out)).rejects.toMatchObject({ code: "ENOENT" }); + // The full history carries it. + const full = await exportArchive({ projectDir, out: "full.cloudhal" }); + const back = await readBack(full.file); + expect(back.bundle.snapshots.map((s) => s.id)).toContain(operationId); + await withStore(projectDir, async ({ store }) => { + const done = await continueFieldOperation(store, fx.siteId, operationId, { actor: person }); + expect(done.done).toBe(true); + }); + const template = await exportArchive({ + projectDir, + out: "template.cloudhal", + latestOnly: true, + }); + expect(template.manifest.counts.snapshots).toBe(2); + }); +}); + +describe("importSite and a crafted project row (the step 3 path)", () => { + /** `bundle` written as an archive (graph only) and read back, as `cloudhal import` will. */ + async function throughArchive(bundle: SiteBundle): Promise { + const content = bundleToArchive(bundle, { + createdBy: "0.0.0", + mediaOmitted: true, + openMedia: () => { + throw new Error("no media in a --no-media archive"); + }, + }); + return (await bundleFromArchive(readArchive(archiveStream(content).bytes))).bundle; + } + + it("refuses a project that is not the site's, not valid, or whose columns disagree", async () => { + const bundle = await bundleOf(fx.projectDir, fx.siteId); + const project = bundle.project as NonNullable; + const body = JSON.parse(project.body) as Record; + const id = project.id; + const cases: [NonNullable, string][] = [ + [{ ...project, id: "prj_planted00001" }, "the bundle's project is not its site's project"], + [{ ...project, body: "{not json" }, `project ${id}: body is not JSON`], + [{ ...project, body: JSON.stringify({ id }) }, `project ${id}: not a valid project`], + [{ ...project, rev: project.rev + 1 }, `project ${id}: columns do not match its body`], + [ + { ...project, body: JSON.stringify({ ...body, id: "prj_other000001" }) }, + `project ${id}: columns do not match its body`, + ], + ]; + for (const [crafted, message] of cases) { + const back = await throughArchive({ ...bundle, project: crafted }); + expect(back.project).toStrictEqual(crafted); + const target = await tempDir("cloudhal-import-"); + await withStore(target, async (local) => { + await expect(importSite(local, back)).rejects.toThrow(message); + const count = (table: string) => + local.sql.get<{ n: number }>(`SELECT count(*) AS n FROM ${table}`); + expect(await count("projects")).toEqual({ n: 0 }); + expect(await count("nodes")).toEqual({ n: 0 }); + }); + } + }); +}); + +describe("cloudhal export and damaged local media (invariant 3)", () => { + async function expectNoPartial(projectDir: string): Promise { + expect((await readdir(projectDir)).filter((f) => f.includes(".partial-"))).toEqual([]); + } + + it("refuses a changed media file and leaves the existing archive as it was", async () => { + const projectDir = await freshProject(); + const out = path.join(projectDir, "keep.cloudhal"); + await writeFile(out, "an older archive"); + const file = mediaFile(projectDir, fx.siteId, fx.sha.a); + const bytes = await readFile(file); + bytes[bytes.length - 1] = (bytes[bytes.length - 1] as number) ^ 0xff; + await writeFile(file, bytes); + const error = await exportArchive({ projectDir, out: "keep.cloudhal" }).catch((e) => e); + expect(error).toBeInstanceOf(ArchiveError); + expect((error as ArchiveError).code).toBe("damaged"); + expect((error as ArchiveError).message).toMatch( + /the local copy is damaged: run `cloudhal media verify`/, + ); + expect(await readFile(out, "utf8")).toBe("an older archive"); + await expectNoPartial(projectDir); + + await writeFile(file, Buffer.concat([bytes, Buffer.from("more")])); + await expect(exportArchive({ projectDir, out: "keep.cloudhal" })).rejects.toMatchObject({ + code: "damaged", + }); + expect(await readFile(out, "utf8")).toBe("an older archive"); + await expectNoPartial(projectDir); + }); + + it("refuses a missing media file before writing anything", async () => { + const projectDir = await freshProject(); + await rm(mediaFile(projectDir, fx.siteId, fx.sha.c)); + await expect(exportArchive({ projectDir, out: "new.cloudhal" })).rejects.toThrow( + new RegExp(`local media ${fx.sha.c} is missing; run \`cloudhal media verify\``), + ); + await expect(stat(path.join(projectDir, "new.cloudhal"))).rejects.toMatchObject({ + code: "ENOENT", + }); + await expectNoPartial(projectDir); + // --no-media does not need the bytes. + expect( + (await exportArchive({ projectDir, out: "new.cloudhal", noMedia: true })).bytes, + ).toBeGreaterThan(0); + }); +}); + +describe("cloudhal export: what it says and never holds (§4)", () => { + it("names the file, the counts and what an archive never holds; no secrets anywhere", async () => { + const projectDir = await freshProject(); + const marker = "placeholder-marker-not-a-secret-4f1c"; + await mkdir(path.join(projectDir, ".cloudhal", "deploy"), { recursive: true }); + await writeFile( + path.join(projectDir, ".cloudhal", "deploy", "secrets.json"), + JSON.stringify({ CLOUDHAL_DEPLOY_TOKEN: marker }), + ); + const result = await exportArchive({ projectDir, out: "site.cloudhal" }); + const lines = describeExport(result, projectDir); + const { counts } = result.manifest; + expect(lines[0]).toMatch(/^wrote site\.cloudhal \(\d+(\.\d)? (B|KiB)\)$/); + expect(counts).toMatchObject({ snapshots: 3, media: 2 }); + expect(lines[1]).toMatch( + new RegExp( + `^site ${fx.siteId} \\(english\\): ${counts.nodes} node\\(s\\), 3 snapshot\\(s\\), ${counts.blobs} blob\\(s\\), 2 media file\\(s\\) \\(\\d+ B\\), 1 taken down$`, + ), + ); + const never = lines.find((l) => l.startsWith("not in the archive (RFC-0039 §4): ")); + for (const item of NEVER_IN_ARCHIVE) expect(never).toContain(item); + expect(never).toContain("user accounts"); + expect(lines.at(-1)).toBe(`in the archive: ${NAMED_IN_ARCHIVE}`); + const tar = Buffer.from(gunzipSync(await readFile(result.file))).toString("latin1"); + expect(tar).not.toContain(marker); + expect(lines.join("\n")).not.toContain(marker); + // What the last line says: the history names who made each change. + // (A snapshot row's body is JSON inside the NDJSON line, so its quotes are escaped.) + expect(tar.includes(`\\"name\\":\\"${person.name}\\"`)).toBe(true); + }); +}); + +describe("cloudhal export: the command", () => { + it("is in --help and refuses positionals", async () => { + expect(HELP).toMatch( + /^ {2}cloudhal export \[--out \] \[--site \] \[--no-media\] \[--latest-only\]$/m, + ); + await expect(main(["export", "x.cloudhal"])).rejects.toThrow( + "usage: cloudhal export [--out ] [--site ] [--no-media] [--latest-only]", + ); + }); + + it("runs from the project folder with its flags and prints the summary", async () => { + const projectDir = await freshProject(); + vi.spyOn(process, "cwd").mockReturnValue(projectDir); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const argv = ["export", "--no-media", "--latest-only", "--out", "sub/x.cloudhal"]; + expect(await main(argv)).toBe(0); + expect((await stat(path.join(projectDir, "sub", "x.cloudhal"))).isFile()).toBe(true); + const printed = log.mock.calls.map((c) => String(c[0])); + expect(printed[0]).toMatch(/^wrote sub\/x\.cloudhal \(/); + expect(printed).toContain( + "media left out (--no-media): an import needs the media from elsewhere", + ); + expect(printed).toContain( + "history: the published and the latest snapshot only (--latest-only)", + ); + await expect(exportArchive({ projectDir, out: "sub" })).rejects.toThrow( + "--out sub is a directory; name a file", + ); + }); + + it("refuses --site, --no-media and --latest-only on every other command", async () => { + await expect(main(["push", "--replace", "--site", "site_other00001"])).rejects.toThrow( + "cloudhal push does not take --site (only cloudhal export does); every other command acts on cloudhal.json's siteId", + ); + await expect(main(["pull", "--replace", "--no-media"])).rejects.toThrow( + "cloudhal pull does not take --no-media (only cloudhal export does)", + ); + await expect(main(["build", "--latest-only"])).rejects.toThrow( + "cloudhal build does not take --latest-only (only cloudhal export does)", + ); + }); + + it("stops on Ctrl-C: exit 130, no file and no partial file, the listener removed", async () => { + const projectDir = await freshProject(); + vi.spyOn(process, "cwd").mockReturnValue(projectDir); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const before = process.listeners("SIGINT"); + const beforeTerm = process.listeners("SIGTERM"); + const running = main(["export", "--out", "stopped.cloudhal"]); + const ours = process.listeners("SIGINT").filter((l) => !before.includes(l)); + expect(ours).toHaveLength(1); + (ours[0] as () => void)(); + expect(await running).toBe(130); + expect(error).toHaveBeenCalledWith(expect.stringMatching(/^cloudhal: export interrupted;/)); + await expect(stat(path.join(projectDir, "stopped.cloudhal"))).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(await partials(projectDir)).toEqual([]); + expect(process.listeners("SIGINT")).toEqual(before); + expect(process.listeners("SIGTERM")).toEqual(beforeTerm); + }); +}); diff --git a/packages/cli/test/import-cli.test.ts b/packages/cli/test/import-cli.test.ts new file mode 100644 index 00000000..712e3ca5 --- /dev/null +++ b/packages/cli/test/import-cli.test.ts @@ -0,0 +1,779 @@ +/** + * `cloudhal import` (RFC-0039 §2, rollout step 3) on real local projects, in bootstrap mode: an + * archive comes back as the same site (invariant 1) and a second import changes nothing; rows and + * node bodies are validated and printed clean (invariant 2); a damaged file, a file that changes + * between the passes, a `--no-media` archive without its media, a newer format, a conflict, + * another site's archive, content taken down here, and Ctrl-C are refused or stopped before + * anything is written (invariants 3, 4, 5, 6), and a storage failure after the first write says + * so; `--dry-run` shows the difference by kind and title and writes nothing. + */ +import { writeFileSync } from "node:fs"; +import { cp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { Page } from "@cloudhal/core-contracts"; +import { type NodeRow, type SiteBundle, SyncConflictError } from "@cloudhal/core-ts"; +import { makePng } from "@cloudhal/core-ts/media/fixtures"; +import { FsBlobStore } from "@cloudhal/storage/fs"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { writeArchiveFile } from "../src/archive/index.ts"; +import { HELP } from "../src/cli/help.ts"; +import { main } from "../src/cli.ts"; +import { bundleToArchive } from "../src/commands/archive-bundle.ts"; +import { exportArchive } from "../src/commands/export.ts"; +import { + type ImportArchiveResult, + ImportInterruptedError, + importArchive, +} from "../src/commands/import.ts"; +import { importFailure } from "../src/commands/import-checks.ts"; +import { describeImport } from "../src/commands/import-describe.ts"; +import { type ProjectConfig, writeProjectConfig } from "../src/config.ts"; +import { archiveOf, baseFiles, bodyOffset, flip, retar, sha256, utf8 } from "./archive-fixture.ts"; +import { + blobListing, + bundleOf, + copyProject, + type Fixture, + mediaFile, + person, + removeTempDirs, + seedFixture, + tempDir, + withStore, +} from "./site-fixture.ts"; + +let fx: Fixture; +let files: string; +const archives = { full: "", graph: "", tr: "" }; + +beforeAll(async () => { + fx = await seedFixture("cloudhal-import-src-"); + files = await tempDir("cloudhal-archives-"); + const out = (name: string) => path.join(files, name); + const { projectDir } = fx; + archives.full = (await exportArchive({ projectDir, out: out("english.cloudhal") })).file; + archives.graph = ( + await exportArchive({ projectDir, out: out("graph.cloudhal"), noMedia: true }) + ).file; + archives.tr = ( + await exportArchive({ projectDir, out: out("tr.cloudhal"), siteId: fx.trId }) + ).file; +}); +afterAll(removeTempDirs); +afterEach(() => vi.restoreAllMocks()); + +/** A project with only its cloudhal.json (no .cloudhal): the fixture's sites unless `cfg` says. */ +async function emptyProject(cfg?: Pick): Promise { + const dir = await tempDir("cloudhal-import-"); + await writeProjectConfig(dir, { + dataDir: ".cloudhal", + ...(cfg ?? { siteId: fx.siteId, sites: [fx.siteId, fx.trId] }), + }); + return dir; +} + +const exists = (file: string) => + stat(file).then( + () => true, + () => false, + ); + +async function expectNoStore(dir: string): Promise { + expect(await exists(path.join(dir, ".cloudhal"))).toBe(false); +} + +/** What a refused import must leave as it was (opens the store: call it only where one exists). */ +async function stateOf(dir: string) { + const rows = await withStore(dir, async ({ sql }) => ({ + nodes: await sql.all("SELECT id, site_id, rev, hash FROM nodes ORDER BY id"), + snapshots: await sql.all("SELECT id FROM snapshots ORDER BY id"), + projects: await sql.all("SELECT id, rev FROM projects ORDER BY id"), + media: await sql.all("SELECT site_id, sha256, state FROM media_blobs ORDER BY site_id, sha256"), + })); + return { + rows, + blobs: await blobListing(dir), + config: await readFile(path.join(dir, "cloudhal.json"), "utf8"), + }; +} + +const sourceBytes = (sha: string) => readFile(mediaFile(fx.projectDir, fx.siteId, sha)); + +/** An archive in its own folder. */ +async function archiveFile(bytes: Uint8Array): Promise { + const file = path.join(await tempDir("cloudhal-crafted-"), "crafted.cloudhal"); + await writeFile(file, bytes); + return file; +} + +/** A `--no-media` archive of a crafted bundle, written as `cloudhal export` writes one. */ +async function craftedArchive(bundle: SiteBundle, name = "crafted.cloudhal"): Promise { + const content = bundleToArchive(bundle, { + createdBy: "0.0.0", + mediaOmitted: true, + openMedia: () => { + throw new Error("no media in a --no-media archive"); + }, + }); + const file = path.join(await tempDir("cloudhal-crafted-"), name); + await writeArchiveFile(file, content); + return file; +} + +/** The fixture's bundle with the first ref of `kind` in its published snapshot (s2) at `text`. */ +async function publishedRefAt(kind: string, text: string) { + const bundle = await bundleOf(fx.projectDir, fx.siteId); + const hash = sha256(utf8(text)); + const snapshots = bundle.snapshots.map((s) => { + if (s.id !== fx.snapshots.s2) return s; + const body = JSON.parse(s.body) as { nodes: { kind: string; hash: string }[] }; + const ref = body.nodes.find((r) => r.kind === kind); + if (!ref) throw new Error(`no ${kind} ref`); + ref.hash = hash; + return { ...s, body: JSON.stringify(body) }; + }); + return { hash, bundle: { ...bundle, snapshots, blobs: { ...bundle.blobs, [hash]: text } } }; +} + +describe("cloudhal import: the same site comes back (invariant 1)", () => { + it("imports into an empty project: nodes, history, media, takedowns and the publish pointer", async () => { + const dir = await emptyProject(); + const lines: string[] = []; + const result = await importArchive({ + projectDir: dir, + file: archives.full, + log: (l) => lines.push(l), + }); + expect(result.newStore).toBe(true); + expect(await bundleOf(dir, fx.siteId)).toStrictEqual(await bundleOf(fx.projectDir, fx.siteId)); + for (const sha of [fx.sha.a, fx.sha.c]) + expect( + Buffer.compare(await readFile(mediaFile(dir, fx.siteId, sha)), await sourceBytes(sha)), + ).toBe(0); + // Invariant 5: the taken-down content stays down, its ledger row as the archive has it. + expect(await exists(mediaFile(dir, fx.siteId, fx.sha.b))).toBe(false); + const source = await bundleOf(fx.projectDir, fx.siteId); + const ledger = await withStore(dir, async ({ sql, store }) => ({ + row: await sql.get( + 'SELECT state, purge_reason AS reason, purged_by AS "by", purged_at AS "at" FROM media_blobs WHERE site_id = ? AND sha256 = ?', + [fx.siteId, fx.sha.b], + ), + published: (await store.getSite(fx.siteId))?.publishedSnapshotId, + })); + const [purge] = source.purged ?? []; + expect(ledger.row).toEqual({ + state: "purged", + reason: "personal data", + by: purge?.by, + at: purge?.at, + }); + expect(ledger.published).toBe(fx.snapshots.s2); + expect(result.imported).toMatchObject({ + nodes: { unchanged: false, deleted: 0 }, + snapshots: 3, + media: { copied: 2, present: 0 }, + takedowns: 1, + searchRefreshed: true, + }); + const printed = describeImport(result); + expect(printed[0]).toBe( + `imported ${archives.full}: site ${fx.siteId} (english) is in this project`, + ); + expect(printed).toContain( + `media: 2 file(s) copied (${result.imported?.media.bytes} B), 0 already here; 1 taken down, never imported`, + ); + expect(lines[0]).toMatch(/^verified .*english\.cloudhal: 8 file\(s\), \d+(\.\d)? (B|KiB)$/); + expect(lines[1]).toMatch(/^copying 2 media file\(s\) \(\d+ B\)$/); + + // Another listed site of the same project. + await importArchive({ projectDir: dir, file: archives.tr }); + expect(await bundleOf(dir, fx.trId)).toStrictEqual(await bundleOf(fx.projectDir, fx.trId)); + }); + + it("changes nothing the second time", async () => { + const dir = await emptyProject(); + await importArchive({ projectDir: dir, file: archives.full }); + const before = { ...(await stateOf(dir)), bundle: await bundleOf(dir, fx.siteId) }; + const lines: string[] = []; + const again = await importArchive({ + projectDir: dir, + file: archives.full, + log: (l) => lines.push(l), + }); + expect(again.newStore).toBe(false); + expect(again.imported).toMatchObject({ + nodes: { inserted: 0, deleted: 0, unchanged: true }, + snapshots: 0, + blobs: 0, + media: { copied: 0, present: 2 }, + }); + expect(describeImport(again)).toEqual([ + `nothing to import: this project already holds ${archives.full} (site ${fx.siteId}, english)`, + ]); + expect(lines.some((l) => l.startsWith("copying"))).toBe(false); + expect({ ...(await stateOf(dir)), bundle: await bundleOf(dir, fx.siteId) }).toEqual(before); + }); + + it("replaces a local media file of another size from a full archive", async () => { + const dir = await emptyProject(); + const file = mediaFile(dir, fx.siteId, fx.sha.a); + await cp(path.dirname(mediaFile(fx.projectDir, fx.siteId, fx.sha.a)), path.dirname(file), { + recursive: true, + }); + await writeFile(file, "not the picture"); + const result = await importArchive({ projectDir: dir, file: archives.full }); + expect(result.imported?.media).toMatchObject({ copied: 1, present: 1 }); + expect(Buffer.compare(await readFile(file), await sourceBytes(fx.sha.a))).toBe(0); + }); +}); + +describe("cloudhal import: untrusted rows (invariant 2)", () => { + it("refuses a row whose id carries escapes, before any write, and prints it clean", async () => { + const bundle = await bundleOf(fx.projectDir, fx.siteId); + const page = bundle.nodes.find((n) => n.kind === "page") as NodeRow; + const crafted = { + ...bundle, + nodes: bundle.nodes.map((n) => (n === page ? { ...n, id: "page_\u001b[2J‮evil" } : n)), + }; + const file = await craftedArchive(crafted, "rows.cloudhal"); + const dir = await emptyProject(); + const error = (await importArchive({ projectDir: dir, file }).catch( + (e: unknown) => e, + )) as Error; + expect(error.message).toMatch( + /^cannot import .*rows\.cloudhal: node page_ \[2J evil: columns do not match its body; nothing was imported$/, + ); + expect(error.message).not.toMatch(/[\p{Cc}\p{Cf}]/u); + await expectNoStore(dir); + }); + + it("refuses a node body behind the published snapshot that is not a valid node, dry run too", async () => { + // What the renderer would read for the live site, with its correct hash. + const text = '{"blocks":"","id":"not-a-page","title":42}'; + const { hash, bundle } = await publishedRefAt("page", text); + const file = await craftedArchive(bundle); + const dir = await emptyProject(); + for (const dryRun of [true, false]) + await expect(importArchive({ projectDir: dir, file, dryRun })).rejects.toThrow( + `cannot import ${file}: blob ${hash} is not a valid page; nothing was imported`, + ); + await expectNoStore(dir); + // An asset body the media ledger reads is refused while the archive is read. + const asset = await publishedRefAt("asset", '{"a":1}'); + const assetFile = await craftedArchive(asset.bundle); + for (const dryRun of [true, false]) + await expect(importArchive({ projectDir: dir, file: assetFile, dryRun })).rejects.toThrow( + /^cannot import .*crafted\.cloudhal: snapshots\/0001\.ndjson line \d+: an asset body it references needs storage, an object with a kind; nothing was imported$/, + ); + await expectNoStore(dir); + }); +}); + +describe("cloudhal import: every file verified before use (invariant 3)", () => { + it("refuses a damaged archive before anything is written", async () => { + const last = [fx.sha.a, fx.sha.c].sort().at(-1) as string; + const damaged = await archiveFile( + retar(await readFile(archives.full), (tar) => { + flip(tar, bodyOffset(tar, `media/${last}`) + 10); + return tar; + }), + ); + const refused = new RegExp( + `^cannot import .*crafted\\.cloudhal: media/${last}: its SHA-256 does not match manifest\\.json .*; the file is damaged or incomplete, export it again; nothing was imported$`, + ); + const fresh = await emptyProject(); + await expect(importArchive({ projectDir: fresh, file: damaged })).rejects.toThrow(refused); + await expectNoStore(fresh); + // Into a project with a store: no media file was written, so the first pass refused it. + const dir = await emptyProject(); + await withStore(dir, async () => {}); + const before = await stateOf(dir); + await expect(importArchive({ projectDir: dir, file: damaged })).rejects.toThrow(refused); + expect(await stateOf(dir)).toEqual(before); + expect(before.blobs).toEqual([]); + }); + + it("refuses a file that changes between the passes: another archive, or a media byte", async () => { + const file = path.join(await tempDir("cloudhal-swap-"), "swap.cloudhal"); + // Pass 2 opens the file right after this line. + const swapTo = (bytes: Uint8Array) => (line: string) => { + if (line.startsWith("copying")) writeFileSync(file, bytes); + }; + // Another site's archive, which this project's cloudhal.json does not list. + await cp(archives.full, file); + const dir = await emptyProject({ siteId: fx.siteId }); + await expect( + importArchive({ projectDir: dir, file, log: swapTo(await readFile(archives.tr)) }), + ).rejects.toThrow( + `cannot import ${file}: the file changed while it was being imported; nothing was imported`, + ); + expect(await exists(path.join(dir, ".cloudhal", "data.db"))).toBe(false); + expect(await blobListing(dir)).toEqual([]); + // The same manifest, one media byte flipped: verified again as it is copied, never stored. + const last = [fx.sha.a, fx.sha.c].sort().at(-1) as string; + const flipped = retar(await readFile(archives.full), (tar) => { + flip(tar, bodyOffset(tar, `media/${last}`) + 10); + return tar; + }); + await cp(archives.full, file); + const other = await emptyProject(); + await expect(importArchive({ projectDir: other, file, log: swapTo(flipped) })).rejects.toThrow( + new RegExp( + `^cannot import .*swap\\.cloudhal: media/${last}: its SHA-256 does not match manifest\\.json .*; nothing was imported; 1 media file\\(s\\) already copied stay in `, + ), + ); + expect(await exists(mediaFile(other, fx.siteId, last))).toBe(false); + expect(await exists(path.join(other, ".cloudhal", "data.db"))).toBe(false); + }); + + it("refuses a --no-media archive the project lacks media for, and imports it when it has them", async () => { + const dir = await emptyProject(); + await expect(importArchive({ projectDir: dir, file: archives.graph })).rejects.toThrow( + `the archive has no media (written with --no-media) and this project lacks 2 of the 2 media file(s) it needs (first ${[fx.sha.a, fx.sha.c].sort()[0]}); copy them into .cloudhal/blobs/media/${fx.siteId}/ or import an archive with media; nothing was imported`, + ); + await expectNoStore(dir); + const dry = await importArchive({ projectDir: dir, file: archives.graph, dryRun: true }); + const printed = describeImport(dry); + expect(printed).toContain( + "warning: the archive was written with --no-media; its 2 media file(s) (" + + `${dry.mediaNeeded.bytes} B) must already be in this project`, + ); + expect(printed).toContain( + "media: none in the archive (--no-media); 0 of 2 already here, 2 missing", + ); + await expectNoStore(dir); + + // With the media folder copied (and nothing else), it imports without a second read. + const media = path.dirname(mediaFile(dir, fx.siteId, fx.sha.a)); + await cp(path.dirname(mediaFile(fx.projectDir, fx.siteId, fx.sha.a)), media, { + recursive: true, + }); + const lines: string[] = []; + const result = await importArchive({ + projectDir: dir, + file: archives.graph, + log: (l) => lines.push(l), + }); + expect(lines.some((l) => l.startsWith("copying"))).toBe(false); + expect(result.imported?.media).toMatchObject({ copied: 0, present: 2 }); + expect(await bundleOf(dir, fx.siteId)).toStrictEqual(await bundleOf(fx.projectDir, fx.siteId)); + }); + + it("refuses a --no-media archive over a local media file of another size", async () => { + const dir = await emptyProject(); + const media = path.dirname(mediaFile(dir, fx.siteId, fx.sha.a)); + await cp(path.dirname(mediaFile(fx.projectDir, fx.siteId, fx.sha.a)), media, { + recursive: true, + }); + await writeFile(mediaFile(dir, fx.siteId, fx.sha.c), "short"); + await expect(importArchive({ projectDir: dir, file: archives.graph })).rejects.toThrow( + /this project lacks or has a damaged copy of 1 of the 2 media file\(s\) it needs/, + ); + expect(await exists(path.join(dir, ".cloudhal", "data.db"))).toBe(false); + }); +}); + +describe("cloudhal import: newer archives (invariant 4)", () => { + it("refuses a newer archive format or contracts version, naming the CLI version needed", async () => { + const newer: ((m: Record) => void)[] = [ + (m) => { + m.format = "cloudhal.site-archive@2"; + }, + (m) => { + m.contracts = { schemaVersion: 2 }; + }, + ]; + for (const edit of newer) { + const file = await archiveFile( + archiveOf(baseFiles(), { + manifest: (m) => { + edit(m); + m.createdBy = "9.9.9"; + }, + }), + ); + const dir = await emptyProject(); + await expect(importArchive({ projectDir: dir, file })).rejects.toThrow( + /^cannot import .*: this archive was written by a newer Cloudhal \(.*\); update @cloudhal\/cli to 9\.9\.9 or later; nothing was imported$/, + ); + await expectNoStore(dir); + } + }); +}); + +describe("cloudhal import: taken-down content stays down (invariant 5)", () => { + it("deletes local bytes of content the archive's ledger took down", async () => { + const dir = await emptyProject(); + const b = makePng({ width: 3, height: 3 }); + expect(sha256(b)).toBe(fx.sha.b); + const file = mediaFile(dir, fx.siteId, fx.sha.b); + await cp(path.dirname(mediaFile(fx.projectDir, fx.siteId, fx.sha.a)), path.dirname(file), { + recursive: true, + }); + await writeFile(file, b); + const dry = await importArchive({ projectDir: dir, file: archives.full, dryRun: true }); + expect(describeImport(dry)).toContain( + "taken down: 1 file(s) in the archive's ledger, never imported; 1 of them are in this project and would be deleted", + ); + expect(await exists(file)).toBe(true); + await importArchive({ projectDir: dir, file: archives.full }); + expect(await exists(file)).toBe(false); + }); + + it("says so when it deletes a local copy of content the source took down since", async () => { + const dir = await emptyProject(); + await importArchive({ projectDir: dir, file: archives.full }); + expect(await exists(mediaFile(dir, fx.siteId, fx.sha.c))).toBe(true); + // C is in the first snapshot only: taking it down changes no node. + const source = await copyProject(fx); + await withStore(source, ({ store }) => + store.purgeBlob(fx.siteId, fx.sha.c, person, { text: "court order", legal: true }), + ); + const v2 = path.join(source, "v2.cloudhal"); + await exportArchive({ projectDir: source, out: v2 }); + const dry = await importArchive({ projectDir: dir, file: v2, dryRun: true }); + expect(describeImport(dry)).toContain( + "taken down: 2 file(s) in the archive's ledger, never imported; 1 of them are in this project and would be deleted", + ); + const result = await importArchive({ projectDir: dir, file: v2 }); + expect(result.imported).toMatchObject({ + nodes: { unchanged: true }, + snapshots: 0, + blobs: 0, + media: { copied: 0, present: 1 }, + takedowns: 2, + takedownsDeleted: 1, + }); + expect(describeImport(result)).toEqual([ + `imported ${v2}: site ${fx.siteId} (english) is in this project`, + "nodes: unchanged; snapshots: 0 added; node bodies: 0 added", + "media: 0 file(s) copied (0 B), 1 already here; 2 taken down, never imported; 1 of them deleted from this project", + ]); + expect(await exists(mediaFile(dir, fx.siteId, fx.sha.c))).toBe(false); + const row = await withStore(dir, ({ sql }) => + sql.get( + "SELECT state, purge_legal AS legal FROM media_blobs WHERE site_id = ? AND sha256 = ?", + [fx.siteId, fx.sha.c], + ), + ); + expect(row).toEqual({ state: "purged", legal: 1 }); + }); + + it("refuses an archive carrying content this project has taken down", async () => { + const dir = await emptyProject(); + await importArchive({ projectDir: dir, file: archives.full }); + await withStore(dir, ({ store }) => + store.purgeBlob(fx.siteId, fx.sha.c, person, { text: "notice", legal: false }), + ); + const before = await stateOf(dir); + await expect(importArchive({ projectDir: dir, file: archives.full })).rejects.toThrow( + `the archive carries 1 media file(s) this project has taken down (first ${fx.sha.c}); an import never brings taken-down content back; nothing was imported`, + ); + expect(await exists(mediaFile(dir, fx.siteId, fx.sha.c))).toBe(false); + expect(await stateOf(dir)).toEqual(before); + }); +}); + +/** A project that imported the archive and edited /about; an archive of a source that moved on. */ +async function divergedProject() { + const target = await emptyProject(); + await importArchive({ projectDir: target, file: archives.full }); + const page = await withStore(target, async ({ store }) => { + const about = (await store.listPages(fx.siteId)).find((p) => p.path === "/about") as Page; + return store.updatePage({ ...about, title: "Edited here" }); + }); + const source = await copyProject(fx); + const d = await withStore(source, async ({ store }) => { + const { asset } = await store.ingestAsset({ + siteId: fx.siteId, + bytes: makePng({ width: 7, height: 7 }), + filename: "d.png", + uploadedBy: person, + source: { kind: "upload" }, + }); + await store.createSnapshot(fx.siteId, { actor: person, reason: "manual" }); + return asset; + }); + const moved = path.join(source, "moved.cloudhal"); + await exportArchive({ projectDir: source, out: moved }); + return { target, page, d, moved }; +} + +describe("cloudhal import: a refused import leaves the project as it was (invariant 6)", () => { + it("refuses a project whose site differs: no node body, snapshot or media lands", async () => { + const { target, page, d, moved } = await divergedProject(); + const before = { ...(await stateOf(target)), bundle: await bundleOf(target, fx.siteId) }; + const error = (await importArchive({ projectDir: target, file: moved }).catch( + (e: unknown) => e, + )) as Error; + expect(error.message).toMatch( + new RegExp( + `^cannot import .*moved\\.cloudhal: this project's site ${fx.siteId} differs from the archive in 2 node\\(s\\) \\(.*\\); bootstrap imports only into a project without the site or holding exactly its nodes, and --replace is not available yet; nothing was imported$`, + ), + ); + expect(error.message).toContain(page.id); + expect(error.message).toContain(d.id); + expect({ ...(await stateOf(target)), bundle: await bundleOf(target, fx.siteId) }).toEqual( + before, + ); + expect(await exists(mediaFile(target, fx.siteId, d.sha256))).toBe(false); + }); + + it("says a storage failure after the first write may have left part of it; a second run completes it", async () => { + const dir = await emptyProject(); + // The last write: deleting the bytes of the content the archive's ledger took down. + vi.spyOn(FsBlobStore.prototype, "delete").mockRejectedValueOnce(new Error("disk on fire")); + const error = (await importArchive({ projectDir: dir, file: archives.full }).catch( + (e: unknown) => e, + )) as Error; + expect(error.message).toBe( + `cannot import ${archives.full}: disk on fire; the import stopped after it began writing, so this project may hold part of it; run it again to finish (the 2 media file(s) copied are reused)`, + ); + vi.restoreAllMocks(); + const again = await importArchive({ projectDir: dir, file: archives.full }); + expect(again.imported).toMatchObject({ + media: { copied: 0, present: 2 }, + searchRefreshed: true, + }); + expect(await bundleOf(dir, fx.siteId)).toStrictEqual(await bundleOf(fx.projectDir, fx.siteId)); + // The node set had landed, so only the search index was behind: it is brought to the live one. + const search = await withStore(dir, ({ store }) => store.searchStatus(fx.siteId)); + expect(search?.snapshotId).toBe(fx.snapshots.s2); + // A refusal raised before importSite's first write still says nothing was imported. + const refused = importFailure(new SyncConflictError(fx.siteId, ["page_aaaaaa01"]), { + file: "x.cloudhal", + mediaCopied: 0, + mediaDir: "", + siteId: fx.siteId, + applying: true, + }); + expect(refused.message).toMatch(/; nothing was imported$/); + }); + + it("stops on an abort between the passes", async () => { + const abortAfterVerify = () => { + const controller = new AbortController(); + const log = (l: string) => { + if (l.startsWith("verified ")) controller.abort(); + }; + return { signal: controller.signal, log }; + }; + const fresh = await emptyProject(); + const error = await importArchive({ + projectDir: fresh, + file: archives.full, + ...abortAfterVerify(), + }).catch((e: unknown) => e); + expect(error).toBeInstanceOf(ImportInterruptedError); + expect(error).toMatchObject({ + mediaCopied: 0, + message: "import interrupted; nothing was imported", + }); + await expectNoStore(fresh); + const dir = await emptyProject(); + await withStore(dir, async () => {}); + const before = await stateOf(dir); + await expect( + importArchive({ projectDir: dir, file: archives.full, ...abortAfterVerify() }), + ).rejects.toBeInstanceOf(ImportInterruptedError); + expect(await stateOf(dir)).toEqual(before); + }); + + it("stops during the media copy: the copied file stays unreferenced, the next import reuses it", async () => { + const dir = await emptyProject(); + const controller = new AbortController(); + const putStream = FsBlobStore.prototype.putStream; + vi.spyOn(FsBlobStore.prototype, "putStream").mockImplementation(async function ( + this: FsBlobStore, + ...args: Parameters + ) { + await putStream.apply(this, args); + controller.abort(); + }); + const error = await importArchive({ + projectDir: dir, + file: archives.full, + signal: controller.signal, + }).catch((e: unknown) => e); + expect(error).toBeInstanceOf(ImportInterruptedError); + expect(error).toMatchObject({ + mediaCopied: 1, + message: `import interrupted; nothing was imported; 1 media file(s) already copied stay in .cloudhal/blobs/media/${fx.siteId}/, unreferenced, and the next import reuses them`, + }); + expect((await blobListing(dir)).filter((f) => f.startsWith("media"))).toHaveLength(1); + expect(await exists(path.join(dir, ".cloudhal", "data.db"))).toBe(false); + vi.restoreAllMocks(); + const result = await importArchive({ projectDir: dir, file: archives.full }); + expect(result.imported?.media).toMatchObject({ copied: 1, present: 1 }); + }); + + it("stops on Ctrl-C through the command: exit 130, the listeners removed", async () => { + const dir = await emptyProject(); + vi.spyOn(process, "cwd").mockReturnValue(dir); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const before = process.listeners("SIGINT"); + const beforeTerm = process.listeners("SIGTERM"); + const running = main(["import", archives.full]); + const ours = process.listeners("SIGINT").filter((l) => !before.includes(l)); + expect(ours).toHaveLength(1); + (ours[0] as () => void)(); + expect(await running).toBe(130); + expect(error).toHaveBeenCalledWith("cloudhal: import interrupted; nothing was imported"); + await expectNoStore(dir); + expect(process.listeners("SIGINT")).toEqual(before); + expect(process.listeners("SIGTERM")).toEqual(beforeTerm); + }); +}); + +describe("cloudhal import: which site (the archive's id is kept)", () => { + it("refuses another site's archive, naming both ids; the hint works without a store", async () => { + const dir = await emptyProject({ siteId: "site_other00001" }); + const config = await readFile(path.join(dir, "cloudhal.json")); + await expect(importArchive({ projectDir: dir, file: archives.full })).rejects.toThrow( + `it holds site ${fx.siteId}, and this project's site is site_other00001 (cloudhal.json siteId); importing another site into a project comes with \`cloudhal import --replace\`, not available yet; this project has no local store yet, so setting cloudhal.json siteId to ${fx.siteId} also works (push and deploy then use ${fx.siteId}); nothing was imported`, + ); + await expectNoStore(dir); + expect(Buffer.compare(await readFile(path.join(dir, "cloudhal.json")), config)).toBe(0); + // A dry run reads the whole archive and says so, without a plan. + const dry = await importArchive({ projectDir: dir, file: archives.full, dryRun: true }); + expect(dry.plan).toBeUndefined(); + expect(describeImport(dry).at(-2)).toMatch(/^would refuse: it holds site /); + // As the hint says. + await writeProjectConfig(dir, { siteId: fx.siteId, dataDir: ".cloudhal" }); + await importArchive({ projectDir: dir, file: archives.full }); + expect(await bundleOf(dir, fx.siteId)).toStrictEqual(await bundleOf(fx.projectDir, fx.siteId)); + }); + + it("refuses another site's archive into a project with a store, without the hint", async () => { + const dir = await copyProject(fx); + await writeProjectConfig(dir, { siteId: fx.trId, dataDir: ".cloudhal" }); + const before = await stateOf(dir); + const error = (await importArchive({ projectDir: dir, file: archives.full }).catch( + (e: unknown) => e, + )) as Error; + expect(error.message).toContain( + `it holds site ${fx.siteId}, and this project's site is ${fx.trId} (cloudhal.json siteId); importing another site into a project comes with \`cloudhal import --replace\`, not available yet; nothing was imported`, + ); + expect(await stateOf(dir)).toEqual(before); + }); +}); + +describe("cloudhal import --dry-run", () => { + it("lists what a fresh import adds, by kind and title, and writes nothing", async () => { + const dir = await emptyProject(); + const result = await importArchive({ projectDir: dir, file: archives.full, dryRun: true }); + const lines = describeImport(result); + expect(result.refusals).toEqual([]); + expect(lines[0]).toMatch( + /^archive .*english\.cloudhal: cloudhal\.site-archive@1 \(contracts 1\), written by @cloudhal\/cli \S+ at 2026-01-02T/, + ); + expect(lines[1]).toMatch( + new RegExp(`^site ${fx.siteId} "English" \\(english\\): \\d+ node\\(s\\), 3 snapshot\\(s\\)`), + ); + expect(lines).toContain( + `dry run: bootstrap import into site ${fx.siteId}, which is new to this project`, + ); + expect(lines).toContain(' + page "About us" (/about)'); + expect(lines).toContain(' + media "pixel.png"'); + expect(lines).toContain(' + site "English"'); + expect(lines.find((l) => l.startsWith("nodes: "))).toMatch( + /^nodes: \d+ to add \(1 site, .*2 media, 2 pages.*\), 0 to change, 0 to remove$/, + ); + // In bundle order (created_at, id), as the archive holds them. + const ids = (await bundleOf(fx.projectDir, fx.siteId)).snapshots.map((x) => x.id); + expect(lines.find((l) => l.startsWith("snapshots: "))).toMatch( + new RegExp(`^snapshots: 3 to add \\(${ids.join(", ")}\\), with \\d+ node bodies$`), + ); + expect(lines).toContain( + `media: 2 file(s) to copy (${result.mediaNeeded.bytes} B), 0 already here`, + ); + expect(lines).toContain("taken down: 1 file(s) in the archive's ledger, never imported"); + expect(lines.at(-1)).toBe("nothing written (dry run)"); + await expectNoStore(dir); + }); + + it("shows the change and the refusal for a diverged project and exits 1, writing nothing", async () => { + const { target, moved } = await divergedProject(); + const before = { ...(await stateOf(target)), bundle: await bundleOf(target, fx.siteId) }; + vi.spyOn(process, "cwd").mockReturnValue(target); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(process.stderr, "write").mockImplementation(() => true); + expect(await main(["import", moved, "--dry-run"])).toBe(1); + const printed = log.mock.calls.map((c) => String(c[0])); + expect(printed).toContainEqual( + expect.stringMatching( + /^ {2}~ page "About us" \(\/about\), rev \d+ here, \d+ in the archive$/, + ), + ); + expect(printed).toContain(' + media "d.png"'); + expect(printed).toContainEqual( + expect.stringMatching(new RegExp(`^would refuse: this project's site ${fx.siteId} differs`)), + ); + expect(printed.at(-1)).toBe("nothing written (dry run)"); + expect({ ...(await stateOf(target)), bundle: await bundleOf(target, fx.siteId) }).toEqual( + before, + ); + }); + + it("prints titles clean and caps each group at 20 lines", async () => { + const source = await copyProject(fx); + await withStore(source, async ({ store }) => { + const about = (await store.listPages(fx.siteId)).find((p) => p.path === "/about") as Page; + await store.updatePage({ ...about, title: "Evil\u001b[31m‮title​" }); + }); + const file = path.join(source, "titles.cloudhal"); + await exportArchive({ projectDir: source, out: file, noMedia: true }); + const dry = await importArchive({ projectDir: await emptyProject(), file, dryRun: true }); + const lines = describeImport(dry); + expect(lines).toContain(' + page "Evil [31m title" (/about)'); + for (const line of lines) expect(line).not.toMatch(/[\p{Cc}\p{Cf}]/u); + + const page = dry.plan?.nodes.added.find((n) => n.kind === "page") as NodeRow; + const many = Array.from({ length: 25 }, (_, i) => ({ + ...page, + id: `page_many${String(i).padStart(4, "0")}`, + })); + const plan = { ...(dry.plan as NonNullable) }; + const capped = describeImport({ + ...dry, + plan: { ...plan, nodes: { ...plan.nodes, added: many } }, + }); + const added = capped.filter((l) => l.startsWith(" + ")); + expect(added).toHaveLength(20); + expect(capped[capped.lastIndexOf(added.at(-1) as string) + 1]).toBe(" … and 5 more"); + }); +}); + +describe("cloudhal import: the command", () => { + it("is in --help and checks its arguments and flags", async () => { + expect(HELP).toMatch(/^ {2}cloudhal import \[--dry-run\] +a \.cloudhal file/m); + for (const argv of [["import"], ["import", "a", "b"]]) + await expect(main(argv)).rejects.toThrow("usage: cloudhal import [--dry-run]"); + await expect(main(["import", "x.cloudhal", "--replace"])).rejects.toThrow( + "cloudhal import --replace is not available yet; this version imports in bootstrap mode only", + ); + await expect(main(["import", "x.cloudhal", "--site", "site_other00001"])).rejects.toThrow( + "cloudhal import does not take --site (only cloudhal export does)", + ); + }); + + it("names a missing file or a folder, and imports from the project folder", async () => { + const dir = await emptyProject(); + vi.spyOn(process, "cwd").mockReturnValue(dir); + await expect(main(["import", "x.cloudhal"])).rejects.toThrow( + "cannot import x.cloudhal: no such file; nothing was imported", + ); + await expect(main(["import", "."])).rejects.toThrow( + "cannot import .: it is a directory; name a .cloudhal file; nothing was imported", + ); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const err = vi.spyOn(process.stderr, "write").mockImplementation(() => true); + await cp(archives.full, path.join(dir, "site.cloudhal")); + expect(await main(["import", "site.cloudhal"])).toBe(0); + expect(log).toHaveBeenCalledWith( + `imported site.cloudhal: site ${fx.siteId} (english) is in this project`, + ); + expect(err).toHaveBeenCalledWith(expect.stringMatching(/^verified site\.cloudhal: /)); + await rm(path.join(dir, "site.cloudhal")); + }); +}); diff --git a/packages/cli/test/site-fixture.ts b/packages/cli/test/site-fixture.ts new file mode 100644 index 00000000..0645b185 --- /dev/null +++ b/packages/cli/test/site-fixture.ts @@ -0,0 +1,148 @@ +/** + * A seeded local project for the `cloudhal export` and `cloudhal import` tests (RFC-0039): the + * English site with three uploads (A the entry's cover; B taken down; C deleted, kept by the first + * snapshot only), three snapshots (the second published, the third after an edit of /about) and a + * second listed site (`tr`), all written on `SEEDED_AT`. Helpers open its store, copy it and list + * its blobs; every folder they make is removed by `removeTempDirs`. + */ +import { cp, mkdtemp, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { Actor, ContentEntry, Page, Site } from "@cloudhal/core-contracts"; +import { exportSite, type SiteBundle, seedStarterSite } from "@cloudhal/core-ts"; +import { type LocalStore, openLocalStore } from "@cloudhal/core-ts/local"; +import { makePng, PNG_1X1 } from "@cloudhal/core-ts/media/fixtures"; +import { vi } from "vitest"; +import { writeProjectConfig } from "../src/config.ts"; + +export const person: Actor = { kind: "human", id: "user_01HZX8K3Q9CD", name: "Halil" }; +export const robot: Actor = { kind: "agent", id: "agent_01HZX8K3Q9YZ", name: "Newsroom Agent" }; + +export interface Fixture { + projectDir: string; + siteId: string; + trId: string; + /** A: the entry's cover; B: taken down; C: deleted, kept by the first snapshot only. */ + sha: { a: string; b: string; c: string }; + snapshots: { s1: string; s2: string; s3: string }; +} + +/** The fixture's rows are written on this day, so a file name dated by the clock would show. */ +export const SEEDED_AT = new Date("2026-01-02T03:04:05.000Z"); +export const SEEDED_DAY = "20260102"; + +const dirs: string[] = []; + +export async function tempDir(prefix: string): Promise { + const dir = await mkdtemp(path.join(tmpdir(), prefix)); + dirs.push(dir); + return dir; +} + +export async function removeTempDirs(): Promise { + for (const d of dirs.splice(0)) await rm(d, { recursive: true, force: true }); +} + +async function seed(projectDir: string): Promise> { + const local = await openLocalStore(path.join(projectDir, ".cloudhal")); + const { store } = local; + const site = await seedStarterSite(store, { name: "English", slug: "english" }); + const upload = (bytes: Uint8Array, filename: string) => + store.ingestAsset({ + siteId: site.id, + bytes, + filename, + uploadedBy: robot, + source: { kind: "upload" }, + }); + const a = await upload(PNG_1X1, "pixel.png"); + const entry = (await store.listContentEntries(site.id))[0] as ContentEntry; + await store.updateContentEntry({ + ...entry, + coverImage: { assetId: a.asset.id, alt: "" } as never, + }); + const b = await upload(makePng({ width: 3, height: 3 }), "b.png"); + const c = await upload(makePng({ width: 5, height: 4 }), "c.png"); + const s1 = await store.createSnapshot(site.id, { actor: person, reason: "manual" }); + await store.deleteAsset(c.asset); + await store.purgeBlob(site.id, b.asset.sha256, person, { text: "personal data", legal: false }); + const s2 = await store.createSnapshot(site.id, { actor: person, reason: "publish" }); + await store.publish((await store.getSite(site.id)) as Site, s2.id); + const about = (await store.listPages(site.id)).find((p) => p.path === "/about") as Page; + await store.updatePage({ ...about, title: "About us" }); + const s3 = await store.createSnapshot(site.id, { actor: person, reason: "manual" }); + const tr = await seedStarterSite(store, { + name: "Türkçe", + slug: "turkce", + locale: "tr", + projectId: site.projectId, + }); + await local.close(); + await writeProjectConfig(projectDir, { + siteId: site.id, + dataDir: ".cloudhal", + sites: [site.id, tr.id], + }); + return { + siteId: site.id, + trId: tr.id, + sha: { a: a.asset.sha256, b: b.asset.sha256, c: c.asset.sha256 }, + snapshots: { s1: s1.id, s2: s2.id, s3: s3.id }, + }; +} + +/** The fixture project in a new folder, its rows dated `SEEDED_AT`. */ +export async function seedFixture(prefix: string): Promise { + const projectDir = await tempDir(prefix); + vi.useFakeTimers({ toFake: ["Date"], now: SEEDED_AT, shouldAdvanceTime: true }); + try { + return { projectDir, ...(await seed(projectDir)) }; + } finally { + vi.useRealTimers(); + } +} + +/** A copy of the fixture project (without the archives other tests wrote), for tests that change it. */ +export async function copyProject(fx: Fixture): Promise { + const dir = await tempDir("cloudhal-copy-"); + await cp(fx.projectDir, dir, { recursive: true }); + for (const f of await readdir(dir)) + if (f !== ".cloudhal" && f.endsWith(".cloudhal")) await rm(path.join(dir, f)); + return dir; +} + +/** The local store of `projectDir`, open for `fn` only. */ +export async function withStore( + projectDir: string, + fn: (local: LocalStore) => Promise, +): Promise { + const local = await openLocalStore(path.join(projectDir, ".cloudhal")); + try { + return await fn(local); + } finally { + await local.close(); + } +} + +export const bundleOf = (projectDir: string, siteId: string): Promise => + withStore(projectDir, (local) => exportSite(local, siteId)); + +/** A media file of the site in the project's blob store. */ +export const mediaFile = (projectDir: string, siteId: string, sha: string): string => + path.join(projectDir, ".cloudhal", "blobs", "media", siteId, sha); + +/** Every file under the project's blob store but its temporary folder (`blobs/.tmp`). */ +export async function blobListing(projectDir: string): Promise { + const root = path.join(projectDir, ".cloudhal", "blobs"); + const entries = await readdir(root, { recursive: true, withFileTypes: true }).catch( + (e: NodeJS.ErrnoException) => { + if (e.code === "ENOENT") return []; + throw e; + }, + ); + return entries + .filter((e) => e.isFile()) + .map((e) => path.relative(root, path.join(e.parentPath, e.name))) + .filter((f) => f.split(path.sep)[0] !== ".tmp") + .sort(); +} diff --git a/packages/core-ts/src/sync-bundle.ts b/packages/core-ts/src/sync-bundle.ts new file mode 100644 index 00000000..11c48bdc --- /dev/null +++ b/packages/core-ts/src/sync-bundle.ts @@ -0,0 +1,115 @@ +/** + * The site bundle (`cloudhal.site-bundle@1`): what `exportSite` returns, `importSite` and + * `planImport` take, and `cloudhal push`/`pull` send over HTTP; with the copy options, the copy + * result and the bootstrap conflict. Types and values only, so `sync-checks.ts` and + * `sync-plan.ts` share them without importing `sync.ts`. Worker-safe. + */ +import type { BlobStore, SqlDriver } from "@cloudhal/storage"; +import type { GateSupport } from "./availability.ts"; + +export interface StoragePair { + sql: SqlDriver; + blobs: BlobStore; +} + +export interface CopySiteOptions { + /** + * `bootstrap` (default): copy when the target has no such site or already holds exactly the same + * nodes; otherwise throw `SyncConflictError`. `replace`: make the target's nodes equal the source. + */ + mode?: "bootstrap" | "replace"; + log?: (line: string) => void; + /** + * RFC-0008 S15: when given, a bundle whose pages or entries carry `availability` is refused + * (`gates_unsupported`) unless the target's renderer enforces gates. The admin Worker passes its + * store's check; local targets have the renderer in the same process. + */ + gates?: GateSupport; + /** + * Whose `publishedSnapshotId` a written site node carries. `source` (default, `pull`): the + * bundle's, so a local copy mirrors the deployment. `target` (`push`): once the target has + * published, its pointer is kept, so a push carries content but never moves production back to + * an older snapshot (publishing stays a person's action in Studio); a target that never + * published takes the bundle's, as a first deploy does. + */ + published?: "source" | "target"; +} + +export interface CopySiteResult { + nodes: { inserted: number; deleted: number; unchanged: boolean }; + snapshots: number; + blobs: number; +} + +export class SyncConflictError extends Error { + override readonly name = "SyncConflictError"; + readonly differing: string[]; + constructor(siteId: string, differing: string[]) { + super( + `target already has site ${siteId} with different content (${differing.length} node(s) differ: ${differing.slice(0, 5).join(", ")}${differing.length > 5 ? ", …" : ""}); pull first or pass mode "replace"`, + ); + this.differing = differing; + } +} + +export type NodeRow = { + id: string; + site_id: string; + kind: string; + key: string; + rev: number; + hash: string; + body: string; + created_at: string; + updated_at: string; +}; +export type SnapshotRow = { + id: string; + site_id: string; + parent_id: string | null; + site_rev: number; + body: string; + created_at: string; +}; +export type ProjectRow = { + id: string; + rev: number; + body: string; + created_at: string; + updated_at: string; +}; + +/** A self-contained, JSON-serialisable copy of one site (what push/pull send over HTTP). */ +export interface SiteBundle { + format: "cloudhal.site-bundle@1"; + siteId: string; + project?: ProjectRow | undefined; + nodes: NodeRow[]; + snapshots: SnapshotRow[]; + /** Node bodies referenced by the snapshots, keyed by content hash (UTF-8 JSON text). */ + blobs: Record; + /** + * RFC-0004 §4.7: content the bundle's blob assets (current and in snapshots) need. The bytes + * travel separately (`copyMediaBlobs`, or the admin media routes); a target that lacks them + * refuses the bundle (`missing_media`). + */ + media?: BundleMedia[]; + /** Taken-down content: travels as ledger rows, so a copy can never bring the bytes back. */ + purged?: BundlePurge[]; +} + +export interface BundleMedia { + sha256: string; + bytes: number; + mediaType: string; +} + +export interface BundlePurge { + sha256: string; + reason: string; + legal: boolean; + by: string; + at: string; +} + +export const SITE_BUNDLE_FORMAT = "cloudhal.site-bundle@1" as const; diff --git a/packages/core-ts/src/sync-checks.ts b/packages/core-ts/src/sync-checks.ts new file mode 100644 index 00000000..a2e588ed --- /dev/null +++ b/packages/core-ts/src/sync-checks.ts @@ -0,0 +1,358 @@ +/** + * The checks a site bundle passes before anything of it is written (`planImport`): the format, the + * site node, rows of this site only, every row a valid contract document whose columns agree with + * its body (precompiled validators only, AGENTS.md rule 6), rows the node batch's keys accept, + * gates, the time zone, the media and takedowns the bundle lists, and every node body a snapshot + * references. Worker-safe; not exported from the package (`planImport` is the public surface). + */ +import { type Asset, assertValid, type NodeRef } from "@cloudhal/core-contracts"; +import { + checkAvailability, + checkTimeZone, + EXPIRY_GATES_VERSION, + type GateSupport, + isTimeZone, +} from "./availability.ts"; +import { InvariantError } from "./errors.ts"; +import { nodeHash } from "./hash.ts"; +import { validateMenuTree } from "./menu.ts"; +import { isNodeKind, NODE_SPEC, type NodeKind, type NodeTypes } from "./nodes.ts"; +import { + type BundleMedia, + type NodeRow, + SITE_BUNDLE_FORMAT, + type SiteBundle, + type SnapshotRow, + type StoragePair, +} from "./sync-bundle.ts"; + +const SHA256 = /^[0-9a-f]{64}$/; + +/** A node kind's spec with its node type widened, for rows whose kind is only known at run time. */ +type AnySpec = { + schema: (typeof NODE_SPEC)[NodeKind]["schema"]; + key(n: NodeTypes[NodeKind]): string; + siteId(n: NodeTypes[NodeKind]): string; +}; +const specOf = (kind: NodeKind) => NODE_SPEC[kind] as AnySpec; + +/** Gates in a bundle follow the write rules (§4.1) and the writer guard (S15). */ +async function assertGates( + siteId: string, + bundle: SiteBundle, + gates: GateSupport | undefined, +): Promise { + let gated = false; + for (const row of bundle.nodes) { + if (row.kind !== "page" && row.kind !== "contentEntry") continue; + const availability = (JSON.parse(row.body) as { availability?: { from?: string } }) + .availability; + if (!availability) continue; + checkAvailability(availability); + gated = true; + } + if (gated && gates && !(await gates.enforced(siteId))) + throw new InvariantError( + "gates_unsupported", + "the bundle has time gates (availability) and this site's renderer has not confirmed it enforces them", + { siteId }, + ); + // RFC-0042 §3.6: a collection's `expiry` needs a renderer with derived gates. + const site = bundle.nodes.find((r) => r.kind === "site"); + const collections = site + ? (JSON.parse(site.body) as { collections?: Record }).collections + : undefined; + const expiring = Object.values(collections ?? {}).some((c) => c?.expiry !== undefined); + if (expiring && gates && !(await gates.enforced(siteId, EXPIRY_GATES_VERSION))) + throw new InvariantError( + "gates_unsupported", + "the bundle has a collection expiry and this site's renderer has not confirmed it enforces derived gates", + { siteId }, + ); +} + +/** + * `Site.timeZone` must be a zone this runtime knows (RFC-0008 §4.1), as on every other write; the + * zone the target already stores for the site is accepted unchanged. + */ +export async function assertTimeZone( + to: StoragePair, + siteId: string, + bundle: SiteBundle, +): Promise { + const row = bundle.nodes.find((n) => n.kind === "site" && n.id === siteId); + const zone = row ? (JSON.parse(row.body) as { timeZone?: string }).timeZone : undefined; + if (zone === undefined || isTimeZone(zone)) return; + const stored = await to.sql.get<{ body: string }>( + "SELECT body FROM nodes WHERE id = ? AND kind = 'site'", + [siteId], + ); + checkTimeZone( + zone, + stored ? (JSON.parse(stored.body) as { timeZone?: string }).timeZone : undefined, + ); +} + +/** + * The project row is inserted as given when the target lacks it, so it must be the site node's + * project (`projectId`), a valid Project whose `id` and `rev` agree with its columns. Its id is + * echoed only once it equals the validated site node's. + */ +function assertProjectRow(siteId: string, bundle: SiteBundle): void { + const p = bundle.project; + if (!p) return; + const siteRow = bundle.nodes.find((n) => n.kind === "site" && n.id === siteId); + const projectId = siteRow + ? (JSON.parse(siteRow.body) as { projectId?: unknown }).projectId + : undefined; + if (p.id !== projectId) throw new Error("the bundle's project is not its site's project"); + let body: { id?: unknown; rev?: unknown }; + try { + body = JSON.parse(p.body) as { id?: unknown; rev?: unknown }; + } catch { + throw new Error(`project ${p.id}: body is not JSON`); + } + try { + assertValid("project", body); + } catch { + throw new Error(`project ${p.id}: not a valid project`); + } + if (body.id !== p.id || body.rev !== p.rev) + throw new Error(`project ${p.id}: columns do not match its body`); +} + +/** + * A bundle is untrusted input (whoever holds the deployment token): every row must be a valid + * contract document of the kind its `kind` column declares (RFC-0002 §4.1: the node schemas are + * mutually exclusive, so a body cannot pass as another kind), whose columns agree with its body, + * for this site only; the project row too (`assertProjectRow`). + */ +async function assertValidRows(siteId: string, bundle: SiteBundle): Promise { + for (const row of bundle.nodes) { + if (!isNodeKind(row.kind)) throw new Error(`node ${row.id}: unknown kind ${row.kind}`); + const kind: NodeKind = row.kind; + const spec = specOf(kind); + const schema = spec.schema; + let body: NodeTypes[NodeKind]; + try { + body = JSON.parse(row.body) as NodeTypes[NodeKind]; + } catch { + throw new Error(`node ${row.id}: body is not JSON`); + } + try { + assertValid(schema, body); + } catch (e) { + throw new Error( + `node ${row.id}: not a valid ${schema} (${e instanceof Error ? e.message.slice(0, 200) : e})`, + ); + } + if ( + body.id !== row.id || + body.rev !== row.rev || + spec.siteId(body) !== row.site_id || + row.site_id !== siteId || + spec.key(body) !== row.key + ) + throw new Error(`node ${row.id}: columns do not match its body`); + if ((await nodeHash(body)) !== row.hash) + throw new Error(`node ${row.id}: hash does not match its body`); + if (kind === "menu") { + // RFC-0003 §4.2: the tree rules the schema cannot express; targets are not checked (hidden + // items are a legal state), and labels of a locale the site dropped are kept. + const siteRow = bundle.nodes.find((n) => n.kind === "site" && n.id === siteId); + const menu = body as NodeTypes["menu"]; + try { + if (siteRow) validateMenuTree(menu, JSON.parse(siteRow.body), menu); + } catch (e) { + throw new Error(`node ${row.id}: ${e instanceof Error ? e.message : e}`); + } + } + } + for (const row of bundle.snapshots) { + let body: { id?: string; siteId?: string }; + try { + body = JSON.parse(row.body) as { id?: string; siteId?: string }; + } catch { + throw new Error(`snapshot ${row.id}: body is not JSON`); + } + try { + assertValid("snapshot", body); + } catch { + throw new Error(`snapshot ${row.id}: not a valid snapshot`); + } + if (body.id !== row.id || body.siteId !== siteId || row.site_id !== siteId) + throw new Error(`snapshot ${row.id}: columns do not match its body`); + } + assertProjectRow(siteId, bundle); +} + +/** A page's or entry's translation group and locale, as `nodes_translation_locale` indexes them. */ +function translationSlot(row: NodeRow): string | undefined { + if (row.kind !== "page" && row.kind !== "contentEntry") return undefined; + const body = JSON.parse(row.body) as { translation?: { group?: unknown }; locale?: unknown }; + const group = body.translation?.group; + return typeof group === "string" && typeof body.locale === "string" + ? `t\u0000${row.kind}\u0000${group}\u0000${body.locale}` + : undefined; +} + +/** + * What the node batch's unique keys would refuse after the node bodies and snapshots landed, + * checked on the validated rows first: each node and snapshot id once, one node per kind and key + * (`nodes_site_kind_key`), one page or entry per translation group and locale + * (`nodes_translation_locale`). Echoes only validated ids and kinds. + */ +function assertDistinctRows(bundle: SiteBundle): void { + const ids = new Set(); + const slots = new Set(); + for (const row of bundle.nodes) { + if (ids.has(row.id)) throw new Error(`node ${row.id} appears twice`); + ids.add(row.id); + const key = `k\u0000${row.kind}\u0000${row.key}`; + if (slots.has(key)) throw new Error(`node ${row.id}: another ${row.kind} has its key`); + slots.add(key); + const translation = translationSlot(row); + if (translation === undefined) continue; + if (slots.has(translation)) + throw new Error( + `node ${row.id}: another ${row.kind} of its translation group has its locale`, + ); + slots.add(translation); + } + const snapshots = new Set(); + for (const row of bundle.snapshots) { + if (snapshots.has(row.id)) throw new Error(`snapshot ${row.id} appears twice`); + snapshots.add(row.id); + } +} + +/** + * Node and snapshot ids are keys of the whole installation (one store may hold several sites), so + * none of the bundle's may be another site's in the target. Ids go 500 to a query, as one JSON + * parameter. + */ +export async function assertIdsFree(to: StoragePair, bundle: SiteBundle): Promise { + const tables = [ + ["nodes", "node", bundle.nodes], + ["snapshots", "snapshot", bundle.snapshots], + ] as const; + for (const [table, what, rows] of tables) + for (let i = 0; i < rows.length; i += 500) { + const taken = await to.sql.get<{ id: string }>( + `SELECT id FROM ${table} WHERE site_id <> ? AND id IN (SELECT value FROM json_each(?)) LIMIT 1`, + [bundle.siteId, JSON.stringify(rows.slice(i, i + 500).map((r) => r.id))], + ); + if (taken) + throw new Error(`${what} ${taken.id} belongs to another site of this installation`); + } +} + +/** + * The checks that need no target, in `importSite`'s order: the format, the site node, rows of this + * site only, valid rows (`assertValidRows`), distinct rows (`assertDistinctRows`) and gates. + * Returns the validated site row. + */ +export async function checkBundle( + bundle: SiteBundle, + gates: GateSupport | undefined, +): Promise { + if (bundle.format !== SITE_BUNDLE_FORMAT) + throw new Error(`unsupported bundle format ${bundle.format}`); + const { siteId } = bundle; + const siteRow = bundle.nodes.find((n) => n.kind === "site" && n.id === siteId); + if (!siteRow) throw new Error(`bundle has no site node for ${siteId}`); + if ( + bundle.nodes.some((n) => n.site_id !== siteId) || + bundle.snapshots.some((s) => s.site_id !== siteId) + ) + throw new Error("bundle contains rows of another site"); + await assertValidRows(siteId, bundle); + assertDistinctRows(bundle); + await assertGates(siteId, bundle, gates); + return siteRow; +} + +/** + * RFC-0004 §4.7, the part that needs no target: the bundle's media entries (each content once, in + * bundle order) and takedowns are well formed (every field the ledger writes, so writing it cannot + * fail on them), and every current blob asset's content is listed or taken down; otherwise + * `missing_media`. The target must then hold every listed content that is not taken down + * (`planImport`). + */ +export function listedMedia(bundle: SiteBundle): { + listed: BundleMedia[]; + purged: ReadonlySet; +} { + const listed = new Map(); + for (const m of bundle.media ?? []) { + if ( + typeof m?.sha256 !== "string" || + !SHA256.test(m.sha256) || + !Number.isInteger(m.bytes) || + m.bytes < 0 || + typeof m.mediaType !== "string" || + m.mediaType.length > 255 + ) + throw new Error("bundle media entry is malformed"); + if (!listed.has(m.sha256)) listed.set(m.sha256, m); + } + const purged = new Set(); + for (const p of bundle.purged ?? []) { + if ( + typeof p?.sha256 !== "string" || + !SHA256.test(p.sha256) || + typeof p.reason !== "string" || + typeof p.legal !== "boolean" || + typeof p.by !== "string" || + typeof p.at !== "string" + ) + throw new Error("bundle takedown entry is malformed"); + purged.add(p.sha256); + } + for (const n of bundle.nodes) + if (n.kind === "asset") { + const a = JSON.parse(n.body) as Asset; + if (a.storage.kind === "blob" && !listed.has(a.sha256) && !purged.has(a.sha256)) + throw new InvariantError( + "missing_media", + `asset ${a.id} needs content the bundle does not list`, + { + sha256: a.sha256, + }, + ); + } + return { listed: [...listed.values()], purged }; +} + +/** A snapshot row's node refs; its body was validated as a snapshot (`checkBundle`). */ +export function snapshotRefs(row: SnapshotRow): NodeRef[] { + return (JSON.parse(row.body) as { nodes: NodeRef[] }).nodes; +} + +/** + * Invariant 2 for the node bodies (RFC-0039): a body the target lacks is written as given and read + * back as published content (the renderer reads a snapshot's nodes from these bodies), so it must + * be a valid contract document of the kind of every snapshot ref that points to it, with the ref's + * id and rev, of this site. A body no snapshot references is refused. Names only the validated + * hash and the ref's kind. + */ +export function assertNodeBody( + siteId: string, + hash: string, + body: unknown, + refs: readonly NodeRef[] | undefined, +): void { + if (!refs || refs.length === 0) throw new Error(`blob ${hash} is not referenced by any snapshot`); + for (const ref of refs) { + if (!isNodeKind(ref.kind)) throw new Error(`blob ${hash}: unknown kind`); + const spec = specOf(ref.kind); + try { + assertValid(spec.schema, body); + } catch { + throw new Error(`blob ${hash} is not a valid ${ref.kind}`); + } + const node = body as NodeTypes[NodeKind]; + if (node.id !== ref.id || node.rev !== ref.rev || spec.siteId(node) !== siteId) + throw new Error(`blob ${hash} is not the ${ref.kind} its snapshot ref names`); + } +} diff --git a/packages/core-ts/src/sync-ledger.ts b/packages/core-ts/src/sync-ledger.ts new file mode 100644 index 00000000..8011da99 --- /dev/null +++ b/packages/core-ts/src/sync-ledger.ts @@ -0,0 +1,103 @@ +/** + * The media ledger writes of an import (RFC-0004 §4.7), planned with the rest before the first + * write (`prepareImport`), so applying them cannot fail on the bundle's data: a `stored` row for + * each listed content and a `purged` row for each takedown (`listedMedia` checked every field), + * and `snapshot_blobs` for the bundle's snapshots. The content a snapshot captured is read from + * verified asset bodies only: the plan's for the bodies the target lacks (validated as assets), + * the target's own otherwise, never the bundle's text for a body the target already holds. + * Worker-safe. + */ +import type { Asset, NodeRef } from "@cloudhal/core-contracts"; +import type { SqlStatement } from "@cloudhal/storage"; +import type { BundleMedia, BundlePurge, SiteBundle, StoragePair } from "./sync-bundle.ts"; + +export interface LedgerOptions { + /** Each bundle snapshot's id and node refs, in bundle order. */ + refs: readonly { id: string; refs: readonly NodeRef[] }[]; + /** The verified bodies the import writes (the target lacks them), by hash. */ + written: ReadonlyMap; + /** The import adds at least one snapshot. */ + snapshotsAdded: boolean; +} + +function storedRow(siteId: string, m: BundleMedia, ts: string): SqlStatement { + return { + sql: `INSERT INTO media_blobs (site_id, sha256, bytes, media_type, state, created_at, updated_at) + VALUES (?, ?, ?, ?, 'stored', ?, ?) + ON CONFLICT (site_id, sha256) DO UPDATE SET state = 'stored', updated_at = excluded.updated_at + WHERE media_blobs.state IN ('uploading', 'deleting')`, + params: [siteId, m.sha256, m.bytes, m.mediaType, ts, ts], + }; +} + +function purgedRow(siteId: string, p: BundlePurge, ts: string): SqlStatement { + return { + sql: `INSERT INTO media_blobs (site_id, sha256, bytes, media_type, state, purge_reason, purge_legal, purged_by, purged_at, created_at, updated_at) + VALUES (?, ?, 0, 'application/octet-stream', 'purged', ?, ?, ?, ?, ?, ?) + ON CONFLICT (site_id, sha256) DO UPDATE SET state = 'purged', lease_until = NULL, + purge_reason = excluded.purge_reason, purge_legal = excluded.purge_legal, + purged_by = excluded.purged_by, purged_at = excluded.purged_at, updated_at = excluded.updated_at`, + params: [ + siteId, + p.sha256, + p.reason.slice(0, 1000), + p.legal ? 1 : 0, + p.by.slice(0, 200), + p.at, + ts, + ts, + ], + }; +} + +/** The media content an asset body stores in the blob store, if it is a blob asset. */ +async function capturedContent( + to: StoragePair, + hash: string, + written: ReadonlyMap, +): Promise { + let body = written.get(hash); + if (body === undefined) { + const data = await to.blobs.get(`nodes/${hash}.json`); + try { + body = data ? JSON.parse(new TextDecoder().decode(data)) : undefined; + } catch { + body = undefined; + } + } + const a = body as Partial | null | undefined; + return a?.storage?.kind === "blob" && typeof a.sha256 === "string" ? a.sha256 : undefined; +} + +/** Ledger rows for the bundle's media and takedowns, and `snapshot_blobs` for its snapshots. */ +export async function planLedger( + to: StoragePair, + bundle: SiteBundle, + options: LedgerOptions, +): Promise { + const { siteId } = bundle; + const ts = new Date().toISOString(); + const statements: SqlStatement[] = [ + ...(bundle.media ?? []).map((m) => storedRow(siteId, m, ts)), + ...(bundle.purged ?? []).map((p) => purgedRow(siteId, p, ts)), + ]; + if (!options.snapshotsAdded && (bundle.media ?? []).length === 0) return statements; + // Which content each snapshot captured: from its asset node bodies, each read once. + const content = new Map(); + for (const snap of options.refs) { + const shas = new Set(); + for (const r of snap.refs) { + if (r.kind !== "asset") continue; + if (!content.has(r.hash)) + content.set(r.hash, await capturedContent(to, r.hash, options.written)); + const sha = content.get(r.hash); + if (sha) shas.add(sha); + } + if (shas.size > 0) + statements.push({ + sql: "INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, site_id, sha256) SELECT ?, ?, value FROM json_each(?)", + params: [snap.id, siteId, JSON.stringify([...shas])], + }); + } + return statements; +} diff --git a/packages/core-ts/src/sync-plan.ts b/packages/core-ts/src/sync-plan.ts new file mode 100644 index 00000000..8250b015 --- /dev/null +++ b/packages/core-ts/src/sync-plan.ts @@ -0,0 +1,384 @@ +/** + * What importing a site bundle would change, without writing (RFC-0039 invariants 3 and 6): + * every check and every target read `importSite` does, in its order, with the writes held apart. + * `importSite` applies them only once all checks passed, so a refused bundle (a bootstrap + * conflict, missing media, a taken host, an id another site holds, a node body that is not the + * node its snapshot ref names, a malformed takedown, the 1000-statement limit) leaves the target + * as it was. The media ledger is planned here too (`sync-ledger.ts`), from verified bodies only. + * `cloudhal import` plans with `report: true` before it copies any media, and prints the plan for + * `--dry-run`. Worker-safe. + */ +import type { NodeRef, Site } from "@cloudhal/core-contracts"; +import type { SqlStatement } from "@cloudhal/storage"; +import { mediaBlobKey } from "./assets.ts"; +import type { GateSupport } from "./availability.ts"; +import { InvariantError } from "./errors.ts"; +import { canonicalJson, sha256Hex } from "./hash.ts"; +import { checkHosts, hostStatements, hostsOf, type SiteHost } from "./hosts.ts"; +import { refsRebuild } from "./media/refs-rebuild.ts"; +import { + type BundleMedia, + type NodeRow, + type SiteBundle, + type StoragePair, + SyncConflictError, +} from "./sync-bundle.ts"; +import { + assertIdsFree, + assertNodeBody, + assertTimeZone, + checkBundle, + listedMedia, + snapshotRefs, +} from "./sync-checks.ts"; +import { planLedger } from "./sync-ledger.ts"; +import { keepTargetPublished } from "./sync-published.ts"; + +export interface ImportPlanOptions { + /** As `importSite`'s (`CopySiteOptions`). */ + mode?: "bootstrap" | "replace" | undefined; + gates?: GateSupport | undefined; + published?: "source" | "target" | undefined; + /** + * Return a bootstrap conflict (`conflict`) and missing media (`media.missing`) instead of + * throwing them (`cloudhal import` before it copies media, and `--dry-run`). Every other + * refusal still throws. + */ + report?: boolean | undefined; +} + +export interface ImportPlan { + siteId: string; + mode: "bootstrap" | "replace"; + /** The target holds no node of the site. */ + fresh: boolean; + /** The target's nodes are exactly the bundle's (id, rev and hash). */ + unchanged: boolean; + /** By id: nodes only the bundle has, both have at another rev or hash, only the target has. */ + nodes: { added: NodeRow[]; changed: { from: NodeRow; to: NodeRow }[]; removed: NodeRow[] }; + /** Bootstrap into a target whose nodes differ: the ids (`SyncConflictError.differing`). */ + conflict?: string[]; + /** Node bodies the target lacks. */ + blobs: number; + /** Ids of the snapshots the target lacks, in bundle order. */ + snapshots: string[]; + /** The target lacks the bundle's project row. */ + projectAdded: boolean; + /** + * The listed media not taken down by the bundle, by what the target holds: the content with its + * size, nothing, or a file of another size (`wrongSize`); `takenDownHere` are the ones the + * target's own ledger has taken down (also counted in one of the other three). + */ + media: { + present: BundleMedia[]; + missing: BundleMedia[]; + wrongSize: BundleMedia[]; + takenDownHere: BundleMedia[]; + }; + /** + * The bundle's takedowns: how many, those whose bytes the target still holds (an import deletes + * them), and those the target's ledger does not yet hold as taken down (an import records them). + */ + takedowns: { listed: number; bytesHere: string[]; recorded: string[] }; +} + +/** The writes of a planned import, in `importSite`'s order. */ +export interface ImportWrites { + /** Node bodies (`nodes/.json`), verified against their hash and their snapshot refs. */ + blobs: VerifiedBody[]; + /** Snapshot inserts. */ + snapshots: SqlStatement[]; + /** One atomic batch: the project row if missing, the node set, its reference index, its hosts. */ + batch: SqlStatement[]; + inserted: number; + deleted: number; + /** The media ledger and `snapshot_blobs` (`planLedger`), after the batch. */ + ledger: SqlStatement[]; + /** Media keys to delete: the bundle's takedowns (taken-down bytes never survive a copy). */ + takedowns: string[]; +} + +export interface VerifiedBody { + hash: string; + text: string; + /** The parsed body, a valid node of every snapshot ref to it. */ + body: unknown; +} + +/** Each bundle snapshot's node refs, parsed once. */ +type SnapshotRefs = { id: string; refs: NodeRef[] }[]; + +const HASH = /^[0-9a-f]{64}$/; + +/** The content the target's ledger has taken down. */ +async function takenDown(to: StoragePair, siteId: string): Promise> { + const down = await to.sql.all<{ sha256: string }>( + "SELECT sha256 FROM media_blobs WHERE site_id = ? AND state = 'purged'", + [siteId], + ); + return new Set(down.map((r) => r.sha256)); +} + +/** The listed media against the target's blob store and ledger; missing ones throw unless `report`. */ +async function mediaOnTarget( + to: StoragePair, + bundle: SiteBundle, + options: { report: boolean; down: ReadonlySet }, +): Promise { + const { listed, purged } = listedMedia(bundle); + const wanted = listed.filter((m) => !purged.has(m.sha256)); + const media: ImportPlan["media"] = { present: [], missing: [], wrongSize: [], takenDownHere: [] }; + for (const m of wanted) { + const head = await to.blobs.head(mediaBlobKey(bundle.siteId, m.sha256)); + if (!head) media.missing.push(m); + else if (head.size === m.bytes) media.present.push(m); + else media.wrongSize.push(m); + } + media.takenDownHere = wanted.filter((m) => options.down.has(m.sha256)); + if (media.missing.length > 0 && !options.report) + throw new InvariantError( + "missing_media", + `the target is missing ${media.missing.length} media file(s); copy them first`, + { missing: media.missing.map((m) => m.sha256) }, + ); + return media; +} + +/** + * The node bodies the target lacks, each verified against its content hash and validated as the + * node of every snapshot ref to it (`assertNodeBody`, invariant 2). A body the target holds is + * its own and is neither read from the bundle nor written. + */ +async function bodiesToWrite( + to: StoragePair, + bundle: SiteBundle, + snapshots: SnapshotRefs, +): Promise { + // The distinct refs to each body (most bodies appear in many snapshots). + const refs = new Map>(); + for (const s of snapshots) + for (const r of s.refs) { + const forBody = refs.get(r.hash) ?? new Map(); + forBody.set(`${r.kind}\u0000${r.id}\u0000${r.rev}`, r); + refs.set(r.hash, forBody); + } + const out: VerifiedBody[] = []; + for (const [hash, text] of Object.entries(bundle.blobs).sort(([a], [b]) => a.localeCompare(b))) { + if (!HASH.test(hash)) throw new Error(`invalid blob hash ${hash}`); + if (await to.blobs.head(`nodes/${hash}.json`)) continue; + let body: unknown; + try { + body = JSON.parse(text); + } catch { + // Never the parser's message: it can quote the body. + throw new Error(`blob ${hash} is not JSON`); + } + if ((await sha256Hex(canonicalJson(body))) !== hash) + throw new Error(`blob ${hash} does not match its content`); + assertNodeBody(bundle.siteId, hash, body, [...(refs.get(hash)?.values() ?? [])]); + out.push({ hash, text, body }); + } + return out; +} + +/** Every snapshot must be fully backed by blobs: the bundle's or the target's. */ +async function assertSnapshotsBacked( + to: StoragePair, + bundle: SiteBundle, + snapshots: SnapshotRefs, +): Promise { + for (const snap of snapshots) + for (const n of snap.refs) + if (!Object.hasOwn(bundle.blobs, n.hash) && !(await to.blobs.head(`nodes/${n.hash}.json`))) + throw new Error(`snapshot ${snap.id} references a missing blob ${n.hash}`); +} + +/** Snapshots are immutable: only the missing ones are inserted. */ +async function snapshotsToAdd( + to: StoragePair, + bundle: SiteBundle, +): Promise<{ ids: string[]; statements: SqlStatement[] }> { + const existing = new Set( + ( + await to.sql.all<{ id: string }>("SELECT id FROM snapshots WHERE site_id = ?", [ + bundle.siteId, + ]) + ).map((r) => r.id), + ); + const added = bundle.snapshots.filter((s) => !existing.has(s.id)); + return { + ids: added.map((s) => s.id), + statements: added.map((s) => ({ + sql: "INSERT INTO snapshots (id, site_id, parent_id, site_rev, body, created_at) VALUES (?, ?, ?, ?, ?, ?)", + params: [s.id, s.site_id, s.parent_id, s.site_rev, s.body, s.created_at], + })), + }; +} + +/** The project row is inserted when missing, never overwritten. */ +async function projectToAdd( + to: StoragePair, + bundle: SiteBundle, +): Promise { + const p = bundle.project; + if (!p || (await to.sql.get("SELECT id FROM projects WHERE id = ?", [p.id]))) return undefined; + return { + sql: "INSERT INTO projects (id, rev, body, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", + params: [p.id, p.rev, p.body, p.created_at, p.updated_at], + }; +} + +interface NodeChanges extends Pick { + /** The rows to write (the bundle's, with the target's publish pointer for `published: "target"`). */ + rows: NodeRow[]; + target: NodeRow[]; +} + +/** The bundle's nodes against the target's, by (id, rev, hash); a bootstrap conflict throws unless `report`. */ +async function nodeChanges( + to: StoragePair, + bundle: SiteBundle, + mode: ImportPlan["mode"], + options: ImportPlanOptions, +): Promise { + const { siteId } = bundle; + const target = await to.sql.all("SELECT * FROM nodes WHERE site_id = ? ORDER BY id", [ + siteId, + ]); + // A push keeps the deployment's publish pointer once it has one (`published: "target"`). + const rows = + options.published === "target" + ? await keepTargetPublished(bundle.nodes, target, siteId) + : bundle.nodes; + const sig = (n: NodeRow) => `${n.id}@${n.rev}:${n.hash}`; + const srcSig = new Set(rows.map(sig)); + const dstSig = new Set(target.map(sig)); + const unchanged = srcSig.size === dstSig.size && [...srcSig].every((x) => dstSig.has(x)); + const byId = new Map(target.map((n) => [n.id, n])); + const ids = new Set(rows.map((n) => n.id)); + const nodes: ImportPlan["nodes"] = { + added: rows.filter((n) => !byId.has(n.id)), + changed: rows.flatMap((n) => { + const from = byId.get(n.id); + return from && sig(from) !== sig(n) ? [{ from, to: n }] : []; + }), + removed: target.filter((n) => !ids.has(n.id)), + }; + if (unchanged || target.length === 0 || mode === "replace") + return { unchanged, nodes, rows, target }; + const conflict = [ + ...new Set([ + ...rows.filter((n) => !dstSig.has(sig(n))).map((n) => n.id), + ...target.filter((n) => !srcSig.has(sig(n))).map((n) => n.id), + ]), + ]; + if (!options.report) throw new SyncConflictError(siteId, conflict); + return { unchanged, nodes, conflict, rows, target }; +} + +/** The atomic batch (D1 takes at most 1000 statements; AGENTS.md rule 4). */ +function nodeBatch( + bundle: SiteBundle, + siteRow: NodeRow, + changes: NodeChanges, + extra: { project: SqlStatement | undefined; hosts: readonly SiteHost[] }, +): Pick { + const { siteId } = bundle; + const batch: SqlStatement[] = extra.project ? [extra.project] : []; + let inserted = 0; + let deleted = 0; + if (!changes.unchanged && !changes.conflict) { + batch.push({ sql: "DELETE FROM nodes WHERE site_id = ?", params: [siteId] }); + deleted = changes.target.length; + for (const n of changes.rows) { + batch.push({ + sql: "INSERT INTO nodes (id, site_id, kind, key, rev, hash, body, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + params: [n.id, n.site_id, n.kind, n.key, n.rev, n.hash, n.body, n.created_at, n.updated_at], + }); + inserted++; + } + } + // The derived reference index follows the node set in the same batch (RFC-0004 §4.4). + batch.push(...refsRebuild(siteId, changes.rows)); + const written = changes.rows.find((n) => n.kind === "site" && n.id === siteId) ?? siteRow; + batch.push(...hostStatements(extra.hosts, { siteId, hash: written.hash })); + if (batch.length > 1000) + throw new Error( + `site ${siteId} has too many nodes (${bundle.nodes.length}) for one atomic copy`, + ); + return { batch, inserted, deleted }; +} + +/** The bundle's takedowns against the target's blob store and ledger. */ +async function takedownsOnTarget( + to: StoragePair, + bundle: SiteBundle, + down: ReadonlySet, +): Promise { + const purged = bundle.purged ?? []; + const bytesHere: string[] = []; + for (const p of purged) + if (await to.blobs.head(mediaBlobKey(bundle.siteId, p.sha256))) bytesHere.push(p.sha256); + const recorded = purged.filter((p) => !down.has(p.sha256)).map((p) => p.sha256); + return { listed: purged.length, bytesHere, recorded }; +} + +/** Every check and target read of an import, in `importSite`'s order; nothing is written. */ +export async function prepareImport( + to: StoragePair, + bundle: SiteBundle, + options: ImportPlanOptions, +): Promise<{ plan: ImportPlan; writes: ImportWrites }> { + const mode = options.mode ?? "bootstrap"; + const { siteId } = bundle; + const siteRow = await checkBundle(bundle, options.gates); + await assertTimeZone(to, siteId, bundle); + const down = await takenDown(to, siteId); + const media = await mediaOnTarget(to, bundle, { report: options.report === true, down }); + // RFC-0034 §4: the site's hostnames must be free in the target installation. + const hosts = hostsOf(JSON.parse(siteRow.body) as Site); + await checkHosts(to.sql, hosts); + await assertIdsFree(to, bundle); + const refs = bundle.snapshots.map((s) => ({ id: s.id, refs: snapshotRefs(s) })); + const blobs = await bodiesToWrite(to, bundle, refs); + await assertSnapshotsBacked(to, bundle, refs); + const snapshots = await snapshotsToAdd(to, bundle); + const project = await projectToAdd(to, bundle); + const changes = await nodeChanges(to, bundle, mode, options); + const nodes = nodeBatch(bundle, siteRow, changes, { project, hosts }); + const ledger = await planLedger(to, bundle, { + refs, + written: new Map(blobs.map((b) => [b.hash, b.body])), + snapshotsAdded: snapshots.statements.length > 0, + }); + const takedowns = (bundle.purged ?? []).map((p) => mediaBlobKey(siteId, p.sha256)); + const plan: ImportPlan = { + siteId, + mode, + fresh: changes.target.length === 0, + unchanged: changes.unchanged, + nodes: changes.nodes, + ...(changes.conflict ? { conflict: changes.conflict } : {}), + blobs: blobs.length, + snapshots: snapshots.ids, + projectAdded: project !== undefined, + media, + takedowns: await takedownsOnTarget(to, bundle, down), + }; + return { + plan, + writes: { blobs, snapshots: snapshots.statements, ...nodes, ledger, takedowns }, + }; +} + +/** + * What `importSite(to, bundle, options)` would change, with every refusal it would throw; writes + * nothing. With `report: true` a bootstrap conflict and missing media are returned instead. + */ +export async function planImport( + to: StoragePair, + bundle: SiteBundle, + options: ImportPlanOptions = {}, +): Promise { + return (await prepareImport(to, bundle, options)).plan; +} diff --git a/packages/core-ts/src/sync-published.ts b/packages/core-ts/src/sync-published.ts index 83628319..73aefe3f 100644 --- a/packages/core-ts/src/sync-published.ts +++ b/packages/core-ts/src/sync-published.ts @@ -6,7 +6,7 @@ */ import type { Site } from "@cloudhal/core-contracts"; import { canonicalJson, nodeHash } from "./hash.ts"; -import type { NodeRow } from "./sync.ts"; +import type { NodeRow } from "./sync-bundle.ts"; const VOLATILE = new Set(["rev", "updatedAt"]); diff --git a/packages/core-ts/src/sync.ts b/packages/core-ts/src/sync.ts index 4ec66b15..fcb93c14 100644 --- a/packages/core-ts/src/sync.ts +++ b/packages/core-ts/src/sync.ts @@ -1,267 +1,45 @@ /** * Copies one site — project row, nodes, snapshots and the blobs they reference — from one storage * pair to another. This is what `cloudhal push` (local → Cloudflare) and `cloudhal pull` - * (Cloudflare → local) are built on; it only uses the `SqlDriver`/`BlobStore` interfaces, so any - * combination of SQLite, D1 and fs/R2 works. + * (Cloudflare → local) are built on, and `cloudhal import` applies an archive's bundle with it; it + * only uses the `SqlDriver`/`BlobStore` interfaces, so any combination of SQLite, D1 and fs/R2 + * works. * * Safety: the target is the source of truth once a site is deployed (agents edit it there). The * copy therefore refuses to overwrite a target whose nodes differ from the source unless - * `mode: "replace"` is given. Snapshots are immutable and only ever added. + * `mode: "replace"` is given. Snapshots are immutable and only ever added. Every check runs before + * the first write (`sync-plan.ts`), so a refused bundle leaves the target as it was. + * + * The bundle format lives in `sync-bundle.ts`, its checks in `sync-checks.ts`, the plan of an + * import in `sync-plan.ts` (its media ledger in `sync-ledger.ts`); this module re-exports the + * public parts. */ -import { type Asset, assertValid, type Site } from "@cloudhal/core-contracts"; +import type { Asset } from "@cloudhal/core-contracts"; import type { BlobStore, SqlDriver, SqlStatement } from "@cloudhal/storage"; import { mediaBlobKey } from "./assets.ts"; -import { - checkAvailability, - checkTimeZone, - EXPIRY_GATES_VERSION, - type GateSupport, - isTimeZone, -} from "./availability.ts"; import { InvariantError } from "./errors.ts"; -import { canonicalJson, nodeHash, sha256Hex } from "./hash.ts"; -import { checkHosts, hostStatements, hostsOf } from "./hosts.ts"; -import { refsRebuild } from "./media/refs-rebuild.ts"; -import { validateMenuTree } from "./menu.ts"; -import { isNodeKind, NODE_SPEC, type NodeKind, type NodeTypes } from "./nodes.ts"; -import { keepTargetPublished } from "./sync-published.ts"; - -/** Gates in a bundle follow the write rules (§4.1) and the writer guard (S15). */ -async function assertGates( - siteId: string, - bundle: SiteBundle, - gates: GateSupport | undefined, -): Promise { - let gated = false; - for (const row of bundle.nodes) { - if (row.kind !== "page" && row.kind !== "contentEntry") continue; - const availability = (JSON.parse(row.body) as { availability?: { from?: string } }) - .availability; - if (!availability) continue; - checkAvailability(availability); - gated = true; - } - if (gated && gates && !(await gates.enforced(siteId))) - throw new InvariantError( - "gates_unsupported", - "the bundle has time gates (availability) and this site's renderer has not confirmed it enforces them", - { siteId }, - ); - // RFC-0042 §3.6: a collection's `expiry` needs a renderer with derived gates. - const site = bundle.nodes.find((r) => r.kind === "site"); - const collections = site - ? (JSON.parse(site.body) as { collections?: Record }).collections - : undefined; - const expiring = Object.values(collections ?? {}).some((c) => c?.expiry !== undefined); - if (expiring && gates && !(await gates.enforced(siteId, EXPIRY_GATES_VERSION))) - throw new InvariantError( - "gates_unsupported", - "the bundle has a collection expiry and this site's renderer has not confirmed it enforces derived gates", - { siteId }, - ); -} - -/** - * `Site.timeZone` must be a zone this runtime knows (RFC-0008 §4.1), as on every other write; the - * zone the target already stores for the site is accepted unchanged. - */ -async function assertTimeZone(to: StoragePair, siteId: string, bundle: SiteBundle): Promise { - const row = bundle.nodes.find((n) => n.kind === "site" && n.id === siteId); - const zone = row ? (JSON.parse(row.body) as { timeZone?: string }).timeZone : undefined; - if (zone === undefined || isTimeZone(zone)) return; - const stored = await to.sql.get<{ body: string }>( - "SELECT body FROM nodes WHERE id = ? AND kind = 'site'", - [siteId], - ); - checkTimeZone( - zone, - stored ? (JSON.parse(stored.body) as { timeZone?: string }).timeZone : undefined, - ); -} - -/** - * A bundle is untrusted input (whoever holds the deployment token): every row must be a valid - * contract document of the kind its `kind` column declares (RFC-0002 §4.1: the node schemas are - * mutually exclusive, so a body cannot pass as another kind), whose columns agree with its body, - * for this site only. - */ -async function assertValidRows(siteId: string, bundle: SiteBundle): Promise { - for (const row of bundle.nodes) { - if (!isNodeKind(row.kind)) throw new Error(`node ${row.id}: unknown kind ${row.kind}`); - const kind: NodeKind = row.kind; - const spec = NODE_SPEC[kind] as { - schema: (typeof NODE_SPEC)[NodeKind]["schema"]; - key(n: NodeTypes[NodeKind]): string; - siteId(n: NodeTypes[NodeKind]): string; - }; - const schema = spec.schema; - let body: NodeTypes[NodeKind]; - try { - body = JSON.parse(row.body) as NodeTypes[NodeKind]; - } catch { - throw new Error(`node ${row.id}: body is not JSON`); - } - try { - assertValid(schema, body); - } catch (e) { - throw new Error( - `node ${row.id}: not a valid ${schema} (${e instanceof Error ? e.message.slice(0, 200) : e})`, - ); - } - if ( - body.id !== row.id || - body.rev !== row.rev || - spec.siteId(body) !== row.site_id || - row.site_id !== siteId || - spec.key(body) !== row.key - ) - throw new Error(`node ${row.id}: columns do not match its body`); - if ((await nodeHash(body)) !== row.hash) - throw new Error(`node ${row.id}: hash does not match its body`); - if (kind === "menu") { - // RFC-0003 §4.2: the tree rules the schema cannot express; targets are not checked (hidden - // items are a legal state), and labels of a locale the site dropped are kept. - const siteRow = bundle.nodes.find((n) => n.kind === "site" && n.id === siteId); - const menu = body as NodeTypes["menu"]; - try { - if (siteRow) validateMenuTree(menu, JSON.parse(siteRow.body), menu); - } catch (e) { - throw new Error(`node ${row.id}: ${e instanceof Error ? e.message : e}`); - } - } - } - for (const row of bundle.snapshots) { - let body: { id?: string; siteId?: string }; - try { - body = JSON.parse(row.body) as { id?: string; siteId?: string }; - } catch { - throw new Error(`snapshot ${row.id}: body is not JSON`); - } - try { - assertValid("snapshot", body); - } catch { - throw new Error(`snapshot ${row.id}: not a valid snapshot`); - } - if (body.id !== row.id || body.siteId !== siteId || row.site_id !== siteId) - throw new Error(`snapshot ${row.id}: columns do not match its body`); - } -} - -export interface StoragePair { - sql: SqlDriver; - blobs: BlobStore; -} - -export interface CopySiteOptions { - /** - * `bootstrap` (default): copy when the target has no such site or already holds exactly the same - * nodes; otherwise throw `SyncConflictError`. `replace`: make the target's nodes equal the source. - */ - mode?: "bootstrap" | "replace"; - log?: (line: string) => void; - /** - * RFC-0008 S15: when given, a bundle whose pages or entries carry `availability` is refused - * (`gates_unsupported`) unless the target's renderer enforces gates. The admin Worker passes its - * store's check; local targets have the renderer in the same process. - */ - gates?: GateSupport; - /** - * Whose `publishedSnapshotId` a written site node carries. `source` (default, `pull`): the - * bundle's, so a local copy mirrors the deployment. `target` (`push`): once the target has - * published, its pointer is kept, so a push carries content but never moves production back to - * an older snapshot (publishing stays a person's action in Studio); a target that never - * published takes the bundle's, as a first deploy does. - */ - published?: "source" | "target"; -} - -export interface CopySiteResult { - nodes: { inserted: number; deleted: number; unchanged: boolean }; - snapshots: number; - blobs: number; -} - -export class SyncConflictError extends Error { - override readonly name = "SyncConflictError"; - readonly differing: string[]; - constructor(siteId: string, differing: string[]) { - super( - `target already has site ${siteId} with different content (${differing.length} node(s) differ: ${differing.slice(0, 5).join(", ")}${differing.length > 5 ? ", …" : ""}); pull first or pass mode "replace"`, - ); - this.differing = differing; - } -} - -export type NodeRow = { - id: string; - site_id: string; - kind: string; - key: string; - rev: number; - hash: string; - body: string; - created_at: string; - updated_at: string; -}; -export type SnapshotRow = { - id: string; - site_id: string; - parent_id: string | null; - site_rev: number; - body: string; - created_at: string; -}; -export type ProjectRow = { - id: string; - rev: number; - body: string; - created_at: string; - updated_at: string; -}; +import { + type BundleMedia, + type CopySiteOptions, + type CopySiteResult, + type NodeRow, + type ProjectRow, + SITE_BUNDLE_FORMAT, + type SiteBundle, + type SnapshotRow, + type StoragePair, +} from "./sync-bundle.ts"; +import { prepareImport } from "./sync-plan.ts"; + +export * from "./sync-bundle.ts"; +export { type ImportPlan, type ImportPlanOptions, planImport } from "./sync-plan.ts"; /** Splits statements into batches that respect D1's per-batch limit. */ async function runBatches(sql: SqlDriver, statements: SqlStatement[], size = 500): Promise { for (let i = 0; i < statements.length; i += size) await sql.batch(statements.slice(i, i + size)); } -/** A self-contained, JSON-serialisable copy of one site (what push/pull send over HTTP). */ -export interface SiteBundle { - format: "cloudhal.site-bundle@1"; - siteId: string; - project?: ProjectRow | undefined; - nodes: NodeRow[]; - snapshots: SnapshotRow[]; - /** Node bodies referenced by the snapshots, keyed by content hash (UTF-8 JSON text). */ - blobs: Record; - /** - * RFC-0004 §4.7: content the bundle's blob assets (current and in snapshots) need. The bytes - * travel separately (`copyMediaBlobs`, or the admin media routes); a target that lacks them - * refuses the bundle (`missing_media`). - */ - media?: BundleMedia[]; - /** Taken-down content: travels as ledger rows, so a copy can never bring the bytes back. */ - purged?: BundlePurge[]; -} - -export interface BundleMedia { - sha256: string; - bytes: number; - mediaType: string; -} - -export interface BundlePurge { - sha256: string; - reason: string; - legal: boolean; - by: string; - at: string; -} - -const SHA256 = /^[0-9a-f]{64}$/; - -export const SITE_BUNDLE_FORMAT = "cloudhal.site-bundle@1" as const; - /** Reads everything needed to recreate `siteId` elsewhere. */ export async function exportSite( from: StoragePair, @@ -306,6 +84,8 @@ export async function exportSite( [siteId], ); const purged = new Set(purgedRows.map((r) => r.sha256)); + // packages/cli/src/commands/archive-bundle.ts bundleMedia rebuilds this from an archive; keep + // the two in step (export-cli.test.ts compares them). const media = new Map(); const addAsset = (body: string) => { const a = JSON.parse(body) as Asset; @@ -375,131 +155,42 @@ export async function copyMediaBlobs( } /** - * Applies a bundle to `to`. Blobs and snapshots land first, then the project row and the node set - * in one atomic batch. Refuses to overwrite diverged nodes unless `mode: "replace"`. + * Applies a bundle to `to`. Every check and read runs first (`prepareImport`), so a refused bundle + * writes nothing; then node bodies and snapshots land, then the project row and the node set in + * one atomic batch, then the media ledger, all planned. Only a storage failure can stop it + * between those writes; the next import of the bundle completes it. Refuses to overwrite diverged + * nodes unless `mode: "replace"`. */ export async function importSite( to: StoragePair, bundle: SiteBundle, options: CopySiteOptions = {}, ): Promise { - if (bundle.format !== SITE_BUNDLE_FORMAT) - throw new Error(`unsupported bundle format ${bundle.format}`); const log = options.log ?? (() => {}); - const mode = options.mode ?? "bootstrap"; - const { siteId } = bundle; - if (!bundle.nodes.some((n) => n.kind === "site" && n.id === siteId)) - throw new Error(`bundle has no site node for ${siteId}`); - if ( - bundle.nodes.some((n) => n.site_id !== siteId) || - bundle.snapshots.some((s) => s.site_id !== siteId) - ) - throw new Error("bundle contains rows of another site"); - await assertValidRows(siteId, bundle); - await assertGates(siteId, bundle, options.gates); - await assertTimeZone(to, siteId, bundle); - await assertMedia(to, bundle); - // RFC-0034 §4: the site's hostnames must be free in the target installation. - const siteRow = bundle.nodes.find((n) => n.kind === "site" && n.id === siteId) as NodeRow; - const hosts = hostsOf(JSON.parse(siteRow.body) as Site); - await checkHosts(to.sql, hosts); + const { plan, writes } = await prepareImport(to, bundle, { ...options, report: false }); - // 1. Blobs: verify content addressing so a bundle cannot plant mismatched bodies. - let blobs = 0; + // 1. Node bodies the target lacks, verified against their content hash by the plan. const encoder = new TextEncoder(); - for (const [hash, text] of Object.entries(bundle.blobs).sort(([a], [b]) => a.localeCompare(b))) { - if (!/^[0-9a-f]{64}$/.test(hash)) throw new Error(`invalid blob hash ${hash}`); - const key = `nodes/${hash}.json`; - if (await to.blobs.head(key)) continue; - if ((await sha256Hex(canonicalJson(JSON.parse(text)))) !== hash) - throw new Error(`blob ${hash} does not match its content`); - await to.blobs.put(key, encoder.encode(text), { contentType: "application/json" }); - blobs++; - } - // Every snapshot must be fully backed by blobs on the target. - for (const snap of bundle.snapshots) - for (const n of (JSON.parse(snap.body) as { nodes: { hash: string }[] }).nodes) - if (!(n.hash in bundle.blobs) && !(await to.blobs.head(`nodes/${n.hash}.json`))) - throw new Error(`snapshot ${snap.id} references a missing blob ${n.hash}`); - log(`blobs: ${blobs} uploaded`); - - // 2. Snapshots: immutable, insert the missing ones. - const existingSnaps = new Set( - (await to.sql.all<{ id: string }>("SELECT id FROM snapshots WHERE site_id = ?", [siteId])).map( - (r) => r.id, - ), - ); - const snapInserts: SqlStatement[] = bundle.snapshots - .filter((s) => !existingSnaps.has(s.id)) - .map((s) => ({ - sql: "INSERT INTO snapshots (id, site_id, parent_id, site_rev, body, created_at) VALUES (?, ?, ?, ?, ?, ?)", - params: [s.id, s.site_id, s.parent_id, s.site_rev, s.body, s.created_at], - })); - - // 3. Project row (insert if missing; never overwritten). - const statements: SqlStatement[] = []; - if ( - bundle.project && - !(await to.sql.get("SELECT id FROM projects WHERE id = ?", [bundle.project.id])) - ) { - const p = bundle.project; - statements.push({ - sql: "INSERT INTO projects (id, rev, body, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", - params: [p.id, p.rev, p.body, p.created_at, p.updated_at], + for (const b of writes.blobs) + await to.blobs.put(`nodes/${b.hash}.json`, encoder.encode(b.text), { + contentType: "application/json", }); - } - - // 4. Nodes: compare by (id, rev, hash). - const dstNodes = await to.sql.all("SELECT * FROM nodes WHERE site_id = ? ORDER BY id", [ - siteId, - ]); - // A push keeps the deployment's publish pointer once it has one (`published: "target"`). - const rows = - options.published === "target" - ? await keepTargetPublished(bundle.nodes, dstNodes, siteId) - : bundle.nodes; - const sig = (n: NodeRow) => `${n.id}@${n.rev}:${n.hash}`; - const srcSig = new Set(rows.map(sig)); - const dstSig = new Set(dstNodes.map(sig)); - const unchanged = srcSig.size === dstSig.size && [...srcSig].every((x) => dstSig.has(x)); - let inserted = 0; - let deleted = 0; - if (!unchanged) { - if (dstNodes.length > 0 && mode !== "replace") { - const differing = [ - ...rows.filter((n) => !dstSig.has(sig(n))).map((n) => n.id), - ...dstNodes.filter((n) => !srcSig.has(sig(n))).map((n) => n.id), - ]; - throw new SyncConflictError(siteId, [...new Set(differing)]); - } - statements.push({ sql: "DELETE FROM nodes WHERE site_id = ?", params: [siteId] }); - deleted = dstNodes.length; - for (const n of rows) { - statements.push({ - sql: "INSERT INTO nodes (id, site_id, kind, key, rev, hash, body, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", - params: [n.id, n.site_id, n.kind, n.key, n.rev, n.hash, n.body, n.created_at, n.updated_at], - }); - inserted++; - } - } - - await runBatches(to.sql, snapInserts); - // The derived reference index follows the node set in the same batch (RFC-0004 §4.4). - statements.push(...refsRebuild(siteId, rows)); - const written = rows.find((n) => n.kind === "site" && n.id === siteId) ?? siteRow; - statements.push(...hostStatements(hosts, { siteId, hash: written.hash })); - if (statements.length > 0) { - if (statements.length > 1000) - throw new Error( - `site ${siteId} has too many nodes (${bundle.nodes.length}) for one atomic copy`, - ); - await to.sql.batch(statements); - } - await importMediaLedger(to, bundle, snapInserts.length > 0); + log(`blobs: ${writes.blobs.length} uploaded`); + // 2. Snapshots: immutable, the missing ones. + await runBatches(to.sql, writes.snapshots); + // 3. The project row (if missing), the node set, its reference index and hostnames: one batch. + if (writes.batch.length > 0) await to.sql.batch(writes.batch); + // 4. The media ledger and `snapshot_blobs`; taken-down bytes never survive a copy. + await runBatches(to.sql, writes.ledger); + for (const key of writes.takedowns) await to.blobs.delete(key); log( - `snapshots: ${snapInserts.length} added; nodes: ${unchanged ? "unchanged" : `${inserted} written, ${deleted} replaced`}`, + `snapshots: ${writes.snapshots.length} added; nodes: ${plan.unchanged ? "unchanged" : `${writes.inserted} written, ${writes.deleted} replaced`}`, ); - return { nodes: { inserted, deleted, unchanged }, snapshots: snapInserts.length, blobs }; + return { + nodes: { inserted: writes.inserted, deleted: writes.deleted, unchanged: plan.unchanged }, + snapshots: writes.snapshots.length, + blobs: writes.blobs.length, + }; } /** `importSite(to, exportSite(from))`, skipping blobs the target already has. */ @@ -516,112 +207,3 @@ export async function copySite( await copyMediaBlobs(from.blobs, to.blobs, siteId, bundle.media ?? []); return importSite(to, bundle, options); } - -/** - * RFC-0004 §4.7: every listed content must be in the target's blob store already (copied first), - * and every current blob asset must be listed (or taken down). Otherwise `missing_media`. - */ -async function assertMedia(to: StoragePair, bundle: SiteBundle): Promise { - const listed = new Set(); - for (const m of bundle.media ?? []) { - if ( - !SHA256.test(m.sha256) || - !Number.isInteger(m.bytes) || - m.bytes < 0 || - typeof m.mediaType !== "string" || - m.mediaType.length > 255 - ) - throw new Error("bundle media entry is malformed"); - listed.add(m.sha256); - } - const purged = new Set(); - for (const p of bundle.purged ?? []) { - if (!SHA256.test(p.sha256)) throw new Error("bundle takedown entry is malformed"); - purged.add(p.sha256); - } - for (const n of bundle.nodes) - if (n.kind === "asset") { - const a = JSON.parse(n.body) as Asset; - if (a.storage.kind === "blob" && !listed.has(a.sha256) && !purged.has(a.sha256)) - throw new InvariantError( - "missing_media", - `asset ${a.id} needs content the bundle does not list`, - { - sha256: a.sha256, - }, - ); - } - const missing: string[] = []; - for (const sha of listed) - if (!purged.has(sha) && !(await to.blobs.head(mediaBlobKey(bundle.siteId, sha)))) - missing.push(sha); - if (missing.length > 0) - throw new InvariantError( - "missing_media", - `the target is missing ${missing.length} media file(s); copy them first`, - { missing }, - ); -} - -/** Ledger rows for the bundle's media and takedowns, and `snapshot_blobs` for its snapshots. */ -async function importMediaLedger( - to: StoragePair, - bundle: SiteBundle, - snapshotsAdded: boolean, -): Promise { - const { siteId } = bundle; - const ts = new Date().toISOString(); - const statements: SqlStatement[] = []; - for (const m of bundle.media ?? []) - statements.push({ - sql: `INSERT INTO media_blobs (site_id, sha256, bytes, media_type, state, created_at, updated_at) - VALUES (?, ?, ?, ?, 'stored', ?, ?) - ON CONFLICT (site_id, sha256) DO UPDATE SET state = 'stored', updated_at = excluded.updated_at - WHERE media_blobs.state IN ('uploading', 'deleting')`, - params: [siteId, m.sha256, m.bytes, m.mediaType, ts, ts], - }); - for (const p of bundle.purged ?? []) - statements.push({ - sql: `INSERT INTO media_blobs (site_id, sha256, bytes, media_type, state, purge_reason, purge_legal, purged_by, purged_at, created_at, updated_at) - VALUES (?, ?, 0, 'application/octet-stream', 'purged', ?, ?, ?, ?, ?, ?) - ON CONFLICT (site_id, sha256) DO UPDATE SET state = 'purged', lease_until = NULL, - purge_reason = excluded.purge_reason, purge_legal = excluded.purge_legal, - purged_by = excluded.purged_by, purged_at = excluded.purged_at, updated_at = excluded.updated_at`, - params: [ - siteId, - p.sha256, - p.reason.slice(0, 1000), - p.legal ? 1 : 0, - p.by.slice(0, 200), - p.at, - ts, - ts, - ], - }); - if (snapshotsAdded || (bundle.media ?? []).length > 0) { - // Which content each snapshot captured: from its asset node bodies. - const assetBodies = new Map(); - for (const [hash, text] of Object.entries(bundle.blobs)) assetBodies.set(hash, text); - for (const snap of bundle.snapshots) { - const refs = (JSON.parse(snap.body) as { nodes: { kind: string; hash: string }[] }).nodes; - const shas = new Set(); - for (const r of refs) { - if (r.kind !== "asset") continue; - const text = - assetBodies.get(r.hash) ?? - new TextDecoder().decode((await to.blobs.get(`nodes/${r.hash}.json`)) ?? undefined); - if (!text) continue; - const a = JSON.parse(text) as Asset; - if (a.storage.kind === "blob") shas.add(a.sha256); - } - if (shas.size > 0) - statements.push({ - sql: "INSERT OR IGNORE INTO snapshot_blobs (snapshot_id, site_id, sha256) SELECT ?, ?, value FROM json_each(?)", - params: [snap.id, siteId, JSON.stringify([...shas])], - }); - } - } - await runBatches(to.sql, statements); - // Taken-down bytes never survive a copy. - for (const p of bundle.purged ?? []) await to.blobs.delete(mediaBlobKey(siteId, p.sha256)); -} diff --git a/packages/core-ts/test/sync-plan.test.ts b/packages/core-ts/test/sync-plan.test.ts new file mode 100644 index 00000000..8cd43b72 --- /dev/null +++ b/packages/core-ts/test/sync-plan.test.ts @@ -0,0 +1,382 @@ +/** + * `planImport` (RFC-0039 invariants 2, 3 and 6): what an import would change, with every refusal + * `importSite` throws, and no write; and `importSite` refusing before its first write (node bodies + * that are not the nodes their snapshot refs name, ids another site holds, rows the node batch's + * keys refuse, malformed takedowns), so a refused push or pull leaves the target as it was. + */ +import type { ContentEntry, NodeRef, Page, Site } from "@cloudhal/core-contracts"; +import { MemoryBlobStore } from "@cloudhal/storage"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { describe, expect, it } from "vitest"; +import { mediaBlobKey } from "../src/assets.ts"; +import { InvariantError } from "../src/errors.ts"; +import { canonicalJson, nodeHash, sha256Hex } from "../src/hash.ts"; +import { makePng, PNG_1X1 } from "../src/media/fixtures.ts"; +import { uniquenessKey } from "../src/nodes.ts"; +import { seedStarterSite } from "../src/seed.ts"; +import { SiteGraphStore } from "../src/store.ts"; +import { + copySite, + exportSite, + importSite, + type NodeRow, + planImport, + type SiteBundle, + SyncConflictError, +} from "../src/sync.ts"; + +const actor = { kind: "human" as const, id: "user_plan000001" }; + +async function pair() { + const sql = await BetterSqlite3Driver.open(":memory:"); + const blobs = new MemoryBlobStore(); + const store = new SiteGraphStore({ sql, blobs }); + await store.init(); + return { sql, blobs, store }; +} + +type Pair = Awaited>; + +async function counts(p: Pair) { + const n = async (table: string) => + (await p.sql.get<{ n: number }>(`SELECT count(*) AS n FROM ${table}`))?.n; + return { + nodes: await n("nodes"), + snapshots: await n("snapshots"), + projects: await n("projects"), + blobs: await p.blobs.list(), + }; +} + +async function upload(p: Pair, siteId: string, bytes: Uint8Array, filename: string) { + const { asset } = await p.store.ingestAsset({ + siteId, + bytes, + filename, + uploadedBy: actor, + source: { kind: "upload" }, + }); + return asset; +} + +async function editAbout(p: Pair, siteId: string, title: string): Promise { + const about = (await p.store.listPages(siteId)).find((x) => x.path === "/about") as Page; + return p.store.updatePage({ ...about, title }); +} + +const refsOf = (bundle: SiteBundle, snapshotId: string): NodeRef[] => + ( + JSON.parse(bundle.snapshots.find((s) => s.id === snapshotId)?.body ?? "{}") as { + nodes: NodeRef[]; + } + ).nodes; + +/** `bundle` with one ref of snapshot `snapshotId` pointing at `body`, added to the blobs. */ +async function pointAt(bundle: SiteBundle, snapshotId: string, refId: string, body: unknown) { + const text = canonicalJson(body); + const hash = await sha256Hex(text); + const snapshots = bundle.snapshots.map((s) => { + if (s.id !== snapshotId) return s; + const snap = JSON.parse(s.body) as { nodes: NodeRef[] }; + for (const ref of snap.nodes) if (ref.id === refId) ref.hash = hash; + return { ...s, body: JSON.stringify(snap) }; + }); + return { hash, bundle: { ...bundle, snapshots, blobs: { ...bundle.blobs, [hash]: text } } }; +} + +/** A row with another id (and `edit`ed body), its key and hash recomputed. */ +async function rowWith(row: NodeRow, id: string, edit: (body: Page) => void = () => {}) { + const body = { ...(JSON.parse(row.body) as Page), id }; + edit(body); + return { + ...row, + id, + key: uniquenessKey("page", body), + body: canonicalJson(body), + hash: await nodeHash(body), + }; +} + +describe("planImport", () => { + it("plans a fresh import without writing anything", async () => { + const src = await pair(); + const site = await seedStarterSite(src.store, { name: "Plan", slug: "plan" }); + const asset = await upload(src, site.id, PNG_1X1, "dot.png"); + const entry = (await src.store.listContentEntries(site.id))[0] as ContentEntry; + await src.store.updateContentEntry({ + ...entry, + coverImage: { assetId: asset.id, alt: "" } as NonNullable, + }); + const snap = await src.store.createSnapshot(site.id, { actor, reason: "manual" }); + const bundle = await exportSite(src, site.id); + const dst = await pair(); + const empty = await counts(dst); + + const plan = await planImport(dst, bundle, { report: true }); + expect(plan).toMatchObject({ + siteId: site.id, + mode: "bootstrap", + fresh: true, + unchanged: false, + snapshots: [snap.id], + blobs: Object.keys(bundle.blobs).length, + projectAdded: true, + takedowns: { listed: 0, bytesHere: [] }, + }); + expect(plan.conflict).toBeUndefined(); + expect(plan.nodes.added.map((n) => n.id).sort()).toEqual(bundle.nodes.map((n) => n.id).sort()); + expect(plan.nodes.changed).toEqual([]); + expect(plan.nodes.removed).toEqual([]); + expect(plan.media.missing.map((m) => m.sha256)).toEqual([asset.sha256]); + expect(await counts(dst)).toEqual(empty); + expect(empty).toMatchObject({ nodes: 0, snapshots: 0, projects: 0, blobs: [] }); + // Without `report`, missing media throw as importSite does. + await expect(planImport(dst, bundle)).rejects.toMatchObject({ code: "missing_media" }); + }); + + it("refuses a diverged bootstrap before any write; report returns the conflict (invariant 6)", async () => { + const src = await pair(); + const dst = await pair(); + const site = await seedStarterSite(src.store, { name: "Plan", slug: "plan" }); + await src.store.createSnapshot(site.id, { actor, reason: "manual" }); + await copySite(src, dst, site.id); + const there = await editAbout(dst, site.id, "Edited on the target"); + await editAbout(src, site.id, "Edited here"); + const here = await editAbout(src, site.id, "Edited here again"); + await src.store.createSnapshot(site.id, { actor, reason: "manual" }); + const bundle = await exportSite(src, site.id); + const before = await counts(dst); + + const error = await importSite(dst, bundle).catch((e: unknown) => e); + expect(error).toBeInstanceOf(SyncConflictError); + // No node body and no snapshot of the refused bundle landed. + expect(await counts(dst)).toEqual(before); + expect(await dst.blobs.list("nodes/")).toEqual(before.blobs); + + const plan = await planImport(dst, bundle, { report: true }); + expect(plan.conflict).toEqual((error as SyncConflictError).differing); + expect(plan.conflict).toContain(there.id); + expect(plan.fresh).toBe(false); + expect(plan.snapshots).toHaveLength(1); + expect(plan.nodes.changed.map((c) => [c.to.id, c.from.rev, c.to.rev])).toContainEqual([ + here.id, + there.rev, + here.rev, + ]); + expect(here.rev).not.toBe(there.rev); + await expect(planImport(dst, bundle)).rejects.toBeInstanceOf(SyncConflictError); + expect(await counts(dst)).toEqual(before); + }); + + it("reports media present, missing, of another size and taken down on the target", async () => { + const src = await pair(); + const dst = await pair(); + const site = await seedStarterSite(src.store, { name: "Media", slug: "media" }); + const a = await upload(src, site.id, PNG_1X1, "a.png"); + const b = await upload(src, site.id, makePng({ width: 2, height: 2 }), "b.png"); + const c = await upload(src, site.id, makePng({ width: 3, height: 2 }), "c.png"); + const d = await upload(src, site.id, makePng({ width: 4, height: 2 }), "d.png"); + await src.store.createSnapshot(site.id, { actor, reason: "manual" }); + // D stays in the snapshot only, so taking it down on the target changes no node. + await src.store.deleteAsset(d); + await copySite(src, dst, site.id); + await dst.blobs.delete(mediaBlobKey(site.id, b.sha256)); + await dst.blobs.put(mediaBlobKey(site.id, c.sha256), new Uint8Array([1, 2, 3])); + await dst.store.purgeBlob(site.id, d.sha256, actor, { text: "notice", legal: false }); + const bundle = await exportSite(src, site.id); + + const plan = await planImport(dst, bundle, { report: true }); + const shas = (list: { sha256: string }[]) => list.map((m) => m.sha256).sort(); + expect(plan.unchanged).toBe(true); + expect(shas(plan.media.present)).toEqual([a.sha256]); + expect(shas(plan.media.missing)).toEqual([b.sha256, d.sha256].sort()); + expect(shas(plan.media.wrongSize)).toEqual([c.sha256]); + expect(shas(plan.media.takenDownHere)).toEqual([d.sha256]); + const error = await importSite(dst, bundle).catch((e: unknown) => e); + expect(error).toBeInstanceOf(InvariantError); + expect(error).toMatchObject({ + code: "missing_media", + message: "the target is missing 2 media file(s); copy them first", + }); + }); + + it("refuses a node body that is not JSON without quoting it, before any write", async () => { + const src = await pair(); + const site = await seedStarterSite(src.store, { name: "Bodies", slug: "bodies" }); + await src.store.createSnapshot(site.id, { actor, reason: "manual" }); + const bundle = await exportSite(src, site.id); + const [hash] = Object.keys(bundle.blobs).sort() as [string]; + const crafted = { ...bundle, blobs: { ...bundle.blobs, [hash]: "\u001b[31m{" } }; + const dst = await pair(); + const error = (await importSite(dst, crafted).catch((e: unknown) => e)) as Error; + expect(error.message).toBe(`blob ${hash} is not JSON`); + expect(error.message).not.toMatch(/\p{Cc}/u); + expect(await counts(dst)).toMatchObject({ nodes: 0, snapshots: 0, blobs: [] }); + }); + + it("refuses a node body a snapshot references that is not the node its ref names (invariant 2)", async () => { + const src = await pair(); + const site = await seedStarterSite(src.store, { name: "Bodies", slug: "bodies" }); + const asset = await upload(src, site.id, PNG_1X1, "dot.png"); + const v1 = await src.store.createSnapshot(site.id, { actor, reason: "publish" }); + await src.store.publish((await src.store.getSite(site.id)) as Site, v1.id); + // v2 keeps the original bodies referenced once v1's refs are pointed elsewhere. + await src.store.createSnapshot(site.id, { actor, reason: "manual" }); + const bundle = await exportSite(src, site.id); + const [first, second] = refsOf(bundle, v1.id).filter((r) => r.kind === "page") as [ + NodeRef, + NodeRef, + ]; + const secondBody = JSON.parse(bundle.blobs[second.hash] as string) as unknown; + const dst = await pair(); + await dst.blobs.put(mediaBlobKey(site.id, asset.sha256), PNG_1X1); + const before = await counts(dst); + const cases = [ + // The published snapshot's page: what the renderer would read. + [ + first.id, + { id: "not-a-page", title: 42, blocks: "" }, + "is not a valid page", + ], + // A valid page, but another one. + [first.id, secondBody, "is not the page its snapshot ref names"], + // An asset the media ledger would read after the node set landed. + [asset.id, {}, "is not a valid asset"], + ] as const; + for (const [refId, body, what] of cases) { + const crafted = await pointAt(bundle, v1.id, refId, body); + for (const run of [ + () => importSite(dst, crafted.bundle), + () => planImport(dst, crafted.bundle, { report: true }), + ]) { + const error = (await run().catch((e: unknown) => e)) as Error; + expect(error.message).toBe(`blob ${crafted.hash} ${what}`); + } + expect(await counts(dst)).toEqual(before); + } + // A body no snapshot references. + const stray = canonicalJson({ stray: true }); + const strayHash = await sha256Hex(stray); + await expect( + importSite(dst, { ...bundle, blobs: { ...bundle.blobs, [strayHash]: stray } }), + ).rejects.toThrow(`blob ${strayHash} is not referenced by any snapshot`); + expect(await counts(dst)).toEqual(before); + expect(before).toMatchObject({ nodes: 0, snapshots: 0, projects: 0 }); + // The bundle itself imports. + await importSite(dst, bundle); + }); + + it("refuses ids another site holds, rows the node batch's keys refuse and malformed takedowns, before any write", async () => { + const src = await pair(); + const site = await seedStarterSite(src.store, { name: "Rows", slug: "rows" }); + await src.store.createSnapshot(site.id, { actor, reason: "manual" }); + const bundle = await exportSite(src, site.id); + const dst = await pair(); + const other = await seedStarterSite(dst.store, { name: "Other", slug: "other" }); + const theirs = await dst.store.createSnapshot(other.id, { actor, reason: "manual" }); + const theirPage = (await dst.store.listPages(other.id))[0] as Page; + const before = await counts(dst); + const page = bundle.nodes.find((n) => n.kind === "page") as NodeRow; + const nodes = (...extra: NodeRow[]) => [...bundle.nodes.filter((n) => n !== page), ...extra]; + const [snap] = bundle.snapshots as [SiteBundle["snapshots"][number]]; + const snapBody = { ...(JSON.parse(snap.body) as object), id: theirs.id }; + const group = { group: page.id, method: "original" } as const; + const cases: [Partial, string][] = [ + [ + { nodes: nodes(await rowWith(page, theirPage.id)) }, + `node ${theirPage.id} belongs to another site of this installation`, + ], + [ + { snapshots: [{ ...snap, id: theirs.id, body: JSON.stringify(snapBody) }] }, + `snapshot ${theirs.id} belongs to another site of this installation`, + ], + [ + { nodes: nodes(page, await rowWith(page, "page_dup0000001")) }, + "node page_dup0000001: another page has its key", + ], + [ + { + nodes: nodes( + await rowWith(page, page.id, (b) => { + b.translation = group; + }), + await rowWith(page, "page_tr00000002", (b) => { + b.path = "/copy"; + b.translation = { ...group, method: "human" }; + }), + ), + }, + "node page_tr00000002: another page of its translation group has its locale", + ], + [{ snapshots: [snap, snap] }, `snapshot ${snap.id} appears twice`], + [ + { + purged: [{ sha256: "a".repeat(64), reason: null as never, legal: false, by: "", at: "" }], + }, + "bundle takedown entry is malformed", + ], + ]; + for (const [edit, message] of cases) { + await expect(importSite(dst, { ...bundle, ...edit })).rejects.toThrow(message); + expect(await counts(dst)).toEqual(before); + } + }); + + it("reads the target's own asset bodies for the media ledger, never the bundle's text for them", async () => { + const src = await pair(); + const dst = await pair(); + const site = await seedStarterSite(src.store, { name: "Ledger", slug: "ledger" }); + const asset = await upload(src, site.id, PNG_1X1, "dot.png"); + const snap = await src.store.createSnapshot(site.id, { actor, reason: "manual" }); + await copySite(src, dst, site.id); + const bundle = await exportSite(src, site.id); + const ref = refsOf(bundle, snap.id).find((r) => r.kind === "asset") as NodeRef; + // Not JSON: before, the ledger parsed it after the node set had landed. + const crafted = { ...bundle, blobs: { ...bundle.blobs, [ref.hash]: "\u001b[31m{" } }; + expect(await importSite(dst, crafted)).toMatchObject({ + nodes: { unchanged: true }, + snapshots: 0, + blobs: 0, + }); + expect( + await dst.sql.all("SELECT sha256 FROM snapshot_blobs WHERE snapshot_id = ?", [snap.id]), + ).toEqual([{ sha256: asset.sha256 }]); + }); + + it("refuses a site over the batch limit before any snapshot lands", async () => { + const src = await pair(); + const site = await seedStarterSite(src.store, { name: "Big", slug: "big" }); + const nodes = (await exportSite(src, site.id)).nodes.length; + for (let i = 0; nodes + i <= 1000; i++) + await src.store.createRedirect({ + siteId: site.id, + from: `/old-${i}`, + to: "/about", + status: 301, + match: "exact", + reason: "manual", + }); + await src.store.createSnapshot(site.id, { actor, reason: "manual" }); + const dst = await pair(); + await expect(importSite(dst, await exportSite(src, site.id))).rejects.toThrow( + /too many nodes .* for one atomic copy/, + ); + expect(await counts(dst)).toMatchObject({ nodes: 0, snapshots: 0, blobs: [] }); + }); + + it("keeps the published pointer of a published target for a push (published: target)", async () => { + const src = await pair(); + const dst = await pair(); + const site = await seedStarterSite(src.store, { name: "Pointer", slug: "pointer" }); + const v1 = await src.store.createSnapshot(site.id, { actor, reason: "publish" }); + await src.store.publish((await src.store.getSite(site.id)) as Site, v1.id); + await copySite(src, dst, site.id); + const v2 = await dst.store.createSnapshot(site.id, { actor, reason: "publish" }); + await dst.store.publish((await dst.store.getSite(site.id)) as Site, v2.id); + const bundle = await exportSite(src, site.id); + const plan = await planImport(dst, bundle, { mode: "replace", published: "target" }); + // Only the pointer differs, and the target's is kept: nothing changes. + expect(plan.unchanged).toBe(true); + expect(plan.nodes.changed).toEqual([]); + }); +});