From 7223152615ac901aada2a578fe149f31000195fe Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:42:36 +0300 Subject: [PATCH 1/3] fix(core-ts): check a bundle's project row before importing it importSite inserted bundle.project as given when the target lacked it, so a crafted bundle could plant any projects row. The row must now be the site node's project, a valid Project, with id and rev matching its body. Covers pull, the admin bundle route behind push, and the coming `cloudhal import`. Refs #47 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/bundle-project-row.md | 5 +++++ packages/core-ts/src/sync.ts | 33 +++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 .changeset/bundle-project-row.md diff --git a/.changeset/bundle-project-row.md b/.changeset/bundle-project-row.md new file mode 100644 index 00000000..3d995c71 --- /dev/null +++ b/.changeset/bundle-project-row.md @@ -0,0 +1,5 @@ +--- +"@cloudhal/core-ts": patch +--- + +`importSite` (behind `cloudhal pull`, the admin bundle route `cloudhal push` uses, and the coming `cloudhal import`) now checks a bundle's project row before inserting it: it must be the site node's project (`projectId`), a valid Project, with its `id` and `rev` columns matching its body. Before, a bundle could plant any projects row. diff --git a/packages/core-ts/src/sync.ts b/packages/core-ts/src/sync.ts index 4ec66b15..6daa5263 100644 --- a/packages/core-ts/src/sync.ts +++ b/packages/core-ts/src/sync.ts @@ -80,11 +80,39 @@ async function assertTimeZone(to: StoragePair, siteId: string, bundle: SiteBundl ); } +/** + * The project row is inserted as given when the target lacks it, so it must be the site node's + * project (`projectId`), a valid Project whose `id` and `rev` agree with its columns. Its id is + * echoed only once it equals the validated site node's. + */ +function assertProjectRow(siteId: string, bundle: SiteBundle): void { + const p = bundle.project; + if (!p) return; + const siteRow = bundle.nodes.find((n) => n.kind === "site" && n.id === siteId); + const projectId = siteRow + ? (JSON.parse(siteRow.body) as { projectId?: unknown }).projectId + : undefined; + if (p.id !== projectId) throw new Error("the bundle's project is not its site's project"); + let body: { id?: unknown; rev?: unknown }; + try { + body = JSON.parse(p.body) as { id?: unknown; rev?: unknown }; + } catch { + throw new Error(`project ${p.id}: body is not JSON`); + } + try { + assertValid("project", body); + } catch { + throw new Error(`project ${p.id}: not a valid project`); + } + if (body.id !== p.id || body.rev !== p.rev) + throw new Error(`project ${p.id}: columns do not match its body`); +} + /** * A bundle is untrusted input (whoever holds the deployment token): every row must be a valid * contract document of the kind its `kind` column declares (RFC-0002 §4.1: the node schemas are * mutually exclusive, so a body cannot pass as another kind), whose columns agree with its body, - * for this site only. + * for this site only; the project row too (`assertProjectRow`). */ async function assertValidRows(siteId: string, bundle: SiteBundle): Promise { for (const row of bundle.nodes) { @@ -146,6 +174,7 @@ async function assertValidRows(siteId: string, bundle: SiteBundle): Promise r.sha256)); + // packages/cli/src/commands/archive-bundle.ts bundleMedia rebuilds this from an archive; keep + // the two in step (export-cli.test.ts compares them). const media = new Map(); const addAsset = (body: string) => { const a = JSON.parse(body) as Asset; From 330b6505b4a365d677a25d261934b4a90ccf2b14 Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Mon, 26 Oct 2026 00:42:37 +0300 Subject: [PATCH 2/3] feat(cli): cloudhal export writes a site archive (#47) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RFC-0039 rollout step 2. `cloudhal export [--out ] [--site ] [--no-media] [--latest-only]` writes one site of the local project to a .cloudhal file: exportSite's bundle as NDJSON parts, media streamed from the local blob store and verified, and the takedown ledger. - Default name -.cloudhal, dated by the newest row it holds (never the clock), so an unchanged site exports identically. A file at the default name is replaced only by the same archive; --out replaces any. - --site takes a site listed in cloudhal.json sites; --site, --no-media and --latest-only are refused on every other command. - --latest-only keeps the published and the latest snapshot and what they need; refused while a field operation is unfinished. - Ctrl-C and SIGTERM remove the partial file. - The summary lists what an archive never holds (RFC-0039 §4) and that its history names who made each change. archive-bundle.ts and archive-reassemble.ts convert a bundle to archive content and back (step 3's import reuses the second); the tests check the round trip deep-equals exportSite's bundle. Refs #47 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/site-archive-export.md | 12 + packages/cli/src/archive/index.ts | 7 +- packages/cli/src/archive/write.ts | 17 +- packages/cli/src/assets.ts | 8 + packages/cli/src/cli.ts | 24 + packages/cli/src/cli/commands-content.ts | 33 +- packages/cli/src/cli/help.ts | 11 + packages/cli/src/cli/types.ts | 30 + packages/cli/src/commands/archive-bundle.ts | 294 +++++++++ .../cli/src/commands/archive-reassemble.ts | 347 ++++++++++ packages/cli/src/commands/export.ts | 337 ++++++++++ packages/cli/test/archive-bundle.test.ts | 508 +++++++++++++++ packages/cli/test/archive-roundtrip.test.ts | 21 + packages/cli/test/export-cli.test.ts | 613 ++++++++++++++++++ 14 files changed, 2256 insertions(+), 6 deletions(-) create mode 100644 .changeset/site-archive-export.md create mode 100644 packages/cli/src/commands/archive-bundle.ts create mode 100644 packages/cli/src/commands/archive-reassemble.ts create mode 100644 packages/cli/src/commands/export.ts create mode 100644 packages/cli/test/archive-bundle.test.ts create mode 100644 packages/cli/test/export-cli.test.ts diff --git a/.changeset/site-archive-export.md b/.changeset/site-archive-export.md new file mode 100644 index 00000000..9a4a3381 --- /dev/null +++ b/.changeset/site-archive-export.md @@ -0,0 +1,12 @@ +--- +"@cloudhal/cli": patch +--- + +New `cloudhal export` (RFC-0039, #47) writes one site of the local project to a single `.cloudhal` file. The file is a gzipped tar holding the site's content graph (nodes, snapshots and the node bodies they reference, as NDJSON parts), its media files and its takedown ledger, with every file's SHA-256 listed in the manifest. The flags: + +- `--out ` names the file, and replaces a file already there once the new one is complete. By default the file is `-.cloudhal` in the current folder, dated by the newest timestamp among the rows it holds (deleting content alone does not move that date; the next edit or snapshot does). A file already at the default name is replaced only by the same archive; any other (a later edit the same day, other flags, a deletion) is left alone and the export asks for `--out`. +- `--site ` exports another site listed in `cloudhal.json` `sites`. `--site`, `--no-media` and `--latest-only` are refused on every other command. +- `--no-media` writes the graph only. +- `--latest-only` keeps only the published and the latest snapshot, and only the media they and the current content need. It is refused while a field operation is unfinished (`cloudhal fields continue|revert`). + +A damaged or missing local media file stops the export (run `cloudhal media verify`); so do Ctrl-C and SIGTERM. Either way no file is written and an existing one is left as it was. Exporting an unchanged site again with the same CLI and Node version gives the same bytes. An archive never contains user accounts, sessions, roles, agent keys, comments, readers, analytics, the audit log, automations, schedule runs, deployment secrets or webhook credentials. Its history does name who made each change (the user or agent of each snapshot, upload, block edit, review and takedown). `cloudhal import` comes in a later release. diff --git a/packages/cli/src/archive/index.ts b/packages/cli/src/archive/index.ts index 646e2970..51ffb3ad 100644 --- a/packages/cli/src/archive/index.ts +++ b/packages/cli/src/archive/index.ts @@ -14,4 +14,9 @@ export { export { encodeNdjson, type NdjsonSplitOptions, splitNdjson } from "./parts.ts"; export type { ArchiveContent, ArchiveMediaSource } from "./plan.ts"; export { type ArchiveItem, type ReadArchiveOptions, readArchive } from "./read.ts"; -export { archiveStream, type WriteArchiveOptions, writeArchiveFile } from "./write.ts"; +export { + archiveStream, + type WriteArchiveFileOptions, + type WriteArchiveOptions, + writeArchiveFile, +} from "./write.ts"; diff --git a/packages/cli/src/archive/write.ts b/packages/cli/src/archive/write.ts index a9786cf6..0ad7553c 100644 --- a/packages/cli/src/archive/write.ts +++ b/packages/cli/src/archive/write.ts @@ -33,6 +33,14 @@ export interface WriteArchiveOptions { limits?: Partial; } +export interface WriteArchiveFileOptions extends WriteArchiveOptions { + /** + * Stops the write (`cloudhal export` aborts it on Ctrl-C and SIGTERM): the temporary file is + * removed, an existing `file` stays as it was, and the promise rejects. + */ + signal?: AbortSignal | undefined; +} + /** A media body as it is written: no more than `bytes`, then the SHA-256. */ async function* verifiedMedia( file: string, @@ -96,13 +104,16 @@ export function archiveStream( /** * Writes the archive of `content` to `file`: to a temporary file next to it first, synced and then - * renamed over `file`, so a failure leaves no file behind and an existing `file` unchanged. + * renamed over `file`, so a failure or an abort leaves no file behind and an existing `file` + * unchanged. Only a killed process (SIGKILL, power loss) can leave the `..partial-…` file. */ export async function writeArchiveFile( file: string, content: ArchiveContent, - options: WriteArchiveOptions = {}, + options: WriteArchiveFileOptions = {}, ): Promise { + const { signal } = options; + signal?.throwIfAborted(); const { manifest, bytes } = archiveStream(content, options); const temp = path.join( path.dirname(file), @@ -112,7 +123,7 @@ export async function writeArchiveFile( try { handle = await open(temp, "wx"); // The stream owns the handle: it fsyncs (flush) and closes it, on errors too. - await pipeline(bytes, handle.createWriteStream({ flush: true })); + await pipeline(bytes, handle.createWriteStream({ flush: true }), signal ? { signal } : {}); await rename(temp, file); return manifest; } catch (e) { diff --git a/packages/cli/src/assets.ts b/packages/cli/src/assets.ts index 58ab6664..fc491c89 100644 --- a/packages/cli/src/assets.ts +++ b/packages/cli/src/assets.ts @@ -33,6 +33,14 @@ async function packageRoot(): Promise { throw new Error("could not locate the @cloudhal/cli package root"); } +/** The `@cloudhal/cli` version (its package.json's), from either layout. */ +export async function cliVersion(): Promise { + const pkg = path.join(await packageRoot(), "package.json"); + const version = (JSON.parse(await readFile(pkg, "utf8")) as { version?: unknown }).version; + if (typeof version !== "string") throw new Error(`${pkg} has no version`); + return version; +} + export type AssetKind = "studio" | "starters" | "skills"; const MONOREPO: Record = { diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index d7fd9047..a667fd9c 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -38,6 +38,9 @@ export const OPTIONS = { views: { type: "string" }, from: { type: "string" }, out: { type: "string" }, + site: { type: "string" }, + "no-media": { type: "boolean" }, + "latest-only": { type: "boolean" }, jurisdiction: { type: "string" }, in: { type: "string" }, file: { type: "string" }, @@ -70,6 +73,26 @@ export const OPTIONS = { clear: { type: "boolean" }, } as const; +/** + * Flags only some commands take. `OPTIONS` is one table for every command, so without this a + * `cloudhal push --replace --site ` would parse and act on cloudhal.json's `siteId`. + */ +const ONLY_FOR: Readonly< + Partial> +> = { + site: { commands: ["export"], note: "; every other command acts on cloudhal.json's siteId" }, + "no-media": { commands: ["export"] }, + "latest-only": { commands: ["export"] }, +}; + +function refuseForeignFlags(command: string, values: Record): void { + for (const [flag, only] of Object.entries(ONLY_FOR)) + if (values[flag] !== undefined && !only.commands.includes(command)) + throw new Error( + `cloudhal ${command} does not take --${flag} (only cloudhal ${only.commands.join(", ")} does)${only.note ?? ""}`, + ); +} + /** `cloudhal ` → its handler. */ export const COMMANDS: Record = { ...LOCAL_COMMANDS, @@ -95,5 +118,6 @@ export async function main(argv: string[]): Promise { process.stderr.write(`unknown command "${command}"\n\n${HELP}`); return 1; } + refuseForeignFlags(command, values); return run(rest, values, { projectDir: process.cwd() }); } diff --git a/packages/cli/src/cli/commands-content.ts b/packages/cli/src/cli/commands-content.ts index b062d873..0790eadf 100644 --- a/packages/cli/src/cli/commands-content.ts +++ b/packages/cli/src/cli/commands-content.ts @@ -4,14 +4,15 @@ import { commentsStatusCli, describeCommentsStatus, } from "../commands/comments.ts"; +import { describeExport, exportArchive } from "../commands/export.ts"; import { fieldsCommand } from "../commands/fields.ts"; import { describeMediaImport, mediaGc, mediaImportUrls, mediaVerify } from "../commands/media.ts"; import { describePlaceholders, mediaPlaceholders } from "../commands/media-placeholders.ts"; import { describeMigration, migrateTags } from "../commands/migrate.ts"; import { describeReindex, searchReindex } from "../commands/search.ts"; -import { type CliCommand, remoteFlags, subcommands } from "./types.ts"; +import { abortOnSignal, type CliCommand, remoteFlags, subcommands } from "./types.ts"; -/** `cloudhal migrate`, `media`, `search`, `comments`: the site's content and its stores. */ +/** `cloudhal migrate`, `fields`, `media`, `search`, `comments`, `export`: the site's content and its stores. */ const migrateCommand: CliCommand = async (rest, values, { projectDir }) => { const [what] = rest; @@ -138,10 +139,38 @@ const commentsCommand = subcommands( "usage: cloudhal comments status|export|import", ); +/** Ctrl-C or SIGTERM stops the export: no file is written, an existing one stays (exit 130/143). */ +const exportCommand: CliCommand = async (rest, values, { projectDir }) => { + if (rest.length > 0) + throw new Error( + "usage: cloudhal export [--out ] [--site ] [--no-media] [--latest-only]", + ); + const interrupt = abortOnSignal(); + try { + const result = await exportArchive({ + projectDir, + ...(values.out !== undefined ? { out: values.out } : {}), + ...(values.site !== undefined ? { siteId: values.site } : {}), + ...(values["no-media"] ? { noMedia: true } : {}), + ...(values["latest-only"] ? { latestOnly: true } : {}), + signal: interrupt.signal, + }); + for (const line of describeExport(result, projectDir)) console.log(line); + return 0; + } catch (e) { + if (!interrupt.signal.aborted) throw e; + console.error("cloudhal: export interrupted; no file written, an existing one left as it was"); + return interrupt.exitCode(); + } finally { + interrupt.dispose(); + } +}; + export const CONTENT_COMMANDS = { migrate: migrateCommand, fields: fieldsCommand, media: mediaCommand, search: searchCommand, comments: commentsCommand, + export: exportCommand, } satisfies Record; diff --git a/packages/cli/src/cli/help.ts b/packages/cli/src/cli/help.ts index be7bb068..b75e2768 100644 --- a/packages/cli/src/cli/help.ts +++ b/packages/cli/src/cli/help.ts @@ -27,6 +27,17 @@ Usage: local site → deployment (refused while releases are open unless --cancel-schedules cancels them) cloudhal pull [--replace] [--admin-url ] deployment → local site + cloudhal export [--out ] [--site ] [--no-media] [--latest-only] + one site as a .cloudhal file (RFC-0039): its graph, + snapshots and media; never user accounts, comments, + analytics or secrets (its history names who made each + change). Default -.cloudhal, dated + by the newest row it holds (a deletion alone does not + move it); a file there is replaced only by the same + archive, --out replaces any; + --site: a site listed in cloudhal.json sites; + --no-media: the graph only; --latest-only: only the + published and the latest snapshot cloudhal owner transfer --to [--admin-url ] make a member the deployment's Owner (lockout recovery; uses the deployment token) diff --git a/packages/cli/src/cli/types.ts b/packages/cli/src/cli/types.ts index 9d6bda8e..f8462e75 100644 --- a/packages/cli/src/cli/types.ts +++ b/packages/cli/src/cli/types.ts @@ -20,6 +20,36 @@ export function untilSignal(also?: (resolve: () => void) => void): Promise }); } +/** + * An `AbortSignal` that Ctrl-C and SIGTERM abort while a command writes a file. Listening replaces + * Node's default exit, which runs no `finally`, so the writer can remove its temporary file; a + * second Ctrl-C exits at once. `exitCode` is the shell's code for the signal received (130, 143); + * `dispose` stops listening. + */ +export function abortOnSignal(): { + signal: AbortSignal; + exitCode(): number; + dispose(): void; +} { + const controller = new AbortController(); + let code = 1; + const handlers = (["SIGINT", "SIGTERM"] as const).map((sig) => { + const handler = () => { + code = sig === "SIGINT" ? 130 : 143; + controller.abort(); + }; + process.once(sig, handler); + return { sig, handler }; + }); + return { + signal: controller.signal, + exitCode: () => code, + dispose: () => { + for (const { sig, handler } of handlers) process.off(sig, handler); + }, + }; +} + /** * A command whose first positional picks an action (`cloudhal media gc`); an unknown or missing * action throws `usage`. The action gets the positionals after its name. diff --git a/packages/cli/src/commands/archive-bundle.ts b/packages/cli/src/commands/archive-bundle.ts new file mode 100644 index 00000000..4a66c099 --- /dev/null +++ b/packages/cli/src/commands/archive-bundle.ts @@ -0,0 +1,294 @@ +/** + * RFC-0039 rollout step 2: a `SiteBundle` (what `exportSite` returns and `importSite` takes) as the + * archive container's content, and the parts both directions share; `archive-reassemble.ts` is the + * way back. Pure (no I/O), so the container in `../archive/` stays generic and these run in tests + * without a store. + * + * - Rows are written with the keys of the field tables below, in their order, so the archive's + * bytes do not depend on the column order a driver returns; the reader checks every row against + * the same tables. + * - `bundleMedia` rebuilds `bundle.media` from the rows the way `exportSite` does: the archive + * holds the media files, not their media types. + * + * Known limitation: `exportSite` returns a whole bundle, so the graph rows and the blob texts are + * held in memory. `importSite`'s single atomic batch already bounds a site to about 1000 nodes; + * snapshots are not bounded that way. Only the media bytes stream. + */ +import type { Asset } from "@cloudhal/core-contracts"; +import type { BundleMedia, BundlePurge, SiteBundle } from "@cloudhal/core-ts"; +import { type ArchiveContent, encodeNdjson, splitNdjson } from "../archive/index.ts"; + +/** How a row field is checked on the way back. */ +export type FieldType = "string" | "string|null" | "integer" | "boolean"; + +type Fields = Readonly>; + +/** `NodeRow` (core-ts sync.ts), in the `nodes` table's column order. */ +export const NODE_FIELDS = { + id: "string", + site_id: "string", + kind: "string", + key: "string", + rev: "integer", + hash: "string", + body: "string", + created_at: "string", + updated_at: "string", +} as const satisfies Fields; + +/** `SnapshotRow`. */ +export const SNAPSHOT_FIELDS = { + id: "string", + site_id: "string", + parent_id: "string|null", + site_rev: "integer", + body: "string", + created_at: "string", +} as const satisfies Fields; + +/** `ProjectRow`. */ +export const PROJECT_FIELDS = { + id: "string", + rev: "integer", + body: "string", + created_at: "string", + updated_at: "string", +} as const satisfies Fields; + +/** A line of `blobs/NNNN.ndjson`: a node body the snapshots reference, by its content hash. */ +export const BLOB_FIELDS = { hash: "string", body: "string" } as const satisfies Fields; + +/** `BundlePurge`: a line of `ledger.ndjson`. */ +export const PURGE_FIELDS = { + sha256: "string", + reason: "string", + legal: "boolean", + by: "string", + at: "string", +} as const satisfies Fields; + +/** A node ref of a snapshot body, as far as the archive needs it. */ +export interface SnapshotRef { + kind: string; + hash: string; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** Code-unit order: the same on every machine (and, for lowercase hex, `localeCompare`'s). */ +export function byCodeUnit(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +/** `row` with exactly the keys of `fields`, in their order. */ +function ordered(row: object, fields: Fields): Record { + const out: Record = {}; + for (const key of Object.keys(fields)) out[key] = (row as Record)[key]; + return out; +} + +/** The node refs of a snapshot body, or undefined when it is not `{nodes: [{kind, hash}, …]}`. */ +export function snapshotRefs(body: string): SnapshotRef[] | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(body); + } catch { + return undefined; + } + const nodes = isRecord(parsed) ? parsed.nodes : undefined; + if (!Array.isArray(nodes)) return undefined; + const refs: SnapshotRef[] = []; + for (const n of nodes) { + if (!isRecord(n) || typeof n.kind !== "string" || typeof n.hash !== "string") return undefined; + refs.push({ kind: n.kind, hash: n.hash }); + } + return refs; +} + +/** What the archive and its file name need from the bundle's site node. */ +export function siteFacts(bundle: Pick): { + name: string; + slug: string; + publishedSnapshotId?: string | undefined; +} { + const row = bundle.nodes.find((n) => n.kind === "site" && n.id === bundle.siteId); + if (!row) throw new Error(`the bundle has no site node for ${bundle.siteId}`); + const site = JSON.parse(row.body) as { + name?: unknown; + slug?: unknown; + publishedSnapshotId?: unknown; + }; + const published = site.publishedSnapshotId; + return { + name: typeof site.name === "string" ? site.name : "", + slug: typeof site.slug === "string" ? site.slug : "", + ...(typeof published === "string" ? { publishedSnapshotId: published } : {}), + }; +} + +/** + * `bundle.media` as `exportSite` builds it (core-ts sync.ts): the blob assets of the current nodes + * (in id order), then of each snapshot (in bundle order, refs in body order, bodies from `blobs`), + * the last one seen for a hash winning; taken-down content left out; sorted by hash; undefined + * when there is none. Keep the two in step: export-cli.test.ts compares them. + */ +export function bundleMedia( + rows: Pick, + purged: ReadonlySet, +): BundleMedia[] | undefined { + const media = new Map(); + const add = (body: string) => { + const a = JSON.parse(body) as Asset; + if (a.storage?.kind === "blob" && !purged.has(a.sha256)) + media.set(a.sha256, { sha256: a.sha256, bytes: a.bytes, mediaType: a.mediaType }); + }; + for (const n of rows.nodes) if (n.kind === "asset") add(n.body); + for (const snap of rows.snapshots) + for (const ref of snapshotRefs(snap.body) ?? []) { + const text = Object.hasOwn(rows.blobs, ref.hash) ? rows.blobs[ref.hash] : undefined; + if (ref.kind === "asset" && text) add(text); + } + if (media.size === 0) return undefined; + return [...media.values()].sort((a, b) => byCodeUnit(a.sha256, b.sha256)); +} + +/** + * The archive's `createdAt` (invariant 7 as amended): the newest timestamp among the rows it holds + * (nodes, snapshots, the project and the takedowns, since a takedown alone changes the ledger), + * never the clock, so an unchanged site exports identically. Not "the newest change": a deletion + * stamps no remaining row, so it alone leaves the date as it was (or moves it back, when the + * newest row went); the next edit or snapshot moves it on. Values that do not parse (the ledger's + * `''` defaults) are skipped. + */ +export function contentCreatedAt(bundle: SiteBundle): string { + let newest = Number.NEGATIVE_INFINITY; + const see = (at: string) => { + const t = Date.parse(at); + if (t > newest) newest = t; + }; + for (const n of bundle.nodes) { + see(n.created_at); + see(n.updated_at); + } + for (const s of bundle.snapshots) see(s.created_at); + if (bundle.project) { + see(bundle.project.created_at); + see(bundle.project.updated_at); + } + for (const p of bundle.purged ?? []) see(p.at); + if (!Number.isFinite(newest)) + throw new Error(`site ${bundle.siteId} has no row with a timestamp to date the archive by`); + return new Date(newest).toISOString(); +} + +/** + * RFC-0041 §6: a field operation whose marker (`migration`) is still on a field of the site node + * needs its `before` snapshot (the marker's id) to be continued or reverted, and blocks any other + * operation on its collection until then. Dropping that snapshot would leave the collection stuck + * after an import, so `--latest-only` is refused until the operation is finished or reverted. + */ +function refuseUnfinishedOperations(bundle: SiteBundle): void { + const row = bundle.nodes.find((n) => n.kind === "site" && n.id === bundle.siteId); + const site: unknown = row ? JSON.parse(row.body) : undefined; + const collections = isRecord(site) && isRecord(site.collections) ? site.collections : {}; + for (const [collection, c] of Object.entries(collections)) { + const fields: unknown[] = isRecord(c) && Array.isArray(c.fields) ? c.fields : []; + const migration = fields.map((f) => (isRecord(f) ? f.migration : undefined)).find(isRecord); + if (!migration) continue; + const id = String(migration.id); + throw new Error( + `field operation ${id} on ${collection} has not finished, and --latest-only would drop the snapshot it needs; finish or revert it first (\`cloudhal fields continue|revert ${collection} ${id}\`), or export the full history`, + ); + } +} + +/** + * `cloudhal export --latest-only`: only the published snapshot (the site node's pointer) and the + * latest one (`latestId`, the store's `latestSnapshot`; the bundle's last row when not given), in + * bundle order, and only the blobs they reference. Media are what the current nodes and those + * snapshots need; nodes, the project and the takedowns stay. A kept snapshot's `parent_id` may + * name a dropped one: nothing follows that link. Refused while a field operation is unfinished + * (`refuseUnfinishedOperations`). + */ +export function latestOnly(bundle: SiteBundle, latestId?: string): SiteBundle { + refuseUnfinishedOperations(bundle); + const latest = latestId ?? bundle.snapshots.at(-1)?.id; + const keep = new Set([siteFacts(bundle).publishedSnapshotId, latest]); + const snapshots = bundle.snapshots.filter((s) => keep.has(s.id)); + const hashes = new Set(snapshots.flatMap((s) => (snapshotRefs(s.body) ?? []).map((r) => r.hash))); + const blobs: Record = {}; + for (const [hash, text] of Object.entries(bundle.blobs)) if (hashes.has(hash)) blobs[hash] = text; + const purged = new Set((bundle.purged ?? []).map((p) => p.sha256)); + const media = bundleMedia({ nodes: bundle.nodes, snapshots, blobs }, purged); + return { + format: bundle.format, + siteId: bundle.siteId, + ...(bundle.project ? { project: bundle.project } : {}), + nodes: bundle.nodes, + snapshots, + blobs, + ...(media ? { media } : {}), + ...(bundle.purged ? { purged: bundle.purged } : {}), + }; +} + +/** Archives are self-contained: every snapshot ref has its body in the bundle's blobs. */ +function assertSelfContained(bundle: SiteBundle): void { + for (const snap of bundle.snapshots) { + const refs = snapshotRefs(snap.body); + if (!refs) throw new Error(`snapshot ${snap.id}: its body is not a snapshot`); + for (const ref of refs) + if (!Object.hasOwn(bundle.blobs, ref.hash)) + throw new Error(`snapshot ${snap.id} references node body ${ref.hash}, not in the bundle`); + } +} + +export interface BundleArchiveOptions { + /** The `@cloudhal/cli` version. */ + createdBy: string; + /** The bytes of a media file, opened when the writer reaches it. */ + openMedia(m: BundleMedia): AsyncIterable; + /** `--no-media`: no media files, `mediaOmitted` in the manifest. */ + mediaOmitted?: boolean | undefined; + /** NDJSON part size (tests; default 8 MiB). */ + partBytes?: number | undefined; +} + +/** + * `bundle` as the container's content (RFC-0039 §1): `project.json`, the node, snapshot and blob + * parts in the bundle's order (blobs sorted by hash), one media source per `bundle.media` entry + * and `ledger.ndjson` when content was taken down. + */ +export function bundleToArchive(bundle: SiteBundle, options: BundleArchiveOptions): ArchiveContent { + assertSelfContained(bundle); + const split = (rows: unknown[]) => + splitNdjson(rows, options.partBytes === undefined ? {} : { partBytes: options.partBytes }); + const blobs = Object.keys(bundle.blobs) + .sort(byCodeUnit) + .map((hash) => ({ hash, body: bundle.blobs[hash] })); + const purged: readonly BundlePurge[] = bundle.purged ?? []; + return { + siteId: bundle.siteId, + siteName: siteFacts(bundle).name, + createdBy: options.createdBy, + createdAt: contentCreatedAt(bundle), + ...(bundle.project ? { project: encodeNdjson([ordered(bundle.project, PROJECT_FIELDS)]) } : {}), + nodes: split(bundle.nodes.map((n) => ordered(n, NODE_FIELDS))), + snapshots: split(bundle.snapshots.map((s) => ordered(s, SNAPSHOT_FIELDS))), + blobs: split(blobs), + ...(purged.length > 0 + ? { ledger: encodeNdjson(purged.map((p) => ordered(p, PURGE_FIELDS))) } + : {}), + ...(options.mediaOmitted + ? { mediaOmitted: true as const } + : { + media: (bundle.media ?? []).map((m) => ({ + sha256: m.sha256, + bytes: m.bytes, + open: () => options.openMedia(m), + })), + }), + }; +} diff --git a/packages/cli/src/commands/archive-reassemble.ts b/packages/cli/src/commands/archive-reassemble.ts new file mode 100644 index 00000000..1f03f1ac --- /dev/null +++ b/packages/cli/src/commands/archive-reassemble.ts @@ -0,0 +1,347 @@ +/** + * RFC-0039 §1: a site archive's items (from `readArchive`) back into exactly the `SiteBundle` that + * `exportSite` returned when it was written. `cloudhal import` (rollout step 3) hands the result + * to `importSite`; the tests compare it with `exportSite`. + * + * Every row is checked against the field tables of `archive-bundle.ts` (the exact keys, strings, + * non-negative integers), belongs to the manifest's site and appears once; every snapshot ref has + * its body in the archive. Every asset body `bundleMedia` reads (asset nodes, the bodies of + * snapshots' asset refs) is a JSON object and, for a blob asset, has a hex SHA-256, a size and a + * media type, so the media rule never throws or quotes. The media files must match the graph: + * none that no asset needs, none taken down by the archive's own ledger (invariant 5), each with + * its asset's size, and, unless the manifest says `mediaOmitted`, none missing. + * + * The ledger comes after the media in the layout, so a media file for taken-down content is + * refused only at the end, after `onMedia` saw it: a caller stages media and removes them when + * this throws. Nothing is returned before `readArchive` has verified the whole archive. + * + * Errors name the archive path, the line, our own field names and validated hex hashes; never ids, + * keys, bodies or JSON.parse messages, which can quote untrusted input. + */ +import { + type BundleMedia, + type BundlePurge, + type NodeRow, + type ProjectRow, + SITE_BUNDLE_FORMAT, + type SiteBundle, + type SnapshotRow, +} from "@cloudhal/core-ts"; +import type { ArchiveItem, PartKind, SiteArchiveManifest } from "../archive/index.ts"; +import { + BLOB_FIELDS, + bundleMedia, + type FieldType, + isRecord, + NODE_FIELDS, + PROJECT_FIELDS, + PURGE_FIELDS, + SNAPSHOT_FIELDS, + type SnapshotRef, + snapshotRefs, +} from "./archive-bundle.ts"; + +export type ArchiveMediaItem = Extract; +type FileItem = Extract; + +export interface ReassembleOptions { + /** + * Called for each media file, in archive order, after it was matched against the graph; reads + * (or ignores) its body before resolving. The body is verified as it is read. + */ + onMedia?: ((item: ArchiveMediaItem) => Promise) | undefined; +} + +export interface ReassembledArchive { + manifest: SiteArchiveManifest; + bundle: SiteBundle; + /** The media files the archive held, in archive order. */ + media: { sha256: string; size: number }[]; +} + +const SHA256 = /^[0-9a-f]{64}$/; +const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + +/** What has been read so far. */ +interface Draft { + /** Blob texts already checked as asset bodies. */ + assetBlobs: Set; + manifest: SiteArchiveManifest; + project?: ProjectRow | undefined; + nodes: NodeRow[]; + nodeIds: Set; + snapshots: SnapshotRow[]; + snapshotIds: Set; + /** Each snapshot's refs and where it was read, for the checks once the blobs are in. */ + refs: { at: string; refs: SnapshotRef[] }[]; + blobs: Record; + purged: BundlePurge[]; + purgedShas: Set; + media: { sha256: string; size: number }[]; + /** Set once the graph is complete (at the first media file, or at the end). */ + needed?: Map | undefined; +} + +type Row>> = { + -readonly [K in keyof F]: F[K] extends "integer" + ? number + : F[K] extends "boolean" + ? boolean + : F[K] extends "string|null" + ? string | null + : string; +}; + +const FIELD_CHECKS: Record boolean> = { + string: (v) => typeof v === "string", + "string|null": (v) => v === null || typeof v === "string", + integer: (v) => Number.isSafeInteger(v) && (v as number) >= 0, + boolean: (v) => typeof v === "boolean", +}; + +const FIELD_WORDS: Record = { + string: "a string", + "string|null": "a string or null", + integer: "a non-negative integer", + boolean: "true or false", +}; + +/** A new row with exactly `fields`' keys, in their order, each of its type; or the reason not. */ +function rowOf>>( + value: unknown, + fields: F, + at: string, +): Row { + if (!isRecord(value)) throw new Error(`${at}: not a JSON object`); + const keys = Object.keys(fields); + if (Object.keys(value).length !== keys.length || !keys.every((k) => Object.hasOwn(value, k))) + throw new Error(`${at}: needs exactly the fields ${keys.join(", ")}`); + const row: Record = {}; + for (const key of keys) { + const type = fields[key] as FieldType; + if (!FIELD_CHECKS[type](value[key])) + throw new Error(`${at}: ${key} must be ${FIELD_WORDS[type]}`); + row[key] = value[key]; + } + return row as Row; +} + +/** The JSON value of each line of an NDJSON file, with where it is (`path line n`). */ +function* lines(item: FileItem): Generator<{ value: unknown; at: string }> { + let text: string; + try { + text = decoder.decode(item.bytes); + } catch { + throw new Error(`${item.path}: not UTF-8`); + } + // The reader checks this for the parts and the ledger, not for project.json. + if (!text.endsWith("\n")) throw new Error(`${item.path}: not NDJSON (no final newline)`); + const all = text.split("\n"); + for (let i = 0; i < all.length - 1; i++) { + const at = `${item.path} line ${i + 1}`; + let value: unknown; + try { + value = JSON.parse(all[i] as string); + } catch { + throw new Error(`${at}: not JSON`); + } + yield { value, at }; + } +} + +function ownSite(draft: Draft, siteId: string, at: string): void { + if (siteId !== draft.manifest.siteId) throw new Error(`${at}: site_id is not the archive's site`); +} + +/** + * An asset body as `bundleMedia` reads it: a JSON object and, when its bytes are in the blob + * store, a hex `sha256`, a non-negative `bytes` and a string `mediaType`. Never quotes the body. + */ +function checkAssetBody(body: string, at: string, what: string): void { + let asset: unknown; + try { + asset = JSON.parse(body); + } catch { + throw new Error(`${at}: ${what} is not JSON`); + } + if (!isRecord(asset)) throw new Error(`${at}: ${what} is not a JSON object`); + if (!isRecord(asset.storage) || asset.storage.kind !== "blob") return; + if (typeof asset.sha256 !== "string" || !SHA256.test(asset.sha256)) + throw new Error(`${at}: ${what} needs a sha256 of 64 lowercase hex digits`); + if (!FIELD_CHECKS.integer(asset.bytes)) + throw new Error(`${at}: ${what} needs bytes, a non-negative integer`); + if (typeof asset.mediaType !== "string") + throw new Error(`${at}: ${what} needs a mediaType string`); +} + +function addNode(draft: Draft, value: unknown, at: string): void { + const row = rowOf(value, NODE_FIELDS, at); + ownSite(draft, row.site_id, at); + if (row.kind === "asset") checkAssetBody(row.body, at, "the asset body"); + if (draft.nodeIds.has(row.id)) throw new Error(`${at}: a node id appears twice`); + draft.nodeIds.add(row.id); + draft.nodes.push(row); +} + +function addSnapshot(draft: Draft, value: unknown, at: string): void { + const row = rowOf(value, SNAPSHOT_FIELDS, at); + ownSite(draft, row.site_id, at); + if (draft.snapshotIds.has(row.id)) throw new Error(`${at}: a snapshot id appears twice`); + const refs = snapshotRefs(row.body); + if (!refs) throw new Error(`${at}: body is not a snapshot (no {nodes: [{kind, hash}]})`); + draft.snapshotIds.add(row.id); + draft.snapshots.push(row); + draft.refs.push({ at, refs }); +} + +function addBlob(draft: Draft, value: unknown, at: string): void { + const row = rowOf(value, BLOB_FIELDS, at); + if (!SHA256.test(row.hash)) throw new Error(`${at}: hash must be 64 lowercase hex digits`); + if (Object.hasOwn(draft.blobs, row.hash)) + throw new Error(`${at}: blob ${row.hash} appears twice`); + draft.blobs[row.hash] = row.body; +} + +function addPurge(draft: Draft, value: unknown, at: string): void { + const row = rowOf(value, PURGE_FIELDS, at); + if (!SHA256.test(row.sha256)) throw new Error(`${at}: sha256 must be 64 lowercase hex digits`); + if (draft.purgedShas.has(row.sha256)) throw new Error(`${at}: ${row.sha256} appears twice`); + draft.purgedShas.add(row.sha256); + draft.purged.push(row); +} + +const PART_ROWS: Record void> = { + nodes: addNode, + snapshots: addSnapshot, + blobs: addBlob, +}; + +/** One handler per kind of buffered file. */ +const FILES: Record void> = { + project: (draft, item) => { + for (const { value, at } of lines(item)) { + if (draft.project) throw new Error(`${at}: project.json holds more than one row`); + draft.project = rowOf(value, PROJECT_FIELDS, at); + } + }, + part: (draft, item) => { + const slot = item.slot; + if (slot.kind !== "part") return; + if (draft.needed) throw new Error(`${item.path}: comes after the media`); + const add = PART_ROWS[slot.part]; + for (const { value, at } of lines(item)) add(draft, value, at); + }, + ledger: (draft, item) => { + for (const { value, at } of lines(item)) addPurge(draft, value, at); + }, +}; + +/** + * The graph checks that need every row, run once: the site node, every snapshot ref's body. Then + * the media the graph needs, taken-down content not yet left out (the ledger comes last). + */ +function needed(draft: Draft): Map { + if (draft.needed) return draft.needed; + const siteId = draft.manifest.siteId; + if (!draft.nodes.some((n) => n.kind === "site" && n.id === siteId)) + throw new Error(`nodes: the archive has no site node for ${siteId}`); + for (const { at, refs } of draft.refs) + for (const ref of refs) { + if (!Object.hasOwn(draft.blobs, ref.hash)) + throw new Error(`${at}: references a node body that is not in the archive's blobs`); + const text = draft.blobs[ref.hash]; + // `bundleMedia` reads only non-empty bodies of asset refs. + if (ref.kind !== "asset" || !text || draft.assetBlobs.has(ref.hash)) continue; + checkAssetBody(text, at, "an asset body it references"); + draft.assetBlobs.add(ref.hash); + } + const media = bundleMedia(draft, new Set()) ?? []; + draft.needed = new Map(media.map((m) => [m.sha256, m])); + return draft.needed; +} + +async function addMedia( + draft: Draft, + item: ArchiveMediaItem, + options: ReassembleOptions, +): Promise { + const asset = needed(draft).get(item.sha256); + if (!asset) throw new Error(`${item.path}: no asset of the archive needs this file`); + if (asset.bytes !== item.size) + throw new Error(`${item.path}: ${item.size} bytes, its asset says ${asset.bytes}`); + await options.onMedia?.(item); + draft.media.push({ sha256: item.sha256, size: item.size }); +} + +/** The media files against the final media (the ledger applied); `bundle.media`. */ +function finalMedia(draft: Draft): BundleMedia[] | undefined { + needed(draft); + const media = bundleMedia(draft, draft.purgedShas); + for (const m of draft.media) + if (draft.purgedShas.has(m.sha256)) + throw new Error(`media/${m.sha256}: the archive's ledger has taken this content down`); + if (!draft.manifest.mediaOmitted) { + const present = new Set(draft.media.map((m) => m.sha256)); + for (const m of media ?? []) + if (!present.has(m.sha256)) + throw new Error(`media/${m.sha256}: an asset needs it, and the archive does not hold it`); + } + return media; +} + +async function firstManifest(iterator: AsyncIterator): Promise { + const first = await iterator.next(); + if (first.done || first.value.type !== "manifest") + throw new Error("the archive's items do not start with its manifest"); + return { + assetBlobs: new Set(), + manifest: first.value.manifest, + nodes: [], + nodeIds: new Set(), + snapshots: [], + snapshotIds: new Set(), + refs: [], + blobs: {}, + purged: [], + purgedShas: new Set(), + media: [], + }; +} + +function assemble(draft: Draft): ReassembledArchive { + const media = finalMedia(draft); + const bundle: SiteBundle = { + format: SITE_BUNDLE_FORMAT, + siteId: draft.manifest.siteId, + ...(draft.project ? { project: draft.project } : {}), + nodes: draft.nodes, + snapshots: draft.snapshots, + blobs: draft.blobs, + ...(media ? { media } : {}), + ...(draft.purged.length > 0 ? { purged: draft.purged } : {}), + }; + return { manifest: draft.manifest, bundle, media: draft.media }; +} + +/** + * The bundle of an archive: reads `items` to the end (the reader verifies the whole archive only + * then) and hands each media file to `onMedia` on the way. A refusal stops the reader. + */ +export async function bundleFromArchive( + items: AsyncIterable, + options: ReassembleOptions = {}, +): Promise { + const iterator = items[Symbol.asyncIterator](); + try { + const draft = await firstManifest(iterator); + for (let next = await iterator.next(); !next.done; next = await iterator.next()) { + const item = next.value; + if (item.type === "manifest") throw new Error("manifest.json: appears twice"); + if (item.type === "media") await addMedia(draft, item, options); + else FILES[item.slot.kind](draft, item); + } + return assemble(draft); + } finally { + await iterator.return?.(); + } +} diff --git a/packages/cli/src/commands/export.ts b/packages/cli/src/commands/export.ts new file mode 100644 index 00000000..0b5f259c --- /dev/null +++ b/packages/cli/src/commands/export.ts @@ -0,0 +1,337 @@ +/** + * `cloudhal export [--out ] [--site ] [--no-media] [--latest-only]` (RFC-0039 §2, + * rollout step 2): one site of the local project as a `.cloudhal` file, its content graph, + * snapshots, the node bodies they reference, its media and its takedown ledger. + * + * - **Site:** the project's (`cloudhal.json` `siteId`), or with `--site` another one listed in + * `sites`; any other id is refused before the store is opened. + * - **File:** `-.cloudhal` in the current folder unless `--out` names one; + * the date is that of the archive's `createdAt`. The default name says only the site and a day, + * and the same day can hold other content (a later edit that day, another flag, or a deletion, + * which moves no timestamp), so a file already there is replaced only by the same archive (its + * manifest, which pins every file's SHA-256, is the new one's); any other is refused, and + * `--out` naming it replaces it. A file is replaced only once the new one is complete + * (`writeArchiveFile` writes a temporary file beside it and renames it over): a failed or + * interrupted (Ctrl-C, SIGTERM) export leaves no file and an existing one unchanged. + * - **createdAt** (invariant 7 as amended): the newest timestamp among the exported rows, never + * the clock, so an unchanged site exports to the same name and, with the same CLI and Node/zlib + * build, the same bytes. Deleting content stamps no remaining row, so it alone does not move the + * date; the next edit or snapshot does. + * - **`--latest-only`:** only the published and the latest snapshot (`latestSnapshot`, what + * `cloudhal build` falls back to), only the blobs they reference, and only the media the current + * nodes and those snapshots need; the takedown ledger still travels. Refused while an RFC-0041 + * field operation is unfinished: it needs its `before` snapshot. + * - **`--no-media`:** the graph only; the manifest says `mediaOmitted`. + * - **Never in an archive (§4):** user accounts, sessions, roles, agent keys, comments, readers, + * analytics, the audit log, automations, schedule runs, deployment secrets and webhook + * credentials. None of their tables or files are read. The graph does name who made each change, + * as its contracts require: snapshots, uploads, block edits, reviews, term proposals and + * takedowns carry user and agent ids (most with display names), entries their authors and the + * project its owner's id. + * + * A media file whose local bytes are missing, or differ from its asset's size and SHA-256, stops + * the export (`cloudhal media verify` finds them). Known limitation: the graph rows are held in + * memory (`exportSite` returns a whole bundle); media bytes stream from the blob store. + */ +import { createReadStream } from "node:fs"; +import { mkdir, stat } from "node:fs/promises"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import type { Site } from "@cloudhal/core-contracts"; +import { type BundleMedia, exportSite, mediaBlobKey, slugify } from "@cloudhal/core-ts"; +import { openLocalStore } from "@cloudhal/core-ts/local"; +import type { BlobStore } from "@cloudhal/storage"; +import { + type ArchiveContent, + ArchiveError, + archiveStream, + readArchive, + type SiteArchiveManifest, + writeArchiveFile, +} from "../archive/index.ts"; +import { cliVersion } from "../assets.ts"; +import { type ProjectConfig, readProjectConfig } from "../config.ts"; +import { bundleToArchive, latestOnly } from "./archive-bundle.ts"; + +export interface ExportArchiveOptions { + readonly projectDir: string; + /** The file to write, relative to `projectDir`; default `-.cloudhal`. */ + readonly out?: string | undefined; + /** A site listed in `cloudhal.json` `sites`; default the project's `siteId`. */ + readonly siteId?: string | undefined; + readonly noMedia?: boolean | undefined; + readonly latestOnly?: boolean | undefined; + /** Aborts the export (the command's Ctrl-C and SIGTERM): no file is written. */ + readonly signal?: AbortSignal | undefined; +} + +export interface ExportArchiveResult { + /** Absolute path of the archive. */ + readonly file: string; + readonly bytes: number; + /** A file was there before and has been replaced. */ + readonly replaced: boolean; + readonly manifest: SiteArchiveManifest; + readonly slug: string; + /** Takedown ledger rows. */ + readonly purged: number; + readonly latestOnly: boolean; +} + +/** RFC-0039 §4: what an archive never contains. */ +export const NEVER_IN_ARCHIVE: readonly string[] = [ + "user accounts", + "sessions", + "roles", + "agent keys", + "comments", + "readers", + "analytics", + "the audit log", + "automations", + "schedule runs", + "deployment secrets", + "webhook credentials", +]; + +/** What the graph does carry about people (the summary says so beside §4's list). */ +export const NAMED_IN_ARCHIVE = + "who made each change: snapshots, uploads, block edits, reviews and takedowns name their user or agent (by id, most also by display name)"; + +const SLUG = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +const SAFE_ID = /^[A-Za-z0-9_-]{1,64}$/; + +/** + * `-.cloudhal`, dated by `createdAt` in UTC. The slug is the site's (always + * valid in the store); then the slug of its name; then its id. + */ +export function archiveFileName( + site: Pick, + createdAt: string, +): string { + const base = + [site.slug, slugify(site.name)].find((s) => SLUG.test(s) && s.length <= 128) ?? + (SAFE_ID.test(site.id) ? site.id : "site"); + const date = new Date(createdAt).toISOString().slice(0, 10).replaceAll("-", ""); + return `${base}-${date}.cloudhal`; +} + +function missingMedia(sha256: string): Error { + return new Error( + `local media ${sha256} is missing; run \`cloudhal media verify\` (or export with --no-media)`, + ); +} + +/** One media file's bytes: streamed when the store can, read whole otherwise. */ +async function* mediaBytes( + blobs: BlobStore, + siteId: string, + sha256: string, +): AsyncGenerator { + const key = mediaBlobKey(siteId, sha256); + if (!blobs.getStream) { + const bytes = await blobs.get(key); + if (!bytes) throw missingMedia(sha256); + yield bytes; + return; + } + const stream = await blobs.getStream(key); + if (!stream) throw missingMedia(sha256); + const reader = stream.body.getReader(); + try { + for (let read = await reader.read(); !read.done; read = await reader.read()) yield read.value; + } finally { + // Stopped early (the writer refused the file): closes the file it reads. + await reader.cancel().catch(() => {}); + } +} + +/** The media sources' bytes, from the site's local blob store. */ +export function localMediaReader( + blobs: BlobStore, + siteId: string, +): (m: BundleMedia) => AsyncIterable { + return (m) => mediaBytes(blobs, siteId, m.sha256); +} + +/** The site to export: the project's, or one listed in `sites`. Checked before any I/O. */ +function chooseSite(cfg: ProjectConfig, requested: string | undefined): string { + const siteId = requested ?? cfg.siteId; + const allowed = cfg.sites ?? [cfg.siteId]; + if (!allowed.includes(siteId)) + throw new Error( + `site ${siteId} is not listed in cloudhal.json \`sites\` (this project's sites: ${allowed.join(", ")})`, + ); + return siteId; +} + +/** Every media file the archive will hold must be in the local blob store; one message for all. */ +async function assertLocalMedia( + blobs: BlobStore, + siteId: string, + media: readonly BundleMedia[], +): Promise { + const missing: string[] = []; + for (const m of media) + if (!(await blobs.head(mediaBlobKey(siteId, m.sha256)))) missing.push(m.sha256); + const [first] = missing; + if (first === undefined) return; + if (missing.length === 1) throw missingMedia(first); + throw new Error( + `local media ${first} and ${missing.length - 1} more are missing; run \`cloudhal media verify\` (or export with --no-media)`, + ); +} + +/** The archive's path (its folder created), and whether a file is there already. */ +async function outputFile( + projectDir: string, + out: string | undefined, + name: string, +): Promise<{ file: string; replaced: boolean }> { + const shown = out ?? name; + if (shown === "") throw new Error("--out needs a file name"); + if (shown.endsWith("/") || shown.endsWith(path.sep)) + throw new Error(`--out ${shown} is a directory; name a file`); + const file = path.resolve(projectDir, shown); + const existing = await stat(file).catch((e: NodeJS.ErrnoException) => { + if (e.code === "ENOENT") return undefined; + throw e; + }); + if (existing?.isDirectory()) throw new Error(`--out ${shown} is a directory; name a file`); + await mkdir(path.dirname(file), { recursive: true }); + return { file, replaced: existing !== undefined }; +} + +/** The manifest of the archive at `file`; undefined when it is not one (or does not read as one). */ +async function manifestAt(file: string): Promise { + const stream = createReadStream(file); + const items = readArchive(stream); + try { + const first = await items.next(); + return !first.done && first.value.type === "manifest" ? first.value.manifest : undefined; + } catch { + return undefined; + } finally { + await items.return(undefined).catch(() => {}); + stream.destroy(); + } +} + +/** + * A file at the default name is replaced only by the same archive: its manifest lists the same + * files with the same SHA-256s, so its tar is byte for byte the new one's (only another zlib + * build's gzip bytes can differ). Anything else stays; `--out` naming it replaces it. + */ +async function keepOtherArchive( + file: string, + name: string, + content: ArchiveContent, +): Promise { + const existing = await manifestAt(file); + const plain = (m: SiteArchiveManifest) => JSON.parse(JSON.stringify(m)) as unknown; + if (existing && isDeepStrictEqual(plain(existing), plain(archiveStream(content).manifest))) + return; + throw new Error( + `${name} already exists and is not this archive (other content or flags, or another CLI version); pass --out ${name} to replace it, or --out to keep both`, + ); +} + +/** `writeArchiveFile`, with a damaged local media file named as such. */ +async function writeArchive( + file: string, + content: ArchiveContent, + signal: AbortSignal | undefined, +): Promise { + try { + return await writeArchiveFile(file, content, { signal }); + } catch (e) { + if (e instanceof ArchiveError && e.code === "damaged" && e.path?.startsWith("media/")) + throw new ArchiveError( + "damaged", + `${e.message}; the local copy is damaged: run \`cloudhal media verify\` (or export with --no-media)`, + { path: e.path, cause: e }, + ); + throw e; + } +} + +/** Writes the archive; see the module doc for every rule. */ +export async function exportArchive(options: ExportArchiveOptions): Promise { + const cfg = await readProjectConfig(options.projectDir); + const siteId = chooseSite(cfg, options.siteId); + const local = await openLocalStore(path.join(options.projectDir, cfg.dataDir)); + // Open until the archive is written: the media are opened as the writer reaches them. + try { + const site = await local.store.getSite(siteId); + if (!site) + throw new Error( + `site ${siteId} is listed in cloudhal.json but the local store (${cfg.dataDir}) has no such site`, + ); + const full = await exportSite(local, siteId); + const bundle = options.latestOnly + ? latestOnly(full, (await local.store.latestSnapshot(siteId))?.id) + : full; + if (!options.noMedia) await assertLocalMedia(local.blobs, siteId, bundle.media ?? []); + const content = bundleToArchive(bundle, { + createdBy: await cliVersion(), + mediaOmitted: options.noMedia === true, + openMedia: localMediaReader(local.blobs, siteId), + }); + const name = archiveFileName(site, content.createdAt); + const { file, replaced } = await outputFile(options.projectDir, options.out, name); + if (replaced && options.out === undefined) await keepOtherArchive(file, name, content); + const manifest = await writeArchive(file, content, options.signal); + return { + file, + bytes: (await stat(file)).size, + replaced, + manifest, + slug: site.slug, + purged: bundle.purged?.length ?? 0, + latestOnly: options.latestOnly === true, + }; + } finally { + await local.close(); + } +} + +/** 1536 → `1.5 KiB`. */ +function size(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + const units = ["KiB", "MiB", "GiB", "TiB"]; + let value = bytes / 1024; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit++; + } + return `${value.toFixed(1)} ${units[unit]}`; +} + +/** + * The summary `cloudhal export` prints: paths, sizes, counts, the site id and slug, and what an + * archive never holds. Never the site name or any row content. + */ +export function describeExport(result: ExportArchiveResult, projectDir: string): string[] { + const { manifest } = result; + const { counts } = manifest; + const relative = path.relative(projectDir, result.file); + const shown = + relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative) + ? relative + : result.file; + const media = manifest.mediaOmitted + ? "" + : `, ${counts.media} media file(s) (${size(counts.mediaBytes)})`; + const lines = [ + `wrote ${shown} (${size(result.bytes)}${result.replaced ? "; replaced the existing file" : ""})`, + `site ${manifest.siteId} (${result.slug}): ${counts.nodes} node(s), ${counts.snapshots} snapshot(s), ${counts.blobs} blob(s)${media}${result.purged > 0 ? `, ${result.purged} taken down` : ""}`, + ]; + if (manifest.mediaOmitted) + lines.push("media left out (--no-media): an import needs the media from elsewhere"); + if (result.latestOnly) + lines.push("history: the published and the latest snapshot only (--latest-only)"); + lines.push(`not in the archive (RFC-0039 §4): ${NEVER_IN_ARCHIVE.join(", ")}`); + lines.push(`in the archive: ${NAMED_IN_ARCHIVE}`); + return lines; +} diff --git a/packages/cli/test/archive-bundle.test.ts b/packages/cli/test/archive-bundle.test.ts new file mode 100644 index 00000000..ea0776b3 --- /dev/null +++ b/packages/cli/test/archive-bundle.test.ts @@ -0,0 +1,508 @@ +/** + * RFC-0039 rollout step 2, the pure half: a `SiteBundle` to the container's content and back + * (`archive-bundle.ts`, `archive-reassemble.ts`) on hand-built bundles, `exportSite`'s media rule, + * `--latest-only`, the archive's date and file name, the local media reader, and every refusal of + * the reassembly. The store-backed command is in export-cli.test.ts. + */ +import type { BundleMedia, BundlePurge, NodeRow, SiteBundle, SnapshotRow } from "@cloudhal/core-ts"; +import { MemoryBlobStore } from "@cloudhal/storage"; +import { describe, expect, it } from "vitest"; +import { + type ArchiveContent, + type ArchiveMediaSource, + archiveStream, + encodeNdjson, + readArchive, + splitNdjson, +} from "../src/archive/index.ts"; +import { + bundleMedia, + bundleToArchive, + contentCreatedAt, + latestOnly, +} from "../src/commands/archive-bundle.ts"; +import { bundleFromArchive } from "../src/commands/archive-reassemble.ts"; +import { archiveFileName, localMediaReader } from "../src/commands/export.ts"; +import { chunked, collect, pattern, sha256, utf8 } from "./archive-fixture.ts"; + +const SITE_ID = "site_demo0001"; +const BYTES = { a: pattern(300, 1), b: pattern(500, 2), extra: pattern(100, 7) }; +const SHA = { + a: sha256(BYTES.a), + b: sha256(BYTES.b), + extra: sha256(BYTES.extra), + gone: sha256(utf8("taken down")), + ext: sha256(utf8("external")), +}; +const BY_SHA = new Map([ + [SHA.a, BYTES.a], + [SHA.b, BYTES.b], + [SHA.extra, BYTES.extra], +]); + +const at = (minute: number) => `2026-10-20T12:${String(minute).padStart(2, "0")}:00.000Z`; + +function asset( + id: string, + sha: string, + o: { bytes?: number; type?: string; external?: boolean } = {}, +) { + return JSON.stringify({ + id, + kind: "image", + sha256: sha, + bytes: o.bytes ?? BY_SHA.get(sha)?.length ?? 10, + mediaType: o.type ?? "image/png", + storage: o.external ? { kind: "external", url: "https://example.com/x.png" } : { kind: "blob" }, + }); +} + +function node(id: string, kind: string, body: string, minute = 1): NodeRow { + const stamp = at(minute); + return { + id, + site_id: SITE_ID, + kind, + key: id, + rev: 1, + hash: sha256(utf8(body)), + body, + created_at: stamp, + updated_at: stamp, + }; +} + +function snapshot(id: string, refs: { kind: string; text: string }[], minute: number): SnapshotRow { + const nodes = refs.map((r) => ({ kind: r.kind, hash: sha256(utf8(r.text)) })); + return { + id, + site_id: SITE_ID, + parent_id: null, + site_rev: 1, + body: JSON.stringify({ id, nodes }), + created_at: at(minute), + }; +} + +function blobsOf(...texts: string[]): Record { + const out: Record = {}; + for (const text of texts + .map((t) => [sha256(utf8(t)), t] as const) + .sort(([x], [y]) => (x < y ? -1 : 1))) + out[text[0]] = text[1]; + return out; +} + +const SITE_BODY = (published?: string) => + JSON.stringify({ + id: SITE_ID, + name: "Demo site", + slug: "demo", + ...(published ? { publishedSnapshotId: published } : {}), + }); + +/** + * A bundle as `exportSite` returns it: blob asset A current and in both snapshots, B only in the + * second, an external asset, a taken-down blob asset, a project and a ledger row. + */ +function sampleBundle(o: { aBytes?: number } = {}): SiteBundle { + const aBody = asset("asset_aaaaaa01", SHA.a, o.aBytes === undefined ? {} : { bytes: o.aBytes }); + const bBody = asset("asset_bbbbbb01", SHA.b, { type: "image/webp" }); + const site = SITE_BODY("snap_00000001"); + const nodes = [ + node("asset_aaaaaa01", "asset", aBody), + node("asset_eeeeee01", "asset", asset("asset_eeeeee01", SHA.ext, { external: true })), + node("asset_gggggg01", "asset", asset("asset_gggggg01", SHA.gone)), + node(SITE_ID, "site", site, 3), + ]; + const snapshots = [ + snapshot( + "snap_00000001", + [ + { kind: "asset", text: aBody }, + { kind: "site", text: site }, + ], + 4, + ), + snapshot( + "snap_00000002", + [ + { kind: "asset", text: aBody }, + { kind: "asset", text: bBody }, + ], + 5, + ), + ]; + const blobs = blobsOf(aBody, bBody, site); + const purged = [ + { sha256: SHA.gone, reason: "notice", legal: false, by: "human:user_x", at: at(6) }, + ]; + const media = bundleMedia({ nodes, snapshots, blobs }, new Set([SHA.gone])); + return { + format: "cloudhal.site-bundle@1", + siteId: SITE_ID, + project: { id: "prj_demo0001", rev: 2, body: "{}", created_at: at(0), updated_at: at(2) }, + nodes, + snapshots, + blobs, + ...(media ? { media } : {}), + purged, + }; +} + +const openMedia = (m: BundleMedia) => chunked(BY_SHA.get(m.sha256) as Uint8Array, 64); + +async function roundTrip( + content: ArchiveContent, + onMedia?: (sha: string) => void, +): Promise { + const bytes = await collect(archiveStream(content).bytes); + const result = await bundleFromArchive(readArchive(chunked(bytes, 1000)), { + onMedia: async (item) => { + onMedia?.(item.sha256); + await collect(item.body); + }, + }); + return result.bundle; +} + +describe("bundle → archive → bundle", () => { + it("comes back exactly, over several parts, with media streamed", async () => { + const bundle = sampleBundle(); + expect(bundle.media?.map((m) => m.sha256)).toEqual([SHA.a, SHA.b].sort()); + const content = bundleToArchive(bundle, { createdBy: "0.6.1", partBytes: 200, openMedia }); + expect(content.nodes.length).toBeGreaterThan(1); + expect(content.createdAt).toBe(at(6)); + expect(content.siteName).toBe("Demo site"); + const seen: string[] = []; + expect(await roundTrip(content, (sha) => seen.push(sha))).toStrictEqual(bundle); + expect(seen).toEqual([SHA.a, SHA.b].sort()); + }); + + it("leaves out project, media and purged when the bundle has none", async () => { + const site = SITE_BODY(); + const bundle: SiteBundle = { + format: "cloudhal.site-bundle@1", + siteId: SITE_ID, + nodes: [ + node("asset_eeeeee01", "asset", asset("asset_eeeeee01", SHA.ext, { external: true })), + node(SITE_ID, "site", site), + ], + snapshots: [snapshot("snap_00000001", [{ kind: "site", text: site }], 2)], + blobs: blobsOf(site), + }; + const content = bundleToArchive(bundle, { createdBy: "0.6.1", openMedia }); + const { manifest } = archiveStream(content); + expect(manifest.files.map((f) => f.path)).toEqual([ + "nodes/0001.ndjson", + "snapshots/0001.ndjson", + "blobs/0001.ndjson", + ]); + const back = await roundTrip(content); + expect(back).toStrictEqual(bundle); + expect(Object.keys(back)).toEqual(["format", "siteId", "nodes", "snapshots", "blobs"]); + }); + + it("--no-media: no media files, and the media rebuilt from the rows", async () => { + const bundle = sampleBundle(); + const content = bundleToArchive(bundle, { createdBy: "0.6.1", mediaOmitted: true, openMedia }); + expect(content.media).toBeUndefined(); + expect(archiveStream(content).manifest.mediaOmitted).toBe(true); + expect(await roundTrip(content)).toStrictEqual(bundle); + }); + + it("refuses to write a snapshot whose node body the bundle lacks", () => { + const bundle = sampleBundle(); + const { [Object.keys(bundle.blobs)[0] as string]: _dropped, ...blobs } = bundle.blobs; + expect(() => bundleToArchive({ ...bundle, blobs }, { createdBy: "0.6.1", openMedia })).toThrow( + /not in the bundle/, + ); + }); +}); + +describe("bundleMedia (exportSite's rule)", () => { + it("lets the last body seen win, skips external and taken-down content, sorts by hash", () => { + const current = asset("asset_aaaaaa01", SHA.a, { type: "image/png" }); + const later = asset("asset_aaaaaa01", SHA.a, { type: "image/webp" }); + const rows = { + nodes: [ + node("asset_aaaaaa01", "asset", current), + node("asset_bbbbbb01", "asset", asset("asset_bbbbbb01", SHA.b)), + node("asset_eeeeee01", "asset", asset("asset_eeeeee01", SHA.ext, { external: true })), + node("asset_gggggg01", "asset", asset("asset_gggggg01", SHA.gone)), + ], + snapshots: [snapshot("snap_00000001", [{ kind: "asset", text: later }], 2)], + blobs: blobsOf(later), + }; + expect(bundleMedia(rows, new Set([SHA.gone]))).toEqual( + [ + { sha256: SHA.a, bytes: 300, mediaType: "image/webp" }, + { sha256: SHA.b, bytes: 500, mediaType: "image/png" }, + ].sort((x, y) => (x.sha256 < y.sha256 ? -1 : 1)), + ); + expect(bundleMedia({ nodes: [], snapshots: [], blobs: {} }, new Set())).toBeUndefined(); + }); +}); + +describe("latestOnly", () => { + function threeSnapshots(published: string) { + const site = SITE_BODY(published); + const a = asset("asset_aaaaaa01", SHA.a); + const b = asset("asset_bbbbbb01", SHA.b); + const s2 = { ...snapshot("snap_00000002", [{ kind: "asset", text: b }], 5) }; + const s3 = { + ...snapshot("snap_00000003", [{ kind: "site", text: site }], 6), + parent_id: s2.id, + }; + const bundle: SiteBundle = { + format: "cloudhal.site-bundle@1", + siteId: SITE_ID, + nodes: [node(SITE_ID, "site", site)], + snapshots: [snapshot("snap_00000001", [{ kind: "asset", text: a }], 4), s2, s3], + blobs: blobsOf(a, b, site), + media: [], + purged: [{ sha256: SHA.gone, reason: "", legal: true, by: "", at: "" }], + }; + return { bundle, a, b, site }; + } + + it("keeps the published and the latest snapshot, their blobs and media; purged stays", () => { + const { bundle, a, site } = threeSnapshots("snap_00000001"); + const kept = latestOnly(bundle, "snap_00000003"); + expect(kept.snapshots.map((s) => s.id)).toEqual(["snap_00000001", "snap_00000003"]); + expect(kept.snapshots[1]?.parent_id).toBe("snap_00000002"); + expect(kept.blobs).toEqual(blobsOf(a, site)); + expect(kept.media).toEqual([{ sha256: SHA.a, bytes: 300, mediaType: "image/png" }]); + expect(kept.purged).toEqual(bundle.purged); + expect(kept.nodes).toBe(bundle.nodes); + }); + + it("refuses a site whose field operation is unfinished (its before snapshot would go)", () => { + const { bundle } = threeSnapshots("snap_00000003"); + const marker = { id: "snap_00000002", fromKey: "speaker" }; + const field = { key: "presenter", type: "text", localized: false, migration: marker }; + const site = JSON.stringify({ + ...JSON.parse(SITE_BODY("snap_00000003")), + collections: { posts: { fields: [] }, events: { fields: [field] } }, + }); + expect(() => latestOnly({ ...bundle, nodes: [node(SITE_ID, "site", site)] })).toThrow( + "field operation snap_00000002 on events has not finished, and --latest-only would drop the snapshot it needs", + ); + }); + + it("keeps one when published is latest, and falls back to the last row without an id", () => { + const { bundle, site } = threeSnapshots("snap_00000003"); + const one = latestOnly(bundle); + expect(one.snapshots.map((s) => s.id)).toEqual(["snap_00000003"]); + expect(one.blobs).toEqual(blobsOf(site)); + expect(one).not.toHaveProperty("media"); + }); +}); + +describe("contentCreatedAt", () => { + it("is the newest row timestamp, the ledger's included, never the clock", () => { + const bundle = sampleBundle(); + expect(contentCreatedAt(bundle)).toBe(at(6)); + const purge = bundle.purged?.[0] as BundlePurge; + expect(contentCreatedAt({ ...bundle, purged: [{ ...purge, at: "" }] })).toBe(at(5)); + const offset = { ...purge, at: "2026-10-21T01:30:00+02:00" }; + expect(contentCreatedAt({ ...bundle, purged: [offset] })).toBe("2026-10-20T23:30:00.000Z"); + }); + + it("refuses a bundle without a single timestamp", () => { + const bare = { + format: "cloudhal.site-bundle@1" as const, + siteId: SITE_ID, + nodes: [{ ...node(SITE_ID, "site", SITE_BODY()), created_at: "", updated_at: "" }], + snapshots: [], + blobs: {}, + }; + expect(() => contentCreatedAt(bare)).toThrow(/no row with a timestamp/); + }); +}); + +describe("archiveFileName", () => { + it("is -.cloudhal, falling back to the name's slug, then the id", () => { + const site = { id: SITE_ID, slug: "english", name: "English" }; + expect(archiveFileName(site, "2026-10-25T09:00:00.000Z")).toBe("english-20261025.cloudhal"); + expect(archiveFileName(site, "2026-10-25T01:30:00+02:00")).toBe("english-20261024.cloudhal"); + expect(archiveFileName({ ...site, slug: "Not A Slug", name: "Türkçe Blog" }, at(0))).toBe( + "turkce-blog-20261020.cloudhal", + ); + expect(archiveFileName({ ...site, slug: "", name: "日本" }, at(0))).toBe( + `${SITE_ID}-20261020.cloudhal`, + ); + }); +}); + +describe("localMediaReader", () => { + const key = (sha: string) => `media/${SITE_ID}/${sha}`; + const m = { sha256: SHA.a, bytes: BYTES.a.length, mediaType: "image/png" }; + + it("streams through getStream, falls back to get, and names a missing file", async () => { + const store = new MemoryBlobStore(); + await store.put(key(SHA.a), BYTES.a); + expect(Buffer.compare(await collect(localMediaReader(store, SITE_ID)(m)), BYTES.a)).toBe(0); + const plain = { + name: "plain", + get: (k: string) => store.get(k), + head: (k: string) => store.head(k), + put: (k: string, d: Uint8Array) => store.put(k, d), + delete: (k: string) => store.delete(k), + list: (p?: string) => store.list(p), + }; + expect(Buffer.compare(await collect(localMediaReader(plain, SITE_ID)(m)), BYTES.a)).toBe(0); + const absent = { ...m, sha256: SHA.b }; + await expect(collect(localMediaReader(store, SITE_ID)(absent))).rejects.toThrow( + /local media [0-9a-f]{64} is missing/, + ); + await expect(collect(localMediaReader(plain, SITE_ID)(absent))).rejects.toThrow(/is missing/); + }); +}); + +/** Raw rows of the sample bundle, for crafted archives the writer does not judge. */ +function rawContent(edit: { + nodes?: (rows: Record[]) => unknown[]; + blobs?: (rows: { hash: string; body: string }[]) => unknown[]; + ledger?: unknown[]; + media?: (sources: ArchiveMediaSource[]) => ArchiveMediaSource[]; + bundle?: SiteBundle; +}): ArchiveContent { + const bundle = edit.bundle ?? sampleBundle(); + const content = bundleToArchive(bundle, { createdBy: "0.6.1", openMedia }); + const nodes = bundle.nodes.map((n) => ({ ...n }) as Record); + const blobs = Object.entries(bundle.blobs).map(([hash, body]) => ({ hash, body })); + return { + ...content, + nodes: splitNdjson(edit.nodes ? edit.nodes(nodes) : nodes), + blobs: splitNdjson(edit.blobs ? edit.blobs(blobs) : blobs), + ...(edit.ledger ? { ledger: encodeNdjson(edit.ledger) } : {}), + media: edit.media ? edit.media([...(content.media ?? [])]) : content.media, + }; +} + +async function refusal(content: ArchiveContent, seen: string[] = []): Promise { + const error = await roundTrip(content, (sha) => seen.push(sha)).then( + () => undefined, + (e: unknown) => e, + ); + expect(error).toBeInstanceOf(Error); + return (error as Error).message; +} + +describe("bundleFromArchive refuses", () => { + it("rows that are not exactly the bundle's", async () => { + expect(await refusal(rawContent({ nodes: (r) => [...r, r[0]] }))).toMatch( + /^nodes\/0001\.ndjson line 5: a node id appears twice$/, + ); + expect( + await refusal(rawContent({ nodes: (r) => [{ ...r[0], extra: 1 }, ...r.slice(1)] })), + ).toMatch(/^nodes\/0001\.ndjson line 1: needs exactly the fields id, site_id, kind/); + expect( + await refusal(rawContent({ nodes: (r) => [{ ...r[0], rev: "1" }, ...r.slice(1)] })), + ).toBe("nodes/0001.ndjson line 1: rev must be a non-negative integer"); + expect( + await refusal( + rawContent({ nodes: (r) => [...r.slice(0, 3), { ...r[3], site_id: "site_other001" }] }), + ), + ).toBe("nodes/0001.ndjson line 4: site_id is not the archive's site"); + expect(await refusal(rawContent({ nodes: (r) => r.slice(0, 3) }))).toBe( + `nodes: the archive has no site node for ${SITE_ID}`, + ); + }); + + it("lines that are not JSON or not UTF-8", async () => { + expect(await refusal({ ...rawContent({}), nodes: [utf8('{"id":\n')] })).toBe( + "nodes/0001.ndjson line 1: not JSON", + ); + expect( + await refusal({ ...rawContent({}), nodes: [Uint8Array.from([0x7b, 0xff, 0x7d, 0x0a])] }), + ).toBe("nodes/0001.ndjson: not UTF-8"); + }); + + it("a snapshot ref whose node body is not in the archive", async () => { + const message = await refusal(rawContent({ blobs: (rows) => rows.slice(1) })); + expect(message).toMatch(/^snapshots\/0001\.ndjson line \d: references a node body/); + }); + + it("media no asset needs, before onMedia sees them", async () => { + const seen: string[] = []; + const extra = { + sha256: SHA.extra, + bytes: BYTES.extra.length, + open: () => chunked(BYTES.extra, 64), + }; + const message = await refusal(rawContent({ media: (m) => [...m, extra] }), seen); + expect(message).toBe(`media/${SHA.extra}: no asset of the archive needs this file`); + expect(seen).not.toContain(SHA.extra); + }); + + it("media the ledger took down (invariant 5), missing media and a size the asset disagrees with", async () => { + const ledger = [ + { sha256: SHA.a, reason: "notice", legal: true, by: "human:user_x", at: at(7) }, + ]; + expect(await refusal(rawContent({ ledger }))).toBe( + `media/${SHA.a}: the archive's ledger has taken this content down`, + ); + expect(await refusal(rawContent({ media: (m) => m.filter((s) => s.sha256 !== SHA.b) }))).toBe( + `media/${SHA.b}: an asset needs it, and the archive does not hold it`, + ); + const bundle = sampleBundle({ aBytes: 301 }); + const sources = (m: ArchiveMediaSource[]) => + m.map((s) => (s.sha256 === SHA.a ? { ...s, bytes: BYTES.a.length } : s)); + expect(await refusal(rawContent({ bundle, media: sources }))).toBe( + `media/${SHA.a}: 300 bytes, its asset says 301`, + ); + }); + + it("asset bodies the media rule cannot trust, without quoting them", async () => { + const evil = "\u001b[31mEVIL"; + const blobAsset = (extra: Record) => + JSON.stringify({ ...JSON.parse(asset("asset_aaaaaa01", SHA.a)), ...extra }); + const withBody = (body: string) => + rawContent({ + nodes: (rows) => rows.map((n) => (n.id === "asset_aaaaaa01" ? { ...n, body } : n)), + }); + const line = "nodes/0001.ndjson line 1: the asset body"; + for (const [body, message] of [ + [evil, `${line} is not JSON`], + ["null", `${line} is not a JSON object`], + [blobAsset({ sha256: evil }), `${line} needs a sha256 of 64 lowercase hex digits`], + [blobAsset({ bytes: evil }), `${line} needs bytes, a non-negative integer`], + [blobAsset({ bytes: -1 }), `${line} needs bytes, a non-negative integer`], + [blobAsset({ mediaType: 7 }), `${line} needs a mediaType string`], + ] as const) { + const got = await refusal(withBody(body)); + expect(got).toBe(message); + expect(got).not.toContain("\u001b"); + } + // The body a snapshot's asset ref points at. + const aHash = sha256(utf8(asset("asset_aaaaaa01", SHA.a))); + const got = await refusal( + rawContent({ + blobs: (rows) => rows.map((b) => (b.hash === aHash ? { ...b, body: evil } : b)), + }), + ); + expect(got).toBe("snapshots/0001.ndjson line 1: an asset body it references is not JSON"); + // An external asset's hash is not the media rule's business. + const external = asset("asset_eeeeee01", SHA.ext, { external: true }); + const odd = JSON.stringify({ ...JSON.parse(external), sha256: evil }); + const bundle = sampleBundle(); + const nodes = bundle.nodes.map((n) => (n.id === "asset_eeeeee01" ? { ...n, body: odd } : n)); + expect( + await roundTrip(bundleToArchive({ ...bundle, nodes }, { createdBy: "0.6.1", openMedia })), + ).toStrictEqual({ + ...bundle, + nodes, + }); + }); + + it("and never echoes untrusted ids", async () => { + const evil = "page_\u001b[31mred\u001b[0m"; + const message = await refusal( + rawContent({ nodes: (r) => [{ ...r[0], id: evil }, { ...r[0], id: evil }, ...r.slice(1)] }), + ); + expect(message).toBe("nodes/0001.ndjson line 2: a node id appears twice"); + expect(message).not.toContain("\u001b"); + }); +}); diff --git a/packages/cli/test/archive-roundtrip.test.ts b/packages/cli/test/archive-roundtrip.test.ts index 6dfa7c26..ffe07857 100644 --- a/packages/cli/test/archive-roundtrip.test.ts +++ b/packages/cli/test/archive-roundtrip.test.ts @@ -283,6 +283,27 @@ describe("media verified while writing", () => { await expectArchiveError(readAll(cut), "truncated"); }); + it("stops on an abort, removing its temporary file and leaving an existing file as it was", async () => { + await writeFile(target(), "old"); + const m = MEDIA[0] as Uint8Array; + const controller = new AbortController(); + async function* interrupted() { + yield m.subarray(0, 100); + controller.abort(); + await new Promise((resolve) => setTimeout(resolve, 20)); + yield m.subarray(100); + } + const write = writeArchiveFile(target(), withMedia(interrupted), { signal: controller.signal }); + await expect(write).rejects.toMatchObject({ name: "AbortError" }); + expect(await readdir(dir)).toEqual(["site.cloudhal"]); + expect(await readFile(target(), "utf8")).toBe("old"); + await rm(target()); + // Aborted before it starts: no file is opened. + const early = writeArchiveFile(target(), sampleContent(), { signal: AbortSignal.abort() }); + await expect(early).rejects.toMatchObject({ name: "AbortError" }); + expect(await readdir(dir)).toEqual([]); + }); + it("writes a file that reads back from disk", async () => { const manifest = await writeArchiveFile(target(), sampleContent()); let read: unknown; diff --git a/packages/cli/test/export-cli.test.ts b/packages/cli/test/export-cli.test.ts new file mode 100644 index 00000000..87e3b715 --- /dev/null +++ b/packages/cli/test/export-cli.test.ts @@ -0,0 +1,613 @@ +/** + * `cloudhal export` (RFC-0039 §2, rollout step 2) on a real local project: what it writes reads + * back into exactly the bundle `exportSite` returns (§1, invariant 1), the same content gives the + * same bytes (invariant 7), the file is dated by its rows (seeded on a fixed day, not today), + * `--no-media`, `--latest-only` and `--site`, a file at the default name is replaced only by the + * same archive, a damaged local media file or Ctrl-C fails the export and leaves no file + * (invariant 3), taken-down content never travels as bytes (invariant 5), nothing from §4's list + * ends up in the file or the summary, and `importSite` refuses a crafted project row (step 3's + * path). + */ +import { createReadStream } from "node:fs"; +import { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { gunzipSync } from "node:zlib"; +import type { Actor, ContentEntry, Page, Site } from "@cloudhal/core-contracts"; +import { + continueFieldOperation, + exportSite, + importSite, + runFieldOperation, + type SiteBundle, + seedStarterSite, +} from "@cloudhal/core-ts"; +import { openLocalStore } from "@cloudhal/core-ts/local"; +import { makePng, PNG_1X1 } from "@cloudhal/core-ts/media/fixtures"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { ArchiveError, archiveStream, readArchive } from "../src/archive/index.ts"; +import { HELP } from "../src/cli/help.ts"; +import { main } from "../src/cli.ts"; +import { bundleToArchive, latestOnly, snapshotRefs } from "../src/commands/archive-bundle.ts"; +import { bundleFromArchive } from "../src/commands/archive-reassemble.ts"; +import { + describeExport, + exportArchive, + NAMED_IN_ARCHIVE, + NEVER_IN_ARCHIVE, +} from "../src/commands/export.ts"; +import { writeProjectConfig } from "../src/config.ts"; + +const person: Actor = { kind: "human", id: "user_01HZX8K3Q9CD", name: "Halil" }; +const robot: Actor = { kind: "agent", id: "agent_01HZX8K3Q9YZ", name: "Newsroom Agent" }; + +interface Fixture { + projectDir: string; + siteId: string; + trId: string; + /** A: the entry's cover; B: taken down; C: deleted, kept by the first snapshot only. */ + sha: { a: string; b: string; c: string }; + snapshots: { s1: string; s2: string; s3: string }; +} + +const dirs: string[] = []; +async function tempDir(prefix: string): Promise { + const dir = await mkdtemp(path.join(tmpdir(), prefix)); + dirs.push(dir); + return dir; +} + +async function seed(projectDir: string): Promise> { + const local = await openLocalStore(path.join(projectDir, ".cloudhal")); + const { store } = local; + const site = await seedStarterSite(store, { name: "English", slug: "english" }); + const upload = (bytes: Uint8Array, filename: string) => + store.ingestAsset({ + siteId: site.id, + bytes, + filename, + uploadedBy: robot, + source: { kind: "upload" }, + }); + const a = await upload(PNG_1X1, "pixel.png"); + const entry = (await store.listContentEntries(site.id))[0] as ContentEntry; + await store.updateContentEntry({ + ...entry, + coverImage: { assetId: a.asset.id, alt: "" } as never, + }); + const b = await upload(makePng({ width: 3, height: 3 }), "b.png"); + const c = await upload(makePng({ width: 5, height: 4 }), "c.png"); + const s1 = await store.createSnapshot(site.id, { actor: person, reason: "manual" }); + await store.deleteAsset(c.asset); + await store.purgeBlob(site.id, b.asset.sha256, person, { text: "personal data", legal: false }); + const s2 = await store.createSnapshot(site.id, { actor: person, reason: "publish" }); + await store.publish((await store.getSite(site.id)) as Site, s2.id); + const about = (await store.listPages(site.id)).find((p) => p.path === "/about") as Page; + await store.updatePage({ ...about, title: "About us" }); + const s3 = await store.createSnapshot(site.id, { actor: person, reason: "manual" }); + const tr = await seedStarterSite(store, { + name: "Türkçe", + slug: "turkce", + locale: "tr", + projectId: site.projectId, + }); + await local.close(); + await writeProjectConfig(projectDir, { + siteId: site.id, + dataDir: ".cloudhal", + sites: [site.id, tr.id], + }); + return { + siteId: site.id, + trId: tr.id, + sha: { a: a.asset.sha256, b: b.asset.sha256, c: c.asset.sha256 }, + snapshots: { s1: s1.id, s2: s2.id, s3: s3.id }, + }; +} + +/** The fixture's rows are written on this day, so a file name dated by the clock would show. */ +const SEEDED_AT = new Date("2026-01-02T03:04:05.000Z"); +const SEEDED_DAY = "20260102"; + +let fx: Fixture; +beforeAll(async () => { + const projectDir = await tempDir("cloudhal-export-"); + vi.useFakeTimers({ toFake: ["Date"], now: SEEDED_AT, shouldAdvanceTime: true }); + try { + fx = { projectDir, ...(await seed(projectDir)) }; + } finally { + vi.useRealTimers(); + } +}); +afterAll(async () => { + for (const d of dirs.splice(0)) await rm(d, { recursive: true, force: true }); +}); +afterEach(() => vi.restoreAllMocks()); + +/** A copy of the fixture project (without the archives other tests wrote), for tests that change it. */ +async function freshProject(): Promise { + const dir = await tempDir("cloudhal-export-copy-"); + await cp(fx.projectDir, dir, { recursive: true }); + for (const f of await readdir(dir)) + if (f !== ".cloudhal" && f.endsWith(".cloudhal")) await rm(path.join(dir, f)); + return dir; +} + +/** The local store of `projectDir`, open for `fn` only. */ +async function withStore( + projectDir: string, + fn: (local: Awaited>) => Promise, +): Promise { + const local = await openLocalStore(path.join(projectDir, ".cloudhal")); + try { + return await fn(local); + } finally { + await local.close(); + } +} + +const bundleOf = (projectDir: string, siteId: string): Promise => + withStore(projectDir, (local) => exportSite(local, siteId)); + +const latestId = (projectDir: string, siteId: string): Promise => + withStore( + projectDir, + async (local) => ((await local.store.latestSnapshot(siteId)) as { id: string }).id, + ); + +const partials = async (dir: string) => (await readdir(dir)).filter((f) => f.includes(".partial-")); + +const readBack = (file: string) => bundleFromArchive(readArchive(createReadStream(file))); +const day = (createdAt: string) => createdAt.slice(0, 10).replaceAll("-", ""); + +/** The newest timestamp of the bundle's rows, computed apart from `contentCreatedAt`. */ +function newest(bundle: SiteBundle): string { + const stamps = [ + ...bundle.nodes.flatMap((n) => [n.created_at, n.updated_at]), + ...bundle.snapshots.map((s) => s.created_at), + ...(bundle.project ? [bundle.project.created_at, bundle.project.updated_at] : []), + ...(bundle.purged ?? []).map((p) => p.at), + ]; + return new Date( + Math.max(...stamps.map((s) => Date.parse(s)).filter((t) => !Number.isNaN(t))), + ).toISOString(); +} + +describe("cloudhal export", () => { + it("writes -.cloudhal that reads back into exactly exportSite's bundle", async () => { + const result = await exportArchive({ projectDir: fx.projectDir }); + const { manifest } = result; + // Dated by the rows (seeded on SEEDED_AT), not by today. + expect(result.file).toBe(path.join(fx.projectDir, `english-${SEEDED_DAY}.cloudhal`)); + expect(day(manifest.createdAt)).toBe(SEEDED_DAY); + expect(result.replaced).toBe(false); + expect((await stat(result.file)).size).toBe(result.bytes); + const pkg = JSON.parse( + await readFile(path.join(import.meta.dirname, "../package.json"), "utf8"), + ) as { + version: string; + }; + expect(manifest).toMatchObject({ + siteId: fx.siteId, + siteName: "English", + createdBy: pkg.version, + }); + const media = [fx.sha.a, fx.sha.c].sort(); + expect(manifest.files.map((f) => f.path)).toEqual([ + "project.json", + "nodes/0001.ndjson", + "snapshots/0001.ndjson", + "blobs/0001.ndjson", + ...media.map((sha) => `media/${sha}`), + "ledger.ndjson", + ]); + const back = await readBack(result.file); + const bundle = await bundleOf(fx.projectDir, fx.siteId); + expect(back.bundle).toStrictEqual(bundle); + expect(back.bundle.media?.map((m) => m.sha256)).toEqual(media); + expect(back.bundle.purged?.map((p) => p.sha256)).toEqual([fx.sha.b]); + expect(back.media.map((m) => m.sha256)).not.toContain(fx.sha.b); + expect(manifest.createdAt).toBe(newest(bundle)); + }); + + it("gives the same bytes for the same content, and replaces the default file", async () => { + const first = await exportArchive({ projectDir: fx.projectDir, out: "one.cloudhal" }); + await new Promise((resolve) => setTimeout(resolve, 20)); + const second = await exportArchive({ projectDir: fx.projectDir, out: "two.cloudhal" }); + const bytes = await readFile(first.file); + expect(Buffer.compare(bytes, await readFile(second.file))).toBe(0); + await exportArchive({ projectDir: fx.projectDir }); + const again = await exportArchive({ projectDir: fx.projectDir }); + expect(again.replaced).toBe(true); + expect(Buffer.compare(bytes, await readFile(again.file))).toBe(0); + expect(describeExport(again, fx.projectDir)[0]).toMatch(/; replaced the existing file\)$/); + }); + + it("--no-media writes the graph only; the media list is rebuilt from it", async () => { + const result = await exportArchive({ + projectDir: fx.projectDir, + out: "graph.cloudhal", + noMedia: true, + }); + expect(result.manifest.mediaOmitted).toBe(true); + expect(result.manifest.counts.media).toBe(0); + expect(result.manifest.files.some((f) => f.path.startsWith("media/"))).toBe(false); + const back = await readBack(result.file); + expect(back.media).toEqual([]); + expect(back.bundle).toStrictEqual(await bundleOf(fx.projectDir, fx.siteId)); + }); + + it("--latest-only keeps the published and the latest snapshot and what they need", async () => { + const result = await exportArchive({ + projectDir: fx.projectDir, + out: "latest.cloudhal", + latestOnly: true, + }); + const { bundle } = await readBack(result.file); + const latest = await latestId(fx.projectDir, fx.siteId); + expect(latest).toBe(fx.snapshots.s3); + expect(new Set(bundle.snapshots.map((s) => s.id))).toEqual( + new Set([fx.snapshots.s2, fx.snapshots.s3]), + ); + const refs = bundle.snapshots.flatMap((s) => (snapshotRefs(s.body) ?? []).map((r) => r.hash)); + expect(Object.keys(bundle.blobs).sort()).toEqual([...new Set(refs)].sort()); + expect(bundle.media?.map((m) => m.sha256)).toEqual([fx.sha.a]); + expect(bundle.purged?.map((p) => p.sha256)).toEqual([fx.sha.b]); + expect(bundle).toStrictEqual(latestOnly(await bundleOf(fx.projectDir, fx.siteId), latest)); + expect(describeExport(result, fx.projectDir)).toContain( + "history: the published and the latest snapshot only (--latest-only)", + ); + }); + + it("--site exports another listed site and refuses any other id", async () => { + const result = await exportArchive({ projectDir: fx.projectDir, siteId: fx.trId }); + expect(result.manifest.siteId).toBe(fx.trId); + expect(path.basename(result.file)).toBe(`turkce-${SEEDED_DAY}.cloudhal`); + expect((await readBack(result.file)).bundle).toStrictEqual( + await bundleOf(fx.projectDir, fx.trId), + ); + const before = await readdir(fx.projectDir); + await expect( + exportArchive({ projectDir: fx.projectDir, siteId: "site_notlisted01" }), + ).rejects.toThrow(/site site_notlisted01 is not listed in cloudhal\.json `sites`/); + expect(await readdir(fx.projectDir)).toEqual(before); + + const projectDir = await freshProject(); + await writeProjectConfig(projectDir, { + siteId: fx.siteId, + dataDir: ".cloudhal", + sites: [fx.siteId, fx.trId, "site_ghost00001"], + }); + await expect(exportArchive({ projectDir, siteId: "site_ghost00001" })).rejects.toThrow( + /the local store \(\.cloudhal\) has no such site/, + ); + }); +}); + +describe("cloudhal export and a file already at the default name", () => { + const name = `english-${SEEDED_DAY}.cloudhal`; + const refused = new RegExp( + `^${name.replaceAll(".", "\\.")} already exists and is not this archive .*; pass --out ${name.replaceAll(".", "\\.")} to replace it`, + ); + + it("is replaced only by the same archive: not by another flag, a deletion or another file", async () => { + const projectDir = await freshProject(); + const first = await exportArchive({ projectDir }); + expect(path.basename(first.file)).toBe(name); + const bytes = await readFile(first.file); + const unchanged = async () => { + expect(Buffer.compare(await readFile(first.file), bytes)).toBe(0); + expect(await partials(projectDir)).toEqual([]); + }; + // Same content, same day, other flags: the full backup stays. + for (const flags of [{ latestOnly: true }, { noMedia: true }]) { + await expect(exportArchive({ projectDir, ...flags })).rejects.toThrow(refused); + await unchanged(); + } + // A deletion stamps no remaining row, so the date stays and the name is the same. + await withStore(projectDir, async ({ store }) => { + const [entry] = await store.listContentEntries(fx.siteId); + await store.deleteContentEntry(entry as ContentEntry); + }); + await expect(exportArchive({ projectDir })).rejects.toThrow(refused); + await unchanged(); + // A file that is no archive at all stays too. + const other = path.join(projectDir, `turkce-${SEEDED_DAY}.cloudhal`); + await writeFile(other, "my notes"); + await expect(exportArchive({ projectDir, siteId: fx.trId })).rejects.toThrow( + /^turkce-\d{8}\.cloudhal already exists and is not this archive/, + ); + expect(await readFile(other, "utf8")).toBe("my notes"); + // --out naming it replaces it. + const replaced = await exportArchive({ projectDir, out: name }); + expect(replaced).toMatchObject({ file: first.file, replaced: true }); + expect((await readBack(first.file)).bundle).toStrictEqual( + await bundleOf(projectDir, fx.siteId), + ); + }); +}); + +describe("cloudhal export --latest-only and the store's history", () => { + it("keeps the snapshot latestSnapshot picks when two share a created_at", async () => { + const projectDir = await freshProject(); + // The newest snapshot (by insertion) ties with an earlier one whose id sorts after it, so the + // bundle's last row (created_at, id) is not the store's latest (created_at, rowid). + const tie = await withStore(projectDir, async ({ store, sql }) => { + for (let i = 0; i < 40; i++) { + const rows = await sql.all<{ id: string; created_at: string }>( + "SELECT id, created_at FROM snapshots WHERE site_id = ? ORDER BY rowid", + [fx.siteId], + ); + const newest = rows.at(-1) as { id: string; created_at: string }; + const earlier = rows.slice(0, -1).find((r) => r.id > newest.id); + if (earlier) { + await sql.run("UPDATE snapshots SET created_at = ? WHERE id = ?", [ + newest.created_at, + earlier.id, + ]); + return { newest: newest.id, earlier: earlier.id }; + } + await store.createSnapshot(fx.siteId, { actor: person, reason: "manual" }); + } + throw new Error("no two snapshots to tie"); + }); + expect(await latestId(projectDir, fx.siteId)).toBe(tie.newest); + expect((await bundleOf(projectDir, fx.siteId)).snapshots.at(-1)?.id).toBe(tie.earlier); + const result = await exportArchive({ projectDir, out: "tie.cloudhal", latestOnly: true }); + const { bundle } = await readBack(result.file); + expect(new Set(bundle.snapshots.map((s) => s.id))).toEqual( + new Set([fx.snapshots.s2, tie.newest]), + ); + }); + + it("is refused while a field operation is unfinished, and works once it is done", async () => { + const projectDir = await freshProject(); + const op = { + op: "rename_field" as const, + collection: "events", + from: "speaker", + to: "presenter", + }; + const operationId = await withStore(projectDir, async ({ store }) => { + const site = (await store.getSite(fx.siteId)) as Site; + const speaker = { key: "speaker", type: "text", labels: { en: { label: "Speaker" } } }; + await store.updateSite({ + ...site, + collections: { + ...site.collections, + events: { fields: [{ ...speaker, localized: false }] }, + } as never, + }); + for (const slug of ["one", "two", "three"]) + await store.createContentEntry({ + siteId: fx.siteId, + collection: "events", + slug, + locale: "en", + status: "draft", + title: slug, + body: { format: "markdown", value: "Body" }, + fields: { speaker: slug }, + tags: [], + author: person, + } as never); + const part = await runFieldOperation(store, fx.siteId, op, { + actor: person, + budget: { entries: 1 }, + }); + expect(part.done).toBe(false); + // A later snapshot: the operation's `before` is neither the published nor the latest one. + await store.createSnapshot(fx.siteId, { actor: person, reason: "manual" }); + return part.operationId; + }); + const out = path.join(projectDir, "template.cloudhal"); + await expect( + exportArchive({ projectDir, out: "template.cloudhal", latestOnly: true }), + ).rejects.toThrow( + `field operation ${operationId} on events has not finished, and --latest-only would drop the snapshot it needs; finish or revert it first (\`cloudhal fields continue|revert events ${operationId}\`)`, + ); + await expect(stat(out)).rejects.toMatchObject({ code: "ENOENT" }); + // The full history carries it. + const full = await exportArchive({ projectDir, out: "full.cloudhal" }); + const back = await readBack(full.file); + expect(back.bundle.snapshots.map((s) => s.id)).toContain(operationId); + await withStore(projectDir, async ({ store }) => { + const done = await continueFieldOperation(store, fx.siteId, operationId, { actor: person }); + expect(done.done).toBe(true); + }); + const template = await exportArchive({ + projectDir, + out: "template.cloudhal", + latestOnly: true, + }); + expect(template.manifest.counts.snapshots).toBe(2); + }); +}); + +describe("importSite and a crafted project row (the step 3 path)", () => { + /** `bundle` written as an archive (graph only) and read back, as `cloudhal import` will. */ + async function throughArchive(bundle: SiteBundle): Promise { + const content = bundleToArchive(bundle, { + createdBy: "0.0.0", + mediaOmitted: true, + openMedia: () => { + throw new Error("no media in a --no-media archive"); + }, + }); + return (await bundleFromArchive(readArchive(archiveStream(content).bytes))).bundle; + } + + it("refuses a project that is not the site's, not valid, or whose columns disagree", async () => { + const bundle = await bundleOf(fx.projectDir, fx.siteId); + const project = bundle.project as NonNullable; + const body = JSON.parse(project.body) as Record; + const id = project.id; + const cases: [NonNullable, string][] = [ + [{ ...project, id: "prj_planted00001" }, "the bundle's project is not its site's project"], + [{ ...project, body: "{not json" }, `project ${id}: body is not JSON`], + [{ ...project, body: JSON.stringify({ id }) }, `project ${id}: not a valid project`], + [{ ...project, rev: project.rev + 1 }, `project ${id}: columns do not match its body`], + [ + { ...project, body: JSON.stringify({ ...body, id: "prj_other000001" }) }, + `project ${id}: columns do not match its body`, + ], + ]; + for (const [crafted, message] of cases) { + const back = await throughArchive({ ...bundle, project: crafted }); + expect(back.project).toStrictEqual(crafted); + const target = await tempDir("cloudhal-import-"); + await withStore(target, async (local) => { + await expect(importSite(local, back)).rejects.toThrow(message); + const count = (table: string) => + local.sql.get<{ n: number }>(`SELECT count(*) AS n FROM ${table}`); + expect(await count("projects")).toEqual({ n: 0 }); + expect(await count("nodes")).toEqual({ n: 0 }); + }); + } + }); +}); + +describe("cloudhal export and damaged local media (invariant 3)", () => { + const mediaFile = (projectDir: string, sha: string) => + path.join(projectDir, ".cloudhal", "blobs", "media", fx.siteId, sha); + + async function expectNoPartial(projectDir: string): Promise { + expect((await readdir(projectDir)).filter((f) => f.includes(".partial-"))).toEqual([]); + } + + it("refuses a changed media file and leaves the existing archive as it was", async () => { + const projectDir = await freshProject(); + const out = path.join(projectDir, "keep.cloudhal"); + await writeFile(out, "an older archive"); + const file = mediaFile(projectDir, fx.sha.a); + const bytes = await readFile(file); + bytes[bytes.length - 1] = (bytes[bytes.length - 1] as number) ^ 0xff; + await writeFile(file, bytes); + const error = await exportArchive({ projectDir, out: "keep.cloudhal" }).catch((e) => e); + expect(error).toBeInstanceOf(ArchiveError); + expect((error as ArchiveError).code).toBe("damaged"); + expect((error as ArchiveError).message).toMatch( + /the local copy is damaged: run `cloudhal media verify`/, + ); + expect(await readFile(out, "utf8")).toBe("an older archive"); + await expectNoPartial(projectDir); + + await writeFile(file, Buffer.concat([bytes, Buffer.from("more")])); + await expect(exportArchive({ projectDir, out: "keep.cloudhal" })).rejects.toMatchObject({ + code: "damaged", + }); + expect(await readFile(out, "utf8")).toBe("an older archive"); + await expectNoPartial(projectDir); + }); + + it("refuses a missing media file before writing anything", async () => { + const projectDir = await freshProject(); + await rm(mediaFile(projectDir, fx.sha.c)); + await expect(exportArchive({ projectDir, out: "new.cloudhal" })).rejects.toThrow( + new RegExp(`local media ${fx.sha.c} is missing; run \`cloudhal media verify\``), + ); + await expect(stat(path.join(projectDir, "new.cloudhal"))).rejects.toMatchObject({ + code: "ENOENT", + }); + await expectNoPartial(projectDir); + // --no-media does not need the bytes. + expect( + (await exportArchive({ projectDir, out: "new.cloudhal", noMedia: true })).bytes, + ).toBeGreaterThan(0); + }); +}); + +describe("cloudhal export: what it says and never holds (§4)", () => { + it("names the file, the counts and what an archive never holds; no secrets anywhere", async () => { + const projectDir = await freshProject(); + const marker = "placeholder-marker-not-a-secret-4f1c"; + await mkdir(path.join(projectDir, ".cloudhal", "deploy"), { recursive: true }); + await writeFile( + path.join(projectDir, ".cloudhal", "deploy", "secrets.json"), + JSON.stringify({ CLOUDHAL_DEPLOY_TOKEN: marker }), + ); + const result = await exportArchive({ projectDir, out: "site.cloudhal" }); + const lines = describeExport(result, projectDir); + const { counts } = result.manifest; + expect(lines[0]).toMatch(/^wrote site\.cloudhal \(\d+(\.\d)? (B|KiB)\)$/); + expect(counts).toMatchObject({ snapshots: 3, media: 2 }); + expect(lines[1]).toMatch( + new RegExp( + `^site ${fx.siteId} \\(english\\): ${counts.nodes} node\\(s\\), 3 snapshot\\(s\\), ${counts.blobs} blob\\(s\\), 2 media file\\(s\\) \\(\\d+ B\\), 1 taken down$`, + ), + ); + const never = lines.find((l) => l.startsWith("not in the archive (RFC-0039 §4): ")); + for (const item of NEVER_IN_ARCHIVE) expect(never).toContain(item); + expect(never).toContain("user accounts"); + expect(lines.at(-1)).toBe(`in the archive: ${NAMED_IN_ARCHIVE}`); + const tar = Buffer.from(gunzipSync(await readFile(result.file))).toString("latin1"); + expect(tar).not.toContain(marker); + expect(lines.join("\n")).not.toContain(marker); + // What the last line says: the history names who made each change. + // (A snapshot row's body is JSON inside the NDJSON line, so its quotes are escaped.) + expect(tar.includes(`\\"name\\":\\"${person.name}\\"`)).toBe(true); + }); +}); + +describe("cloudhal export: the command", () => { + it("is in --help and refuses positionals", async () => { + expect(HELP).toMatch( + /^ {2}cloudhal export \[--out \] \[--site \] \[--no-media\] \[--latest-only\]$/m, + ); + await expect(main(["export", "x.cloudhal"])).rejects.toThrow( + "usage: cloudhal export [--out ] [--site ] [--no-media] [--latest-only]", + ); + }); + + it("runs from the project folder with its flags and prints the summary", async () => { + const projectDir = await freshProject(); + vi.spyOn(process, "cwd").mockReturnValue(projectDir); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const argv = ["export", "--no-media", "--latest-only", "--out", "sub/x.cloudhal"]; + expect(await main(argv)).toBe(0); + expect((await stat(path.join(projectDir, "sub", "x.cloudhal"))).isFile()).toBe(true); + const printed = log.mock.calls.map((c) => String(c[0])); + expect(printed[0]).toMatch(/^wrote sub\/x\.cloudhal \(/); + expect(printed).toContain( + "media left out (--no-media): an import needs the media from elsewhere", + ); + expect(printed).toContain( + "history: the published and the latest snapshot only (--latest-only)", + ); + await expect(exportArchive({ projectDir, out: "sub" })).rejects.toThrow( + "--out sub is a directory; name a file", + ); + }); + + it("refuses --site, --no-media and --latest-only on every other command", async () => { + await expect(main(["push", "--replace", "--site", "site_other00001"])).rejects.toThrow( + "cloudhal push does not take --site (only cloudhal export does); every other command acts on cloudhal.json's siteId", + ); + await expect(main(["pull", "--replace", "--no-media"])).rejects.toThrow( + "cloudhal pull does not take --no-media (only cloudhal export does)", + ); + await expect(main(["build", "--latest-only"])).rejects.toThrow( + "cloudhal build does not take --latest-only (only cloudhal export does)", + ); + }); + + it("stops on Ctrl-C: exit 130, no file and no partial file, the listener removed", async () => { + const projectDir = await freshProject(); + vi.spyOn(process, "cwd").mockReturnValue(projectDir); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const before = process.listeners("SIGINT"); + const beforeTerm = process.listeners("SIGTERM"); + const running = main(["export", "--out", "stopped.cloudhal"]); + const ours = process.listeners("SIGINT").filter((l) => !before.includes(l)); + expect(ours).toHaveLength(1); + (ours[0] as () => void)(); + expect(await running).toBe(130); + expect(error).toHaveBeenCalledWith(expect.stringMatching(/^cloudhal: export interrupted;/)); + await expect(stat(path.join(projectDir, "stopped.cloudhal"))).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(await partials(projectDir)).toEqual([]); + expect(process.listeners("SIGINT")).toEqual(before); + expect(process.listeners("SIGTERM")).toEqual(beforeTerm); + }); +}); From c5914723f8b2a6c7b7540c2147834dbc2626f355 Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:34:05 +0300 Subject: [PATCH 3/3] test(cli): pick the true (created_at, id) maximum in the snapshot tie test The seed's clock moves in 20 ms steps, so other snapshots can already share the newest one's created_at; the test picked the first earlier row with a larger id instead of the largest, and failed when another tied row sorted after it (CI on #130). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- packages/cli/test/export-cli.test.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/cli/test/export-cli.test.ts b/packages/cli/test/export-cli.test.ts index 87e3b715..0e5367a5 100644 --- a/packages/cli/test/export-cli.test.ts +++ b/packages/cli/test/export-cli.test.ts @@ -331,7 +331,9 @@ describe("cloudhal export --latest-only and the store's history", () => { it("keeps the snapshot latestSnapshot picks when two share a created_at", async () => { const projectDir = await freshProject(); // The newest snapshot (by insertion) ties with an earlier one whose id sorts after it, so the - // bundle's last row (created_at, id) is not the store's latest (created_at, rowid). + // bundle's last row (created_at, id) is not the store's latest (created_at, rowid). The + // earlier one is the largest such id: other rows may already share newest's created_at (the + // seed's clock moves in 20 ms steps), and it must still sort last among them. const tie = await withStore(projectDir, async ({ store, sql }) => { for (let i = 0; i < 40; i++) { const rows = await sql.all<{ id: string; created_at: string }>( @@ -339,7 +341,14 @@ describe("cloudhal export --latest-only and the store's history", () => { [fx.siteId], ); const newest = rows.at(-1) as { id: string; created_at: string }; - const earlier = rows.slice(0, -1).find((r) => r.id > newest.id); + const earlier = rows + .slice(0, -1) + .filter((r) => r.id > newest.id) + // Code-unit order, as SQLite's BINARY collation sorts ids. + .reduce<{ id: string } | undefined>( + (max, r) => (max && max.id > r.id ? max : r), + undefined, + ); if (earlier) { await sql.run("UPDATE snapshots SET created_at = ? WHERE id = ?", [ newest.created_at,