From 7321cdba9d31a1664018a4f730d327f3fff7c82c Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:18:23 +0300 Subject: [PATCH] feat(cli): site archive container for export and import (#47) RFC-0039 rollout step 1: the .cloudhal container that `cloudhal export` will write and `cloudhal import` read. A POSIX ustar stream (regular files only, fixed modes, zero times, 10240-byte records) compressed with gzip at a fixed level and a fixed OS byte, holding manifest.json first and then project.json, nodes/snapshots/blobs NDJSON parts of at most 8 MiB, media/ and ledger.ndjson in one fixed order. The reader treats the file as untrusted input: a closed list of paths, no links, devices or extended headers, header checksums, every file checked against the manifest's size and SHA-256 (media also against its name), row counts, caps on the manifest, parts, media, total decompressed bytes and entry count, and no data after the tar or gzip end. Parts reach the caller only once verified; media bodies stream and fail at their end on a mismatch. The writer streams media and verifies them on the way, and writes to a temporary file that is renamed only when complete. RFC-0039 invariant 7 and ADR-0041 rule 6 are amended: gzip output differs between zlib builds, so identical .cloudhal bytes hold for the same Node/zlib build, while the tar inside and every manifest SHA-256 are identical on every platform. No command yet; `cloudhal export` comes next. Refs #47 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- docs/adr/0041-site-archives.md | 3 +- docs/rfcs/0039-site-archive-export-import.md | 5 +- packages/cli/src/archive/byte-source.ts | 97 ++++ packages/cli/src/archive/errors.ts | 60 +++ packages/cli/src/archive/gzip.ts | 143 ++++++ packages/cli/src/archive/index.ts | 17 + packages/cli/src/archive/layout.ts | 81 ++++ packages/cli/src/archive/limits.ts | 52 ++ packages/cli/src/archive/manifest-files.ts | 121 +++++ packages/cli/src/archive/manifest.ts | 216 +++++++++ packages/cli/src/archive/parts.ts | 98 ++++ packages/cli/src/archive/plan.ts | 164 +++++++ packages/cli/src/archive/read.ts | 257 ++++++++++ packages/cli/src/archive/tar-read.ts | 121 +++++ packages/cli/src/archive/tar.ts | 176 +++++++ packages/cli/src/archive/write.ts | 126 +++++ packages/cli/test/archive-fixture.ts | 248 ++++++++++ packages/cli/test/archive-hostile.test.ts | 471 +++++++++++++++++++ packages/cli/test/archive-interop.test.ts | 45 ++ packages/cli/test/archive-manifest.test.ts | 214 +++++++++ packages/cli/test/archive-parts.test.ts | 89 ++++ packages/cli/test/archive-roundtrip.test.ts | 310 ++++++++++++ packages/cli/test/archive-tar.test.ts | 160 +++++++ 23 files changed, 3271 insertions(+), 3 deletions(-) create mode 100644 packages/cli/src/archive/byte-source.ts create mode 100644 packages/cli/src/archive/errors.ts create mode 100644 packages/cli/src/archive/gzip.ts create mode 100644 packages/cli/src/archive/index.ts create mode 100644 packages/cli/src/archive/layout.ts create mode 100644 packages/cli/src/archive/limits.ts create mode 100644 packages/cli/src/archive/manifest-files.ts create mode 100644 packages/cli/src/archive/manifest.ts create mode 100644 packages/cli/src/archive/parts.ts create mode 100644 packages/cli/src/archive/plan.ts create mode 100644 packages/cli/src/archive/read.ts create mode 100644 packages/cli/src/archive/tar-read.ts create mode 100644 packages/cli/src/archive/tar.ts create mode 100644 packages/cli/src/archive/write.ts create mode 100644 packages/cli/test/archive-fixture.ts create mode 100644 packages/cli/test/archive-hostile.test.ts create mode 100644 packages/cli/test/archive-interop.test.ts create mode 100644 packages/cli/test/archive-manifest.test.ts create mode 100644 packages/cli/test/archive-parts.test.ts create mode 100644 packages/cli/test/archive-roundtrip.test.ts create mode 100644 packages/cli/test/archive-tar.test.ts diff --git a/docs/adr/0041-site-archives.md b/docs/adr/0041-site-archives.md index 7a881381..8b22c2e5 100644 --- a/docs/adr/0041-site-archives.md +++ b/docs/adr/0041-site-archives.md @@ -1,6 +1,7 @@ # ADR-0041: Site archives: a streaming `.cloudhal` tar of NDJSON parts and media, ids kept, graph only - Status: Accepted - Date: 2026-10-22 (decided by Halil) +- Amendment (2026-10-25, RFC-0039 rollout step 1): deterministic bytes hold for the same Node/zlib build; the tar stream and the manifest's SHA-256s on every platform. See rule 6. - Related: RFC-0039, RFC-0004, RFC-0010, ADR-0002, #47 - Sources: [Sanity datasets](https://www.sanity.io/docs/cli-reference/dataset) · [Strapi data management](https://docs.strapi.io/cms/data-management/export) · [Contentful CLI export](https://github.com/contentful/contentful-cli/tree/main/docs/space/export) · [Ghost migration](https://docs.ghost.org/migration/ghost) · [WordPress export](https://wordpress.org/documentation/article/tools-export-screen/) @@ -18,7 +19,7 @@ A site's content can only leave a project through a deployment (`push`, `pull`), Never erase-then-import. 5. **Graph only.** No users, sessions, roles, agent keys, comments, readers, analytics, audit log, automations, secrets or webhook credentials. That is why there is no built-in encryption; people can pipe the file through `age` or `gpg`. -6. **Deterministic bytes:** a fixed order, zero timestamps and fixed modes, so an unchanged site exports identically. +6. **Deterministic bytes:** a fixed order, zero timestamps and fixed modes, so an unchanged site exports identically (amended: byte for byte with the same Node/zlib build, since gzip output differs between zlib builds; the tar stream inside, and every SHA-256 in the manifest, on every platform). 7. **Untrusted input:** the layout is a closed list of paths (no traversal, links or devices); sizes are capped; rows go through the precompiled validators and gates, as for `push`. 8. **Later, already decided:** `init --from`, `--new-ids` for templates, partial export, `--from `, Studio download, and signed archives. diff --git a/docs/rfcs/0039-site-archive-export-import.md b/docs/rfcs/0039-site-archive-export-import.md index c14a2a87..8778eefb 100644 --- a/docs/rfcs/0039-site-archive-export-import.md +++ b/docs/rfcs/0039-site-archive-export-import.md @@ -1,6 +1,7 @@ # RFC-0039: Site archives: `cloudhal export` and `cloudhal import` - Status: Accepted (2026-10-22, by Halil; ADR-0041). Open questions decided as recommended. Revised before acceptance after a comparison with Sanity, Strapi, Contentful, Ghost, WordPress, Directus and Webflow (see Prior art): NDJSON parts instead of one JSON file, per-file checksums, contract versions in the manifest, and a snapshot before `--replace`. +- Amendment (2026-10-25, rollout step 1): identical `.cloudhal` bytes need the same Node/zlib build; the tar stream inside, and so every SHA-256 in the manifest, is identical on every platform. See invariant 7. - Author: Halil (Madraka), drafted with Claude · Date: 2026-10-21 - Related: - #47 (found while building cloudhal.com: no way to keep or move a site's content without a deployment, so it was written as code). @@ -88,7 +89,7 @@ cloudhal import [--replace] [--dry-run] 4. An archive written by a newer format or a newer contracts `schemaVersion` is refused with the CLI version needed. Older ones are read, and migrated forward if a later schema version requires it. 5. Taken-down content stays taken down: the `purged` ledger travels and the import never restores those bytes. 6. Import writes the graph in atomic batches (no interactive transactions, AGENTS.md rule 4), guarded like `importSite` today. A failure leaves the project as it was, and `--replace` can always be undone through its `before import` snapshot. -7. Same content, same file: rows and files are written in a fixed order, with zero timestamps and fixed modes in the tar headers, so exporting an unchanged site twice gives identical bytes (good for git and for diffing). +7. Same content, same file: rows and files are written in a fixed order, with zero timestamps and fixed modes in the tar headers, so exporting an unchanged site twice gives identical bytes (good for git and for diffing). *Amended 2026-10-25:* identical bytes hold for the same Node/zlib build. gzip output differs between zlib builds (Homebrew's zlib and the Chromium zlib that official Node binaries ship compress the same input differently), and only a dependency or uncompressed output would avoid that. The tar stream inside the gzip, and so every SHA-256 in the manifest, is identical on every platform. A committed archive (step 5's demo) is regenerated on one pinned Node in CI, or compared through its decompressed tar. ### 4. What is not in an archive @@ -125,7 +126,7 @@ The manifest says so, and `export` prints it. People and personal data never lea 2. `export`. 3. `import` with `--dry-run`. 4. `--replace`, the `before import` snapshot, and the site id switch. - 5. Docs and a starter demo archive. + 5. Docs and a starter demo archive (regenerated on one pinned Node, or compared through its decompressed tar: invariant 7). ## Security & policy impact diff --git a/packages/cli/src/archive/byte-source.ts b/packages/cli/src/archive/byte-source.ts new file mode 100644 index 00000000..715f4867 --- /dev/null +++ b/packages/cli/src/archive/byte-source.ts @@ -0,0 +1,97 @@ +/** + * Exact byte counts from an async iterable of chunks of any size: gunzip emits 16 KiB chunks that + * do not line up with tar's 512-byte blocks. Internal to the archive reader. + */ +import { ArchiveError } from "./errors.ts"; + +const EMPTY = new Uint8Array(0); + +function truncated(): ArchiveError { + return new ArchiveError("truncated", "the archive ends in the middle of an entry (truncated)"); +} + +export class ByteSource { + readonly #iterator: AsyncIterator; + #chunk: Uint8Array = EMPTY; + #offset = 0; + #position = 0; + #done = false; + + constructor(source: AsyncIterable) { + this.#iterator = source[Symbol.asyncIterator](); + } + + /** Bytes consumed so far. */ + get position(): number { + return this.#position; + } + + /** Makes sure an unread byte is buffered; false at the end of the source. */ + async #fill(): Promise { + while (this.#offset >= this.#chunk.length) { + if (this.#done) return false; + const next = await this.#iterator.next(); + if (next.done) { + this.#done = true; + return false; + } + this.#chunk = next.value; + this.#offset = 0; + } + return true; + } + + #take(max: number): Uint8Array { + const n = Math.min(max, this.#chunk.length - this.#offset); + const view = this.#chunk.subarray(this.#offset, this.#offset + n); + this.#offset += n; + this.#position += n; + return view; + } + + /** + * Exactly `n` bytes in a fresh array; undefined when the source ended before the first of them, + * `truncated` when it ends part-way. + */ + async read(n: number): Promise { + const out = new Uint8Array(n); + let filled = 0; + while (filled < n) { + if (!(await this.#fill())) { + if (filled === 0) return undefined; + throw truncated(); + } + const view = this.#take(n - filled); + out.set(view, filled); + filled += view.length; + } + return out; + } + + /** + * Between 1 and `max` bytes, as a view of the source's chunk (Node streams never reuse a chunk + * they emitted); `truncated` at the end of the source. + */ + async readSome(max: number): Promise { + if (!(await this.#fill())) throw truncated(); + return this.#take(max); + } + + /** Discards exactly `n` bytes; `truncated` when the source has fewer. */ + async skip(n: number): Promise { + let left = n; + while (left > 0) left -= (await this.readSome(left)).length; + } + + /** True when every byte has been consumed and the source has ended. */ + async atEnd(): Promise { + return !(await this.#fill()); + } + + /** Stops the source (runs its `finally`); safe to call more than once. */ + async close(): Promise { + if (this.#done) return; + this.#done = true; + await this.#iterator.return?.(); + } +} diff --git a/packages/cli/src/archive/errors.ts b/packages/cli/src/archive/errors.ts new file mode 100644 index 00000000..b1a176e2 --- /dev/null +++ b/packages/cli/src/archive/errors.ts @@ -0,0 +1,60 @@ +/** + * The one error type of the site archive container (RFC-0039): a code that `cloudhal import` maps + * to its messages, the archive path when there is one, and a safe echo of untrusted names. + */ + +/** + * - `not_archive`: empty, not gzip, or no Cloudhal manifest as the first entry. + * - `unsupported_version`: written by a newer Cloudhal (archive, contracts or bundle format). + * - `invalid_manifest`: `manifest.json` is malformed or disagrees with itself. + * - `unsafe_entry`: a tar header or path Cloudhal never writes (links, devices, traversal). + * - `mismatch`: the entries disagree with the manifest (unlisted, duplicate, order, size, missing). + * - `damaged`: a checksum, SHA-256 or zlib check failed, or data follows the tar or gzip end. + * - `truncated`, `too_large`: the input ends early, or is over a cap. + * - `invalid_input`: the writer was given content it must not write, or a body was misused. + */ +export type ArchiveErrorCode = + | "not_archive" + | "unsupported_version" + | "invalid_manifest" + | "unsafe_entry" + | "mismatch" + | "damaged" + | "truncated" + | "too_large" + | "invalid_input"; + +/** Every failure of the archive writer and reader. */ +export class ArchiveError extends Error { + override readonly name = "ArchiveError"; + readonly code: ArchiveErrorCode; + /** The archive path the error is about, when there is one (a path of the closed layout). */ + readonly path: string | undefined; + constructor( + code: ArchiveErrorCode, + message: string, + options: { path?: string; cause?: unknown } = {}, + ) { + super(message, options.cause === undefined ? undefined : { cause: options.cause }); + this.code = code; + this.path = options.path; + } +} + +const MAX_ECHO = 120; + +/** + * An untrusted name (an entry name by default) as it may appear in a message: JSON-quoted and cut + * when every character is printable ASCII, otherwise only `described`, so a name cannot carry + * terminal escapes (C0 or C1 controls), bidi overrides or megabytes into a message. + */ +export function printable( + raw: string, + described = "an entry name with a non-printable byte", +): string { + for (let i = 0; i < raw.length; i++) { + const c = raw.charCodeAt(i); + if (c < 0x21 || c > 0x7e) return described; + } + return JSON.stringify(raw.length > MAX_ECHO ? `${raw.slice(0, MAX_ECHO)}...` : raw); +} diff --git a/packages/cli/src/archive/gzip.ts b/packages/cli/src/archive/gzip.ts new file mode 100644 index 00000000..bd9cee9b --- /dev/null +++ b/packages/cli/src/archive/gzip.ts @@ -0,0 +1,143 @@ +/** + * gzip for site archives. The writer compresses with fixed parameters and a fixed OS byte, so the + * same tar stream gives the same bytes on every platform that runs the same zlib build (gzip + * output is not identical across zlib builds; the tar stream inside it is). The reader accepts any + * gzip and nothing after its last member, counts the decompressed bytes against a cap, and maps + * zlib's errors to archive errors. + */ +import { pipeline } from "node:stream/promises"; +import { constants, createGunzip, createGzip, type ZlibOptions } from "node:zlib"; +import { ArchiveError } from "./errors.ts"; + +/** + * Level 6 (zlib's default) writes XFL 0. Level 9 costs about twice the CPU on NDJSON for about 3% + * less; level 0 is not used because stored blocks follow the input's chunking. + */ +export const GZIP_OPTIONS: Readonly = Object.freeze({ + level: 6, + memLevel: 8, + windowBits: 15, + strategy: constants.Z_DEFAULT_STRATEGY, +}); + +/** RFC 1952 OS "unknown": the same on every platform (zlib writes 3 on Unix, 10 on Windows). */ +export const GZIP_OS = 255; + +const GZIP_HEADER = 10; + +/** Checks zlib's header (no name, comment or header CRC: FLG 0) and sets the OS byte. */ +function patchHeader(head: Uint8Array): Uint8Array { + if (head[0] !== 0x1f || head[1] !== 0x8b || head[2] !== 8 || head[3] !== 0) + throw new ArchiveError("invalid_input", "zlib wrote an unexpected gzip header"); + head[9] = GZIP_OS; + return head; +} + +/** `source` gzip-compressed with GZIP_OPTIONS and the OS byte set to GZIP_OS. */ +export async function* gzipFixed( + source: AsyncIterable, +): AsyncGenerator { + const gzip = createGzip(GZIP_OPTIONS); + const piping = pipeline(source, gzip); + piping.catch(() => {}); // surfaced by the loop below (pipeline destroys gzip with the error) + let head = new Uint8Array(0); + let patched = false; + try { + for await (const chunk of gzip as AsyncIterable) { + if (patched) { + yield chunk; + continue; + } + head = Buffer.concat([head, chunk]); + if (head.length < GZIP_HEADER) continue; + patched = true; + yield patchHeader(head); + } + await piping; + if (!patched) throw new ArchiveError("invalid_input", "zlib wrote no gzip header"); + } finally { + gzip.destroy(); + } +} + +function zlibCode(error: unknown): string | undefined { + const code = (error as { code?: unknown } | null)?.code; + return typeof code === "string" && code.startsWith("Z_") ? code : undefined; +} + +/** What the source handed to gunzip: its first two bytes, its length, and whether it ended. */ +interface Feed { + head: number[]; + bytes: number; + ended: boolean; +} + +/** `source` unchanged, counted into `feed`. */ +async function* counted( + source: AsyncIterable, + feed: Feed, +): AsyncGenerator { + for await (const chunk of source) { + for (const b of chunk.subarray(0, 2 - feed.head.length)) feed.head.push(b); + feed.bytes += chunk.length; + yield chunk; + } + feed.ended = true; +} + +/** + * A zlib error as an archive error. Whether the input is gzip at all is judged from its first two + * bytes (the magic 1f 8b), not from how much of it was decompressed before the error. + */ +function mapZlibError(error: unknown, feed: Feed): unknown { + const code = zlibCode(error); + if (code === undefined) return error; + if (feed.bytes === 0) + return new ArchiveError("not_archive", "not a Cloudhal site archive (it is empty)", { + cause: error, + }); + if (feed.head[0] !== 0x1f || feed.head[1] !== 0x8b) + return new ArchiveError("not_archive", "not a Cloudhal site archive (not gzip)", { + cause: error, + }); + if (code === "Z_BUF_ERROR") + return new ArchiveError("truncated", "the archive is truncated (gzip ends early)", { + cause: error, + }); + return new ArchiveError("damaged", "the archive is damaged (gzip data is corrupt)", { + cause: error, + }); +} + +/** + * `source` decompressed, `too_large` beyond `maxBytes` decompressed bytes. It returns only after + * gunzip ended cleanly (its CRC and length check passed) and consumed the whole source: zlib stops + * quietly before input that starts with a zero byte, so anything left over is refused here as + * `damaged`. Other errors (of the source, such as a file read) pass through unchanged. + */ +export async function* gunzipCapped( + source: AsyncIterable, + maxBytes: number, +): AsyncGenerator { + const feed: Feed = { head: [], bytes: 0, ended: false }; + const gunzip = createGunzip(); + const piping = pipeline(counted(source, feed), gunzip); + piping.catch(() => {}); // surfaced by the loop below + let produced = 0; + try { + for await (const chunk of gunzip as AsyncIterable) { + produced += chunk.length; + if (produced > maxBytes) + throw new ArchiveError("too_large", `the archive is above ${maxBytes} bytes decompressed`); + yield chunk; + } + // Checked before `piping`, which fails (premature close, abort) when gunzip ended early. + if (!feed.ended || feed.bytes !== gunzip.bytesWritten) + throw new ArchiveError("damaged", "the archive has data after its gzip stream"); + await piping; + } catch (error) { + throw mapZlibError(error, feed); + } finally { + gunzip.destroy(); + } +} diff --git a/packages/cli/src/archive/index.ts b/packages/cli/src/archive/index.ts new file mode 100644 index 00000000..646e2970 --- /dev/null +++ b/packages/cli/src/archive/index.ts @@ -0,0 +1,17 @@ +/** + * Site archives (RFC-0039, ADR-0041): the `.cloudhal` container that `cloudhal export` writes and + * `cloudhal import` reads. Node-only; nothing a Worker runs imports it. + */ +export { ArchiveError, type ArchiveErrorCode } from "./errors.ts"; +export { type ArchiveSlot, classifyPath, type FileSlot, type PartKind } from "./layout.ts"; +export { ARCHIVE_LIMITS, type ArchiveLimits, PART_SPLIT_BYTES } from "./limits.ts"; +export { + type ArchiveCounts, + type ArchiveFile, + SITE_ARCHIVE_FORMAT, + type SiteArchiveManifest, +} from "./manifest.ts"; +export { encodeNdjson, type NdjsonSplitOptions, splitNdjson } from "./parts.ts"; +export type { ArchiveContent, ArchiveMediaSource } from "./plan.ts"; +export { type ArchiveItem, type ReadArchiveOptions, readArchive } from "./read.ts"; +export { archiveStream, type WriteArchiveOptions, writeArchiveFile } from "./write.ts"; diff --git a/packages/cli/src/archive/layout.ts b/packages/cli/src/archive/layout.ts new file mode 100644 index 00000000..ec307337 --- /dev/null +++ b/packages/cli/src/archive/layout.ts @@ -0,0 +1,81 @@ +/** + * The closed list of paths a site archive holds (RFC-0039 §1 and Security) and the one order the + * writer writes them in, which the manifest lists and the reader checks: + * + * `manifest.json`, `project.json`, `nodes/NNNN.ndjson`, `snapshots/NNNN.ndjson`, + * `blobs/NNNN.ndjson`, `media/` (ascending), `ledger.ndjson`. + * + * Any other path, including `..`, absolute paths, backslashes, `./` and part number `0000`, is + * not part of the layout. + */ +import { ArchiveError } from "./errors.ts"; +import { MAX_PART_INDEX } from "./limits.ts"; + +export type PartKind = "nodes" | "snapshots" | "blobs"; + +/** The part kinds in archive order. */ +export const PART_KINDS: readonly PartKind[] = ["nodes", "snapshots", "blobs"]; + +/** What a layout path names. */ +export type ArchiveSlot = + | { kind: "manifest" } + | { kind: "project" } + | { kind: "part"; part: PartKind; index: number } + | { kind: "media"; sha256: string } + | { kind: "ledger" }; + +/** A slot whose file the reader buffers and hands over whole. */ +export type FileSlot = Exclude; + +export const MANIFEST_PATH = "manifest.json"; +export const PROJECT_PATH = "project.json"; +export const LEDGER_PATH = "ledger.ndjson"; + +export const SHA256_HEX = /^[0-9a-f]{64}$/; + +const PART_PATH = /^(nodes|snapshots|blobs)\/([0-9]{4})\.ndjson$/; +const MEDIA_PATH = /^media\/([0-9a-f]{64})$/; + +/** The slot `path` names, or undefined when the path is not in the layout. */ +export function classifyPath(path: string): ArchiveSlot | undefined { + if (path === MANIFEST_PATH) return { kind: "manifest" }; + if (path === PROJECT_PATH) return { kind: "project" }; + if (path === LEDGER_PATH) return { kind: "ledger" }; + const part = PART_PATH.exec(path); + if (part) { + const index = Number(part[2]); + return index >= 1 ? { kind: "part", part: part[1] as PartKind, index } : undefined; + } + const media = MEDIA_PATH.exec(path); + return media ? { kind: "media", sha256: media[1] as string } : undefined; +} + +/** `nodes/0001.ndjson` for part 1 of the nodes. */ +export function partPath(part: PartKind, index: number): string { + if (!Number.isInteger(index) || index < 1 || index > MAX_PART_INDEX) + throw new ArchiveError( + "too_large", + `${part}: an archive holds at most ${MAX_PART_INDEX} parts of each kind`, + ); + return `${part}/${String(index).padStart(4, "0")}.ndjson`; +} + +export function mediaPath(sha256: string): string { + return `media/${sha256}`; +} + +const RANK = { manifest: 0, project: 1, nodes: 2, snapshots: 3, blobs: 4, media: 5, ledger: 6 }; + +function rank(slot: ArchiveSlot): number { + return slot.kind === "part" ? RANK[slot.part] : RANK[slot.kind]; +} + +/** Archive order; 0 only for the same slot. Media hashes compare by code unit, never by locale. */ +export function compareSlots(a: ArchiveSlot, b: ArchiveSlot): number { + const byRank = rank(a) - rank(b); + if (byRank !== 0) return byRank; + if (a.kind === "part" && b.kind === "part") return a.index - b.index; + if (a.kind === "media" && b.kind === "media") + return a.sha256 < b.sha256 ? -1 : a.sha256 > b.sha256 ? 1 : 0; + return 0; +} diff --git a/packages/cli/src/archive/limits.ts b/packages/cli/src/archive/limits.ts new file mode 100644 index 00000000..64dc83ec --- /dev/null +++ b/packages/cli/src/archive/limits.ts @@ -0,0 +1,52 @@ +/** + * Caps for reading untrusted archives (RFC-0039 Security) and the NDJSON part size of the writer. + * Tests lower the caps through `archiveLimits`; the writer refuses content its own reader would + * refuse under the same limits. + */ +import { ArchiveError } from "./errors.ts"; + +const MiB = 1024 * 1024; + +export interface ArchiveLimits { + /** `manifest.json`. */ + manifestBytes: number; + /** `project.json`, `ledger.ndjson` and any one `nodes|snapshots|blobs` part. */ + partBytes: number; + /** Any one `media/` file. */ + mediaBytes: number; + /** Decompressed bytes of the whole archive (gzip bomb backstop). */ + totalBytes: number; + /** Tar entries, the manifest included. */ + entries: number; +} + +export const ARCHIVE_LIMITS: Readonly = Object.freeze({ + // About 200 bytes per listed file: room for 80 000 media files. + manifestBytes: 16 * MiB, + partBytes: 16 * MiB, + // = MAX_MEDIA_PUT (packages/api/src/routes/admin/sync.ts) = MEDIA_LIMITS.upload + // (packages/core-ts/src/media/types.ts): no larger file can reach a site. + mediaBytes: 100 * MiB, + // Above the 10 GiB default site media quota (MEDIA_LIMITS.siteQuota). + totalBytes: 32 * 1024 * MiB, + entries: 100_000, +}); + +/** RFC-0039 §1: a new part every 8 MB. */ +export const PART_SPLIT_BYTES = 8 * MiB; + +/** Part numbers have 4 digits. */ +export const MAX_PART_INDEX = 9999; + +/** The default limits with `overrides` applied; each override must be a positive safe integer. */ +export function archiveLimits(overrides: Partial = {}): ArchiveLimits { + const limits: ArchiveLimits = { ...ARCHIVE_LIMITS }; + for (const [key, value] of Object.entries(overrides)) { + if (!(key in ARCHIVE_LIMITS)) + throw new ArchiveError("invalid_input", `unknown archive limit ${JSON.stringify(key)}`); + if (!Number.isSafeInteger(value) || (value as number) <= 0) + throw new ArchiveError("invalid_input", `archive limit ${key} must be a positive integer`); + limits[key as keyof ArchiveLimits] = value as number; + } + return limits; +} diff --git a/packages/cli/src/archive/manifest-files.ts b/packages/cli/src/archive/manifest-files.ts new file mode 100644 index 00000000..58296b13 --- /dev/null +++ b/packages/cli/src/archive/manifest-files.ts @@ -0,0 +1,121 @@ +/** + * The manifest's `files` list against the layout: every path in it, in archive order, once; part + * numbers contiguous from 0001; each size within its cap; media named by their own hash; and the + * media counts equal to the media listed. + */ +import { ArchiveError, printable } from "./errors.ts"; +import { + type ArchiveSlot, + classifyPath, + compareSlots, + MANIFEST_PATH, + type PartKind, + SHA256_HEX, +} from "./layout.ts"; +import type { ArchiveLimits } from "./limits.ts"; +import type { ArchiveCounts, ArchiveFile } from "./manifest.ts"; +import { tarLength } from "./tar.ts"; + +const FILE_KEYS = ["path", "sha256", "bytes"]; + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** An `invalid_manifest` error about `manifest.json`. */ +export function manifestError(message: string): ArchiveError { + return new ArchiveError("invalid_manifest", `manifest.json: ${message}`, { + path: MANIFEST_PATH, + }); +} + +/** What an unknown key that is not safe to print is replaced with. */ +const HIDDEN_KEY = "(not shown: it has spaces, controls or non-ASCII characters)"; + +/** Refuses a key of `value` that is not `allowed`, echoing it only when that is safe. */ +export function checkKeys( + value: Record, + allowed: readonly string[], + where: string, +): void { + for (const key of Object.keys(value)) + if (!allowed.includes(key)) + throw manifestError(`${where} has an unknown key ${printable(key, HIDDEN_KEY)}`); +} + +function tooLarge(message: string): ArchiveError { + return new ArchiveError("too_large", `manifest.json: ${message}`, { path: MANIFEST_PATH }); +} + +/** One entry of `files`: its shape, its path, its hash and its size cap. */ +function checkEntry( + raw: unknown, + at: string, + limits: ArchiveLimits, +): { file: ArchiveFile; slot: ArchiveSlot } { + if (!isRecord(raw)) throw manifestError(`${at} must be an object`); + checkKeys(raw, FILE_KEYS, at); + const { path, sha256, bytes } = raw; + if (typeof path !== "string") throw manifestError(`${at}.path must be a string`); + const slot = classifyPath(path); + if (slot === undefined || slot.kind === "manifest") + throw manifestError(`${at} names ${printable(path)}, which is not a path a site archive holds`); + if (typeof sha256 !== "string" || !SHA256_HEX.test(sha256)) + throw manifestError(`${at} (${path}) needs a lowercase hex SHA-256`); + if (slot.kind === "media" && sha256 !== slot.sha256) + throw manifestError(`${at} (${path}) is listed with a SHA-256 other than its name`); + if (!Number.isSafeInteger(bytes) || (bytes as number) < 0) + throw manifestError(`${at} (${path}) needs a non-negative integer size`); + const size = bytes as number; + if (slot.kind !== "media" && size === 0) throw manifestError(`${at} (${path}) is empty`); + const cap = slot.kind === "media" ? limits.mediaBytes : limits.partBytes; + if (size > cap) throw tooLarge(`${path} is ${size} bytes, above the ${cap}-byte cap`); + return { file: { path, sha256, bytes: size }, slot }; +} + +/** The checked `files` list of a manifest. */ +export function checkManifestFiles( + raw: unknown, + context: { + counts: ArchiveCounts; + mediaOmitted: boolean; + limits: ArchiveLimits; + /** The manifest's own size, for the decompressed total. */ + manifestBytes: number; + }, +): ArchiveFile[] { + const { counts, limits } = context; + if (!Array.isArray(raw)) throw manifestError("files must be an array"); + if (raw.length + 1 > limits.entries) + throw tooLarge(`${raw.length + 1} entries, above the cap of ${limits.entries}`); + const files: ArchiveFile[] = []; + const nextPart: Record = { nodes: 1, snapshots: 1, blobs: 1 }; + let previous: ArchiveSlot = { kind: "manifest" }; + let media = 0; + let mediaBytes = 0; + for (const [i, entry] of raw.entries()) { + const { file, slot } = checkEntry(entry, `files[${i}]`, limits); + if (compareSlots(previous, slot) >= 0) + throw manifestError(`files[${i}] (${file.path}) is listed twice or out of order`); + if (slot.kind === "part") { + if (slot.index !== nextPart[slot.part]) + throw manifestError(`files[${i}] (${file.path}) leaves a gap in the ${slot.part} parts`); + nextPart[slot.part] = slot.index + 1; + } + if (slot.kind === "media") { + media++; + mediaBytes += file.bytes; + } + previous = slot; + files.push(file); + } + const total = tarLength([context.manifestBytes, ...files.map((f) => f.bytes)]); + if (total > limits.totalBytes) + throw tooLarge( + `the archive is ${total} bytes decompressed, above the ${limits.totalBytes}-byte cap`, + ); + if (counts.media !== media || counts.mediaBytes !== mediaBytes) + throw manifestError(`counts.media and counts.mediaBytes must be ${media} and ${mediaBytes}`); + if (context.mediaOmitted && media > 0) throw manifestError("mediaOmitted, yet media are listed"); + return files; +} diff --git a/packages/cli/src/archive/manifest.ts b/packages/cli/src/archive/manifest.ts new file mode 100644 index 00000000..1bfd4048 --- /dev/null +++ b/packages/cli/src/archive/manifest.ts @@ -0,0 +1,216 @@ +/** + * `manifest.json`, the first entry of a site archive (RFC-0039 §1): its type, its canonical bytes + * and the hand-written check of untrusted ones. Not a contract schema (like `SiteBundle`), so no + * Ajv: the version gate runs first, so a newer archive is refused with the CLI version it needs + * before its shape is judged; then the shape is checked strictly, unknown keys included. + */ +import { CONTRACT_SCHEMA_VERSION } from "@cloudhal/core-contracts"; +import { SITE_BUNDLE_FORMAT } from "@cloudhal/core-ts"; +import { ArchiveError } from "./errors.ts"; +import type { ArchiveLimits } from "./limits.ts"; +import { checkKeys, checkManifestFiles, isRecord, manifestError } from "./manifest-files.ts"; + +export const SITE_ARCHIVE_FORMAT = "cloudhal.site-archive@1" as const; + +/** One file of the archive other than the manifest. */ +export interface ArchiveFile { + path: string; + sha256: string; + bytes: number; +} + +/** Rows of the NDJSON parts, and the media files with their total size. */ +export interface ArchiveCounts { + nodes: number; + snapshots: number; + blobs: number; + media: number; + mediaBytes: number; +} + +export interface SiteArchiveManifest { + format: typeof SITE_ARCHIVE_FORMAT; + /** The contracts' schemaVersion the rows were written with. */ + contracts: { schemaVersion: number }; + /** The bundle format the parts reassemble into. */ + bundle: typeof SITE_BUNDLE_FORMAT; + siteId: string; + siteName: string; + /** The `@cloudhal/cli` version that wrote it. */ + createdBy: string; + /** RFC 3339; from the content, never the wall clock (invariant 7). */ + createdAt: string; + counts: ArchiveCounts; + /** Every other file, in archive order, with its SHA-256 and size. */ + files: ArchiveFile[]; + /** Written without media: they must come from elsewhere. */ + mediaOmitted?: true; +} + +const TOP_KEYS = [ + "format", + "contracts", + "bundle", + "siteId", + "siteName", + "createdBy", + "createdAt", + "counts", + "files", + "mediaOmitted", +] as const; +const COUNT_KEYS = ["nodes", "snapshots", "blobs", "media", "mediaBytes"] as const; + +const FORMAT = /^cloudhal\.site-archive@([1-9][0-9]{0,5})$/; +const BUNDLE = /^cloudhal\.site-bundle@([1-9][0-9]{0,5})$/; +/** A version as `createdBy` may carry it, safe to print. */ +const VERSION_TOKEN = /^[0-9A-Za-z.+-]{1,64}$/; +/** common.schema.json `$defs/Id`. */ +const ID = /^[a-z]+_[A-Za-z0-9]{6,32}$/; +const RFC3339 = + /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{1,9})?(Z|[+-][0-9]{2}:[0-9]{2})$/; +/** Site.name (site.schema.json). */ +const MAX_SITE_NAME = 200; + +/** The manifest's bytes: RFC key order, two-space JSON and a final newline, so it reads and diffs. */ +export function encodeManifest(m: SiteArchiveManifest): Uint8Array { + const { nodes, snapshots, blobs, media, mediaBytes } = m.counts; + const canonical = { + format: m.format, + contracts: { schemaVersion: m.contracts.schemaVersion }, + bundle: m.bundle, + siteId: m.siteId, + siteName: m.siteName, + createdBy: m.createdBy, + createdAt: m.createdAt, + counts: { nodes, snapshots, blobs, media, mediaBytes }, + files: m.files.map((f) => ({ path: f.path, sha256: f.sha256, bytes: f.bytes })), + ...(m.mediaOmitted === true ? { mediaOmitted: true } : {}), + }; + return new TextEncoder().encode(`${JSON.stringify(canonical, null, 2)}\n`); +} + +function versionNumber(value: unknown, pattern: RegExp): number | undefined { + const match = typeof value === "string" ? pattern.exec(value) : null; + return match ? Number(match[1]) : undefined; +} + +/** What in `raw` is newer than this reader, if anything. */ +function newerPart(raw: Record, format: number): string | undefined { + if (format > 1) return `archive format ${raw.format as string}`; + const contracts = isRecord(raw.contracts) ? raw.contracts.schemaVersion : undefined; + if (Number.isInteger(contracts) && (contracts as number) > CONTRACT_SCHEMA_VERSION) + return `contracts schemaVersion ${contracts as number}`; + const bundle = versionNumber(raw.bundle, BUNDLE); + if (bundle !== undefined && bundle > 1) return `bundle format ${raw.bundle as string}`; + return undefined; +} + +/** Refuses what only a newer Cloudhal can read, naming the CLI version that wrote it. */ +function checkVersion(raw: Record): void { + const format = versionNumber(raw.format, FORMAT); + if (format === undefined) + throw new ArchiveError("not_archive", "not a Cloudhal site archive (unknown manifest format)"); + const newer = newerPart(raw, format); + if (newer === undefined) return; + const version = + typeof raw.createdBy === "string" && VERSION_TOKEN.test(raw.createdBy) + ? `${raw.createdBy} or later` + : "a newer version"; + throw new ArchiveError( + "unsupported_version", + `this archive was written by a newer Cloudhal (${newer}); update @cloudhal/cli to ${version}`, + ); +} + +function checkCounts(raw: unknown): ArchiveCounts { + if (!isRecord(raw)) throw manifestError("counts must be an object"); + checkKeys(raw, COUNT_KEYS, "counts"); + for (const key of COUNT_KEYS) { + const n = raw[key]; + if (!Number.isSafeInteger(n) || (n as number) < 0) + throw manifestError(`counts.${key} must be a non-negative integer`); + } + return raw as unknown as ArchiveCounts; +} + +function checkStrings(raw: Record): void { + if (raw.bundle !== SITE_BUNDLE_FORMAT) + throw manifestError(`bundle must be ${SITE_BUNDLE_FORMAT}`); + if (typeof raw.siteId !== "string" || !ID.test(raw.siteId)) + throw manifestError("siteId must be a site id"); + const name = raw.siteName; + if (typeof name !== "string" || name.length === 0 || [...name].length > MAX_SITE_NAME) + throw manifestError(`siteName must be a string of 1-${MAX_SITE_NAME} characters`); + if (typeof raw.createdBy !== "string" || !VERSION_TOKEN.test(raw.createdBy)) + throw manifestError("createdBy must be a version (letters, digits, '.', '+', '-')"); + const at = raw.createdAt; + if (typeof at !== "string" || !RFC3339.test(at) || Number.isNaN(Date.parse(at))) + throw manifestError("createdAt must be an RFC 3339 timestamp"); +} + +/** The shape check, after the version gate. */ +function checkShape( + raw: Record, + size: number, + limits: ArchiveLimits, +): SiteArchiveManifest { + checkKeys(raw, TOP_KEYS, "the manifest"); + for (const key of TOP_KEYS) + if (key !== "mediaOmitted" && !(key in raw)) throw manifestError(`${key} is missing`); + if (!isRecord(raw.contracts)) throw manifestError("contracts must be an object"); + checkKeys(raw.contracts, ["schemaVersion"], "contracts"); + const schemaVersion = raw.contracts.schemaVersion; + if (!Number.isInteger(schemaVersion) || (schemaVersion as number) < 1) + throw manifestError("contracts.schemaVersion must be a positive integer"); + checkStrings(raw); + if (raw.mediaOmitted !== undefined && raw.mediaOmitted !== true) + throw manifestError("mediaOmitted must be true when present"); + const counts = checkCounts(raw.counts); + const mediaOmitted = raw.mediaOmitted === true; + const files = checkManifestFiles(raw.files, { + counts, + mediaOmitted, + limits, + manifestBytes: size, + }); + return { + format: SITE_ARCHIVE_FORMAT, + contracts: { schemaVersion: schemaVersion as number }, + bundle: SITE_BUNDLE_FORMAT, + siteId: raw.siteId as string, + siteName: raw.siteName as string, + createdBy: raw.createdBy as string, + createdAt: raw.createdAt as string, + counts: { + nodes: counts.nodes, + snapshots: counts.snapshots, + blobs: counts.blobs, + media: counts.media, + mediaBytes: counts.mediaBytes, + }, + files, + ...(mediaOmitted ? { mediaOmitted: true as const } : {}), + }; +} + +/** Untrusted manifest bytes to a manifest, or the reason they are refused. */ +export function parseManifest(bytes: Uint8Array, limits: ArchiveLimits): SiteArchiveManifest { + if (bytes.length > limits.manifestBytes) + throw new ArchiveError("too_large", `manifest.json is above ${limits.manifestBytes} bytes`, { + path: "manifest.json", + }); + let raw: unknown; + try { + raw = JSON.parse(new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes)); + } catch { + throw manifestError("it is not UTF-8 JSON"); + } + if (!isRecord(raw)) + throw new ArchiveError( + "not_archive", + "not a Cloudhal site archive (the manifest is not an object)", + ); + checkVersion(raw); + return checkShape(raw, bytes.length, limits); +} diff --git a/packages/cli/src/archive/parts.ts b/packages/cli/src/archive/parts.ts new file mode 100644 index 00000000..6b4c9aca --- /dev/null +++ b/packages/cli/src/archive/parts.ts @@ -0,0 +1,98 @@ +/** + * NDJSON parts (RFC-0039 §1): one JSON row per line, a new part every 8 MB. A row's line is + * `JSON.stringify(row)` and a newline; JSON.stringify never writes a raw newline, so the rows of a + * part are exactly its newlines. The writer and the reader count them the same way. + */ +import { ArchiveError, type ArchiveErrorCode } from "./errors.ts"; +import { ARCHIVE_LIMITS, PART_SPLIT_BYTES } from "./limits.ts"; + +const LF = 0x0a; +const encoder = new TextEncoder(); + +export interface NdjsonSplitOptions { + /** Start a new part when the next line would cross this size (default 8 MiB). */ + partBytes?: number; + /** Refuse a line above this size (default: the reader's part cap, 16 MiB). */ + maxLineBytes?: number; +} + +function encodeLine(row: unknown, index: number): Uint8Array { + const text = JSON.stringify(row); + if (text === undefined) + throw new ArchiveError("invalid_input", `row ${index} has no JSON form (${typeof row})`); + return encoder.encode(`${text}\n`); +} + +/** + * `rows` as NDJSON parts, in the given order (the caller sorts): a new part when the current one + * is not empty and the next line would take it above `partBytes`. A line larger than `partBytes` + * goes alone in its own part; a line above `maxLineBytes` is refused. No rows give no parts. + */ +export function splitNdjson( + rows: Iterable, + options: NdjsonSplitOptions = {}, +): Uint8Array[] { + const partBytes = options.partBytes ?? PART_SPLIT_BYTES; + const maxLine = options.maxLineBytes ?? ARCHIVE_LIMITS.partBytes; + const parts: Uint8Array[] = []; + let lines: Uint8Array[] = []; + let size = 0; + let index = 0; + for (const row of rows) { + const line = encodeLine(row, index); + if (line.length > maxLine) + throw new ArchiveError( + "too_large", + `row ${index} is ${line.length} bytes as JSON, above the ${maxLine}-byte part cap`, + ); + if (size > 0 && size + line.length > partBytes) { + parts.push(Buffer.concat(lines, size)); + lines = []; + size = 0; + } + lines.push(line); + size += line.length; + index++; + } + if (size > 0) parts.push(Buffer.concat(lines, size)); + return parts; +} + +/** `rows` as one NDJSON file (`ledger.ndjson`), refused above `maxBytes`. */ +export function encodeNdjson( + rows: Iterable, + maxBytes: number = ARCHIVE_LIMITS.partBytes, +): Uint8Array { + const lines: Uint8Array[] = []; + let size = 0; + let index = 0; + for (const row of rows) { + const line = encodeLine(row, index++); + size += line.length; + if (size > maxBytes) + throw new ArchiveError("too_large", `the rows are above ${maxBytes} bytes as NDJSON`); + lines.push(line); + } + return Buffer.concat(lines, size); +} + +/** + * The rows of an NDJSON file: its newlines. An empty file, an empty line or a missing final + * newline is refused with `code` (`damaged` when reading, `invalid_input` when writing). + */ +export function countRows( + bytes: Uint8Array, + path: string, + code: ArchiveErrorCode = "damaged", +): number { + if (bytes.length === 0 || bytes[bytes.length - 1] !== LF) + throw new ArchiveError(code, `${path}: not NDJSON (empty, or no final newline)`, { path }); + let rows = 0; + let previous = -1; + for (let i = bytes.indexOf(LF); i >= 0; i = bytes.indexOf(LF, i + 1)) { + if (i === previous + 1) throw new ArchiveError(code, `${path}: has an empty line`, { path }); + previous = i; + rows++; + } + return rows; +} diff --git a/packages/cli/src/archive/plan.ts b/packages/cli/src/archive/plan.ts new file mode 100644 index 00000000..5f122aad --- /dev/null +++ b/packages/cli/src/archive/plan.ts @@ -0,0 +1,164 @@ +/** + * The synchronous front half of the archive writer: checks the content, hashes the in-memory + * files, counts the rows and media, and builds the manifest, which it checks with the reader's own + * parser, so the writer never starts an archive its reader would refuse under the same limits. + */ +import { createHash } from "node:crypto"; +import { CONTRACT_SCHEMA_VERSION } from "@cloudhal/core-contracts"; +import { SITE_BUNDLE_FORMAT } from "@cloudhal/core-ts"; +import { ArchiveError } from "./errors.ts"; +import { + LEDGER_PATH, + mediaPath, + PART_KINDS, + type PartKind, + PROJECT_PATH, + partPath, + SHA256_HEX, +} from "./layout.ts"; +import { type ArchiveLimits, MAX_PART_INDEX } from "./limits.ts"; +import { + type ArchiveFile, + encodeManifest, + parseManifest, + SITE_ARCHIVE_FORMAT, + type SiteArchiveManifest, +} from "./manifest.ts"; +import { countRows } from "./parts.ts"; + +/** A media file, streamed when the writer reaches it and verified against `sha256` and `bytes`. */ +export interface ArchiveMediaSource { + readonly sha256: string; + readonly bytes: number; + open(): AsyncIterable | Promise>; +} + +/** What an archive holds. The caller orders the rows (RFC-0039 §1); the writer sorts media. */ +export interface ArchiveContent { + siteId: string; + siteName: string; + /** The `@cloudhal/cli` version. */ + createdBy: string; + /** RFC 3339, taken from the content (the newest row), never from the clock. */ + createdAt: string; + /** `project.json`: the project row as JSON. */ + project?: Uint8Array | undefined; + /** NDJSON parts, from `splitNdjson`. */ + nodes: readonly Uint8Array[]; + snapshots: readonly Uint8Array[]; + blobs: readonly Uint8Array[]; + /** `ledger.ndjson`, from `encodeNdjson`, when there are purged rows. */ + ledger?: Uint8Array | undefined; + media?: readonly ArchiveMediaSource[] | undefined; + mediaOmitted?: true | undefined; +} + +export type PlannedFile = + | { path: string; data: Uint8Array } + | { path: string; media: ArchiveMediaSource }; + +export interface ArchivePlan { + manifest: SiteArchiveManifest; + manifestBytes: Uint8Array; + /** Every file after the manifest, in archive order. */ + files: PlannedFile[]; +} + +interface Draft { + files: PlannedFile[]; + listed: ArchiveFile[]; + limits: ArchiveLimits; +} + +function sha256Of(data: Uint8Array): string { + return createHash("sha256").update(data).digest("hex"); +} + +function addData(draft: Draft, path: string, data: Uint8Array): void { + if (data.length === 0) throw new ArchiveError("invalid_input", `${path} is empty`, { path }); + if (data.length > draft.limits.partBytes) + throw new ArchiveError("too_large", `${path} is above ${draft.limits.partBytes} bytes`, { + path, + }); + draft.files.push({ path, data }); + draft.listed.push({ path, sha256: sha256Of(data), bytes: data.length }); +} + +/** Adds the parts of one kind and returns their rows. */ +function addParts(draft: Draft, kind: PartKind, parts: readonly Uint8Array[]): number { + if (parts.length > MAX_PART_INDEX) + throw new ArchiveError("too_large", `${kind}: more than ${MAX_PART_INDEX} parts`); + let rows = 0; + for (const [i, part] of parts.entries()) { + const path = partPath(kind, i + 1); + addData(draft, path, part); + rows += countRows(part, path, "invalid_input"); + } + return rows; +} + +/** The media in archive order (ascending hash), checked. */ +function sortedMedia(content: ArchiveContent, limits: ArchiveLimits): ArchiveMediaSource[] { + const media = [...(content.media ?? [])]; + if (media.length > 0 && content.mediaOmitted) + throw new ArchiveError("invalid_input", "media were given together with mediaOmitted"); + const seen = new Set(); + for (const m of media) { + if (!SHA256_HEX.test(m.sha256)) + throw new ArchiveError("invalid_input", "a media sha256 is not 64 lowercase hex digits"); + if (seen.has(m.sha256)) + throw new ArchiveError("invalid_input", `media ${m.sha256} is given twice`); + seen.add(m.sha256); + if (!Number.isSafeInteger(m.bytes) || m.bytes < 0) + throw new ArchiveError("invalid_input", `media ${m.sha256} has an invalid size`); + if (m.bytes > limits.mediaBytes) + throw new ArchiveError("too_large", `media ${m.sha256} is above ${limits.mediaBytes} bytes`); + } + return media.sort((a, b) => (a.sha256 < b.sha256 ? -1 : a.sha256 > b.sha256 ? 1 : 0)); +} + +/** The manifest checked by the reader's parser; refusals become the writer's `invalid_input`. */ +function checkedManifest(manifest: SiteArchiveManifest, limits: ArchiveLimits): Uint8Array { + const bytes = encodeManifest(manifest); + try { + parseManifest(bytes, limits); + } catch (e) { + if (e instanceof ArchiveError && e.code !== "too_large") + throw new ArchiveError("invalid_input", e.message, { cause: e }); + throw e; + } + return bytes; +} + +/** Checks `content` and lays out the archive; throws before any byte is written. */ +export function planArchive(content: ArchiveContent, limits: ArchiveLimits): ArchivePlan { + const draft: Draft = { files: [], listed: [], limits }; + if (content.project !== undefined) addData(draft, PROJECT_PATH, content.project); + const rows: Record = { nodes: 0, snapshots: 0, blobs: 0 }; + for (const kind of PART_KINDS) rows[kind] = addParts(draft, kind, content[kind]); + const media = sortedMedia(content, limits); + let mediaBytes = 0; + for (const m of media) { + const path = mediaPath(m.sha256); + draft.files.push({ path, media: m }); + draft.listed.push({ path, sha256: m.sha256, bytes: m.bytes }); + mediaBytes += m.bytes; + } + if (content.ledger !== undefined) { + addData(draft, LEDGER_PATH, content.ledger); + countRows(content.ledger, LEDGER_PATH, "invalid_input"); + } + const manifest: SiteArchiveManifest = { + format: SITE_ARCHIVE_FORMAT, + contracts: { schemaVersion: CONTRACT_SCHEMA_VERSION }, + bundle: SITE_BUNDLE_FORMAT, + siteId: content.siteId, + siteName: content.siteName, + createdBy: content.createdBy, + createdAt: content.createdAt, + counts: { ...rows, media: media.length, mediaBytes }, + files: draft.listed, + ...(content.mediaOmitted ? { mediaOmitted: true as const } : {}), + }; + return { manifest, manifestBytes: checkedManifest(manifest, limits), files: draft.files }; +} diff --git a/packages/cli/src/archive/read.ts b/packages/cli/src/archive/read.ts new file mode 100644 index 00000000..591bef02 --- /dev/null +++ b/packages/cli/src/archive/read.ts @@ -0,0 +1,257 @@ +/** + * The verifying site archive reader (RFC-0039 §1, invariant 3). It checks as it streams and hands + * the caller each item only as far as it can be trusted: + * + * 1. The manifest first, after only `manifest.json` has been read (a dry run can stop there). + * 2. `project.json`, the NDJSON parts and `ledger.ndjson` buffered (they are capped) and yielded + * only after their size, SHA-256 and rows were checked against the manifest. + * 3. Media with a streaming body, hashed as it is read: the body is unverified until its last + * `next()` resolves, which throws `damaged` on a mismatch. A body the caller does not read is + * read and verified by the reader before the next item; a body read after the next item was + * requested, or still being read then, throws `invalid_input` instead of ending early. + * + * The archive as a whole is verified only when the generator returns: every listed file present, + * the row counts equal, the end of the archive well formed and gzip's CRC checked. A caller + * applies nothing (no `importSite`) before that; media it streamed somewhere content-addressed + * stay unreferenced if the archive then fails. + */ +import { createHash } from "node:crypto"; +import { ArchiveError, printable } from "./errors.ts"; +import { gunzipCapped } from "./gzip.ts"; +import { classifyPath, type FileSlot, MANIFEST_PATH, PART_KINDS, type PartKind } from "./layout.ts"; +import { type ArchiveLimits, archiveLimits } from "./limits.ts"; +import { type ArchiveFile, parseManifest, type SiteArchiveManifest } from "./manifest.ts"; +import { countRows } from "./parts.ts"; +import { readTar, type TarEntry } from "./tar-read.ts"; + +export type ArchiveItem = + | { type: "manifest"; manifest: SiteArchiveManifest } + | { type: "file"; path: string; slot: FileSlot; bytes: Uint8Array } + | { + type: "media"; + path: string; + sha256: string; + size: number; + /** + * Readable once, and only before the next item is requested (await the sink first): a read + * after that, or one still running then, throws `invalid_input`. Unverified until its last + * `next()` resolves, which throws `damaged` on a mismatch. + */ + body: AsyncIterable; + }; + +export interface ReadArchiveOptions { + /** Lower caps for untrusted input (the defaults are ARCHIVE_LIMITS). */ + limits?: Partial; +} + +type Entries = AsyncGenerator; + +async function readBody(entry: TarEntry): Promise { + const bytes = new Uint8Array(entry.size); + let filled = 0; + for await (const chunk of entry.body) { + bytes.set(chunk, filled); + filled += chunk.length; + } + return bytes; +} + +async function readManifest(entries: Entries, limits: ArchiveLimits): Promise { + const first = await entries.next(); + if (first.done || first.value.path !== MANIFEST_PATH) + throw new ArchiveError( + "not_archive", + "not a Cloudhal site archive (manifest.json is not its first entry)", + ); + if (first.value.size > limits.manifestBytes) + throw new ArchiveError("too_large", `manifest.json is above ${limits.manifestBytes} bytes`, { + path: MANIFEST_PATH, + }); + return parseManifest(await readBody(first.value), limits); +} + +/** The entry must be the next listed file, with its listed size: checked before its body. */ +function expectEntry(entry: TarEntry, files: readonly ArchiveFile[], index: number) { + const slot = classifyPath(entry.path); + if (slot === undefined) + throw new ArchiveError( + "unsafe_entry", + `the archive holds ${printable(entry.path)}, which is not a path a site archive holds`, + ); + if (slot.kind === "manifest") + throw new ArchiveError("mismatch", "manifest.json: appears twice", { path: MANIFEST_PATH }); + const listed = files[index]; + if (listed?.path !== entry.path) { + const known = files.some((f) => f.path === entry.path); + throw new ArchiveError( + "mismatch", + `${entry.path}: ${known ? "appears twice or out of order" : "is not listed in manifest.json"}`, + { path: entry.path }, + ); + } + if (entry.size !== listed.bytes) + throw new ArchiveError( + "mismatch", + `${entry.path}: ${entry.size} bytes in the archive, ${listed.bytes} in manifest.json`, + { path: entry.path }, + ); + return { slot, listed }; +} + +function damaged(path: string): ArchiveError { + return new ArchiveError( + "damaged", + `${path}: its SHA-256 does not match manifest.json (the file is damaged or was edited)`, + { path }, + ); +} + +/** + * A media body hashed as it is read; its last `next()` throws on a mismatch. Like a tar entry's + * body it is stale once the next item is requested: `drain` marks it so, refuses to run beside a + * read in flight, and reads and verifies the rest itself. Its first error is thrown again by every + * later read, so a caller that retries never sees a clean end of a partial body. + */ +class MediaBody implements AsyncIterable { + readonly #inner: AsyncIterator; + readonly #listed: ArchiveFile; + readonly #hash = createHash("sha256"); + #claimed = false; + /** The caller's reads in flight. */ + #reading = 0; + #stale = false; + #finished = false; + #failure: { error: unknown } | undefined; + + constructor(entry: TarEntry, listed: ArchiveFile) { + this.#inner = entry.body[Symbol.asyncIterator](); + this.#listed = listed; + } + + [Symbol.asyncIterator](): AsyncIterator { + if (this.#claimed) throw this.#misuse("can be read once"); + this.#claimed = true; + return { + next: async () => { + if (this.#stale) throw this.#misuse("was read after the next item was requested"); + this.#reading++; + try { + const value = await this.#pull(); + return value === undefined ? { done: true, value: undefined } : { done: false, value }; + } finally { + this.#reading--; + } + }, + // Stopping early leaves the rest to `drain`, which still verifies it. + return: async () => ({ done: true, value: undefined }), + }; + } + + async #pull(): Promise { + if (this.#failure) throw this.#failure.error; + if (this.#finished) return undefined; + let next: IteratorResult; + try { + next = await this.#inner.next(); + } catch (error) { + this.#failure = { error }; + throw error; + } + if (!next.done) { + this.#hash.update(next.value); + return next.value; + } + this.#finished = true; + if (this.#hash.digest("hex") === this.#listed.sha256) return undefined; + this.#failure = { error: damaged(this.#listed.path) }; + throw this.#failure.error; + } + + /** Reads and verifies what the caller left; the body is stale from here on. */ + async drain(): Promise { + this.#stale = true; + if (this.#reading > 0) + throw this.#misuse("is still being read (await it before the next item)"); + for (;;) if ((await this.#pull()) === undefined) return; + } + + #misuse(what: string): ArchiveError { + const path = this.#listed.path; + return new ArchiveError("invalid_input", `${path}: a media body ${what}`, { path }); + } +} + +/** A buffered file, checked against the manifest; its rows when it is a part. */ +async function readFile( + entry: TarEntry, + listed: ArchiveFile, + slot: FileSlot, +): Promise<{ bytes: Uint8Array; rows: number }> { + const bytes = await readBody(entry); + if (createHash("sha256").update(bytes).digest("hex") !== listed.sha256) throw damaged(entry.path); + const ndjson = slot.kind === "part" || slot.kind === "ledger"; + return { bytes, rows: ndjson ? countRows(bytes, entry.path) : 0 }; +} + +function checkComplete( + manifest: SiteArchiveManifest, + index: number, + rows: Record, +): void { + const missing = manifest.files[index]; + if (missing) + throw new ArchiveError( + "mismatch", + `${missing.path}: listed in manifest.json but not in the archive`, + { + path: missing.path, + }, + ); + for (const kind of PART_KINDS) + if (rows[kind] !== manifest.counts[kind]) + throw new ArchiveError( + "mismatch", + `${kind}: the parts hold ${rows[kind]} rows, manifest.json counts ${manifest.counts[kind]}`, + ); +} + +async function* readEntries( + entries: Entries, + manifest: SiteArchiveManifest, +): AsyncGenerator { + const rows: Record = { nodes: 0, snapshots: 0, blobs: 0 }; + let index = 0; + for await (const entry of entries) { + const { slot, listed } = expectEntry(entry, manifest.files, index++); + if (slot.kind === "media") { + const body = new MediaBody(entry, listed); + yield { type: "media", path: entry.path, sha256: slot.sha256, size: entry.size, body }; + await body.drain(); + continue; + } + const file = await readFile(entry, listed, slot); + if (slot.kind === "part") rows[slot.part] += file.rows; + yield { type: "file", path: entry.path, slot, bytes: file.bytes }; + } + checkComplete(manifest, index, rows); +} + +/** + * The verified items of a `.cloudhal` stream (gzip bytes, for example a file's read stream): + * the manifest, then every file in archive order. Stopping early (break) closes the source. + */ +export async function* readArchive( + source: AsyncIterable, + options: ReadArchiveOptions = {}, +): AsyncGenerator { + const limits = archiveLimits(options.limits); + const entries = readTar(gunzipCapped(source, limits.totalBytes), { maxEntries: limits.entries }); + try { + const manifest = await readManifest(entries, limits); + yield { type: "manifest", manifest }; + yield* readEntries(entries, manifest); + } finally { + await entries.return(undefined); + } +} diff --git a/packages/cli/src/archive/tar-read.ts b/packages/cli/src/archive/tar-read.ts new file mode 100644 index 00000000..d1a97aee --- /dev/null +++ b/packages/cli/src/archive/tar-read.ts @@ -0,0 +1,121 @@ +/** + * Streaming ustar reader: yields the regular-file entries of a decompressed tar stream, each with + * a lazy body, and checks the end of the archive (two zero blocks, then at most one record of zero + * padding, then nothing). It reads its source to the end before it returns, so gunzip's CRC and + * length check has run when the last entry is done. + */ + +import { ByteSource } from "./byte-source.ts"; +import { ArchiveError } from "./errors.ts"; +import { parseTarHeader, TAR_BLOCK, TAR_RECORD, tarPadding } from "./tar.ts"; + +export interface TarEntry { + readonly path: string; + readonly size: number; + /** + * Exactly `size` bytes, readable once and only until the next entry is requested: the reader + * skips what is left of it then, and a later read throws. + */ + readonly body: AsyncIterable; +} + +class EntryBody implements AsyncIterable { + readonly #source: ByteSource; + readonly #path: string; + #remaining: number; + #claimed = false; + #reading = false; + #stale = false; + + constructor(source: ByteSource, entry: { path: string; size: number }) { + this.#source = source; + this.#path = entry.path; + this.#remaining = entry.size; + } + + async *[Symbol.asyncIterator](): AsyncGenerator { + if (this.#claimed) throw this.#misuse("can be read once"); + this.#claimed = true; + for (;;) { + if (this.#stale) throw this.#misuse("was read after the next entry was requested"); + if (this.#remaining === 0) return; + this.#reading = true; + let chunk: Uint8Array; + try { + chunk = await this.#source.readSome(this.#remaining); + } finally { + this.#reading = false; + } + this.#remaining -= chunk.length; + yield chunk; + } + } + + /** Skips what the caller did not read; the body is stale from here on. */ + async finish(): Promise { + if (this.#reading) throw this.#misuse("is still being read"); + this.#stale = true; + await this.#source.skip(this.#remaining); + this.#remaining = 0; + } + + #misuse(what: string): ArchiveError { + return new ArchiveError("invalid_input", `the body of a tar entry ${what}`, { + path: this.#path, + }); + } +} + +function isZero(bytes: Uint8Array): boolean { + for (const b of bytes) if (b !== 0) return false; + return true; +} + +/** After the first zero block: the second one, then at most one record of zeros, then the end. */ +async function readEnd(source: ByteSource): Promise { + const second = await source.read(TAR_BLOCK); + if (second === undefined) + throw new ArchiveError("truncated", "the archive ends inside its end-of-archive marker"); + if (!isZero(second)) + throw new ArchiveError("damaged", "the archive has a zero block between its entries"); + let tail = 0; + while (!(await source.atEnd())) { + const chunk = await source.readSome(TAR_RECORD - tail + 1); + tail += chunk.length; + if (!isZero(chunk) || tail > TAR_RECORD) + throw new ArchiveError("damaged", "the archive has data after its end"); + } +} + +/** The regular files of a tar stream, in order. */ +export async function* readTar( + input: AsyncIterable, + options: { maxEntries: number }, +): AsyncGenerator { + const source = new ByteSource(input); + try { + for (let count = 0; ; count++) { + const block = await source.read(TAR_BLOCK); + if (block === undefined && source.position === 0) + throw new ArchiveError("not_archive", "not a Cloudhal site archive (it is empty)"); + if (block === undefined) + throw new ArchiveError("truncated", "the archive ends without its end-of-archive marker"); + const header = parseTarHeader(block); + if (header === "zero") { + await readEnd(source); + return; + } + if (count >= options.maxEntries) + throw new ArchiveError( + "too_large", + `the archive has more than ${options.maxEntries} entries`, + ); + const body = new EntryBody(source, header); + yield { path: header.path, size: header.size, body }; + await body.finish(); + await source.skip(tarPadding(header.size).length); + } + } finally { + await source.close(); + } +} diff --git a/packages/cli/src/archive/tar.ts b/packages/cli/src/archive/tar.ts new file mode 100644 index 00000000..eee00632 --- /dev/null +++ b/packages/cli/src/archive/tar.ts @@ -0,0 +1,176 @@ +/** + * POSIX ustar blocks for site archives (RFC-0039 §1): regular files only, no I/O. + * + * The writer's headers carry nothing that varies between machines or runs (invariant 7): mode + * 0644, uid/gid 0, mtime 0, empty user, group, link name and prefix. The reader accepts exactly the + * headers Cloudhal writes: a valid checksum, `ustar\0` + `00`, typeflag `0` (or NUL), no link name, + * no prefix and a printable ASCII name. Links, devices, directories, FIFOs and pax or GNU + * extension headers are refused, so an archive can never point outside itself. + */ +import { ArchiveError } from "./errors.ts"; + +export const TAR_BLOCK = 512; +export const TAR_RECORD = 10240; +/** The largest size 11 octal digits hold (8 GiB - 1). */ +export const TAR_MAX_SIZE = 0o77777777777; + +// Field offsets and widths of a ustar header. +const NAME = [0, 100] as const; +const MODE = [100, 8] as const; +const UID = [108, 8] as const; +const GID = [116, 8] as const; +const SIZE = [124, 12] as const; +const MTIME = [136, 12] as const; +const CHECKSUM = [148, 8] as const; +const TYPEFLAG = 156; +const LINKNAME = [157, 100] as const; +const MAGIC = [257, 8] as const; // "ustar\0" + version "00" +const DEVMAJOR = [329, 8] as const; +const DEVMINOR = [337, 8] as const; +const PREFIX = [345, 155] as const; + +const USTAR_MAGIC = "ustar\u000000"; + +const TYPE_NAMES: Record = { + "1": "a hard link", + "2": "a symbolic link", + "3": "a character device", + "4": "a block device", + "5": "a directory", + "6": "a FIFO", + "7": "a contiguous file", + x: "a pax extended header", + g: "a pax global header", + L: "a GNU long name", + K: "a GNU long link name", +}; + +function writeAscii(block: Uint8Array, [offset]: readonly [number, number], text: string): void { + for (let i = 0; i < text.length; i++) block[offset + i] = text.charCodeAt(i); +} + +function octal(value: number, digits: number): string { + return `${value.toString(8).padStart(digits, "0")}\u0000`; +} + +function isPrintableName(name: string): boolean { + for (let i = 0; i < name.length; i++) { + const c = name.charCodeAt(i); + if (c < 0x21 || c > 0x7e) return false; + } + return name.length > 0; +} + +/** The unsigned header checksum: every byte, with the checksum field counted as spaces. */ +function checksumOf(block: Uint8Array): number { + let sum = 0; + for (let i = 0; i < TAR_BLOCK; i++) + sum += i >= CHECKSUM[0] && i < CHECKSUM[0] + CHECKSUM[1] ? 0x20 : (block[i] as number); + return sum; +} + +/** Writes the checksum of `block` into it (`%06o\0 `) and returns it. */ +export function sealTarHeader(block: Uint8Array): Uint8Array { + writeAscii(block, CHECKSUM, `${checksumOf(block).toString(8).padStart(6, "0")}\u0000 `); + return block; +} + +/** The header of a regular file: `path` is printable ASCII of at most 99 bytes. */ +export function tarHeader(path: string, size: number): Uint8Array { + if (!isPrintableName(path) || path.length >= NAME[1]) + throw new ArchiveError("invalid_input", "a tar entry name must be 1-99 printable ASCII bytes"); + if (!Number.isSafeInteger(size) || size < 0 || size > TAR_MAX_SIZE) + throw new ArchiveError("too_large", `${path}: ${size} bytes do not fit a ustar header`, { + path, + }); + const block = new Uint8Array(TAR_BLOCK); + writeAscii(block, NAME, path); + writeAscii(block, MODE, octal(0o644, 7)); + writeAscii(block, UID, octal(0, 7)); + writeAscii(block, GID, octal(0, 7)); + writeAscii(block, SIZE, octal(size, 11)); + writeAscii(block, MTIME, octal(0, 11)); + block[TYPEFLAG] = 0x30; // "0": regular file + writeAscii(block, MAGIC, USTAR_MAGIC); + writeAscii(block, DEVMAJOR, octal(0, 7)); + writeAscii(block, DEVMINOR, octal(0, 7)); + return sealTarHeader(block); +} + +/** Zeros from the end of a `size`-byte body to the next block boundary. */ +export function tarPadding(size: number): Uint8Array { + return new Uint8Array((TAR_BLOCK - (size % TAR_BLOCK)) % TAR_BLOCK); +} + +/** Two zero blocks, then zeros so the archive is a whole number of 10240-byte records. */ +export function tarEnd(written: number): Uint8Array { + const total = Math.ceil((written + 2 * TAR_BLOCK) / TAR_RECORD) * TAR_RECORD; + return new Uint8Array(total - written); +} + +/** The length of the tar stream of files of these sizes, the end of the archive included. */ +export function tarLength(sizes: Iterable): number { + let written = 0; + for (const size of sizes) written += TAR_BLOCK + size + tarPadding(size).length; + return written + tarEnd(written).length; +} + +function isZero(bytes: Uint8Array, [offset, length]: readonly [number, number]): boolean { + for (let i = offset; i < offset + length; i++) if (bytes[i] !== 0) return false; + return true; +} + +/** An octal number field: optional leading spaces, digits, then NUL or space padding only. */ +function parseOctal(block: Uint8Array, [offset, length]: readonly [number, number]): number { + let i = offset; + const end = offset + length; + while (i < end && block[i] === 0x20) i++; + let value = 0; + let digits = 0; + for (; i < end && (block[i] as number) >= 0x30 && (block[i] as number) <= 0x37; i++, digits++) + value = value * 8 + ((block[i] as number) - 0x30); + for (; i < end; i++) if (block[i] !== 0 && block[i] !== 0x20) return Number.NaN; + return digits === 0 ? Number.NaN : value; +} + +function unsafe(message: string): ArchiveError { + return new ArchiveError("unsafe_entry", message); +} + +function checkType(block: Uint8Array): void { + const flag = block[TYPEFLAG] as number; + if (flag === 0x30 || flag === 0) return; + const char = String.fromCharCode(flag); + const what = TYPE_NAMES[char] ?? `an entry of type ${flag < 0x21 || flag > 0x7e ? flag : char}`; + throw unsafe(`the archive holds ${what}; site archives hold regular files only`); +} + +function parseName(block: Uint8Array): string { + const nul = block.indexOf(0, NAME[0]); + if (nul < 0 || nul >= NAME[1] || !isZero(block, [nul, NAME[1] - nul])) + throw unsafe("a tar entry name is not NUL-terminated"); + const name = String.fromCharCode(...block.subarray(NAME[0], nul)); + if (!isPrintableName(name)) throw unsafe("a tar entry name is empty or not printable ASCII"); + return name; +} + +/** + * One header block: `"zero"` for an all-zero block (the end of the archive), otherwise the regular + * file it describes. A bad checksum is `damaged`; anything Cloudhal does not write is + * `unsafe_entry`. + */ +export function parseTarHeader(block: Uint8Array): { path: string; size: number } | "zero" { + if (isZero(block, [0, TAR_BLOCK])) return "zero"; + if (parseOctal(block, CHECKSUM) !== checksumOf(block)) + throw new ArchiveError("damaged", "a tar header checksum does not match (the file is damaged)"); + if (String.fromCharCode(...block.subarray(MAGIC[0], MAGIC[0] + MAGIC[1])) !== USTAR_MAGIC) + throw unsafe("a tar header is not POSIX ustar"); + checkType(block); + if (!isZero(block, LINKNAME)) throw unsafe("a tar entry has a link name"); + if (!isZero(block, PREFIX)) throw unsafe("a tar entry has a name prefix"); + const path = parseName(block); + if (((block[SIZE[0]] as number) & 0x80) !== 0) throw unsafe("a tar entry size is base-256"); + const size = parseOctal(block, SIZE); + if (Number.isNaN(size)) throw unsafe("a tar entry size is not an octal number"); + return { path, size }; +} diff --git a/packages/cli/src/archive/write.ts b/packages/cli/src/archive/write.ts new file mode 100644 index 00000000..a9786cf6 --- /dev/null +++ b/packages/cli/src/archive/write.ts @@ -0,0 +1,126 @@ +/** + * The site archive writer (RFC-0039 §1): `manifest.json` first, then the files in the one order of + * the layout, as a POSIX ustar stream compressed by `gzipFixed`. Media are streamed and verified + * against their SHA-256 and size on the way; a mismatch ends the stream before the gzip trailer, so + * a partial output never reads as an archive, and `writeArchiveFile` removes it. + * + * Determinism (invariant 7): nothing here reads the clock or the machine. `createdAt` and + * `createdBy` come from the caller (`cloudhal export` passes the newest row timestamp of the site + * and its own version); rows come in the caller's order, media are sorted by hash, and tar headers + * carry fixed modes and zero times. The same content written by the same Node/zlib build gives + * identical bytes; the tar stream inside the gzip, and so every SHA-256 in the manifest, is + * identical on every platform. + */ +import { createHash, randomUUID } from "node:crypto"; +import { type FileHandle, open, rename, rm } from "node:fs/promises"; +import path from "node:path"; +import { pipeline } from "node:stream/promises"; +import { ArchiveError } from "./errors.ts"; +import { gzipFixed } from "./gzip.ts"; +import { MANIFEST_PATH } from "./layout.ts"; +import { type ArchiveLimits, archiveLimits } from "./limits.ts"; +import type { SiteArchiveManifest } from "./manifest.ts"; +import { + type ArchiveContent, + type ArchiveMediaSource, + type ArchivePlan, + planArchive, +} from "./plan.ts"; +import { TAR_BLOCK, tarEnd, tarHeader, tarPadding } from "./tar.ts"; + +export interface WriteArchiveOptions { + /** Lower caps (tests); the writer refuses what a reader with the same caps would refuse. */ + limits?: Partial; +} + +/** A media body as it is written: no more than `bytes`, then the SHA-256. */ +async function* verifiedMedia( + file: string, + media: ArchiveMediaSource, +): AsyncGenerator { + const hash = createHash("sha256"); + let seen = 0; + for await (const chunk of await media.open()) { + seen += chunk.length; + if (seen > media.bytes) + throw new ArchiveError( + "damaged", + `${file}: the media source is longer than ${media.bytes} bytes`, + { + path: file, + }, + ); + hash.update(chunk); + yield chunk; + } + if (seen < media.bytes) + throw new ArchiveError( + "damaged", + `${file}: the media source has ${seen} of ${media.bytes} bytes`, + { + path: file, + }, + ); + if (hash.digest("hex") !== media.sha256) + throw new ArchiveError("damaged", `${file}: the media source does not match its SHA-256`, { + path: file, + }); +} + +/** The tar stream of a plan. */ +async function* tarChunks(plan: ArchivePlan): AsyncGenerator { + let written = 0; + for (const file of [{ path: MANIFEST_PATH, data: plan.manifestBytes }, ...plan.files]) { + const size = "data" in file ? file.data.length : file.media.bytes; + yield tarHeader(file.path, size); + if ("data" in file) yield file.data; + else yield* verifiedMedia(file.path, file.media); + const padding = tarPadding(size); + if (padding.length > 0) yield padding; + written += TAR_BLOCK + size + padding.length; + } + yield tarEnd(written); +} + +/** + * The archive of `content` as a byte stream, with its manifest. The content is checked before + * this returns (it throws, and nothing is streamed); a media mismatch fails the stream. + */ +export function archiveStream( + content: ArchiveContent, + options: WriteArchiveOptions = {}, +): { manifest: SiteArchiveManifest; bytes: AsyncGenerator } { + const plan = planArchive(content, archiveLimits(options.limits)); + return { manifest: plan.manifest, bytes: gzipFixed(tarChunks(plan)) }; +} + +/** + * Writes the archive of `content` to `file`: to a temporary file next to it first, synced and then + * renamed over `file`, so a failure leaves no file behind and an existing `file` unchanged. + */ +export async function writeArchiveFile( + file: string, + content: ArchiveContent, + options: WriteArchiveOptions = {}, +): Promise { + const { manifest, bytes } = archiveStream(content, options); + const temp = path.join( + path.dirname(file), + `.${path.basename(file)}.partial-${process.pid}-${randomUUID().slice(0, 8)}`, + ); + let handle: FileHandle | undefined; + try { + handle = await open(temp, "wx"); + // The stream owns the handle: it fsyncs (flush) and closes it, on errors too. + await pipeline(bytes, handle.createWriteStream({ flush: true })); + await rename(temp, file); + return manifest; + } catch (e) { + await bytes.return(); + if (handle !== undefined) { + await handle.close().catch(() => {}); + await rm(temp, { force: true }); + } + throw e; + } +} diff --git a/packages/cli/test/archive-fixture.ts b/packages/cli/test/archive-fixture.ts new file mode 100644 index 00000000..b65c195a --- /dev/null +++ b/packages/cli/test/archive-fixture.ts @@ -0,0 +1,248 @@ +/** + * Helpers for the site archive tests (RFC-0039 step 1): fixed sample content, chunked sources, a + * full read-back, and hand-built tar streams for hostile input. The ustar offsets used to patch + * headers are spelled out here, apart from src/archive/tar.ts, as an independent reading of the + * format. + */ +import { createHash } from "node:crypto"; +import { gunzipSync, gzipSync } from "node:zlib"; +import { expect } from "vitest"; +import { + type ArchiveContent, + ArchiveError, + type ArchiveErrorCode, + type ArchiveMediaSource, + archiveStream, + encodeNdjson, + type ReadArchiveOptions, + readArchive, + type SiteArchiveManifest, + splitNdjson, + type WriteArchiveOptions, +} from "../src/archive/index.ts"; +import { sealTarHeader, tarHeader, tarPadding } from "../src/archive/tar.ts"; + +/** ustar header offsets (POSIX.1-1988). */ +export const USTAR = { + name: 0, + size: 124, + checksum: 148, + typeflag: 156, + linkname: 157, + magic: 257, + version: 263, + prefix: 345, +} as const; + +export const utf8 = (text: string): Uint8Array => new TextEncoder().encode(text); +export const sha256 = (bytes: Uint8Array): string => + createHash("sha256").update(bytes).digest("hex"); + +export const SITE = { + siteId: "site_demo0001", + siteName: "Demo site", + createdBy: "0.6.1", + createdAt: "2026-10-20T12:00:00.000Z", +}; + +/** Deterministic bytes of a given length. */ +export function pattern(length: number, seed: number): Uint8Array { + const out = new Uint8Array(length); + for (let i = 0; i < length; i++) out[i] = (i * 31 + seed * 7) % 251; + return out; +} + +/** Byte-for-byte equality. */ +export const same = (a: Uint8Array | undefined, b: Uint8Array): boolean => + a !== undefined && Buffer.compare(a, b) === 0; + +/** XORs one byte in place. */ +export function flip(bytes: Uint8Array, at: number, mask = 1): void { + bytes[at] = (bytes[at] as number) ^ mask; +} + +/** `files[index]` of a manifest object under test. */ +export function listedAt( + manifest: Record, + index: number, +): Record { + return (manifest.files as Record[])[index] as Record; +} + +export async function* chunked(bytes: Uint8Array, size: number): AsyncGenerator { + for (let i = 0; i < bytes.length; i += size) yield bytes.subarray(i, i + size); +} + +export async function collect(iter: AsyncIterable): Promise { + const chunks: Uint8Array[] = []; + for await (const chunk of iter) chunks.push(chunk); + return Buffer.concat(chunks); +} + +export function memoryMedia(bytes: Uint8Array, chunk = 64 * 1024): ArchiveMediaSource { + return { sha256: sha256(bytes), bytes: bytes.length, open: () => chunked(bytes, chunk) }; +} + +export const MEDIA = [pattern(3000, 1), pattern(700, 2), pattern(0, 3), pattern(70_000, 4)]; + +export const NODE_ROWS = Array.from({ length: 12 }, (_, i) => ({ + id: `page_${String(i).padStart(8, "0")}`, + kind: "page", + body: JSON.stringify({ title: `Page ${i}`, text: "lorem ipsum ".repeat(5) }), +})); + +/** Sample content: nodes in several parts, snapshots, blobs, project, ledger and media. */ +export function sampleContent(overrides: Partial = {}): ArchiveContent { + return { + ...SITE, + project: utf8(JSON.stringify({ id: "prj_demo0001", rev: 1 })), + nodes: splitNdjson(NODE_ROWS, { partBytes: 512 }), + snapshots: splitNdjson([{ id: "snap_00000001" }, { id: "snap_00000002" }]), + blobs: splitNdjson([{ hash: "a".repeat(64), body: "{}" }]), + ledger: encodeNdjson([{ sha256: "b".repeat(64), reason: "dmca", legal: true }]), + media: MEDIA.map((m) => memoryMedia(m)), + ...overrides, + }; +} + +export async function writeBytes( + content: ArchiveContent, + options: WriteArchiveOptions = {}, +): Promise { + return collect(archiveStream(content, options).bytes); +} + +export interface ReadBack { + manifest: SiteArchiveManifest; + files: { path: string; bytes: Buffer }[]; +} + +/** Every item of an archive, media bodies read in full. */ +export async function readAll( + bytes: Uint8Array, + options: ReadArchiveOptions & { chunk?: number } = {}, +): Promise { + let manifest: SiteArchiveManifest | undefined; + const files: { path: string; bytes: Buffer }[] = []; + for await (const item of readArchive(chunked(bytes, options.chunk ?? 4096), options)) { + if (item.type === "manifest") manifest = item.manifest; + else if (item.type === "file") files.push({ path: item.path, bytes: Buffer.from(item.bytes) }); + else files.push({ path: item.path, bytes: await collect(item.body) }); + } + if (!manifest) throw new Error("no manifest"); + return { manifest, files }; +} + +export interface RawEntry { + path: string; + body: Uint8Array | string; + /** Declared size, when it should differ from the body's. */ + size?: number; + /** Edits the header before it is sealed with a fresh checksum. */ + patch?: (header: Uint8Array) => void; +} + +function setName(header: Uint8Array, path: Uint8Array): void { + header.fill(0, USTAR.name, USTAR.name + 100); + header.set(path.subarray(0, 100), USTAR.name); +} + +/** A tar stream built entry by entry; `end: false` leaves out the two zero blocks. */ +export function rawTar( + entries: RawEntry[], + options: { end?: boolean; tail?: Uint8Array } = {}, +): Buffer { + const out: Uint8Array[] = []; + for (const entry of entries) { + const body = typeof entry.body === "string" ? utf8(entry.body) : entry.body; + const header = tarHeader("x", entry.size ?? body.length); + setName(header, utf8(entry.path)); + entry.patch?.(header); + out.push(sealTarHeader(header), body, tarPadding(body.length)); + } + if (options.end !== false) out.push(new Uint8Array(1024)); + if (options.tail) out.push(options.tail); + return Buffer.concat(out); +} + +const LF = 0x0a; +function rows(body: Uint8Array): number { + return body.filter((b) => b === LF).length; +} + +/** A manifest (as a plain object, for tests to break) listing `files`. */ +export function manifestFor(files: RawEntry[]): Record { + const counts = { nodes: 0, snapshots: 0, blobs: 0, media: 0, mediaBytes: 0 }; + const listed = files.map((f) => { + const body = typeof f.body === "string" ? utf8(f.body) : f.body; + const kind = f.path.split("/")[0] as string; + if (kind === "nodes" || kind === "snapshots" || kind === "blobs") counts[kind] += rows(body); + if (kind === "media") { + counts.media++; + counts.mediaBytes += body.length; + } + return { path: f.path, sha256: sha256(body), bytes: body.length }; + }); + return { + format: "cloudhal.site-archive@1", + contracts: { schemaVersion: 1 }, + bundle: "cloudhal.site-bundle@1", + ...SITE, + counts, + files: listed, + }; +} + +/** A small valid file set: project, two node parts, one snapshot part and a media file. */ +export function baseFiles(): RawEntry[] { + const media = pattern(900, 9); + return [ + { path: "project.json", body: '{"id":"prj_demo0001"}' }, + { path: "nodes/0001.ndjson", body: '{"id":"n1"}\n{"id":"n2"}\n' }, + { path: "nodes/0002.ndjson", body: '{"id":"n3"}\n' }, + { path: "snapshots/0001.ndjson", body: '{"id":"s1"}\n' }, + { path: `media/${sha256(media)}`, body: media }, + ]; +} + +/** + * A gzipped archive of `entries` whose manifest lists `listed` (default: the same entries); + * `manifest` edits the manifest object first. + */ +export function archiveOf( + entries: RawEntry[], + options: { + listed?: RawEntry[]; + manifest?: (m: Record) => void; + end?: boolean; + tail?: Uint8Array; + } = {}, +): Buffer { + const manifest = manifestFor(options.listed ?? entries); + options.manifest?.(manifest); + const head: RawEntry = { path: "manifest.json", body: JSON.stringify(manifest) }; + return gzipSync(rawTar([head, ...entries], options)); +} + +/** A valid archive's tar, edited, compressed again. */ +export function retar(archive: Uint8Array, edit: (tar: Buffer) => Buffer | undefined): Buffer { + const tar = gunzipSync(archive); + return gzipSync(edit(tar) ?? tar); +} + +/** Awaits a rejection and checks it is an ArchiveError with `code` (and `text`). */ +export async function expectArchiveError( + promise: Promise, + code: ArchiveErrorCode, + text?: string | RegExp, +): Promise { + const error = await promise.then( + () => undefined, + (e: unknown) => e, + ); + expect(error, `expected an ArchiveError ${code}`).toBeInstanceOf(ArchiveError); + const archiveError = error as ArchiveError; + expect({ code: archiveError.code, message: archiveError.message }).toMatchObject({ code }); + if (text !== undefined) expect(archiveError.message).toMatch(text); + return archiveError; +} diff --git a/packages/cli/test/archive-hostile.test.ts b/packages/cli/test/archive-hostile.test.ts new file mode 100644 index 00000000..fde6b52f --- /dev/null +++ b/packages/cli/test/archive-hostile.test.ts @@ -0,0 +1,471 @@ +/** + * RFC-0039 invariant 3 and Security: an archive is untrusted input. Damaged, edited, reordered or + * oversized archives, paths outside the layout, links and devices, and broken or padded gzip are + * refused, and nothing unverified is handed over as verified. + */ +import { createReadStream } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { gzipSync } from "node:zlib"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { type ArchiveErrorCode, type ArchiveItem, readArchive } from "../src/archive/index.ts"; +import { encodeManifest } from "../src/archive/manifest.ts"; +import { tarLength } from "../src/archive/tar.ts"; +import { + archiveOf, + baseFiles, + chunked, + collect, + expectArchiveError, + flip, + listedAt, + MEDIA, + pattern, + type RawEntry, + rawTar, + readAll, + retar, + sampleContent, + sha256, + USTAR, + writeBytes, +} from "./archive-fixture.ts"; + +const valid = await writeBytes(sampleContent()); + +let dir: string; +beforeAll(async () => { + dir = await mkdtemp(path.join(os.tmpdir(), "archive-hostile-")); +}); +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +/** The offset of the body of `path` in a tar stream. */ +function bodyOffset(tar: Buffer, path: string): number { + for (let at = 0; at < tar.length; at += 512) + if ( + Buffer.from(tar.subarray(at, at + 100)) + .toString("latin1") + .split("\0")[0] === path + ) + return at + 512; + throw new Error(`no ${path}`); +} + +/** Items read until the first error, which is returned with them. */ +async function readUntilError(archive: Uint8Array, drainMedia = true) { + const seen: ArchiveItem[] = []; + try { + for await (const item of readArchive(chunked(archive, 4096))) { + seen.push(item); + if (item.type === "media" && drainMedia) await collect(item.body); + } + } catch (error) { + return { seen, error }; + } + return { seen, error: undefined }; +} + +describe("damaged and edited files", () => { + it("refuses a part with a flipped byte and never hands it over", async () => { + const damaged = retar(valid, (tar) => { + flip(tar, bodyOffset(tar, "nodes/0002.ndjson") + 10); + return tar; + }); + const { seen, error } = await readUntilError(damaged); + await expectArchiveError(Promise.reject(error), "damaged", /^nodes\/0002\.ndjson: .*SHA-256/); + expect(seen.map((i) => (i.type === "manifest" ? "manifest.json" : i.path))).toEqual([ + "manifest.json", + "project.json", + "nodes/0001.ndjson", + ]); + }); + + it("refuses a media file with a flipped byte, read or not", async () => { + const { manifest } = await readAll(valid); + const media = manifest.files.find((f) => f.path.startsWith("media/") && f.bytes > 0); + const path = media?.path as string; + const damaged = retar(valid, (tar) => { + flip(tar, bodyOffset(tar, path) + 5); + return tar; + }); + // Read: the body's last next() throws. + let bodyError: unknown; + for await (const item of readArchive(chunked(damaged, 1000))) { + if (item.type !== "media" || item.path !== path) continue; + bodyError = await collect(item.body).then( + () => undefined, + (e: unknown) => e, + ); + break; + } + await expectArchiveError(Promise.reject(bodyError), "damaged", path); + // Not read: the reader drains it and throws before the next item. + const { seen, error } = await readUntilError(damaged, false); + await expectArchiveError(Promise.reject(error), "damaged", path); + expect(seen.at(-1)).toMatchObject({ type: "media", path }); + }); + + it("refuses a listed hash other than the media name, a size or a row count that disagrees", async () => { + const files = baseFiles(); + const edited = (edit: (m: Record) => void) => + readAll(archiveOf(files, { manifest: edit })); + await expectArchiveError( + edited((m) => { + listedAt(m, 4).sha256 = "0".repeat(64); + }), + "invalid_manifest", + /SHA-256 other than its name/, + ); + await expectArchiveError( + edited((m) => { + listedAt(m, 1).bytes = 25; + }), + "mismatch", + /^nodes\/0001\.ndjson: 24 bytes in the archive, 25 in manifest\.json/, + ); + await expectArchiveError( + edited((m) => { + (m.counts as { nodes: number }).nodes = 2; + }), + "mismatch", + /nodes: the parts hold 3 rows, manifest\.json counts 2/, + ); + }); +}); + +const HEX = "ab".repeat(32); +const BAD_PATHS = [ + "../x", + "/etc/passwd", + "nodes/../manifest.json", + "nodes\\0001.ndjson", + "./manifest.json", + "nodes/0000.ndjson", + "nodes/1.ndjson", + "nodes/00001.ndjson", + "nodes/0001.json", + `media/${HEX.toUpperCase()}`, + `media/${HEX.slice(1)}`, + `media/${HEX}.png`, + `media/${HEX}/`, + "Manifest.json", + "project.json/", + "x.txt", + "ledger.ndjson.gz", + "", +]; + +describe("paths outside the layout", () => { + it.each(BAD_PATHS)("refuses %j as an entry and in the manifest", async (bad) => { + const files = baseFiles(); + await expectArchiveError( + readAll(archiveOf([...files, { path: bad, body: "x" }], { listed: files })), + "unsafe_entry", + ); + await expectArchiveError( + readAll( + archiveOf(files, { + manifest: (m) => { + (m.files as unknown[]).push({ path: bad, sha256: HEX, bytes: 1 }); + }, + }), + ), + "invalid_manifest", + ); + }); + + it("refuses symlinks, hard links and directories inside a valid archive", async () => { + const files = baseFiles(); + for (const flag of ["2", "1", "5"]) { + const link: RawEntry = { + path: "nodes/0003.ndjson", + body: "", + patch: (h) => { + h[USTAR.typeflag] = flag.charCodeAt(0); + if (flag !== "5") h.set(Buffer.from("/etc/passwd"), USTAR.linkname); + }, + }; + await expectArchiveError( + readAll(archiveOf([...files.slice(0, 3), link, ...files.slice(3)], { listed: files })), + "unsafe_entry", + ); + } + }); +}); + +describe("entries against the manifest", () => { + const files = baseFiles(); + const [project, n1, n2, s1, media] = files as [RawEntry, RawEntry, RawEntry, RawEntry, RawEntry]; + const other = Buffer.from("other media"); + const second: RawEntry = { path: `media/${sha256(other)}`, body: other }; + const [low, high] = [media, second].sort((a, b) => (a.path < b.path ? -1 : 1)) as [ + RawEntry, + RawEntry, + ]; + const cases: [string, RawEntry[], RawEntry[], RegExp][] = [ + [ + "an unlisted extra entry", + [...files, { path: "blobs/0001.ndjson", body: "{}\n" }], + files, + /not listed/, + ], + ["a duplicate entry", [project, n1, n1, n2, s1, media], files, /twice or out of order/], + ["snapshots before nodes", [project, s1, n1, n2, media], files, /twice or out of order/], + [ + "unsorted media", + [project, n1, n2, s1, high, low], + [project, n1, n2, s1, low, high], + /twice or out of order/, + ], + [ + "a listed but missing file", + files.slice(0, 4), + files, + /listed in manifest\.json but not in the archive/, + ], + [ + "a second manifest", + [...files, { path: "manifest.json", body: "{}" }], + files, + /appears twice/, + ], + ]; + it.each(cases)("refuses %s", async (_, entries, listed, text) => { + await expectArchiveError(readAll(archiveOf(entries, { listed })), "mismatch", text); + }); +}); + +describe("caps", () => { + it("refuses an oversized manifest before reading it", async () => { + // A declared size of 1 GiB and no body: the cap answers before the truncation could. + const tar = rawTar([{ path: "manifest.json", body: "{}", size: 1 << 30 }], { end: false }); + await expectArchiveError(readAll(gzipSync(tar)), "too_large", /manifest\.json is above/); + await expectArchiveError(readAll(valid, { limits: { manifestBytes: 100 } }), "too_large"); + }); + + it("refuses parts, media, totals and entry counts above the caps at manifest time", async () => { + await expectArchiveError(readAll(valid, { limits: { partBytes: 100 } }), "too_large", /cap/); + await expectArchiveError( + readAll(valid, { limits: { mediaBytes: 1000 } }), + "too_large", + /media\//, + ); + await expectArchiveError( + readAll(valid, { limits: { totalBytes: 50_000 } }), + "too_large", + /decompressed/, + ); + await expectArchiveError(readAll(valid, { limits: { entries: 5 } }), "too_large", /entries/); + }); + + it("counts decompressed bytes while streaming", async () => { + const { manifest } = await readAll(valid); + const exact = tarLength([ + encodeManifest(manifest).length, + ...manifest.files.map((f) => f.bytes), + ]); + expect((await readAll(valid, { limits: { totalBytes: exact } })).files).toHaveLength( + manifest.files.length, + ); + // 512 more zeros after the end: within the padding the reader allows, beyond the cap. + const padded = retar(valid, (tar) => Buffer.concat([tar, new Uint8Array(512)])); + expect((await readAll(padded, { limits: { totalBytes: exact + 512 } })).files).toHaveLength( + manifest.files.length, + ); + await expectArchiveError( + readAll(padded, { limits: { totalBytes: exact + 511 } }), + "too_large", + /decompressed/, + ); + }); +}); + +describe("gzip", () => { + it("refuses a truncated stream, even after the last item", async () => { + for (const cut of [5, Math.floor(valid.length / 2), valid.length - 4]) { + await expectArchiveError(readAll(valid.subarray(0, cut)), "truncated"); + } + }); + + it("refuses corrupt data, a wrong CRC, non-gzip input and garbage after the trailer", async () => { + const crc = Buffer.from(valid); + flip(crc, crc.length - 8); + await expectArchiveError(readAll(crc), "damaged", /gzip/); + const mid = Buffer.from(valid); + flip(mid, Math.floor(mid.length / 2), 0xff); + await expectArchiveError(readAll(mid), "damaged"); + await expectArchiveError(readAll(Buffer.from("hello")), "not_archive", /not gzip/); + await expectArchiveError(readAll(Buffer.concat([valid, Buffer.from("garbage")])), "damaged"); + }); + + it("tells damaged gzip from input that is not gzip by its first bytes, at any size", async () => { + // Read in one chunk, an archive under 16 KiB decompressed fails in zlib before any output. + const small = archiveOf(baseFiles()); + const crc = Buffer.from(small); + flip(crc, crc.length - 8); + const mid = Buffer.from(small); + flip(mid, Math.floor(mid.length / 2), 0xff); + const cases: [Uint8Array, ArchiveErrorCode, number[], RegExp?][] = [ + [crc, "damaged", [crc.length, 64], /gzip data is corrupt/], + // In 64-byte chunks the damaged manifest is parsed first: invalid_manifest. + [mid, "damaged", [mid.length]], + [Buffer.concat([small, Buffer.from("garbage")]), "damaged", [small.length + 7, 64]], + [small.subarray(0, small.length - 4), "truncated", [small.length, 64]], + [Buffer.from("hello"), "not_archive", [5, 1], /not gzip/], + [Buffer.from([0x1f]), "not_archive", [1], /not gzip/], + [new Uint8Array(0), "not_archive", [1], /it is empty/], + ]; + for (const [bytes, code, chunks, text] of cases) + for (const chunk of chunks) await expectArchiveError(readAll(bytes, { chunk }), code, text); + }); + + it("refuses data after the gzip stream that starts with a zero byte, however it is chunked", async () => { + const tails = [ + Buffer.from([0]), + Buffer.from("\0junk"), + Buffer.alloc(64), + Buffer.concat([Buffer.alloc(64), Buffer.from("junk")]), + Buffer.concat([Buffer.from([0]), Buffer.alloc(1_200_000, "A")]), + ]; + for (const tail of tails) { + const padded = Buffer.concat([valid, tail]); + for (const chunk of [padded.length, 4096, 64]) + await expectArchiveError(readAll(padded, { chunk }), "damaged", /after its gzip stream/); + async function* apart() { + yield valid; + yield* chunked(tail, 16); + } + await expectArchiveError(collectItems(readArchive(apart())), "damaged", /gzip stream/); + } + }); + + it("refuses data after the gzip stream of a file read stream", async () => { + const tails = [ + Buffer.alloc(8), + Buffer.from("\0hidden"), + Buffer.concat([Buffer.from([0]), Buffer.alloc(200_000, "A")]), + Buffer.alloc(1 << 20), + ]; + for (const [i, tail] of tails.entries()) { + const file = path.join(dir, `tail-${i}.cloudhal`); + await writeFile(file, Buffer.concat([valid, tail])); + await expectArchiveError( + collectItems(readArchive(createReadStream(file))), + "damaged", + /after its gzip stream/, + ); + } + const clean = path.join(dir, "clean.cloudhal"); + await writeFile(clean, valid); + expect(await collectItems(readArchive(createReadStream(clean)))).toBeGreaterThan(1); + }); + + it("stops a gzip bomb after the end of the archive", async () => { + const member = gzipSync(new Uint8Array(1 << 20)); // 1 MiB of zeros in about 1 KiB + let pulled = 0; + async function* bomb() { + yield valid; + for (let i = 0; i < 1024; i++) { + pulled++; + yield member; + } + } + await expectArchiveError(collectItems(readArchive(bomb())), "damaged", /after its end/); + expect(pulled).toBeLessThan(64); + }); + + it("refuses a second gzip member that holds another entry", async () => { + const extra = gzipSync(rawTar([{ path: "nodes/0004.ndjson", body: "{}\n" }])); + await expectArchiveError(readAll(Buffer.concat([valid, extra])), "damaged", /after its end/); + }); +}); + +async function collectItems(items: AsyncIterable): Promise { + let n = 0; + for await (const item of items) { + if (item.type === "media") await collect(item.body); + n++; + } + return n; +} + +type MediaItem = Extract; + +/** The first media item of `items` with at least `size` bytes. */ +async function firstMedia(items: AsyncGenerator, size = 0): Promise { + for (;;) { + const next = await items.next(); + if (next.done) throw new Error("no media item"); + if (next.value.type === "media" && next.value.size >= size) return next.value; + } +} + +/** Reads an iterator to its end. */ +async function toEnd(body: AsyncIterator): Promise { + for (;;) if ((await body.next()).done) return; +} + +describe("media bodies", () => { + it("refuses a body read after the next item was requested, an empty one too", async () => { + const held: MediaItem[] = []; + for await (const item of readArchive(chunked(valid, 4096))) + if (item.type === "media") held.push(item); + expect(held.map((m) => m.size).sort((a, b) => a - b)).toEqual( + MEDIA.map((m) => m.length).sort((a, b) => a - b), + ); + for (const item of held) + await expectArchiveError( + collect(item.body), + "invalid_input", + `${item.path}: a media body was read after the next item was requested`, + ); + }); + + it("refuses to move on while a body is still being read", async () => { + const items = readArchive(chunked(valid, 1000)); + const body = (await firstMedia(items, 1)).body[Symbol.asyncIterator](); + const pending = body.next(); // a sink started without awaiting it + await expectArchiveError(items.next(), "invalid_input", /is still being read/); + await pending.catch(() => undefined); + await expectArchiveError(body.next(), "invalid_input", /after the next item/); + }); + + it("keeps a read error, so a retry never ends a partial body cleanly", async () => { + // Named and listed by the hash of the 512 bytes the archive holds, declared 1024 long. + const held = pattern(512, 5); + const cut: RawEntry = { path: `media/${sha256(held)}`, body: held, size: 1024 }; + const archive = archiveOf([...baseFiles().slice(0, 4), cut], { + end: false, + manifest: (m) => { + listedAt(m, 4).bytes = 1024; + (m.counts as { mediaBytes: number }).mediaBytes = 1024; + }, + }); + const items = readArchive(chunked(archive, 4096)); + const body = (await firstMedia(items)).body[Symbol.asyncIterator](); + await expectArchiveError(toEnd(body), "truncated"); + await expectArchiveError(body.next(), "truncated"); + await expectArchiveError(items.next(), "truncated"); + }); +}); + +describe("stopping early", () => { + it("closes the source when the caller stops after the manifest", async () => { + let closed = false; + async function* source() { + try { + yield* chunked(valid, 512); + } finally { + closed = true; + } + } + for await (const item of readArchive(source())) { + expect(item.type).toBe("manifest"); + break; + } + await vi.waitFor(() => expect(closed).toBe(true)); + }); +}); diff --git a/packages/cli/test/archive-interop.test.ts b/packages/cli/test/archive-interop.test.ts new file mode 100644 index 00000000..a5aef3ce --- /dev/null +++ b/packages/cli/test/archive-interop.test.ts @@ -0,0 +1,45 @@ +/** + * RFC-0039 §1: a `.cloudhal` file is a standard gzipped ustar. The system `tar` (bsdtar on macOS, + * GNU tar on Linux CI) lists exactly the archive's paths and extracts identical bytes. Skipped + * where no `tar` is installed. + */ +import { spawnSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { writeArchiveFile } from "../src/archive/index.ts"; +import { encodeManifest } from "../src/archive/manifest.ts"; +import { MEDIA, same, sampleContent, sha256 } from "./archive-fixture.ts"; + +const hasTar = spawnSync("tar", ["--version"], { stdio: "ignore" }).status === 0; + +describe.skipIf(!hasTar)("system tar", () => { + it("lists and extracts the archive", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "archive-interop-")); + try { + const file = path.join(dir, "site.cloudhal"); + const content = sampleContent(); + const manifest = await writeArchiveFile(file, content); + const list = spawnSync("tar", ["-tzf", file], { encoding: "utf8" }); + expect(list.status, list.stderr).toBe(0); + expect(list.stdout.split("\n").filter(Boolean)).toEqual([ + "manifest.json", + ...manifest.files.map((f) => f.path), + ]); + const out = path.join(dir, "out"); + await mkdir(out); + const extract = spawnSync("tar", ["-xzf", file, "-C", out], { encoding: "utf8" }); + expect(extract.status, extract.stderr).toBe(0); + const read = (p: string) => readFile(path.join(out, p)); + expect(same(await read("manifest.json"), encodeManifest(manifest))).toBe(true); + expect(same(await read("project.json"), content.project as Uint8Array)).toBe(true); + expect(same(await read("ledger.ndjson"), content.ledger as Uint8Array)).toBe(true); + for (const [i, part] of content.nodes.entries()) + expect(same(await read(`nodes/${String(i + 1).padStart(4, "0")}.ndjson`), part)).toBe(true); + for (const m of MEDIA) expect(same(await read(`media/${sha256(m)}`), m)).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/test/archive-manifest.test.ts b/packages/cli/test/archive-manifest.test.ts new file mode 100644 index 00000000..b36e4d49 --- /dev/null +++ b/packages/cli/test/archive-manifest.test.ts @@ -0,0 +1,214 @@ +/** + * RFC-0039 §1 and invariant 4: `manifest.json` comes first and is checked strictly; an archive of + * a newer format, contracts schemaVersion or bundle format is refused with the CLI version needed. + */ +import { gzipSync } from "node:zlib"; +import { describe, expect, it } from "vitest"; +import type { ArchiveErrorCode } from "../src/archive/index.ts"; +import { + archiveOf, + baseFiles, + expectArchiveError, + listedAt, + manifestFor, + pattern, + type RawEntry, + rawTar, + readAll, + sha256, +} from "./archive-fixture.ts"; + +type Edit = (m: Record) => void; + +const files = baseFiles(); +const withManifest = (edit: Edit) => readAll(archiveOf(files, { manifest: edit })); +const counts = (m: Record) => m.counts as Record; +/** Edits that set one value: on the manifest, its counts, or one listed file. */ +const set = + (key: string, value: unknown): Edit => + (m) => { + m[key] = value; + }; +const setCount = + (key: string, value: unknown): Edit => + (m) => { + counts(m)[key] = value; + }; +const setFile = + (index: number, key: string, value: unknown): Edit => + (m) => { + listedAt(m, index)[key] = value; + }; +const drop = + (key: string): Edit => + (m) => { + delete m[key]; + }; + +describe("the first entry", () => { + it("must be manifest.json", async () => { + const tar = rawTar([ + ...files, + { path: "manifest.json", body: JSON.stringify(manifestFor(files)) }, + ]); + await expectArchiveError(readAll(gzipSync(tar)), "not_archive", /first entry/); + await expectArchiveError(readAll(gzipSync(new Uint8Array(10240))), "not_archive"); + await expectArchiveError(readAll(gzipSync(new Uint8Array(0))), "not_archive"); + }); + + it("must be UTF-8 JSON without a byte order mark", async () => { + const raw = (body: Uint8Array | string) => + readAll(gzipSync(rawTar([{ path: "manifest.json", body }]))); + await expectArchiveError(raw("{not json"), "invalid_manifest", /UTF-8 JSON/); + await expectArchiveError(raw(new Uint8Array([0x7b, 0xff, 0x7d])), "invalid_manifest"); + const bom = Buffer.concat([ + Buffer.from([0xef, 0xbb, 0xbf]), + Buffer.from(JSON.stringify(manifestFor(files))), + ]); + await expectArchiveError(raw(bom), "invalid_manifest"); + await expectArchiveError(raw("[]"), "not_archive"); + }); +}); + +describe("the manifest's shape", () => { + const cases: [string, Edit][] = [ + ["an unknown key", set("extra", 1)], + ["an unknown counts key", setCount("pages", 1)], + ["an unknown files key", setFile(0, "mode", 420)], + ["a missing field", drop("siteName")], + ["a missing contracts version", set("contracts", {})], + ["a contracts version 0", set("contracts", { schemaVersion: 0 })], + ["a negative count", setCount("blobs", -1)], + ["a fractional count", setCount("blobs", 0.5)], + ["a site id that is not an id", set("siteId", "../x")], + ["an empty site name", set("siteName", "")], + ["a createdAt that is not RFC 3339", set("createdAt", "2026-10-20")], + ["a bundle that is not a bundle format", set("bundle", "zip")], + ["files that are not an array", set("files", {})], + [ + "a file listed twice", + (m) => { + (m.files as unknown[]).splice(1, 0, listedAt(m, 1)); + }, + ], + [ + "files out of order", + (m) => { + (m.files as unknown[]).reverse(); + }, + ], + ["a gap in the part numbers", setFile(2, "path", "nodes/0003.ndjson")], + ["an empty part", setFile(2, "bytes", 0)], + ["an uppercase hash", setFile(0, "sha256", "A".repeat(64))], + ["a size that is not an integer", setFile(0, "bytes", "21")], + ["a media count that disagrees", setCount("media", 2)], + ["a media size that disagrees", setCount("mediaBytes", 1)], + ["mediaOmitted false", set("mediaOmitted", false)], + ["mediaOmitted with media", set("mediaOmitted", true)], + [ + "a manifest that lists itself", + (m) => { + (m.files as unknown[]).unshift({ path: "manifest.json", sha256: "0".repeat(64), bytes: 1 }); + }, + ], + ]; + it.each(cases)("refuses %s", async (_, edit) => { + await expectArchiveError(withManifest(edit), "invalid_manifest", /^manifest\.json: /); + }); + + it("echoes an unknown key only when it is short, printable ASCII", async () => { + const places: [string, (key: string) => Edit][] = [ + ["the manifest", (key) => set(key, 1)], + ["contracts", (key) => set("contracts", { schemaVersion: 1, [key]: 1 })], + ["counts", (key) => setCount(key, 1)], + ["files[0]", (key) => setFile(0, key, 1)], + ]; + const hidden = ["\u001b[2J", "\u009b2J\u009b31m", "a\u202eb", "\u0085", "two words"]; + const long = "k".repeat(2_000_000); + for (const [where, edit] of places) { + const at = `manifest.json: ${where} has an unknown key `; + const shown = await expectArchiveError(withManifest(edit("extra")), "invalid_manifest"); + expect(shown.message).toBe(`${at}"extra"`); + const cut = await expectArchiveError(withManifest(edit(long)), "invalid_manifest"); + expect(cut.message).toBe(`${at}"${"k".repeat(120)}..."`); + for (const key of hidden) { + const error = await expectArchiveError(withManifest(edit(key)), "invalid_manifest"); + expect(error.message).toBe( + `${at}(not shown: it has spaces, controls or non-ASCII characters)`, + ); + } + } + }); + + it("accepts mediaOmitted without media", async () => { + const noMedia = files.slice(0, 4); + const back = await readAll( + archiveOf(noMedia, { + manifest: (m) => { + m.mediaOmitted = true; + }, + }), + ); + expect(back.manifest.mediaOmitted).toBe(true); + expect(back.files.map((f) => f.path)).toEqual(noMedia.map((f) => f.path)); + }); + + it("accepts an empty media file", async () => { + const empty: RawEntry = { path: `media/${sha256(new Uint8Array(0))}`, body: new Uint8Array(0) }; + const media = pattern(10, 1); + const entries = [ + ...files.slice(0, 4), + ...[empty, { path: `media/${sha256(media)}`, body: media }].sort((a, b) => + a.path < b.path ? -1 : 1, + ), + ]; + expect((await readAll(archiveOf(entries))).files).toHaveLength(6); + }); +}); + +describe("versions (invariant 4)", () => { + const notArchive: [string, Edit][] = [ + ["no format", drop("format")], + ["another format", set("format", "zip")], + ["the bundle format as the archive format", set("format", "cloudhal.site-bundle@1")], + ["a format version 0", set("format", "cloudhal.site-archive@0")], + ]; + it.each(notArchive)("refuses %s as not an archive", async (_, edit) => { + await expectArchiveError(withManifest(edit), "not_archive", /not a Cloudhal site archive/); + }); + + const newer: [string, Edit][] = [ + [ + "archive format 2 of any shape", + (m) => { + for (const key of Object.keys(m)) if (key !== "createdBy") delete m[key]; + m.format = "cloudhal.site-archive@2"; + m.parts = ["whatever @2 holds"]; + }, + ], + ["contracts schemaVersion 2", set("contracts", { schemaVersion: 2 })], + ["bundle format 2", set("bundle", "cloudhal.site-bundle@2")], + ]; + it.each(newer)("refuses %s, naming the CLI version", async (_, edit) => { + await expectArchiveError( + withManifest((m) => { + m.createdBy = "9.9.9"; + edit(m); + }), + "unsupported_version" as ArchiveErrorCode, + /newer Cloudhal .*; update @cloudhal\/cli to 9\.9\.9 or later$/, + ); + }); + + it("does not echo a createdBy that is not a version", async () => { + const error = await expectArchiveError( + withManifest((m) => { + m.format = "cloudhal.site-archive@3"; + m.createdBy = "1.0\u001b[2J"; + }), + "unsupported_version", + /to a newer version$/, + ); + expect(error.message).not.toContain("\u001b"); + }); +}); diff --git a/packages/cli/test/archive-parts.test.ts b/packages/cli/test/archive-parts.test.ts new file mode 100644 index 00000000..15c7369f --- /dev/null +++ b/packages/cli/test/archive-parts.test.ts @@ -0,0 +1,89 @@ +/** RFC-0039 §1: NDJSON parts of at most 8 MB, one row per line. */ +import { describe, expect, it } from "vitest"; +import { encodeNdjson, PART_SPLIT_BYTES, splitNdjson } from "../src/archive/index.ts"; +import { countRows } from "../src/archive/parts.ts"; +import { expectArchiveError } from "./archive-fixture.ts"; + +const text = (part: Uint8Array) => Buffer.from(part).toString("utf8"); +/** A row whose NDJSON line is exactly `bytes` long: `"xxx…"` plus the newline. */ +const rowOf = (bytes: number) => "x".repeat(bytes - 3); + +describe("splitNdjson", () => { + it("starts a new part when the next line would cross the limit", () => { + // Lines of 32 bytes: two fill a 64-byte part exactly, a third starts the next one. + const two = splitNdjson([rowOf(32), rowOf(32)], { partBytes: 64 }); + expect(two.map((p) => p.length)).toEqual([64]); + const three = splitNdjson([rowOf(32), rowOf(32), rowOf(1 + 3)], { partBytes: 64 }); + expect(three.map((p) => p.length)).toEqual([64, 4]); + const over = splitNdjson([rowOf(32), rowOf(33)], { partBytes: 64 }); + expect(over.map((p) => p.length)).toEqual([32, 33]); + }); + + it("puts a line larger than a part alone in its own part", () => { + const parts = splitNdjson([rowOf(10), rowOf(100), rowOf(10), rowOf(10)], { partBytes: 64 }); + expect(parts.map((p) => p.length)).toEqual([10, 100, 20]); + expect(splitNdjson([rowOf(100), rowOf(10)], { partBytes: 64 }).map((p) => p.length)).toEqual([ + 100, 10, + ]); + }); + + it("refuses a line above the reader's part cap, naming the row", async () => { + await expectArchiveError( + Promise.resolve().then(() => + splitNdjson([rowOf(10), rowOf(200)], { partBytes: 64, maxLineBytes: 199 }), + ), + "too_large", + /row 1 is 200 bytes/, + ); + }); + + it("keeps row order, counts bytes not characters, and ends every part with a newline", () => { + const rows = [{ a: "ü".repeat(20) }, { b: "
 and \ud800" }, ["x"], 7, null, "é"]; + const parts = splitNdjson(rows, { partBytes: 48 }); + expect(parts.length).toBeGreaterThan(1); + for (const part of parts) { + expect(part.length).toBeGreaterThan(0); + expect(part[part.length - 1]).toBe(0x0a); + } + const lines = rows.map((r) => `${JSON.stringify(r)}\n`); + expect(parts.map(text).join("")).toBe(lines.join("")); + // ü is two bytes: the first line is 49 bytes, above 48, and so alone. + expect(parts[0]?.length).toBe(Buffer.byteLength(lines[0] as string)); + expect(parts.reduce((n, p) => n + countRows(p, "p"), 0)).toBe(rows.length); + expect(splitNdjson([])).toEqual([]); + expect(() => splitNdjson([undefined])).toThrow(/row 0 has no JSON form/); + }); + + it("splits at the real 8 MiB threshold", () => { + const half = PART_SPLIT_BYTES / 2; + expect(splitNdjson([rowOf(half), rowOf(half)]).map((p) => p.length)).toEqual([ + PART_SPLIT_BYTES, + ]); + expect(splitNdjson([rowOf(half), rowOf(half + 1)]).map((p) => p.length)).toEqual([ + half, + half + 1, + ]); + }); +}); + +describe("encodeNdjson and countRows", () => { + it("writes one file and refuses one above the cap", async () => { + expect(text(encodeNdjson([{ a: 1 }, { b: 2 }]))).toBe('{"a":1}\n{"b":2}\n'); + expect(encodeNdjson([]).length).toBe(0); + await expectArchiveError( + Promise.resolve().then(() => encodeNdjson([rowOf(10), rowOf(10)], 19)), + "too_large", + ); + }); + + it("counts newlines and refuses what JSON.stringify never writes", async () => { + const enc = (s: string) => new TextEncoder().encode(s); + expect(countRows(enc("{}\n[]\n"), "nodes/0001.ndjson")).toBe(2); + for (const bad of ["", "{}", "{}\n\n", "\n{}\n"]) + await expectArchiveError( + Promise.resolve().then(() => countRows(enc(bad), "nodes/0001.ndjson")), + "damaged", + /^nodes\/0001\.ndjson: /, + ); + }); +}); diff --git a/packages/cli/test/archive-roundtrip.test.ts b/packages/cli/test/archive-roundtrip.test.ts new file mode 100644 index 00000000..6dfa7c26 --- /dev/null +++ b/packages/cli/test/archive-roundtrip.test.ts @@ -0,0 +1,310 @@ +/** + * RFC-0039 §1 and invariants 3, 5 and 7: what the writer writes, the reader reads back byte for + * byte, the same content gives the same bytes, and content the reader would refuse is never + * written. + */ +import { createReadStream } from "node:fs"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { gunzipSync } from "node:zlib"; +import { CONTRACT_SCHEMA_VERSION } from "@cloudhal/core-contracts"; +import { SITE_BUNDLE_FORMAT } from "@cloudhal/core-ts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + type ArchiveContent, + ArchiveError, + archiveStream, + readArchive, + SITE_ARCHIVE_FORMAT, + splitNdjson, + writeArchiveFile, +} from "../src/archive/index.ts"; +import { encodeManifest } from "../src/archive/manifest.ts"; +import { + collect, + expectArchiveError, + MEDIA, + memoryMedia, + NODE_ROWS, + readAll, + SITE, + same, + sampleContent, + sha256, + utf8, + writeBytes, +} from "./archive-fixture.ts"; + +let dir: string; +beforeAll(async () => { + dir = await mkdtemp(path.join(os.tmpdir(), "archive-roundtrip-")); +}); +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +/** The fixed order of RFC-0039 §1 for the sample content. */ +const ORDER = [ + "project.json", + ...sampleContent().nodes.map((_, i) => `nodes/${String(i + 1).padStart(4, "0")}.ndjson`), + "snapshots/0001.ndjson", + "blobs/0001.ndjson", + ...MEDIA.map((m) => `media/${sha256(m)}`).sort(), + "ledger.ndjson", +]; + +describe("round trip", () => { + it("reads back the manifest first and every file in the fixed order", async () => { + const content = sampleContent(); + const { manifest, bytes } = archiveStream(content); + const archive = await collect(bytes); + expect(ORDER.filter((p) => p.startsWith("nodes/")).length).toBeGreaterThan(1); + for (const chunk of [1, 333, 64 * 1024]) { + const back = await readAll(archive, { chunk }); + expect(back.manifest).toEqual(manifest); + expect(back.files.map((f) => f.path)).toEqual(ORDER); + const byPath = new Map(back.files.map((f) => [f.path, f.bytes])); + expect(same(byPath.get("project.json"), content.project as Uint8Array)).toBe(true); + expect(same(byPath.get("ledger.ndjson"), content.ledger as Uint8Array)).toBe(true); + const nodes = Buffer.concat( + ORDER.filter((p) => p.startsWith("nodes/")).map((p) => byPath.get(p) as Buffer), + ); + expect(same(nodes, Buffer.concat(content.nodes))).toBe(true); + for (const m of MEDIA) expect(same(byPath.get(`media/${sha256(m)}`), m)).toBe(true); + } + }); + + it("writes the RFC manifest with counts it computes", async () => { + const { manifest } = archiveStream(sampleContent()); + expect(Object.keys(manifest)).toEqual([ + "format", + "contracts", + "bundle", + "siteId", + "siteName", + "createdBy", + "createdAt", + "counts", + "files", + ]); + expect(manifest).toMatchObject({ + format: SITE_ARCHIVE_FORMAT, + contracts: { schemaVersion: CONTRACT_SCHEMA_VERSION }, + bundle: SITE_BUNDLE_FORMAT, + ...SITE, + counts: { + nodes: NODE_ROWS.length, + snapshots: 2, + blobs: 1, + media: MEDIA.length, + mediaBytes: MEDIA.reduce((n, m) => n + m.length, 0), + }, + }); + const text = Buffer.from(encodeManifest(manifest)).toString("utf8"); + expect(text.startsWith('{\n "format": "cloudhal.site-archive@1",\n')).toBe(true); + expect(text.endsWith("}\n")).toBe(true); + }); + + it("writes mediaOmitted archives without media, and minimal ones", async () => { + const omitted = sampleContent({ media: undefined, mediaOmitted: true }); + const back = await readAll(await writeBytes(omitted)); + expect(back.manifest.mediaOmitted).toBe(true); + expect(back.manifest.counts).toMatchObject({ media: 0, mediaBytes: 0 }); + expect(back.files.some((f) => f.path.startsWith("media/"))).toBe(false); + const minimal: ArchiveContent = { + ...SITE, + nodes: splitNdjson([{ id: "n" }]), + snapshots: [], + blobs: [], + }; + const small = await readAll(await writeBytes(minimal)); + expect(small.files.map((f) => f.path)).toEqual(["nodes/0001.ndjson"]); + expect("mediaOmitted" in small.manifest).toBe(false); + }); +}); + +describe("deterministic bytes (invariant 7)", () => { + it("gives identical bytes for the same content, whatever the media order or chunking", async () => { + const first = await writeBytes(sampleContent()); + expect(same(await writeBytes(sampleContent()), first)).toBe(true); + const reordered = sampleContent({ media: [...MEDIA].reverse().map((m) => memoryMedia(m, 1)) }); + expect(same(await writeBytes(reordered), first)).toBe(true); + // Rows are written in the caller's order: sorting them is the caller's job. + const nodes = splitNdjson([...NODE_ROWS].reverse(), { partBytes: 512 }); + expect(same(await writeBytes(sampleContent({ nodes })), first)).toBe(false); + }); + + it("writes a fixed gzip header and a platform-independent tar stream", async () => { + const archive = await writeBytes(sampleContent()); + expect(Buffer.from(archive.subarray(0, 10)).toString("hex")).toBe("1f8b08000000000000ff"); + const tar = gunzipSync(archive); + expect(tar.length % 10240).toBe(0); + // A golden hash of the uncompressed tar (gzip bytes differ between zlib builds, the tar does + // not). Update it on purpose, only when the format changes. + expect(sha256(tar)).toBe("81d8e9868fa915e24229bc3c4fc1a1acdfbfc41fd862fc335aa583b85e16a716"); + }); +}); + +describe("writer refusals", () => { + const bad: [string, Partial, string][] = [ + [ + "a media hash that is not hex", + { media: [{ ...memoryMedia(MEDIA[0] as Uint8Array), sha256: "A".repeat(64) }] }, + "invalid_input", + ], + [ + "duplicate media", + { media: [memoryMedia(MEDIA[0] as Uint8Array), memoryMedia(MEDIA[0] as Uint8Array)] }, + "invalid_input", + ], + ["media with mediaOmitted", { mediaOmitted: true }, "invalid_input"], + ["an empty part", { nodes: [new Uint8Array(0)] }, "invalid_input"], + ["a part without a final newline", { nodes: [utf8('{"id":"n"}')] }, "invalid_input"], + ["an empty project", { project: new Uint8Array(0) }, "invalid_input"], + ["10000 parts", { blobs: Array.from({ length: 10_000 }, () => utf8("{}\n")) }, "too_large"], + ["a createdAt that is not RFC 3339", { createdAt: "yesterday" }, "invalid_input"], + ["a createdBy that is not a version", { createdBy: "1.0 \u001b[2J" }, "invalid_input"], + ["a siteId that is not an id", { siteId: "../site" }, "invalid_input"], + ]; + + it.each(bad)("refuses %s before writing anything", async (_, overrides, code) => { + const content = sampleContent(overrides); + expect(() => archiveStream(content)).toThrow(ArchiveError); + const target = path.join(dir, "refused.cloudhal"); + await expectArchiveError(writeArchiveFile(target, content), code as "invalid_input"); + expect(await readdir(dir)).toEqual([]); + }); + + it("refuses what a reader with the same limits would refuse", async () => { + const content = sampleContent(); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { manifestBytes: 500 } })), + "too_large", + ); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { partBytes: 100 } })), + "too_large", + ); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { mediaBytes: 1000 } })), + "too_large", + ); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { entries: 5 } })), + "too_large", + ); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { totalBytes: 50_000 } })), + "too_large", + ); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { entries: 0 } })), + "invalid_input", + ); + }); +}); + +describe("media verified while writing", () => { + const target = () => path.join(dir, "site.cloudhal"); + const withMedia = (open: () => AsyncIterable) => { + const m = MEDIA[0] as Uint8Array; + return sampleContent({ media: [{ sha256: sha256(m), bytes: m.length, open }] }); + }; + + it("aborts a longer body without reading it all", async () => { + let pulled = 0; + let closed = false; + async function* endless() { + try { + for (;;) { + pulled++; + yield new Uint8Array(1024); + } + } finally { + closed = true; + } + } + await expectArchiveError(writeArchiveFile(target(), withMedia(endless)), "damaged", /longer/); + expect(pulled).toBeLessThan(10); + expect(closed).toBe(true); + expect(await readdir(dir)).toEqual([]); + }); + + it("refuses a short or wrong body, leaving an existing file as it was", async () => { + await writeFile(target(), "old"); + const m = MEDIA[0] as Uint8Array; + const wrong = Buffer.from(m); + wrong[0] = (wrong[0] as number) ^ 1; + for (const [body, text] of [ + [m.subarray(1), /has 2999 of 3000 bytes/], + [wrong, /SHA-256/], + ] as const) { + const error = await expectArchiveError( + writeArchiveFile( + target(), + withMedia(() => yieldOnce(body)), + ), + "damaged", + text, + ); + expect(error.path).toBe(`media/${sha256(m)}`); + expect(await readdir(dir)).toEqual(["site.cloudhal"]); + expect(await readFile(target(), "utf8")).toBe("old"); + } + await rm(target()); + }); + + it("ends a failed stream before the gzip trailer, so its bytes never read as an archive", async () => { + /** What the stream emitted before a media source one byte short failed it. */ + const partial = async (media: Uint8Array) => { + const short = { + sha256: sha256(media), + bytes: media.length, + open: () => yieldOnce(media.subarray(1)), + }; + const { bytes } = archiveStream(sampleContent({ media: [short] })); + const chunks: Uint8Array[] = []; + await expectArchiveError( + (async () => { + for await (const chunk of bytes) chunks.push(chunk); + })(), + "damaged", + ); + return Buffer.concat(chunks); + }; + // zlib had emitted nothing yet: an empty file. + const empty = await partial(MEDIA[0] as Uint8Array); + expect(empty.length).toBe(0); + await expectArchiveError(readAll(empty), "not_archive", /it is empty/); + // zlib had emitted part of the stream: no trailer, so it is truncated. + const cut = await partial(noise(256 * 1024)); + expect(cut.length).toBeGreaterThan(0); + await expectArchiveError(readAll(cut), "truncated"); + }); + + it("writes a file that reads back from disk", async () => { + const manifest = await writeArchiveFile(target(), sampleContent()); + let read: unknown; + for await (const item of readArchive(createReadStream(target()))) + if (item.type === "manifest") read = item.manifest; + expect(read).toEqual(manifest); + expect(await readdir(dir)).toEqual(["site.cloudhal"]); + await rm(target()); + }); +}); + +/** Bytes that do not compress (a fixed LCG), so gzip emits output as it goes. */ +function noise(length: number): Uint8Array { + const out = new Uint8Array(length); + let x = 1; + for (let i = 0; i < length; i++) { + x = (Math.imul(x, 1103515245) + 12345) >>> 0; + out[i] = x >>> 24; + } + return out; +} + +async function* yieldOnce(bytes: Uint8Array): AsyncGenerator { + yield bytes; +} diff --git a/packages/cli/test/archive-tar.test.ts b/packages/cli/test/archive-tar.test.ts new file mode 100644 index 00000000..d8f9dd0f --- /dev/null +++ b/packages/cli/test/archive-tar.test.ts @@ -0,0 +1,160 @@ +/** RFC-0039 §1: the ustar blocks Cloudhal writes, and the streaming reader's refusals. */ +import { describe, expect, it } from "vitest"; +import { parseTarHeader, TAR_RECORD, tarEnd, tarHeader } from "../src/archive/tar.ts"; +import { readTar } from "../src/archive/tar-read.ts"; +import { + chunked, + collect, + expectArchiveError, + pattern, + type RawEntry, + rawTar, + same, + USTAR, +} from "./archive-fixture.ts"; + +const ascii = (bytes: Uint8Array, start: number, length: number) => + Buffer.from(bytes.subarray(start, start + length)).toString("latin1"); + +async function entriesOf(tar: Uint8Array, chunk = 512, maxEntries = 100) { + const out: { path: string; body: Buffer }[] = []; + for await (const entry of readTar(chunked(tar, chunk), { maxEntries })) + out.push({ path: entry.path, body: await collect(entry.body) }); + return out; +} + +const drain = (tar: Uint8Array, maxEntries = 100) => entriesOf(tar, 512, maxEntries); + +describe("tar header bytes", () => { + it("writes fixed fields, an octal size and an unsigned checksum", () => { + const h = tarHeader("nodes/0001.ndjson", 1234); + expect(h.length).toBe(512); + expect(ascii(h, 0, 100)).toBe(`nodes/0001.ndjson${"\0".repeat(83)}`); + expect(ascii(h, 100, 8)).toBe("0000644\0"); + expect(ascii(h, 108, 8)).toBe("0000000\0"); + expect(ascii(h, 116, 8)).toBe("0000000\0"); + expect(ascii(h, 124, 12)).toBe(`${(1234).toString(8).padStart(11, "0")}\0`); + expect(ascii(h, 136, 12)).toBe("00000000000\0"); + expect(ascii(h, 156, 1)).toBe("0"); + expect(ascii(h, 157, 100)).toBe("\0".repeat(100)); + expect(ascii(h, 257, 8)).toBe("ustar\u000000"); + expect(ascii(h, 265, 64)).toBe("\0".repeat(64)); // uname, gname + expect(ascii(h, 345, 155)).toBe("\0".repeat(155)); // prefix + let sum = 0; + for (let i = 0; i < 512; i++) sum += i >= 148 && i < 156 ? 0x20 : (h[i] as number); + expect(ascii(h, 148, 8)).toBe(`${sum.toString(8).padStart(6, "0")}\0 `); + expect(parseTarHeader(h)).toEqual({ path: "nodes/0001.ndjson", size: 1234 }); + }); + + it("ends with two zero blocks padded to a 10240-byte record", () => { + for (const written of [0, 512, 9216, 9728, 10240, 20480 + 512]) { + const end = tarEnd(written); + expect((written + end.length) % TAR_RECORD).toBe(0); + expect(end.length).toBeGreaterThanOrEqual(1024); + expect(end.every((b) => b === 0)).toBe(true); + } + }); + + it("refuses names it cannot write as a plain ustar name", () => { + expect(() => tarHeader("x".repeat(100), 1)).toThrow(/1-99 printable/); + expect(() => tarHeader("a b", 1)).toThrow(/printable/); + expect(() => tarHeader("", 1)).toThrow(/printable/); + }); +}); + +describe("readTar", () => { + const sizes = [0, 1, 511, 512, 513, 4096 + 17]; + const entries: RawEntry[] = sizes.map((size, i) => ({ + path: `f${i}`, + body: pattern(size, i), + })); + + it("reads bodies of every size in any chunking", async () => { + const tar = rawTar(entries, { tail: new Uint8Array(4096) }); + for (const chunk of [1, 7, 512, 64 * 1024]) { + const got = await entriesOf(tar, chunk); + expect(got.map((e) => e.path)).toEqual(entries.map((e) => e.path)); + for (const [i, e] of got.entries()) + expect(same(e.body, entries[i]?.body as Uint8Array)).toBe(true); + } + }); + + it("skips bodies that are not read and refuses a stale body", async () => { + const tar = rawTar(entries); + const paths: string[] = []; + let stale: AsyncIterable | undefined; + for await (const entry of readTar(chunked(tar, 100), { maxEntries: 100 })) { + paths.push(entry.path); + if (entry.path === "f3") stale = entry.body; + } + expect(paths).toEqual(entries.map((e) => e.path)); + await expectArchiveError(collect(stale as AsyncIterable), "invalid_input", /after/); + }); + + it("refuses links, devices, directories, FIFOs and extension headers", async () => { + for (const flag of ["1", "2", "3", "4", "5", "6", "7", "x", "g", "L", "K", "A"]) { + const tar = rawTar([ + { + path: "f", + body: "x", + patch: (h) => { + h[USTAR.typeflag] = flag.charCodeAt(0); + }, + }, + ]); + await expectArchiveError(drain(tar), "unsafe_entry", /regular files only/); + } + }); + + it("refuses prefixes, link names, other magic, base-256 sizes and odd names", async () => { + const patches: ((h: Uint8Array) => void)[] = [ + (h) => h.fill(0x61, USTAR.prefix, USTAR.prefix + 1), + (h) => h.fill(0x61, USTAR.linkname, USTAR.linkname + 1), + (h) => h.set(Buffer.from("ustar \0"), USTAR.magic), // GNU tar's old format + (h) => h.set(Buffer.from("01"), USTAR.version), + (h) => h.fill(0x80, USTAR.size, USTAR.size + 1), + (h) => h.set(Buffer.from("12x"), USTAR.size), + (h) => h.set(Buffer.from("f\0junk"), USTAR.name), + (h) => h.set(Buffer.from("f\u001b[31m"), USTAR.name), + (h) => h.fill(0x61, USTAR.name, USTAR.name + 100), + (h) => h.fill(0, USTAR.name, USTAR.name + 100), + ]; + for (const patch of patches) { + const error = await expectArchiveError( + drain(rawTar([{ path: "f", body: "x", patch }])), + "unsafe_entry", + ); + expect(error.message).not.toContain("\u001b"); + } + }); + + it("refuses a bad checksum as damage", async () => { + const tar = rawTar([{ path: "nodes/0001.ndjson", body: "x" }]); + tar[3] = (tar[3] as number) ^ 1; // a flipped name bit, checksum unchanged + await expectArchiveError(drain(tar), "damaged", /checksum/); + }); + + it("checks the end of the archive", async () => { + const one: RawEntry[] = [{ path: "f", body: "abc" }]; + const body = rawTar(one, { end: false }); + const zero = new Uint8Array(512); + const cases: [Uint8Array, string][] = [ + [Buffer.concat([body, zero]), "truncated"], + [Buffer.concat([body, zero, rawTar(one, { end: false })]), "damaged"], + [body, "truncated"], + [body.subarray(0, 700), "truncated"], + [body.subarray(0, 300), "truncated"], + [rawTar(one, { tail: Buffer.from([0, 0, 1]) }), "damaged"], + [rawTar(one, { tail: new Uint8Array(TAR_RECORD + 1) }), "damaged"], + [new Uint8Array(0), "not_archive"], + ]; + for (const [tar, code] of cases) + await expectArchiveError(drain(tar), code as "truncated" | "damaged" | "not_archive"); + expect(await drain(rawTar(one, { tail: new Uint8Array(TAR_RECORD) }))).toHaveLength(1); + }); + + it("caps the number of entries", async () => { + await expectArchiveError(drain(rawTar(entries), 3), "too_large", /more than 3 entries/); + expect(await drain(rawTar(entries), entries.length)).toHaveLength(entries.length); + }); +});