diff --git a/docs/adr/0041-site-archives.md b/docs/adr/0041-site-archives.md index 7a881381..8b22c2e5 100644 --- a/docs/adr/0041-site-archives.md +++ b/docs/adr/0041-site-archives.md @@ -1,6 +1,7 @@ # ADR-0041: Site archives: a streaming `.cloudhal` tar of NDJSON parts and media, ids kept, graph only - Status: Accepted - Date: 2026-10-22 (decided by Halil) +- Amendment (2026-10-25, RFC-0039 rollout step 1): deterministic bytes hold for the same Node/zlib build; the tar stream and the manifest's SHA-256s on every platform. See rule 6. - Related: RFC-0039, RFC-0004, RFC-0010, ADR-0002, #47 - Sources: [Sanity datasets](https://www.sanity.io/docs/cli-reference/dataset) · [Strapi data management](https://docs.strapi.io/cms/data-management/export) · [Contentful CLI export](https://github.com/contentful/contentful-cli/tree/main/docs/space/export) · [Ghost migration](https://docs.ghost.org/migration/ghost) · [WordPress export](https://wordpress.org/documentation/article/tools-export-screen/) @@ -18,7 +19,7 @@ A site's content can only leave a project through a deployment (`push`, `pull`), Never erase-then-import. 5. **Graph only.** No users, sessions, roles, agent keys, comments, readers, analytics, audit log, automations, secrets or webhook credentials. That is why there is no built-in encryption; people can pipe the file through `age` or `gpg`. -6. **Deterministic bytes:** a fixed order, zero timestamps and fixed modes, so an unchanged site exports identically. +6. **Deterministic bytes:** a fixed order, zero timestamps and fixed modes, so an unchanged site exports identically (amended: byte for byte with the same Node/zlib build, since gzip output differs between zlib builds; the tar stream inside, and every SHA-256 in the manifest, on every platform). 7. **Untrusted input:** the layout is a closed list of paths (no traversal, links or devices); sizes are capped; rows go through the precompiled validators and gates, as for `push`. 8. **Later, already decided:** `init --from`, `--new-ids` for templates, partial export, `--from `, Studio download, and signed archives. diff --git a/docs/rfcs/0039-site-archive-export-import.md b/docs/rfcs/0039-site-archive-export-import.md index c14a2a87..8778eefb 100644 --- a/docs/rfcs/0039-site-archive-export-import.md +++ b/docs/rfcs/0039-site-archive-export-import.md @@ -1,6 +1,7 @@ # RFC-0039: Site archives: `cloudhal export` and `cloudhal import` - Status: Accepted (2026-10-22, by Halil; ADR-0041). Open questions decided as recommended. Revised before acceptance after a comparison with Sanity, Strapi, Contentful, Ghost, WordPress, Directus and Webflow (see Prior art): NDJSON parts instead of one JSON file, per-file checksums, contract versions in the manifest, and a snapshot before `--replace`. +- Amendment (2026-10-25, rollout step 1): identical `.cloudhal` bytes need the same Node/zlib build; the tar stream inside, and so every SHA-256 in the manifest, is identical on every platform. See invariant 7. - Author: Halil (Madraka), drafted with Claude · Date: 2026-10-21 - Related: - #47 (found while building cloudhal.com: no way to keep or move a site's content without a deployment, so it was written as code). @@ -88,7 +89,7 @@ cloudhal import [--replace] [--dry-run] 4. An archive written by a newer format or a newer contracts `schemaVersion` is refused with the CLI version needed. Older ones are read, and migrated forward if a later schema version requires it. 5. Taken-down content stays taken down: the `purged` ledger travels and the import never restores those bytes. 6. Import writes the graph in atomic batches (no interactive transactions, AGENTS.md rule 4), guarded like `importSite` today. A failure leaves the project as it was, and `--replace` can always be undone through its `before import` snapshot. -7. Same content, same file: rows and files are written in a fixed order, with zero timestamps and fixed modes in the tar headers, so exporting an unchanged site twice gives identical bytes (good for git and for diffing). +7. Same content, same file: rows and files are written in a fixed order, with zero timestamps and fixed modes in the tar headers, so exporting an unchanged site twice gives identical bytes (good for git and for diffing). *Amended 2026-10-25:* identical bytes hold for the same Node/zlib build. gzip output differs between zlib builds (Homebrew's zlib and the Chromium zlib that official Node binaries ship compress the same input differently), and only a dependency or uncompressed output would avoid that. The tar stream inside the gzip, and so every SHA-256 in the manifest, is identical on every platform. A committed archive (step 5's demo) is regenerated on one pinned Node in CI, or compared through its decompressed tar. ### 4. What is not in an archive @@ -125,7 +126,7 @@ The manifest says so, and `export` prints it. People and personal data never lea 2. `export`. 3. `import` with `--dry-run`. 4. `--replace`, the `before import` snapshot, and the site id switch. - 5. Docs and a starter demo archive. + 5. Docs and a starter demo archive (regenerated on one pinned Node, or compared through its decompressed tar: invariant 7). ## Security & policy impact diff --git a/packages/cli/src/archive/byte-source.ts b/packages/cli/src/archive/byte-source.ts new file mode 100644 index 00000000..715f4867 --- /dev/null +++ b/packages/cli/src/archive/byte-source.ts @@ -0,0 +1,97 @@ +/** + * Exact byte counts from an async iterable of chunks of any size: gunzip emits 16 KiB chunks that + * do not line up with tar's 512-byte blocks. Internal to the archive reader. + */ +import { ArchiveError } from "./errors.ts"; + +const EMPTY = new Uint8Array(0); + +function truncated(): ArchiveError { + return new ArchiveError("truncated", "the archive ends in the middle of an entry (truncated)"); +} + +export class ByteSource { + readonly #iterator: AsyncIterator; + #chunk: Uint8Array = EMPTY; + #offset = 0; + #position = 0; + #done = false; + + constructor(source: AsyncIterable) { + this.#iterator = source[Symbol.asyncIterator](); + } + + /** Bytes consumed so far. */ + get position(): number { + return this.#position; + } + + /** Makes sure an unread byte is buffered; false at the end of the source. */ + async #fill(): Promise { + while (this.#offset >= this.#chunk.length) { + if (this.#done) return false; + const next = await this.#iterator.next(); + if (next.done) { + this.#done = true; + return false; + } + this.#chunk = next.value; + this.#offset = 0; + } + return true; + } + + #take(max: number): Uint8Array { + const n = Math.min(max, this.#chunk.length - this.#offset); + const view = this.#chunk.subarray(this.#offset, this.#offset + n); + this.#offset += n; + this.#position += n; + return view; + } + + /** + * Exactly `n` bytes in a fresh array; undefined when the source ended before the first of them, + * `truncated` when it ends part-way. + */ + async read(n: number): Promise { + const out = new Uint8Array(n); + let filled = 0; + while (filled < n) { + if (!(await this.#fill())) { + if (filled === 0) return undefined; + throw truncated(); + } + const view = this.#take(n - filled); + out.set(view, filled); + filled += view.length; + } + return out; + } + + /** + * Between 1 and `max` bytes, as a view of the source's chunk (Node streams never reuse a chunk + * they emitted); `truncated` at the end of the source. + */ + async readSome(max: number): Promise { + if (!(await this.#fill())) throw truncated(); + return this.#take(max); + } + + /** Discards exactly `n` bytes; `truncated` when the source has fewer. */ + async skip(n: number): Promise { + let left = n; + while (left > 0) left -= (await this.readSome(left)).length; + } + + /** True when every byte has been consumed and the source has ended. */ + async atEnd(): Promise { + return !(await this.#fill()); + } + + /** Stops the source (runs its `finally`); safe to call more than once. */ + async close(): Promise { + if (this.#done) return; + this.#done = true; + await this.#iterator.return?.(); + } +} diff --git a/packages/cli/src/archive/errors.ts b/packages/cli/src/archive/errors.ts new file mode 100644 index 00000000..b1a176e2 --- /dev/null +++ b/packages/cli/src/archive/errors.ts @@ -0,0 +1,60 @@ +/** + * The one error type of the site archive container (RFC-0039): a code that `cloudhal import` maps + * to its messages, the archive path when there is one, and a safe echo of untrusted names. + */ + +/** + * - `not_archive`: empty, not gzip, or no Cloudhal manifest as the first entry. + * - `unsupported_version`: written by a newer Cloudhal (archive, contracts or bundle format). + * - `invalid_manifest`: `manifest.json` is malformed or disagrees with itself. + * - `unsafe_entry`: a tar header or path Cloudhal never writes (links, devices, traversal). + * - `mismatch`: the entries disagree with the manifest (unlisted, duplicate, order, size, missing). + * - `damaged`: a checksum, SHA-256 or zlib check failed, or data follows the tar or gzip end. + * - `truncated`, `too_large`: the input ends early, or is over a cap. + * - `invalid_input`: the writer was given content it must not write, or a body was misused. + */ +export type ArchiveErrorCode = + | "not_archive" + | "unsupported_version" + | "invalid_manifest" + | "unsafe_entry" + | "mismatch" + | "damaged" + | "truncated" + | "too_large" + | "invalid_input"; + +/** Every failure of the archive writer and reader. */ +export class ArchiveError extends Error { + override readonly name = "ArchiveError"; + readonly code: ArchiveErrorCode; + /** The archive path the error is about, when there is one (a path of the closed layout). */ + readonly path: string | undefined; + constructor( + code: ArchiveErrorCode, + message: string, + options: { path?: string; cause?: unknown } = {}, + ) { + super(message, options.cause === undefined ? undefined : { cause: options.cause }); + this.code = code; + this.path = options.path; + } +} + +const MAX_ECHO = 120; + +/** + * An untrusted name (an entry name by default) as it may appear in a message: JSON-quoted and cut + * when every character is printable ASCII, otherwise only `described`, so a name cannot carry + * terminal escapes (C0 or C1 controls), bidi overrides or megabytes into a message. + */ +export function printable( + raw: string, + described = "an entry name with a non-printable byte", +): string { + for (let i = 0; i < raw.length; i++) { + const c = raw.charCodeAt(i); + if (c < 0x21 || c > 0x7e) return described; + } + return JSON.stringify(raw.length > MAX_ECHO ? `${raw.slice(0, MAX_ECHO)}...` : raw); +} diff --git a/packages/cli/src/archive/gzip.ts b/packages/cli/src/archive/gzip.ts new file mode 100644 index 00000000..bd9cee9b --- /dev/null +++ b/packages/cli/src/archive/gzip.ts @@ -0,0 +1,143 @@ +/** + * gzip for site archives. The writer compresses with fixed parameters and a fixed OS byte, so the + * same tar stream gives the same bytes on every platform that runs the same zlib build (gzip + * output is not identical across zlib builds; the tar stream inside it is). The reader accepts any + * gzip and nothing after its last member, counts the decompressed bytes against a cap, and maps + * zlib's errors to archive errors. + */ +import { pipeline } from "node:stream/promises"; +import { constants, createGunzip, createGzip, type ZlibOptions } from "node:zlib"; +import { ArchiveError } from "./errors.ts"; + +/** + * Level 6 (zlib's default) writes XFL 0. Level 9 costs about twice the CPU on NDJSON for about 3% + * less; level 0 is not used because stored blocks follow the input's chunking. + */ +export const GZIP_OPTIONS: Readonly = Object.freeze({ + level: 6, + memLevel: 8, + windowBits: 15, + strategy: constants.Z_DEFAULT_STRATEGY, +}); + +/** RFC 1952 OS "unknown": the same on every platform (zlib writes 3 on Unix, 10 on Windows). */ +export const GZIP_OS = 255; + +const GZIP_HEADER = 10; + +/** Checks zlib's header (no name, comment or header CRC: FLG 0) and sets the OS byte. */ +function patchHeader(head: Uint8Array): Uint8Array { + if (head[0] !== 0x1f || head[1] !== 0x8b || head[2] !== 8 || head[3] !== 0) + throw new ArchiveError("invalid_input", "zlib wrote an unexpected gzip header"); + head[9] = GZIP_OS; + return head; +} + +/** `source` gzip-compressed with GZIP_OPTIONS and the OS byte set to GZIP_OS. */ +export async function* gzipFixed( + source: AsyncIterable, +): AsyncGenerator { + const gzip = createGzip(GZIP_OPTIONS); + const piping = pipeline(source, gzip); + piping.catch(() => {}); // surfaced by the loop below (pipeline destroys gzip with the error) + let head = new Uint8Array(0); + let patched = false; + try { + for await (const chunk of gzip as AsyncIterable) { + if (patched) { + yield chunk; + continue; + } + head = Buffer.concat([head, chunk]); + if (head.length < GZIP_HEADER) continue; + patched = true; + yield patchHeader(head); + } + await piping; + if (!patched) throw new ArchiveError("invalid_input", "zlib wrote no gzip header"); + } finally { + gzip.destroy(); + } +} + +function zlibCode(error: unknown): string | undefined { + const code = (error as { code?: unknown } | null)?.code; + return typeof code === "string" && code.startsWith("Z_") ? code : undefined; +} + +/** What the source handed to gunzip: its first two bytes, its length, and whether it ended. */ +interface Feed { + head: number[]; + bytes: number; + ended: boolean; +} + +/** `source` unchanged, counted into `feed`. */ +async function* counted( + source: AsyncIterable, + feed: Feed, +): AsyncGenerator { + for await (const chunk of source) { + for (const b of chunk.subarray(0, 2 - feed.head.length)) feed.head.push(b); + feed.bytes += chunk.length; + yield chunk; + } + feed.ended = true; +} + +/** + * A zlib error as an archive error. Whether the input is gzip at all is judged from its first two + * bytes (the magic 1f 8b), not from how much of it was decompressed before the error. + */ +function mapZlibError(error: unknown, feed: Feed): unknown { + const code = zlibCode(error); + if (code === undefined) return error; + if (feed.bytes === 0) + return new ArchiveError("not_archive", "not a Cloudhal site archive (it is empty)", { + cause: error, + }); + if (feed.head[0] !== 0x1f || feed.head[1] !== 0x8b) + return new ArchiveError("not_archive", "not a Cloudhal site archive (not gzip)", { + cause: error, + }); + if (code === "Z_BUF_ERROR") + return new ArchiveError("truncated", "the archive is truncated (gzip ends early)", { + cause: error, + }); + return new ArchiveError("damaged", "the archive is damaged (gzip data is corrupt)", { + cause: error, + }); +} + +/** + * `source` decompressed, `too_large` beyond `maxBytes` decompressed bytes. It returns only after + * gunzip ended cleanly (its CRC and length check passed) and consumed the whole source: zlib stops + * quietly before input that starts with a zero byte, so anything left over is refused here as + * `damaged`. Other errors (of the source, such as a file read) pass through unchanged. + */ +export async function* gunzipCapped( + source: AsyncIterable, + maxBytes: number, +): AsyncGenerator { + const feed: Feed = { head: [], bytes: 0, ended: false }; + const gunzip = createGunzip(); + const piping = pipeline(counted(source, feed), gunzip); + piping.catch(() => {}); // surfaced by the loop below + let produced = 0; + try { + for await (const chunk of gunzip as AsyncIterable) { + produced += chunk.length; + if (produced > maxBytes) + throw new ArchiveError("too_large", `the archive is above ${maxBytes} bytes decompressed`); + yield chunk; + } + // Checked before `piping`, which fails (premature close, abort) when gunzip ended early. + if (!feed.ended || feed.bytes !== gunzip.bytesWritten) + throw new ArchiveError("damaged", "the archive has data after its gzip stream"); + await piping; + } catch (error) { + throw mapZlibError(error, feed); + } finally { + gunzip.destroy(); + } +} diff --git a/packages/cli/src/archive/index.ts b/packages/cli/src/archive/index.ts new file mode 100644 index 00000000..646e2970 --- /dev/null +++ b/packages/cli/src/archive/index.ts @@ -0,0 +1,17 @@ +/** + * Site archives (RFC-0039, ADR-0041): the `.cloudhal` container that `cloudhal export` writes and + * `cloudhal import` reads. Node-only; nothing a Worker runs imports it. + */ +export { ArchiveError, type ArchiveErrorCode } from "./errors.ts"; +export { type ArchiveSlot, classifyPath, type FileSlot, type PartKind } from "./layout.ts"; +export { ARCHIVE_LIMITS, type ArchiveLimits, PART_SPLIT_BYTES } from "./limits.ts"; +export { + type ArchiveCounts, + type ArchiveFile, + SITE_ARCHIVE_FORMAT, + type SiteArchiveManifest, +} from "./manifest.ts"; +export { encodeNdjson, type NdjsonSplitOptions, splitNdjson } from "./parts.ts"; +export type { ArchiveContent, ArchiveMediaSource } from "./plan.ts"; +export { type ArchiveItem, type ReadArchiveOptions, readArchive } from "./read.ts"; +export { archiveStream, type WriteArchiveOptions, writeArchiveFile } from "./write.ts"; diff --git a/packages/cli/src/archive/layout.ts b/packages/cli/src/archive/layout.ts new file mode 100644 index 00000000..ec307337 --- /dev/null +++ b/packages/cli/src/archive/layout.ts @@ -0,0 +1,81 @@ +/** + * The closed list of paths a site archive holds (RFC-0039 §1 and Security) and the one order the + * writer writes them in, which the manifest lists and the reader checks: + * + * `manifest.json`, `project.json`, `nodes/NNNN.ndjson`, `snapshots/NNNN.ndjson`, + * `blobs/NNNN.ndjson`, `media/` (ascending), `ledger.ndjson`. + * + * Any other path, including `..`, absolute paths, backslashes, `./` and part number `0000`, is + * not part of the layout. + */ +import { ArchiveError } from "./errors.ts"; +import { MAX_PART_INDEX } from "./limits.ts"; + +export type PartKind = "nodes" | "snapshots" | "blobs"; + +/** The part kinds in archive order. */ +export const PART_KINDS: readonly PartKind[] = ["nodes", "snapshots", "blobs"]; + +/** What a layout path names. */ +export type ArchiveSlot = + | { kind: "manifest" } + | { kind: "project" } + | { kind: "part"; part: PartKind; index: number } + | { kind: "media"; sha256: string } + | { kind: "ledger" }; + +/** A slot whose file the reader buffers and hands over whole. */ +export type FileSlot = Exclude; + +export const MANIFEST_PATH = "manifest.json"; +export const PROJECT_PATH = "project.json"; +export const LEDGER_PATH = "ledger.ndjson"; + +export const SHA256_HEX = /^[0-9a-f]{64}$/; + +const PART_PATH = /^(nodes|snapshots|blobs)\/([0-9]{4})\.ndjson$/; +const MEDIA_PATH = /^media\/([0-9a-f]{64})$/; + +/** The slot `path` names, or undefined when the path is not in the layout. */ +export function classifyPath(path: string): ArchiveSlot | undefined { + if (path === MANIFEST_PATH) return { kind: "manifest" }; + if (path === PROJECT_PATH) return { kind: "project" }; + if (path === LEDGER_PATH) return { kind: "ledger" }; + const part = PART_PATH.exec(path); + if (part) { + const index = Number(part[2]); + return index >= 1 ? { kind: "part", part: part[1] as PartKind, index } : undefined; + } + const media = MEDIA_PATH.exec(path); + return media ? { kind: "media", sha256: media[1] as string } : undefined; +} + +/** `nodes/0001.ndjson` for part 1 of the nodes. */ +export function partPath(part: PartKind, index: number): string { + if (!Number.isInteger(index) || index < 1 || index > MAX_PART_INDEX) + throw new ArchiveError( + "too_large", + `${part}: an archive holds at most ${MAX_PART_INDEX} parts of each kind`, + ); + return `${part}/${String(index).padStart(4, "0")}.ndjson`; +} + +export function mediaPath(sha256: string): string { + return `media/${sha256}`; +} + +const RANK = { manifest: 0, project: 1, nodes: 2, snapshots: 3, blobs: 4, media: 5, ledger: 6 }; + +function rank(slot: ArchiveSlot): number { + return slot.kind === "part" ? RANK[slot.part] : RANK[slot.kind]; +} + +/** Archive order; 0 only for the same slot. Media hashes compare by code unit, never by locale. */ +export function compareSlots(a: ArchiveSlot, b: ArchiveSlot): number { + const byRank = rank(a) - rank(b); + if (byRank !== 0) return byRank; + if (a.kind === "part" && b.kind === "part") return a.index - b.index; + if (a.kind === "media" && b.kind === "media") + return a.sha256 < b.sha256 ? -1 : a.sha256 > b.sha256 ? 1 : 0; + return 0; +} diff --git a/packages/cli/src/archive/limits.ts b/packages/cli/src/archive/limits.ts new file mode 100644 index 00000000..64dc83ec --- /dev/null +++ b/packages/cli/src/archive/limits.ts @@ -0,0 +1,52 @@ +/** + * Caps for reading untrusted archives (RFC-0039 Security) and the NDJSON part size of the writer. + * Tests lower the caps through `archiveLimits`; the writer refuses content its own reader would + * refuse under the same limits. + */ +import { ArchiveError } from "./errors.ts"; + +const MiB = 1024 * 1024; + +export interface ArchiveLimits { + /** `manifest.json`. */ + manifestBytes: number; + /** `project.json`, `ledger.ndjson` and any one `nodes|snapshots|blobs` part. */ + partBytes: number; + /** Any one `media/` file. */ + mediaBytes: number; + /** Decompressed bytes of the whole archive (gzip bomb backstop). */ + totalBytes: number; + /** Tar entries, the manifest included. */ + entries: number; +} + +export const ARCHIVE_LIMITS: Readonly = Object.freeze({ + // About 200 bytes per listed file: room for 80 000 media files. + manifestBytes: 16 * MiB, + partBytes: 16 * MiB, + // = MAX_MEDIA_PUT (packages/api/src/routes/admin/sync.ts) = MEDIA_LIMITS.upload + // (packages/core-ts/src/media/types.ts): no larger file can reach a site. + mediaBytes: 100 * MiB, + // Above the 10 GiB default site media quota (MEDIA_LIMITS.siteQuota). + totalBytes: 32 * 1024 * MiB, + entries: 100_000, +}); + +/** RFC-0039 §1: a new part every 8 MB. */ +export const PART_SPLIT_BYTES = 8 * MiB; + +/** Part numbers have 4 digits. */ +export const MAX_PART_INDEX = 9999; + +/** The default limits with `overrides` applied; each override must be a positive safe integer. */ +export function archiveLimits(overrides: Partial = {}): ArchiveLimits { + const limits: ArchiveLimits = { ...ARCHIVE_LIMITS }; + for (const [key, value] of Object.entries(overrides)) { + if (!(key in ARCHIVE_LIMITS)) + throw new ArchiveError("invalid_input", `unknown archive limit ${JSON.stringify(key)}`); + if (!Number.isSafeInteger(value) || (value as number) <= 0) + throw new ArchiveError("invalid_input", `archive limit ${key} must be a positive integer`); + limits[key as keyof ArchiveLimits] = value as number; + } + return limits; +} diff --git a/packages/cli/src/archive/manifest-files.ts b/packages/cli/src/archive/manifest-files.ts new file mode 100644 index 00000000..58296b13 --- /dev/null +++ b/packages/cli/src/archive/manifest-files.ts @@ -0,0 +1,121 @@ +/** + * The manifest's `files` list against the layout: every path in it, in archive order, once; part + * numbers contiguous from 0001; each size within its cap; media named by their own hash; and the + * media counts equal to the media listed. + */ +import { ArchiveError, printable } from "./errors.ts"; +import { + type ArchiveSlot, + classifyPath, + compareSlots, + MANIFEST_PATH, + type PartKind, + SHA256_HEX, +} from "./layout.ts"; +import type { ArchiveLimits } from "./limits.ts"; +import type { ArchiveCounts, ArchiveFile } from "./manifest.ts"; +import { tarLength } from "./tar.ts"; + +const FILE_KEYS = ["path", "sha256", "bytes"]; + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** An `invalid_manifest` error about `manifest.json`. */ +export function manifestError(message: string): ArchiveError { + return new ArchiveError("invalid_manifest", `manifest.json: ${message}`, { + path: MANIFEST_PATH, + }); +} + +/** What an unknown key that is not safe to print is replaced with. */ +const HIDDEN_KEY = "(not shown: it has spaces, controls or non-ASCII characters)"; + +/** Refuses a key of `value` that is not `allowed`, echoing it only when that is safe. */ +export function checkKeys( + value: Record, + allowed: readonly string[], + where: string, +): void { + for (const key of Object.keys(value)) + if (!allowed.includes(key)) + throw manifestError(`${where} has an unknown key ${printable(key, HIDDEN_KEY)}`); +} + +function tooLarge(message: string): ArchiveError { + return new ArchiveError("too_large", `manifest.json: ${message}`, { path: MANIFEST_PATH }); +} + +/** One entry of `files`: its shape, its path, its hash and its size cap. */ +function checkEntry( + raw: unknown, + at: string, + limits: ArchiveLimits, +): { file: ArchiveFile; slot: ArchiveSlot } { + if (!isRecord(raw)) throw manifestError(`${at} must be an object`); + checkKeys(raw, FILE_KEYS, at); + const { path, sha256, bytes } = raw; + if (typeof path !== "string") throw manifestError(`${at}.path must be a string`); + const slot = classifyPath(path); + if (slot === undefined || slot.kind === "manifest") + throw manifestError(`${at} names ${printable(path)}, which is not a path a site archive holds`); + if (typeof sha256 !== "string" || !SHA256_HEX.test(sha256)) + throw manifestError(`${at} (${path}) needs a lowercase hex SHA-256`); + if (slot.kind === "media" && sha256 !== slot.sha256) + throw manifestError(`${at} (${path}) is listed with a SHA-256 other than its name`); + if (!Number.isSafeInteger(bytes) || (bytes as number) < 0) + throw manifestError(`${at} (${path}) needs a non-negative integer size`); + const size = bytes as number; + if (slot.kind !== "media" && size === 0) throw manifestError(`${at} (${path}) is empty`); + const cap = slot.kind === "media" ? limits.mediaBytes : limits.partBytes; + if (size > cap) throw tooLarge(`${path} is ${size} bytes, above the ${cap}-byte cap`); + return { file: { path, sha256, bytes: size }, slot }; +} + +/** The checked `files` list of a manifest. */ +export function checkManifestFiles( + raw: unknown, + context: { + counts: ArchiveCounts; + mediaOmitted: boolean; + limits: ArchiveLimits; + /** The manifest's own size, for the decompressed total. */ + manifestBytes: number; + }, +): ArchiveFile[] { + const { counts, limits } = context; + if (!Array.isArray(raw)) throw manifestError("files must be an array"); + if (raw.length + 1 > limits.entries) + throw tooLarge(`${raw.length + 1} entries, above the cap of ${limits.entries}`); + const files: ArchiveFile[] = []; + const nextPart: Record = { nodes: 1, snapshots: 1, blobs: 1 }; + let previous: ArchiveSlot = { kind: "manifest" }; + let media = 0; + let mediaBytes = 0; + for (const [i, entry] of raw.entries()) { + const { file, slot } = checkEntry(entry, `files[${i}]`, limits); + if (compareSlots(previous, slot) >= 0) + throw manifestError(`files[${i}] (${file.path}) is listed twice or out of order`); + if (slot.kind === "part") { + if (slot.index !== nextPart[slot.part]) + throw manifestError(`files[${i}] (${file.path}) leaves a gap in the ${slot.part} parts`); + nextPart[slot.part] = slot.index + 1; + } + if (slot.kind === "media") { + media++; + mediaBytes += file.bytes; + } + previous = slot; + files.push(file); + } + const total = tarLength([context.manifestBytes, ...files.map((f) => f.bytes)]); + if (total > limits.totalBytes) + throw tooLarge( + `the archive is ${total} bytes decompressed, above the ${limits.totalBytes}-byte cap`, + ); + if (counts.media !== media || counts.mediaBytes !== mediaBytes) + throw manifestError(`counts.media and counts.mediaBytes must be ${media} and ${mediaBytes}`); + if (context.mediaOmitted && media > 0) throw manifestError("mediaOmitted, yet media are listed"); + return files; +} diff --git a/packages/cli/src/archive/manifest.ts b/packages/cli/src/archive/manifest.ts new file mode 100644 index 00000000..1bfd4048 --- /dev/null +++ b/packages/cli/src/archive/manifest.ts @@ -0,0 +1,216 @@ +/** + * `manifest.json`, the first entry of a site archive (RFC-0039 §1): its type, its canonical bytes + * and the hand-written check of untrusted ones. Not a contract schema (like `SiteBundle`), so no + * Ajv: the version gate runs first, so a newer archive is refused with the CLI version it needs + * before its shape is judged; then the shape is checked strictly, unknown keys included. + */ +import { CONTRACT_SCHEMA_VERSION } from "@cloudhal/core-contracts"; +import { SITE_BUNDLE_FORMAT } from "@cloudhal/core-ts"; +import { ArchiveError } from "./errors.ts"; +import type { ArchiveLimits } from "./limits.ts"; +import { checkKeys, checkManifestFiles, isRecord, manifestError } from "./manifest-files.ts"; + +export const SITE_ARCHIVE_FORMAT = "cloudhal.site-archive@1" as const; + +/** One file of the archive other than the manifest. */ +export interface ArchiveFile { + path: string; + sha256: string; + bytes: number; +} + +/** Rows of the NDJSON parts, and the media files with their total size. */ +export interface ArchiveCounts { + nodes: number; + snapshots: number; + blobs: number; + media: number; + mediaBytes: number; +} + +export interface SiteArchiveManifest { + format: typeof SITE_ARCHIVE_FORMAT; + /** The contracts' schemaVersion the rows were written with. */ + contracts: { schemaVersion: number }; + /** The bundle format the parts reassemble into. */ + bundle: typeof SITE_BUNDLE_FORMAT; + siteId: string; + siteName: string; + /** The `@cloudhal/cli` version that wrote it. */ + createdBy: string; + /** RFC 3339; from the content, never the wall clock (invariant 7). */ + createdAt: string; + counts: ArchiveCounts; + /** Every other file, in archive order, with its SHA-256 and size. */ + files: ArchiveFile[]; + /** Written without media: they must come from elsewhere. */ + mediaOmitted?: true; +} + +const TOP_KEYS = [ + "format", + "contracts", + "bundle", + "siteId", + "siteName", + "createdBy", + "createdAt", + "counts", + "files", + "mediaOmitted", +] as const; +const COUNT_KEYS = ["nodes", "snapshots", "blobs", "media", "mediaBytes"] as const; + +const FORMAT = /^cloudhal\.site-archive@([1-9][0-9]{0,5})$/; +const BUNDLE = /^cloudhal\.site-bundle@([1-9][0-9]{0,5})$/; +/** A version as `createdBy` may carry it, safe to print. */ +const VERSION_TOKEN = /^[0-9A-Za-z.+-]{1,64}$/; +/** common.schema.json `$defs/Id`. */ +const ID = /^[a-z]+_[A-Za-z0-9]{6,32}$/; +const RFC3339 = + /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{1,9})?(Z|[+-][0-9]{2}:[0-9]{2})$/; +/** Site.name (site.schema.json). */ +const MAX_SITE_NAME = 200; + +/** The manifest's bytes: RFC key order, two-space JSON and a final newline, so it reads and diffs. */ +export function encodeManifest(m: SiteArchiveManifest): Uint8Array { + const { nodes, snapshots, blobs, media, mediaBytes } = m.counts; + const canonical = { + format: m.format, + contracts: { schemaVersion: m.contracts.schemaVersion }, + bundle: m.bundle, + siteId: m.siteId, + siteName: m.siteName, + createdBy: m.createdBy, + createdAt: m.createdAt, + counts: { nodes, snapshots, blobs, media, mediaBytes }, + files: m.files.map((f) => ({ path: f.path, sha256: f.sha256, bytes: f.bytes })), + ...(m.mediaOmitted === true ? { mediaOmitted: true } : {}), + }; + return new TextEncoder().encode(`${JSON.stringify(canonical, null, 2)}\n`); +} + +function versionNumber(value: unknown, pattern: RegExp): number | undefined { + const match = typeof value === "string" ? pattern.exec(value) : null; + return match ? Number(match[1]) : undefined; +} + +/** What in `raw` is newer than this reader, if anything. */ +function newerPart(raw: Record, format: number): string | undefined { + if (format > 1) return `archive format ${raw.format as string}`; + const contracts = isRecord(raw.contracts) ? raw.contracts.schemaVersion : undefined; + if (Number.isInteger(contracts) && (contracts as number) > CONTRACT_SCHEMA_VERSION) + return `contracts schemaVersion ${contracts as number}`; + const bundle = versionNumber(raw.bundle, BUNDLE); + if (bundle !== undefined && bundle > 1) return `bundle format ${raw.bundle as string}`; + return undefined; +} + +/** Refuses what only a newer Cloudhal can read, naming the CLI version that wrote it. */ +function checkVersion(raw: Record): void { + const format = versionNumber(raw.format, FORMAT); + if (format === undefined) + throw new ArchiveError("not_archive", "not a Cloudhal site archive (unknown manifest format)"); + const newer = newerPart(raw, format); + if (newer === undefined) return; + const version = + typeof raw.createdBy === "string" && VERSION_TOKEN.test(raw.createdBy) + ? `${raw.createdBy} or later` + : "a newer version"; + throw new ArchiveError( + "unsupported_version", + `this archive was written by a newer Cloudhal (${newer}); update @cloudhal/cli to ${version}`, + ); +} + +function checkCounts(raw: unknown): ArchiveCounts { + if (!isRecord(raw)) throw manifestError("counts must be an object"); + checkKeys(raw, COUNT_KEYS, "counts"); + for (const key of COUNT_KEYS) { + const n = raw[key]; + if (!Number.isSafeInteger(n) || (n as number) < 0) + throw manifestError(`counts.${key} must be a non-negative integer`); + } + return raw as unknown as ArchiveCounts; +} + +function checkStrings(raw: Record): void { + if (raw.bundle !== SITE_BUNDLE_FORMAT) + throw manifestError(`bundle must be ${SITE_BUNDLE_FORMAT}`); + if (typeof raw.siteId !== "string" || !ID.test(raw.siteId)) + throw manifestError("siteId must be a site id"); + const name = raw.siteName; + if (typeof name !== "string" || name.length === 0 || [...name].length > MAX_SITE_NAME) + throw manifestError(`siteName must be a string of 1-${MAX_SITE_NAME} characters`); + if (typeof raw.createdBy !== "string" || !VERSION_TOKEN.test(raw.createdBy)) + throw manifestError("createdBy must be a version (letters, digits, '.', '+', '-')"); + const at = raw.createdAt; + if (typeof at !== "string" || !RFC3339.test(at) || Number.isNaN(Date.parse(at))) + throw manifestError("createdAt must be an RFC 3339 timestamp"); +} + +/** The shape check, after the version gate. */ +function checkShape( + raw: Record, + size: number, + limits: ArchiveLimits, +): SiteArchiveManifest { + checkKeys(raw, TOP_KEYS, "the manifest"); + for (const key of TOP_KEYS) + if (key !== "mediaOmitted" && !(key in raw)) throw manifestError(`${key} is missing`); + if (!isRecord(raw.contracts)) throw manifestError("contracts must be an object"); + checkKeys(raw.contracts, ["schemaVersion"], "contracts"); + const schemaVersion = raw.contracts.schemaVersion; + if (!Number.isInteger(schemaVersion) || (schemaVersion as number) < 1) + throw manifestError("contracts.schemaVersion must be a positive integer"); + checkStrings(raw); + if (raw.mediaOmitted !== undefined && raw.mediaOmitted !== true) + throw manifestError("mediaOmitted must be true when present"); + const counts = checkCounts(raw.counts); + const mediaOmitted = raw.mediaOmitted === true; + const files = checkManifestFiles(raw.files, { + counts, + mediaOmitted, + limits, + manifestBytes: size, + }); + return { + format: SITE_ARCHIVE_FORMAT, + contracts: { schemaVersion: schemaVersion as number }, + bundle: SITE_BUNDLE_FORMAT, + siteId: raw.siteId as string, + siteName: raw.siteName as string, + createdBy: raw.createdBy as string, + createdAt: raw.createdAt as string, + counts: { + nodes: counts.nodes, + snapshots: counts.snapshots, + blobs: counts.blobs, + media: counts.media, + mediaBytes: counts.mediaBytes, + }, + files, + ...(mediaOmitted ? { mediaOmitted: true as const } : {}), + }; +} + +/** Untrusted manifest bytes to a manifest, or the reason they are refused. */ +export function parseManifest(bytes: Uint8Array, limits: ArchiveLimits): SiteArchiveManifest { + if (bytes.length > limits.manifestBytes) + throw new ArchiveError("too_large", `manifest.json is above ${limits.manifestBytes} bytes`, { + path: "manifest.json", + }); + let raw: unknown; + try { + raw = JSON.parse(new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes)); + } catch { + throw manifestError("it is not UTF-8 JSON"); + } + if (!isRecord(raw)) + throw new ArchiveError( + "not_archive", + "not a Cloudhal site archive (the manifest is not an object)", + ); + checkVersion(raw); + return checkShape(raw, bytes.length, limits); +} diff --git a/packages/cli/src/archive/parts.ts b/packages/cli/src/archive/parts.ts new file mode 100644 index 00000000..6b4c9aca --- /dev/null +++ b/packages/cli/src/archive/parts.ts @@ -0,0 +1,98 @@ +/** + * NDJSON parts (RFC-0039 §1): one JSON row per line, a new part every 8 MB. A row's line is + * `JSON.stringify(row)` and a newline; JSON.stringify never writes a raw newline, so the rows of a + * part are exactly its newlines. The writer and the reader count them the same way. + */ +import { ArchiveError, type ArchiveErrorCode } from "./errors.ts"; +import { ARCHIVE_LIMITS, PART_SPLIT_BYTES } from "./limits.ts"; + +const LF = 0x0a; +const encoder = new TextEncoder(); + +export interface NdjsonSplitOptions { + /** Start a new part when the next line would cross this size (default 8 MiB). */ + partBytes?: number; + /** Refuse a line above this size (default: the reader's part cap, 16 MiB). */ + maxLineBytes?: number; +} + +function encodeLine(row: unknown, index: number): Uint8Array { + const text = JSON.stringify(row); + if (text === undefined) + throw new ArchiveError("invalid_input", `row ${index} has no JSON form (${typeof row})`); + return encoder.encode(`${text}\n`); +} + +/** + * `rows` as NDJSON parts, in the given order (the caller sorts): a new part when the current one + * is not empty and the next line would take it above `partBytes`. A line larger than `partBytes` + * goes alone in its own part; a line above `maxLineBytes` is refused. No rows give no parts. + */ +export function splitNdjson( + rows: Iterable, + options: NdjsonSplitOptions = {}, +): Uint8Array[] { + const partBytes = options.partBytes ?? PART_SPLIT_BYTES; + const maxLine = options.maxLineBytes ?? ARCHIVE_LIMITS.partBytes; + const parts: Uint8Array[] = []; + let lines: Uint8Array[] = []; + let size = 0; + let index = 0; + for (const row of rows) { + const line = encodeLine(row, index); + if (line.length > maxLine) + throw new ArchiveError( + "too_large", + `row ${index} is ${line.length} bytes as JSON, above the ${maxLine}-byte part cap`, + ); + if (size > 0 && size + line.length > partBytes) { + parts.push(Buffer.concat(lines, size)); + lines = []; + size = 0; + } + lines.push(line); + size += line.length; + index++; + } + if (size > 0) parts.push(Buffer.concat(lines, size)); + return parts; +} + +/** `rows` as one NDJSON file (`ledger.ndjson`), refused above `maxBytes`. */ +export function encodeNdjson( + rows: Iterable, + maxBytes: number = ARCHIVE_LIMITS.partBytes, +): Uint8Array { + const lines: Uint8Array[] = []; + let size = 0; + let index = 0; + for (const row of rows) { + const line = encodeLine(row, index++); + size += line.length; + if (size > maxBytes) + throw new ArchiveError("too_large", `the rows are above ${maxBytes} bytes as NDJSON`); + lines.push(line); + } + return Buffer.concat(lines, size); +} + +/** + * The rows of an NDJSON file: its newlines. An empty file, an empty line or a missing final + * newline is refused with `code` (`damaged` when reading, `invalid_input` when writing). + */ +export function countRows( + bytes: Uint8Array, + path: string, + code: ArchiveErrorCode = "damaged", +): number { + if (bytes.length === 0 || bytes[bytes.length - 1] !== LF) + throw new ArchiveError(code, `${path}: not NDJSON (empty, or no final newline)`, { path }); + let rows = 0; + let previous = -1; + for (let i = bytes.indexOf(LF); i >= 0; i = bytes.indexOf(LF, i + 1)) { + if (i === previous + 1) throw new ArchiveError(code, `${path}: has an empty line`, { path }); + previous = i; + rows++; + } + return rows; +} diff --git a/packages/cli/src/archive/plan.ts b/packages/cli/src/archive/plan.ts new file mode 100644 index 00000000..5f122aad --- /dev/null +++ b/packages/cli/src/archive/plan.ts @@ -0,0 +1,164 @@ +/** + * The synchronous front half of the archive writer: checks the content, hashes the in-memory + * files, counts the rows and media, and builds the manifest, which it checks with the reader's own + * parser, so the writer never starts an archive its reader would refuse under the same limits. + */ +import { createHash } from "node:crypto"; +import { CONTRACT_SCHEMA_VERSION } from "@cloudhal/core-contracts"; +import { SITE_BUNDLE_FORMAT } from "@cloudhal/core-ts"; +import { ArchiveError } from "./errors.ts"; +import { + LEDGER_PATH, + mediaPath, + PART_KINDS, + type PartKind, + PROJECT_PATH, + partPath, + SHA256_HEX, +} from "./layout.ts"; +import { type ArchiveLimits, MAX_PART_INDEX } from "./limits.ts"; +import { + type ArchiveFile, + encodeManifest, + parseManifest, + SITE_ARCHIVE_FORMAT, + type SiteArchiveManifest, +} from "./manifest.ts"; +import { countRows } from "./parts.ts"; + +/** A media file, streamed when the writer reaches it and verified against `sha256` and `bytes`. */ +export interface ArchiveMediaSource { + readonly sha256: string; + readonly bytes: number; + open(): AsyncIterable | Promise>; +} + +/** What an archive holds. The caller orders the rows (RFC-0039 §1); the writer sorts media. */ +export interface ArchiveContent { + siteId: string; + siteName: string; + /** The `@cloudhal/cli` version. */ + createdBy: string; + /** RFC 3339, taken from the content (the newest row), never from the clock. */ + createdAt: string; + /** `project.json`: the project row as JSON. */ + project?: Uint8Array | undefined; + /** NDJSON parts, from `splitNdjson`. */ + nodes: readonly Uint8Array[]; + snapshots: readonly Uint8Array[]; + blobs: readonly Uint8Array[]; + /** `ledger.ndjson`, from `encodeNdjson`, when there are purged rows. */ + ledger?: Uint8Array | undefined; + media?: readonly ArchiveMediaSource[] | undefined; + mediaOmitted?: true | undefined; +} + +export type PlannedFile = + | { path: string; data: Uint8Array } + | { path: string; media: ArchiveMediaSource }; + +export interface ArchivePlan { + manifest: SiteArchiveManifest; + manifestBytes: Uint8Array; + /** Every file after the manifest, in archive order. */ + files: PlannedFile[]; +} + +interface Draft { + files: PlannedFile[]; + listed: ArchiveFile[]; + limits: ArchiveLimits; +} + +function sha256Of(data: Uint8Array): string { + return createHash("sha256").update(data).digest("hex"); +} + +function addData(draft: Draft, path: string, data: Uint8Array): void { + if (data.length === 0) throw new ArchiveError("invalid_input", `${path} is empty`, { path }); + if (data.length > draft.limits.partBytes) + throw new ArchiveError("too_large", `${path} is above ${draft.limits.partBytes} bytes`, { + path, + }); + draft.files.push({ path, data }); + draft.listed.push({ path, sha256: sha256Of(data), bytes: data.length }); +} + +/** Adds the parts of one kind and returns their rows. */ +function addParts(draft: Draft, kind: PartKind, parts: readonly Uint8Array[]): number { + if (parts.length > MAX_PART_INDEX) + throw new ArchiveError("too_large", `${kind}: more than ${MAX_PART_INDEX} parts`); + let rows = 0; + for (const [i, part] of parts.entries()) { + const path = partPath(kind, i + 1); + addData(draft, path, part); + rows += countRows(part, path, "invalid_input"); + } + return rows; +} + +/** The media in archive order (ascending hash), checked. */ +function sortedMedia(content: ArchiveContent, limits: ArchiveLimits): ArchiveMediaSource[] { + const media = [...(content.media ?? [])]; + if (media.length > 0 && content.mediaOmitted) + throw new ArchiveError("invalid_input", "media were given together with mediaOmitted"); + const seen = new Set(); + for (const m of media) { + if (!SHA256_HEX.test(m.sha256)) + throw new ArchiveError("invalid_input", "a media sha256 is not 64 lowercase hex digits"); + if (seen.has(m.sha256)) + throw new ArchiveError("invalid_input", `media ${m.sha256} is given twice`); + seen.add(m.sha256); + if (!Number.isSafeInteger(m.bytes) || m.bytes < 0) + throw new ArchiveError("invalid_input", `media ${m.sha256} has an invalid size`); + if (m.bytes > limits.mediaBytes) + throw new ArchiveError("too_large", `media ${m.sha256} is above ${limits.mediaBytes} bytes`); + } + return media.sort((a, b) => (a.sha256 < b.sha256 ? -1 : a.sha256 > b.sha256 ? 1 : 0)); +} + +/** The manifest checked by the reader's parser; refusals become the writer's `invalid_input`. */ +function checkedManifest(manifest: SiteArchiveManifest, limits: ArchiveLimits): Uint8Array { + const bytes = encodeManifest(manifest); + try { + parseManifest(bytes, limits); + } catch (e) { + if (e instanceof ArchiveError && e.code !== "too_large") + throw new ArchiveError("invalid_input", e.message, { cause: e }); + throw e; + } + return bytes; +} + +/** Checks `content` and lays out the archive; throws before any byte is written. */ +export function planArchive(content: ArchiveContent, limits: ArchiveLimits): ArchivePlan { + const draft: Draft = { files: [], listed: [], limits }; + if (content.project !== undefined) addData(draft, PROJECT_PATH, content.project); + const rows: Record = { nodes: 0, snapshots: 0, blobs: 0 }; + for (const kind of PART_KINDS) rows[kind] = addParts(draft, kind, content[kind]); + const media = sortedMedia(content, limits); + let mediaBytes = 0; + for (const m of media) { + const path = mediaPath(m.sha256); + draft.files.push({ path, media: m }); + draft.listed.push({ path, sha256: m.sha256, bytes: m.bytes }); + mediaBytes += m.bytes; + } + if (content.ledger !== undefined) { + addData(draft, LEDGER_PATH, content.ledger); + countRows(content.ledger, LEDGER_PATH, "invalid_input"); + } + const manifest: SiteArchiveManifest = { + format: SITE_ARCHIVE_FORMAT, + contracts: { schemaVersion: CONTRACT_SCHEMA_VERSION }, + bundle: SITE_BUNDLE_FORMAT, + siteId: content.siteId, + siteName: content.siteName, + createdBy: content.createdBy, + createdAt: content.createdAt, + counts: { ...rows, media: media.length, mediaBytes }, + files: draft.listed, + ...(content.mediaOmitted ? { mediaOmitted: true as const } : {}), + }; + return { manifest, manifestBytes: checkedManifest(manifest, limits), files: draft.files }; +} diff --git a/packages/cli/src/archive/read.ts b/packages/cli/src/archive/read.ts new file mode 100644 index 00000000..591bef02 --- /dev/null +++ b/packages/cli/src/archive/read.ts @@ -0,0 +1,257 @@ +/** + * The verifying site archive reader (RFC-0039 §1, invariant 3). It checks as it streams and hands + * the caller each item only as far as it can be trusted: + * + * 1. The manifest first, after only `manifest.json` has been read (a dry run can stop there). + * 2. `project.json`, the NDJSON parts and `ledger.ndjson` buffered (they are capped) and yielded + * only after their size, SHA-256 and rows were checked against the manifest. + * 3. Media with a streaming body, hashed as it is read: the body is unverified until its last + * `next()` resolves, which throws `damaged` on a mismatch. A body the caller does not read is + * read and verified by the reader before the next item; a body read after the next item was + * requested, or still being read then, throws `invalid_input` instead of ending early. + * + * The archive as a whole is verified only when the generator returns: every listed file present, + * the row counts equal, the end of the archive well formed and gzip's CRC checked. A caller + * applies nothing (no `importSite`) before that; media it streamed somewhere content-addressed + * stay unreferenced if the archive then fails. + */ +import { createHash } from "node:crypto"; +import { ArchiveError, printable } from "./errors.ts"; +import { gunzipCapped } from "./gzip.ts"; +import { classifyPath, type FileSlot, MANIFEST_PATH, PART_KINDS, type PartKind } from "./layout.ts"; +import { type ArchiveLimits, archiveLimits } from "./limits.ts"; +import { type ArchiveFile, parseManifest, type SiteArchiveManifest } from "./manifest.ts"; +import { countRows } from "./parts.ts"; +import { readTar, type TarEntry } from "./tar-read.ts"; + +export type ArchiveItem = + | { type: "manifest"; manifest: SiteArchiveManifest } + | { type: "file"; path: string; slot: FileSlot; bytes: Uint8Array } + | { + type: "media"; + path: string; + sha256: string; + size: number; + /** + * Readable once, and only before the next item is requested (await the sink first): a read + * after that, or one still running then, throws `invalid_input`. Unverified until its last + * `next()` resolves, which throws `damaged` on a mismatch. + */ + body: AsyncIterable; + }; + +export interface ReadArchiveOptions { + /** Lower caps for untrusted input (the defaults are ARCHIVE_LIMITS). */ + limits?: Partial; +} + +type Entries = AsyncGenerator; + +async function readBody(entry: TarEntry): Promise { + const bytes = new Uint8Array(entry.size); + let filled = 0; + for await (const chunk of entry.body) { + bytes.set(chunk, filled); + filled += chunk.length; + } + return bytes; +} + +async function readManifest(entries: Entries, limits: ArchiveLimits): Promise { + const first = await entries.next(); + if (first.done || first.value.path !== MANIFEST_PATH) + throw new ArchiveError( + "not_archive", + "not a Cloudhal site archive (manifest.json is not its first entry)", + ); + if (first.value.size > limits.manifestBytes) + throw new ArchiveError("too_large", `manifest.json is above ${limits.manifestBytes} bytes`, { + path: MANIFEST_PATH, + }); + return parseManifest(await readBody(first.value), limits); +} + +/** The entry must be the next listed file, with its listed size: checked before its body. */ +function expectEntry(entry: TarEntry, files: readonly ArchiveFile[], index: number) { + const slot = classifyPath(entry.path); + if (slot === undefined) + throw new ArchiveError( + "unsafe_entry", + `the archive holds ${printable(entry.path)}, which is not a path a site archive holds`, + ); + if (slot.kind === "manifest") + throw new ArchiveError("mismatch", "manifest.json: appears twice", { path: MANIFEST_PATH }); + const listed = files[index]; + if (listed?.path !== entry.path) { + const known = files.some((f) => f.path === entry.path); + throw new ArchiveError( + "mismatch", + `${entry.path}: ${known ? "appears twice or out of order" : "is not listed in manifest.json"}`, + { path: entry.path }, + ); + } + if (entry.size !== listed.bytes) + throw new ArchiveError( + "mismatch", + `${entry.path}: ${entry.size} bytes in the archive, ${listed.bytes} in manifest.json`, + { path: entry.path }, + ); + return { slot, listed }; +} + +function damaged(path: string): ArchiveError { + return new ArchiveError( + "damaged", + `${path}: its SHA-256 does not match manifest.json (the file is damaged or was edited)`, + { path }, + ); +} + +/** + * A media body hashed as it is read; its last `next()` throws on a mismatch. Like a tar entry's + * body it is stale once the next item is requested: `drain` marks it so, refuses to run beside a + * read in flight, and reads and verifies the rest itself. Its first error is thrown again by every + * later read, so a caller that retries never sees a clean end of a partial body. + */ +class MediaBody implements AsyncIterable { + readonly #inner: AsyncIterator; + readonly #listed: ArchiveFile; + readonly #hash = createHash("sha256"); + #claimed = false; + /** The caller's reads in flight. */ + #reading = 0; + #stale = false; + #finished = false; + #failure: { error: unknown } | undefined; + + constructor(entry: TarEntry, listed: ArchiveFile) { + this.#inner = entry.body[Symbol.asyncIterator](); + this.#listed = listed; + } + + [Symbol.asyncIterator](): AsyncIterator { + if (this.#claimed) throw this.#misuse("can be read once"); + this.#claimed = true; + return { + next: async () => { + if (this.#stale) throw this.#misuse("was read after the next item was requested"); + this.#reading++; + try { + const value = await this.#pull(); + return value === undefined ? { done: true, value: undefined } : { done: false, value }; + } finally { + this.#reading--; + } + }, + // Stopping early leaves the rest to `drain`, which still verifies it. + return: async () => ({ done: true, value: undefined }), + }; + } + + async #pull(): Promise { + if (this.#failure) throw this.#failure.error; + if (this.#finished) return undefined; + let next: IteratorResult; + try { + next = await this.#inner.next(); + } catch (error) { + this.#failure = { error }; + throw error; + } + if (!next.done) { + this.#hash.update(next.value); + return next.value; + } + this.#finished = true; + if (this.#hash.digest("hex") === this.#listed.sha256) return undefined; + this.#failure = { error: damaged(this.#listed.path) }; + throw this.#failure.error; + } + + /** Reads and verifies what the caller left; the body is stale from here on. */ + async drain(): Promise { + this.#stale = true; + if (this.#reading > 0) + throw this.#misuse("is still being read (await it before the next item)"); + for (;;) if ((await this.#pull()) === undefined) return; + } + + #misuse(what: string): ArchiveError { + const path = this.#listed.path; + return new ArchiveError("invalid_input", `${path}: a media body ${what}`, { path }); + } +} + +/** A buffered file, checked against the manifest; its rows when it is a part. */ +async function readFile( + entry: TarEntry, + listed: ArchiveFile, + slot: FileSlot, +): Promise<{ bytes: Uint8Array; rows: number }> { + const bytes = await readBody(entry); + if (createHash("sha256").update(bytes).digest("hex") !== listed.sha256) throw damaged(entry.path); + const ndjson = slot.kind === "part" || slot.kind === "ledger"; + return { bytes, rows: ndjson ? countRows(bytes, entry.path) : 0 }; +} + +function checkComplete( + manifest: SiteArchiveManifest, + index: number, + rows: Record, +): void { + const missing = manifest.files[index]; + if (missing) + throw new ArchiveError( + "mismatch", + `${missing.path}: listed in manifest.json but not in the archive`, + { + path: missing.path, + }, + ); + for (const kind of PART_KINDS) + if (rows[kind] !== manifest.counts[kind]) + throw new ArchiveError( + "mismatch", + `${kind}: the parts hold ${rows[kind]} rows, manifest.json counts ${manifest.counts[kind]}`, + ); +} + +async function* readEntries( + entries: Entries, + manifest: SiteArchiveManifest, +): AsyncGenerator { + const rows: Record = { nodes: 0, snapshots: 0, blobs: 0 }; + let index = 0; + for await (const entry of entries) { + const { slot, listed } = expectEntry(entry, manifest.files, index++); + if (slot.kind === "media") { + const body = new MediaBody(entry, listed); + yield { type: "media", path: entry.path, sha256: slot.sha256, size: entry.size, body }; + await body.drain(); + continue; + } + const file = await readFile(entry, listed, slot); + if (slot.kind === "part") rows[slot.part] += file.rows; + yield { type: "file", path: entry.path, slot, bytes: file.bytes }; + } + checkComplete(manifest, index, rows); +} + +/** + * The verified items of a `.cloudhal` stream (gzip bytes, for example a file's read stream): + * the manifest, then every file in archive order. Stopping early (break) closes the source. + */ +export async function* readArchive( + source: AsyncIterable, + options: ReadArchiveOptions = {}, +): AsyncGenerator { + const limits = archiveLimits(options.limits); + const entries = readTar(gunzipCapped(source, limits.totalBytes), { maxEntries: limits.entries }); + try { + const manifest = await readManifest(entries, limits); + yield { type: "manifest", manifest }; + yield* readEntries(entries, manifest); + } finally { + await entries.return(undefined); + } +} diff --git a/packages/cli/src/archive/tar-read.ts b/packages/cli/src/archive/tar-read.ts new file mode 100644 index 00000000..d1a97aee --- /dev/null +++ b/packages/cli/src/archive/tar-read.ts @@ -0,0 +1,121 @@ +/** + * Streaming ustar reader: yields the regular-file entries of a decompressed tar stream, each with + * a lazy body, and checks the end of the archive (two zero blocks, then at most one record of zero + * padding, then nothing). It reads its source to the end before it returns, so gunzip's CRC and + * length check has run when the last entry is done. + */ + +import { ByteSource } from "./byte-source.ts"; +import { ArchiveError } from "./errors.ts"; +import { parseTarHeader, TAR_BLOCK, TAR_RECORD, tarPadding } from "./tar.ts"; + +export interface TarEntry { + readonly path: string; + readonly size: number; + /** + * Exactly `size` bytes, readable once and only until the next entry is requested: the reader + * skips what is left of it then, and a later read throws. + */ + readonly body: AsyncIterable; +} + +class EntryBody implements AsyncIterable { + readonly #source: ByteSource; + readonly #path: string; + #remaining: number; + #claimed = false; + #reading = false; + #stale = false; + + constructor(source: ByteSource, entry: { path: string; size: number }) { + this.#source = source; + this.#path = entry.path; + this.#remaining = entry.size; + } + + async *[Symbol.asyncIterator](): AsyncGenerator { + if (this.#claimed) throw this.#misuse("can be read once"); + this.#claimed = true; + for (;;) { + if (this.#stale) throw this.#misuse("was read after the next entry was requested"); + if (this.#remaining === 0) return; + this.#reading = true; + let chunk: Uint8Array; + try { + chunk = await this.#source.readSome(this.#remaining); + } finally { + this.#reading = false; + } + this.#remaining -= chunk.length; + yield chunk; + } + } + + /** Skips what the caller did not read; the body is stale from here on. */ + async finish(): Promise { + if (this.#reading) throw this.#misuse("is still being read"); + this.#stale = true; + await this.#source.skip(this.#remaining); + this.#remaining = 0; + } + + #misuse(what: string): ArchiveError { + return new ArchiveError("invalid_input", `the body of a tar entry ${what}`, { + path: this.#path, + }); + } +} + +function isZero(bytes: Uint8Array): boolean { + for (const b of bytes) if (b !== 0) return false; + return true; +} + +/** After the first zero block: the second one, then at most one record of zeros, then the end. */ +async function readEnd(source: ByteSource): Promise { + const second = await source.read(TAR_BLOCK); + if (second === undefined) + throw new ArchiveError("truncated", "the archive ends inside its end-of-archive marker"); + if (!isZero(second)) + throw new ArchiveError("damaged", "the archive has a zero block between its entries"); + let tail = 0; + while (!(await source.atEnd())) { + const chunk = await source.readSome(TAR_RECORD - tail + 1); + tail += chunk.length; + if (!isZero(chunk) || tail > TAR_RECORD) + throw new ArchiveError("damaged", "the archive has data after its end"); + } +} + +/** The regular files of a tar stream, in order. */ +export async function* readTar( + input: AsyncIterable, + options: { maxEntries: number }, +): AsyncGenerator { + const source = new ByteSource(input); + try { + for (let count = 0; ; count++) { + const block = await source.read(TAR_BLOCK); + if (block === undefined && source.position === 0) + throw new ArchiveError("not_archive", "not a Cloudhal site archive (it is empty)"); + if (block === undefined) + throw new ArchiveError("truncated", "the archive ends without its end-of-archive marker"); + const header = parseTarHeader(block); + if (header === "zero") { + await readEnd(source); + return; + } + if (count >= options.maxEntries) + throw new ArchiveError( + "too_large", + `the archive has more than ${options.maxEntries} entries`, + ); + const body = new EntryBody(source, header); + yield { path: header.path, size: header.size, body }; + await body.finish(); + await source.skip(tarPadding(header.size).length); + } + } finally { + await source.close(); + } +} diff --git a/packages/cli/src/archive/tar.ts b/packages/cli/src/archive/tar.ts new file mode 100644 index 00000000..eee00632 --- /dev/null +++ b/packages/cli/src/archive/tar.ts @@ -0,0 +1,176 @@ +/** + * POSIX ustar blocks for site archives (RFC-0039 §1): regular files only, no I/O. + * + * The writer's headers carry nothing that varies between machines or runs (invariant 7): mode + * 0644, uid/gid 0, mtime 0, empty user, group, link name and prefix. The reader accepts exactly the + * headers Cloudhal writes: a valid checksum, `ustar\0` + `00`, typeflag `0` (or NUL), no link name, + * no prefix and a printable ASCII name. Links, devices, directories, FIFOs and pax or GNU + * extension headers are refused, so an archive can never point outside itself. + */ +import { ArchiveError } from "./errors.ts"; + +export const TAR_BLOCK = 512; +export const TAR_RECORD = 10240; +/** The largest size 11 octal digits hold (8 GiB - 1). */ +export const TAR_MAX_SIZE = 0o77777777777; + +// Field offsets and widths of a ustar header. +const NAME = [0, 100] as const; +const MODE = [100, 8] as const; +const UID = [108, 8] as const; +const GID = [116, 8] as const; +const SIZE = [124, 12] as const; +const MTIME = [136, 12] as const; +const CHECKSUM = [148, 8] as const; +const TYPEFLAG = 156; +const LINKNAME = [157, 100] as const; +const MAGIC = [257, 8] as const; // "ustar\0" + version "00" +const DEVMAJOR = [329, 8] as const; +const DEVMINOR = [337, 8] as const; +const PREFIX = [345, 155] as const; + +const USTAR_MAGIC = "ustar\u000000"; + +const TYPE_NAMES: Record = { + "1": "a hard link", + "2": "a symbolic link", + "3": "a character device", + "4": "a block device", + "5": "a directory", + "6": "a FIFO", + "7": "a contiguous file", + x: "a pax extended header", + g: "a pax global header", + L: "a GNU long name", + K: "a GNU long link name", +}; + +function writeAscii(block: Uint8Array, [offset]: readonly [number, number], text: string): void { + for (let i = 0; i < text.length; i++) block[offset + i] = text.charCodeAt(i); +} + +function octal(value: number, digits: number): string { + return `${value.toString(8).padStart(digits, "0")}\u0000`; +} + +function isPrintableName(name: string): boolean { + for (let i = 0; i < name.length; i++) { + const c = name.charCodeAt(i); + if (c < 0x21 || c > 0x7e) return false; + } + return name.length > 0; +} + +/** The unsigned header checksum: every byte, with the checksum field counted as spaces. */ +function checksumOf(block: Uint8Array): number { + let sum = 0; + for (let i = 0; i < TAR_BLOCK; i++) + sum += i >= CHECKSUM[0] && i < CHECKSUM[0] + CHECKSUM[1] ? 0x20 : (block[i] as number); + return sum; +} + +/** Writes the checksum of `block` into it (`%06o\0 `) and returns it. */ +export function sealTarHeader(block: Uint8Array): Uint8Array { + writeAscii(block, CHECKSUM, `${checksumOf(block).toString(8).padStart(6, "0")}\u0000 `); + return block; +} + +/** The header of a regular file: `path` is printable ASCII of at most 99 bytes. */ +export function tarHeader(path: string, size: number): Uint8Array { + if (!isPrintableName(path) || path.length >= NAME[1]) + throw new ArchiveError("invalid_input", "a tar entry name must be 1-99 printable ASCII bytes"); + if (!Number.isSafeInteger(size) || size < 0 || size > TAR_MAX_SIZE) + throw new ArchiveError("too_large", `${path}: ${size} bytes do not fit a ustar header`, { + path, + }); + const block = new Uint8Array(TAR_BLOCK); + writeAscii(block, NAME, path); + writeAscii(block, MODE, octal(0o644, 7)); + writeAscii(block, UID, octal(0, 7)); + writeAscii(block, GID, octal(0, 7)); + writeAscii(block, SIZE, octal(size, 11)); + writeAscii(block, MTIME, octal(0, 11)); + block[TYPEFLAG] = 0x30; // "0": regular file + writeAscii(block, MAGIC, USTAR_MAGIC); + writeAscii(block, DEVMAJOR, octal(0, 7)); + writeAscii(block, DEVMINOR, octal(0, 7)); + return sealTarHeader(block); +} + +/** Zeros from the end of a `size`-byte body to the next block boundary. */ +export function tarPadding(size: number): Uint8Array { + return new Uint8Array((TAR_BLOCK - (size % TAR_BLOCK)) % TAR_BLOCK); +} + +/** Two zero blocks, then zeros so the archive is a whole number of 10240-byte records. */ +export function tarEnd(written: number): Uint8Array { + const total = Math.ceil((written + 2 * TAR_BLOCK) / TAR_RECORD) * TAR_RECORD; + return new Uint8Array(total - written); +} + +/** The length of the tar stream of files of these sizes, the end of the archive included. */ +export function tarLength(sizes: Iterable): number { + let written = 0; + for (const size of sizes) written += TAR_BLOCK + size + tarPadding(size).length; + return written + tarEnd(written).length; +} + +function isZero(bytes: Uint8Array, [offset, length]: readonly [number, number]): boolean { + for (let i = offset; i < offset + length; i++) if (bytes[i] !== 0) return false; + return true; +} + +/** An octal number field: optional leading spaces, digits, then NUL or space padding only. */ +function parseOctal(block: Uint8Array, [offset, length]: readonly [number, number]): number { + let i = offset; + const end = offset + length; + while (i < end && block[i] === 0x20) i++; + let value = 0; + let digits = 0; + for (; i < end && (block[i] as number) >= 0x30 && (block[i] as number) <= 0x37; i++, digits++) + value = value * 8 + ((block[i] as number) - 0x30); + for (; i < end; i++) if (block[i] !== 0 && block[i] !== 0x20) return Number.NaN; + return digits === 0 ? Number.NaN : value; +} + +function unsafe(message: string): ArchiveError { + return new ArchiveError("unsafe_entry", message); +} + +function checkType(block: Uint8Array): void { + const flag = block[TYPEFLAG] as number; + if (flag === 0x30 || flag === 0) return; + const char = String.fromCharCode(flag); + const what = TYPE_NAMES[char] ?? `an entry of type ${flag < 0x21 || flag > 0x7e ? flag : char}`; + throw unsafe(`the archive holds ${what}; site archives hold regular files only`); +} + +function parseName(block: Uint8Array): string { + const nul = block.indexOf(0, NAME[0]); + if (nul < 0 || nul >= NAME[1] || !isZero(block, [nul, NAME[1] - nul])) + throw unsafe("a tar entry name is not NUL-terminated"); + const name = String.fromCharCode(...block.subarray(NAME[0], nul)); + if (!isPrintableName(name)) throw unsafe("a tar entry name is empty or not printable ASCII"); + return name; +} + +/** + * One header block: `"zero"` for an all-zero block (the end of the archive), otherwise the regular + * file it describes. A bad checksum is `damaged`; anything Cloudhal does not write is + * `unsafe_entry`. + */ +export function parseTarHeader(block: Uint8Array): { path: string; size: number } | "zero" { + if (isZero(block, [0, TAR_BLOCK])) return "zero"; + if (parseOctal(block, CHECKSUM) !== checksumOf(block)) + throw new ArchiveError("damaged", "a tar header checksum does not match (the file is damaged)"); + if (String.fromCharCode(...block.subarray(MAGIC[0], MAGIC[0] + MAGIC[1])) !== USTAR_MAGIC) + throw unsafe("a tar header is not POSIX ustar"); + checkType(block); + if (!isZero(block, LINKNAME)) throw unsafe("a tar entry has a link name"); + if (!isZero(block, PREFIX)) throw unsafe("a tar entry has a name prefix"); + const path = parseName(block); + if (((block[SIZE[0]] as number) & 0x80) !== 0) throw unsafe("a tar entry size is base-256"); + const size = parseOctal(block, SIZE); + if (Number.isNaN(size)) throw unsafe("a tar entry size is not an octal number"); + return { path, size }; +} diff --git a/packages/cli/src/archive/write.ts b/packages/cli/src/archive/write.ts new file mode 100644 index 00000000..a9786cf6 --- /dev/null +++ b/packages/cli/src/archive/write.ts @@ -0,0 +1,126 @@ +/** + * The site archive writer (RFC-0039 §1): `manifest.json` first, then the files in the one order of + * the layout, as a POSIX ustar stream compressed by `gzipFixed`. Media are streamed and verified + * against their SHA-256 and size on the way; a mismatch ends the stream before the gzip trailer, so + * a partial output never reads as an archive, and `writeArchiveFile` removes it. + * + * Determinism (invariant 7): nothing here reads the clock or the machine. `createdAt` and + * `createdBy` come from the caller (`cloudhal export` passes the newest row timestamp of the site + * and its own version); rows come in the caller's order, media are sorted by hash, and tar headers + * carry fixed modes and zero times. The same content written by the same Node/zlib build gives + * identical bytes; the tar stream inside the gzip, and so every SHA-256 in the manifest, is + * identical on every platform. + */ +import { createHash, randomUUID } from "node:crypto"; +import { type FileHandle, open, rename, rm } from "node:fs/promises"; +import path from "node:path"; +import { pipeline } from "node:stream/promises"; +import { ArchiveError } from "./errors.ts"; +import { gzipFixed } from "./gzip.ts"; +import { MANIFEST_PATH } from "./layout.ts"; +import { type ArchiveLimits, archiveLimits } from "./limits.ts"; +import type { SiteArchiveManifest } from "./manifest.ts"; +import { + type ArchiveContent, + type ArchiveMediaSource, + type ArchivePlan, + planArchive, +} from "./plan.ts"; +import { TAR_BLOCK, tarEnd, tarHeader, tarPadding } from "./tar.ts"; + +export interface WriteArchiveOptions { + /** Lower caps (tests); the writer refuses what a reader with the same caps would refuse. */ + limits?: Partial; +} + +/** A media body as it is written: no more than `bytes`, then the SHA-256. */ +async function* verifiedMedia( + file: string, + media: ArchiveMediaSource, +): AsyncGenerator { + const hash = createHash("sha256"); + let seen = 0; + for await (const chunk of await media.open()) { + seen += chunk.length; + if (seen > media.bytes) + throw new ArchiveError( + "damaged", + `${file}: the media source is longer than ${media.bytes} bytes`, + { + path: file, + }, + ); + hash.update(chunk); + yield chunk; + } + if (seen < media.bytes) + throw new ArchiveError( + "damaged", + `${file}: the media source has ${seen} of ${media.bytes} bytes`, + { + path: file, + }, + ); + if (hash.digest("hex") !== media.sha256) + throw new ArchiveError("damaged", `${file}: the media source does not match its SHA-256`, { + path: file, + }); +} + +/** The tar stream of a plan. */ +async function* tarChunks(plan: ArchivePlan): AsyncGenerator { + let written = 0; + for (const file of [{ path: MANIFEST_PATH, data: plan.manifestBytes }, ...plan.files]) { + const size = "data" in file ? file.data.length : file.media.bytes; + yield tarHeader(file.path, size); + if ("data" in file) yield file.data; + else yield* verifiedMedia(file.path, file.media); + const padding = tarPadding(size); + if (padding.length > 0) yield padding; + written += TAR_BLOCK + size + padding.length; + } + yield tarEnd(written); +} + +/** + * The archive of `content` as a byte stream, with its manifest. The content is checked before + * this returns (it throws, and nothing is streamed); a media mismatch fails the stream. + */ +export function archiveStream( + content: ArchiveContent, + options: WriteArchiveOptions = {}, +): { manifest: SiteArchiveManifest; bytes: AsyncGenerator } { + const plan = planArchive(content, archiveLimits(options.limits)); + return { manifest: plan.manifest, bytes: gzipFixed(tarChunks(plan)) }; +} + +/** + * Writes the archive of `content` to `file`: to a temporary file next to it first, synced and then + * renamed over `file`, so a failure leaves no file behind and an existing `file` unchanged. + */ +export async function writeArchiveFile( + file: string, + content: ArchiveContent, + options: WriteArchiveOptions = {}, +): Promise { + const { manifest, bytes } = archiveStream(content, options); + const temp = path.join( + path.dirname(file), + `.${path.basename(file)}.partial-${process.pid}-${randomUUID().slice(0, 8)}`, + ); + let handle: FileHandle | undefined; + try { + handle = await open(temp, "wx"); + // The stream owns the handle: it fsyncs (flush) and closes it, on errors too. + await pipeline(bytes, handle.createWriteStream({ flush: true })); + await rename(temp, file); + return manifest; + } catch (e) { + await bytes.return(); + if (handle !== undefined) { + await handle.close().catch(() => {}); + await rm(temp, { force: true }); + } + throw e; + } +} diff --git a/packages/cli/test/archive-fixture.ts b/packages/cli/test/archive-fixture.ts new file mode 100644 index 00000000..b65c195a --- /dev/null +++ b/packages/cli/test/archive-fixture.ts @@ -0,0 +1,248 @@ +/** + * Helpers for the site archive tests (RFC-0039 step 1): fixed sample content, chunked sources, a + * full read-back, and hand-built tar streams for hostile input. The ustar offsets used to patch + * headers are spelled out here, apart from src/archive/tar.ts, as an independent reading of the + * format. + */ +import { createHash } from "node:crypto"; +import { gunzipSync, gzipSync } from "node:zlib"; +import { expect } from "vitest"; +import { + type ArchiveContent, + ArchiveError, + type ArchiveErrorCode, + type ArchiveMediaSource, + archiveStream, + encodeNdjson, + type ReadArchiveOptions, + readArchive, + type SiteArchiveManifest, + splitNdjson, + type WriteArchiveOptions, +} from "../src/archive/index.ts"; +import { sealTarHeader, tarHeader, tarPadding } from "../src/archive/tar.ts"; + +/** ustar header offsets (POSIX.1-1988). */ +export const USTAR = { + name: 0, + size: 124, + checksum: 148, + typeflag: 156, + linkname: 157, + magic: 257, + version: 263, + prefix: 345, +} as const; + +export const utf8 = (text: string): Uint8Array => new TextEncoder().encode(text); +export const sha256 = (bytes: Uint8Array): string => + createHash("sha256").update(bytes).digest("hex"); + +export const SITE = { + siteId: "site_demo0001", + siteName: "Demo site", + createdBy: "0.6.1", + createdAt: "2026-10-20T12:00:00.000Z", +}; + +/** Deterministic bytes of a given length. */ +export function pattern(length: number, seed: number): Uint8Array { + const out = new Uint8Array(length); + for (let i = 0; i < length; i++) out[i] = (i * 31 + seed * 7) % 251; + return out; +} + +/** Byte-for-byte equality. */ +export const same = (a: Uint8Array | undefined, b: Uint8Array): boolean => + a !== undefined && Buffer.compare(a, b) === 0; + +/** XORs one byte in place. */ +export function flip(bytes: Uint8Array, at: number, mask = 1): void { + bytes[at] = (bytes[at] as number) ^ mask; +} + +/** `files[index]` of a manifest object under test. */ +export function listedAt( + manifest: Record, + index: number, +): Record { + return (manifest.files as Record[])[index] as Record; +} + +export async function* chunked(bytes: Uint8Array, size: number): AsyncGenerator { + for (let i = 0; i < bytes.length; i += size) yield bytes.subarray(i, i + size); +} + +export async function collect(iter: AsyncIterable): Promise { + const chunks: Uint8Array[] = []; + for await (const chunk of iter) chunks.push(chunk); + return Buffer.concat(chunks); +} + +export function memoryMedia(bytes: Uint8Array, chunk = 64 * 1024): ArchiveMediaSource { + return { sha256: sha256(bytes), bytes: bytes.length, open: () => chunked(bytes, chunk) }; +} + +export const MEDIA = [pattern(3000, 1), pattern(700, 2), pattern(0, 3), pattern(70_000, 4)]; + +export const NODE_ROWS = Array.from({ length: 12 }, (_, i) => ({ + id: `page_${String(i).padStart(8, "0")}`, + kind: "page", + body: JSON.stringify({ title: `Page ${i}`, text: "lorem ipsum ".repeat(5) }), +})); + +/** Sample content: nodes in several parts, snapshots, blobs, project, ledger and media. */ +export function sampleContent(overrides: Partial = {}): ArchiveContent { + return { + ...SITE, + project: utf8(JSON.stringify({ id: "prj_demo0001", rev: 1 })), + nodes: splitNdjson(NODE_ROWS, { partBytes: 512 }), + snapshots: splitNdjson([{ id: "snap_00000001" }, { id: "snap_00000002" }]), + blobs: splitNdjson([{ hash: "a".repeat(64), body: "{}" }]), + ledger: encodeNdjson([{ sha256: "b".repeat(64), reason: "dmca", legal: true }]), + media: MEDIA.map((m) => memoryMedia(m)), + ...overrides, + }; +} + +export async function writeBytes( + content: ArchiveContent, + options: WriteArchiveOptions = {}, +): Promise { + return collect(archiveStream(content, options).bytes); +} + +export interface ReadBack { + manifest: SiteArchiveManifest; + files: { path: string; bytes: Buffer }[]; +} + +/** Every item of an archive, media bodies read in full. */ +export async function readAll( + bytes: Uint8Array, + options: ReadArchiveOptions & { chunk?: number } = {}, +): Promise { + let manifest: SiteArchiveManifest | undefined; + const files: { path: string; bytes: Buffer }[] = []; + for await (const item of readArchive(chunked(bytes, options.chunk ?? 4096), options)) { + if (item.type === "manifest") manifest = item.manifest; + else if (item.type === "file") files.push({ path: item.path, bytes: Buffer.from(item.bytes) }); + else files.push({ path: item.path, bytes: await collect(item.body) }); + } + if (!manifest) throw new Error("no manifest"); + return { manifest, files }; +} + +export interface RawEntry { + path: string; + body: Uint8Array | string; + /** Declared size, when it should differ from the body's. */ + size?: number; + /** Edits the header before it is sealed with a fresh checksum. */ + patch?: (header: Uint8Array) => void; +} + +function setName(header: Uint8Array, path: Uint8Array): void { + header.fill(0, USTAR.name, USTAR.name + 100); + header.set(path.subarray(0, 100), USTAR.name); +} + +/** A tar stream built entry by entry; `end: false` leaves out the two zero blocks. */ +export function rawTar( + entries: RawEntry[], + options: { end?: boolean; tail?: Uint8Array } = {}, +): Buffer { + const out: Uint8Array[] = []; + for (const entry of entries) { + const body = typeof entry.body === "string" ? utf8(entry.body) : entry.body; + const header = tarHeader("x", entry.size ?? body.length); + setName(header, utf8(entry.path)); + entry.patch?.(header); + out.push(sealTarHeader(header), body, tarPadding(body.length)); + } + if (options.end !== false) out.push(new Uint8Array(1024)); + if (options.tail) out.push(options.tail); + return Buffer.concat(out); +} + +const LF = 0x0a; +function rows(body: Uint8Array): number { + return body.filter((b) => b === LF).length; +} + +/** A manifest (as a plain object, for tests to break) listing `files`. */ +export function manifestFor(files: RawEntry[]): Record { + const counts = { nodes: 0, snapshots: 0, blobs: 0, media: 0, mediaBytes: 0 }; + const listed = files.map((f) => { + const body = typeof f.body === "string" ? utf8(f.body) : f.body; + const kind = f.path.split("/")[0] as string; + if (kind === "nodes" || kind === "snapshots" || kind === "blobs") counts[kind] += rows(body); + if (kind === "media") { + counts.media++; + counts.mediaBytes += body.length; + } + return { path: f.path, sha256: sha256(body), bytes: body.length }; + }); + return { + format: "cloudhal.site-archive@1", + contracts: { schemaVersion: 1 }, + bundle: "cloudhal.site-bundle@1", + ...SITE, + counts, + files: listed, + }; +} + +/** A small valid file set: project, two node parts, one snapshot part and a media file. */ +export function baseFiles(): RawEntry[] { + const media = pattern(900, 9); + return [ + { path: "project.json", body: '{"id":"prj_demo0001"}' }, + { path: "nodes/0001.ndjson", body: '{"id":"n1"}\n{"id":"n2"}\n' }, + { path: "nodes/0002.ndjson", body: '{"id":"n3"}\n' }, + { path: "snapshots/0001.ndjson", body: '{"id":"s1"}\n' }, + { path: `media/${sha256(media)}`, body: media }, + ]; +} + +/** + * A gzipped archive of `entries` whose manifest lists `listed` (default: the same entries); + * `manifest` edits the manifest object first. + */ +export function archiveOf( + entries: RawEntry[], + options: { + listed?: RawEntry[]; + manifest?: (m: Record) => void; + end?: boolean; + tail?: Uint8Array; + } = {}, +): Buffer { + const manifest = manifestFor(options.listed ?? entries); + options.manifest?.(manifest); + const head: RawEntry = { path: "manifest.json", body: JSON.stringify(manifest) }; + return gzipSync(rawTar([head, ...entries], options)); +} + +/** A valid archive's tar, edited, compressed again. */ +export function retar(archive: Uint8Array, edit: (tar: Buffer) => Buffer | undefined): Buffer { + const tar = gunzipSync(archive); + return gzipSync(edit(tar) ?? tar); +} + +/** Awaits a rejection and checks it is an ArchiveError with `code` (and `text`). */ +export async function expectArchiveError( + promise: Promise, + code: ArchiveErrorCode, + text?: string | RegExp, +): Promise { + const error = await promise.then( + () => undefined, + (e: unknown) => e, + ); + expect(error, `expected an ArchiveError ${code}`).toBeInstanceOf(ArchiveError); + const archiveError = error as ArchiveError; + expect({ code: archiveError.code, message: archiveError.message }).toMatchObject({ code }); + if (text !== undefined) expect(archiveError.message).toMatch(text); + return archiveError; +} diff --git a/packages/cli/test/archive-hostile.test.ts b/packages/cli/test/archive-hostile.test.ts new file mode 100644 index 00000000..fde6b52f --- /dev/null +++ b/packages/cli/test/archive-hostile.test.ts @@ -0,0 +1,471 @@ +/** + * RFC-0039 invariant 3 and Security: an archive is untrusted input. Damaged, edited, reordered or + * oversized archives, paths outside the layout, links and devices, and broken or padded gzip are + * refused, and nothing unverified is handed over as verified. + */ +import { createReadStream } from "node:fs"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { gzipSync } from "node:zlib"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { type ArchiveErrorCode, type ArchiveItem, readArchive } from "../src/archive/index.ts"; +import { encodeManifest } from "../src/archive/manifest.ts"; +import { tarLength } from "../src/archive/tar.ts"; +import { + archiveOf, + baseFiles, + chunked, + collect, + expectArchiveError, + flip, + listedAt, + MEDIA, + pattern, + type RawEntry, + rawTar, + readAll, + retar, + sampleContent, + sha256, + USTAR, + writeBytes, +} from "./archive-fixture.ts"; + +const valid = await writeBytes(sampleContent()); + +let dir: string; +beforeAll(async () => { + dir = await mkdtemp(path.join(os.tmpdir(), "archive-hostile-")); +}); +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +/** The offset of the body of `path` in a tar stream. */ +function bodyOffset(tar: Buffer, path: string): number { + for (let at = 0; at < tar.length; at += 512) + if ( + Buffer.from(tar.subarray(at, at + 100)) + .toString("latin1") + .split("\0")[0] === path + ) + return at + 512; + throw new Error(`no ${path}`); +} + +/** Items read until the first error, which is returned with them. */ +async function readUntilError(archive: Uint8Array, drainMedia = true) { + const seen: ArchiveItem[] = []; + try { + for await (const item of readArchive(chunked(archive, 4096))) { + seen.push(item); + if (item.type === "media" && drainMedia) await collect(item.body); + } + } catch (error) { + return { seen, error }; + } + return { seen, error: undefined }; +} + +describe("damaged and edited files", () => { + it("refuses a part with a flipped byte and never hands it over", async () => { + const damaged = retar(valid, (tar) => { + flip(tar, bodyOffset(tar, "nodes/0002.ndjson") + 10); + return tar; + }); + const { seen, error } = await readUntilError(damaged); + await expectArchiveError(Promise.reject(error), "damaged", /^nodes\/0002\.ndjson: .*SHA-256/); + expect(seen.map((i) => (i.type === "manifest" ? "manifest.json" : i.path))).toEqual([ + "manifest.json", + "project.json", + "nodes/0001.ndjson", + ]); + }); + + it("refuses a media file with a flipped byte, read or not", async () => { + const { manifest } = await readAll(valid); + const media = manifest.files.find((f) => f.path.startsWith("media/") && f.bytes > 0); + const path = media?.path as string; + const damaged = retar(valid, (tar) => { + flip(tar, bodyOffset(tar, path) + 5); + return tar; + }); + // Read: the body's last next() throws. + let bodyError: unknown; + for await (const item of readArchive(chunked(damaged, 1000))) { + if (item.type !== "media" || item.path !== path) continue; + bodyError = await collect(item.body).then( + () => undefined, + (e: unknown) => e, + ); + break; + } + await expectArchiveError(Promise.reject(bodyError), "damaged", path); + // Not read: the reader drains it and throws before the next item. + const { seen, error } = await readUntilError(damaged, false); + await expectArchiveError(Promise.reject(error), "damaged", path); + expect(seen.at(-1)).toMatchObject({ type: "media", path }); + }); + + it("refuses a listed hash other than the media name, a size or a row count that disagrees", async () => { + const files = baseFiles(); + const edited = (edit: (m: Record) => void) => + readAll(archiveOf(files, { manifest: edit })); + await expectArchiveError( + edited((m) => { + listedAt(m, 4).sha256 = "0".repeat(64); + }), + "invalid_manifest", + /SHA-256 other than its name/, + ); + await expectArchiveError( + edited((m) => { + listedAt(m, 1).bytes = 25; + }), + "mismatch", + /^nodes\/0001\.ndjson: 24 bytes in the archive, 25 in manifest\.json/, + ); + await expectArchiveError( + edited((m) => { + (m.counts as { nodes: number }).nodes = 2; + }), + "mismatch", + /nodes: the parts hold 3 rows, manifest\.json counts 2/, + ); + }); +}); + +const HEX = "ab".repeat(32); +const BAD_PATHS = [ + "../x", + "/etc/passwd", + "nodes/../manifest.json", + "nodes\\0001.ndjson", + "./manifest.json", + "nodes/0000.ndjson", + "nodes/1.ndjson", + "nodes/00001.ndjson", + "nodes/0001.json", + `media/${HEX.toUpperCase()}`, + `media/${HEX.slice(1)}`, + `media/${HEX}.png`, + `media/${HEX}/`, + "Manifest.json", + "project.json/", + "x.txt", + "ledger.ndjson.gz", + "", +]; + +describe("paths outside the layout", () => { + it.each(BAD_PATHS)("refuses %j as an entry and in the manifest", async (bad) => { + const files = baseFiles(); + await expectArchiveError( + readAll(archiveOf([...files, { path: bad, body: "x" }], { listed: files })), + "unsafe_entry", + ); + await expectArchiveError( + readAll( + archiveOf(files, { + manifest: (m) => { + (m.files as unknown[]).push({ path: bad, sha256: HEX, bytes: 1 }); + }, + }), + ), + "invalid_manifest", + ); + }); + + it("refuses symlinks, hard links and directories inside a valid archive", async () => { + const files = baseFiles(); + for (const flag of ["2", "1", "5"]) { + const link: RawEntry = { + path: "nodes/0003.ndjson", + body: "", + patch: (h) => { + h[USTAR.typeflag] = flag.charCodeAt(0); + if (flag !== "5") h.set(Buffer.from("/etc/passwd"), USTAR.linkname); + }, + }; + await expectArchiveError( + readAll(archiveOf([...files.slice(0, 3), link, ...files.slice(3)], { listed: files })), + "unsafe_entry", + ); + } + }); +}); + +describe("entries against the manifest", () => { + const files = baseFiles(); + const [project, n1, n2, s1, media] = files as [RawEntry, RawEntry, RawEntry, RawEntry, RawEntry]; + const other = Buffer.from("other media"); + const second: RawEntry = { path: `media/${sha256(other)}`, body: other }; + const [low, high] = [media, second].sort((a, b) => (a.path < b.path ? -1 : 1)) as [ + RawEntry, + RawEntry, + ]; + const cases: [string, RawEntry[], RawEntry[], RegExp][] = [ + [ + "an unlisted extra entry", + [...files, { path: "blobs/0001.ndjson", body: "{}\n" }], + files, + /not listed/, + ], + ["a duplicate entry", [project, n1, n1, n2, s1, media], files, /twice or out of order/], + ["snapshots before nodes", [project, s1, n1, n2, media], files, /twice or out of order/], + [ + "unsorted media", + [project, n1, n2, s1, high, low], + [project, n1, n2, s1, low, high], + /twice or out of order/, + ], + [ + "a listed but missing file", + files.slice(0, 4), + files, + /listed in manifest\.json but not in the archive/, + ], + [ + "a second manifest", + [...files, { path: "manifest.json", body: "{}" }], + files, + /appears twice/, + ], + ]; + it.each(cases)("refuses %s", async (_, entries, listed, text) => { + await expectArchiveError(readAll(archiveOf(entries, { listed })), "mismatch", text); + }); +}); + +describe("caps", () => { + it("refuses an oversized manifest before reading it", async () => { + // A declared size of 1 GiB and no body: the cap answers before the truncation could. + const tar = rawTar([{ path: "manifest.json", body: "{}", size: 1 << 30 }], { end: false }); + await expectArchiveError(readAll(gzipSync(tar)), "too_large", /manifest\.json is above/); + await expectArchiveError(readAll(valid, { limits: { manifestBytes: 100 } }), "too_large"); + }); + + it("refuses parts, media, totals and entry counts above the caps at manifest time", async () => { + await expectArchiveError(readAll(valid, { limits: { partBytes: 100 } }), "too_large", /cap/); + await expectArchiveError( + readAll(valid, { limits: { mediaBytes: 1000 } }), + "too_large", + /media\//, + ); + await expectArchiveError( + readAll(valid, { limits: { totalBytes: 50_000 } }), + "too_large", + /decompressed/, + ); + await expectArchiveError(readAll(valid, { limits: { entries: 5 } }), "too_large", /entries/); + }); + + it("counts decompressed bytes while streaming", async () => { + const { manifest } = await readAll(valid); + const exact = tarLength([ + encodeManifest(manifest).length, + ...manifest.files.map((f) => f.bytes), + ]); + expect((await readAll(valid, { limits: { totalBytes: exact } })).files).toHaveLength( + manifest.files.length, + ); + // 512 more zeros after the end: within the padding the reader allows, beyond the cap. + const padded = retar(valid, (tar) => Buffer.concat([tar, new Uint8Array(512)])); + expect((await readAll(padded, { limits: { totalBytes: exact + 512 } })).files).toHaveLength( + manifest.files.length, + ); + await expectArchiveError( + readAll(padded, { limits: { totalBytes: exact + 511 } }), + "too_large", + /decompressed/, + ); + }); +}); + +describe("gzip", () => { + it("refuses a truncated stream, even after the last item", async () => { + for (const cut of [5, Math.floor(valid.length / 2), valid.length - 4]) { + await expectArchiveError(readAll(valid.subarray(0, cut)), "truncated"); + } + }); + + it("refuses corrupt data, a wrong CRC, non-gzip input and garbage after the trailer", async () => { + const crc = Buffer.from(valid); + flip(crc, crc.length - 8); + await expectArchiveError(readAll(crc), "damaged", /gzip/); + const mid = Buffer.from(valid); + flip(mid, Math.floor(mid.length / 2), 0xff); + await expectArchiveError(readAll(mid), "damaged"); + await expectArchiveError(readAll(Buffer.from("hello")), "not_archive", /not gzip/); + await expectArchiveError(readAll(Buffer.concat([valid, Buffer.from("garbage")])), "damaged"); + }); + + it("tells damaged gzip from input that is not gzip by its first bytes, at any size", async () => { + // Read in one chunk, an archive under 16 KiB decompressed fails in zlib before any output. + const small = archiveOf(baseFiles()); + const crc = Buffer.from(small); + flip(crc, crc.length - 8); + const mid = Buffer.from(small); + flip(mid, Math.floor(mid.length / 2), 0xff); + const cases: [Uint8Array, ArchiveErrorCode, number[], RegExp?][] = [ + [crc, "damaged", [crc.length, 64], /gzip data is corrupt/], + // In 64-byte chunks the damaged manifest is parsed first: invalid_manifest. + [mid, "damaged", [mid.length]], + [Buffer.concat([small, Buffer.from("garbage")]), "damaged", [small.length + 7, 64]], + [small.subarray(0, small.length - 4), "truncated", [small.length, 64]], + [Buffer.from("hello"), "not_archive", [5, 1], /not gzip/], + [Buffer.from([0x1f]), "not_archive", [1], /not gzip/], + [new Uint8Array(0), "not_archive", [1], /it is empty/], + ]; + for (const [bytes, code, chunks, text] of cases) + for (const chunk of chunks) await expectArchiveError(readAll(bytes, { chunk }), code, text); + }); + + it("refuses data after the gzip stream that starts with a zero byte, however it is chunked", async () => { + const tails = [ + Buffer.from([0]), + Buffer.from("\0junk"), + Buffer.alloc(64), + Buffer.concat([Buffer.alloc(64), Buffer.from("junk")]), + Buffer.concat([Buffer.from([0]), Buffer.alloc(1_200_000, "A")]), + ]; + for (const tail of tails) { + const padded = Buffer.concat([valid, tail]); + for (const chunk of [padded.length, 4096, 64]) + await expectArchiveError(readAll(padded, { chunk }), "damaged", /after its gzip stream/); + async function* apart() { + yield valid; + yield* chunked(tail, 16); + } + await expectArchiveError(collectItems(readArchive(apart())), "damaged", /gzip stream/); + } + }); + + it("refuses data after the gzip stream of a file read stream", async () => { + const tails = [ + Buffer.alloc(8), + Buffer.from("\0hidden"), + Buffer.concat([Buffer.from([0]), Buffer.alloc(200_000, "A")]), + Buffer.alloc(1 << 20), + ]; + for (const [i, tail] of tails.entries()) { + const file = path.join(dir, `tail-${i}.cloudhal`); + await writeFile(file, Buffer.concat([valid, tail])); + await expectArchiveError( + collectItems(readArchive(createReadStream(file))), + "damaged", + /after its gzip stream/, + ); + } + const clean = path.join(dir, "clean.cloudhal"); + await writeFile(clean, valid); + expect(await collectItems(readArchive(createReadStream(clean)))).toBeGreaterThan(1); + }); + + it("stops a gzip bomb after the end of the archive", async () => { + const member = gzipSync(new Uint8Array(1 << 20)); // 1 MiB of zeros in about 1 KiB + let pulled = 0; + async function* bomb() { + yield valid; + for (let i = 0; i < 1024; i++) { + pulled++; + yield member; + } + } + await expectArchiveError(collectItems(readArchive(bomb())), "damaged", /after its end/); + expect(pulled).toBeLessThan(64); + }); + + it("refuses a second gzip member that holds another entry", async () => { + const extra = gzipSync(rawTar([{ path: "nodes/0004.ndjson", body: "{}\n" }])); + await expectArchiveError(readAll(Buffer.concat([valid, extra])), "damaged", /after its end/); + }); +}); + +async function collectItems(items: AsyncIterable): Promise { + let n = 0; + for await (const item of items) { + if (item.type === "media") await collect(item.body); + n++; + } + return n; +} + +type MediaItem = Extract; + +/** The first media item of `items` with at least `size` bytes. */ +async function firstMedia(items: AsyncGenerator, size = 0): Promise { + for (;;) { + const next = await items.next(); + if (next.done) throw new Error("no media item"); + if (next.value.type === "media" && next.value.size >= size) return next.value; + } +} + +/** Reads an iterator to its end. */ +async function toEnd(body: AsyncIterator): Promise { + for (;;) if ((await body.next()).done) return; +} + +describe("media bodies", () => { + it("refuses a body read after the next item was requested, an empty one too", async () => { + const held: MediaItem[] = []; + for await (const item of readArchive(chunked(valid, 4096))) + if (item.type === "media") held.push(item); + expect(held.map((m) => m.size).sort((a, b) => a - b)).toEqual( + MEDIA.map((m) => m.length).sort((a, b) => a - b), + ); + for (const item of held) + await expectArchiveError( + collect(item.body), + "invalid_input", + `${item.path}: a media body was read after the next item was requested`, + ); + }); + + it("refuses to move on while a body is still being read", async () => { + const items = readArchive(chunked(valid, 1000)); + const body = (await firstMedia(items, 1)).body[Symbol.asyncIterator](); + const pending = body.next(); // a sink started without awaiting it + await expectArchiveError(items.next(), "invalid_input", /is still being read/); + await pending.catch(() => undefined); + await expectArchiveError(body.next(), "invalid_input", /after the next item/); + }); + + it("keeps a read error, so a retry never ends a partial body cleanly", async () => { + // Named and listed by the hash of the 512 bytes the archive holds, declared 1024 long. + const held = pattern(512, 5); + const cut: RawEntry = { path: `media/${sha256(held)}`, body: held, size: 1024 }; + const archive = archiveOf([...baseFiles().slice(0, 4), cut], { + end: false, + manifest: (m) => { + listedAt(m, 4).bytes = 1024; + (m.counts as { mediaBytes: number }).mediaBytes = 1024; + }, + }); + const items = readArchive(chunked(archive, 4096)); + const body = (await firstMedia(items)).body[Symbol.asyncIterator](); + await expectArchiveError(toEnd(body), "truncated"); + await expectArchiveError(body.next(), "truncated"); + await expectArchiveError(items.next(), "truncated"); + }); +}); + +describe("stopping early", () => { + it("closes the source when the caller stops after the manifest", async () => { + let closed = false; + async function* source() { + try { + yield* chunked(valid, 512); + } finally { + closed = true; + } + } + for await (const item of readArchive(source())) { + expect(item.type).toBe("manifest"); + break; + } + await vi.waitFor(() => expect(closed).toBe(true)); + }); +}); diff --git a/packages/cli/test/archive-interop.test.ts b/packages/cli/test/archive-interop.test.ts new file mode 100644 index 00000000..a5aef3ce --- /dev/null +++ b/packages/cli/test/archive-interop.test.ts @@ -0,0 +1,45 @@ +/** + * RFC-0039 §1: a `.cloudhal` file is a standard gzipped ustar. The system `tar` (bsdtar on macOS, + * GNU tar on Linux CI) lists exactly the archive's paths and extracts identical bytes. Skipped + * where no `tar` is installed. + */ +import { spawnSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { writeArchiveFile } from "../src/archive/index.ts"; +import { encodeManifest } from "../src/archive/manifest.ts"; +import { MEDIA, same, sampleContent, sha256 } from "./archive-fixture.ts"; + +const hasTar = spawnSync("tar", ["--version"], { stdio: "ignore" }).status === 0; + +describe.skipIf(!hasTar)("system tar", () => { + it("lists and extracts the archive", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "archive-interop-")); + try { + const file = path.join(dir, "site.cloudhal"); + const content = sampleContent(); + const manifest = await writeArchiveFile(file, content); + const list = spawnSync("tar", ["-tzf", file], { encoding: "utf8" }); + expect(list.status, list.stderr).toBe(0); + expect(list.stdout.split("\n").filter(Boolean)).toEqual([ + "manifest.json", + ...manifest.files.map((f) => f.path), + ]); + const out = path.join(dir, "out"); + await mkdir(out); + const extract = spawnSync("tar", ["-xzf", file, "-C", out], { encoding: "utf8" }); + expect(extract.status, extract.stderr).toBe(0); + const read = (p: string) => readFile(path.join(out, p)); + expect(same(await read("manifest.json"), encodeManifest(manifest))).toBe(true); + expect(same(await read("project.json"), content.project as Uint8Array)).toBe(true); + expect(same(await read("ledger.ndjson"), content.ledger as Uint8Array)).toBe(true); + for (const [i, part] of content.nodes.entries()) + expect(same(await read(`nodes/${String(i + 1).padStart(4, "0")}.ndjson`), part)).toBe(true); + for (const m of MEDIA) expect(same(await read(`media/${sha256(m)}`), m)).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/test/archive-manifest.test.ts b/packages/cli/test/archive-manifest.test.ts new file mode 100644 index 00000000..b36e4d49 --- /dev/null +++ b/packages/cli/test/archive-manifest.test.ts @@ -0,0 +1,214 @@ +/** + * RFC-0039 §1 and invariant 4: `manifest.json` comes first and is checked strictly; an archive of + * a newer format, contracts schemaVersion or bundle format is refused with the CLI version needed. + */ +import { gzipSync } from "node:zlib"; +import { describe, expect, it } from "vitest"; +import type { ArchiveErrorCode } from "../src/archive/index.ts"; +import { + archiveOf, + baseFiles, + expectArchiveError, + listedAt, + manifestFor, + pattern, + type RawEntry, + rawTar, + readAll, + sha256, +} from "./archive-fixture.ts"; + +type Edit = (m: Record) => void; + +const files = baseFiles(); +const withManifest = (edit: Edit) => readAll(archiveOf(files, { manifest: edit })); +const counts = (m: Record) => m.counts as Record; +/** Edits that set one value: on the manifest, its counts, or one listed file. */ +const set = + (key: string, value: unknown): Edit => + (m) => { + m[key] = value; + }; +const setCount = + (key: string, value: unknown): Edit => + (m) => { + counts(m)[key] = value; + }; +const setFile = + (index: number, key: string, value: unknown): Edit => + (m) => { + listedAt(m, index)[key] = value; + }; +const drop = + (key: string): Edit => + (m) => { + delete m[key]; + }; + +describe("the first entry", () => { + it("must be manifest.json", async () => { + const tar = rawTar([ + ...files, + { path: "manifest.json", body: JSON.stringify(manifestFor(files)) }, + ]); + await expectArchiveError(readAll(gzipSync(tar)), "not_archive", /first entry/); + await expectArchiveError(readAll(gzipSync(new Uint8Array(10240))), "not_archive"); + await expectArchiveError(readAll(gzipSync(new Uint8Array(0))), "not_archive"); + }); + + it("must be UTF-8 JSON without a byte order mark", async () => { + const raw = (body: Uint8Array | string) => + readAll(gzipSync(rawTar([{ path: "manifest.json", body }]))); + await expectArchiveError(raw("{not json"), "invalid_manifest", /UTF-8 JSON/); + await expectArchiveError(raw(new Uint8Array([0x7b, 0xff, 0x7d])), "invalid_manifest"); + const bom = Buffer.concat([ + Buffer.from([0xef, 0xbb, 0xbf]), + Buffer.from(JSON.stringify(manifestFor(files))), + ]); + await expectArchiveError(raw(bom), "invalid_manifest"); + await expectArchiveError(raw("[]"), "not_archive"); + }); +}); + +describe("the manifest's shape", () => { + const cases: [string, Edit][] = [ + ["an unknown key", set("extra", 1)], + ["an unknown counts key", setCount("pages", 1)], + ["an unknown files key", setFile(0, "mode", 420)], + ["a missing field", drop("siteName")], + ["a missing contracts version", set("contracts", {})], + ["a contracts version 0", set("contracts", { schemaVersion: 0 })], + ["a negative count", setCount("blobs", -1)], + ["a fractional count", setCount("blobs", 0.5)], + ["a site id that is not an id", set("siteId", "../x")], + ["an empty site name", set("siteName", "")], + ["a createdAt that is not RFC 3339", set("createdAt", "2026-10-20")], + ["a bundle that is not a bundle format", set("bundle", "zip")], + ["files that are not an array", set("files", {})], + [ + "a file listed twice", + (m) => { + (m.files as unknown[]).splice(1, 0, listedAt(m, 1)); + }, + ], + [ + "files out of order", + (m) => { + (m.files as unknown[]).reverse(); + }, + ], + ["a gap in the part numbers", setFile(2, "path", "nodes/0003.ndjson")], + ["an empty part", setFile(2, "bytes", 0)], + ["an uppercase hash", setFile(0, "sha256", "A".repeat(64))], + ["a size that is not an integer", setFile(0, "bytes", "21")], + ["a media count that disagrees", setCount("media", 2)], + ["a media size that disagrees", setCount("mediaBytes", 1)], + ["mediaOmitted false", set("mediaOmitted", false)], + ["mediaOmitted with media", set("mediaOmitted", true)], + [ + "a manifest that lists itself", + (m) => { + (m.files as unknown[]).unshift({ path: "manifest.json", sha256: "0".repeat(64), bytes: 1 }); + }, + ], + ]; + it.each(cases)("refuses %s", async (_, edit) => { + await expectArchiveError(withManifest(edit), "invalid_manifest", /^manifest\.json: /); + }); + + it("echoes an unknown key only when it is short, printable ASCII", async () => { + const places: [string, (key: string) => Edit][] = [ + ["the manifest", (key) => set(key, 1)], + ["contracts", (key) => set("contracts", { schemaVersion: 1, [key]: 1 })], + ["counts", (key) => setCount(key, 1)], + ["files[0]", (key) => setFile(0, key, 1)], + ]; + const hidden = ["\u001b[2J", "\u009b2J\u009b31m", "a\u202eb", "\u0085", "two words"]; + const long = "k".repeat(2_000_000); + for (const [where, edit] of places) { + const at = `manifest.json: ${where} has an unknown key `; + const shown = await expectArchiveError(withManifest(edit("extra")), "invalid_manifest"); + expect(shown.message).toBe(`${at}"extra"`); + const cut = await expectArchiveError(withManifest(edit(long)), "invalid_manifest"); + expect(cut.message).toBe(`${at}"${"k".repeat(120)}..."`); + for (const key of hidden) { + const error = await expectArchiveError(withManifest(edit(key)), "invalid_manifest"); + expect(error.message).toBe( + `${at}(not shown: it has spaces, controls or non-ASCII characters)`, + ); + } + } + }); + + it("accepts mediaOmitted without media", async () => { + const noMedia = files.slice(0, 4); + const back = await readAll( + archiveOf(noMedia, { + manifest: (m) => { + m.mediaOmitted = true; + }, + }), + ); + expect(back.manifest.mediaOmitted).toBe(true); + expect(back.files.map((f) => f.path)).toEqual(noMedia.map((f) => f.path)); + }); + + it("accepts an empty media file", async () => { + const empty: RawEntry = { path: `media/${sha256(new Uint8Array(0))}`, body: new Uint8Array(0) }; + const media = pattern(10, 1); + const entries = [ + ...files.slice(0, 4), + ...[empty, { path: `media/${sha256(media)}`, body: media }].sort((a, b) => + a.path < b.path ? -1 : 1, + ), + ]; + expect((await readAll(archiveOf(entries))).files).toHaveLength(6); + }); +}); + +describe("versions (invariant 4)", () => { + const notArchive: [string, Edit][] = [ + ["no format", drop("format")], + ["another format", set("format", "zip")], + ["the bundle format as the archive format", set("format", "cloudhal.site-bundle@1")], + ["a format version 0", set("format", "cloudhal.site-archive@0")], + ]; + it.each(notArchive)("refuses %s as not an archive", async (_, edit) => { + await expectArchiveError(withManifest(edit), "not_archive", /not a Cloudhal site archive/); + }); + + const newer: [string, Edit][] = [ + [ + "archive format 2 of any shape", + (m) => { + for (const key of Object.keys(m)) if (key !== "createdBy") delete m[key]; + m.format = "cloudhal.site-archive@2"; + m.parts = ["whatever @2 holds"]; + }, + ], + ["contracts schemaVersion 2", set("contracts", { schemaVersion: 2 })], + ["bundle format 2", set("bundle", "cloudhal.site-bundle@2")], + ]; + it.each(newer)("refuses %s, naming the CLI version", async (_, edit) => { + await expectArchiveError( + withManifest((m) => { + m.createdBy = "9.9.9"; + edit(m); + }), + "unsupported_version" as ArchiveErrorCode, + /newer Cloudhal .*; update @cloudhal\/cli to 9\.9\.9 or later$/, + ); + }); + + it("does not echo a createdBy that is not a version", async () => { + const error = await expectArchiveError( + withManifest((m) => { + m.format = "cloudhal.site-archive@3"; + m.createdBy = "1.0\u001b[2J"; + }), + "unsupported_version", + /to a newer version$/, + ); + expect(error.message).not.toContain("\u001b"); + }); +}); diff --git a/packages/cli/test/archive-parts.test.ts b/packages/cli/test/archive-parts.test.ts new file mode 100644 index 00000000..15c7369f --- /dev/null +++ b/packages/cli/test/archive-parts.test.ts @@ -0,0 +1,89 @@ +/** RFC-0039 §1: NDJSON parts of at most 8 MB, one row per line. */ +import { describe, expect, it } from "vitest"; +import { encodeNdjson, PART_SPLIT_BYTES, splitNdjson } from "../src/archive/index.ts"; +import { countRows } from "../src/archive/parts.ts"; +import { expectArchiveError } from "./archive-fixture.ts"; + +const text = (part: Uint8Array) => Buffer.from(part).toString("utf8"); +/** A row whose NDJSON line is exactly `bytes` long: `"xxx…"` plus the newline. */ +const rowOf = (bytes: number) => "x".repeat(bytes - 3); + +describe("splitNdjson", () => { + it("starts a new part when the next line would cross the limit", () => { + // Lines of 32 bytes: two fill a 64-byte part exactly, a third starts the next one. + const two = splitNdjson([rowOf(32), rowOf(32)], { partBytes: 64 }); + expect(two.map((p) => p.length)).toEqual([64]); + const three = splitNdjson([rowOf(32), rowOf(32), rowOf(1 + 3)], { partBytes: 64 }); + expect(three.map((p) => p.length)).toEqual([64, 4]); + const over = splitNdjson([rowOf(32), rowOf(33)], { partBytes: 64 }); + expect(over.map((p) => p.length)).toEqual([32, 33]); + }); + + it("puts a line larger than a part alone in its own part", () => { + const parts = splitNdjson([rowOf(10), rowOf(100), rowOf(10), rowOf(10)], { partBytes: 64 }); + expect(parts.map((p) => p.length)).toEqual([10, 100, 20]); + expect(splitNdjson([rowOf(100), rowOf(10)], { partBytes: 64 }).map((p) => p.length)).toEqual([ + 100, 10, + ]); + }); + + it("refuses a line above the reader's part cap, naming the row", async () => { + await expectArchiveError( + Promise.resolve().then(() => + splitNdjson([rowOf(10), rowOf(200)], { partBytes: 64, maxLineBytes: 199 }), + ), + "too_large", + /row 1 is 200 bytes/, + ); + }); + + it("keeps row order, counts bytes not characters, and ends every part with a newline", () => { + const rows = [{ a: "ü".repeat(20) }, { b: "
 and \ud800" }, ["x"], 7, null, "é"]; + const parts = splitNdjson(rows, { partBytes: 48 }); + expect(parts.length).toBeGreaterThan(1); + for (const part of parts) { + expect(part.length).toBeGreaterThan(0); + expect(part[part.length - 1]).toBe(0x0a); + } + const lines = rows.map((r) => `${JSON.stringify(r)}\n`); + expect(parts.map(text).join("")).toBe(lines.join("")); + // ü is two bytes: the first line is 49 bytes, above 48, and so alone. + expect(parts[0]?.length).toBe(Buffer.byteLength(lines[0] as string)); + expect(parts.reduce((n, p) => n + countRows(p, "p"), 0)).toBe(rows.length); + expect(splitNdjson([])).toEqual([]); + expect(() => splitNdjson([undefined])).toThrow(/row 0 has no JSON form/); + }); + + it("splits at the real 8 MiB threshold", () => { + const half = PART_SPLIT_BYTES / 2; + expect(splitNdjson([rowOf(half), rowOf(half)]).map((p) => p.length)).toEqual([ + PART_SPLIT_BYTES, + ]); + expect(splitNdjson([rowOf(half), rowOf(half + 1)]).map((p) => p.length)).toEqual([ + half, + half + 1, + ]); + }); +}); + +describe("encodeNdjson and countRows", () => { + it("writes one file and refuses one above the cap", async () => { + expect(text(encodeNdjson([{ a: 1 }, { b: 2 }]))).toBe('{"a":1}\n{"b":2}\n'); + expect(encodeNdjson([]).length).toBe(0); + await expectArchiveError( + Promise.resolve().then(() => encodeNdjson([rowOf(10), rowOf(10)], 19)), + "too_large", + ); + }); + + it("counts newlines and refuses what JSON.stringify never writes", async () => { + const enc = (s: string) => new TextEncoder().encode(s); + expect(countRows(enc("{}\n[]\n"), "nodes/0001.ndjson")).toBe(2); + for (const bad of ["", "{}", "{}\n\n", "\n{}\n"]) + await expectArchiveError( + Promise.resolve().then(() => countRows(enc(bad), "nodes/0001.ndjson")), + "damaged", + /^nodes\/0001\.ndjson: /, + ); + }); +}); diff --git a/packages/cli/test/archive-roundtrip.test.ts b/packages/cli/test/archive-roundtrip.test.ts new file mode 100644 index 00000000..6dfa7c26 --- /dev/null +++ b/packages/cli/test/archive-roundtrip.test.ts @@ -0,0 +1,310 @@ +/** + * RFC-0039 §1 and invariants 3, 5 and 7: what the writer writes, the reader reads back byte for + * byte, the same content gives the same bytes, and content the reader would refuse is never + * written. + */ +import { createReadStream } from "node:fs"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { gunzipSync } from "node:zlib"; +import { CONTRACT_SCHEMA_VERSION } from "@cloudhal/core-contracts"; +import { SITE_BUNDLE_FORMAT } from "@cloudhal/core-ts"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + type ArchiveContent, + ArchiveError, + archiveStream, + readArchive, + SITE_ARCHIVE_FORMAT, + splitNdjson, + writeArchiveFile, +} from "../src/archive/index.ts"; +import { encodeManifest } from "../src/archive/manifest.ts"; +import { + collect, + expectArchiveError, + MEDIA, + memoryMedia, + NODE_ROWS, + readAll, + SITE, + same, + sampleContent, + sha256, + utf8, + writeBytes, +} from "./archive-fixture.ts"; + +let dir: string; +beforeAll(async () => { + dir = await mkdtemp(path.join(os.tmpdir(), "archive-roundtrip-")); +}); +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +/** The fixed order of RFC-0039 §1 for the sample content. */ +const ORDER = [ + "project.json", + ...sampleContent().nodes.map((_, i) => `nodes/${String(i + 1).padStart(4, "0")}.ndjson`), + "snapshots/0001.ndjson", + "blobs/0001.ndjson", + ...MEDIA.map((m) => `media/${sha256(m)}`).sort(), + "ledger.ndjson", +]; + +describe("round trip", () => { + it("reads back the manifest first and every file in the fixed order", async () => { + const content = sampleContent(); + const { manifest, bytes } = archiveStream(content); + const archive = await collect(bytes); + expect(ORDER.filter((p) => p.startsWith("nodes/")).length).toBeGreaterThan(1); + for (const chunk of [1, 333, 64 * 1024]) { + const back = await readAll(archive, { chunk }); + expect(back.manifest).toEqual(manifest); + expect(back.files.map((f) => f.path)).toEqual(ORDER); + const byPath = new Map(back.files.map((f) => [f.path, f.bytes])); + expect(same(byPath.get("project.json"), content.project as Uint8Array)).toBe(true); + expect(same(byPath.get("ledger.ndjson"), content.ledger as Uint8Array)).toBe(true); + const nodes = Buffer.concat( + ORDER.filter((p) => p.startsWith("nodes/")).map((p) => byPath.get(p) as Buffer), + ); + expect(same(nodes, Buffer.concat(content.nodes))).toBe(true); + for (const m of MEDIA) expect(same(byPath.get(`media/${sha256(m)}`), m)).toBe(true); + } + }); + + it("writes the RFC manifest with counts it computes", async () => { + const { manifest } = archiveStream(sampleContent()); + expect(Object.keys(manifest)).toEqual([ + "format", + "contracts", + "bundle", + "siteId", + "siteName", + "createdBy", + "createdAt", + "counts", + "files", + ]); + expect(manifest).toMatchObject({ + format: SITE_ARCHIVE_FORMAT, + contracts: { schemaVersion: CONTRACT_SCHEMA_VERSION }, + bundle: SITE_BUNDLE_FORMAT, + ...SITE, + counts: { + nodes: NODE_ROWS.length, + snapshots: 2, + blobs: 1, + media: MEDIA.length, + mediaBytes: MEDIA.reduce((n, m) => n + m.length, 0), + }, + }); + const text = Buffer.from(encodeManifest(manifest)).toString("utf8"); + expect(text.startsWith('{\n "format": "cloudhal.site-archive@1",\n')).toBe(true); + expect(text.endsWith("}\n")).toBe(true); + }); + + it("writes mediaOmitted archives without media, and minimal ones", async () => { + const omitted = sampleContent({ media: undefined, mediaOmitted: true }); + const back = await readAll(await writeBytes(omitted)); + expect(back.manifest.mediaOmitted).toBe(true); + expect(back.manifest.counts).toMatchObject({ media: 0, mediaBytes: 0 }); + expect(back.files.some((f) => f.path.startsWith("media/"))).toBe(false); + const minimal: ArchiveContent = { + ...SITE, + nodes: splitNdjson([{ id: "n" }]), + snapshots: [], + blobs: [], + }; + const small = await readAll(await writeBytes(minimal)); + expect(small.files.map((f) => f.path)).toEqual(["nodes/0001.ndjson"]); + expect("mediaOmitted" in small.manifest).toBe(false); + }); +}); + +describe("deterministic bytes (invariant 7)", () => { + it("gives identical bytes for the same content, whatever the media order or chunking", async () => { + const first = await writeBytes(sampleContent()); + expect(same(await writeBytes(sampleContent()), first)).toBe(true); + const reordered = sampleContent({ media: [...MEDIA].reverse().map((m) => memoryMedia(m, 1)) }); + expect(same(await writeBytes(reordered), first)).toBe(true); + // Rows are written in the caller's order: sorting them is the caller's job. + const nodes = splitNdjson([...NODE_ROWS].reverse(), { partBytes: 512 }); + expect(same(await writeBytes(sampleContent({ nodes })), first)).toBe(false); + }); + + it("writes a fixed gzip header and a platform-independent tar stream", async () => { + const archive = await writeBytes(sampleContent()); + expect(Buffer.from(archive.subarray(0, 10)).toString("hex")).toBe("1f8b08000000000000ff"); + const tar = gunzipSync(archive); + expect(tar.length % 10240).toBe(0); + // A golden hash of the uncompressed tar (gzip bytes differ between zlib builds, the tar does + // not). Update it on purpose, only when the format changes. + expect(sha256(tar)).toBe("81d8e9868fa915e24229bc3c4fc1a1acdfbfc41fd862fc335aa583b85e16a716"); + }); +}); + +describe("writer refusals", () => { + const bad: [string, Partial, string][] = [ + [ + "a media hash that is not hex", + { media: [{ ...memoryMedia(MEDIA[0] as Uint8Array), sha256: "A".repeat(64) }] }, + "invalid_input", + ], + [ + "duplicate media", + { media: [memoryMedia(MEDIA[0] as Uint8Array), memoryMedia(MEDIA[0] as Uint8Array)] }, + "invalid_input", + ], + ["media with mediaOmitted", { mediaOmitted: true }, "invalid_input"], + ["an empty part", { nodes: [new Uint8Array(0)] }, "invalid_input"], + ["a part without a final newline", { nodes: [utf8('{"id":"n"}')] }, "invalid_input"], + ["an empty project", { project: new Uint8Array(0) }, "invalid_input"], + ["10000 parts", { blobs: Array.from({ length: 10_000 }, () => utf8("{}\n")) }, "too_large"], + ["a createdAt that is not RFC 3339", { createdAt: "yesterday" }, "invalid_input"], + ["a createdBy that is not a version", { createdBy: "1.0 \u001b[2J" }, "invalid_input"], + ["a siteId that is not an id", { siteId: "../site" }, "invalid_input"], + ]; + + it.each(bad)("refuses %s before writing anything", async (_, overrides, code) => { + const content = sampleContent(overrides); + expect(() => archiveStream(content)).toThrow(ArchiveError); + const target = path.join(dir, "refused.cloudhal"); + await expectArchiveError(writeArchiveFile(target, content), code as "invalid_input"); + expect(await readdir(dir)).toEqual([]); + }); + + it("refuses what a reader with the same limits would refuse", async () => { + const content = sampleContent(); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { manifestBytes: 500 } })), + "too_large", + ); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { partBytes: 100 } })), + "too_large", + ); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { mediaBytes: 1000 } })), + "too_large", + ); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { entries: 5 } })), + "too_large", + ); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { totalBytes: 50_000 } })), + "too_large", + ); + await expectArchiveError( + Promise.resolve().then(() => archiveStream(content, { limits: { entries: 0 } })), + "invalid_input", + ); + }); +}); + +describe("media verified while writing", () => { + const target = () => path.join(dir, "site.cloudhal"); + const withMedia = (open: () => AsyncIterable) => { + const m = MEDIA[0] as Uint8Array; + return sampleContent({ media: [{ sha256: sha256(m), bytes: m.length, open }] }); + }; + + it("aborts a longer body without reading it all", async () => { + let pulled = 0; + let closed = false; + async function* endless() { + try { + for (;;) { + pulled++; + yield new Uint8Array(1024); + } + } finally { + closed = true; + } + } + await expectArchiveError(writeArchiveFile(target(), withMedia(endless)), "damaged", /longer/); + expect(pulled).toBeLessThan(10); + expect(closed).toBe(true); + expect(await readdir(dir)).toEqual([]); + }); + + it("refuses a short or wrong body, leaving an existing file as it was", async () => { + await writeFile(target(), "old"); + const m = MEDIA[0] as Uint8Array; + const wrong = Buffer.from(m); + wrong[0] = (wrong[0] as number) ^ 1; + for (const [body, text] of [ + [m.subarray(1), /has 2999 of 3000 bytes/], + [wrong, /SHA-256/], + ] as const) { + const error = await expectArchiveError( + writeArchiveFile( + target(), + withMedia(() => yieldOnce(body)), + ), + "damaged", + text, + ); + expect(error.path).toBe(`media/${sha256(m)}`); + expect(await readdir(dir)).toEqual(["site.cloudhal"]); + expect(await readFile(target(), "utf8")).toBe("old"); + } + await rm(target()); + }); + + it("ends a failed stream before the gzip trailer, so its bytes never read as an archive", async () => { + /** What the stream emitted before a media source one byte short failed it. */ + const partial = async (media: Uint8Array) => { + const short = { + sha256: sha256(media), + bytes: media.length, + open: () => yieldOnce(media.subarray(1)), + }; + const { bytes } = archiveStream(sampleContent({ media: [short] })); + const chunks: Uint8Array[] = []; + await expectArchiveError( + (async () => { + for await (const chunk of bytes) chunks.push(chunk); + })(), + "damaged", + ); + return Buffer.concat(chunks); + }; + // zlib had emitted nothing yet: an empty file. + const empty = await partial(MEDIA[0] as Uint8Array); + expect(empty.length).toBe(0); + await expectArchiveError(readAll(empty), "not_archive", /it is empty/); + // zlib had emitted part of the stream: no trailer, so it is truncated. + const cut = await partial(noise(256 * 1024)); + expect(cut.length).toBeGreaterThan(0); + await expectArchiveError(readAll(cut), "truncated"); + }); + + it("writes a file that reads back from disk", async () => { + const manifest = await writeArchiveFile(target(), sampleContent()); + let read: unknown; + for await (const item of readArchive(createReadStream(target()))) + if (item.type === "manifest") read = item.manifest; + expect(read).toEqual(manifest); + expect(await readdir(dir)).toEqual(["site.cloudhal"]); + await rm(target()); + }); +}); + +/** Bytes that do not compress (a fixed LCG), so gzip emits output as it goes. */ +function noise(length: number): Uint8Array { + const out = new Uint8Array(length); + let x = 1; + for (let i = 0; i < length; i++) { + x = (Math.imul(x, 1103515245) + 12345) >>> 0; + out[i] = x >>> 24; + } + return out; +} + +async function* yieldOnce(bytes: Uint8Array): AsyncGenerator { + yield bytes; +} diff --git a/packages/cli/test/archive-tar.test.ts b/packages/cli/test/archive-tar.test.ts new file mode 100644 index 00000000..d8f9dd0f --- /dev/null +++ b/packages/cli/test/archive-tar.test.ts @@ -0,0 +1,160 @@ +/** RFC-0039 §1: the ustar blocks Cloudhal writes, and the streaming reader's refusals. */ +import { describe, expect, it } from "vitest"; +import { parseTarHeader, TAR_RECORD, tarEnd, tarHeader } from "../src/archive/tar.ts"; +import { readTar } from "../src/archive/tar-read.ts"; +import { + chunked, + collect, + expectArchiveError, + pattern, + type RawEntry, + rawTar, + same, + USTAR, +} from "./archive-fixture.ts"; + +const ascii = (bytes: Uint8Array, start: number, length: number) => + Buffer.from(bytes.subarray(start, start + length)).toString("latin1"); + +async function entriesOf(tar: Uint8Array, chunk = 512, maxEntries = 100) { + const out: { path: string; body: Buffer }[] = []; + for await (const entry of readTar(chunked(tar, chunk), { maxEntries })) + out.push({ path: entry.path, body: await collect(entry.body) }); + return out; +} + +const drain = (tar: Uint8Array, maxEntries = 100) => entriesOf(tar, 512, maxEntries); + +describe("tar header bytes", () => { + it("writes fixed fields, an octal size and an unsigned checksum", () => { + const h = tarHeader("nodes/0001.ndjson", 1234); + expect(h.length).toBe(512); + expect(ascii(h, 0, 100)).toBe(`nodes/0001.ndjson${"\0".repeat(83)}`); + expect(ascii(h, 100, 8)).toBe("0000644\0"); + expect(ascii(h, 108, 8)).toBe("0000000\0"); + expect(ascii(h, 116, 8)).toBe("0000000\0"); + expect(ascii(h, 124, 12)).toBe(`${(1234).toString(8).padStart(11, "0")}\0`); + expect(ascii(h, 136, 12)).toBe("00000000000\0"); + expect(ascii(h, 156, 1)).toBe("0"); + expect(ascii(h, 157, 100)).toBe("\0".repeat(100)); + expect(ascii(h, 257, 8)).toBe("ustar\u000000"); + expect(ascii(h, 265, 64)).toBe("\0".repeat(64)); // uname, gname + expect(ascii(h, 345, 155)).toBe("\0".repeat(155)); // prefix + let sum = 0; + for (let i = 0; i < 512; i++) sum += i >= 148 && i < 156 ? 0x20 : (h[i] as number); + expect(ascii(h, 148, 8)).toBe(`${sum.toString(8).padStart(6, "0")}\0 `); + expect(parseTarHeader(h)).toEqual({ path: "nodes/0001.ndjson", size: 1234 }); + }); + + it("ends with two zero blocks padded to a 10240-byte record", () => { + for (const written of [0, 512, 9216, 9728, 10240, 20480 + 512]) { + const end = tarEnd(written); + expect((written + end.length) % TAR_RECORD).toBe(0); + expect(end.length).toBeGreaterThanOrEqual(1024); + expect(end.every((b) => b === 0)).toBe(true); + } + }); + + it("refuses names it cannot write as a plain ustar name", () => { + expect(() => tarHeader("x".repeat(100), 1)).toThrow(/1-99 printable/); + expect(() => tarHeader("a b", 1)).toThrow(/printable/); + expect(() => tarHeader("", 1)).toThrow(/printable/); + }); +}); + +describe("readTar", () => { + const sizes = [0, 1, 511, 512, 513, 4096 + 17]; + const entries: RawEntry[] = sizes.map((size, i) => ({ + path: `f${i}`, + body: pattern(size, i), + })); + + it("reads bodies of every size in any chunking", async () => { + const tar = rawTar(entries, { tail: new Uint8Array(4096) }); + for (const chunk of [1, 7, 512, 64 * 1024]) { + const got = await entriesOf(tar, chunk); + expect(got.map((e) => e.path)).toEqual(entries.map((e) => e.path)); + for (const [i, e] of got.entries()) + expect(same(e.body, entries[i]?.body as Uint8Array)).toBe(true); + } + }); + + it("skips bodies that are not read and refuses a stale body", async () => { + const tar = rawTar(entries); + const paths: string[] = []; + let stale: AsyncIterable | undefined; + for await (const entry of readTar(chunked(tar, 100), { maxEntries: 100 })) { + paths.push(entry.path); + if (entry.path === "f3") stale = entry.body; + } + expect(paths).toEqual(entries.map((e) => e.path)); + await expectArchiveError(collect(stale as AsyncIterable), "invalid_input", /after/); + }); + + it("refuses links, devices, directories, FIFOs and extension headers", async () => { + for (const flag of ["1", "2", "3", "4", "5", "6", "7", "x", "g", "L", "K", "A"]) { + const tar = rawTar([ + { + path: "f", + body: "x", + patch: (h) => { + h[USTAR.typeflag] = flag.charCodeAt(0); + }, + }, + ]); + await expectArchiveError(drain(tar), "unsafe_entry", /regular files only/); + } + }); + + it("refuses prefixes, link names, other magic, base-256 sizes and odd names", async () => { + const patches: ((h: Uint8Array) => void)[] = [ + (h) => h.fill(0x61, USTAR.prefix, USTAR.prefix + 1), + (h) => h.fill(0x61, USTAR.linkname, USTAR.linkname + 1), + (h) => h.set(Buffer.from("ustar \0"), USTAR.magic), // GNU tar's old format + (h) => h.set(Buffer.from("01"), USTAR.version), + (h) => h.fill(0x80, USTAR.size, USTAR.size + 1), + (h) => h.set(Buffer.from("12x"), USTAR.size), + (h) => h.set(Buffer.from("f\0junk"), USTAR.name), + (h) => h.set(Buffer.from("f\u001b[31m"), USTAR.name), + (h) => h.fill(0x61, USTAR.name, USTAR.name + 100), + (h) => h.fill(0, USTAR.name, USTAR.name + 100), + ]; + for (const patch of patches) { + const error = await expectArchiveError( + drain(rawTar([{ path: "f", body: "x", patch }])), + "unsafe_entry", + ); + expect(error.message).not.toContain("\u001b"); + } + }); + + it("refuses a bad checksum as damage", async () => { + const tar = rawTar([{ path: "nodes/0001.ndjson", body: "x" }]); + tar[3] = (tar[3] as number) ^ 1; // a flipped name bit, checksum unchanged + await expectArchiveError(drain(tar), "damaged", /checksum/); + }); + + it("checks the end of the archive", async () => { + const one: RawEntry[] = [{ path: "f", body: "abc" }]; + const body = rawTar(one, { end: false }); + const zero = new Uint8Array(512); + const cases: [Uint8Array, string][] = [ + [Buffer.concat([body, zero]), "truncated"], + [Buffer.concat([body, zero, rawTar(one, { end: false })]), "damaged"], + [body, "truncated"], + [body.subarray(0, 700), "truncated"], + [body.subarray(0, 300), "truncated"], + [rawTar(one, { tail: Buffer.from([0, 0, 1]) }), "damaged"], + [rawTar(one, { tail: new Uint8Array(TAR_RECORD + 1) }), "damaged"], + [new Uint8Array(0), "not_archive"], + ]; + for (const [tar, code] of cases) + await expectArchiveError(drain(tar), code as "truncated" | "damaged" | "not_archive"); + expect(await drain(rawTar(one, { tail: new Uint8Array(TAR_RECORD) }))).toHaveLength(1); + }); + + it("caps the number of entries", async () => { + await expectArchiveError(drain(rawTar(entries), 3), "too_large", /more than 3 entries/); + expect(await drain(rawTar(entries), entries.length)).toHaveLength(entries.length); + }); +});