From 6397baaa937f1285377bccfbd97bca4f9df5e7d0 Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:27:48 +0300 Subject: [PATCH 1/6] test(cli): reference links through the real EntryFields and locale middleware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds the starter under prefix-except-default and prefix-all with a Turkish referrer and a fallback card: the venue links /tr/places/… exactly once, the fallback the default locale's member. Reverting EntryFields.astro to href={c.url} fails it with /tr/tr/…. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- packages/cli/test/reference-links.test.ts | 152 ++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 packages/cli/test/reference-links.test.ts diff --git a/packages/cli/test/reference-links.test.ts b/packages/cli/test/reference-links.test.ts new file mode 100644 index 00000000..842e73df --- /dev/null +++ b/packages/cli/test/reference-links.test.ts @@ -0,0 +1,152 @@ +/** + * RFC-0045 §4 through the real pipeline: the starter's `EntryFields.astro` draws a reference's + * card by its graph path, and the locale middleware (`localizeLinks`) gives it the page's locale + * exactly once. Built with `cloudhal build` under `prefix-except-default` and `prefix-all`: a + * Turkish referrer links its venue's Turkish member at `/tr/places/…` (never `/tr/tr/…`), and a + * fallback card (`default-with-notice`) links the default locale's member. The types stay refused + * in production; the test enables them on its own store, as the shared suites do. + */ +import { spawn } from "node:child_process"; +import { mkdtemp, readFile, rm, symlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { ContentEntry, Site } from "@cloudhal/core-contracts"; +import { GATES_ENFORCED, nodeHash, SiteGraphStore } from "@cloudhal/core-ts"; +import { openLocalStore } from "@cloudhal/core-ts/local"; +import { afterAll, describe, expect, it } from "vitest"; +import { init, readProjectConfig, snapshot } from "../src/index.ts"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const STARTER_NODE_MODULES = path.resolve(here, "../../../starters/blog/node_modules"); +const BIN = path.resolve(here, "../bin/cloudhal.ts"); +const author = { kind: "human", id: "user_01HZX8K3Q9CD", name: "Halil" }; + +async function runBuild(projectDir: string): Promise<{ code: number | null; output: string }> { + const { VITEST: _v, NODE_ENV: _n, ...env } = process.env; + const child = spawn(process.execPath, [BIN, "build"], { + cwd: projectDir, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout?.on("data", (c: Buffer) => (output += c.toString())); + child.stderr?.on("data", (c: Buffer) => (output += c.toString())); + const code = await new Promise((resolve) => child.once("exit", resolve)); + return { code, output }; +} + +/** Places with a Turkish member (`hall`/`salon`) and without (`pier`), and events naming them. */ +async function addReferences( + projectDir: string, + routing: "prefix-except-default" | "prefix-all", +): Promise { + const cfg = await readProjectConfig(projectDir); + const local = await openLocalStore(path.join(projectDir, cfg.dataDir)); + const store = new SiteGraphStore({ + sql: local.sql, + blobs: local.blobs, + gates: GATES_ENFORCED, + fieldTypes: { supported: () => true }, + }); + try { + const site = (await store.getSite(cfg.siteId)) as Site; + // the seed's required category applies to every collection: reuse the seeded post's terms + const [hello] = await store.listContentEntries(cfg.siteId, "posts"); + await store.updateSite({ + ...site, + locales: ["en", "tr"], + localization: { ...site.localization, routing, fallback: "default-with-notice" }, + collections: { + ...site.collections, + places: {}, + events: { + fields: [ + { + key: "venue", + type: "reference", + labels: { en: { label: "Venue" }, tr: { label: "Mekan" } }, + localized: false, + collections: ["places"], + }, + ], + }, + }, + }); + const make = (collection: string, slug: string, over: Partial) => + store.createContentEntry({ + siteId: cfg.siteId, + collection, + slug, + locale: "en", + status: "published", + title: `The ${slug}`, + body: { format: "markdown", value: `A page about the ${slug}, with enough words to read.` }, + fields: {}, + tags: [], + ...(hello?.terms ? { terms: hello.terms } : {}), + author, + ...over, + } as never) as Promise; + const translate = async (of: ContentEntry, slug: string) => + make(of.collection, slug, { + locale: "tr", + title: `Türkçe ${slug}`, + translation: { + group: of.id, + method: "human", + source: { id: of.id, locale: of.locale, rev: of.rev, hash: await nodeHash(of) }, + }, + }); + const hall = await make("places", "hall", {}); + await translate(hall, "salon"); + const pier = await make("places", "pier", {}); + await translate(await make("events", "launch", { fields: { venue: hall.id } }), "lansman"); + await translate(await make("events", "gig", { fields: { venue: pier.id } }), "konser"); + } finally { + await local.close(); + } + await snapshot({ projectDir, label: "references", publish: true }); +} + +/** The hrefs inside a page's typed-fields list. */ +const fieldLinks = (html: string) => + [ + ...(html.match(/
/)?.[0] ?? "").matchAll(/href="([^"]*)"/g), + ].map((m) => m[1]); + +describe.each(["prefix-except-default", "prefix-all"] as const)( + "reference links in built pages (%s)", + (routing) => { + let tmp: string; + afterAll(async () => { + if (tmp) await rm(tmp, { recursive: true, force: true }); + }); + + it("links each card once in the page's locale, the fallback in the default locale", async () => { + tmp = await mkdtemp(path.join(os.tmpdir(), "cloudhal-ref-links-")); + const projectDir = path.join(tmp, "site"); + await init({ dir: projectDir, name: "Reference Links" }); + await symlink(STARTER_NODE_MODULES, path.join(projectDir, "node_modules"), "dir"); + await addReferences(projectDir, routing); + const { code, output } = await runBuild(projectDir); + expect(code, output.slice(-4000)).toBe(0); + const read = (...p: string[]) => readFile(path.join(projectDir, "dist", ...p), "utf8"); + const en = routing === "prefix-all" ? ["en"] : []; + // a Turkish referrer: the venue's Turkish member, prefixed once + expect(fieldLinks(await read("tr", "events", "lansman", "index.html"))).toEqual([ + "/tr/places/salon", + ]); + // no Turkish member: the default locale's, through `default-with-notice` + expect(fieldLinks(await read("tr", "events", "konser", "index.html"))).toEqual([ + `${en.map((l) => `/${l}`).join("")}/places/pier`, + ]); + // the default locale's own page + expect(fieldLinks(await read(...en, "events", "launch", "index.html"))).toEqual([ + `${en.map((l) => `/${l}`).join("")}/places/hall`, + ]); + const all = await read("tr", "events", "lansman", "index.html"); + expect(all).not.toMatch(/href="\/(en|tr)\/(en|tr)\//); + }); + }, +); From 9aa67d3944a174e584bc7cebed474a46f1fb7f0c Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:38:27 +0300 Subject: [PATCH 2/6] feat(core-ts): purge referrers from the snapshots' bodies (RFC-0045 step 3b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - references/purge.ts: the purge planner's referrer closure. Seeds are the groups of changed entries and of entries the asset rule reached; two hops of referrers (with every member of each referrer's group); "used by" from resolved values, before and after. Reads only the from/to bodies: one hash-keyed cache, one bundle (to's, else from's) streamed and cut to what the closure needs, then the missing hashes. Byte and node budgets are checked before each parse; past them, or past maxReads/maxTags, or with any field-operation marker on either site node, it purges everything. Sites without reference definitions skip it (no bundle read). - readSnapshotBundle takes maxBytes / maxNodes / keep (BundleBudgetError); the store and SnapshotReader pass them. No new tags. - references/release-purge.ts: release runs re-plan their purge at run time, from the pointer they moved; RELEASE_EXECUTOR_COST 16 → 18. - planCachePurge stays under its baseline (diffNodes, countedReader, ancestry helpers). - tests: every purge case of the RFC's Renderer test plan, and the release re-plan in the shared release suite (SQLite, D1). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/rfc45-step3b-purges.md | 21 ++ ...ctions-stage-4-references-and-repeaters.md | 7 + packages/core-ts/src/bundles.ts | 40 +- packages/core-ts/src/cache-purge-assets.ts | 12 +- packages/core-ts/src/cache-purge.ts | 134 ++++--- packages/core-ts/src/references/purge.ts | 262 +++++++++++++ .../core-ts/src/references/release-purge.ts | 32 ++ packages/core-ts/src/release-executor.ts | 29 +- packages/core-ts/src/store.ts | 8 +- packages/core-ts/src/testing-releases.ts | 53 ++- .../core-ts/test/references-purge.test.ts | 347 ++++++++++++++++++ 11 files changed, 874 insertions(+), 71 deletions(-) create mode 100644 .changeset/rfc45-step3b-purges.md create mode 100644 packages/core-ts/src/references/purge.ts create mode 100644 packages/core-ts/src/references/release-purge.ts create mode 100644 packages/core-ts/test/references-purge.test.ts diff --git a/.changeset/rfc45-step3b-purges.md b/.changeset/rfc45-step3b-purges.md new file mode 100644 index 00000000..ff6eb090 --- /dev/null +++ b/.changeset/rfc45-step3b-purges.md @@ -0,0 +1,21 @@ +--- +"@cloudhal/core-ts": patch +--- + +RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothing changes for sites without reference definitions: their purge plans are the same, and no bundle is read. + +- **The referrer closure** (`references/purge.ts`, called by `planCachePurge`). + - The seeds are the groups of every changed, added or removed entry, and of every entry whose shown asset changed. + - It purges the referrers of those groups, two hops deep, with every language version of each referrer. + - It also purges every member of each group a changed entry names, before or after ("used by"). + - Values are resolved, so a translation that only inherits counts. + - It reads only the `from` and `to` snapshots' bodies, never the working copy or `entry_refs`. +- **Reads and budgets.** One hash-keyed cache: one bundle (`to`'s, else `from`'s), streamed and cut down as each shard is parsed, then the missing hashes. A passed budget purges everything: + - bytes (`maxBodyBytes`, 32 MB); + - nodes (`maxNodes`, 50,000); + - reads (`maxReads`, shard reads included); + - tags (`maxTags`). + + So does any field-operation marker on either snapshot's site node. +- **`readSnapshotBundle`** (`SnapshotReader.readSnapshotBundle`, `SiteGraphStore.readSnapshotBundle`, `readSnapshotBundle` in bundles) takes optional `maxBytes`, `maxNodes` and `keep`. A passed budget throws `BundleBudgetError` before the shard is parsed. No new cache tags. +- **Release runs re-plan their purge at run time**, from the pointer they moved to the snapshot they made, instead of sending the plan stored at pin time (`references/release-purge.ts`). `RELEASE_EXECUTOR_COST` is 18. diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index b2bb0163..c66220f3 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -1210,3 +1210,10 @@ Rollout step 2 shipped in two PRs: 2a, store and integrity, and 2b, the consumer - **`ReferencedEntry.path`** (§4): the member's graph path (`entryPath`, no locale prefix). Themes link with it, so the locale middleware localizes it once, like every other entry link. `url` stays the canonical URL, already localized, and absolute only on a locale's own domain. - **`referencedBy` follows each page** (§4): a translation resolves with its same-site original while the view shows that original, and without it otherwise, as its page reads it through the gated `entryById`. Ties on date order by id, so both sources agree on order and on what `limit` keeps. + +### Amended in 3b (2026-11-12) + +- **Seeds purge their own groups' members** (§2 Purges, point 1). A translation inheriting a changed original's reference shows a new card, but it is neither a referrer of the seed nor changed itself. +- **The marker rule follows the short-circuit.** A field-operation marker purges everything only on sites whose `from` or `to` site node defines a reference. Sites without references keep today's purge, as the short-circuit promises. A marker's own publish changes the site node, which already purges the site. +- **The node budget** (`maxNodes`) counts the entry bodies the closure holds across both snapshots, and the node count of the one bundle it reads. +- **The release re-plan** plans from the pointer the batch moved (`from`, the live snapshot it replaced) to the new snapshot. That is `baseSnapshotId` whenever something was live. A first publish purges everything, as before. `RELEASE_EXECUTOR_COST` rises from 16 to 18 (two `getSnapshot` reads); bundle shards and node bodies are blob reads, bounded by `maxReads`. diff --git a/packages/core-ts/src/bundles.ts b/packages/core-ts/src/bundles.ts index b5614ecb..47e976d6 100644 --- a/packages/core-ts/src/bundles.ts +++ b/packages/core-ts/src/bundles.ts @@ -58,18 +58,41 @@ export async function writeSnapshotBundle( } /** - * The snapshot's bodies by node hash, or `undefined` unless every shard is there and matches the - * snapshot (version, snapshot id, the refs' hashes in order). Never throws for a bad bundle: the - * caller falls back to `nodes/.json`. + * RFC-0045 §2 "Purges": a streaming read. Budgets are checked before each shard is parsed, so a + * bundle too large for the isolate fails loudly (`BundleBudgetError`) instead of running it out + * of memory; `keep` reduces each body as its shard is parsed (`undefined` drops it). + */ +export interface BundleReadOptions { + /** More shard bytes than this: `BundleBudgetError`, before the shard that would pass it. */ + readonly maxBytes?: number | undefined; + /** A snapshot of more nodes than this: `BundleBudgetError`, before any shard is read. */ + readonly maxNodes?: number | undefined; + /** What to keep of each body (default: the body). */ + readonly keep?: ((body: unknown, ref: NodeRef) => unknown) | undefined; +} + +/** A bundle read passed `BundleReadOptions.maxBytes` or `maxNodes`. */ +export class BundleBudgetError extends Error {} + +/** + * The snapshot's bodies by node hash (or what `options.keep` keeps of them), or `undefined` unless + * every shard is there and matches the snapshot (version, snapshot id, the refs' hashes in order). + * Never throws for a bad bundle, so the caller falls back to `nodes/.json`; only a passed + * budget throws (`BundleBudgetError`). */ export async function readSnapshotBundle( blobs: BlobStore, snapshot: Snapshot, + options: BundleReadOptions = {}, ): Promise | undefined> { const count = bundleShardCount(snapshot); if (count === 0) return undefined; + const { maxBytes, maxNodes, keep } = options; + if (maxNodes !== undefined && snapshot.nodes.length > maxNodes) + throw new BundleBudgetError(`${snapshot.nodes.length} nodes (more than ${maxNodes})`); const bodies = new Map(); let complete = true; + let bytesSeen = 0; const decoder = new TextDecoder(); try { await eachLimited( @@ -82,6 +105,9 @@ export async function readSnapshotBundle( complete = false; return; } + bytesSeen += bytes.byteLength; + if (maxBytes !== undefined && bytesSeen > maxBytes) + throw new BundleBudgetError(`more than ${maxBytes} bytes of bodies`); const parsed = JSON.parse(decoder.decode(bytes)) as { v?: unknown; snapshotId?: unknown; @@ -105,10 +131,14 @@ export async function readSnapshotBundle( complete = false; return; } - for (const [i, r] of refs.entries()) bodies.set(r.hash, list[i]); + for (const [i, r] of refs.entries()) { + const kept = keep ? keep(list[i], r) : list[i]; + if (kept !== undefined) bodies.set(r.hash, kept); + } }, ); - } catch { + } catch (e) { + if (e instanceof BundleBudgetError) throw e; return undefined; // unreadable or unparsable shard: the bundle is absent } return complete ? bodies : undefined; diff --git a/packages/core-ts/src/cache-purge-assets.ts b/packages/core-ts/src/cache-purge-assets.ts index 556a6d25..80f2664d 100644 --- a/packages/core-ts/src/cache-purge-assets.ts +++ b/packages/core-ts/src/cache-purge-assets.ts @@ -28,8 +28,12 @@ export interface AssetPurgeInput { readonly why: string[]; } -/** Adds the asset rule's tags and log lines to `input.tags` and `input.why`. */ -export async function planAssetPurge(input: AssetPurgeInput): Promise { +/** + * Adds the asset rule's tags and log lines to `input.tags` and `input.why`; returns the entries it + * reached (RFC-0045: their referrers' cards show the changed cover). + */ +export async function planAssetPurge(input: AssetPurgeInput): Promise> { + const reached = new Set(); const { tags, why, read } = input; const shown = new Set(); for (const c of input.changed) { @@ -45,7 +49,7 @@ export async function planAssetPurge(input: AssetPurgeInput): Promise { else why.push(`asset ${id} changed`); shown.add(id); } - if (shown.size === 0) return; + if (shown.size === 0) return reached; const refsOf = await referencesIn(input.to, read); for (const ref of input.to.nodes) { if (ref.kind !== "contentEntry" && ref.kind !== "page" && ref.kind !== "term") continue; @@ -54,7 +58,9 @@ export async function planAssetPurge(input: AssetPurgeInput): Promise { if (ref.kind === "term") for (const id of input.withAncestors(ref.id)) tags.add(termCacheTag(id)); else tags.add(nodeCacheTag(ref.id)); + if (ref.kind === "contentEntry") reached.add(ref.id); } + return reached; } /** A node's asset references in `snapshot`: its own, and an entry's typed fields' by its defs. */ diff --git a/packages/core-ts/src/cache-purge.ts b/packages/core-ts/src/cache-purge.ts index b99854c7..ba6509f9 100644 --- a/packages/core-ts/src/cache-purge.ts +++ b/packages/core-ts/src/cache-purge.ts @@ -24,6 +24,11 @@ * nothing (404s are never cached, and whatever starts showing it changed itself). Media never * carry the site tag, so the site-wide purges above leave them. * + * - RFC-0045 **references**: the referrers of every changed entry's group, two hops deep, and the + * targets a changed entry names ("used by"), found from the two snapshots' own bodies, never + * the working copy (`references/purge.ts`; one bundle, then only the missing hashes, within + * byte and node budgets). Any field-operation marker on either site node purges everything. + * * Falls back to purging everything when there is no previous snapshot, a snapshot or node cannot * be read, reading the diff would take more than `maxReads` node reads, or the result has more * than `maxTags` tags. @@ -39,13 +44,18 @@ import type { import { planAssetPurge } from "./cache-purge-assets.ts"; import { listCacheTag, nodeCacheTag, siteCacheTag, termCacheTag } from "./cache-tags.ts"; import { canonicalJson } from "./hash.ts"; +import { + planReferencePurge, + type ReferencePurgeLimits, + type ReferencePurgeReader, +} from "./references/purge.ts"; -export interface SnapshotReader { +export interface SnapshotReader extends ReferencePurgeReader { getSnapshot(id: string): Promise; readSnapshotNode(ref: NodeRef): Promise; } -export interface CachePurgeOptions { +export interface CachePurgeOptions extends ReferencePurgeLimits { /** More tags than this purge everything instead (default 300: three requests of 100). */ readonly maxTags?: number; /** More node reads than this purge everything instead (default 2000). */ @@ -97,6 +107,69 @@ function filedUnder(e: ContentEntry | undefined): string[] { return Object.values(e.terms).flatMap((a) => a?.ids ?? []); } +type Change = { before?: NodeRef; after?: NodeRef }; + +/** The planner's one hash-keyed body cache; reads past `maxReads` (bundle shards too) throw. */ +function countedReader(reader: SnapshotReader, maxReads: number) { + let reads = 0; + const bodies = new Map(); + const spend = (n: number) => { + reads += n; + if (reads > maxReads) throw new TooManyReads(); + }; + const read = async (ref: NodeRef): Promise => { + const hit = bodies.get(ref.hash); + if (hit !== undefined) return hit as T; + spend(1); + const body = (await reader.readSnapshotNode(ref)) as T; + bodies.set(ref.hash, body); + return body; + }; + return { read, spend, cached: (hash: string) => bodies.get(hash) }; +} + +/** Both snapshots' vocabularies, and a term with its ancestors in either (archives follow parents). */ +async function ancestry( + before: Map, + after: Map, + read: (ref: NodeRef) => Promise, +) { + const termsOf = async (refs: Map) => { + const out = new Map(); + for (const ref of refs.values()) + if (ref.kind === "term") out.set(ref.id, await read(ref)); + return out; + }; + const oldTerms = await termsOf(before); + const newTerms = await termsOf(after); + const withAncestors = (id: string): string[] => { + const out = new Set(); + for (const terms of [oldTerms, newTerms]) { + let cursor: string | undefined = id; + while (cursor && !out.has(cursor)) { + out.add(cursor); + cursor = terms.get(cursor)?.parentId; + } + } + return [...out]; + }; + return { oldTerms, newTerms, withAncestors }; +} + +/** The nodes of both snapshots by id, and every node that differs between them. */ +function diffNodes(from: Snapshot, to: Snapshot) { + const before = new Map(from.nodes.map((n) => [n.id, n])); + const after = new Map(to.nodes.map((n) => [n.id, n])); + const changed: Change[] = []; + for (const [id, ref] of after) { + const prev = before.get(id); + if (!prev || prev.hash !== ref.hash) + changed.push({ ...(prev ? { before: prev } : {}), after: ref }); + } + for (const [id, ref] of before) if (!after.has(id)) changed.push({ before: ref }); + return { before, after, changed }; +} + export async function planCachePurge( reader: SnapshotReader, fromId: string | undefined, @@ -113,16 +186,7 @@ export async function planCachePurge( if (!to) return { mode: "everything", reason: `snapshot ${toId} not found` }; if (from.siteId !== to.siteId) return { mode: "everything", reason: "different sites" }; - let reads = 0; - const bodies = new Map(); - const read = async (ref: NodeRef): Promise => { - const hit = bodies.get(ref.hash); - if (hit !== undefined) return hit as T; - if (++reads > maxReads) throw new TooManyReads(); - const body = (await reader.readSnapshotNode(ref)) as T; - bodies.set(ref.hash, body); - return body; - }; + const { read, spend, cached } = countedReader(reader, maxReads); try { return await plan(); @@ -138,15 +202,7 @@ export async function planCachePurge( async function plan(): Promise { const site = siteCacheTag(to?.siteId as string); const wholeSite = (why: string): CachePurgePlan => ({ mode: "tags", tags: [site], why: [why] }); - const before = new Map((from as Snapshot).nodes.map((n) => [n.id, n])); - const after = new Map((to as Snapshot).nodes.map((n) => [n.id, n])); - const changed: Array<{ before?: NodeRef; after?: NodeRef }> = []; - for (const [id, ref] of after) { - const prev = before.get(id); - if (!prev || prev.hash !== ref.hash) - changed.push({ ...(prev ? { before: prev } : {}), after: ref }); - } - for (const [id, ref] of before) if (!after.has(id)) changed.push({ before: ref }); + const { before, after, changed } = diffNodes(from as Snapshot, to as Snapshot); if (changed.length === 0) return { mode: "tags", tags: [], why: [] }; // Structure first: no node body needs reading to know these purge the site. @@ -170,34 +226,22 @@ export async function planCachePurge( return wholeSite("the site settings or navigation changed"); } - // The vocabulary of both snapshots, for ancestors (archives and breadcrumbs follow parents). - const termsOf = async (refs: Map) => { - const out = new Map(); - for (const ref of refs.values()) - if (ref.kind === "term") out.set(ref.id, await read(ref)); - return out; - }; const entryChanges = changed.filter((c) => (c.after ?? c.before)?.kind === "contentEntry"); const termChanges = changed.filter((c) => (c.after ?? c.before)?.kind === "term"); - const oldTerms = await termsOf(before); - const newTerms = await termsOf(after); - const withAncestors = (id: string): string[] => { - const out = new Set(); - for (const terms of [oldTerms, newTerms]) { - let cursor: string | undefined = id; - while (cursor && !out.has(cursor)) { - out.add(cursor); - cursor = terms.get(cursor)?.parentId; - } - } - return [...out]; - }; + const { oldTerms, newTerms, withAncestors } = await ancestry(before, after, read); const tags = new Set(); const why: string[] = []; // Media (§7.4): an asset that left the live snapshot or changed (`cache-purge-assets.ts`). - await planAssetPurge({ changed, to: to as Snapshot, read, withAncestors, tags, why }); + const reached = await planAssetPurge({ + changed, + to: to as Snapshot, + read, + withAncestors, + tags, + why, + }); for (const c of termChanges) { const a = c.before ? await read(c.before) : undefined; @@ -230,6 +274,12 @@ export async function planCachePurge( if (flip) return wholeSite(flip); } + // RFC-0045: the referrer closure, from the two snapshots' bodies (`references/purge.ts`). + const all = await planReferencePurge({ + ...{ reader, read, cached, spend, limits: options, entries, reached, tags, why }, + ...{ from: from as Snapshot, to: to as Snapshot }, + }); + if (all) return { mode: "everything", reason: all }; if (tags.size > maxTags) return { mode: "everything", reason: `${tags.size} tags (more than ${maxTags})` }; return { mode: "tags", tags: [...tags], why }; diff --git a/packages/core-ts/src/references/purge.ts b/packages/core-ts/src/references/purge.ts new file mode 100644 index 00000000..fd63f124 --- /dev/null +++ b/packages/core-ts/src/references/purge.ts @@ -0,0 +1,262 @@ +/** + * RFC-0045 §2 "Purges": the purge planner's referrer closure. Pages carry only the node tags they + * already carry (`cloudhal-node-`); no new tags and no theme code. Exact by construction: it + * reads only the immutable `from` and `to` snapshots' bodies, never the working copy or + * `entry_refs`, so a referrer edited away or deleted since is still found where a cached page + * came from. + * + * 1. **Seeds:** the groups (`COALESCE(translation.group, id)`) of every changed, added or removed + * entry, before and after (a group change seeds both), and of every entry the asset rule + * reached (its cover changed). Their members' pages go too (a translation inheriting a changed + * original's reference shows a new card). + * 2. **Two hops:** the referrers of the seed groups, then the referrers of *their* groups (pages + * draw cards to depth 2), from both snapshots, whatever their status. Each referrer purges its + * node tag and every member of its group's. + * 3. **"Used by":** every member of each group a changed or reached entry names, before or after, + * so a target page listing `referencedBy` follows its referrers. + * + * "Names" means the resolved values (`resolveFields`, with the same snapshot's same-site original + * and without it, since a page reads whichever its view shows), so a translation that only + * inherits counts. + * + * Reads: one hash-keyed cache over both snapshots, the planner's own (`cached`). It loads one + * bundle, `to`'s, else `from`'s (RFC-0010), streamed: each body is cut to what the closure needs + * as its shard is parsed, and the byte and node budgets are checked before each parse. Only the + * hashes still missing are read one by one, through the planner's counted reader. Shard reads count + * against `maxReads` too. + * + * Purges everything (`string` result) when a budget is passed, and whenever a field-operation + * marker is on either site node (renames and conversions in flight read through markers). Skipped + * entirely for sites whose two snapshots define no reference (top-level or inside a repeater). + */ +import type { + ContentEntry, + FieldDefinition, + NodeRef, + Site, + Snapshot, +} from "@cloudhal/core-contracts"; +import { BundleBudgetError, type BundleReadOptions, bundleShardCount } from "../bundles.ts"; +import { nodeCacheTag } from "../cache-tags.ts"; +import { originalsOf } from "../fields/entry-gates.ts"; +import { resolveFields } from "../fields/resolve.ts"; +import { isReferenceFieldType } from "../fields/types.ts"; +import { memberGroup } from "./members.ts"; +import { referenceUses } from "./values.ts"; + +/** What the closure reads of a snapshot (the planner's `SnapshotReader`). */ +export interface ReferencePurgeReader { + readSnapshotBundle?( + snapshot: Snapshot, + options?: BundleReadOptions, + ): Promise | undefined>; +} + +export interface ReferencePurgeLimits { + /** Bytes of bundle bodies (default 32 MB). */ + readonly maxBodyBytes?: number | undefined; + /** Entry bodies across both snapshots, and nodes of the bundle read (default 50,000). */ + readonly maxNodes?: number | undefined; +} + +export const DEFAULT_PURGE_MAX_BODY_BYTES = 32 * 1024 * 1024; +export const DEFAULT_PURGE_MAX_NODES = 50_000; + +export interface ReferencePurgeInput { + readonly reader: ReferencePurgeReader; + readonly from: Snapshot; + readonly to: Snapshot; + /** The planner's counted, hash-keyed reader. */ + readonly read: (ref: NodeRef) => Promise; + /** A body the planner already holds, by hash (no read). */ + readonly cached: (hash: string) => unknown; + /** Counts `n` reads against the planner's `maxReads` (throws past it). */ + readonly spend: (n: number) => void; + /** Every changed, added or removed entry: before (in `from`) and after (in `to`). */ + readonly entries: readonly { before?: ContentEntry; after?: ContentEntry }[]; + /** Entries of `to` the asset rule reached (a shown asset of theirs changed). */ + readonly reached: ReadonlySet; + readonly tags: Set; + readonly why: string[]; + readonly limits?: ReferencePurgeLimits | undefined; +} + +/** What the closure keeps of an entry body. */ +type Slim = Pick & { + translation?: ContentEntry["translation"]; +}; + +const defsOf = (site: Site): FieldDefinition[] => + Object.values(site.collections ?? {}).flatMap((c) => c?.fields ?? []); + +/** A `reference` field, or a repeater with a `reference` sub-field. */ +const namesEntries = (d: FieldDefinition): boolean => + d.type === "reference" || (d.type === "repeater" && d.fields.some((s) => s.type === "reference")); + +/** True when some collection of the site holds references (top-level or in a repeater). */ +export function definesReferences(site: Pick): boolean { + return defsOf(site as Site).some(namesEntries); +} + +/** Any field-operation marker on the site node. */ +const hasMarker = (site: Site): boolean => defsOf(site).some((d) => d.migration !== undefined); + +/** Per collection, the field keys the closure keeps (references and repeaters of either site). */ +function keptKeys(sites: readonly Site[]): Map> { + const out = new Map>(); + for (const site of sites) + for (const [c, col] of Object.entries(site.collections ?? {})) { + const keys = out.get(c) ?? new Set(); + for (const d of col?.fields ?? []) if (isReferenceFieldType(d.type)) keys.add(d.key); + out.set(c, keys); + } + return out; +} + +function slimmer(keys: Map>): (body: unknown) => Slim { + return (body) => { + const e = body as ContentEntry; + const own = keys.get(e.collection); + const fields: Record = {}; + for (const [k, v] of Object.entries(e.fields ?? {})) if (own?.has(k)) fields[k] = v; + return { + id: e.id, + collection: e.collection, + locale: e.locale, + status: e.status, + fields, + ...(e.translation ? { translation: e.translation } : {}), + }; + }; +} + +/** One snapshot's referrer index: names by entry, referrers and members by group. */ +interface Index { + readonly named: Map>; + readonly referrers: Map>; + readonly members: Map>; + readonly groupOf: Map; +} + +function indexOf(site: Site, entries: readonly Slim[]): Index { + const index: Index = { + named: new Map(), + referrers: new Map(), + members: new Map(), + groupOf: new Map(), + }; + const add = (map: Map>, key: string, value: string) => { + const set = map.get(key) ?? new Set(); + set.add(value); + map.set(key, set); + }; + const originalOf = originalsOf(entries as ContentEntry[]); + for (const e of entries) { + const g = memberGroup(e); + index.groupOf.set(e.id, g); + add(index.members, g, e.id); + const names = new Set(); + const original = originalOf(e as ContentEntry); + for (const r of original ? [original, undefined] : [undefined]) + for (const f of Object.values(resolveFields(site, e, r))) + if (isReferenceFieldType(f.def.type)) + for (const u of referenceUses(f.def, f.value, "")) names.add(u.groupId); + index.named.set(e.id, names); + for (const n of names) add(index.referrers, n, e.id); + } + return index; +} + +/** Loads every entry body of both snapshots as `Slim`, by hash (see the module comment). */ +async function slimBodies( + input: ReferencePurgeInput, + slim: (body: unknown) => Slim, +): Promise> { + const { from, to, reader, limits } = input; + const maxNodes = limits?.maxNodes ?? DEFAULT_PURGE_MAX_NODES; + const out = new Map(); + const bundleOf = async (snapshot: Snapshot) => { + if (!reader.readSnapshotBundle) return undefined; + input.spend(bundleShardCount(snapshot)); + return reader.readSnapshotBundle(snapshot, { + maxBytes: limits?.maxBodyBytes ?? DEFAULT_PURGE_MAX_BODY_BYTES, + maxNodes, + keep: (body, ref) => (ref.kind === "contentEntry" ? slim(body) : undefined), + }); + }; + const bundle = (await bundleOf(to)) ?? (await bundleOf(from)); + for (const [hash, s] of bundle ?? []) out.set(hash, s as Slim); + for (const snapshot of [from, to]) + for (const ref of snapshot.nodes) { + if (ref.kind !== "contentEntry" || out.has(ref.hash)) continue; + if (out.size >= maxNodes) throw new BundleBudgetError(`more than ${maxNodes} entries`); + out.set(ref.hash, slim(input.cached(ref.hash) ?? (await input.read(ref)))); + } + return out; +} + +/** + * Adds the referrer closure's node tags to `input.tags`; returns a reason to purge everything + * instead, or undefined. + */ +export async function planReferencePurge(input: ReferencePurgeInput): Promise { + const { from, to, read, tags } = input; + const siteOf = (s: Snapshot) => { + const ref = s.nodes.find((n) => n.kind === "site" && n.id === s.siteId); + return ref ? read(ref) : Promise.resolve(undefined); + }; + const [a, b] = await Promise.all([siteOf(from), siteOf(to)]); + if (!a || !b) return "a snapshot has no site node"; + if (!definesReferences(a) && !definesReferences(b)) return undefined; + if (hasMarker(a) || hasMarker(b)) return "a field operation is running in a published snapshot"; + let bodies: Map; + try { + bodies = await slimBodies(input, slimmer(keptKeys([a, b]))); + } catch (e) { + if (e instanceof BundleBudgetError) return `the referrer closure is too large: ${e.message}`; + throw e; + } + const entriesOf = (s: Snapshot) => + s.nodes.flatMap((n) => (n.kind === "contentEntry" ? [bodies.get(n.hash) as Slim] : [])); + const indexes = [indexOf(a, entriesOf(from)), indexOf(b, entriesOf(to))]; + const before = input.entries.length; + const purge = (id: string) => tags.add(nodeCacheTag(id)); + const members = (g: string) => { + for (const ix of indexes) for (const id of ix.members.get(g) ?? []) purge(id); + }; + const referrersOf = (groups: Iterable) => { + const out = new Set(); + for (const g of groups) + for (const ix of indexes) for (const id of ix.referrers.get(g) ?? []) out.add(id); + return out; + }; + const groupsOf = (ids: Iterable) => { + const out = new Set(); + for (const id of ids) + for (const ix of indexes) { + const g = ix.groupOf.get(id); + if (g) out.add(g); + } + return out; + }; + // 1. seeds + const touched = new Set(input.reached); + for (const { before: x, after: y } of input.entries) + for (const e of [x, y]) if (e) touched.add(e.id); + const seeds = new Set(groupsOf(touched)); + for (const { before: x, after: y } of input.entries) + for (const e of [x, y]) if (e) seeds.add(memberGroup(e)); + for (const g of seeds) members(g); + // 2. two hops: referrers of the seeds, then of their groups; each with its group's members + const hop1 = referrersOf(seeds); + const hop2 = referrersOf(groupsOf(hop1)); + for (const g of groupsOf([...hop1, ...hop2])) members(g); + for (const id of [...hop1, ...hop2]) purge(id); + // 3. "used by": the members of every group a touched entry names, before or after + for (const id of touched) + for (const ix of indexes) for (const g of ix.named.get(id) ?? []) members(g); + input.why.push( + `references: ${hop1.size + hop2.size} referrers of ${seeds.size} groups (${before} entries changed)`, + ); + return undefined; +} diff --git a/packages/core-ts/src/references/release-purge.ts b/packages/core-ts/src/references/release-purge.ts new file mode 100644 index 00000000..56996e16 --- /dev/null +++ b/packages/core-ts/src/references/release-purge.ts @@ -0,0 +1,32 @@ +/** + * RFC-0045 §2 "Purges", point 4: a release run re-plans its purge at run time. The plan stored + * when the release was pinned diffs the snapshot that was live *then* against a virtual target; by + * the run, other publishes may have moved the pointer, and referrers (whose pages show the + * release's targets as cards) are found only from the two real snapshots' bodies. So after + * `publishRelease` commits, the executor plans `from → snapshotId` (`from`: the pointer the batch + * moved), exactly like every other publish path, and sends that. + * + * The stored plan still decides the outcome's `purge` written in the batch (it must be known + * before the batch); the executor corrects it when the re-plan purges differently. Reads: two + * `getSnapshot` statements, then bundle shards and node bodies (blob reads, bounded by + * `maxReads`), counted in `RELEASE_EXECUTOR_COST`. + */ +import type { CachePurgePlan } from "../cache-purge.ts"; +import type { ReleasePurge } from "../release-apply.ts"; +import type { SiteGraphStore } from "../store.ts"; + +/** The purge a release run sends: planned now, from the pointer it moved to the snapshot it made. */ +export function releasePurgePlan( + graph: Pick, + published: { readonly from?: string | undefined; readonly snapshotId: string }, +): Promise { + const { from, snapshotId: to } = published; + return graph.planCachePurge({ ...(from ? { from } : {}), to }); +} + +/** What a plan does to the outcome's `purge` (`none` where nothing is cached or nothing changed). */ +export function purgeOutcome(plan: CachePurgePlan, purging: boolean): ReleasePurge { + if (!purging) return "none"; + if (plan.mode === "everything") return "everything"; + return plan.tags.length > 0 ? "tags" : "none"; +} diff --git a/packages/core-ts/src/release-executor.ts b/packages/core-ts/src/release-executor.ts index a8a99b75..98b94d9a 100644 --- a/packages/core-ts/src/release-executor.ts +++ b/packages/core-ts/src/release-executor.ts @@ -1,12 +1,15 @@ /** * The `ReleaseExecutor` seam of `runDue` (RFC-0008 §6.3, §8): `SiteGraphStore.publishRelease`, - * then the stored purge plan, sent after the batch, then the release-published hook + * then the purge, re-planned after the batch from the pointer it moved (RFC-0045 §2, + * `references/release-purge.ts`; the stored plan only fills the batch's outcome), then the + * release-published hook * (`release-after.ts`; the Node server rebuilds its static site, #150). `runScheduledTick` and * "Publish now" (`ReleaseService.publishNow`) build one; both live in `releases.ts`. */ import type { ReleaseExecution, ReleaseExecutor } from "@cloudhal/schedules"; import type { SqlScheduleStore } from "@cloudhal/schedules/sql"; import type { CachePurgePlan } from "./cache-purge.ts"; +import { purgeOutcome, releasePurgePlan } from "./references/release-purge.ts"; import * as after from "./release-after.ts"; import type { ReleasePurge } from "./release-apply.ts"; import type { ReleasePurger } from "./releases.ts"; @@ -14,11 +17,13 @@ import type { SiteGraphStore } from "./store.ts"; /** * Statements one release execution takes, for `runDue`'s budget check: the stored purge plan (1), - * the site, the live snapshot and the parent (3), the eight-statement batch, and the purge's own - * bookkeeping after it (up to 4). A site that moved between the reads and the batch repeats the - * reads and the batch once; that rare path may end the tick early (`more`), never mid-batch. + * the site, the live snapshot and the parent (3), the eight-statement batch, the run-time re-plan's + * two `getSnapshot` reads (RFC-0045 §2; its bundle shards and node bodies are blob reads, bounded + * by the planner's `maxReads`), and the purge's own bookkeeping after it (up to 4). A site that + * moved between the reads and the batch repeats the reads and the batch once; that rare path may + * end the tick early (`more`), never mid-batch. */ -export const RELEASE_EXECUTOR_COST = 16; +export const RELEASE_EXECUTOR_COST = 18; export interface ReleaseExecutorOptions { readonly graph: SiteGraphStore; @@ -48,13 +53,7 @@ export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { const plan: CachePurgePlan = after.isPurgePlan(stored) ? stored : { mode: "everything", reason: "no purge plan was stored with the release" }; - const expected: ReleasePurge = !o.purge - ? "none" - : plan.mode === "everything" - ? "everything" - : plan.tags.length > 0 - ? "tags" - : "none"; + const expected = purgeOutcome(plan, o.purge !== undefined); const r = await o.graph.publishRelease({ release: input.release, items: input.items, @@ -70,9 +69,11 @@ export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { switch (r.kind) { case "published": { let purge: ReleasePurge = expected; - if (r.committed && o.purge && expected !== "none") { + if (r.committed && o.purge) { try { - purge = await o.purge(input.release.siteId, plan); + const now = await releasePurgePlan(o.graph, r); + purge = purgeOutcome(now, true); + if (purge !== "none") purge = await o.purge(input.release.siteId, now); } catch (e) { purge = "failed"; log( diff --git a/packages/core-ts/src/store.ts b/packages/core-ts/src/store.ts index 1f5b8b35..3709b029 100644 --- a/packages/core-ts/src/store.ts +++ b/packages/core-ts/src/store.ts @@ -36,7 +36,7 @@ import { } from "./assets.ts"; import { checkAvailability, type GateSupport, sameAvailability } from "./availability.ts"; import { type BlockRegistry, createBlockRegistry, type PropsSchema } from "./blocks.ts"; -import { readSnapshotBundle, writeSnapshotBundle } from "./bundles.ts"; +import { type BundleReadOptions, readSnapshotBundle, writeSnapshotBundle } from "./bundles.ts"; import { type CachePurgeOptions, type CachePurgePlan, planCachePurge } from "./cache-purge.ts"; import type { CollectionInfo } from "./collections.ts"; import { addedCorrections, type CorrectionInput } from "./corrections.ts"; @@ -945,9 +945,9 @@ export class SiteGraphStore { return this.#snapshots.create(siteId, input); } - /** The bodies of `snapshot` by node hash from its bundle, or undefined (RFC-0010). */ - readSnapshotBundle(snapshot: Snapshot): Promise | undefined> { - return readSnapshotBundle(this.#blobs, snapshot); + /** Bodies by node hash from `snapshot`'s bundle, or undefined (RFC-0010; budgets: RFC-0045). */ + readSnapshotBundle(snapshot: Snapshot, options?: BundleReadOptions) { + return readSnapshotBundle(this.#blobs, snapshot, options); } /** diff --git a/packages/core-ts/src/testing-releases.ts b/packages/core-ts/src/testing-releases.ts index b36ec656..3ba30997 100644 --- a/packages/core-ts/src/testing-releases.ts +++ b/packages/core-ts/src/testing-releases.ts @@ -4,7 +4,14 @@ * purge plan, and the tick that runs them (`runScheduledTick`) — on SQLite and Miniflare D1. * Test names cite the invariants of RFC-0008 §17 they prove. */ -import type { Actor, ContentEntry, Release, Site, Snapshot } from "@cloudhal/core-contracts"; +import type { + Actor, + ContentEntry, + FieldDefinition, + Release, + Site, + Snapshot, +} from "@cloudhal/core-contracts"; import { isValid } from "@cloudhal/core-contracts"; import { TICK_BUDGETS } from "@cloudhal/schedules"; import type { BlobStore, SqlDriver } from "@cloudhal/storage"; @@ -48,7 +55,9 @@ export function releasesSuite( await dropAllTables(sql); now = T0; logs.length = 0; - store = new SiteGraphStore({ sql, blobs, now: () => new Date(now) }); + // RFC-0045: references are enabled here (refused in production) for the purge re-plan case + const fieldTypes = { supported: () => true }; + store = new SiteGraphStore({ sql, blobs, now: () => new Date(now), fieldTypes }); await store.init(); site = await seedStarterSite(store, { name: "Releases", slug: "releases" }); news = (await store.getTermBySlug(site.id, "category", "news"))?.id as string; @@ -485,7 +494,45 @@ export function releasesSuite( expect((await get(release.id)).state).toBe("published"); }); - it("sends the purge plan stored at pin time and records a deferred purge", async () => { + it("re-plans the purge at run time: a referrer published after the pin is purged (RFC-0045)", async () => { + const s = await fresh(); + const venue: FieldDefinition = { + key: "venue", + type: "reference", + labels: { en: { label: "Venue" } }, + localized: false, + collections: ["places"], + }; + site = await store.updateSite({ + ...s, + collections: { ...s.collections, places: {}, events: { fields: [venue] } }, + }); + const hall = await story({ collection: "places", slug: "hall", title: "Hall" }); + await publishSite(); + // the release retitles the hall; at pin time nothing names it + await store.updateContentEntry({ ...hall, title: "Main hall" }); + await scheduled([hall.id]); + // a referrer goes live after the pin, by an ordinary publish + const event = await story({ + collection: "events", + slug: "launch", + fields: { venue: hall.id }, + }); + await publishSite(); + const plans: CachePurgePlan[] = []; + const report = await tick(T, { + purge: () => async (_siteId, plan) => { + plans.push(plan); + return "tags"; + }, + }); + expect(report.ran).toBe(1); + const tags = plans[0]?.mode === "tags" ? plans[0].tags : []; + expect(tags).toContain(`cloudhal-node-${hall.id}`); + expect(tags).toContain(`cloudhal-node-${event.id}`); + }); + + it("sends the purge re-planned at run time and records a deferred purge", async () => { const s = await story(); await scheduled([s.id]); const plans: CachePurgePlan[] = []; diff --git a/packages/core-ts/test/references-purge.test.ts b/packages/core-ts/test/references-purge.test.ts new file mode 100644 index 00000000..8329d077 --- /dev/null +++ b/packages/core-ts/test/references-purge.test.ts @@ -0,0 +1,347 @@ +/** + * RFC-0045 step 3b (§2 "Purges", Test plan "Renderer"): the purge planner's referrer closure. + * Pages carry only the node tags they already carry, so a theme that emits no new tags is purged: + * every case asserts `cloudhal-node-` tags of referrers, "used by" targets and their language + * versions. The closure reads only the `from`/`to` snapshots' bodies (bundles first), never the + * working copy, so edits after a snapshot never hide a referrer. The types stay refused in + * production; this store enables them (`fieldTypes`), as the shared suites do. + */ +import type { + ContentEntry, + FieldDefinition, + NodeRef, + Site, + Snapshot, +} from "@cloudhal/core-contracts"; +import { MemoryBlobStore } from "@cloudhal/storage"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { beforeEach, describe, expect, it } from "vitest"; +import { GATES_ENFORCED } from "../src/availability.ts"; +import type { BundleReadOptions } from "../src/bundles.ts"; +import { type CachePurgeOptions, planCachePurge, type SnapshotReader } from "../src/cache-purge.ts"; +import { nodeCacheTag, siteCacheTag } from "../src/cache-tags.ts"; +import { continueFieldOperation, runFieldOperation } from "../src/fields-operations/index.ts"; +import { nodeHash } from "../src/hash.ts"; +import { PNG_1X1 } from "../src/media/fixtures.ts"; +import { seedStarterSite } from "../src/seed.ts"; +import { type PublishChange, SiteGraphStore } from "../src/store.ts"; + +const human = { kind: "human" as const, id: "user_refpurge001", name: "Editor" }; +const L = (label: string) => ({ en: { label } }); +const VENUE: FieldDefinition = { + key: "venue", + type: "reference", + labels: L("Venue"), + localized: false, + collections: ["places", "events"], +}; +const RELATED: FieldDefinition = { + key: "related", + type: "reference", + labels: L("Related"), + localized: false, + multiple: true, + collections: ["events"], +}; +const NOTES: FieldDefinition = { key: "notes", type: "text", labels: L("Notes"), localized: true }; + +describe("planCachePurge: the referrer closure (RFC-0045 §2)", () => { + let store: SiteGraphStore; + let site: Site; + let terms: ContentEntry["terms"]; + const changes: PublishChange[] = []; + /** What the planner read: node reads, bundle reads. */ + const counts = { nodes: 0, bundles: 0 }; + const reader: SnapshotReader = { + getSnapshot: (id) => store.getSnapshot(id), + readSnapshotNode: (ref: NodeRef) => { + counts.nodes++; + return store.readSnapshotNode(ref); + }, + readSnapshotBundle: (s: Snapshot, o?: BundleReadOptions) => { + counts.bundles++; + return store.readSnapshotBundle(s, o); + }, + }; + + const fresh = async () => (await store.getSite(site.id)) as Site; + const get = async (e: ContentEntry) => (await store.getContentEntry(e.id)) as ContentEntry; + const snap = (reason: "publish" | "manual") => + store.createSnapshot(site.id, { actor: human, reason }); + /** Moves the pointer to `id` (a publish, an approved request, a restore). */ + const point = async (id: string): Promise => { + site = await store.publish(await fresh(), id); + return changes.at(-1) as PublishChange; + }; + const publish = async () => point((await snap("publish")).id); + const plan = (change: PublishChange, o: CachePurgeOptions = {}) => { + counts.nodes = 0; + counts.bundles = 0; + return planCachePurge(reader, change.from, change.to, o); + }; + const tagsOf = async (change: PublishChange, o: CachePurgeOptions = {}) => { + const p = await plan(change, o); + if (p.mode !== "tags") throw new Error(`expected tags, got everything: ${p.reason}`); + return new Set(p.tags); + }; + const node = (e: { id: string }) => nodeCacheTag(e.id); + const make = (collection: string, slug: string, over: Partial = {}) => + store.createContentEntry({ + siteId: site.id, + collection, + slug, + locale: "en", + status: "published", + title: slug, + body: { format: "markdown", value: `About ${slug}.` }, + fields: {}, + tags: [], + ...(terms ? { terms } : {}), + author: human, + ...over, + } as never) as Promise; + const translate = async (of: ContentEntry, slug: string, over: Partial = {}) => + make(of.collection, slug, { + locale: "tr", + translation: { + group: of.id, + method: "human", + source: { id: of.id, locale: of.locale, rev: of.rev, hash: await nodeHash(of) }, + }, + ...over, + }); + const edit = async (e: ContentEntry, over: Partial) => + store.updateContentEntry({ ...(await get(e)), ...over }); + + beforeEach(async () => { + changes.length = 0; + store = new SiteGraphStore({ + sql: await BetterSqlite3Driver.open(":memory:"), + blobs: new MemoryBlobStore(), + gates: GATES_ENFORCED, + fieldTypes: { supported: () => true }, + onPublished: async (_site, change) => { + changes.push(change); + }, + }); + await store.init(); + site = await seedStarterSite(store, { name: "Refs", slug: "refs" }); + site = await store.updateSite({ + ...site, + locales: ["en", "tr"], + collections: { + ...site.collections, + places: {}, + events: { fields: [VENUE, RELATED, NOTES] }, + }, + }); + terms = (await store.listContentEntries(site.id, "posts"))[0]?.terms; + await publish(); + }); + + /** hall, an event naming it, and its Turkish translation (inheriting the venue). */ + async function world() { + const hall = await make("places", "hall"); + const event = await make("events", "launch", { fields: { venue: hall.id } }); + const eventTr = await translate(event, "lansman"); + const other = await make("events", "other"); + await publish(); + return { hall, event, eventTr, other }; + } + + it("a target retitled, unpublished or newly published purges its referrers' pages", async () => { + const { hall, event, eventTr, other } = await world(); + await edit(hall, { title: "Main hall" }); + const retitled = await tagsOf(await publish()); + for (const e of [hall, event, eventTr]) expect(retitled.has(node(e))).toBe(true); + expect(retitled.has(node(other))).toBe(false); + expect(retitled.has(siteCacheTag(site.id))).toBe(false); + await edit(hall, { status: "draft" }); + const unpublished = await tagsOf(await publish()); + for (const e of [hall, event, eventTr]) expect(unpublished.has(node(e))).toBe(true); + await edit(hall, { status: "published" }); + const republished = await tagsOf(await publish()); + for (const e of [hall, event, eventTr]) expect(republished.has(node(e))).toBe(true); + }); + + it("a target newly translated, or with a changed cover, purges its referrers", async () => { + const { hall, event, eventTr } = await world(); + await translate(hall, "salon"); + const translated = await tagsOf(await publish()); + for (const e of [event, eventTr]) expect(translated.has(node(e))).toBe(true); + const { asset } = await store.ingestAsset({ + siteId: site.id, + bytes: PNG_1X1, + filename: "hall.png", + uploadedBy: human, + source: { kind: "upload" }, + fields: { decorative: true }, + }); + await edit(hall, { coverImage: { assetId: asset.id, alt: "" } as never }); + await publish(); + // only the asset changes: the asset rule reaches hall, the closure its referrers + await store.updateAsset({ ...asset, slug: "hall-renamed" }); + const cover = await tagsOf(await publish()); + for (const e of [hall, event, eventTr]) expect(cover.has(node(e))).toBe(true); + }); + + it("depth 2: A → B → C, with C unpublished, purges A", async () => { + const c = await make("events", "c"); + const b = await make("events", "b", { fields: { related: [c.id] } }); + const a = await make("events", "a", { fields: { related: [b.id] } }); + const far = await make("events", "far", { fields: { related: [a.id] } }); + await publish(); + await edit(c, { status: "draft" }); + const tags = await tagsOf(await publish()); + for (const e of [c, b, a]) expect(tags.has(node(e)), e.slug).toBe(true); + // three hops away: its page shows a's card, not a's fields + expect(tags.has(node(far))).toBe(false); + }); + + it('a "used by" page follows referrers that appear or go, a translation that only inherits too', async () => { + const { hall, event } = await world(); + const salon = await translate(hall, "salon"); + await publish(); + const gig = await make("events", "gig", { fields: { venue: hall.id } }); + const appeared = await tagsOf(await publish()); + for (const e of [hall, salon]) expect(appeared.has(node(e))).toBe(true); + // a translation that only inherits the venue appears on its own + const gigTr = await translate(gig, "konser"); + const inherited = await tagsOf(await publish()); + expect(inherited.has(node(gigTr))).toBe(true); + for (const e of [hall, salon]) expect(inherited.has(node(e))).toBe(true); + await store.deleteContentEntry(await get(gigTr)); + await store.deleteContentEntry(await get(gig)); + const gone = await tagsOf(await publish()); + for (const e of [hall, salon]) expect(gone.has(node(e))).toBe(true); + // editing a referrer away from its venue purges the venue it left + await edit(event, { fields: {} }); + const left = await tagsOf(await publish()); + for (const e of [hall, salon]) expect(left.has(node(e))).toBe(true); + }); + + it("a rollback whose referrer was since edited to draft still purges the referrer", async () => { + const { hall, event } = await world(); + const before = site.publishedSnapshotId as string; + await edit(hall, { title: "Main hall" }); + await publish(); + await edit(event, { status: "draft" }); + const { site: back } = await store.rollback(await fresh(), before, human); + site = back; + const tags = await tagsOf(changes.at(-1) as PublishChange); + expect(tags.has(node(event))).toBe(true); + }); + + it("an approved request publishing an older snapshot, after the referrer was edited and the target unpublished, purges the referrer", async () => { + const { hall, event, eventTr } = await world(); + // the request's snapshot: hall unpublished, the event still naming it + await edit(hall, { status: "draft" }); + const request = await snap("manual"); + // meanwhile the referrer stops naming the target (only in the working copy) + await edit(event, { fields: {} }); + const tags = await tagsOf(await point(request.id)); + for (const e of [hall, event, eventTr]) expect(tags.has(node(e))).toBe(true); + }); + + it("a referrer edited away or deleted between snapshot and purge is still purged from to's bodies", async () => { + const { hall, event, eventTr } = await world(); + await edit(hall, { title: "Main hall" }); + const s = await snap("publish"); + await edit(event, { fields: {} }); + await store.deleteContentEntry(await get(eventTr)); + const tags = await tagsOf(await point(s.id)); + for (const e of [event, eventTr]) expect(tags.has(node(e))).toBe(true); + // and deleted outright + await edit(hall, { title: "Great hall" }); + const s2 = await snap("publish"); + await store.deleteContentEntry(await get(event)); + expect((await tagsOf(await point(s2.id))).has(node(event))).toBe(true); + }); + + it("a translation in to that only inherits, deleted since, is still purged", async () => { + const { hall, eventTr } = await world(); + await edit(hall, { title: "Main hall" }); + const s = await snap("publish"); + await store.deleteContentEntry(await get(eventTr)); + expect((await tagsOf(await point(s.id))).has(node(eventTr))).toBe(true); + }); + + it("a snapshot without a bundle costs one bundle plus its items, never purge-everything", async () => { + const { hall, event } = await world(); + for (let i = 0; i < 30; i++) await make("events", `filler-${i}`); + await publish(); + // a release run's `scheduled` snapshot and an approval's manual one have no bundle + await edit(hall, { title: "Main hall" }); + const manual = await snap("manual"); + const tags = await tagsOf(await point(manual.id), { maxReads: 20 }); + expect(tags.has(node(event))).toBe(true); + // `to` has none, `from` has one: two bundle attempts, and node reads only for the diff + expect(counts.bundles).toBe(2); + expect(counts.nodes).toBeLessThan(10); + }); + + it("a site without reference definitions reads no bundle at all", async () => { + const plain = await seedStarterSite(store, { name: "Plain", slug: "plain" }); + const s1 = await store.createSnapshot(plain.id, { actor: human, reason: "publish" }); + let p = await store.publish(plain, s1.id); + const post = (await store.listContentEntries(plain.id, "posts"))[0] as ContentEntry; + await store.updateContentEntry({ ...post, title: "Retitled" }); + const s2 = await store.createSnapshot(plain.id, { actor: human, reason: "publish" }); + p = await store.publish((await store.getSite(plain.id)) as Site, s2.id); + expect(p.publishedSnapshotId).toBe(s2.id); + const out = await plan({ from: s1.id, to: s2.id }); + expect(out.mode).toBe("tags"); + expect(counts.bundles).toBe(0); + }); + + it("the byte, node, read and tag budgets fall back to purge-everything", async () => { + const { hall } = await world(); + await edit(hall, { title: "Main hall" }); + const change = await publish(); + const reason = async (o: CachePurgeOptions) => { + const p = await plan(change, o); + return p.mode === "everything" ? p.reason : "tags"; + }; + expect(await reason({ maxBodyBytes: 10 })).toMatch(/too large/); + expect(await reason({ maxNodes: 2 })).toMatch(/too large/); + expect(await reason({ maxTags: 1 })).toMatch(/tags/); + expect(await reason({})).toBe("tags"); + // without bundles every entry body is a node read + await edit(hall, { title: "Great hall" }); + const manual = await snap("manual"); + await edit(hall, { title: "Grand hall" }); + const manual2 = await snap("manual"); + const p = await plan({ from: manual.id, to: manual2.id }, { maxReads: 3 }); + expect(p).toMatchObject({ mode: "everything" }); + expect(p.mode === "everything" ? p.reason : "").toMatch(/node reads/); + }); + + it("any field-operation marker on from or to purges everything", async () => { + const { hall, event } = await world(); + await edit(event, { fields: { venue: hall.id, notes: "Doors at 7" } }); + await make("events", "second", { fields: { notes: "Late" } }); + await publish(); + // a rename of a text field, stopped by its budget: the marker stays on the site node + const part = await runFieldOperation( + store, + site.id, + { op: "rename_field", collection: "events", from: "notes", to: "remarks" }, + { actor: human, budget: { entries: 1 } }, + ); + expect(part.done).toBe(false); + // purging everything: the whole site's tag, or mode `everything` + const all = (p: Awaited>) => + p.mode === "everything" || (p.tags.length === 1 && p.tags[0] === siteCacheTag(site.id)); + await edit(hall, { title: "Main hall" }); + // the marker's own publish changed the site node: the site tag already + expect(all(await plan(await publish()))).toBe(true); + // the marker on both snapshots, the site node unchanged: the closure purges everything + await edit(hall, { title: "Great hall" }); + const onBoth = await plan(await publish()); + expect(onBoth.mode === "everything" ? onBoth.reason : "").toMatch(/field operation/); + // on `from` only: the operation finished (the site node changed again) + const done = await continueFieldOperation(store, site.id, part.operationId, { actor: human }); + expect(done.done).toBe(true); + expect(all(await plan(await publish()))).toBe(true); + }); +}); From 1cb2520aefd402d2bd7cc45b5df2314d1bc97477 Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:38:02 +0300 Subject: [PATCH 3/6] fix(core-ts): RFC-0045 step 3b review (PR #212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A: no seeds (no changed entry, no reached asset), no reads: term-, asset- and site-only publishes on a reference site never reach the budgets - B: the closure reads missing hashes itself (readSnapshotBundle `only`), six at a time, counted against maxReads and the same byte total before each parse, kept as Slim and never cached whole; a reader that cannot stream purges everything - C: release runs re-plan over the tick's counting driver; a re-plan that throws sends purge-everything, never the stored plan; RELEASE_EXECUTOR_COST 26 (refreshSearch, the re-plan and the bundle's reads, bookkeeping) - D: a release run writes its scheduled snapshot's bundle first (best effort, ≤ 1,000 nodes); one left without is passed as `unbundled`, so no shard reads are spent on it - E: the pin-time plan skips the closure (`references: false`); it is the expected outcome only for reference sites - F: a field still converting from references counts as one (the marker rule applies during reference → text) - G: tests for each, including the closure's own reads hitting maxReads; RFC "Amended in 3b" rewritten Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/rfc45-step3b-purges.md | 16 +++- ...ctions-stage-4-references-and-repeaters.md | 19 +++- packages/core-ts/src/bundles.ts | 39 ++++++++- packages/core-ts/src/references/purge.ts | 74 ++++++++++++---- .../core-ts/src/references/release-purge.ts | 79 +++++++++++++---- packages/core-ts/src/release-executor.ts | 38 +++++--- packages/core-ts/src/releases.ts | 13 ++- packages/core-ts/src/testing-releases.ts | 60 +++++++++++++ .../core-ts/test/references-purge.test.ts | 87 ++++++++++++++++++- 9 files changed, 355 insertions(+), 70 deletions(-) diff --git a/.changeset/rfc45-step3b-purges.md b/.changeset/rfc45-step3b-purges.md index ff6eb090..ee2e2ab8 100644 --- a/.changeset/rfc45-step3b-purges.md +++ b/.changeset/rfc45-step3b-purges.md @@ -10,12 +10,20 @@ RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothin - It also purges every member of each group a changed entry names, before or after ("used by"). - Values are resolved, so a translation that only inherits counts. - It reads only the `from` and `to` snapshots' bodies, never the working copy or `entry_refs`. -- **Reads and budgets.** One hash-keyed cache: one bundle (`to`'s, else `from`'s), streamed and cut down as each shard is parsed, then the missing hashes. A passed budget purges everything: +- **Reads and budgets.** A publish that changes no entry and no shown asset reads nothing. Otherwise the closure keeps one hash-keyed cache: + - one bundle (`to`'s, else `from`'s), streamed and cut down as each shard is parsed; + - then the missing hashes, read by the closure itself, six at a time, and cut down the same way. + + A passed budget purges everything: - bytes (`maxBodyBytes`, 32 MB); - nodes (`maxNodes`, 50,000); - reads (`maxReads`, shard reads included); - tags (`maxTags`). - So does any field-operation marker on either snapshot's site node. -- **`readSnapshotBundle`** (`SnapshotReader.readSnapshotBundle`, `SiteGraphStore.readSnapshotBundle`, `readSnapshotBundle` in bundles) takes optional `maxBytes`, `maxNodes` and `keep`. A passed budget throws `BundleBudgetError` before the shard is parsed. No new cache tags. -- **Release runs re-plan their purge at run time**, from the pointer they moved to the snapshot they made, instead of sending the plan stored at pin time (`references/release-purge.ts`). `RELEASE_EXECUTOR_COST` is 18. + So does any field-operation marker on either snapshot's site node, for sites that hold references or convert from them. +- **`readSnapshotBundle`** (`SnapshotReader.readSnapshotBundle`, `SiteGraphStore.readSnapshotBundle`, `readSnapshotBundle` in bundles) takes optional `maxBytes`, `maxNodes`, `keep`, `only` (bodies node by node) and `spent` (a shared byte total). A passed budget throws `BundleBudgetError` before the shard is parsed. No new cache tags. +- **Release runs re-plan their purge at run time** (`references/release-purge.ts`), from the pointer they moved to the snapshot they made, instead of sending the plan stored at pin time. + - The re-plan runs over the tick's counting driver; if it fails, the run purges everything. + - Each run first writes its snapshot's bundle, best effort, for snapshots of at most 1,000 nodes. + - The pin-time plan skips the closure (`CachePurgeOptions.references: false`): for sites with references it is the expected outcome only. + - `RELEASE_EXECUTOR_COST` is 26 (it counts `refreshSearch` now). diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index c66220f3..d71d3896 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -1211,9 +1211,20 @@ Rollout step 2 shipped in two PRs: 2a, store and integrity, and 2b, the consumer - **`ReferencedEntry.path`** (§4): the member's graph path (`entryPath`, no locale prefix). Themes link with it, so the locale middleware localizes it once, like every other entry link. `url` stays the canonical URL, already localized, and absolute only on a locale's own domain. - **`referencedBy` follows each page** (§4): a translation resolves with its same-site original while the view shows that original, and without it otherwise, as its page reads it through the gated `entryById`. Ties on date order by id, so both sources agree on order and on what `limit` keeps. -### Amended in 3b (2026-11-12) +### Amended in 3b (2026-11-12, after the PR #212 review) - **Seeds purge their own groups' members** (§2 Purges, point 1). A translation inheriting a changed original's reference shows a new card, but it is neither a referrer of the seed nor changed itself. -- **The marker rule follows the short-circuit.** A field-operation marker purges everything only on sites whose `from` or `to` site node defines a reference. Sites without references keep today's purge, as the short-circuit promises. A marker's own publish changes the site node, which already purges the site. -- **The node budget** (`maxNodes`) counts the entry bodies the closure holds across both snapshots, and the node count of the one bundle it reads. -- **The release re-plan** plans from the pointer the batch moved (`from`, the live snapshot it replaced) to the new snapshot. That is `baseSnapshotId` whenever something was live. A first publish purges everything, as before. `RELEASE_EXECUTOR_COST` rises from 16 to 18 (two `getSnapshot` reads); bundle shards and node bodies are blob reads, bounded by `maxReads`. +- **No seeds, no reads.** A publish that changes no entry and no shown asset (term-only, asset-only, site-only) reads nothing for the closure, so the budgets cannot turn it into purge-everything. +- **Which sites count.** A site counts as having references when its `from` or `to` site node defines a reference field, a repeater with a reference sub-field, or a field (or `sub`) still converting *from* references (`migration.fromType: "reference"`). During `change_field_type` reference → text, the converted values render their targets' titles through the conversion maps until phase B rewrites them. On such sites any field-operation marker purges everything; other sites keep today's purge. +- **The closure's own reads.** Bodies the planner already holds are reused. The missing hashes are read by the closure itself (`readSnapshotBundle` with `only`): + - six at a time; + - counted against `maxReads`; + - added to the same byte total as the bundle before each parse; + - cut to what the closure needs, and never cached whole. +- **The node budget** (`maxNodes`) counts the entry bodies the closure holds across both snapshots, plus the node count of the one bundle it reads. +- **Release runs.** + - The purge is re-planned after the batch from the pointer it moved (`from`, the live snapshot it replaced). That is `baseSnapshotId` whenever something was live; a first publish purges everything. + - The re-plan runs over the tick's counting driver, never refused. If it throws, the run sends purge-everything, never the stored plan. + - First, the run writes its `scheduled` snapshot's bundle, best effort, for snapshots of at most 1,000 nodes. So back-to-back releases each read one bundle. A snapshot left without one is passed to the planner as `unbundled`, so no shard reads are spent on it. + - `RELEASE_EXECUTOR_COST` is 26: the batch, `refreshSearch`'s 7, the re-plan's and the bundle's reads, and the bookkeeping. The rare site-moved retry stays outside it, as before. +- **The pin-time plan skips the closure** (`references: false`). It would read every body one by one, and for sites with references it is the batch's *expected* outcome only: the run-time re-plan, or purge-everything, covers referrers. diff --git a/packages/core-ts/src/bundles.ts b/packages/core-ts/src/bundles.ts index 47e976d6..bac4c3bb 100644 --- a/packages/core-ts/src/bundles.ts +++ b/packages/core-ts/src/bundles.ts @@ -69,6 +69,13 @@ export interface BundleReadOptions { readonly maxNodes?: number | undefined; /** What to keep of each body (default: the body). */ readonly keep?: ((body: unknown, ref: NodeRef) => unknown) | undefined; + /** + * Read these refs' bodies from `nodes/.json` instead of the shards, with the same budgets + * and `keep`, at most `BLOB_CONCURRENCY` at a time; a missing body throws. + */ + readonly only?: readonly NodeRef[] | undefined; + /** A running byte total shared across reads, so `maxBytes` bounds them all together. */ + readonly spent?: { bytes: number } | undefined; } /** A bundle read passed `BundleReadOptions.maxBytes` or `maxNodes`. */ @@ -85,14 +92,15 @@ export async function readSnapshotBundle( snapshot: Snapshot, options: BundleReadOptions = {}, ): Promise | undefined> { + if (options.only) return readNodeBodies(blobs, options.only, options); const count = bundleShardCount(snapshot); if (count === 0) return undefined; const { maxBytes, maxNodes, keep } = options; + const spent = options.spent ?? { bytes: 0 }; if (maxNodes !== undefined && snapshot.nodes.length > maxNodes) throw new BundleBudgetError(`${snapshot.nodes.length} nodes (more than ${maxNodes})`); const bodies = new Map(); let complete = true; - let bytesSeen = 0; const decoder = new TextDecoder(); try { await eachLimited( @@ -105,8 +113,8 @@ export async function readSnapshotBundle( complete = false; return; } - bytesSeen += bytes.byteLength; - if (maxBytes !== undefined && bytesSeen > maxBytes) + spent.bytes += bytes.byteLength; + if (maxBytes !== undefined && spent.bytes > maxBytes) throw new BundleBudgetError(`more than ${maxBytes} bytes of bodies`); const parsed = JSON.parse(decoder.decode(bytes)) as { v?: unknown; @@ -143,3 +151,28 @@ export async function readSnapshotBundle( } return complete ? bodies : undefined; } + +/** `only`: bodies node by node, budgets checked before each parse (`readSnapshotBundle`). */ +async function readNodeBodies( + blobs: BlobStore, + refs: readonly NodeRef[], + options: BundleReadOptions, +): Promise> { + const { maxBytes, maxNodes, keep } = options; + const spent = options.spent ?? { bytes: 0 }; + if (maxNodes !== undefined && refs.length > maxNodes) + throw new BundleBudgetError(`${refs.length} nodes (more than ${maxNodes})`); + const out = new Map(); + const decoder = new TextDecoder(); + await eachLimited([...refs], BLOB_CONCURRENCY, async (ref) => { + const bytes = await blobs.get(`nodes/${ref.hash}.json`); + if (!bytes) throw new Error(`node body ${ref.hash} not found`); + spent.bytes += bytes.byteLength; + if (maxBytes !== undefined && spent.bytes > maxBytes) + throw new BundleBudgetError(`more than ${maxBytes} bytes of bodies`); + const body = JSON.parse(decoder.decode(bytes)) as unknown; + const kept = keep ? keep(body, ref) : body; + if (kept !== undefined) out.set(ref.hash, kept); + }); + return out; +} diff --git a/packages/core-ts/src/references/purge.ts b/packages/core-ts/src/references/purge.ts index fd63f124..f962e81b 100644 --- a/packages/core-ts/src/references/purge.ts +++ b/packages/core-ts/src/references/purge.ts @@ -19,15 +19,17 @@ * and without it, since a page reads whichever its view shows), so a translation that only * inherits counts. * - * Reads: one hash-keyed cache over both snapshots, the planner's own (`cached`). It loads one - * bundle, `to`'s, else `from`'s (RFC-0010), streamed: each body is cut to what the closure needs - * as its shard is parsed, and the byte and node budgets are checked before each parse. Only the - * hashes still missing are read one by one, through the planner's counted reader. Shard reads count - * against `maxReads` too. + * Reads, only when something seeds it: one hash-keyed cache over both snapshots. It loads one + * bundle, `to`'s, else `from`'s (RFC-0010; none for a snapshot known to have none, `unbundled`), + * streamed: each body is cut to what the closure needs as its shard is parsed, and the byte and + * node budgets are checked before each parse. Bodies the planner already holds are reused; the + * hashes still missing are read by the closure itself (`only`, six at a time, the same budgets), + * never into the planner's cache of full bodies. Shard and node reads count against `maxReads`. * * Purges everything (`string` result) when a budget is passed, and whenever a field-operation * marker is on either site node (renames and conversions in flight read through markers). Skipped - * entirely for sites whose two snapshots define no reference (top-level or inside a repeater). + * entirely for sites whose two snapshots define no reference (top-level, inside a repeater, or + * still converting from one), and at a release's pin time (`references: false`). */ import type { ContentEntry, @@ -57,6 +59,13 @@ export interface ReferencePurgeLimits { readonly maxBodyBytes?: number | undefined; /** Entry bodies across both snapshots, and nodes of the bundle read (default 50,000). */ readonly maxNodes?: number | undefined; + /** Snapshots known to have no bundle (a release run could not write one): no shard reads. */ + readonly unbundled?: ReadonlySet | undefined; + /** + * `false`: skip the closure (a release's pin-time plan, which is the batch's expected outcome + * only; the run-time re-plan covers referrers, or purges everything). + */ + readonly references?: false | undefined; } export const DEFAULT_PURGE_MAX_BODY_BYTES = 32 * 1024 * 1024; @@ -89,11 +98,17 @@ type Slim = Pick Object.values(site.collections ?? {}).flatMap((c) => c?.fields ?? []); -/** A `reference` field, or a repeater with a `reference` sub-field. */ +/** + * A `reference` field, a repeater with a `reference` sub-field, or a field (or `sub`) still + * converting *from* references: during `change_field_type` reference → text its values render + * the targets' titles through the conversion maps until phase B rewrites them. + */ const namesEntries = (d: FieldDefinition): boolean => - d.type === "reference" || (d.type === "repeater" && d.fields.some((s) => s.type === "reference")); + d.type === "reference" || + (d.type === "repeater" && d.fields.some((s) => s.type === "reference")) || + d.migration?.fromType === "reference"; -/** True when some collection of the site holds references (top-level or in a repeater). */ +/** True when some collection of the site holds or converts references (top-level or in items). */ export function definesReferences(site: Pick): boolean { return defsOf(site as Site).some(namesEntries); } @@ -172,26 +187,42 @@ async function slimBodies( input: ReferencePurgeInput, slim: (body: unknown) => Slim, ): Promise> { - const { from, to, reader, limits } = input; + const { from, to, limits } = input; + const read = input.reader.readSnapshotBundle?.bind(input.reader); + if (!read) throw new Error("the snapshot reader cannot stream bodies"); const maxNodes = limits?.maxNodes ?? DEFAULT_PURGE_MAX_NODES; + // One byte total over the bundle and the node reads: nothing is parsed past it. + const budget = { + maxBytes: limits?.maxBodyBytes ?? DEFAULT_PURGE_MAX_BODY_BYTES, + maxNodes, + spent: { bytes: 0 }, + keep: (body: unknown, ref: NodeRef) => (ref.kind === "contentEntry" ? slim(body) : undefined), + }; const out = new Map(); const bundleOf = async (snapshot: Snapshot) => { - if (!reader.readSnapshotBundle) return undefined; + if (limits?.unbundled?.has(snapshot.id)) return undefined; input.spend(bundleShardCount(snapshot)); - return reader.readSnapshotBundle(snapshot, { - maxBytes: limits?.maxBodyBytes ?? DEFAULT_PURGE_MAX_BODY_BYTES, - maxNodes, - keep: (body, ref) => (ref.kind === "contentEntry" ? slim(body) : undefined), - }); + return read(snapshot, budget); }; const bundle = (await bundleOf(to)) ?? (await bundleOf(from)); for (const [hash, s] of bundle ?? []) out.set(hash, s as Slim); + // The rest: bodies the planner already holds, then the missing hashes, read by this closure + // alone (counted, budgeted, cut to `Slim`; never into the planner's cache of full bodies). + const missing = new Map(); for (const snapshot of [from, to]) for (const ref of snapshot.nodes) { - if (ref.kind !== "contentEntry" || out.has(ref.hash)) continue; - if (out.size >= maxNodes) throw new BundleBudgetError(`more than ${maxNodes} entries`); - out.set(ref.hash, slim(input.cached(ref.hash) ?? (await input.read(ref)))); + if (ref.kind !== "contentEntry" || out.has(ref.hash) || missing.has(ref.hash)) continue; + const held = input.cached(ref.hash); + if (held !== undefined) out.set(ref.hash, slim(held)); + else missing.set(ref.hash, ref); } + if (out.size + missing.size > maxNodes) + throw new BundleBudgetError(`more than ${maxNodes} entries`); + if (missing.size > 0) { + input.spend(missing.size); + const got = await read(to, { ...budget, only: [...missing.values()] }); + for (const [hash, s] of got ?? []) out.set(hash, s as Slim); + } return out; } @@ -201,6 +232,7 @@ async function slimBodies( */ export async function planReferencePurge(input: ReferencePurgeInput): Promise { const { from, to, read, tags } = input; + if (input.limits?.references === false) return undefined; const siteOf = (s: Snapshot) => { const ref = s.nodes.find((n) => n.kind === "site" && n.id === s.siteId); return ref ? read(ref) : Promise.resolve(undefined); @@ -209,6 +241,10 @@ export async function planReferencePurge(input: ReferencePurgeInput): Promise; try { bodies = await slimBodies(input, slimmer(keptKeys([a, b]))); diff --git a/packages/core-ts/src/references/release-purge.ts b/packages/core-ts/src/references/release-purge.ts index 56996e16..eec3806d 100644 --- a/packages/core-ts/src/references/release-purge.ts +++ b/packages/core-ts/src/references/release-purge.ts @@ -1,28 +1,29 @@ /** - * RFC-0045 §2 "Purges", point 4: a release run re-plans its purge at run time. The plan stored - * when the release was pinned diffs the snapshot that was live *then* against a virtual target; by - * the run, other publishes may have moved the pointer, and referrers (whose pages show the - * release's targets as cards) are found only from the two real snapshots' bodies. So after - * `publishRelease` commits, the executor plans `from → snapshotId` (`from`: the pointer the batch - * moved), exactly like every other publish path, and sends that. + * RFC-0045 §2 "Purges", point 4: what a release run purges, decided after its batch committed. * - * The stored plan still decides the outcome's `purge` written in the batch (it must be known - * before the batch); the executor corrects it when the re-plan purges differently. Reads: two - * `getSnapshot` statements, then bundle shards and node bodies (blob reads, bounded by - * `maxReads`), counted in `RELEASE_EXECUTOR_COST`. + * - **Re-planned at run time.** The plan stored when the release was pinned diffs the snapshot + * live *then* against a virtual target, and skips the referrer closure (it would read every + * body one by one): for sites with references it is the batch's *expected* outcome only. After + * the commit the executor plans `from → snapshotId` (`from`: the pointer the batch moved) like + * every other publish path, and sends that. + * - **Over the tick's counting driver.** The re-plan's statements count against the tick but are + * never refused, like the purge's bookkeeping, so a tick near its budget still purges. + * - **Never dropped.** A re-plan that throws (a transient error) sends purge-everything, never the + * stored plan, which may lack referrers. + * - **The scheduled snapshot's bundle** (RFC-0010) is written first, best effort, as + * `createSnapshot` writes one for `publish`: the re-plan, and the next release's (whose `from` + * this snapshot is), then read one bundle instead of every body. Snapshots of more than + * `RELEASE_BUNDLE_MAX_NODES` nodes are left without (the site Worker writes it on its first + * render), and the planner is told so it spends no shard reads on them. */ +import type { SqlDriver } from "@cloudhal/storage"; import type { CachePurgePlan } from "../cache-purge.ts"; import type { ReleasePurge } from "../release-apply.ts"; -import type { SiteGraphStore } from "../store.ts"; +import type { ReleasePurger } from "../releases.ts"; +import { SiteGraphStore } from "../store.ts"; -/** The purge a release run sends: planned now, from the pointer it moved to the snapshot it made. */ -export function releasePurgePlan( - graph: Pick, - published: { readonly from?: string | undefined; readonly snapshotId: string }, -): Promise { - const { from, snapshotId: to } = published; - return graph.planCachePurge({ ...(from ? { from } : {}), to }); -} +/** Above this many nodes a release run leaves its snapshot's bundle to the site Worker. */ +export const RELEASE_BUNDLE_MAX_NODES = 1000; /** What a plan does to the outcome's `purge` (`none` where nothing is cached or nothing changed). */ export function purgeOutcome(plan: CachePurgePlan, purging: boolean): ReleasePurge { @@ -30,3 +31,43 @@ export function purgeOutcome(plan: CachePurgePlan, purging: boolean): ReleasePur if (plan.mode === "everything") return "everything"; return plan.tags.length > 0 ? "tags" : "none"; } + +export interface ReleasePurgeInput { + /** The executor's graph (its blob store is the planner's). */ + readonly graph: SiteGraphStore; + /** The tick's counting driver; absent ("Publish now"): the graph's own. */ + readonly counted?: SqlDriver | undefined; + readonly purge: ReleasePurger; + readonly siteId: string; + /** The pointer the batch moved (absent: nothing was live, so everything). */ + readonly from?: string | undefined; + readonly snapshotId: string; + readonly log: (line: string) => void; +} + +/** Writes the bundle, re-plans and sends the purge; a throw only from the purger itself. */ +export async function sendReleasePurge(input: ReleasePurgeInput): Promise { + const { siteId, from, snapshotId: to, log } = input; + const planner = input.counted + ? new SiteGraphStore({ sql: input.counted, blobs: input.graph.blobs }) + : input.graph; + const unbundled = new Set(); + try { + const snapshot = await planner.getSnapshot(to); + if (!snapshot || snapshot.nodes.length > RELEASE_BUNDLE_MAX_NODES) unbundled.add(to); + else await planner.writeSnapshotBundle(snapshot, new Map()); + } catch (e) { + unbundled.add(to); + log(`schedules: no bundle for ${to}: ${e instanceof Error ? e.message : String(e)}`); + } + let plan: CachePurgePlan; + try { + plan = await planner.planCachePurge({ ...(from ? { from } : {}), to }, { unbundled }); + } catch (e) { + const reason = `the run-time purge plan failed: ${e instanceof Error ? e.message : String(e)}`; + log(`schedules: ${reason}`); + plan = { mode: "everything", reason }; + } + const outcome = purgeOutcome(plan, true); + return outcome === "none" ? "none" : input.purge(siteId, plan); +} diff --git a/packages/core-ts/src/release-executor.ts b/packages/core-ts/src/release-executor.ts index 98b94d9a..4f5b6f48 100644 --- a/packages/core-ts/src/release-executor.ts +++ b/packages/core-ts/src/release-executor.ts @@ -8,22 +8,26 @@ */ import type { ReleaseExecution, ReleaseExecutor } from "@cloudhal/schedules"; import type { SqlScheduleStore } from "@cloudhal/schedules/sql"; +import type { SqlDriver } from "@cloudhal/storage"; import type { CachePurgePlan } from "./cache-purge.ts"; -import { purgeOutcome, releasePurgePlan } from "./references/release-purge.ts"; +import { purgeOutcome, sendReleasePurge } from "./references/release-purge.ts"; import * as after from "./release-after.ts"; import type { ReleasePurge } from "./release-apply.ts"; import type { ReleasePurger } from "./releases.ts"; import type { SiteGraphStore } from "./store.ts"; /** - * Statements one release execution takes, for `runDue`'s budget check: the stored purge plan (1), - * the site, the live snapshot and the parent (3), the eight-statement batch, the run-time re-plan's - * two `getSnapshot` reads (RFC-0045 §2; its bundle shards and node bodies are blob reads, bounded - * by the planner's `maxReads`), and the purge's own bookkeeping after it (up to 4). A site that - * moved between the reads and the batch repeats the reads and the batch once; that rare path may - * end the tick early (`more`), never mid-batch. + * Statements one release execution takes, for `runDue`'s budget check. On the tick's refusing + * driver: the stored purge plan (1), the site, the live snapshot and the parent (3), the batch + * (up to 8, with asset blobs to root), and the search index brought to the new snapshot (7: its + * reads and one guarded batch, RFC-0019 §3). On its counting driver, never refused: the + * scheduled snapshot read for its bundle (1), the run-time re-plan's two `getSnapshot` reads + * (RFC-0045 §2; bundle shards and node bodies are blob reads, bounded by the planner's + * `maxReads`), and the purge's bookkeeping (up to 4). A site that moved between the reads and the + * batch repeats the reads and the batch once (11 more); that rare path may end the tick early + * (`more`), never mid-batch, and the purge after it runs on the counting driver regardless. */ -export const RELEASE_EXECUTOR_COST = 18; +export const RELEASE_EXECUTOR_COST = 26; export interface ReleaseExecutorOptions { readonly graph: SiteGraphStore; @@ -36,6 +40,11 @@ export interface ReleaseExecutorOptions { * gets its purge. */ readonly recordPurge?: ((releaseId: string, purge: ReleasePurge) => Promise) | undefined; + /** + * The tick's counting (never refusing) driver: the run-time purge plan reads over it, and, when + * `recordPurge` is absent, the outcome's correction is written over it. + */ + readonly counted?: SqlDriver | undefined; /** After a committed release's purge (`release-after.ts`; the Node server rebuilds, #150). */ readonly onPublished?: after.ReleasePublishedHook | undefined; readonly log?: ((line: string) => void) | undefined; @@ -44,8 +53,12 @@ export interface ReleaseExecutorOptions { /** The `ReleaseExecutor` of `runDue`: `publishRelease`, then the stored purge plan. */ export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { const log = o.log ?? (() => {}); + const counted = o.counted; const recordPurge = - o.recordPurge ?? ((id: string, purge: ReleasePurge) => o.schedules.recordPurge(id, purge)); + o.recordPurge ?? + (counted + ? (id: string, purge: ReleasePurge) => after.recordPurgeOutcome(counted, id, purge) + : (id: string, purge: ReleasePurge) => o.schedules.recordPurge(id, purge)); return { cost: RELEASE_EXECUTOR_COST, async execute(input): Promise { @@ -71,9 +84,10 @@ export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { let purge: ReleasePurge = expected; if (r.committed && o.purge) { try { - const now = await releasePurgePlan(o.graph, r); - purge = purgeOutcome(now, true); - if (purge !== "none") purge = await o.purge(input.release.siteId, now); + purge = await sendReleasePurge({ + ...{ graph: o.graph, counted, purge: o.purge, log }, + ...{ siteId: input.release.siteId, from: r.from, snapshotId: r.snapshotId }, + }); } catch (e) { purge = "failed"; log( diff --git a/packages/core-ts/src/releases.ts b/packages/core-ts/src/releases.ts index 731be999..00764e19 100644 --- a/packages/core-ts/src/releases.ts +++ b/packages/core-ts/src/releases.ts @@ -71,7 +71,7 @@ import { type BuiltinInputs, graphBuiltinRunner } from "./automations.ts"; import { type CachePurgePlan, planCachePurge } from "./cache-purge.ts"; import { NotFoundError } from "./errors.ts"; import type { NodeHold } from "./holds.ts"; -import * as after from "./release-after.ts"; +import type * as after from "./release-after.ts"; import type { ReleasePurge } from "./release-apply.ts"; import { releaseDepsOf } from "./release-deps.ts"; import { releaseExecutor } from "./release-executor.ts"; @@ -191,7 +191,7 @@ export async function runScheduledTick(o: ScheduledTickOptions): Promise after.recordPurgeOutcome(counted, id, p), + counted, onPublished: o.onReleasePublished, log: o.log, }), @@ -509,17 +509,13 @@ export class ReleaseService { }; } - /** Step 5: `planCachePurge(L → L ⊕ items)` over a virtual snapshot (bodies are in the blob store). */ + /** Step 5: the expected purge `L → L ⊕ items`, over a virtual snapshot, closure aside (RFC-0045). */ async #purgePlan( live: Snapshot | undefined, items: readonly ReleaseItem[], ): Promise { if (!live) return { mode: "everything", reason: "nothing is live yet" }; - const target: Snapshot = { - ...live, - id: "snap_releaseplan", - nodes: composeRelease(live.nodes, items), - }; + const target: Snapshot = { ...live, id: "snap_pin", nodes: composeRelease(live.nodes, items) }; return planCachePurge( { getSnapshot: async (id) => (id === live.id ? live : id === target.id ? target : undefined), @@ -527,6 +523,7 @@ export class ReleaseService { }, live.id, target.id, + { references: false }, ); } diff --git a/packages/core-ts/src/testing-releases.ts b/packages/core-ts/src/testing-releases.ts index 3ba30997..0f29f33c 100644 --- a/packages/core-ts/src/testing-releases.ts +++ b/packages/core-ts/src/testing-releases.ts @@ -532,6 +532,66 @@ export function releasesSuite( expect(tags).toContain(`cloudhal-node-${event.id}`); }); + it("a run-time re-plan that fails still purges everything, never drops the purge", async () => { + const s = await story(); + await scheduled([s.id]); + // after the release's batch, every snapshot read fails (a transient D1 error) + let committed = false; + const failing: SqlDriver = { + ...sql, + get: (q, p) => { + if (committed && /FROM snapshots/.test(q)) throw new Error("D1_ERROR: connection lost"); + return sql.get(q, p); + }, + all: (q, p) => sql.all(q, p), + run: (q, p) => sql.run(q, p), + exec: (q) => sql.exec(q), + close: () => sql.close(), + batch: async (statements) => { + const out = await sql.batch(statements); + if (statements.some((st) => /INSERT INTO snapshots/.test(st.sql))) committed = true; + return out; + }, + }; + const plans: CachePurgePlan[] = []; + const report = await tick(T, { + sql: failing, + purge: () => async (_siteId, plan) => { + plans.push(plan); + return "everything"; + }, + }); + expect(report.ran).toBe(1); + expect(committed).toBe(true); + expect(plans).toHaveLength(1); + expect(plans[0]?.mode).toBe("everything"); + const r = (await svc.list(site.id))[0] as Release; + expect(r.outcome).toMatchObject({ kind: "published", purge: "everything" }); + }); + + it("back-to-back releases: each run writes its snapshot's bundle, so the next plan reads it", async () => { + const first = await story(); + await scheduled([first.id]); + const plans: CachePurgePlan[] = []; + const purge = () => async (_siteId: string, plan: CachePurgePlan) => { + plans.push(plan); + return "tags" as const; + }; + await tick(T, { purge }); + const S1 = await live(); + expect(S1.reason).toBe("scheduled"); + expect(await store.readSnapshotBundle(S1)).toBeDefined(); + const second = await story({ slug: "second-story", title: "Second story" }); + await scheduled([second.id], { at: T + HOUR }); + await tick(T + HOUR, { purge }); + const S2 = await live(); + expect(S2.id).not.toBe(S1.id); + expect(await store.readSnapshotBundle(S2)).toBeDefined(); + expect(plans.map((p) => p.mode)).toEqual(["tags", "tags"]); + const tags = plans[1]?.mode === "tags" ? plans[1].tags : []; + expect(tags).toContain(`cloudhal-node-${second.id}`); + }); + it("sends the purge re-planned at run time and records a deferred purge", async () => { const s = await story(); await scheduled([s.id]); diff --git a/packages/core-ts/test/references-purge.test.ts b/packages/core-ts/test/references-purge.test.ts index 8329d077..a6a683ac 100644 --- a/packages/core-ts/test/references-purge.test.ts +++ b/packages/core-ts/test/references-purge.test.ts @@ -12,6 +12,7 @@ import type { NodeRef, Site, Snapshot, + Term, } from "@cloudhal/core-contracts"; import { MemoryBlobStore } from "@cloudhal/storage"; import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; @@ -21,7 +22,7 @@ import type { BundleReadOptions } from "../src/bundles.ts"; import { type CachePurgeOptions, planCachePurge, type SnapshotReader } from "../src/cache-purge.ts"; import { nodeCacheTag, siteCacheTag } from "../src/cache-tags.ts"; import { continueFieldOperation, runFieldOperation } from "../src/fields-operations/index.ts"; -import { nodeHash } from "../src/hash.ts"; +import { canonicalJson, nodeHash } from "../src/hash.ts"; import { PNG_1X1 } from "../src/media/fixtures.ts"; import { seedStarterSite } from "../src/seed.ts"; import { type PublishChange, SiteGraphStore } from "../src/store.ts"; @@ -344,4 +345,88 @@ describe("planCachePurge: the referrer closure (RFC-0045 §2)", () => { expect(done.done).toBe(true); expect(all(await plan(await publish()))).toBe(true); }); + + it("a publish that changes no entry and no shown asset reads nothing: a term's tags only", async () => { + await world(); + const news = (await store.getTermBySlug(site.id, "category", "news")) as Term; + await store.updateTerm({ ...news, labels: { en: { name: "Latest news" } } }); + const p = await plan(await publish()); + expect(p.mode).toBe("tags"); + expect(p.mode === "tags" ? p.tags : []).toContain(`cloudhal-term-${news.id}`); + expect(counts.bundles).toBe(0); + }); + + it("without bundles the closure's own reads keep to the byte budget", async () => { + const { hall } = await world(); + await edit(hall, { title: "Great hall" }); + const a = await snap("manual"); + await edit(hall, { title: "Grand hall" }); + const b = await snap("manual"); + const p = await plan({ from: a.id, to: b.id }, { maxBodyBytes: 300 }); + expect(p.mode === "everything" ? p.reason : "tags").toMatch(/too large/); + expect((await plan({ from: a.id, to: b.id })).mode).toBe("tags"); + }); + + it("the closure's own reads count against maxReads (not just the planner's)", async () => { + const { hall, event } = await world(); + for (let i = 0; i < 20; i++) await make("events", `filler-${i}`); + await edit(hall, { title: "Great hall" }); + const a = await snap("manual"); + await edit(hall, { title: "Grand hall" }); + const b = await snap("manual"); + const change = { from: a.id, to: b.id }; + // the planner's own reads, the closure aside + await plan(change, { references: false }); + const own = counts.nodes; + const tight = await plan(change, { maxReads: own + 5 }); + expect(tight.mode === "everything" ? tight.reason : "tags").toMatch(/node reads/); + expect((await tagsOf(change, { maxReads: own + 100 })).has(node(event))).toBe(true); + }); + + it("spends no shard reads on a snapshot known to have no bundle", async () => { + const { hall } = await world(); + await edit(hall, { title: "Main hall" }); + const s = await snap("manual"); + await tagsOf(await point(s.id), { unbundled: new Set([s.id]) }); + expect(counts.bundles).toBe(1); + }); + + it("a conversion from references counts as references: its marker purges everything", async () => { + const { hall } = await world(); + // `change_field_type` reference → text, phase A: the definition is text with the marker, and + // no reference field is left (written directly: the operation tools need rollout step 4) + const s = await fresh(); + const events = s.collections?.events; + const fields = (events?.fields ?? []).flatMap((d): FieldDefinition[] => + d.key === "venue" + ? [ + { + key: "venue", + type: "text", + labels: L("Venue"), + localized: false, + migration: { id: "fop_01HZX8K3Q9CD", fromType: "reference" }, + }, + ] + : d.type === "reference" + ? [] + : [d], + ); + const next = { + ...s, + rev: s.rev + 1, + collections: { ...s.collections, events: { ...events, fields } }, + } as unknown as Site; + await store.sql.run("UPDATE nodes SET rev = ?, hash = ?, body = ? WHERE id = ?", [ + next.rev, + await nodeHash(next), + canonicalJson(next), + s.id, + ]); + await publish(); + // a retitled target, both snapshots converting: its referrers' pages show its title + await edit(hall, { title: "Main hall" }); + const p = await plan(await publish()); + expect(p.mode === "everything" ? p.reason : "tags").toMatch(/field operation/); + }); }); From b4b19b0b0b9d96f78b472245489c560775c0e3d3 Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:28:05 +0300 Subject: [PATCH 4/6] fix(core-ts): RFC-0045 step 3b verification round 2 (PR #212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - release runs on sites without references behave exactly as before 3b: the stored pin-time plan, no extra read (decided on the site row publishRelease returns) - reference sites: re-plan within a per-tick blob allowance (400 Free, 4,000 Paid; ≤ 250 per re-plan), purge-everything past it; to's shards never read - the bundle write comes after the purge, only when from has a bundle, built from from's bodies plus ≤ 100 changed nodes; skipped when the re-plan found from bundle-less - RELEASE_EXECUTOR_COST 26 with recordPurge counted (the bundle's own snapshot read is gone), subrequest numbers documented beside it - a discarded partial bundle gives its bytes back; the no-seeds return precedes the site-node reads - tests: blob gets counted through a wrapped BlobStore (no-reference site equals the executor-less run; back-to-back releases with from bundled and bundle-less), the shard spend pinned at own+shards, the byte give-back; changeset and RFC amended Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/rfc45-step3b-purges.md | 12 +- ...ctions-stage-4-references-and-repeaters.md | 15 +- packages/core-ts/src/references/purge.ts | 12 +- .../core-ts/src/references/release-purge.ts | 147 ++++++++++++---- packages/core-ts/src/release-executor.ts | 28 ++- packages/core-ts/src/releases.ts | 2 + packages/core-ts/src/testing-releases.ts | 164 +++++++++++++----- .../core-ts/test/references-purge.test.ts | 40 ++++- 8 files changed, 320 insertions(+), 100 deletions(-) diff --git a/.changeset/rfc45-step3b-purges.md b/.changeset/rfc45-step3b-purges.md index ee2e2ab8..a624854b 100644 --- a/.changeset/rfc45-step3b-purges.md +++ b/.changeset/rfc45-step3b-purges.md @@ -2,7 +2,7 @@ "@cloudhal/core-ts": patch --- -RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothing changes for sites without reference definitions: their purge plans are the same, and no bundle is read. +RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothing changes for sites without reference definitions. Their publish purge plans are the same, with no bundle read. Their release runs send the plan stored at pin time, as before, with no extra read, re-plan or bundle write. - **The referrer closure** (`references/purge.ts`, called by `planCachePurge`). - The seeds are the groups of every changed, added or removed entry, and of every entry whose shown asset changed. @@ -22,8 +22,8 @@ RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothin So does any field-operation marker on either snapshot's site node, for sites that hold references or convert from them. - **`readSnapshotBundle`** (`SnapshotReader.readSnapshotBundle`, `SiteGraphStore.readSnapshotBundle`, `readSnapshotBundle` in bundles) takes optional `maxBytes`, `maxNodes`, `keep`, `only` (bodies node by node) and `spent` (a shared byte total). A passed budget throws `BundleBudgetError` before the shard is parsed. No new cache tags. -- **Release runs re-plan their purge at run time** (`references/release-purge.ts`), from the pointer they moved to the snapshot they made, instead of sending the plan stored at pin time. - - The re-plan runs over the tick's counting driver; if it fails, the run purges everything. - - Each run first writes its snapshot's bundle, best effort, for snapshots of at most 1,000 nodes. - - The pin-time plan skips the closure (`CachePurgeOptions.references: false`): for sites with references it is the expected outcome only. - - `RELEASE_EXECUTOR_COST` is 26 (it counts `refreshSearch` now). +- **Release runs on sites with references re-plan their purge at run time** (`references/release-purge.ts`), from the pointer they moved to the snapshot they made, instead of sending the plan stored at pin time. + - The re-plan runs over the tick's counting driver, within the tick's blob allowance (400 on Workers Free, 4,000 on Paid; at most 250 reads per re-plan). If it fails, the run purges everything. + - After the purge, the run writes its snapshot's bundle, best effort, when `from` has one: from `from`'s bodies plus at most 100 changed nodes, for snapshots of at most 1,000 nodes. + - The pin-time plan skips the closure (`CachePurgeOptions.references: false`): for these sites it is the expected outcome only. + - `RELEASE_EXECUTOR_COST` is 26 (it counts `refreshSearch` and `recordPurge` now). diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index d71d3896..ab0700f8 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -1223,8 +1223,15 @@ Rollout step 2 shipped in two PRs: 2a, store and integrity, and 2b, the consumer - cut to what the closure needs, and never cached whole. - **The node budget** (`maxNodes`) counts the entry bodies the closure holds across both snapshots, plus the node count of the one bundle it reads. - **Release runs.** - - The purge is re-planned after the batch from the pointer it moved (`from`, the live snapshot it replaced). That is `baseSnapshotId` whenever something was live; a first publish purges everything. - - The re-plan runs over the tick's counting driver, never refused. If it throws, the run sends purge-everything, never the stored plan. - - First, the run writes its `scheduled` snapshot's bundle, best effort, for snapshots of at most 1,000 nodes. So back-to-back releases each read one bundle. A snapshot left without one is passed to the planner as `unbundled`, so no shard reads are spent on it. - - `RELEASE_EXECUTOR_COST` is 26: the batch, `refreshSearch`'s 7, the re-plan's and the bundle's reads, and the bookkeeping. The rare site-moved retry stays outside it, as before. + - **Sites without references behave exactly as before 3b.** The run sends the plan stored at pin time, or nothing when it purges nothing, and makes no extra read. The test is the site node the release publishes, which `publishRelease` returns. A site whose last reference field was removed since `from` gets the stored plan, as every site did before. + - **Sites with references** (or converting from them) re-plan after the batch, from the pointer it moved (`from`, the live snapshot it replaced). That is `baseSnapshotId` whenever something was live; a first publish purges everything. + - The re-plan runs over the tick's counting driver, never refused, and spends no shard reads on `to`, which has no bundle yet. + - If it throws, or the tick's blob allowance is spent, the run purges everything, never the stored plan. + - **Order:** send the purge first, then write `to`'s bundle, best effort, for the next run. The write happens only when `from` has a bundle. It is built from `from`'s bodies plus at most 100 changed nodes fetched one by one, for snapshots of at most 1,000 nodes. This run's re-plan reads `from`'s bundle and the changed items. + - **Subrequests:** a tick's release runs share one blob allowance: 400 on Workers Free, under its 1,000 R2 and D1 calls per invocation together with the tick's 40 statements; 4,000 on Paid, under its default 10,000 with 900. A re-plan reads at most 250 blobs, a bundle write at most 108. + - **`RELEASE_EXECUTOR_COST` is 26:** + - the stored plan (1), the reads (3), the batch (8) and `refreshSearch` (7); + - then the re-plan's two `getSnapshot` reads (the bundle write reuses them), the purge's bookkeeping (4) and `recordPurge` (1). + + The rare site-moved retry stays outside it, as before. - **The pin-time plan skips the closure** (`references: false`). It would read every body one by one, and for sites with references it is the batch's *expected* outcome only: the run-time re-plan, or purge-everything, covers referrers. diff --git a/packages/core-ts/src/references/purge.ts b/packages/core-ts/src/references/purge.ts index f962e81b..14f64de2 100644 --- a/packages/core-ts/src/references/purge.ts +++ b/packages/core-ts/src/references/purge.ts @@ -202,7 +202,11 @@ async function slimBodies( const bundleOf = async (snapshot: Snapshot) => { if (limits?.unbundled?.has(snapshot.id)) return undefined; input.spend(bundleShardCount(snapshot)); - return read(snapshot, budget); + const before = budget.spent.bytes; + const got = await read(snapshot, budget); + // A partial or mismatching bundle is discarded: only bodies kept count against the bytes. + if (!got) budget.spent.bytes = before; + return got; }; const bundle = (await bundleOf(to)) ?? (await bundleOf(from)); for (const [hash, s] of bundle ?? []) out.set(hash, s as Slim); @@ -233,6 +237,9 @@ async function slimBodies( export async function planReferencePurge(input: ReferencePurgeInput): Promise { const { from, to, read, tags } = input; if (input.limits?.references === false) return undefined; + // Nothing changed that a card shows: no entry and no shown asset (a term-only, asset-only or + // site-only publish). The closure could add no tag, so it reads nothing at all. + if (input.entries.length === 0 && input.reached.size === 0) return undefined; const siteOf = (s: Snapshot) => { const ref = s.nodes.find((n) => n.kind === "site" && n.id === s.siteId); return ref ? read(ref) : Promise.resolve(undefined); @@ -241,9 +248,6 @@ export async function planReferencePurge(input: ReferencePurgeInput): Promise; try { diff --git a/packages/core-ts/src/references/release-purge.ts b/packages/core-ts/src/references/release-purge.ts index eec3806d..c8e5222a 100644 --- a/packages/core-ts/src/references/release-purge.ts +++ b/packages/core-ts/src/references/release-purge.ts @@ -1,29 +1,47 @@ /** * RFC-0045 §2 "Purges", point 4: what a release run purges, decided after its batch committed. * - * - **Re-planned at run time.** The plan stored when the release was pinned diffs the snapshot - * live *then* against a virtual target, and skips the referrer closure (it would read every - * body one by one): for sites with references it is the batch's *expected* outcome only. After - * the commit the executor plans `from → snapshotId` (`from`: the pointer the batch moved) like - * every other publish path, and sends that. - * - **Over the tick's counting driver.** The re-plan's statements count against the tick but are - * never refused, like the purge's bookkeeping, so a tick near its budget still purges. - * - **Never dropped.** A re-plan that throws (a transient error) sends purge-everything, never the - * stored plan, which may lack referrers. - * - **The scheduled snapshot's bundle** (RFC-0010) is written first, best effort, as - * `createSnapshot` writes one for `publish`: the re-plan, and the next release's (whose `from` - * this snapshot is), then read one bundle instead of every body. Snapshots of more than - * `RELEASE_BUNDLE_MAX_NODES` nodes are left without (the site Worker writes it on its first - * render), and the planner is told so it spends no shard reads on them. + * - **Sites without references: exactly as before 3b.** The plan stored at pin time is sent (nothing + * when it purges nothing), with no read at all: the test is the site row `publishRelease` + * returns, which is the site node the release publishes. (A site whose last reference field was + * removed since `from` gets the stored plan, as every site did before 3b.) + * - **Sites with references (or converting from them): re-planned** `from → snapshotId` (`from`: + * the pointer the batch moved), like every other publish path. The pin-time plan skips the + * referrer closure and is the batch's *expected* outcome only. The re-plan reads over the tick's + * counting driver (never refused) and, for blobs, within the run's share of the tick's blob + * allowance; past it, or if it throws, the run purges everything, never the stored plan. + * - **Then the scheduled snapshot's bundle** (RFC-0010), best effort and bounded, so the *next* + * release run (whose `from` this snapshot is) reads one bundle: only for reference sites, only + * when `from` has a bundle, from `from`'s bodies plus at most `RELEASE_BUNDLE_MAX_FETCH` changed + * nodes, for snapshots of at most `RELEASE_BUNDLE_MAX_NODES` nodes. This run's re-plan reads + * `from`'s bundle and the items, and spends no shard reads on `to` (`unbundled`). + * + * **Subrequests.** R2 and D1 calls are capped per invocation: 1,000 on Workers Free, 10,000 by + * default on Paid (RFC-0010, ADR-0024). A tick spends at most its `maxQueries` D1 statements (40 + * Free, 900 Paid) and the release runs' blob allowance (`releaseBlobAllowance`: 400 Free, 4,000 + * Paid, shared by every run of the tick), plus the purge's own calls: well under either cap. */ +import type { NodeRef, Site, Snapshot } from "@cloudhal/core-contracts"; import type { SqlDriver } from "@cloudhal/storage"; -import type { CachePurgePlan } from "../cache-purge.ts"; +import { type CachePurgePlan, planCachePurge, type SnapshotReader } from "../cache-purge.ts"; import type { ReleasePurge } from "../release-apply.ts"; import type { ReleasePurger } from "../releases.ts"; import { SiteGraphStore } from "../store.ts"; +import { definesReferences } from "./purge.ts"; /** Above this many nodes a release run leaves its snapshot's bundle to the site Worker. */ export const RELEASE_BUNDLE_MAX_NODES = 1000; +/** Changed nodes a bundle write may fetch one by one (the rest come from `from`'s bundle). */ +export const RELEASE_BUNDLE_MAX_FETCH = 100; +/** The most blob reads one re-plan may make (node bodies and bundle shards). */ +export const RELEASE_REPLAN_MAX_READS = 250; +/** What a bundle write may cost in blob calls: `from`'s shards, the fetches, `to`'s shards. */ +const BUNDLE_WRITE_COST = 2 * Math.ceil(RELEASE_BUNDLE_MAX_NODES / 256) + RELEASE_BUNDLE_MAX_FETCH; + +/** The blob calls one tick's release runs may make together (see the module comment). */ +export function releaseBlobAllowance(budget?: { readonly maxQueries: number }): number { + return budget && budget.maxQueries > 40 ? 4000 : 400; +} /** What a plan does to the outcome's `purge` (`none` where nothing is cached or nothing changed). */ export function purgeOutcome(plan: CachePurgePlan, purging: boolean): ReleasePurge { @@ -38,36 +56,103 @@ export interface ReleasePurgeInput { /** The tick's counting driver; absent ("Publish now"): the graph's own. */ readonly counted?: SqlDriver | undefined; readonly purge: ReleasePurger; - readonly siteId: string; + /** The plan stored at pin time. */ + readonly stored: CachePurgePlan; + /** The site row the batch wrote (`to`'s site node). */ + readonly site: Site; /** The pointer the batch moved (absent: nothing was live, so everything). */ readonly from?: string | undefined; readonly snapshotId: string; + /** Blob calls left for this tick's release runs; spent here. */ + readonly allowance: { left: number }; readonly log: (line: string) => void; } -/** Writes the bundle, re-plans and sends the purge; a throw only from the purger itself. */ +const message = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +/** Sends the run's purge (see the module comment); a throw only from the purger itself. */ export async function sendReleasePurge(input: ReleasePurgeInput): Promise { - const { siteId, from, snapshotId: to, log } = input; + const { site, allowance, log } = input; const planner = input.counted ? new SiteGraphStore({ sql: input.counted, blobs: input.graph.blobs }) : input.graph; - const unbundled = new Set(); + const send = (plan: CachePurgePlan) => + purgeOutcome(plan, true) === "none" + ? Promise.resolve("none" as const) + : input.purge(site.id, plan); + if (!definesReferences(site)) return send(input.stored); + let from: Snapshot | undefined; + let to: Snapshot | undefined; + try { + [from, to] = await Promise.all([ + input.from ? planner.getSnapshot(input.from) : undefined, + planner.getSnapshot(input.snapshotId), + ]); + } catch (e) { + log(`schedules: the release's snapshots could not be read: ${message(e)}`); + return send({ mode: "everything", reason: `snapshots unreadable: ${message(e)}` }); + } + const pair: Pair = { from, to, allowance, log }; + const plan = await replan(planner, pair); + const sent = await send(plan); + if (from && to && pair.fromBundle !== "missing") + await writeBundle(planner, { ...pair, from, to }); + return sent; +} + +interface Pair { + readonly from: Snapshot | undefined; + readonly to: Snapshot | undefined; + readonly allowance: { left: number }; + readonly log: (line: string) => void; + /** What the re-plan found of `from`'s bundle (unset: it did not look). */ + fromBundle?: "present" | "missing"; +} + +/** The run-time plan within the run's blob share; purge-everything when it cannot be made. */ +async function replan(planner: SiteGraphStore, p: Pair): Promise { + const { from, to } = p; + if (!from || !to) return { mode: "everything", reason: "nothing was live before the release" }; + const maxReads = Math.min(RELEASE_REPLAN_MAX_READS, p.allowance.left); + p.allowance.left -= maxReads; + if (maxReads < 10) return { mode: "everything", reason: "the tick's blob allowance is spent" }; + const reader: SnapshotReader = { + getSnapshot: async (id) => (id === from.id ? from : id === to.id ? to : undefined), + readSnapshotNode: (ref: NodeRef) => planner.readSnapshotNode(ref), + readSnapshotBundle: async (s, o) => { + const got = await planner.readSnapshotBundle(s, o); + if (s.id === from.id && !o?.only) p.fromBundle = got ? "present" : "missing"; + return got; + }, + }; try { - const snapshot = await planner.getSnapshot(to); - if (!snapshot || snapshot.nodes.length > RELEASE_BUNDLE_MAX_NODES) unbundled.add(to); - else await planner.writeSnapshotBundle(snapshot, new Map()); + // `to` has no bundle yet: it is written after the purge, for the next run. + return await planCachePurge(reader, from.id, to.id, { maxReads, unbundled: new Set([to.id]) }); } catch (e) { - unbundled.add(to); - log(`schedules: no bundle for ${to}: ${e instanceof Error ? e.message : String(e)}`); + const reason = `the run-time purge plan failed: ${message(e)}`; + p.log(`schedules: ${reason}`); + return { mode: "everything", reason }; } - let plan: CachePurgePlan; +} + +/** `to`'s bundle from `from`'s bodies plus the few changed nodes; skipped when it would cost more. */ +async function writeBundle(planner: SiteGraphStore, p: Pair & { from: Snapshot; to: Snapshot }) { + const { from, to } = p; + if (to.nodes.length > RELEASE_BUNDLE_MAX_NODES || from.nodes.length > RELEASE_BUNDLE_MAX_NODES) + return; + if (p.allowance.left < BUNDLE_WRITE_COST) return; + p.allowance.left -= BUNDLE_WRITE_COST; try { - plan = await planner.planCachePurge({ ...(from ? { from } : {}), to }, { unbundled }); + const bodies = await planner.readSnapshotBundle(from, { maxNodes: RELEASE_BUNDLE_MAX_NODES }); + if (!bodies) return; + const known = new Map(); + for (const ref of to.nodes) { + const body = bodies.get(ref.hash); + if (body !== undefined) known.set(ref.hash, JSON.stringify(body)); + } + if (to.nodes.length - known.size > RELEASE_BUNDLE_MAX_FETCH) return; + await planner.writeSnapshotBundle(to, known); } catch (e) { - const reason = `the run-time purge plan failed: ${e instanceof Error ? e.message : String(e)}`; - log(`schedules: ${reason}`); - plan = { mode: "everything", reason }; + p.log(`schedules: no bundle for ${to.id}: ${message(e)}`); } - const outcome = purgeOutcome(plan, true); - return outcome === "none" ? "none" : input.purge(siteId, plan); } diff --git a/packages/core-ts/src/release-executor.ts b/packages/core-ts/src/release-executor.ts index 4f5b6f48..a970e2fb 100644 --- a/packages/core-ts/src/release-executor.ts +++ b/packages/core-ts/src/release-executor.ts @@ -10,7 +10,11 @@ import type { ReleaseExecution, ReleaseExecutor } from "@cloudhal/schedules"; import type { SqlScheduleStore } from "@cloudhal/schedules/sql"; import type { SqlDriver } from "@cloudhal/storage"; import type { CachePurgePlan } from "./cache-purge.ts"; -import { purgeOutcome, sendReleasePurge } from "./references/release-purge.ts"; +import { + purgeOutcome, + releaseBlobAllowance, + sendReleasePurge, +} from "./references/release-purge.ts"; import * as after from "./release-after.ts"; import type { ReleasePurge } from "./release-apply.ts"; import type { ReleasePurger } from "./releases.ts"; @@ -20,12 +24,18 @@ import type { SiteGraphStore } from "./store.ts"; * Statements one release execution takes, for `runDue`'s budget check. On the tick's refusing * driver: the stored purge plan (1), the site, the live snapshot and the parent (3), the batch * (up to 8, with asset blobs to root), and the search index brought to the new snapshot (7: its - * reads and one guarded batch, RFC-0019 §3). On its counting driver, never refused: the - * scheduled snapshot read for its bundle (1), the run-time re-plan's two `getSnapshot` reads - * (RFC-0045 §2; bundle shards and node bodies are blob reads, bounded by the planner's - * `maxReads`), and the purge's bookkeeping (up to 4). A site that moved between the reads and the + * reads and one guarded batch, RFC-0019 §3). On its counting driver, never refused: for sites + * with references, the run-time re-plan's two `getSnapshot` reads (RFC-0045 §2; the bundle + * write reuses them); the purge's bookkeeping (up to 4); and `recordPurge`, correcting the + * outcome after a deferred or differing purge (1): 1 + 3 + 8 + 7 + 2 + 4 + 1 = 26. A site that moved between the reads and the * batch repeats the reads and the batch once (11 more); that rare path may end the tick early * (`more`), never mid-batch, and the purge after it runs on the counting driver regardless. + * + * Blob calls (R2 subrequests) are bounded apart: a tick's release runs share + * `releaseBlobAllowance` (400 on Workers Free, under its 1,000 subrequests with the tick's 40 D1 + * statements; 4,000 on Paid, under its default 10,000 with 900). A re-plan reads at most + * `RELEASE_REPLAN_MAX_READS` (250) blobs, a bundle write at most 108; past the allowance the run + * purges everything (`references/release-purge.ts`). */ export const RELEASE_EXECUTOR_COST = 26; @@ -45,6 +55,8 @@ export interface ReleaseExecutorOptions { * `recordPurge` is absent, the outcome's correction is written over it. */ readonly counted?: SqlDriver | undefined; + /** Blob calls this executor's runs may make together (`releaseBlobAllowance`; default Free's). */ + readonly blobReads?: number | undefined; /** After a committed release's purge (`release-after.ts`; the Node server rebuilds, #150). */ readonly onPublished?: after.ReleasePublishedHook | undefined; readonly log?: ((line: string) => void) | undefined; @@ -54,6 +66,8 @@ export interface ReleaseExecutorOptions { export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { const log = o.log ?? (() => {}); const counted = o.counted; + // RFC-0045: the blob calls every release run of this tick may make together. + const allowance = { left: o.blobReads ?? releaseBlobAllowance() }; const recordPurge = o.recordPurge ?? (counted @@ -85,8 +99,8 @@ export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { if (r.committed && o.purge) { try { purge = await sendReleasePurge({ - ...{ graph: o.graph, counted, purge: o.purge, log }, - ...{ siteId: input.release.siteId, from: r.from, snapshotId: r.snapshotId }, + ...{ graph: o.graph, counted, purge: o.purge, stored: plan, allowance, log }, + ...{ site: r.site, from: r.from, snapshotId: r.snapshotId }, }); } catch (e) { purge = "failed"; diff --git a/packages/core-ts/src/releases.ts b/packages/core-ts/src/releases.ts index 00764e19..026ee6ec 100644 --- a/packages/core-ts/src/releases.ts +++ b/packages/core-ts/src/releases.ts @@ -71,6 +71,7 @@ import { type BuiltinInputs, graphBuiltinRunner } from "./automations.ts"; import { type CachePurgePlan, planCachePurge } from "./cache-purge.ts"; import { NotFoundError } from "./errors.ts"; import type { NodeHold } from "./holds.ts"; +import { releaseBlobAllowance } from "./references/release-purge.ts"; import type * as after from "./release-after.ts"; import type { ReleasePurge } from "./release-apply.ts"; import { releaseDepsOf } from "./release-deps.ts"; @@ -192,6 +193,7 @@ export async function runScheduledTick(o: ScheduledTickOptions): Promise { + /** RFC-0045: a `venue` reference from events to places (enabled on this suite's store). */ + async function venueSite() { const s = await fresh(); const venue: FieldDefinition = { key: "venue", @@ -508,33 +510,79 @@ export function releasesSuite( collections: { ...s.collections, places: {}, events: { fields: [venue] } }, }); const hall = await story({ collection: "places", slug: "hall", title: "Hall" }); - await publishSite(); - // the release retitles the hall; at pin time nothing names it - await store.updateContentEntry({ ...hall, title: "Main hall" }); - await scheduled([hall.id]); - // a referrer goes live after the pin, by an ordinary publish const event = await story({ collection: "events", slug: "launch", fields: { venue: hall.id }, }); await publishSite(); - const plans: CachePurgePlan[] = []; - const report = await tick(T, { - purge: () => async (_siteId, plan) => { - plans.push(plan); - return "tags"; + return { hall, event }; + } + /** A blob store that counts gets by key. */ + const counting = () => { + const gets: string[] = []; + const wrapped: BlobStore = new Proxy(blobs, { + get(target, prop, receiver) { + if (prop === "get") + return (key: string) => { + gets.push(key); + return target.get(key); + }; + const v = Reflect.get(target, prop, receiver); + return typeof v === "function" ? v.bind(target) : v; }, }); - expect(report.ran).toBe(1); + return { gets, blobs: wrapped }; + }; + const capture = + (plans: CachePurgePlan[], answer: "tags" | "everything" = "tags") => + () => + async (_siteId: string, plan: CachePurgePlan) => { + plans.push(plan); + return answer; + }; + + it("re-plans the purge at run time: a referrer published after the pin is purged (RFC-0045)", async () => { + const { hall } = await venueSite(); + // the release retitles the hall; a referrer goes live after the pin, by an ordinary publish + await store.updateContentEntry({ + ...((await store.getContentEntry(hall.id)) as ContentEntry), + title: "Main hall", + }); + await scheduled([hall.id]); + const late = await story({ collection: "events", slug: "late", fields: { venue: hall.id } }); + await publishSite(); + const plans: CachePurgePlan[] = []; + expect((await tick(T, { purge: capture(plans) })).ran).toBe(1); const tags = plans[0]?.mode === "tags" ? plans[0].tags : []; expect(tags).toContain(`cloudhal-node-${hall.id}`); - expect(tags).toContain(`cloudhal-node-${event.id}`); + expect(tags).toContain(`cloudhal-node-${late.id}`); + }); + + it("a site without references sends the pin-time plan and reads no more blobs than before", async () => { + // the same release twice: without a purger (Node: the executor sends nothing), then with + const first = await story(); + await scheduled([first.id]); + const quiet = counting(); + expect((await tick(T, { blobs: quiet.blobs })).ran).toBe(1); + const second = await story({ slug: "second-story", title: "Second story" }); + const { release } = await scheduled([second.id], { at: T + HOUR }); + const stored = await svc.schedules.releasePurgePlan(release.id); + const c = counting(); + const plans: CachePurgePlan[] = []; + expect((await tick(T + HOUR, { blobs: c.blobs, purge: capture(plans) })).ran).toBe(1); + expect(plans).toEqual([stored]); + expect(c.gets.filter((k) => k.startsWith("bundles/"))).toEqual([]); + expect(c.gets).toHaveLength(quiet.gets.length); }); it("a run-time re-plan that fails still purges everything, never drops the purge", async () => { - const s = await story(); - await scheduled([s.id]); + const { hall } = await venueSite(); + await store.updateContentEntry({ + ...((await store.getContentEntry(hall.id)) as ContentEntry), + title: "Main hall", + }); + await scheduled([hall.id]); // after the release's batch, every snapshot read fails (a transient D1 error) let committed = false; const failing: SqlDriver = { @@ -554,42 +602,78 @@ export function releasesSuite( }, }; const plans: CachePurgePlan[] = []; - const report = await tick(T, { - sql: failing, - purge: () => async (_siteId, plan) => { - plans.push(plan); - return "everything"; - }, - }); + const report = await tick(T, { sql: failing, purge: capture(plans, "everything") }); expect(report.ran).toBe(1); expect(committed).toBe(true); - expect(plans).toHaveLength(1); - expect(plans[0]?.mode).toBe("everything"); + expect(plans.map((p) => p.mode)).toEqual(["everything"]); const r = (await svc.list(site.id))[0] as Release; expect(r.outcome).toMatchObject({ kind: "published", purge: "everything" }); }); - it("back-to-back releases: each run writes its snapshot's bundle, so the next plan reads it", async () => { - const first = await story(); - await scheduled([first.id]); + it("back-to-back releases on a reference site: one bundle each, never to's shards", async () => { + const { hall, event } = await venueSite(); + const edit = async (title: string) => + store.updateContentEntry({ + ...((await store.getContentEntry(hall.id)) as ContentEntry), + title, + }); + const shardsOf = (gets: string[], id: string) => gets.filter((k) => k.includes(`/${id}/`)); + // 1: `from` (a publish) has a bundle; `to` gets one after the purge + const L = await live(); + await edit("Main hall"); + await scheduled([hall.id]); + const c1 = counting(); const plans: CachePurgePlan[] = []; - const purge = () => async (_siteId: string, plan: CachePurgePlan) => { - plans.push(plan); - return "tags" as const; - }; - await tick(T, { purge }); + await tick(T, { blobs: c1.blobs, purge: capture(plans) }); const S1 = await live(); expect(S1.reason).toBe("scheduled"); + expect(shardsOf(c1.gets, S1.id)).toEqual([]); + expect(shardsOf(c1.gets, L.id).length).toBeGreaterThan(0); expect(await store.readSnapshotBundle(S1)).toBeDefined(); - const second = await story({ slug: "second-story", title: "Second story" }); - await scheduled([second.id], { at: T + HOUR }); - await tick(T + HOUR, { purge }); + // 2: `from` is the first release's scheduled snapshot, bundled by its run + await edit("Great hall"); + await scheduled([hall.id], { at: T + HOUR }); + const c2 = counting(); + await tick(T + HOUR, { blobs: c2.blobs, purge: capture(plans) }); const S2 = await live(); - expect(S2.id).not.toBe(S1.id); - expect(await store.readSnapshotBundle(S2)).toBeDefined(); - expect(plans.map((p) => p.mode)).toEqual(["tags", "tags"]); - const tags = plans[1]?.mode === "tags" ? plans[1].tags : []; - expect(tags).toContain(`cloudhal-node-${second.id}`); + expect(shardsOf(c2.gets, S2.id)).toEqual([]); + // `from`'s shards: once for the re-plan, once to build `to`'s bundle (the rest of `to` + // comes from them; only changed nodes are fetched one by one) + expect(shardsOf(c2.gets, S1.id)).toHaveLength(2 * bundleShardCount(S1)); + for (const p of plans.slice(0, 2)) { + expect(p.mode).toBe("tags"); + expect(p.mode === "tags" ? p.tags : []).toContain(`cloudhal-node-${event.id}`); + } + }); + + it("back-to-back releases where from has no bundle: from's shards tried once, to's never", async () => { + const { hall, event } = await venueSite(); + const edit = async (title: string) => + store.updateContentEntry({ + ...((await store.getContentEntry(hall.id)) as ContentEntry), + title, + }); + // a first release run with nothing to purge (Node): its snapshot gets no bundle + await edit("Main hall"); + await scheduled([hall.id]); + await tick(T); + const S1 = await live(); + expect(await store.readSnapshotBundle(S1)).toBeUndefined(); + await edit("Great hall"); + await scheduled([hall.id], { at: T + HOUR }); + const c = counting(); + const plans: CachePurgePlan[] = []; + await tick(T + HOUR, { blobs: c.blobs, purge: capture(plans) }); + const S2 = await live(); + expect(c.gets.filter((k) => k.includes(`/${S2.id}/`))).toEqual([]); + expect(c.gets.filter((k) => k.includes(`/${S1.id}/`))).toHaveLength(1); + const p = plans[0]; + expect( + p?.mode === "everything" || + (p?.mode === "tags" && p.tags.includes(`cloudhal-node-${event.id}`)), + ).toBe(true); + // nor does it write `to`'s bundle without `from`'s + expect(await store.readSnapshotBundle(S2)).toBeUndefined(); }); it("sends the purge re-planned at run time and records a deferred purge", async () => { diff --git a/packages/core-ts/test/references-purge.test.ts b/packages/core-ts/test/references-purge.test.ts index a6a683ac..d5939926 100644 --- a/packages/core-ts/test/references-purge.test.ts +++ b/packages/core-ts/test/references-purge.test.ts @@ -18,7 +18,7 @@ import { MemoryBlobStore } from "@cloudhal/storage"; import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; import { beforeEach, describe, expect, it } from "vitest"; import { GATES_ENFORCED } from "../src/availability.ts"; -import type { BundleReadOptions } from "../src/bundles.ts"; +import { type BundleReadOptions, bundleKey, bundleShardCount } from "../src/bundles.ts"; import { type CachePurgeOptions, planCachePurge, type SnapshotReader } from "../src/cache-purge.ts"; import { nodeCacheTag, siteCacheTag } from "../src/cache-tags.ts"; import { continueFieldOperation, runFieldOperation } from "../src/fields-operations/index.ts"; @@ -370,17 +370,41 @@ describe("planCachePurge: the referrer closure (RFC-0045 §2)", () => { it("the closure's own reads count against maxReads (not just the planner's)", async () => { const { hall, event } = await world(); for (let i = 0; i < 20; i++) await make("events", `filler-${i}`); + await publish(); + // a bundled `from` and a manual `to`: the closure spends shard reads only (the changed + // entry's bodies are the planner's, every other one is in `from`'s bundle) + const from = (await fresh()).publishedSnapshotId as string; await edit(hall, { title: "Great hall" }); - const a = await snap("manual"); - await edit(hall, { title: "Grand hall" }); - const b = await snap("manual"); - const change = { from: a.id, to: b.id }; - // the planner's own reads, the closure aside + const to = await snap("manual"); + const change = { from, to: to.id }; await plan(change, { references: false }); const own = counts.nodes; - const tight = await plan(change, { maxReads: own + 5 }); + const snapshots = await Promise.all([store.getSnapshot(from), store.getSnapshot(to.id)]); + const shards = snapshots.reduce((n, s) => n + bundleShardCount(s as Snapshot), 0); + expect((await tagsOf(change, { maxReads: own + shards })).has(node(event))).toBe(true); + expect(counts.nodes).toBe(own); + const tight = await plan(change, { maxReads: own + shards - 1 }); expect(tight.mode === "everything" ? tight.reason : "tags").toMatch(/node reads/); - expect((await tagsOf(change, { maxReads: own + 100 })).has(node(event))).toBe(true); + }); + + it("a discarded partial bundle's bytes do not count against the byte budget", async () => { + const { hall, event } = await world(); + const from = (await fresh()).publishedSnapshotId as string; + const fromSnap = (await store.getSnapshot(from)) as Snapshot; + const fromBytes = (await store.blobs.get(bundleKey(site.id, from, 0)))?.byteLength ?? 0; + await edit(hall, { title: "Great hall" }); + const to = await snap("publish"); + // `to`'s shard is replaced by a mismatching one: within the budget alone, past it together + // with `from`'s, read and then discarded + const junk = JSON.stringify({ + v: 1, + snapshotId: "snap_other0001", + pad: "x".repeat(Math.floor(fromBytes * 0.6)), + }); + await store.blobs.put(bundleKey(site.id, to.id, 0), new TextEncoder().encode(junk)); + expect(bundleShardCount(fromSnap)).toBe(1); + const tags = await tagsOf(await point(to.id), { maxBodyBytes: Math.floor(fromBytes * 1.4) }); + expect(tags.has(node(event))).toBe(true); }); it("spends no shard reads on a snapshot known to have no bundle", async () => { From 740e2e9c3fb5d5683af6046bcfff52d13cec9b9c Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:18:09 +0300 Subject: [PATCH 5/6] fix(core-ts,cloudflare): RFC-0045 step 3b verification round 3 (PR #212) Removes machinery: - the release gate tests the published site node (publishRelease returns its ref), not the working row: a draft that drops the last reference field still re-plans, one that adds a first one still sends the stored plan - the run-time bundle write is gone (writeBundle, its cost, its caps); back-to-back releases with no visitor re-plan node by node or purge everything - blob calls are charged as they happen to one allowance per invocation (350 Free, 3,000 Paid; arithmetic beside the constant), shared by the cron's two ticks (cronTick); each re-plan gets min(2,000, left); RELEASE_REPLAN_MAX_READS is gone - tests: back-to-back within the allowance, both gate directions, 13 Paid releases in one tick, a 260-term reference site on Paid, two ticks sharing the allowance; RFC "Amended in 3b" and the changeset rewritten Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/rfc45-step3b-purges.md | 8 +- ...ctions-stage-4-references-and-repeaters.md | 21 +- packages/cloudflare/src/admin-worker.ts | 4 +- packages/cloudflare/src/scheduler.ts | 22 ++ packages/cloudflare/test/scheduler.test.ts | 14 ++ packages/core-ts/src/index.ts | 4 + .../core-ts/src/references/release-purge.ts | 199 ++++++++---------- packages/core-ts/src/release-apply.ts | 13 +- packages/core-ts/src/release-executor.ts | 29 +-- packages/core-ts/src/releases.ts | 3 +- packages/core-ts/src/testing-releases.ts | 185 +++++++++++----- 11 files changed, 305 insertions(+), 197 deletions(-) diff --git a/.changeset/rfc45-step3b-purges.md b/.changeset/rfc45-step3b-purges.md index a624854b..a35f49cf 100644 --- a/.changeset/rfc45-step3b-purges.md +++ b/.changeset/rfc45-step3b-purges.md @@ -1,8 +1,9 @@ --- "@cloudhal/core-ts": patch +"@cloudhal/cloudflare": patch --- -RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothing changes for sites without reference definitions. Their publish purge plans are the same, with no bundle read. Their release runs send the plan stored at pin time, as before, with no extra read, re-plan or bundle write. +RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothing changes for sites without reference definitions. Their publish purge plans are the same, with no bundle read. Their release runs send the plan stored at pin time, as before, with no re-plan; the only addition is one read of the published site node, to tell them apart. - **The referrer closure** (`references/purge.ts`, called by `planCachePurge`). - The seeds are the groups of every changed, added or removed entry, and of every entry whose shown asset changed. @@ -23,7 +24,8 @@ RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothin So does any field-operation marker on either snapshot's site node, for sites that hold references or convert from them. - **`readSnapshotBundle`** (`SnapshotReader.readSnapshotBundle`, `SiteGraphStore.readSnapshotBundle`, `readSnapshotBundle` in bundles) takes optional `maxBytes`, `maxNodes`, `keep`, `only` (bodies node by node) and `spent` (a shared byte total). A passed budget throws `BundleBudgetError` before the shard is parsed. No new cache tags. - **Release runs on sites with references re-plan their purge at run time** (`references/release-purge.ts`), from the pointer they moved to the snapshot they made, instead of sending the plan stored at pin time. - - The re-plan runs over the tick's counting driver, within the tick's blob allowance (400 on Workers Free, 4,000 on Paid; at most 250 reads per re-plan). If it fails, the run purges everything. - - After the purge, the run writes its snapshot's bundle, best effort, when `from` has one: from `from`'s bodies plus at most 100 changed nodes, for snapshots of at most 1,000 nodes. + - The test is the published site node, which `publishRelease` now returns (`publishedSite`). That costs one blob read per run with a purger. + - The re-plan runs over the tick's counting driver. Its actual blob calls are charged to one allowance per invocation, shared by the cron's ticks (`ScheduledTickOptions.releaseBlobs`): 350 on Workers Free, 3,000 on Paid. Past the allowance, or if the re-plan fails, the run purges everything. - The pin-time plan skips the closure (`CachePurgeOptions.references: false`): for these sites it is the expected outcome only. - `RELEASE_EXECUTOR_COST` is 26 (it counts `refreshSearch` and `recordPurge` now). +- **`@cloudhal/cloudflare`:** the cron handler passes one `releaseBlobs` allowance to both ticks of its invocation (`cronTick`); the alarm and the request backstop run one tick each, with the default allowance. diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index ab0700f8..00529852 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -1223,15 +1223,20 @@ Rollout step 2 shipped in two PRs: 2a, store and integrity, and 2b, the consumer - cut to what the closure needs, and never cached whole. - **The node budget** (`maxNodes`) counts the entry bodies the closure holds across both snapshots, plus the node count of the one bundle it reads. - **Release runs.** - - **Sites without references behave exactly as before 3b.** The run sends the plan stored at pin time, or nothing when it purges nothing, and makes no extra read. The test is the site node the release publishes, which `publishRelease` returns. A site whose last reference field was removed since `from` gets the stored plan, as every site did before. - - **Sites with references** (or converting from them) re-plan after the batch, from the pointer it moved (`from`, the live snapshot it replaced). That is `baseSnapshotId` whenever something was live; a first publish purges everything. - - The re-plan runs over the tick's counting driver, never refused, and spends no shard reads on `to`, which has no bundle yet. - - If it throws, or the tick's blob allowance is spent, the run purges everything, never the stored plan. - - **Order:** send the purge first, then write `to`'s bundle, best effort, for the next run. The write happens only when `from` has a bundle. It is built from `from`'s bodies plus at most 100 changed nodes fetched one by one, for snapshots of at most 1,000 nodes. This run's re-plan reads `from`'s bundle and the changed items. - - **Subrequests:** a tick's release runs share one blob allowance: 400 on Workers Free, under its 1,000 R2 and D1 calls per invocation together with the tick's 40 statements; 4,000 on Paid, under its default 10,000 with 900. A re-plan reads at most 250 blobs, a bundle write at most 108. - - **`RELEASE_EXECUTOR_COST` is 26:** + - **The gate is the published site node,** not the working row. A release never carries the site node, so its snapshot publishes the live one's; `publishRelease` returns that ref. Only a node that defines references, or still converts from them, re-plans. Every other site sends the plan stored at pin time, as before 3b (nothing when it purges nothing). + - That gate costs one blob read per release run that has a purger: a snapshot's site node is captured before the publish flips the pointer, so the working row never equals it. + - **The re-plan** goes from the pointer the batch moved (`from`) to the new snapshot. That is `baseSnapshotId` whenever something was live; a first publish purges everything. + - It runs over the tick's counting driver, never refused, and spends no shard reads on `to`. + - If it throws, the run purges everything, never the stored plan. + - **No bundle write at run time.** The site Worker writes the new snapshot's bundle on its first read. Back-to-back releases of one site with no visitor between them therefore re-plan node by node, within the allowance, or purge everything: an over-purge, never a stale page. + - **Subrequests.** R2 and D1 calls are capped per invocation: 1,000 on Workers Free, by default 10,000 on Paid. + - Release purges charge their *actual* blob calls to one allowance per invocation, shared by both ticks of the cron's invocation. Each re-plan gets `maxReads = min(2,000, left)`. + - Free: 1,000 − 2 × (40 + 7) D1 − 2 × 40 built-in blobs − 2 × (211 `refreshSearch` + 2 + 4 purge sends) = 392, so 350. + - Paid: sized for ticks of up to 10 releases of 100 items, 3,000. Fifty such releases pass the default cap with `refreshSearch` alone, re-plan or not; that is RFC-0008's budget, unchanged here. + - On Free, a reference site with more terms than the allowance leaves purges everything on every release, because the planner reads every term. That is accepted: rare, an over-purge, and no cache priming is built for it. + - **`RELEASE_EXECUTOR_COST` is 26 statements:** - the stored plan (1), the reads (3), the batch (8) and `refreshSearch` (7); - - then the re-plan's two `getSnapshot` reads (the bundle write reuses them), the purge's bookkeeping (4) and `recordPurge` (1). + - then the re-plan's two `getSnapshot` reads, the bookkeeping (4) and `recordPurge` (1). The rare site-moved retry stays outside it, as before. - **The pin-time plan skips the closure** (`references: false`). It would read every body one by one, and for sites with references it is the batch's *expected* outcome only: the run-time re-plan, or purge-everything, covers referrers. diff --git a/packages/cloudflare/src/admin-worker.ts b/packages/cloudflare/src/admin-worker.ts index a28f8cb3..a48de073 100644 --- a/packages/cloudflare/src/admin-worker.ts +++ b/packages/cloudflare/src/admin-worker.ts @@ -71,6 +71,7 @@ import { adminMail, type MailEnv } from "./mail.ts"; import { purgeSiteCache } from "./purge.ts"; import { type BackstopState, + cronTick, followUpDelay, releasePurger, requestBackstop, @@ -571,8 +572,7 @@ export function createAdminWorker(options: AdminWorkerOptions = {}) { await ensureMigrated(sql); const tick = (now: Date) => runAdminTick(env, { - trigger: "cron", - now, + ...cronTick(controller, env, now), log, analytics: (s, at) => adminAnalytics(env, s, { now: () => at, log }), commentsOn: commentsOn(env), diff --git a/packages/cloudflare/src/scheduler.ts b/packages/cloudflare/src/scheduler.ts index c1b40ab0..2ab32c60 100644 --- a/packages/cloudflare/src/scheduler.ts +++ b/packages/cloudflare/src/scheduler.ts @@ -30,6 +30,7 @@ import { SqlCommentStore } from "@cloudhal/comments/sql"; import { type AiSearchEnv, type CachePurgePlan, + releaseBlobAllowance, SiteGraphStore, searchProviderFromEnv, siteCacheTag, @@ -380,10 +381,30 @@ export interface AdminTickOptions { readonly commentsOn: boolean; /** Keeps a deferred purge's flush alive after the tick (the handler's `waitUntil`). */ readonly waitUntil?: ((p: Promise) => void) | undefined; + /** + * RFC-0045: the R2 calls release purges may still make in this invocation, shared by every tick + * it runs (the cron's follow-up); absent: one allowance per tick. + */ + readonly releaseBlobs?: { left: number } | undefined; } const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const invocationBlobs = new WeakMap(); + +/** + * A cron tick's trigger and clock, with RFC-0045's R2 allowance for release purges: one per + * invocation (`controller`), so the cron's follow-up tick shares it (the cap is per invocation). + */ +export function cronTick(controller: object, env: SchedulerEnv, now: Date) { + let releaseBlobs = invocationBlobs.get(controller); + if (!releaseBlobs) { + releaseBlobs = { left: releaseBlobAllowance(tickBudget(env)) }; + invocationBlobs.set(controller, releaseBlobs); + } + return { trigger: "cron" as const, now, releaseBlobs }; +} + /** One tick of the admin Worker (cron, alarm, request backstop or the operator's run-due). */ export function runAdminTick(env: SchedulerEnv, o: AdminTickOptions): Promise { const sql = new D1Driver(env.DB); @@ -425,6 +446,7 @@ export function runAdminTick(env: SchedulerEnv, o: AdminTickOptions): Promise releasePurger(env, counted, { log: o.log, wake: flush }), + ...(o.releaseBlobs ? { releaseBlobs: o.releaseBlobs } : {}), automationsEnabled: env.CLOUDHAL_AUTOMATIONS !== "off", // RFC-0008 §11.1: the built-in jobs read analytics' 404 rows and the site's own feed. builtinInputs: ({ counted }) => adminBuiltinInputs(env, o.analytics(counted, o.now)), diff --git a/packages/cloudflare/test/scheduler.test.ts b/packages/cloudflare/test/scheduler.test.ts index 4e410abe..f9faf606 100644 --- a/packages/cloudflare/test/scheduler.test.ts +++ b/packages/cloudflare/test/scheduler.test.ts @@ -15,6 +15,7 @@ import { createMiniflareD1, dropAllTables } from "@cloudhal/storage/testing"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { adminBuiltinInputs, + cronTick, followUpDelay, nextAlarm, releasePurger, @@ -41,6 +42,19 @@ describe("the ticker's mode and budget", () => { expect(tickBudget(env({ CLOUDHAL_TICK_BUDGET: "paid" })).maxQueries).toBe(900); }); + it("RFC-0045: the cron's ticks of one invocation share one R2 allowance for release purges", () => { + const one = {}; + const a = cronTick(one, env(), new Date(0)); + const b = cronTick(one, env(), new Date(60_000)); + expect(a.trigger).toBe("cron"); + expect(a.releaseBlobs).toBe(b.releaseBlobs); + expect(a.releaseBlobs.left).toBe(350); + expect(cronTick({}, env(), new Date(0)).releaseBlobs).not.toBe(a.releaseBlobs); + expect(cronTick({}, env({ CLOUDHAL_TICK_BUDGET: "paid" }), new Date(0)).releaseBlobs.left).toBe( + 3000, + ); + }); + it("cron ignores pokes; alarm pokes the clock", async () => { expect(schedulerRuntime(env()).mode).toBe("cron"); const seen: string[] = []; diff --git a/packages/core-ts/src/index.ts b/packages/core-ts/src/index.ts index b6710be6..faeb3193 100644 --- a/packages/core-ts/src/index.ts +++ b/packages/core-ts/src/index.ts @@ -84,6 +84,10 @@ export { type ReferrerCandidates, referrerCandidates, } from "./references/reads.ts"; +export { + RELEASE_BLOB_ALLOWANCE, + releaseBlobAllowance, +} from "./references/release-purge.ts"; export type { PublishReleaseInput, PublishReleaseResult, diff --git a/packages/core-ts/src/references/release-purge.ts b/packages/core-ts/src/references/release-purge.ts index c8e5222a..c1957f54 100644 --- a/packages/core-ts/src/references/release-purge.ts +++ b/packages/core-ts/src/references/release-purge.ts @@ -1,46 +1,56 @@ /** * RFC-0045 §2 "Purges", point 4: what a release run purges, decided after its batch committed. * - * - **Sites without references: exactly as before 3b.** The plan stored at pin time is sent (nothing - * when it purges nothing), with no read at all: the test is the site row `publishRelease` - * returns, which is the site node the release publishes. (A site whose last reference field was - * removed since `from` gets the stored plan, as every site did before 3b.) - * - **Sites with references (or converting from them): re-planned** `from → snapshotId` (`from`: - * the pointer the batch moved), like every other publish path. The pin-time plan skips the - * referrer closure and is the batch's *expected* outcome only. The re-plan reads over the tick's - * counting driver (never refused) and, for blobs, within the run's share of the tick's blob - * allowance; past it, or if it throws, the run purges everything, never the stored plan. - * - **Then the scheduled snapshot's bundle** (RFC-0010), best effort and bounded, so the *next* - * release run (whose `from` this snapshot is) reads one bundle: only for reference sites, only - * when `from` has a bundle, from `from`'s bodies plus at most `RELEASE_BUNDLE_MAX_FETCH` changed - * nodes, for snapshots of at most `RELEASE_BUNDLE_MAX_NODES` nodes. This run's re-plan reads - * `from`'s bundle and the items, and spends no shard reads on `to` (`unbundled`). + * - **The gate is the published site node.** A release never carries the site node, so the new + * snapshot publishes the live one's. When the working site row the batch read is that node, it + * is tested as is; otherwise the published body is read once (one blob call, charged). In + * practice that read always happens: a snapshot's site node is captured before the publish + * flips the pointer, so the row differs from it at least in `publishedSnapshotId` and `rev`. + * Only a node that defines references, or still converts from them, re-plans; every other site + * sends the plan stored at pin time, as before 3b (nothing when it purges nothing). + * - **Re-planned** `from → snapshotId` (`from`: the pointer the batch moved), like every other + * publish path. The pin-time plan skips the referrer closure and is the batch's *expected* + * outcome only. The re-plan's statements run on the tick's counting driver (never refused); if + * it throws, the run purges everything, never the stored plan. + * - **Blob calls are charged as they happen** against the invocation's allowance (`allowance`, + * shared by every tick of one invocation), and each re-plan gets `maxReads = min(2,000, left)`. + * Past it, the planner purges everything: an over-purge, never a stale page. The scheduled + * snapshot gets no bundle here (the site Worker writes one on its first read): back-to-back + * releases of one site with no visitor between them re-plan node by node, within the same + * allowance. * - * **Subrequests.** R2 and D1 calls are capped per invocation: 1,000 on Workers Free, 10,000 by - * default on Paid (RFC-0010, ADR-0024). A tick spends at most its `maxQueries` D1 statements (40 - * Free, 900 Paid) and the release runs' blob allowance (`releaseBlobAllowance`: 400 Free, 4,000 - * Paid, shared by every run of the tick), plus the purge's own calls: well under either cap. + * **Sizing the allowance.** R2 and D1 calls are capped per invocation: 1,000 on Workers Free, by + * default 10,000 on Paid (RFC-0010, ADR-0024). An invocation runs up to two ticks (the cron's + * follow-up). Per tick: D1, at most `maxQueries` refused statements plus 7 counted ones per + * release (the re-plan's 2 `getSnapshot`, bookkeeping 4, `recordPurge` 1); the built-in jobs' 40 + * blob reads. Per release: `refreshSearch`'s worst case (`MAX_RELEASE_ITEMS` × 2 + the site + + * ~10 taxonomies ≈ 211), `labelOf` and the publish's own blobs (2), the purge's sends (≤ 4). + * - **Free** (40 statements, 1 release per tick): + * 1,000 − 2 × (40 + 7) − 2 × 40 − 2 × (211 + 2 + 4) = 392, so 350. + * - **Paid** (900 statements, up to 50 releases per tick): 50 releases of 100 items pass the + * default 10,000 with `refreshSearch` alone, re-plan or not (RFC-0008's budgets, unchanged + * here). Sized for ticks of up to 10 such releases: + * 10,000 − 2 × (900 + 10 × 7) − 2 × 40 − 2 × 10 × 217 = 3,640, so 3,000. + * + * A reference site with more terms than the allowance leaves (the planner reads every term of + * both snapshots) purges everything on every release: rare, and an over-purge. */ -import type { NodeRef, Site, Snapshot } from "@cloudhal/core-contracts"; -import type { SqlDriver } from "@cloudhal/storage"; -import { type CachePurgePlan, planCachePurge, type SnapshotReader } from "../cache-purge.ts"; +import type { NodeRef, Site } from "@cloudhal/core-contracts"; +import type { BlobStore, SqlDriver } from "@cloudhal/storage"; +import { type CachePurgePlan, DEFAULT_PURGE_MAX_READS } from "../cache-purge.ts"; import type { ReleasePurge } from "../release-apply.ts"; import type { ReleasePurger } from "../releases.ts"; import { SiteGraphStore } from "../store.ts"; import { definesReferences } from "./purge.ts"; -/** Above this many nodes a release run leaves its snapshot's bundle to the site Worker. */ -export const RELEASE_BUNDLE_MAX_NODES = 1000; -/** Changed nodes a bundle write may fetch one by one (the rest come from `from`'s bundle). */ -export const RELEASE_BUNDLE_MAX_FETCH = 100; -/** The most blob reads one re-plan may make (node bodies and bundle shards). */ -export const RELEASE_REPLAN_MAX_READS = 250; -/** What a bundle write may cost in blob calls: `from`'s shards, the fetches, `to`'s shards. */ -const BUNDLE_WRITE_COST = 2 * Math.ceil(RELEASE_BUNDLE_MAX_NODES / 256) + RELEASE_BUNDLE_MAX_FETCH; +/** Blob calls one invocation's release runs may make (see the module comment). */ +export const RELEASE_BLOB_ALLOWANCE = { free: 350, paid: 3000 } as const; -/** The blob calls one tick's release runs may make together (see the module comment). */ +/** The allowance for a tick budget: Paid's when it allows more than Free's 40 statements. */ export function releaseBlobAllowance(budget?: { readonly maxQueries: number }): number { - return budget && budget.maxQueries > 40 ? 4000 : 400; + return budget && budget.maxQueries > 40 + ? RELEASE_BLOB_ALLOWANCE.paid + : RELEASE_BLOB_ALLOWANCE.free; } /** What a plan does to the outcome's `purge` (`none` where nothing is cached or nothing changed). */ @@ -51,108 +61,71 @@ export function purgeOutcome(plan: CachePurgePlan, purging: boolean): ReleasePur } export interface ReleasePurgeInput { - /** The executor's graph (its blob store is the planner's). */ + /** The executor's graph (its blob store and driver are the planner's). */ readonly graph: SiteGraphStore; /** The tick's counting driver; absent ("Publish now"): the graph's own. */ readonly counted?: SqlDriver | undefined; readonly purge: ReleasePurger; /** The plan stored at pin time. */ readonly stored: CachePurgePlan; - /** The site row the batch wrote (`to`'s site node). */ + /** The site row the batch wrote. */ readonly site: Site; + /** The site node the new snapshot publishes, and whether the working row was it. */ + readonly publishedSite?: { readonly ref: NodeRef; readonly current: boolean } | undefined; /** The pointer the batch moved (absent: nothing was live, so everything). */ readonly from?: string | undefined; readonly snapshotId: string; - /** Blob calls left for this tick's release runs; spent here. */ + /** Blob calls left for this invocation's release runs; charged by actual calls. */ readonly allowance: { left: number }; readonly log: (line: string) => void; } -const message = (e: unknown) => (e instanceof Error ? e.message : String(e)); - -/** Sends the run's purge (see the module comment); a throw only from the purger itself. */ -export async function sendReleasePurge(input: ReleasePurgeInput): Promise { - const { site, allowance, log } = input; - const planner = input.counted - ? new SiteGraphStore({ sql: input.counted, blobs: input.graph.blobs }) - : input.graph; - const send = (plan: CachePurgePlan) => - purgeOutcome(plan, true) === "none" - ? Promise.resolve("none" as const) - : input.purge(site.id, plan); - if (!definesReferences(site)) return send(input.stored); - let from: Snapshot | undefined; - let to: Snapshot | undefined; - try { - [from, to] = await Promise.all([ - input.from ? planner.getSnapshot(input.from) : undefined, - planner.getSnapshot(input.snapshotId), - ]); - } catch (e) { - log(`schedules: the release's snapshots could not be read: ${message(e)}`); - return send({ mode: "everything", reason: `snapshots unreadable: ${message(e)}` }); - } - const pair: Pair = { from, to, allowance, log }; - const plan = await replan(planner, pair); - const sent = await send(plan); - if (from && to && pair.fromBundle !== "missing") - await writeBundle(planner, { ...pair, from, to }); - return sent; -} - -interface Pair { - readonly from: Snapshot | undefined; - readonly to: Snapshot | undefined; - readonly allowance: { left: number }; - readonly log: (line: string) => void; - /** What the re-plan found of `from`'s bundle (unset: it did not look). */ - fromBundle?: "present" | "missing"; +/** A blob store that charges every call to `allowance`. */ +function charged(blobs: BlobStore, allowance: { left: number }): BlobStore { + return new Proxy(blobs, { + get(target, prop, receiver) { + const v = Reflect.get(target, prop, receiver); + if (typeof v !== "function") return v; + return (...args: unknown[]) => { + allowance.left--; + return v.apply(target, args); + }; + }, + }); } -/** The run-time plan within the run's blob share; purge-everything when it cannot be made. */ -async function replan(planner: SiteGraphStore, p: Pair): Promise { - const { from, to } = p; - if (!from || !to) return { mode: "everything", reason: "nothing was live before the release" }; - const maxReads = Math.min(RELEASE_REPLAN_MAX_READS, p.allowance.left); - p.allowance.left -= maxReads; - if (maxReads < 10) return { mode: "everything", reason: "the tick's blob allowance is spent" }; - const reader: SnapshotReader = { - getSnapshot: async (id) => (id === from.id ? from : id === to.id ? to : undefined), - readSnapshotNode: (ref: NodeRef) => planner.readSnapshotNode(ref), - readSnapshotBundle: async (s, o) => { - const got = await planner.readSnapshotBundle(s, o); - if (s.id === from.id && !o?.only) p.fromBundle = got ? "present" : "missing"; - return got; - }, - }; +/** The plan to send: the stored one, a re-plan, or purge-everything; never throws. */ +async function planFor(input: ReleasePurgeInput): Promise { + const { site, allowance } = input; + const planner = new SiteGraphStore({ + sql: input.counted ?? input.graph.sql, + blobs: charged(input.graph.blobs, allowance), + }); try { - // `to` has no bundle yet: it is written after the purge, for the next run. - return await planCachePurge(reader, from.id, to.id, { maxReads, unbundled: new Set([to.id]) }); + const published = input.publishedSite; + const read = published && !published.current; + if (read && allowance.left < 1) + return { mode: "everything", reason: "the blob allowance is spent" }; + const node = read ? await planner.readSnapshotNode(published.ref) : site; + if (!definesReferences(node)) return input.stored; + if (!input.from) return { mode: "everything", reason: "nothing was live before the release" }; + const maxReads = Math.min(DEFAULT_PURGE_MAX_READS, allowance.left); + if (maxReads < 1) return { mode: "everything", reason: "the blob allowance is spent" }; + // `to` has no bundle: no shard reads are spent on it. + const unbundled = new Set([input.snapshotId]); + return await planner.planCachePurge( + { from: input.from, to: input.snapshotId }, + { maxReads, unbundled }, + ); } catch (e) { - const reason = `the run-time purge plan failed: ${message(e)}`; - p.log(`schedules: ${reason}`); + const reason = `the run-time purge plan failed: ${e instanceof Error ? e.message : String(e)}`; + input.log(`schedules: ${reason}`); return { mode: "everything", reason }; } } -/** `to`'s bundle from `from`'s bodies plus the few changed nodes; skipped when it would cost more. */ -async function writeBundle(planner: SiteGraphStore, p: Pair & { from: Snapshot; to: Snapshot }) { - const { from, to } = p; - if (to.nodes.length > RELEASE_BUNDLE_MAX_NODES || from.nodes.length > RELEASE_BUNDLE_MAX_NODES) - return; - if (p.allowance.left < BUNDLE_WRITE_COST) return; - p.allowance.left -= BUNDLE_WRITE_COST; - try { - const bodies = await planner.readSnapshotBundle(from, { maxNodes: RELEASE_BUNDLE_MAX_NODES }); - if (!bodies) return; - const known = new Map(); - for (const ref of to.nodes) { - const body = bodies.get(ref.hash); - if (body !== undefined) known.set(ref.hash, JSON.stringify(body)); - } - if (to.nodes.length - known.size > RELEASE_BUNDLE_MAX_FETCH) return; - await planner.writeSnapshotBundle(to, known); - } catch (e) { - p.log(`schedules: no bundle for ${to.id}: ${message(e)}`); - } +/** Sends the run's purge (see the module comment); a throw only from the purger itself. */ +export async function sendReleasePurge(input: ReleasePurgeInput): Promise { + const plan = await planFor(input); + return purgeOutcome(plan, true) === "none" ? "none" : input.purge(input.site.id, plan); } diff --git a/packages/core-ts/src/release-apply.ts b/packages/core-ts/src/release-apply.ts index acb55a55..377bd4af 100644 --- a/packages/core-ts/src/release-apply.ts +++ b/packages/core-ts/src/release-apply.ts @@ -72,6 +72,11 @@ export type PublishReleaseResult = readonly baseSnapshotId: string; /** The pointer moved from (absent on a site never published before). */ readonly from?: string; + /** + * RFC-0045: the site node the new snapshot publishes (a release never carries the site + * node: the live one's), and whether the working site row read before the batch was it. + */ + readonly publishedSite?: { readonly ref: NodeRef; readonly current: boolean } | undefined; /** False when a recovered run found its snapshot already written (nothing new committed). */ readonly committed: boolean; } @@ -119,6 +124,12 @@ function approverOf(release: Release): Actor | undefined { } /** The snapshot's label: the release title, else "Scheduled: ". */ +/** The site node a snapshot's `nodes` publish, and whether it is the working row (`hash`). */ +function publishedSite(nodes: readonly NodeRef[], hash: string) { + const ref = nodes.find((n) => n.kind === "site"); + return ref ? { ref, current: ref.hash === hash } : undefined; +} + async function labelOf( ctx: ReleaseApplyContext, release: Release, @@ -194,7 +205,6 @@ export async function publishRelease( message: `the live snapshot ${site.publishedSnapshotId} cannot be read`, }; const parent = (await ctx.latestSnapshot(siteId)) ?? live; - // Conflicts, composition and dependencies (§6.3 steps 3–5), pure. const plan = planRelease(live?.nodes ?? [], items); if (plan.kind === "conflict") return plan; @@ -350,6 +360,7 @@ export async function publishRelease( snapshotId, baseSnapshotId: base, ...(live ? { from: live.id } : {}), + publishedSite: publishedSite(nodes, row.hash), committed: true, }; lastError = `the site ${siteId} or the run's lease changed while the release was applied`; diff --git a/packages/core-ts/src/release-executor.ts b/packages/core-ts/src/release-executor.ts index a970e2fb..971108b6 100644 --- a/packages/core-ts/src/release-executor.ts +++ b/packages/core-ts/src/release-executor.ts @@ -25,17 +25,16 @@ import type { SiteGraphStore } from "./store.ts"; * driver: the stored purge plan (1), the site, the live snapshot and the parent (3), the batch * (up to 8, with asset blobs to root), and the search index brought to the new snapshot (7: its * reads and one guarded batch, RFC-0019 §3). On its counting driver, never refused: for sites - * with references, the run-time re-plan's two `getSnapshot` reads (RFC-0045 §2; the bundle - * write reuses them); the purge's bookkeeping (up to 4); and `recordPurge`, correcting the - * outcome after a deferred or differing purge (1): 1 + 3 + 8 + 7 + 2 + 4 + 1 = 26. A site that moved between the reads and the - * batch repeats the reads and the batch once (11 more); that rare path may end the tick early + * with references, the run-time re-plan's two `getSnapshot` reads (RFC-0045 §2); the purge's + * bookkeeping (up to 4); and `recordPurge`, correcting the outcome after a deferred or differing + * purge (1). 1 + 3 + 8 + 7 + 2 + 4 + 1 = 26. A site that moved between the reads and the batch + * repeats the reads and the batch once (11 more); that rare path may end the tick early * (`more`), never mid-batch, and the purge after it runs on the counting driver regardless. * - * Blob calls (R2 subrequests) are bounded apart: a tick's release runs share - * `releaseBlobAllowance` (400 on Workers Free, under its 1,000 subrequests with the tick's 40 D1 - * statements; 4,000 on Paid, under its default 10,000 with 900). A re-plan reads at most - * `RELEASE_REPLAN_MAX_READS` (250) blobs, a bundle write at most 108; past the allowance the run - * purges everything (`references/release-purge.ts`). + * Blob calls (R2 subrequests) are bounded apart, per invocation: every release run of every tick + * in it charges its re-plan's actual blob calls to one allowance (350 on Workers Free, 3,000 on + * Paid; the arithmetic is in `references/release-purge.ts`), and past it the run purges + * everything. */ export const RELEASE_EXECUTOR_COST = 26; @@ -55,8 +54,11 @@ export interface ReleaseExecutorOptions { * `recordPurge` is absent, the outcome's correction is written over it. */ readonly counted?: SqlDriver | undefined; - /** Blob calls this executor's runs may make together (`releaseBlobAllowance`; default Free's). */ - readonly blobReads?: number | undefined; + /** + * Blob calls left for this invocation's release runs, shared by its ticks and charged by actual + * calls (default: Free's `releaseBlobAllowance`). + */ + readonly releaseBlobs?: { left: number } | undefined; /** After a committed release's purge (`release-after.ts`; the Node server rebuilds, #150). */ readonly onPublished?: after.ReleasePublishedHook | undefined; readonly log?: ((line: string) => void) | undefined; @@ -67,7 +69,7 @@ export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { const log = o.log ?? (() => {}); const counted = o.counted; // RFC-0045: the blob calls every release run of this tick may make together. - const allowance = { left: o.blobReads ?? releaseBlobAllowance() }; + const allowance = o.releaseBlobs ?? { left: releaseBlobAllowance() }; const recordPurge = o.recordPurge ?? (counted @@ -100,7 +102,8 @@ export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { try { purge = await sendReleasePurge({ ...{ graph: o.graph, counted, purge: o.purge, stored: plan, allowance, log }, - ...{ site: r.site, from: r.from, snapshotId: r.snapshotId }, + ...{ site: r.site, publishedSite: r.publishedSite }, + ...{ from: r.from, snapshotId: r.snapshotId }, }); } catch (e) { purge = "failed"; diff --git a/packages/core-ts/src/releases.ts b/packages/core-ts/src/releases.ts index 026ee6ec..02fc4cd7 100644 --- a/packages/core-ts/src/releases.ts +++ b/packages/core-ts/src/releases.ts @@ -162,6 +162,7 @@ export interface ScheduledTickOptions { readonly checkApproval?: | ((release: Release, now: Date) => ApprovalVerdict | Promise) | undefined; + readonly releaseBlobs?: { left: number } | undefined; // RFC-0045: one per invocation (R2 calls) /** Sends a published release's purge plan; it gets the counting driver for its bookkeeping. */ readonly purge?: ((ctx: { counted: SqlDriver }) => ReleasePurger) | undefined; /** `CLOUDHAL_AUTOMATIONS=off` → false. */ @@ -193,7 +194,7 @@ export async function runScheduledTick(o: ScheduledTickOptions): Promise { + it("a site without references sends the pin-time plan, reading only the published site node", async () => { // the same release twice: without a purger (Node: the executor sends nothing), then with const first = await story(); await scheduled([first.id]); @@ -573,7 +573,8 @@ export function releasesSuite( expect((await tick(T + HOUR, { blobs: c.blobs, purge: capture(plans) })).ran).toBe(1); expect(plans).toEqual([stored]); expect(c.gets.filter((k) => k.startsWith("bundles/"))).toEqual([]); - expect(c.gets).toHaveLength(quiet.gets.length); + // one read more than a run that purges nothing: the published site node, for the gate + expect(c.gets).toHaveLength(quiet.gets.length + 1); }); it("a run-time re-plan that fails still purges everything, never drops the purge", async () => { @@ -610,70 +611,142 @@ export function releasesSuite( expect(r.outcome).toMatchObject({ kind: "published", purge: "everything" }); }); - it("back-to-back releases on a reference site: one bundle each, never to's shards", async () => { + const retitle = async (id: string, title: string) => + store.updateContentEntry({ ...((await store.getContentEntry(id)) as ContentEntry), title }); + const tagsOf = (p: CachePurgePlan | undefined) => (p?.mode === "tags" ? p.tags : []); + /** Writes the working site row directly (a definition change the store would refuse). */ + async function rawSite(change: (s: Site) => Site) { + const s = await fresh(); + const next = { ...change(s), rev: s.rev + 1 }; + await sql.run("UPDATE nodes SET rev = ?, hash = ?, body = ? WHERE id = ?", [ + next.rev, + await nodeHash(next), + canonicalJson(next), + s.id, + ]); + } + + it("back-to-back releases on a reference site: a purge each, within the allowance", async () => { const { hall, event } = await venueSite(); - const edit = async (title: string) => - store.updateContentEntry({ - ...((await store.getContentEntry(hall.id)) as ContentEntry), - title, - }); - const shardsOf = (gets: string[], id: string) => gets.filter((k) => k.includes(`/${id}/`)); - // 1: `from` (a publish) has a bundle; `to` gets one after the purge - const L = await live(); - await edit("Main hall"); - await scheduled([hall.id]); - const c1 = counting(); + const releaseBlobs = { left: 350 }; const plans: CachePurgePlan[] = []; - await tick(T, { blobs: c1.blobs, purge: capture(plans) }); - const S1 = await live(); - expect(S1.reason).toBe("scheduled"); - expect(shardsOf(c1.gets, S1.id)).toEqual([]); - expect(shardsOf(c1.gets, L.id).length).toBeGreaterThan(0); - expect(await store.readSnapshotBundle(S1)).toBeDefined(); - // 2: `from` is the first release's scheduled snapshot, bundled by its run - await edit("Great hall"); + await retitle(hall.id, "Main hall"); + await scheduled([hall.id]); + await tick(T, { purge: capture(plans), releaseBlobs }); + // the second run's `from` is the first's scheduled snapshot: no bundle, nobody read it + await retitle(hall.id, "Great hall"); await scheduled([hall.id], { at: T + HOUR }); - const c2 = counting(); - await tick(T + HOUR, { blobs: c2.blobs, purge: capture(plans) }); - const S2 = await live(); - expect(shardsOf(c2.gets, S2.id)).toEqual([]); - // `from`'s shards: once for the re-plan, once to build `to`'s bundle (the rest of `to` - // comes from them; only changed nodes are fetched one by one) - expect(shardsOf(c2.gets, S1.id)).toHaveLength(2 * bundleShardCount(S1)); - for (const p of plans.slice(0, 2)) { - expect(p.mode).toBe("tags"); - expect(p.mode === "tags" ? p.tags : []).toContain(`cloudhal-node-${event.id}`); + await tick(T + HOUR, { purge: capture(plans), releaseBlobs }); + expect(plans).toHaveLength(2); + for (const p of plans) { + expect(["tags", "everything"]).toContain(p.mode); + if (p.mode === "tags") expect(p.tags).toContain(`cloudhal-node-${event.id}`); } + expect(releaseBlobs.left).toBeGreaterThanOrEqual(0); + expect(releaseBlobs.left).toBeLessThan(350); }); - it("back-to-back releases where from has no bundle: from's shards tried once, to's never", async () => { + it("gates on the published site node: a draft that drops the last reference field still re-plans", async () => { const { hall, event } = await venueSite(); - const edit = async (title: string) => - store.updateContentEntry({ - ...((await store.getContentEntry(hall.id)) as ContentEntry), - title, - }); - // a first release run with nothing to purge (Node): its snapshot gets no bundle - await edit("Main hall"); + // the working copy drops `events.venue`, unpublished + await rawSite((s) => ({ + ...s, + collections: { ...s.collections, events: {} }, + })); + await retitle(hall.id, "Main hall"); await scheduled([hall.id]); - await tick(T); - const S1 = await live(); - expect(await store.readSnapshotBundle(S1)).toBeUndefined(); - await edit("Great hall"); - await scheduled([hall.id], { at: T + HOUR }); + const plans: CachePurgePlan[] = []; + await tick(T, { purge: capture(plans) }); + expect(tagsOf(plans[0])).toContain(`cloudhal-node-${event.id}`); + }); + + it("gates on the published site node: a draft that adds a first reference field sends the stored plan", async () => { + const first = await story(); + await scheduled([first.id]); + const quiet = counting(); + await tick(T, { blobs: quiet.blobs }); + // the working copy gains a reference field, unpublished + const s = await fresh(); + const venue: FieldDefinition = { + key: "venue", + type: "reference", + labels: { en: { label: "Venue" } }, + localized: false, + collections: ["posts"], + }; + site = await store.updateSite({ + ...s, + collections: { ...s.collections, events: { fields: [venue] } }, + }); + const second = await story({ slug: "second-story", title: "Second story" }); + const { release } = await scheduled([second.id], { at: T + HOUR }); + const stored = await svc.schedules.releasePurgePlan(release.id); const c = counting(); const plans: CachePurgePlan[] = []; await tick(T + HOUR, { blobs: c.blobs, purge: capture(plans) }); - const S2 = await live(); - expect(c.gets.filter((k) => k.includes(`/${S2.id}/`))).toEqual([]); - expect(c.gets.filter((k) => k.includes(`/${S1.id}/`))).toHaveLength(1); - const p = plans[0]; - expect( - p?.mode === "everything" || - (p?.mode === "tags" && p.tags.includes(`cloudhal-node-${event.id}`)), - ).toBe(true); - // nor does it write `to`'s bundle without `from`'s - expect(await store.readSnapshotBundle(S2)).toBeUndefined(); + expect(plans).toEqual([stored]); + // one read more than a run that purges nothing: the published site node, for the gate + expect(c.gets).toHaveLength(quiet.gets.length + 1); + }); + + it("Paid: a tick of 13 reference-site releases purges each by its tags", async () => { + const { hall, event } = await venueSite(); + const ids: string[] = [hall.id]; + for (let i = 0; i < 12; i++) + ids.push( + (await story({ collection: "places", slug: `place-${i}`, title: `Place ${i}` })).id, + ); + await publishSite(); + for (const [i, id] of ids.entries()) { + await retitle(id, `Retitled ${i}`); + await scheduled([id]); + } + const plans: CachePurgePlan[] = []; + const report = await tick(T, { purge: capture(plans), budget: TICK_BUDGETS.paid }); + expect(report.ran).toBe(13); + expect(plans.map((p) => p.mode)).toEqual(Array(13).fill("tags")); + expect(plans.some((p) => tagsOf(p).includes(`cloudhal-node-${event.id}`))).toBe(true); + }); + + it("Paid: a reference site with more than 250 terms gets tags for a one-entry release", async () => { + const { hall, event } = await venueSite(); + for (let i = 0; i < 260; i++) + await store.createTerm({ + siteId: site.id, + taxonomyId: category, + slug: `topic-${i}`, + status: "active", + visibility: "public", + labels: { en: { name: `Topic ${i}` } }, + }); + await publishSite(); + await retitle(hall.id, "Main hall"); + await scheduled([hall.id]); + const plans: CachePurgePlan[] = []; + await tick(T, { purge: capture(plans), budget: TICK_BUDGETS.paid }); + expect(tagsOf(plans[0])).toContain(`cloudhal-node-${event.id}`); + }); + + it("two ticks of one invocation share the allowance, charged by actual calls", async () => { + const { hall } = await venueSite(); + const releaseBlobs = { left: 1000 }; + await retitle(hall.id, "Main hall"); + await scheduled([hall.id]); + await tick(T, { purge: capture([]), releaseBlobs }); + const afterFirst = releaseBlobs.left; + expect(afterFirst).toBeLessThan(1000); + await retitle(hall.id, "Great hall"); + await scheduled([hall.id], { at: T + HOUR }); + await tick(T + HOUR, { purge: capture([]), releaseBlobs }); + expect(releaseBlobs.left).toBeLessThan(afterFirst); + // spent: the next run purges everything without reading + releaseBlobs.left = 0; + await retitle(hall.id, "Grand hall"); + await scheduled([hall.id], { at: T + 2 * HOUR }); + const plans: CachePurgePlan[] = []; + await tick(T + 2 * HOUR, { purge: capture(plans), releaseBlobs }); + expect(plans.map((p) => p.mode)).toEqual(["everything"]); + expect(releaseBlobs.left).toBe(0); }); it("sends the purge re-planned at run time and records a deferred purge", async () => { From 01a2ce7164e5d550540b9fc087907df6a46406cd Mon Sep 17 00:00:00 2001 From: Madraka <8898640+Madraka@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:54:21 +0300 Subject: [PATCH 6/6] fix(core-ts,cloudflare,mcp-server): RFC-0045 step 3b verification round 4 (PR #212) - the gate read is counted but never refused; if it fails, the working row decides (no references: the stored plan): sites without references never depend on the shared allowance - a re-plan whose snapshots hold more distinct terms than reads left purges everything before reading - the tick charges the release graph's blob calls (refreshSearch, labelOf, a first publish's head/put) to the allowance, never refusing them (chargedBlobs) - "Publish now" sizes its allowance from the deployment's tick budget (ReleaseServiceOptions/SchedulesDeps.tickBudget, publishNowPurge); ReleaseServiceOptions moves to release-service-options.ts (re-exported) to keep releases.ts at its baseline - docs: labelOf keeps its JSDoc; "except on a first publish"; Paid fits ~27 releases by D1; accepted over-purges listed (asset rule on Free, AI Search mirror uncounted) - tests: stored plan with the allowance spent and with a failing gate read; the terms pre-check costs only the gate read Co-Authored-By: Claude Opus 5.5 Signed-off-by: Madraka <8898640+Madraka@users.noreply.github.com> --- .changeset/rfc45-step3b-purges.md | 7 +- ...ctions-stage-4-references-and-repeaters.md | 14 +- packages/cloudflare/src/admin-worker.ts | 4 +- packages/cloudflare/src/scheduler.ts | 5 + .../core-ts/src/references/release-purge.ts | 127 ++++++++++++------ packages/core-ts/src/release-apply.ts | 2 +- packages/core-ts/src/release-executor.ts | 2 +- .../core-ts/src/release-service-options.ts | 48 +++++++ packages/core-ts/src/releases.ts | 50 +++---- packages/core-ts/src/testing-releases.ts | 3 +- .../core-ts/test/references-purge.test.ts | 79 +++++++++++ packages/mcp-server/src/schedules-deps.ts | 3 + 12 files changed, 258 insertions(+), 86 deletions(-) create mode 100644 packages/core-ts/src/release-service-options.ts diff --git a/.changeset/rfc45-step3b-purges.md b/.changeset/rfc45-step3b-purges.md index a35f49cf..dafe978c 100644 --- a/.changeset/rfc45-step3b-purges.md +++ b/.changeset/rfc45-step3b-purges.md @@ -1,6 +1,7 @@ --- "@cloudhal/core-ts": patch "@cloudhal/cloudflare": patch +"@cloudhal/mcp-server": patch --- RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothing changes for sites without reference definitions. Their publish purge plans are the same, with no bundle read. Their release runs send the plan stored at pin time, as before, with no re-plan; the only addition is one read of the published site node, to tell them apart. @@ -24,8 +25,10 @@ RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothin So does any field-operation marker on either snapshot's site node, for sites that hold references or convert from them. - **`readSnapshotBundle`** (`SnapshotReader.readSnapshotBundle`, `SiteGraphStore.readSnapshotBundle`, `readSnapshotBundle` in bundles) takes optional `maxBytes`, `maxNodes`, `keep`, `only` (bodies node by node) and `spent` (a shared byte total). A passed budget throws `BundleBudgetError` before the shard is parsed. No new cache tags. - **Release runs on sites with references re-plan their purge at run time** (`references/release-purge.ts`), from the pointer they moved to the snapshot they made, instead of sending the plan stored at pin time. - - The test is the published site node, which `publishRelease` now returns (`publishedSite`). That costs one blob read per run with a purger. - - The re-plan runs over the tick's counting driver. Its actual blob calls are charged to one allowance per invocation, shared by the cron's ticks (`ScheduledTickOptions.releaseBlobs`): 350 on Workers Free, 3,000 on Paid. Past the allowance, or if the re-plan fails, the run purges everything. + - The test is the published site node, which `publishRelease` now returns (`publishedSite`). That costs one blob read per run with a purger, except on a first publish; the read is counted, never refused. If it fails, the working row decides. + - The re-plan runs over the tick's counting driver. + - The release graph's blob calls (`refreshSearch`, `labelOf`, the re-plan) are charged to one allowance per invocation, shared by the cron's ticks (`ScheduledTickOptions.releaseBlobs`): 350 on Workers Free, 3,000 on Paid. "Publish now" sizes it from the deployment's tick budget (`ReleaseService` and `SchedulesDeps.tickBudget`). + - Past the allowance, or if the re-plan fails, the run purges everything. A re-plan with more distinct terms than reads left purges everything before reading anything. - The pin-time plan skips the closure (`CachePurgeOptions.references: false`): for these sites it is the expected outcome only. - `RELEASE_EXECUTOR_COST` is 26 (it counts `refreshSearch` and `recordPurge` now). - **`@cloudhal/cloudflare`:** the cron handler passes one `releaseBlobs` allowance to both ticks of its invocation (`cronTick`); the alarm and the request backstop run one tick each, with the default allowance. diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index 00529852..e62e0b3e 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -1224,16 +1224,22 @@ Rollout step 2 shipped in two PRs: 2a, store and integrity, and 2b, the consumer - **The node budget** (`maxNodes`) counts the entry bodies the closure holds across both snapshots, plus the node count of the one bundle it reads. - **Release runs.** - **The gate is the published site node,** not the working row. A release never carries the site node, so its snapshot publishes the live one's; `publishRelease` returns that ref. Only a node that defines references, or still converts from them, re-plans. Every other site sends the plan stored at pin time, as before 3b (nothing when it purges nothing). - - That gate costs one blob read per release run that has a purger: a snapshot's site node is captured before the publish flips the pointer, so the working row never equals it. + - That gate costs one blob read per release run that has a purger, except on a first publish: a snapshot's site node is captured before the publish flips the pointer, so the working row equals it only on a site's first publish. + - The read is counted but never refused, so a site without references never depends on what other runs left. If it fails, the working row decides: no references, the stored plan; otherwise purge everything. - **The re-plan** goes from the pointer the batch moved (`from`) to the new snapshot. That is `baseSnapshotId` whenever something was live; a first publish purges everything. - It runs over the tick's counting driver, never refused, and spends no shard reads on `to`. - If it throws, the run purges everything, never the stored plan. - **No bundle write at run time.** The site Worker writes the new snapshot's bundle on its first read. Back-to-back releases of one site with no visitor between them therefore re-plan node by node, within the allowance, or purge everything: an over-purge, never a stale page. - **Subrequests.** R2 and D1 calls are capped per invocation: 1,000 on Workers Free, by default 10,000 on Paid. - - Release purges charge their *actual* blob calls to one allowance per invocation, shared by both ticks of the cron's invocation. Each re-plan gets `maxReads = min(2,000, left)`. + - Release runs charge their *actual* blob calls to one allowance per invocation, shared by both ticks of the cron's invocation. That includes `refreshSearch`, `labelOf` and a first publish's own blobs, counted but never refused. Each re-plan gets `maxReads = min(2,000, left)`. + - A re-plan whose two snapshots hold more distinct terms than reads left purges everything before reading anything. + - "Publish now" sizes its allowance from the deployment's tick budget (`CLOUDHAL_TICK_BUDGET`). - Free: 1,000 − 2 × (40 + 7) D1 − 2 × 40 built-in blobs − 2 × (211 `refreshSearch` + 2 + 4 purge sends) = 392, so 350. - - Paid: sized for ticks of up to 10 releases of 100 items, 3,000. Fifty such releases pass the default cap with `refreshSearch` alone, re-plan or not; that is RFC-0008's budget, unchanged here. - - On Free, a reference site with more terms than the allowance leaves purges everything on every release, because the planner reads every term. That is accepted: rare, an over-purge, and no cache priming is built for it. + - Paid: about 27 releases fit a tick's 900 statements (33 each with claim and finish). The allowance is sized for ticks of up to 10 releases of 100 items, 3,000. Twenty-seven such releases pass the default cap with `refreshSearch` alone, re-plan or not; that is RFC-0008's budget, unchanged here. + - **Accepted over-purges, not fixed:** + - On Free, a reference site with more terms than the allowance leaves purges everything on every release, because the planner reads every term. No cache priming is built for it. + - On Free, a reference-site release that changes an existing asset reads every entry, page and term one by one in the asset rule, so above about 350 nodes it purges everything. + - The AI Search mirror's R2 calls (RFC-0029) are not counted. - **`RELEASE_EXECUTOR_COST` is 26 statements:** - the stored plan (1), the reads (3), the batch (8) and `refreshSearch` (7); - then the re-plan's two `getSnapshot` reads, the bookkeeping (4) and `recordPurge` (1). diff --git a/packages/cloudflare/src/admin-worker.ts b/packages/cloudflare/src/admin-worker.ts index a48de073..48559a52 100644 --- a/packages/cloudflare/src/admin-worker.ts +++ b/packages/cloudflare/src/admin-worker.ts @@ -73,7 +73,7 @@ import { type BackstopState, cronTick, followUpDelay, - releasePurger, + publishNowPurge, requestBackstop, runAdminTick, type SchedulerEnv, @@ -538,7 +538,7 @@ export function createAdminWorker(options: AdminWorkerOptions = {}) { // RFC-0008 §12–§13: the schedule routes and tools; "Publish now" purges like a tick. schedules: { store: new SqlScheduleStore(sql), - purge: releasePurger(env, sql, { log }), + ...publishNowPurge(env, sql, log), automationsEnabled: env.CLOUDHAL_AUTOMATIONS !== "off", }, scheduler: { diff --git a/packages/cloudflare/src/scheduler.ts b/packages/cloudflare/src/scheduler.ts index 2ab32c60..4f50e119 100644 --- a/packages/cloudflare/src/scheduler.ts +++ b/packages/cloudflare/src/scheduler.ts @@ -390,6 +390,11 @@ export interface AdminTickOptions { const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +/** "Publish now"'s purge, and the deployment's tick budget that sizes its blob calls (RFC-0045). */ +export function publishNowPurge(env: SchedulerEnv, sql: SqlDriver, log: (line: string) => void) { + return { purge: releasePurger(env, sql, { log }), tickBudget: tickBudget(env) }; +} + const invocationBlobs = new WeakMap(); /** diff --git a/packages/core-ts/src/references/release-purge.ts b/packages/core-ts/src/references/release-purge.ts index c1957f54..7001d9ab 100644 --- a/packages/core-ts/src/references/release-purge.ts +++ b/packages/core-ts/src/references/release-purge.ts @@ -2,19 +2,26 @@ * RFC-0045 §2 "Purges", point 4: what a release run purges, decided after its batch committed. * * - **The gate is the published site node.** A release never carries the site node, so the new - * snapshot publishes the live one's. When the working site row the batch read is that node, it - * is tested as is; otherwise the published body is read once (one blob call, charged). In - * practice that read always happens: a snapshot's site node is captured before the publish - * flips the pointer, so the row differs from it at least in `publishedSnapshotId` and `rev`. - * Only a node that defines references, or still converts from them, re-plans; every other site - * sends the plan stored at pin time, as before 3b (nothing when it purges nothing). + * snapshot publishes the live one's. When the working site row the batch read is that node (a + * first publish), it is tested as is; otherwise the published body is read once. In practice + * that read happens except on a first publish: a snapshot's site node is captured before the + * publish flips the pointer, so the row differs from it at least in `publishedSnapshotId` and + * `rev`. The read is counted but never refused (one call per release, from the sizing's margin), + * so a site without references never depends on what other runs left; if it fails, the working + * row decides (no references: the stored plan; else everything). Only a node that defines + * references, or still converts from them, re-plans; every other site sends the plan stored at + * pin time, as before 3b (nothing when it purges nothing). * - **Re-planned** `from → snapshotId` (`from`: the pointer the batch moved), like every other * publish path. The pin-time plan skips the referrer closure and is the batch's *expected* * outcome only. The re-plan's statements run on the tick's counting driver (never refused); if - * it throws, the run purges everything, never the stored plan. - * - **Blob calls are charged as they happen** against the invocation's allowance (`allowance`, - * shared by every tick of one invocation), and each re-plan gets `maxReads = min(2,000, left)`. - * Past it, the planner purges everything: an over-purge, never a stale page. The scheduled + * it throws, the run purges everything, never the stored plan. A re-plan that could not finish + * (more distinct terms in the two snapshots than reads left, the planner reading every term) + * purges everything before it reads anything. + * - **Blob calls are charged as they happen** against the invocation's allowance, shared by every + * tick of one invocation. The tick wraps the release graph's store with `chargedBlobs`, so the + * run's other calls (`refreshSearch`'s reads, `labelOf`, a first publish's head and put) count + * too, never refused; each re-plan then gets `maxReads = min(2,000, left)` of what is really + * left. Past it, the planner purges everything: an over-purge, never a stale page. The scheduled * snapshot gets no bundle here (the site Worker writes one on its first read): back-to-back * releases of one site with no visitor between them re-plan node by node, within the same * allowance. @@ -23,21 +30,30 @@ * default 10,000 on Paid (RFC-0010, ADR-0024). An invocation runs up to two ticks (the cron's * follow-up). Per tick: D1, at most `maxQueries` refused statements plus 7 counted ones per * release (the re-plan's 2 `getSnapshot`, bookkeeping 4, `recordPurge` 1); the built-in jobs' 40 - * blob reads. Per release: `refreshSearch`'s worst case (`MAX_RELEASE_ITEMS` × 2 + the site + - * ~10 taxonomies ≈ 211), `labelOf` and the publish's own blobs (2), the purge's sends (≤ 4). + * blob reads. Per release, counted from the allowance itself: `refreshSearch`'s worst case + * (`MAX_RELEASE_ITEMS` × 2 + the site + ~10 taxonomies ≈ 211), `labelOf` and the publish's own + * blobs (2), the gate read (1); and the purge's sends (≤ 4). * - **Free** (40 statements, 1 release per tick): * 1,000 − 2 × (40 + 7) − 2 × 40 − 2 × (211 + 2 + 4) = 392, so 350. - * - **Paid** (900 statements, up to 50 releases per tick): 50 releases of 100 items pass the - * default 10,000 with `refreshSearch` alone, re-plan or not (RFC-0008's budgets, unchanged - * here). Sized for ticks of up to 10 such releases: - * 10,000 − 2 × (900 + 10 × 7) − 2 × 40 − 2 × 10 × 217 = 3,640, so 3,000. + * - **Paid** (900 statements; about 27 releases fit a tick's D1, at 33 statements each with claim + * and finish): 27 releases of 100 items pass the default 10,000 with `refreshSearch` alone, + * re-plan or not (RFC-0008's budgets, unchanged here). Sized for ticks of up to 10 such + * releases: 10,000 − 2 × (900 + 10 × 7) − 2 × 40 − 2 × 10 × 217 = 3,640, so 3,000. * - * A reference site with more terms than the allowance leaves (the planner reads every term of - * both snapshots) purges everything on every release: rare, and an over-purge. + * Accepted over-purges (an over-purge, never a stale page; rare, and for a feature not enabled + * yet): a reference site with more terms than the allowance leaves purges everything on every + * release; on Free, a reference-site release that changes an existing asset reads every entry, + * page and term one by one in the asset rule, so above about 350 nodes it purges everything. The + * AI Search mirror's R2 calls (RFC-0029) are not counted. */ import type { NodeRef, Site } from "@cloudhal/core-contracts"; import type { BlobStore, SqlDriver } from "@cloudhal/storage"; -import { type CachePurgePlan, DEFAULT_PURGE_MAX_READS } from "../cache-purge.ts"; +import { + type CachePurgePlan, + DEFAULT_PURGE_MAX_READS, + planCachePurge, + type SnapshotReader, +} from "../cache-purge.ts"; import type { ReleasePurge } from "../release-apply.ts"; import type { ReleasePurger } from "../releases.ts"; import { SiteGraphStore } from "../store.ts"; @@ -80,9 +96,16 @@ export interface ReleasePurgeInput { readonly log: (line: string) => void; } -/** A blob store that charges every call to `allowance`. */ -function charged(blobs: BlobStore, allowance: { left: number }): BlobStore { - return new Proxy(blobs, { +const chargedStores = new WeakSet(); + +/** + * A blob store that charges every call to `allowance`, never refusing one: the tick wraps the + * release graph's store with it, so `refreshSearch`, `labelOf` and the publish's own blobs count + * too, and a re-plan's `maxReads` sees what is really left. + */ +export function chargedBlobs(blobs: BlobStore, allowance: { left: number }): BlobStore { + if (chargedStores.has(blobs)) return blobs; + const out = new Proxy(blobs, { get(target, prop, receiver) { const v = Reflect.get(target, prop, receiver); if (typeof v !== "function") return v; @@ -92,38 +115,62 @@ function charged(blobs: BlobStore, allowance: { left: number }): BlobStore { }; }, }); + chargedStores.add(out); + return out; } /** The plan to send: the stored one, a re-plan, or purge-everything; never throws. */ async function planFor(input: ReleasePurgeInput): Promise { - const { site, allowance } = input; + const { site, allowance, log } = input; const planner = new SiteGraphStore({ sql: input.counted ?? input.graph.sql, - blobs: charged(input.graph.blobs, allowance), + blobs: chargedBlobs(input.graph.blobs, allowance), }); + const everything = (reason: string): CachePurgePlan => { + log(`schedules: ${reason}`); + return { mode: "everything", reason }; + }; + // The gate: counted, never refused (one call per release, from the sizing's margin), so a + // site without references never depends on what other runs left. + let node: Site = site; + const published = input.publishedSite; + if (published && !published.current) + try { + node = await planner.readSnapshotNode(published.ref); + } catch (e) { + // the working row decides: no references there, the stored plan; else everything + if (!definesReferences(site)) return input.stored; + return everything(`the published site node could not be read: ${message(e)}`); + } + if (!definesReferences(node)) return input.stored; + if (!input.from) return { mode: "everything", reason: "nothing was live before the release" }; try { - const published = input.publishedSite; - const read = published && !published.current; - if (read && allowance.left < 1) - return { mode: "everything", reason: "the blob allowance is spent" }; - const node = read ? await planner.readSnapshotNode(published.ref) : site; - if (!definesReferences(node)) return input.stored; - if (!input.from) return { mode: "everything", reason: "nothing was live before the release" }; const maxReads = Math.min(DEFAULT_PURGE_MAX_READS, allowance.left); - if (maxReads < 1) return { mode: "everything", reason: "the blob allowance is spent" }; - // `to` has no bundle: no shard reads are spent on it. - const unbundled = new Set([input.snapshotId]); - return await planner.planCachePurge( - { from: input.from, to: input.snapshotId }, - { maxReads, unbundled }, + const [from, to] = await Promise.all([ + planner.getSnapshot(input.from), + planner.getSnapshot(input.snapshotId), + ]); + if (!from || !to) return everything("a release snapshot is missing"); + // A hopeless re-plan stops before it reads: the planner reads every term of both snapshots. + const terms = new Set( + [...from.nodes, ...to.nodes].filter((n) => n.kind === "term").map((n) => n.hash), ); + if (terms.size >= maxReads) + return everything(`${terms.size} terms need more blob reads than are left (${maxReads})`); + const reader: SnapshotReader = { + getSnapshot: async (id) => (id === from.id ? from : id === to.id ? to : undefined), + readSnapshotNode: (ref: NodeRef) => planner.readSnapshotNode(ref), + readSnapshotBundle: (s, o) => planner.readSnapshotBundle(s, o), + }; + // `to` has no bundle: no shard reads are spent on it. + return await planCachePurge(reader, from.id, to.id, { maxReads, unbundled: new Set([to.id]) }); } catch (e) { - const reason = `the run-time purge plan failed: ${e instanceof Error ? e.message : String(e)}`; - input.log(`schedules: ${reason}`); - return { mode: "everything", reason }; + return everything(`the run-time purge plan failed: ${message(e)}`); } } +const message = (e: unknown) => (e instanceof Error ? e.message : String(e)); + /** Sends the run's purge (see the module comment); a throw only from the purger itself. */ export async function sendReleasePurge(input: ReleasePurgeInput): Promise { const plan = await planFor(input); diff --git a/packages/core-ts/src/release-apply.ts b/packages/core-ts/src/release-apply.ts index 377bd4af..93d8b097 100644 --- a/packages/core-ts/src/release-apply.ts +++ b/packages/core-ts/src/release-apply.ts @@ -123,13 +123,13 @@ function approverOf(release: Release): Actor | undefined { return a.kind === "person" ? { kind: "human", ...a.by } : a.by; } -/** The snapshot's label: the release title, else "Scheduled: ". */ /** The site node a snapshot's `nodes` publish, and whether it is the working row (`hash`). */ function publishedSite(nodes: readonly NodeRef[], hash: string) { const ref = nodes.find((n) => n.kind === "site"); return ref ? { ref, current: ref.hash === hash } : undefined; } +/** The snapshot's label: the release title, else "Scheduled: ". */ async function labelOf( ctx: ReleaseApplyContext, release: Release, diff --git a/packages/core-ts/src/release-executor.ts b/packages/core-ts/src/release-executor.ts index 971108b6..710313e7 100644 --- a/packages/core-ts/src/release-executor.ts +++ b/packages/core-ts/src/release-executor.ts @@ -68,7 +68,7 @@ export interface ReleaseExecutorOptions { export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { const log = o.log ?? (() => {}); const counted = o.counted; - // RFC-0045: the blob calls every release run of this tick may make together. + // RFC-0045: the blob calls every release run of this invocation may make together. const allowance = o.releaseBlobs ?? { left: releaseBlobAllowance() }; const recordPurge = o.recordPurge ?? diff --git a/packages/core-ts/src/release-service-options.ts b/packages/core-ts/src/release-service-options.ts new file mode 100644 index 00000000..4d2e0fdb --- /dev/null +++ b/packages/core-ts/src/release-service-options.ts @@ -0,0 +1,48 @@ +/** + * The options and requests of `ReleaseService` (RFC-0008 §6, §12), split from `releases.ts`, which + * re-exports them. + */ +import type { Actor } from "@cloudhal/core-contracts"; +import type { WhenInput } from "@cloudhal/schedules"; +import type { SqlDriver } from "@cloudhal/storage"; +import type { ReleasePublishedHook } from "./release-after.ts"; +import type { ReleasePurger, SchedulerRuntime } from "./releases.ts"; +import type { SiteGraphStore } from "./store.ts"; + +export interface ReleaseServiceOptions { + readonly graph: SiteGraphStore; + /** The database of `graph` (default `graph.sql`). */ + readonly sql?: SqlDriver | undefined; + /** The clock for lead times and "Publish now" (default `Date.now`). */ + readonly now?: (() => Date) | undefined; + /** The ticker, poked when a release's time is set or moved (default: cron, no pokes). */ + readonly scheduler?: SchedulerRuntime | undefined; + /** "Publish now" purges like a tick does (absent: nothing cached). */ + readonly purge?: ReleasePurger | undefined; + /** New bodies one call may write (§6.1 step 4; Workers Free: 20). `Infinity` on Node. */ + readonly maxNewBodies?: number | undefined; + /** After "Publish now" commits a release, as a tick's `onReleasePublished` (the Node server rebuilds, #150). */ + readonly onReleasePublished?: ReleasePublishedHook | undefined; + readonly log?: ((line: string) => void) | undefined; + /** + * RFC-0045: the deployment's tick budget (`CLOUDHAL_TICK_BUDGET`), which sizes "Publish now"'s + * allowance of blob calls for its purge (default: Workers Free's). + */ + readonly tickBudget?: { readonly maxQueries: number } | undefined; +} + +export interface PinRequest { + /** Nodes whose current working-copy version goes live. */ + readonly set?: readonly string[] | undefined; + /** Entries, pages and redirects taken out of the live site. */ + readonly remove?: readonly string[] | undefined; +} + +export interface CreateReleaseRequest extends PinRequest { + readonly siteId: string; + readonly createdBy: Actor; + readonly title?: string | undefined; + /** An instant, a wall-clock time (in `timeZone`, else the site's zone), or `on-approval`. */ + readonly when?: WhenInput | undefined; + readonly metadata?: Record | undefined; +} diff --git a/packages/core-ts/src/releases.ts b/packages/core-ts/src/releases.ts index 02fc4cd7..946bef14 100644 --- a/packages/core-ts/src/releases.ts +++ b/packages/core-ts/src/releases.ts @@ -71,11 +71,16 @@ import { type BuiltinInputs, graphBuiltinRunner } from "./automations.ts"; import { type CachePurgePlan, planCachePurge } from "./cache-purge.ts"; import { NotFoundError } from "./errors.ts"; import type { NodeHold } from "./holds.ts"; -import { releaseBlobAllowance } from "./references/release-purge.ts"; +import { chargedBlobs, releaseBlobAllowance } from "./references/release-purge.ts"; import type * as after from "./release-after.ts"; import type { ReleasePurge } from "./release-apply.ts"; import { releaseDepsOf } from "./release-deps.ts"; import { releaseExecutor } from "./release-executor.ts"; +import type { + CreateReleaseRequest, + PinRequest, + ReleaseServiceOptions, +} from "./release-service-options.ts"; import { SiteGraphStore, type SiteGraphStoreOptions } from "./store.ts"; export { type ReleasePublishedHook, recordPurgeOutcome } from "./release-after.ts"; @@ -181,7 +186,10 @@ export async function runScheduledTick(o: ScheduledTickOptions): Promise Date) | undefined; - /** The ticker, poked when a release's time is set or moved (default: cron, no pokes). */ - readonly scheduler?: SchedulerRuntime | undefined; - /** "Publish now" purges like a tick does (absent: nothing cached). */ - readonly purge?: ReleasePurger | undefined; - /** New bodies one call may write (§6.1 step 4; Workers Free: 20). `Infinity` on Node. */ - readonly maxNewBodies?: number | undefined; - /** After "Publish now" commits a release, as a tick's `onReleasePublished` (the Node server rebuilds, #150). */ - readonly onReleasePublished?: after.ReleasePublishedHook | undefined; - readonly log?: ((line: string) => void) | undefined; -} - -export interface PinRequest { - /** Nodes whose current working-copy version goes live. */ - readonly set?: readonly string[] | undefined; - /** Entries, pages and redirects taken out of the live site. */ - readonly remove?: readonly string[] | undefined; -} - -export interface CreateReleaseRequest extends PinRequest { - readonly siteId: string; - readonly createdBy: Actor; - readonly title?: string | undefined; - /** An instant, a wall-clock time (in `timeZone`, else the site's zone), or `on-approval`. */ - readonly when?: WhenInput | undefined; - readonly metadata?: Record | undefined; -} +export type { CreateReleaseRequest, PinRequest, ReleaseServiceOptions }; export interface PinnedRelease { readonly release: Release; @@ -331,6 +308,7 @@ export class ReleaseService { readonly #maxNewBodies: number; readonly #onReleasePublished: after.ReleasePublishedHook | undefined; readonly #log: (line: string) => void; + readonly #tickBudget: ReleaseServiceOptions["tickBudget"]; constructor(o: ReleaseServiceOptions) { this.graph = o.graph; @@ -342,6 +320,7 @@ export class ReleaseService { this.#maxNewBodies = o.maxNewBodies ?? MAX_NEW_BODIES_PER_REQUEST; this.#onReleasePublished = o.onReleasePublished; this.#log = o.log ?? (() => {}); + this.#tickBudget = o.tickBudget; } async #site(siteId: string) { @@ -830,6 +809,7 @@ export class ReleaseService { graph: this.graph, schedules: this.schedules, purge: this.#purge, + releaseBlobs: { left: releaseBlobAllowance(this.#tickBudget) }, onPublished: this.#onReleasePublished, log: this.#log, }), diff --git a/packages/core-ts/src/testing-releases.ts b/packages/core-ts/src/testing-releases.ts index 91e9af6f..216b3ca1 100644 --- a/packages/core-ts/src/testing-releases.ts +++ b/packages/core-ts/src/testing-releases.ts @@ -746,7 +746,8 @@ export function releasesSuite( const plans: CachePurgePlan[] = []; await tick(T + 2 * HOUR, { purge: capture(plans), releaseBlobs }); expect(plans.map((p) => p.mode)).toEqual(["everything"]); - expect(releaseBlobs.left).toBe(0); + // the run's own calls (search, label, the gate) are counted, never refused: below 0 + expect(releaseBlobs.left).toBeLessThan(0); }); it("sends the purge re-planned at run time and records a deferred purge", async () => { diff --git a/packages/core-ts/test/references-purge.test.ts b/packages/core-ts/test/references-purge.test.ts index d5939926..4167c52d 100644 --- a/packages/core-ts/test/references-purge.test.ts +++ b/packages/core-ts/test/references-purge.test.ts @@ -24,6 +24,7 @@ import { nodeCacheTag, siteCacheTag } from "../src/cache-tags.ts"; import { continueFieldOperation, runFieldOperation } from "../src/fields-operations/index.ts"; import { canonicalJson, nodeHash } from "../src/hash.ts"; import { PNG_1X1 } from "../src/media/fixtures.ts"; +import { sendReleasePurge } from "../src/references/release-purge.ts"; import { seedStarterSite } from "../src/seed.ts"; import { type PublishChange, SiteGraphStore } from "../src/store.ts"; @@ -453,4 +454,82 @@ describe("planCachePurge: the referrer closure (RFC-0045 §2)", () => { const p = await plan(await publish()); expect(p.mode === "everything" ? p.reason : "tags").toMatch(/field operation/); }); + + describe("a release run's purge (sendReleasePurge)", () => { + const stored = { mode: "tags", tags: ["cloudhal-node-entry_STORED0001"], why: [] } as const; + const siteRefOf = (s: Snapshot) => s.nodes.find((n) => n.kind === "site") as NodeRef; + const run = async (o: { + graph?: SiteGraphStore; + siteId?: string; + left: number; + from: string; + to: string; + }) => { + const graph = o.graph ?? store; + const sent: unknown[] = []; + const allowance = { left: o.left }; + const toSnap = (await graph.getSnapshot(o.to)) as Snapshot; + const outcome = await sendReleasePurge({ + graph, + purge: async (_siteId, plan) => { + sent.push(plan); + return plan.mode === "everything" ? "everything" : "tags"; + }, + stored, + site: (await graph.getSite(o.siteId ?? site.id)) as Site, + publishedSite: { ref: siteRefOf(toSnap), current: false }, + from: o.from, + snapshotId: o.to, + allowance, + log: () => {}, + }); + return { sent, allowance, outcome }; + }; + + it("a site without references sends the stored plan with the allowance spent, or the gate read failing", async () => { + const plain = await seedStarterSite(store, { name: "Plain", slug: "plain-release" }); + const a = await store.createSnapshot(plain.id, { actor: human, reason: "publish" }); + await store.publish(plain, a.id); + const b = await store.createSnapshot(plain.id, { actor: human, reason: "manual" }); + const spent = await run({ siteId: plain.id, left: 0, from: a.id, to: b.id }); + expect(spent.sent).toEqual([stored]); + const failing = new SiteGraphStore({ + sql: store.sql, + blobs: new Proxy(store.blobs, { + get(target, prop, receiver) { + const v = Reflect.get(target, prop, receiver); + if (prop === "get") return async () => Promise.reject(new Error("R2 went away")); + return typeof v === "function" ? v.bind(target) : v; + }, + }), + }); + const broken = await run({ + graph: failing, + siteId: plain.id, + left: 100, + from: a.id, + to: b.id, + }); + expect(broken.sent).toEqual([stored]); + }); + + it("a re-plan with more terms than reads left purges everything before reading anything", async () => { + const { hall, event } = await world(); + const from = (await fresh()).publishedSnapshotId as string; + await edit(hall, { title: "Main hall" }); + const to = await snap("manual"); + const s = (await store.getSnapshot(to.id)) as Snapshot; + const terms = new Set(s.nodes.filter((n) => n.kind === "term").map((n) => n.hash)).size; + expect(terms).toBeGreaterThan(1); + // the gate read is one call; then fewer reads are left than there are terms + const { sent, allowance } = await run({ left: terms, from, to: to.id }); + expect(sent).toHaveLength(1); + expect((sent[0] as { reason?: string }).reason).toMatch(/terms/); + expect(allowance.left).toBe(terms - 1); + // enough left: tags + const ok = await run({ left: 2000, from, to: to.id }); + expect(ok.sent[0]).toMatchObject({ mode: "tags" }); + expect((ok.sent[0] as { tags: string[] }).tags).toContain(node(event)); + }); + }); }); diff --git a/packages/mcp-server/src/schedules-deps.ts b/packages/mcp-server/src/schedules-deps.ts index 8a103105..264e5120 100644 --- a/packages/mcp-server/src/schedules-deps.ts +++ b/packages/mcp-server/src/schedules-deps.ts @@ -31,6 +31,8 @@ export interface SchedulesDeps { readonly maxNewBodies?: number | undefined; /** After "Publish now" commits a release (the Node server rebuilds its static site, #150). */ readonly onReleasePublished?: ReleasePublishedHook | undefined; + /** RFC-0045: the deployment's tick budget, which sizes "Publish now"'s purge (default: Free's). */ + readonly tickBudget?: { readonly maxQueries: number } | undefined; } /** True when the deployment serves schedules. */ @@ -50,6 +52,7 @@ export function releaseService( ...(deps.purge ? { purge: deps.purge } : {}), ...(deps.maxNewBodies !== undefined ? { maxNewBodies: deps.maxNewBodies } : {}), ...(deps.onReleasePublished ? { onReleasePublished: deps.onReleasePublished } : {}), + ...(deps.tickBudget ? { tickBudget: deps.tickBudget } : {}), }); }