diff --git a/.changeset/rfc45-step3b-purges.md b/.changeset/rfc45-step3b-purges.md new file mode 100644 index 00000000..dafe978c --- /dev/null +++ b/.changeset/rfc45-step3b-purges.md @@ -0,0 +1,34 @@ +--- +"@cloudhal/core-ts": patch +"@cloudhal/cloudflare": patch +"@cloudhal/mcp-server": patch +--- + +RFC-0045 rollout step 3b: publishes purge the pages that show references. Nothing changes for sites without reference definitions. Their publish purge plans are the same, with no bundle read. Their release runs send the plan stored at pin time, as before, with no re-plan; the only addition is one read of the published site node, to tell them apart. + +- **The referrer closure** (`references/purge.ts`, called by `planCachePurge`). + - The seeds are the groups of every changed, added or removed entry, and of every entry whose shown asset changed. + - It purges the referrers of those groups, two hops deep, with every language version of each referrer. + - It also purges every member of each group a changed entry names, before or after ("used by"). + - Values are resolved, so a translation that only inherits counts. + - It reads only the `from` and `to` snapshots' bodies, never the working copy or `entry_refs`. +- **Reads and budgets.** A publish that changes no entry and no shown asset reads nothing. Otherwise the closure keeps one hash-keyed cache: + - one bundle (`to`'s, else `from`'s), streamed and cut down as each shard is parsed; + - then the missing hashes, read by the closure itself, six at a time, and cut down the same way. + + A passed budget purges everything: + - bytes (`maxBodyBytes`, 32 MB); + - nodes (`maxNodes`, 50,000); + - reads (`maxReads`, shard reads included); + - tags (`maxTags`). + + So does any field-operation marker on either snapshot's site node, for sites that hold references or convert from them. +- **`readSnapshotBundle`** (`SnapshotReader.readSnapshotBundle`, `SiteGraphStore.readSnapshotBundle`, `readSnapshotBundle` in bundles) takes optional `maxBytes`, `maxNodes`, `keep`, `only` (bodies node by node) and `spent` (a shared byte total). A passed budget throws `BundleBudgetError` before the shard is parsed. No new cache tags. +- **Release runs on sites with references re-plan their purge at run time** (`references/release-purge.ts`), from the pointer they moved to the snapshot they made, instead of sending the plan stored at pin time. + - The test is the published site node, which `publishRelease` now returns (`publishedSite`). That costs one blob read per run with a purger, except on a first publish; the read is counted, never refused. If it fails, the working row decides. + - The re-plan runs over the tick's counting driver. + - The release graph's blob calls (`refreshSearch`, `labelOf`, the re-plan) are charged to one allowance per invocation, shared by the cron's ticks (`ScheduledTickOptions.releaseBlobs`): 350 on Workers Free, 3,000 on Paid. "Publish now" sizes it from the deployment's tick budget (`ReleaseService` and `SchedulesDeps.tickBudget`). + - Past the allowance, or if the re-plan fails, the run purges everything. A re-plan with more distinct terms than reads left purges everything before reading anything. + - The pin-time plan skips the closure (`CachePurgeOptions.references: false`): for these sites it is the expected outcome only. + - `RELEASE_EXECUTOR_COST` is 26 (it counts `refreshSearch` and `recordPurge` now). +- **`@cloudhal/cloudflare`:** the cron handler passes one `releaseBlobs` allowance to both ticks of its invocation (`cronTick`); the alarm and the request backstop run one tick each, with the default allowance. diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index b2bb0163..e62e0b3e 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -1210,3 +1210,39 @@ Rollout step 2 shipped in two PRs: 2a, store and integrity, and 2b, the consumer - **`ReferencedEntry.path`** (§4): the member's graph path (`entryPath`, no locale prefix). Themes link with it, so the locale middleware localizes it once, like every other entry link. `url` stays the canonical URL, already localized, and absolute only on a locale's own domain. - **`referencedBy` follows each page** (§4): a translation resolves with its same-site original while the view shows that original, and without it otherwise, as its page reads it through the gated `entryById`. Ties on date order by id, so both sources agree on order and on what `limit` keeps. + +### Amended in 3b (2026-11-12, after the PR #212 review) + +- **Seeds purge their own groups' members** (§2 Purges, point 1). A translation inheriting a changed original's reference shows a new card, but it is neither a referrer of the seed nor changed itself. +- **No seeds, no reads.** A publish that changes no entry and no shown asset (term-only, asset-only, site-only) reads nothing for the closure, so the budgets cannot turn it into purge-everything. +- **Which sites count.** A site counts as having references when its `from` or `to` site node defines a reference field, a repeater with a reference sub-field, or a field (or `sub`) still converting *from* references (`migration.fromType: "reference"`). During `change_field_type` reference → text, the converted values render their targets' titles through the conversion maps until phase B rewrites them. On such sites any field-operation marker purges everything; other sites keep today's purge. +- **The closure's own reads.** Bodies the planner already holds are reused. The missing hashes are read by the closure itself (`readSnapshotBundle` with `only`): + - six at a time; + - counted against `maxReads`; + - added to the same byte total as the bundle before each parse; + - cut to what the closure needs, and never cached whole. +- **The node budget** (`maxNodes`) counts the entry bodies the closure holds across both snapshots, plus the node count of the one bundle it reads. +- **Release runs.** + - **The gate is the published site node,** not the working row. A release never carries the site node, so its snapshot publishes the live one's; `publishRelease` returns that ref. Only a node that defines references, or still converts from them, re-plans. Every other site sends the plan stored at pin time, as before 3b (nothing when it purges nothing). + - That gate costs one blob read per release run that has a purger, except on a first publish: a snapshot's site node is captured before the publish flips the pointer, so the working row equals it only on a site's first publish. + - The read is counted but never refused, so a site without references never depends on what other runs left. If it fails, the working row decides: no references, the stored plan; otherwise purge everything. + - **The re-plan** goes from the pointer the batch moved (`from`) to the new snapshot. That is `baseSnapshotId` whenever something was live; a first publish purges everything. + - It runs over the tick's counting driver, never refused, and spends no shard reads on `to`. + - If it throws, the run purges everything, never the stored plan. + - **No bundle write at run time.** The site Worker writes the new snapshot's bundle on its first read. Back-to-back releases of one site with no visitor between them therefore re-plan node by node, within the allowance, or purge everything: an over-purge, never a stale page. + - **Subrequests.** R2 and D1 calls are capped per invocation: 1,000 on Workers Free, by default 10,000 on Paid. + - Release runs charge their *actual* blob calls to one allowance per invocation, shared by both ticks of the cron's invocation. That includes `refreshSearch`, `labelOf` and a first publish's own blobs, counted but never refused. Each re-plan gets `maxReads = min(2,000, left)`. + - A re-plan whose two snapshots hold more distinct terms than reads left purges everything before reading anything. + - "Publish now" sizes its allowance from the deployment's tick budget (`CLOUDHAL_TICK_BUDGET`). + - Free: 1,000 − 2 × (40 + 7) D1 − 2 × 40 built-in blobs − 2 × (211 `refreshSearch` + 2 + 4 purge sends) = 392, so 350. + - Paid: about 27 releases fit a tick's 900 statements (33 each with claim and finish). The allowance is sized for ticks of up to 10 releases of 100 items, 3,000. Twenty-seven such releases pass the default cap with `refreshSearch` alone, re-plan or not; that is RFC-0008's budget, unchanged here. + - **Accepted over-purges, not fixed:** + - On Free, a reference site with more terms than the allowance leaves purges everything on every release, because the planner reads every term. No cache priming is built for it. + - On Free, a reference-site release that changes an existing asset reads every entry, page and term one by one in the asset rule, so above about 350 nodes it purges everything. + - The AI Search mirror's R2 calls (RFC-0029) are not counted. + - **`RELEASE_EXECUTOR_COST` is 26 statements:** + - the stored plan (1), the reads (3), the batch (8) and `refreshSearch` (7); + - then the re-plan's two `getSnapshot` reads, the bookkeeping (4) and `recordPurge` (1). + + The rare site-moved retry stays outside it, as before. +- **The pin-time plan skips the closure** (`references: false`). It would read every body one by one, and for sites with references it is the batch's *expected* outcome only: the run-time re-plan, or purge-everything, covers referrers. diff --git a/packages/cli/test/reference-links.test.ts b/packages/cli/test/reference-links.test.ts new file mode 100644 index 00000000..842e73df --- /dev/null +++ b/packages/cli/test/reference-links.test.ts @@ -0,0 +1,152 @@ +/** + * RFC-0045 §4 through the real pipeline: the starter's `EntryFields.astro` draws a reference's + * card by its graph path, and the locale middleware (`localizeLinks`) gives it the page's locale + * exactly once. Built with `cloudhal build` under `prefix-except-default` and `prefix-all`: a + * Turkish referrer links its venue's Turkish member at `/tr/places/…` (never `/tr/tr/…`), and a + * fallback card (`default-with-notice`) links the default locale's member. The types stay refused + * in production; the test enables them on its own store, as the shared suites do. + */ +import { spawn } from "node:child_process"; +import { mkdtemp, readFile, rm, symlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import type { ContentEntry, Site } from "@cloudhal/core-contracts"; +import { GATES_ENFORCED, nodeHash, SiteGraphStore } from "@cloudhal/core-ts"; +import { openLocalStore } from "@cloudhal/core-ts/local"; +import { afterAll, describe, expect, it } from "vitest"; +import { init, readProjectConfig, snapshot } from "../src/index.ts"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const STARTER_NODE_MODULES = path.resolve(here, "../../../starters/blog/node_modules"); +const BIN = path.resolve(here, "../bin/cloudhal.ts"); +const author = { kind: "human", id: "user_01HZX8K3Q9CD", name: "Halil" }; + +async function runBuild(projectDir: string): Promise<{ code: number | null; output: string }> { + const { VITEST: _v, NODE_ENV: _n, ...env } = process.env; + const child = spawn(process.execPath, [BIN, "build"], { + cwd: projectDir, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout?.on("data", (c: Buffer) => (output += c.toString())); + child.stderr?.on("data", (c: Buffer) => (output += c.toString())); + const code = await new Promise((resolve) => child.once("exit", resolve)); + return { code, output }; +} + +/** Places with a Turkish member (`hall`/`salon`) and without (`pier`), and events naming them. */ +async function addReferences( + projectDir: string, + routing: "prefix-except-default" | "prefix-all", +): Promise { + const cfg = await readProjectConfig(projectDir); + const local = await openLocalStore(path.join(projectDir, cfg.dataDir)); + const store = new SiteGraphStore({ + sql: local.sql, + blobs: local.blobs, + gates: GATES_ENFORCED, + fieldTypes: { supported: () => true }, + }); + try { + const site = (await store.getSite(cfg.siteId)) as Site; + // the seed's required category applies to every collection: reuse the seeded post's terms + const [hello] = await store.listContentEntries(cfg.siteId, "posts"); + await store.updateSite({ + ...site, + locales: ["en", "tr"], + localization: { ...site.localization, routing, fallback: "default-with-notice" }, + collections: { + ...site.collections, + places: {}, + events: { + fields: [ + { + key: "venue", + type: "reference", + labels: { en: { label: "Venue" }, tr: { label: "Mekan" } }, + localized: false, + collections: ["places"], + }, + ], + }, + }, + }); + const make = (collection: string, slug: string, over: Partial) => + store.createContentEntry({ + siteId: cfg.siteId, + collection, + slug, + locale: "en", + status: "published", + title: `The ${slug}`, + body: { format: "markdown", value: `A page about the ${slug}, with enough words to read.` }, + fields: {}, + tags: [], + ...(hello?.terms ? { terms: hello.terms } : {}), + author, + ...over, + } as never) as Promise; + const translate = async (of: ContentEntry, slug: string) => + make(of.collection, slug, { + locale: "tr", + title: `Türkçe ${slug}`, + translation: { + group: of.id, + method: "human", + source: { id: of.id, locale: of.locale, rev: of.rev, hash: await nodeHash(of) }, + }, + }); + const hall = await make("places", "hall", {}); + await translate(hall, "salon"); + const pier = await make("places", "pier", {}); + await translate(await make("events", "launch", { fields: { venue: hall.id } }), "lansman"); + await translate(await make("events", "gig", { fields: { venue: pier.id } }), "konser"); + } finally { + await local.close(); + } + await snapshot({ projectDir, label: "references", publish: true }); +} + +/** The hrefs inside a page's typed-fields list. */ +const fieldLinks = (html: string) => + [ + ...(html.match(/
/)?.[0] ?? "").matchAll(/href="([^"]*)"/g), + ].map((m) => m[1]); + +describe.each(["prefix-except-default", "prefix-all"] as const)( + "reference links in built pages (%s)", + (routing) => { + let tmp: string; + afterAll(async () => { + if (tmp) await rm(tmp, { recursive: true, force: true }); + }); + + it("links each card once in the page's locale, the fallback in the default locale", async () => { + tmp = await mkdtemp(path.join(os.tmpdir(), "cloudhal-ref-links-")); + const projectDir = path.join(tmp, "site"); + await init({ dir: projectDir, name: "Reference Links" }); + await symlink(STARTER_NODE_MODULES, path.join(projectDir, "node_modules"), "dir"); + await addReferences(projectDir, routing); + const { code, output } = await runBuild(projectDir); + expect(code, output.slice(-4000)).toBe(0); + const read = (...p: string[]) => readFile(path.join(projectDir, "dist", ...p), "utf8"); + const en = routing === "prefix-all" ? ["en"] : []; + // a Turkish referrer: the venue's Turkish member, prefixed once + expect(fieldLinks(await read("tr", "events", "lansman", "index.html"))).toEqual([ + "/tr/places/salon", + ]); + // no Turkish member: the default locale's, through `default-with-notice` + expect(fieldLinks(await read("tr", "events", "konser", "index.html"))).toEqual([ + `${en.map((l) => `/${l}`).join("")}/places/pier`, + ]); + // the default locale's own page + expect(fieldLinks(await read(...en, "events", "launch", "index.html"))).toEqual([ + `${en.map((l) => `/${l}`).join("")}/places/hall`, + ]); + const all = await read("tr", "events", "lansman", "index.html"); + expect(all).not.toMatch(/href="\/(en|tr)\/(en|tr)\//); + }); + }, +); diff --git a/packages/cloudflare/src/admin-worker.ts b/packages/cloudflare/src/admin-worker.ts index a28f8cb3..48559a52 100644 --- a/packages/cloudflare/src/admin-worker.ts +++ b/packages/cloudflare/src/admin-worker.ts @@ -71,8 +71,9 @@ import { adminMail, type MailEnv } from "./mail.ts"; import { purgeSiteCache } from "./purge.ts"; import { type BackstopState, + cronTick, followUpDelay, - releasePurger, + publishNowPurge, requestBackstop, runAdminTick, type SchedulerEnv, @@ -537,7 +538,7 @@ export function createAdminWorker(options: AdminWorkerOptions = {}) { // RFC-0008 §12–§13: the schedule routes and tools; "Publish now" purges like a tick. schedules: { store: new SqlScheduleStore(sql), - purge: releasePurger(env, sql, { log }), + ...publishNowPurge(env, sql, log), automationsEnabled: env.CLOUDHAL_AUTOMATIONS !== "off", }, scheduler: { @@ -571,8 +572,7 @@ export function createAdminWorker(options: AdminWorkerOptions = {}) { await ensureMigrated(sql); const tick = (now: Date) => runAdminTick(env, { - trigger: "cron", - now, + ...cronTick(controller, env, now), log, analytics: (s, at) => adminAnalytics(env, s, { now: () => at, log }), commentsOn: commentsOn(env), diff --git a/packages/cloudflare/src/scheduler.ts b/packages/cloudflare/src/scheduler.ts index c1b40ab0..4f50e119 100644 --- a/packages/cloudflare/src/scheduler.ts +++ b/packages/cloudflare/src/scheduler.ts @@ -30,6 +30,7 @@ import { SqlCommentStore } from "@cloudhal/comments/sql"; import { type AiSearchEnv, type CachePurgePlan, + releaseBlobAllowance, SiteGraphStore, searchProviderFromEnv, siteCacheTag, @@ -380,10 +381,35 @@ export interface AdminTickOptions { readonly commentsOn: boolean; /** Keeps a deferred purge's flush alive after the tick (the handler's `waitUntil`). */ readonly waitUntil?: ((p: Promise) => void) | undefined; + /** + * RFC-0045: the R2 calls release purges may still make in this invocation, shared by every tick + * it runs (the cron's follow-up); absent: one allowance per tick. + */ + readonly releaseBlobs?: { left: number } | undefined; } const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +/** "Publish now"'s purge, and the deployment's tick budget that sizes its blob calls (RFC-0045). */ +export function publishNowPurge(env: SchedulerEnv, sql: SqlDriver, log: (line: string) => void) { + return { purge: releasePurger(env, sql, { log }), tickBudget: tickBudget(env) }; +} + +const invocationBlobs = new WeakMap(); + +/** + * A cron tick's trigger and clock, with RFC-0045's R2 allowance for release purges: one per + * invocation (`controller`), so the cron's follow-up tick shares it (the cap is per invocation). + */ +export function cronTick(controller: object, env: SchedulerEnv, now: Date) { + let releaseBlobs = invocationBlobs.get(controller); + if (!releaseBlobs) { + releaseBlobs = { left: releaseBlobAllowance(tickBudget(env)) }; + invocationBlobs.set(controller, releaseBlobs); + } + return { trigger: "cron" as const, now, releaseBlobs }; +} + /** One tick of the admin Worker (cron, alarm, request backstop or the operator's run-due). */ export function runAdminTick(env: SchedulerEnv, o: AdminTickOptions): Promise { const sql = new D1Driver(env.DB); @@ -425,6 +451,7 @@ export function runAdminTick(env: SchedulerEnv, o: AdminTickOptions): Promise releasePurger(env, counted, { log: o.log, wake: flush }), + ...(o.releaseBlobs ? { releaseBlobs: o.releaseBlobs } : {}), automationsEnabled: env.CLOUDHAL_AUTOMATIONS !== "off", // RFC-0008 §11.1: the built-in jobs read analytics' 404 rows and the site's own feed. builtinInputs: ({ counted }) => adminBuiltinInputs(env, o.analytics(counted, o.now)), diff --git a/packages/cloudflare/test/scheduler.test.ts b/packages/cloudflare/test/scheduler.test.ts index 4e410abe..f9faf606 100644 --- a/packages/cloudflare/test/scheduler.test.ts +++ b/packages/cloudflare/test/scheduler.test.ts @@ -15,6 +15,7 @@ import { createMiniflareD1, dropAllTables } from "@cloudhal/storage/testing"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { adminBuiltinInputs, + cronTick, followUpDelay, nextAlarm, releasePurger, @@ -41,6 +42,19 @@ describe("the ticker's mode and budget", () => { expect(tickBudget(env({ CLOUDHAL_TICK_BUDGET: "paid" })).maxQueries).toBe(900); }); + it("RFC-0045: the cron's ticks of one invocation share one R2 allowance for release purges", () => { + const one = {}; + const a = cronTick(one, env(), new Date(0)); + const b = cronTick(one, env(), new Date(60_000)); + expect(a.trigger).toBe("cron"); + expect(a.releaseBlobs).toBe(b.releaseBlobs); + expect(a.releaseBlobs.left).toBe(350); + expect(cronTick({}, env(), new Date(0)).releaseBlobs).not.toBe(a.releaseBlobs); + expect(cronTick({}, env({ CLOUDHAL_TICK_BUDGET: "paid" }), new Date(0)).releaseBlobs.left).toBe( + 3000, + ); + }); + it("cron ignores pokes; alarm pokes the clock", async () => { expect(schedulerRuntime(env()).mode).toBe("cron"); const seen: string[] = []; diff --git a/packages/core-ts/src/bundles.ts b/packages/core-ts/src/bundles.ts index b5614ecb..bac4c3bb 100644 --- a/packages/core-ts/src/bundles.ts +++ b/packages/core-ts/src/bundles.ts @@ -58,16 +58,47 @@ export async function writeSnapshotBundle( } /** - * The snapshot's bodies by node hash, or `undefined` unless every shard is there and matches the - * snapshot (version, snapshot id, the refs' hashes in order). Never throws for a bad bundle: the - * caller falls back to `nodes/.json`. + * RFC-0045 §2 "Purges": a streaming read. Budgets are checked before each shard is parsed, so a + * bundle too large for the isolate fails loudly (`BundleBudgetError`) instead of running it out + * of memory; `keep` reduces each body as its shard is parsed (`undefined` drops it). + */ +export interface BundleReadOptions { + /** More shard bytes than this: `BundleBudgetError`, before the shard that would pass it. */ + readonly maxBytes?: number | undefined; + /** A snapshot of more nodes than this: `BundleBudgetError`, before any shard is read. */ + readonly maxNodes?: number | undefined; + /** What to keep of each body (default: the body). */ + readonly keep?: ((body: unknown, ref: NodeRef) => unknown) | undefined; + /** + * Read these refs' bodies from `nodes/.json` instead of the shards, with the same budgets + * and `keep`, at most `BLOB_CONCURRENCY` at a time; a missing body throws. + */ + readonly only?: readonly NodeRef[] | undefined; + /** A running byte total shared across reads, so `maxBytes` bounds them all together. */ + readonly spent?: { bytes: number } | undefined; +} + +/** A bundle read passed `BundleReadOptions.maxBytes` or `maxNodes`. */ +export class BundleBudgetError extends Error {} + +/** + * The snapshot's bodies by node hash (or what `options.keep` keeps of them), or `undefined` unless + * every shard is there and matches the snapshot (version, snapshot id, the refs' hashes in order). + * Never throws for a bad bundle, so the caller falls back to `nodes/.json`; only a passed + * budget throws (`BundleBudgetError`). */ export async function readSnapshotBundle( blobs: BlobStore, snapshot: Snapshot, + options: BundleReadOptions = {}, ): Promise | undefined> { + if (options.only) return readNodeBodies(blobs, options.only, options); const count = bundleShardCount(snapshot); if (count === 0) return undefined; + const { maxBytes, maxNodes, keep } = options; + const spent = options.spent ?? { bytes: 0 }; + if (maxNodes !== undefined && snapshot.nodes.length > maxNodes) + throw new BundleBudgetError(`${snapshot.nodes.length} nodes (more than ${maxNodes})`); const bodies = new Map(); let complete = true; const decoder = new TextDecoder(); @@ -82,6 +113,9 @@ export async function readSnapshotBundle( complete = false; return; } + spent.bytes += bytes.byteLength; + if (maxBytes !== undefined && spent.bytes > maxBytes) + throw new BundleBudgetError(`more than ${maxBytes} bytes of bodies`); const parsed = JSON.parse(decoder.decode(bytes)) as { v?: unknown; snapshotId?: unknown; @@ -105,11 +139,40 @@ export async function readSnapshotBundle( complete = false; return; } - for (const [i, r] of refs.entries()) bodies.set(r.hash, list[i]); + for (const [i, r] of refs.entries()) { + const kept = keep ? keep(list[i], r) : list[i]; + if (kept !== undefined) bodies.set(r.hash, kept); + } }, ); - } catch { + } catch (e) { + if (e instanceof BundleBudgetError) throw e; return undefined; // unreadable or unparsable shard: the bundle is absent } return complete ? bodies : undefined; } + +/** `only`: bodies node by node, budgets checked before each parse (`readSnapshotBundle`). */ +async function readNodeBodies( + blobs: BlobStore, + refs: readonly NodeRef[], + options: BundleReadOptions, +): Promise> { + const { maxBytes, maxNodes, keep } = options; + const spent = options.spent ?? { bytes: 0 }; + if (maxNodes !== undefined && refs.length > maxNodes) + throw new BundleBudgetError(`${refs.length} nodes (more than ${maxNodes})`); + const out = new Map(); + const decoder = new TextDecoder(); + await eachLimited([...refs], BLOB_CONCURRENCY, async (ref) => { + const bytes = await blobs.get(`nodes/${ref.hash}.json`); + if (!bytes) throw new Error(`node body ${ref.hash} not found`); + spent.bytes += bytes.byteLength; + if (maxBytes !== undefined && spent.bytes > maxBytes) + throw new BundleBudgetError(`more than ${maxBytes} bytes of bodies`); + const body = JSON.parse(decoder.decode(bytes)) as unknown; + const kept = keep ? keep(body, ref) : body; + if (kept !== undefined) out.set(ref.hash, kept); + }); + return out; +} diff --git a/packages/core-ts/src/cache-purge-assets.ts b/packages/core-ts/src/cache-purge-assets.ts index 556a6d25..80f2664d 100644 --- a/packages/core-ts/src/cache-purge-assets.ts +++ b/packages/core-ts/src/cache-purge-assets.ts @@ -28,8 +28,12 @@ export interface AssetPurgeInput { readonly why: string[]; } -/** Adds the asset rule's tags and log lines to `input.tags` and `input.why`. */ -export async function planAssetPurge(input: AssetPurgeInput): Promise { +/** + * Adds the asset rule's tags and log lines to `input.tags` and `input.why`; returns the entries it + * reached (RFC-0045: their referrers' cards show the changed cover). + */ +export async function planAssetPurge(input: AssetPurgeInput): Promise> { + const reached = new Set(); const { tags, why, read } = input; const shown = new Set(); for (const c of input.changed) { @@ -45,7 +49,7 @@ export async function planAssetPurge(input: AssetPurgeInput): Promise { else why.push(`asset ${id} changed`); shown.add(id); } - if (shown.size === 0) return; + if (shown.size === 0) return reached; const refsOf = await referencesIn(input.to, read); for (const ref of input.to.nodes) { if (ref.kind !== "contentEntry" && ref.kind !== "page" && ref.kind !== "term") continue; @@ -54,7 +58,9 @@ export async function planAssetPurge(input: AssetPurgeInput): Promise { if (ref.kind === "term") for (const id of input.withAncestors(ref.id)) tags.add(termCacheTag(id)); else tags.add(nodeCacheTag(ref.id)); + if (ref.kind === "contentEntry") reached.add(ref.id); } + return reached; } /** A node's asset references in `snapshot`: its own, and an entry's typed fields' by its defs. */ diff --git a/packages/core-ts/src/cache-purge.ts b/packages/core-ts/src/cache-purge.ts index b99854c7..ba6509f9 100644 --- a/packages/core-ts/src/cache-purge.ts +++ b/packages/core-ts/src/cache-purge.ts @@ -24,6 +24,11 @@ * nothing (404s are never cached, and whatever starts showing it changed itself). Media never * carry the site tag, so the site-wide purges above leave them. * + * - RFC-0045 **references**: the referrers of every changed entry's group, two hops deep, and the + * targets a changed entry names ("used by"), found from the two snapshots' own bodies, never + * the working copy (`references/purge.ts`; one bundle, then only the missing hashes, within + * byte and node budgets). Any field-operation marker on either site node purges everything. + * * Falls back to purging everything when there is no previous snapshot, a snapshot or node cannot * be read, reading the diff would take more than `maxReads` node reads, or the result has more * than `maxTags` tags. @@ -39,13 +44,18 @@ import type { import { planAssetPurge } from "./cache-purge-assets.ts"; import { listCacheTag, nodeCacheTag, siteCacheTag, termCacheTag } from "./cache-tags.ts"; import { canonicalJson } from "./hash.ts"; +import { + planReferencePurge, + type ReferencePurgeLimits, + type ReferencePurgeReader, +} from "./references/purge.ts"; -export interface SnapshotReader { +export interface SnapshotReader extends ReferencePurgeReader { getSnapshot(id: string): Promise; readSnapshotNode(ref: NodeRef): Promise; } -export interface CachePurgeOptions { +export interface CachePurgeOptions extends ReferencePurgeLimits { /** More tags than this purge everything instead (default 300: three requests of 100). */ readonly maxTags?: number; /** More node reads than this purge everything instead (default 2000). */ @@ -97,6 +107,69 @@ function filedUnder(e: ContentEntry | undefined): string[] { return Object.values(e.terms).flatMap((a) => a?.ids ?? []); } +type Change = { before?: NodeRef; after?: NodeRef }; + +/** The planner's one hash-keyed body cache; reads past `maxReads` (bundle shards too) throw. */ +function countedReader(reader: SnapshotReader, maxReads: number) { + let reads = 0; + const bodies = new Map(); + const spend = (n: number) => { + reads += n; + if (reads > maxReads) throw new TooManyReads(); + }; + const read = async (ref: NodeRef): Promise => { + const hit = bodies.get(ref.hash); + if (hit !== undefined) return hit as T; + spend(1); + const body = (await reader.readSnapshotNode(ref)) as T; + bodies.set(ref.hash, body); + return body; + }; + return { read, spend, cached: (hash: string) => bodies.get(hash) }; +} + +/** Both snapshots' vocabularies, and a term with its ancestors in either (archives follow parents). */ +async function ancestry( + before: Map, + after: Map, + read: (ref: NodeRef) => Promise, +) { + const termsOf = async (refs: Map) => { + const out = new Map(); + for (const ref of refs.values()) + if (ref.kind === "term") out.set(ref.id, await read(ref)); + return out; + }; + const oldTerms = await termsOf(before); + const newTerms = await termsOf(after); + const withAncestors = (id: string): string[] => { + const out = new Set(); + for (const terms of [oldTerms, newTerms]) { + let cursor: string | undefined = id; + while (cursor && !out.has(cursor)) { + out.add(cursor); + cursor = terms.get(cursor)?.parentId; + } + } + return [...out]; + }; + return { oldTerms, newTerms, withAncestors }; +} + +/** The nodes of both snapshots by id, and every node that differs between them. */ +function diffNodes(from: Snapshot, to: Snapshot) { + const before = new Map(from.nodes.map((n) => [n.id, n])); + const after = new Map(to.nodes.map((n) => [n.id, n])); + const changed: Change[] = []; + for (const [id, ref] of after) { + const prev = before.get(id); + if (!prev || prev.hash !== ref.hash) + changed.push({ ...(prev ? { before: prev } : {}), after: ref }); + } + for (const [id, ref] of before) if (!after.has(id)) changed.push({ before: ref }); + return { before, after, changed }; +} + export async function planCachePurge( reader: SnapshotReader, fromId: string | undefined, @@ -113,16 +186,7 @@ export async function planCachePurge( if (!to) return { mode: "everything", reason: `snapshot ${toId} not found` }; if (from.siteId !== to.siteId) return { mode: "everything", reason: "different sites" }; - let reads = 0; - const bodies = new Map(); - const read = async (ref: NodeRef): Promise => { - const hit = bodies.get(ref.hash); - if (hit !== undefined) return hit as T; - if (++reads > maxReads) throw new TooManyReads(); - const body = (await reader.readSnapshotNode(ref)) as T; - bodies.set(ref.hash, body); - return body; - }; + const { read, spend, cached } = countedReader(reader, maxReads); try { return await plan(); @@ -138,15 +202,7 @@ export async function planCachePurge( async function plan(): Promise { const site = siteCacheTag(to?.siteId as string); const wholeSite = (why: string): CachePurgePlan => ({ mode: "tags", tags: [site], why: [why] }); - const before = new Map((from as Snapshot).nodes.map((n) => [n.id, n])); - const after = new Map((to as Snapshot).nodes.map((n) => [n.id, n])); - const changed: Array<{ before?: NodeRef; after?: NodeRef }> = []; - for (const [id, ref] of after) { - const prev = before.get(id); - if (!prev || prev.hash !== ref.hash) - changed.push({ ...(prev ? { before: prev } : {}), after: ref }); - } - for (const [id, ref] of before) if (!after.has(id)) changed.push({ before: ref }); + const { before, after, changed } = diffNodes(from as Snapshot, to as Snapshot); if (changed.length === 0) return { mode: "tags", tags: [], why: [] }; // Structure first: no node body needs reading to know these purge the site. @@ -170,34 +226,22 @@ export async function planCachePurge( return wholeSite("the site settings or navigation changed"); } - // The vocabulary of both snapshots, for ancestors (archives and breadcrumbs follow parents). - const termsOf = async (refs: Map) => { - const out = new Map(); - for (const ref of refs.values()) - if (ref.kind === "term") out.set(ref.id, await read(ref)); - return out; - }; const entryChanges = changed.filter((c) => (c.after ?? c.before)?.kind === "contentEntry"); const termChanges = changed.filter((c) => (c.after ?? c.before)?.kind === "term"); - const oldTerms = await termsOf(before); - const newTerms = await termsOf(after); - const withAncestors = (id: string): string[] => { - const out = new Set(); - for (const terms of [oldTerms, newTerms]) { - let cursor: string | undefined = id; - while (cursor && !out.has(cursor)) { - out.add(cursor); - cursor = terms.get(cursor)?.parentId; - } - } - return [...out]; - }; + const { oldTerms, newTerms, withAncestors } = await ancestry(before, after, read); const tags = new Set(); const why: string[] = []; // Media (§7.4): an asset that left the live snapshot or changed (`cache-purge-assets.ts`). - await planAssetPurge({ changed, to: to as Snapshot, read, withAncestors, tags, why }); + const reached = await planAssetPurge({ + changed, + to: to as Snapshot, + read, + withAncestors, + tags, + why, + }); for (const c of termChanges) { const a = c.before ? await read(c.before) : undefined; @@ -230,6 +274,12 @@ export async function planCachePurge( if (flip) return wholeSite(flip); } + // RFC-0045: the referrer closure, from the two snapshots' bodies (`references/purge.ts`). + const all = await planReferencePurge({ + ...{ reader, read, cached, spend, limits: options, entries, reached, tags, why }, + ...{ from: from as Snapshot, to: to as Snapshot }, + }); + if (all) return { mode: "everything", reason: all }; if (tags.size > maxTags) return { mode: "everything", reason: `${tags.size} tags (more than ${maxTags})` }; return { mode: "tags", tags: [...tags], why }; diff --git a/packages/core-ts/src/index.ts b/packages/core-ts/src/index.ts index b6710be6..faeb3193 100644 --- a/packages/core-ts/src/index.ts +++ b/packages/core-ts/src/index.ts @@ -84,6 +84,10 @@ export { type ReferrerCandidates, referrerCandidates, } from "./references/reads.ts"; +export { + RELEASE_BLOB_ALLOWANCE, + releaseBlobAllowance, +} from "./references/release-purge.ts"; export type { PublishReleaseInput, PublishReleaseResult, diff --git a/packages/core-ts/src/references/purge.ts b/packages/core-ts/src/references/purge.ts new file mode 100644 index 00000000..14f64de2 --- /dev/null +++ b/packages/core-ts/src/references/purge.ts @@ -0,0 +1,302 @@ +/** + * RFC-0045 §2 "Purges": the purge planner's referrer closure. Pages carry only the node tags they + * already carry (`cloudhal-node-`); no new tags and no theme code. Exact by construction: it + * reads only the immutable `from` and `to` snapshots' bodies, never the working copy or + * `entry_refs`, so a referrer edited away or deleted since is still found where a cached page + * came from. + * + * 1. **Seeds:** the groups (`COALESCE(translation.group, id)`) of every changed, added or removed + * entry, before and after (a group change seeds both), and of every entry the asset rule + * reached (its cover changed). Their members' pages go too (a translation inheriting a changed + * original's reference shows a new card). + * 2. **Two hops:** the referrers of the seed groups, then the referrers of *their* groups (pages + * draw cards to depth 2), from both snapshots, whatever their status. Each referrer purges its + * node tag and every member of its group's. + * 3. **"Used by":** every member of each group a changed or reached entry names, before or after, + * so a target page listing `referencedBy` follows its referrers. + * + * "Names" means the resolved values (`resolveFields`, with the same snapshot's same-site original + * and without it, since a page reads whichever its view shows), so a translation that only + * inherits counts. + * + * Reads, only when something seeds it: one hash-keyed cache over both snapshots. It loads one + * bundle, `to`'s, else `from`'s (RFC-0010; none for a snapshot known to have none, `unbundled`), + * streamed: each body is cut to what the closure needs as its shard is parsed, and the byte and + * node budgets are checked before each parse. Bodies the planner already holds are reused; the + * hashes still missing are read by the closure itself (`only`, six at a time, the same budgets), + * never into the planner's cache of full bodies. Shard and node reads count against `maxReads`. + * + * Purges everything (`string` result) when a budget is passed, and whenever a field-operation + * marker is on either site node (renames and conversions in flight read through markers). Skipped + * entirely for sites whose two snapshots define no reference (top-level, inside a repeater, or + * still converting from one), and at a release's pin time (`references: false`). + */ +import type { + ContentEntry, + FieldDefinition, + NodeRef, + Site, + Snapshot, +} from "@cloudhal/core-contracts"; +import { BundleBudgetError, type BundleReadOptions, bundleShardCount } from "../bundles.ts"; +import { nodeCacheTag } from "../cache-tags.ts"; +import { originalsOf } from "../fields/entry-gates.ts"; +import { resolveFields } from "../fields/resolve.ts"; +import { isReferenceFieldType } from "../fields/types.ts"; +import { memberGroup } from "./members.ts"; +import { referenceUses } from "./values.ts"; + +/** What the closure reads of a snapshot (the planner's `SnapshotReader`). */ +export interface ReferencePurgeReader { + readSnapshotBundle?( + snapshot: Snapshot, + options?: BundleReadOptions, + ): Promise | undefined>; +} + +export interface ReferencePurgeLimits { + /** Bytes of bundle bodies (default 32 MB). */ + readonly maxBodyBytes?: number | undefined; + /** Entry bodies across both snapshots, and nodes of the bundle read (default 50,000). */ + readonly maxNodes?: number | undefined; + /** Snapshots known to have no bundle (a release run could not write one): no shard reads. */ + readonly unbundled?: ReadonlySet | undefined; + /** + * `false`: skip the closure (a release's pin-time plan, which is the batch's expected outcome + * only; the run-time re-plan covers referrers, or purges everything). + */ + readonly references?: false | undefined; +} + +export const DEFAULT_PURGE_MAX_BODY_BYTES = 32 * 1024 * 1024; +export const DEFAULT_PURGE_MAX_NODES = 50_000; + +export interface ReferencePurgeInput { + readonly reader: ReferencePurgeReader; + readonly from: Snapshot; + readonly to: Snapshot; + /** The planner's counted, hash-keyed reader. */ + readonly read: (ref: NodeRef) => Promise; + /** A body the planner already holds, by hash (no read). */ + readonly cached: (hash: string) => unknown; + /** Counts `n` reads against the planner's `maxReads` (throws past it). */ + readonly spend: (n: number) => void; + /** Every changed, added or removed entry: before (in `from`) and after (in `to`). */ + readonly entries: readonly { before?: ContentEntry; after?: ContentEntry }[]; + /** Entries of `to` the asset rule reached (a shown asset of theirs changed). */ + readonly reached: ReadonlySet; + readonly tags: Set; + readonly why: string[]; + readonly limits?: ReferencePurgeLimits | undefined; +} + +/** What the closure keeps of an entry body. */ +type Slim = Pick & { + translation?: ContentEntry["translation"]; +}; + +const defsOf = (site: Site): FieldDefinition[] => + Object.values(site.collections ?? {}).flatMap((c) => c?.fields ?? []); + +/** + * A `reference` field, a repeater with a `reference` sub-field, or a field (or `sub`) still + * converting *from* references: during `change_field_type` reference → text its values render + * the targets' titles through the conversion maps until phase B rewrites them. + */ +const namesEntries = (d: FieldDefinition): boolean => + d.type === "reference" || + (d.type === "repeater" && d.fields.some((s) => s.type === "reference")) || + d.migration?.fromType === "reference"; + +/** True when some collection of the site holds or converts references (top-level or in items). */ +export function definesReferences(site: Pick): boolean { + return defsOf(site as Site).some(namesEntries); +} + +/** Any field-operation marker on the site node. */ +const hasMarker = (site: Site): boolean => defsOf(site).some((d) => d.migration !== undefined); + +/** Per collection, the field keys the closure keeps (references and repeaters of either site). */ +function keptKeys(sites: readonly Site[]): Map> { + const out = new Map>(); + for (const site of sites) + for (const [c, col] of Object.entries(site.collections ?? {})) { + const keys = out.get(c) ?? new Set(); + for (const d of col?.fields ?? []) if (isReferenceFieldType(d.type)) keys.add(d.key); + out.set(c, keys); + } + return out; +} + +function slimmer(keys: Map>): (body: unknown) => Slim { + return (body) => { + const e = body as ContentEntry; + const own = keys.get(e.collection); + const fields: Record = {}; + for (const [k, v] of Object.entries(e.fields ?? {})) if (own?.has(k)) fields[k] = v; + return { + id: e.id, + collection: e.collection, + locale: e.locale, + status: e.status, + fields, + ...(e.translation ? { translation: e.translation } : {}), + }; + }; +} + +/** One snapshot's referrer index: names by entry, referrers and members by group. */ +interface Index { + readonly named: Map>; + readonly referrers: Map>; + readonly members: Map>; + readonly groupOf: Map; +} + +function indexOf(site: Site, entries: readonly Slim[]): Index { + const index: Index = { + named: new Map(), + referrers: new Map(), + members: new Map(), + groupOf: new Map(), + }; + const add = (map: Map>, key: string, value: string) => { + const set = map.get(key) ?? new Set(); + set.add(value); + map.set(key, set); + }; + const originalOf = originalsOf(entries as ContentEntry[]); + for (const e of entries) { + const g = memberGroup(e); + index.groupOf.set(e.id, g); + add(index.members, g, e.id); + const names = new Set(); + const original = originalOf(e as ContentEntry); + for (const r of original ? [original, undefined] : [undefined]) + for (const f of Object.values(resolveFields(site, e, r))) + if (isReferenceFieldType(f.def.type)) + for (const u of referenceUses(f.def, f.value, "")) names.add(u.groupId); + index.named.set(e.id, names); + for (const n of names) add(index.referrers, n, e.id); + } + return index; +} + +/** Loads every entry body of both snapshots as `Slim`, by hash (see the module comment). */ +async function slimBodies( + input: ReferencePurgeInput, + slim: (body: unknown) => Slim, +): Promise> { + const { from, to, limits } = input; + const read = input.reader.readSnapshotBundle?.bind(input.reader); + if (!read) throw new Error("the snapshot reader cannot stream bodies"); + const maxNodes = limits?.maxNodes ?? DEFAULT_PURGE_MAX_NODES; + // One byte total over the bundle and the node reads: nothing is parsed past it. + const budget = { + maxBytes: limits?.maxBodyBytes ?? DEFAULT_PURGE_MAX_BODY_BYTES, + maxNodes, + spent: { bytes: 0 }, + keep: (body: unknown, ref: NodeRef) => (ref.kind === "contentEntry" ? slim(body) : undefined), + }; + const out = new Map(); + const bundleOf = async (snapshot: Snapshot) => { + if (limits?.unbundled?.has(snapshot.id)) return undefined; + input.spend(bundleShardCount(snapshot)); + const before = budget.spent.bytes; + const got = await read(snapshot, budget); + // A partial or mismatching bundle is discarded: only bodies kept count against the bytes. + if (!got) budget.spent.bytes = before; + return got; + }; + const bundle = (await bundleOf(to)) ?? (await bundleOf(from)); + for (const [hash, s] of bundle ?? []) out.set(hash, s as Slim); + // The rest: bodies the planner already holds, then the missing hashes, read by this closure + // alone (counted, budgeted, cut to `Slim`; never into the planner's cache of full bodies). + const missing = new Map(); + for (const snapshot of [from, to]) + for (const ref of snapshot.nodes) { + if (ref.kind !== "contentEntry" || out.has(ref.hash) || missing.has(ref.hash)) continue; + const held = input.cached(ref.hash); + if (held !== undefined) out.set(ref.hash, slim(held)); + else missing.set(ref.hash, ref); + } + if (out.size + missing.size > maxNodes) + throw new BundleBudgetError(`more than ${maxNodes} entries`); + if (missing.size > 0) { + input.spend(missing.size); + const got = await read(to, { ...budget, only: [...missing.values()] }); + for (const [hash, s] of got ?? []) out.set(hash, s as Slim); + } + return out; +} + +/** + * Adds the referrer closure's node tags to `input.tags`; returns a reason to purge everything + * instead, or undefined. + */ +export async function planReferencePurge(input: ReferencePurgeInput): Promise { + const { from, to, read, tags } = input; + if (input.limits?.references === false) return undefined; + // Nothing changed that a card shows: no entry and no shown asset (a term-only, asset-only or + // site-only publish). The closure could add no tag, so it reads nothing at all. + if (input.entries.length === 0 && input.reached.size === 0) return undefined; + const siteOf = (s: Snapshot) => { + const ref = s.nodes.find((n) => n.kind === "site" && n.id === s.siteId); + return ref ? read(ref) : Promise.resolve(undefined); + }; + const [a, b] = await Promise.all([siteOf(from), siteOf(to)]); + if (!a || !b) return "a snapshot has no site node"; + if (!definesReferences(a) && !definesReferences(b)) return undefined; + if (hasMarker(a) || hasMarker(b)) return "a field operation is running in a published snapshot"; + if (!input.reader.readSnapshotBundle) return "the snapshot reader cannot stream bodies"; + let bodies: Map; + try { + bodies = await slimBodies(input, slimmer(keptKeys([a, b]))); + } catch (e) { + if (e instanceof BundleBudgetError) return `the referrer closure is too large: ${e.message}`; + throw e; + } + const entriesOf = (s: Snapshot) => + s.nodes.flatMap((n) => (n.kind === "contentEntry" ? [bodies.get(n.hash) as Slim] : [])); + const indexes = [indexOf(a, entriesOf(from)), indexOf(b, entriesOf(to))]; + const before = input.entries.length; + const purge = (id: string) => tags.add(nodeCacheTag(id)); + const members = (g: string) => { + for (const ix of indexes) for (const id of ix.members.get(g) ?? []) purge(id); + }; + const referrersOf = (groups: Iterable) => { + const out = new Set(); + for (const g of groups) + for (const ix of indexes) for (const id of ix.referrers.get(g) ?? []) out.add(id); + return out; + }; + const groupsOf = (ids: Iterable) => { + const out = new Set(); + for (const id of ids) + for (const ix of indexes) { + const g = ix.groupOf.get(id); + if (g) out.add(g); + } + return out; + }; + // 1. seeds + const touched = new Set(input.reached); + for (const { before: x, after: y } of input.entries) + for (const e of [x, y]) if (e) touched.add(e.id); + const seeds = new Set(groupsOf(touched)); + for (const { before: x, after: y } of input.entries) + for (const e of [x, y]) if (e) seeds.add(memberGroup(e)); + for (const g of seeds) members(g); + // 2. two hops: referrers of the seeds, then of their groups; each with its group's members + const hop1 = referrersOf(seeds); + const hop2 = referrersOf(groupsOf(hop1)); + for (const g of groupsOf([...hop1, ...hop2])) members(g); + for (const id of [...hop1, ...hop2]) purge(id); + // 3. "used by": the members of every group a touched entry names, before or after + for (const id of touched) + for (const ix of indexes) for (const g of ix.named.get(id) ?? []) members(g); + input.why.push( + `references: ${hop1.size + hop2.size} referrers of ${seeds.size} groups (${before} entries changed)`, + ); + return undefined; +} diff --git a/packages/core-ts/src/references/release-purge.ts b/packages/core-ts/src/references/release-purge.ts new file mode 100644 index 00000000..7001d9ab --- /dev/null +++ b/packages/core-ts/src/references/release-purge.ts @@ -0,0 +1,178 @@ +/** + * RFC-0045 §2 "Purges", point 4: what a release run purges, decided after its batch committed. + * + * - **The gate is the published site node.** A release never carries the site node, so the new + * snapshot publishes the live one's. When the working site row the batch read is that node (a + * first publish), it is tested as is; otherwise the published body is read once. In practice + * that read happens except on a first publish: a snapshot's site node is captured before the + * publish flips the pointer, so the row differs from it at least in `publishedSnapshotId` and + * `rev`. The read is counted but never refused (one call per release, from the sizing's margin), + * so a site without references never depends on what other runs left; if it fails, the working + * row decides (no references: the stored plan; else everything). Only a node that defines + * references, or still converts from them, re-plans; every other site sends the plan stored at + * pin time, as before 3b (nothing when it purges nothing). + * - **Re-planned** `from → snapshotId` (`from`: the pointer the batch moved), like every other + * publish path. The pin-time plan skips the referrer closure and is the batch's *expected* + * outcome only. The re-plan's statements run on the tick's counting driver (never refused); if + * it throws, the run purges everything, never the stored plan. A re-plan that could not finish + * (more distinct terms in the two snapshots than reads left, the planner reading every term) + * purges everything before it reads anything. + * - **Blob calls are charged as they happen** against the invocation's allowance, shared by every + * tick of one invocation. The tick wraps the release graph's store with `chargedBlobs`, so the + * run's other calls (`refreshSearch`'s reads, `labelOf`, a first publish's head and put) count + * too, never refused; each re-plan then gets `maxReads = min(2,000, left)` of what is really + * left. Past it, the planner purges everything: an over-purge, never a stale page. The scheduled + * snapshot gets no bundle here (the site Worker writes one on its first read): back-to-back + * releases of one site with no visitor between them re-plan node by node, within the same + * allowance. + * + * **Sizing the allowance.** R2 and D1 calls are capped per invocation: 1,000 on Workers Free, by + * default 10,000 on Paid (RFC-0010, ADR-0024). An invocation runs up to two ticks (the cron's + * follow-up). Per tick: D1, at most `maxQueries` refused statements plus 7 counted ones per + * release (the re-plan's 2 `getSnapshot`, bookkeeping 4, `recordPurge` 1); the built-in jobs' 40 + * blob reads. Per release, counted from the allowance itself: `refreshSearch`'s worst case + * (`MAX_RELEASE_ITEMS` × 2 + the site + ~10 taxonomies ≈ 211), `labelOf` and the publish's own + * blobs (2), the gate read (1); and the purge's sends (≤ 4). + * - **Free** (40 statements, 1 release per tick): + * 1,000 − 2 × (40 + 7) − 2 × 40 − 2 × (211 + 2 + 4) = 392, so 350. + * - **Paid** (900 statements; about 27 releases fit a tick's D1, at 33 statements each with claim + * and finish): 27 releases of 100 items pass the default 10,000 with `refreshSearch` alone, + * re-plan or not (RFC-0008's budgets, unchanged here). Sized for ticks of up to 10 such + * releases: 10,000 − 2 × (900 + 10 × 7) − 2 × 40 − 2 × 10 × 217 = 3,640, so 3,000. + * + * Accepted over-purges (an over-purge, never a stale page; rare, and for a feature not enabled + * yet): a reference site with more terms than the allowance leaves purges everything on every + * release; on Free, a reference-site release that changes an existing asset reads every entry, + * page and term one by one in the asset rule, so above about 350 nodes it purges everything. The + * AI Search mirror's R2 calls (RFC-0029) are not counted. + */ +import type { NodeRef, Site } from "@cloudhal/core-contracts"; +import type { BlobStore, SqlDriver } from "@cloudhal/storage"; +import { + type CachePurgePlan, + DEFAULT_PURGE_MAX_READS, + planCachePurge, + type SnapshotReader, +} from "../cache-purge.ts"; +import type { ReleasePurge } from "../release-apply.ts"; +import type { ReleasePurger } from "../releases.ts"; +import { SiteGraphStore } from "../store.ts"; +import { definesReferences } from "./purge.ts"; + +/** Blob calls one invocation's release runs may make (see the module comment). */ +export const RELEASE_BLOB_ALLOWANCE = { free: 350, paid: 3000 } as const; + +/** The allowance for a tick budget: Paid's when it allows more than Free's 40 statements. */ +export function releaseBlobAllowance(budget?: { readonly maxQueries: number }): number { + return budget && budget.maxQueries > 40 + ? RELEASE_BLOB_ALLOWANCE.paid + : RELEASE_BLOB_ALLOWANCE.free; +} + +/** What a plan does to the outcome's `purge` (`none` where nothing is cached or nothing changed). */ +export function purgeOutcome(plan: CachePurgePlan, purging: boolean): ReleasePurge { + if (!purging) return "none"; + if (plan.mode === "everything") return "everything"; + return plan.tags.length > 0 ? "tags" : "none"; +} + +export interface ReleasePurgeInput { + /** The executor's graph (its blob store and driver are the planner's). */ + readonly graph: SiteGraphStore; + /** The tick's counting driver; absent ("Publish now"): the graph's own. */ + readonly counted?: SqlDriver | undefined; + readonly purge: ReleasePurger; + /** The plan stored at pin time. */ + readonly stored: CachePurgePlan; + /** The site row the batch wrote. */ + readonly site: Site; + /** The site node the new snapshot publishes, and whether the working row was it. */ + readonly publishedSite?: { readonly ref: NodeRef; readonly current: boolean } | undefined; + /** The pointer the batch moved (absent: nothing was live, so everything). */ + readonly from?: string | undefined; + readonly snapshotId: string; + /** Blob calls left for this invocation's release runs; charged by actual calls. */ + readonly allowance: { left: number }; + readonly log: (line: string) => void; +} + +const chargedStores = new WeakSet(); + +/** + * A blob store that charges every call to `allowance`, never refusing one: the tick wraps the + * release graph's store with it, so `refreshSearch`, `labelOf` and the publish's own blobs count + * too, and a re-plan's `maxReads` sees what is really left. + */ +export function chargedBlobs(blobs: BlobStore, allowance: { left: number }): BlobStore { + if (chargedStores.has(blobs)) return blobs; + const out = new Proxy(blobs, { + get(target, prop, receiver) { + const v = Reflect.get(target, prop, receiver); + if (typeof v !== "function") return v; + return (...args: unknown[]) => { + allowance.left--; + return v.apply(target, args); + }; + }, + }); + chargedStores.add(out); + return out; +} + +/** The plan to send: the stored one, a re-plan, or purge-everything; never throws. */ +async function planFor(input: ReleasePurgeInput): Promise { + const { site, allowance, log } = input; + const planner = new SiteGraphStore({ + sql: input.counted ?? input.graph.sql, + blobs: chargedBlobs(input.graph.blobs, allowance), + }); + const everything = (reason: string): CachePurgePlan => { + log(`schedules: ${reason}`); + return { mode: "everything", reason }; + }; + // The gate: counted, never refused (one call per release, from the sizing's margin), so a + // site without references never depends on what other runs left. + let node: Site = site; + const published = input.publishedSite; + if (published && !published.current) + try { + node = await planner.readSnapshotNode(published.ref); + } catch (e) { + // the working row decides: no references there, the stored plan; else everything + if (!definesReferences(site)) return input.stored; + return everything(`the published site node could not be read: ${message(e)}`); + } + if (!definesReferences(node)) return input.stored; + if (!input.from) return { mode: "everything", reason: "nothing was live before the release" }; + try { + const maxReads = Math.min(DEFAULT_PURGE_MAX_READS, allowance.left); + const [from, to] = await Promise.all([ + planner.getSnapshot(input.from), + planner.getSnapshot(input.snapshotId), + ]); + if (!from || !to) return everything("a release snapshot is missing"); + // A hopeless re-plan stops before it reads: the planner reads every term of both snapshots. + const terms = new Set( + [...from.nodes, ...to.nodes].filter((n) => n.kind === "term").map((n) => n.hash), + ); + if (terms.size >= maxReads) + return everything(`${terms.size} terms need more blob reads than are left (${maxReads})`); + const reader: SnapshotReader = { + getSnapshot: async (id) => (id === from.id ? from : id === to.id ? to : undefined), + readSnapshotNode: (ref: NodeRef) => planner.readSnapshotNode(ref), + readSnapshotBundle: (s, o) => planner.readSnapshotBundle(s, o), + }; + // `to` has no bundle: no shard reads are spent on it. + return await planCachePurge(reader, from.id, to.id, { maxReads, unbundled: new Set([to.id]) }); + } catch (e) { + return everything(`the run-time purge plan failed: ${message(e)}`); + } +} + +const message = (e: unknown) => (e instanceof Error ? e.message : String(e)); + +/** Sends the run's purge (see the module comment); a throw only from the purger itself. */ +export async function sendReleasePurge(input: ReleasePurgeInput): Promise { + const plan = await planFor(input); + return purgeOutcome(plan, true) === "none" ? "none" : input.purge(input.site.id, plan); +} diff --git a/packages/core-ts/src/release-apply.ts b/packages/core-ts/src/release-apply.ts index acb55a55..93d8b097 100644 --- a/packages/core-ts/src/release-apply.ts +++ b/packages/core-ts/src/release-apply.ts @@ -72,6 +72,11 @@ export type PublishReleaseResult = readonly baseSnapshotId: string; /** The pointer moved from (absent on a site never published before). */ readonly from?: string; + /** + * RFC-0045: the site node the new snapshot publishes (a release never carries the site + * node: the live one's), and whether the working site row read before the batch was it. + */ + readonly publishedSite?: { readonly ref: NodeRef; readonly current: boolean } | undefined; /** False when a recovered run found its snapshot already written (nothing new committed). */ readonly committed: boolean; } @@ -118,6 +123,12 @@ function approverOf(release: Release): Actor | undefined { return a.kind === "person" ? { kind: "human", ...a.by } : a.by; } +/** The site node a snapshot's `nodes` publish, and whether it is the working row (`hash`). */ +function publishedSite(nodes: readonly NodeRef[], hash: string) { + const ref = nodes.find((n) => n.kind === "site"); + return ref ? { ref, current: ref.hash === hash } : undefined; +} + /** The snapshot's label: the release title, else "Scheduled: ". */ async function labelOf( ctx: ReleaseApplyContext, @@ -194,7 +205,6 @@ export async function publishRelease( message: `the live snapshot ${site.publishedSnapshotId} cannot be read`, }; const parent = (await ctx.latestSnapshot(siteId)) ?? live; - // Conflicts, composition and dependencies (§6.3 steps 3–5), pure. const plan = planRelease(live?.nodes ?? [], items); if (plan.kind === "conflict") return plan; @@ -350,6 +360,7 @@ export async function publishRelease( snapshotId, baseSnapshotId: base, ...(live ? { from: live.id } : {}), + publishedSite: publishedSite(nodes, row.hash), committed: true, }; lastError = `the site ${siteId} or the run's lease changed while the release was applied`; diff --git a/packages/core-ts/src/release-executor.ts b/packages/core-ts/src/release-executor.ts index a8a99b75..710313e7 100644 --- a/packages/core-ts/src/release-executor.ts +++ b/packages/core-ts/src/release-executor.ts @@ -1,24 +1,42 @@ /** * The `ReleaseExecutor` seam of `runDue` (RFC-0008 §6.3, §8): `SiteGraphStore.publishRelease`, - * then the stored purge plan, sent after the batch, then the release-published hook + * then the purge, re-planned after the batch from the pointer it moved (RFC-0045 §2, + * `references/release-purge.ts`; the stored plan only fills the batch's outcome), then the + * release-published hook * (`release-after.ts`; the Node server rebuilds its static site, #150). `runScheduledTick` and * "Publish now" (`ReleaseService.publishNow`) build one; both live in `releases.ts`. */ import type { ReleaseExecution, ReleaseExecutor } from "@cloudhal/schedules"; import type { SqlScheduleStore } from "@cloudhal/schedules/sql"; +import type { SqlDriver } from "@cloudhal/storage"; import type { CachePurgePlan } from "./cache-purge.ts"; +import { + purgeOutcome, + releaseBlobAllowance, + sendReleasePurge, +} from "./references/release-purge.ts"; import * as after from "./release-after.ts"; import type { ReleasePurge } from "./release-apply.ts"; import type { ReleasePurger } from "./releases.ts"; import type { SiteGraphStore } from "./store.ts"; /** - * Statements one release execution takes, for `runDue`'s budget check: the stored purge plan (1), - * the site, the live snapshot and the parent (3), the eight-statement batch, and the purge's own - * bookkeeping after it (up to 4). A site that moved between the reads and the batch repeats the - * reads and the batch once; that rare path may end the tick early (`more`), never mid-batch. + * Statements one release execution takes, for `runDue`'s budget check. On the tick's refusing + * driver: the stored purge plan (1), the site, the live snapshot and the parent (3), the batch + * (up to 8, with asset blobs to root), and the search index brought to the new snapshot (7: its + * reads and one guarded batch, RFC-0019 §3). On its counting driver, never refused: for sites + * with references, the run-time re-plan's two `getSnapshot` reads (RFC-0045 §2); the purge's + * bookkeeping (up to 4); and `recordPurge`, correcting the outcome after a deferred or differing + * purge (1). 1 + 3 + 8 + 7 + 2 + 4 + 1 = 26. A site that moved between the reads and the batch + * repeats the reads and the batch once (11 more); that rare path may end the tick early + * (`more`), never mid-batch, and the purge after it runs on the counting driver regardless. + * + * Blob calls (R2 subrequests) are bounded apart, per invocation: every release run of every tick + * in it charges its re-plan's actual blob calls to one allowance (350 on Workers Free, 3,000 on + * Paid; the arithmetic is in `references/release-purge.ts`), and past it the run purges + * everything. */ -export const RELEASE_EXECUTOR_COST = 16; +export const RELEASE_EXECUTOR_COST = 26; export interface ReleaseExecutorOptions { readonly graph: SiteGraphStore; @@ -31,6 +49,16 @@ export interface ReleaseExecutorOptions { * gets its purge. */ readonly recordPurge?: ((releaseId: string, purge: ReleasePurge) => Promise) | undefined; + /** + * The tick's counting (never refusing) driver: the run-time purge plan reads over it, and, when + * `recordPurge` is absent, the outcome's correction is written over it. + */ + readonly counted?: SqlDriver | undefined; + /** + * Blob calls left for this invocation's release runs, shared by its ticks and charged by actual + * calls (default: Free's `releaseBlobAllowance`). + */ + readonly releaseBlobs?: { left: number } | undefined; /** After a committed release's purge (`release-after.ts`; the Node server rebuilds, #150). */ readonly onPublished?: after.ReleasePublishedHook | undefined; readonly log?: ((line: string) => void) | undefined; @@ -39,8 +67,14 @@ export interface ReleaseExecutorOptions { /** The `ReleaseExecutor` of `runDue`: `publishRelease`, then the stored purge plan. */ export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { const log = o.log ?? (() => {}); + const counted = o.counted; + // RFC-0045: the blob calls every release run of this invocation may make together. + const allowance = o.releaseBlobs ?? { left: releaseBlobAllowance() }; const recordPurge = - o.recordPurge ?? ((id: string, purge: ReleasePurge) => o.schedules.recordPurge(id, purge)); + o.recordPurge ?? + (counted + ? (id: string, purge: ReleasePurge) => after.recordPurgeOutcome(counted, id, purge) + : (id: string, purge: ReleasePurge) => o.schedules.recordPurge(id, purge)); return { cost: RELEASE_EXECUTOR_COST, async execute(input): Promise { @@ -48,13 +82,7 @@ export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { const plan: CachePurgePlan = after.isPurgePlan(stored) ? stored : { mode: "everything", reason: "no purge plan was stored with the release" }; - const expected: ReleasePurge = !o.purge - ? "none" - : plan.mode === "everything" - ? "everything" - : plan.tags.length > 0 - ? "tags" - : "none"; + const expected = purgeOutcome(plan, o.purge !== undefined); const r = await o.graph.publishRelease({ release: input.release, items: input.items, @@ -70,9 +98,13 @@ export function releaseExecutor(o: ReleaseExecutorOptions): ReleaseExecutor { switch (r.kind) { case "published": { let purge: ReleasePurge = expected; - if (r.committed && o.purge && expected !== "none") { + if (r.committed && o.purge) { try { - purge = await o.purge(input.release.siteId, plan); + purge = await sendReleasePurge({ + ...{ graph: o.graph, counted, purge: o.purge, stored: plan, allowance, log }, + ...{ site: r.site, publishedSite: r.publishedSite }, + ...{ from: r.from, snapshotId: r.snapshotId }, + }); } catch (e) { purge = "failed"; log( diff --git a/packages/core-ts/src/release-service-options.ts b/packages/core-ts/src/release-service-options.ts new file mode 100644 index 00000000..4d2e0fdb --- /dev/null +++ b/packages/core-ts/src/release-service-options.ts @@ -0,0 +1,48 @@ +/** + * The options and requests of `ReleaseService` (RFC-0008 §6, §12), split from `releases.ts`, which + * re-exports them. + */ +import type { Actor } from "@cloudhal/core-contracts"; +import type { WhenInput } from "@cloudhal/schedules"; +import type { SqlDriver } from "@cloudhal/storage"; +import type { ReleasePublishedHook } from "./release-after.ts"; +import type { ReleasePurger, SchedulerRuntime } from "./releases.ts"; +import type { SiteGraphStore } from "./store.ts"; + +export interface ReleaseServiceOptions { + readonly graph: SiteGraphStore; + /** The database of `graph` (default `graph.sql`). */ + readonly sql?: SqlDriver | undefined; + /** The clock for lead times and "Publish now" (default `Date.now`). */ + readonly now?: (() => Date) | undefined; + /** The ticker, poked when a release's time is set or moved (default: cron, no pokes). */ + readonly scheduler?: SchedulerRuntime | undefined; + /** "Publish now" purges like a tick does (absent: nothing cached). */ + readonly purge?: ReleasePurger | undefined; + /** New bodies one call may write (§6.1 step 4; Workers Free: 20). `Infinity` on Node. */ + readonly maxNewBodies?: number | undefined; + /** After "Publish now" commits a release, as a tick's `onReleasePublished` (the Node server rebuilds, #150). */ + readonly onReleasePublished?: ReleasePublishedHook | undefined; + readonly log?: ((line: string) => void) | undefined; + /** + * RFC-0045: the deployment's tick budget (`CLOUDHAL_TICK_BUDGET`), which sizes "Publish now"'s + * allowance of blob calls for its purge (default: Workers Free's). + */ + readonly tickBudget?: { readonly maxQueries: number } | undefined; +} + +export interface PinRequest { + /** Nodes whose current working-copy version goes live. */ + readonly set?: readonly string[] | undefined; + /** Entries, pages and redirects taken out of the live site. */ + readonly remove?: readonly string[] | undefined; +} + +export interface CreateReleaseRequest extends PinRequest { + readonly siteId: string; + readonly createdBy: Actor; + readonly title?: string | undefined; + /** An instant, a wall-clock time (in `timeZone`, else the site's zone), or `on-approval`. */ + readonly when?: WhenInput | undefined; + readonly metadata?: Record | undefined; +} diff --git a/packages/core-ts/src/releases.ts b/packages/core-ts/src/releases.ts index 731be999..946bef14 100644 --- a/packages/core-ts/src/releases.ts +++ b/packages/core-ts/src/releases.ts @@ -71,10 +71,16 @@ import { type BuiltinInputs, graphBuiltinRunner } from "./automations.ts"; import { type CachePurgePlan, planCachePurge } from "./cache-purge.ts"; import { NotFoundError } from "./errors.ts"; import type { NodeHold } from "./holds.ts"; -import * as after from "./release-after.ts"; +import { chargedBlobs, releaseBlobAllowance } from "./references/release-purge.ts"; +import type * as after from "./release-after.ts"; import type { ReleasePurge } from "./release-apply.ts"; import { releaseDepsOf } from "./release-deps.ts"; import { releaseExecutor } from "./release-executor.ts"; +import type { + CreateReleaseRequest, + PinRequest, + ReleaseServiceOptions, +} from "./release-service-options.ts"; import { SiteGraphStore, type SiteGraphStoreOptions } from "./store.ts"; export { type ReleasePublishedHook, recordPurgeOutcome } from "./release-after.ts"; @@ -161,6 +167,7 @@ export interface ScheduledTickOptions { readonly checkApproval?: | ((release: Release, now: Date) => ApprovalVerdict | Promise) | undefined; + readonly releaseBlobs?: { left: number } | undefined; // RFC-0045: one per invocation (R2 calls) /** Sends a published release's purge plan; it gets the counting driver for its bookkeeping. */ readonly purge?: ((ctx: { counted: SqlDriver }) => ReleasePurger) | undefined; /** `CLOUDHAL_AUTOMATIONS=off` → false. */ @@ -179,7 +186,10 @@ export async function runScheduledTick(o: ScheduledTickOptions): Promise after.recordPurgeOutcome(counted, id, p), + counted, + releaseBlobs, onPublished: o.onReleasePublished, log: o.log, }), @@ -265,38 +276,7 @@ export async function schedulerHealthReport( // ───────────────────────────── the service ───────────────────────────── -export interface ReleaseServiceOptions { - readonly graph: SiteGraphStore; - /** The database of `graph` (default `graph.sql`). */ - readonly sql?: SqlDriver | undefined; - /** The clock for lead times and "Publish now" (default `Date.now`). */ - readonly now?: (() => Date) | undefined; - /** The ticker, poked when a release's time is set or moved (default: cron, no pokes). */ - readonly scheduler?: SchedulerRuntime | undefined; - /** "Publish now" purges like a tick does (absent: nothing cached). */ - readonly purge?: ReleasePurger | undefined; - /** New bodies one call may write (§6.1 step 4; Workers Free: 20). `Infinity` on Node. */ - readonly maxNewBodies?: number | undefined; - /** After "Publish now" commits a release, as a tick's `onReleasePublished` (the Node server rebuilds, #150). */ - readonly onReleasePublished?: after.ReleasePublishedHook | undefined; - readonly log?: ((line: string) => void) | undefined; -} - -export interface PinRequest { - /** Nodes whose current working-copy version goes live. */ - readonly set?: readonly string[] | undefined; - /** Entries, pages and redirects taken out of the live site. */ - readonly remove?: readonly string[] | undefined; -} - -export interface CreateReleaseRequest extends PinRequest { - readonly siteId: string; - readonly createdBy: Actor; - readonly title?: string | undefined; - /** An instant, a wall-clock time (in `timeZone`, else the site's zone), or `on-approval`. */ - readonly when?: WhenInput | undefined; - readonly metadata?: Record | undefined; -} +export type { CreateReleaseRequest, PinRequest, ReleaseServiceOptions }; export interface PinnedRelease { readonly release: Release; @@ -328,6 +308,7 @@ export class ReleaseService { readonly #maxNewBodies: number; readonly #onReleasePublished: after.ReleasePublishedHook | undefined; readonly #log: (line: string) => void; + readonly #tickBudget: ReleaseServiceOptions["tickBudget"]; constructor(o: ReleaseServiceOptions) { this.graph = o.graph; @@ -339,6 +320,7 @@ export class ReleaseService { this.#maxNewBodies = o.maxNewBodies ?? MAX_NEW_BODIES_PER_REQUEST; this.#onReleasePublished = o.onReleasePublished; this.#log = o.log ?? (() => {}); + this.#tickBudget = o.tickBudget; } async #site(siteId: string) { @@ -509,17 +491,13 @@ export class ReleaseService { }; } - /** Step 5: `planCachePurge(L → L ⊕ items)` over a virtual snapshot (bodies are in the blob store). */ + /** Step 5: the expected purge `L → L ⊕ items`, over a virtual snapshot, closure aside (RFC-0045). */ async #purgePlan( live: Snapshot | undefined, items: readonly ReleaseItem[], ): Promise { if (!live) return { mode: "everything", reason: "nothing is live yet" }; - const target: Snapshot = { - ...live, - id: "snap_releaseplan", - nodes: composeRelease(live.nodes, items), - }; + const target: Snapshot = { ...live, id: "snap_pin", nodes: composeRelease(live.nodes, items) }; return planCachePurge( { getSnapshot: async (id) => (id === live.id ? live : id === target.id ? target : undefined), @@ -527,6 +505,7 @@ export class ReleaseService { }, live.id, target.id, + { references: false }, ); } @@ -830,6 +809,7 @@ export class ReleaseService { graph: this.graph, schedules: this.schedules, purge: this.#purge, + releaseBlobs: { left: releaseBlobAllowance(this.#tickBudget) }, onPublished: this.#onReleasePublished, log: this.#log, }), diff --git a/packages/core-ts/src/store.ts b/packages/core-ts/src/store.ts index 1f5b8b35..3709b029 100644 --- a/packages/core-ts/src/store.ts +++ b/packages/core-ts/src/store.ts @@ -36,7 +36,7 @@ import { } from "./assets.ts"; import { checkAvailability, type GateSupport, sameAvailability } from "./availability.ts"; import { type BlockRegistry, createBlockRegistry, type PropsSchema } from "./blocks.ts"; -import { readSnapshotBundle, writeSnapshotBundle } from "./bundles.ts"; +import { type BundleReadOptions, readSnapshotBundle, writeSnapshotBundle } from "./bundles.ts"; import { type CachePurgeOptions, type CachePurgePlan, planCachePurge } from "./cache-purge.ts"; import type { CollectionInfo } from "./collections.ts"; import { addedCorrections, type CorrectionInput } from "./corrections.ts"; @@ -945,9 +945,9 @@ export class SiteGraphStore { return this.#snapshots.create(siteId, input); } - /** The bodies of `snapshot` by node hash from its bundle, or undefined (RFC-0010). */ - readSnapshotBundle(snapshot: Snapshot): Promise | undefined> { - return readSnapshotBundle(this.#blobs, snapshot); + /** Bodies by node hash from `snapshot`'s bundle, or undefined (RFC-0010; budgets: RFC-0045). */ + readSnapshotBundle(snapshot: Snapshot, options?: BundleReadOptions) { + return readSnapshotBundle(this.#blobs, snapshot, options); } /** diff --git a/packages/core-ts/src/testing-releases.ts b/packages/core-ts/src/testing-releases.ts index b36ec656..216b3ca1 100644 --- a/packages/core-ts/src/testing-releases.ts +++ b/packages/core-ts/src/testing-releases.ts @@ -4,7 +4,14 @@ * purge plan, and the tick that runs them (`runScheduledTick`) — on SQLite and Miniflare D1. * Test names cite the invariants of RFC-0008 §17 they prove. */ -import type { Actor, ContentEntry, Release, Site, Snapshot } from "@cloudhal/core-contracts"; +import type { + Actor, + ContentEntry, + FieldDefinition, + Release, + Site, + Snapshot, +} from "@cloudhal/core-contracts"; import { isValid } from "@cloudhal/core-contracts"; import { TICK_BUDGETS } from "@cloudhal/schedules"; import type { BlobStore, SqlDriver } from "@cloudhal/storage"; @@ -12,6 +19,7 @@ import { dropAllTables } from "@cloudhal/storage/testing"; import { afterAll, beforeEach, describe, expect, it } from "vitest"; import type { CachePurgePlan } from "./cache-purge.ts"; import { ScheduledLockedError } from "./errors.ts"; +import { canonicalJson, nodeHash } from "./hash.ts"; import { type PersonApproval, ReleaseService, runScheduledTick } from "./releases.ts"; import { seedStarterSite } from "./seed.ts"; import { SiteGraphStore } from "./store.ts"; @@ -48,7 +56,9 @@ export function releasesSuite( await dropAllTables(sql); now = T0; logs.length = 0; - store = new SiteGraphStore({ sql, blobs, now: () => new Date(now) }); + // RFC-0045: references are enabled here (refused in production) for the purge re-plan case + const fieldTypes = { supported: () => true }; + store = new SiteGraphStore({ sql, blobs, now: () => new Date(now), fieldTypes }); await store.init(); site = await seedStarterSite(store, { name: "Releases", slug: "releases" }); news = (await store.getTermBySlug(site.id, "category", "news"))?.id as string; @@ -485,7 +495,262 @@ export function releasesSuite( expect((await get(release.id)).state).toBe("published"); }); - it("sends the purge plan stored at pin time and records a deferred purge", async () => { + /** RFC-0045: a `venue` reference from events to places (enabled on this suite's store). */ + async function venueSite() { + const s = await fresh(); + const venue: FieldDefinition = { + key: "venue", + type: "reference", + labels: { en: { label: "Venue" } }, + localized: false, + collections: ["places"], + }; + site = await store.updateSite({ + ...s, + collections: { ...s.collections, places: {}, events: { fields: [venue] } }, + }); + const hall = await story({ collection: "places", slug: "hall", title: "Hall" }); + const event = await story({ + collection: "events", + slug: "launch", + fields: { venue: hall.id }, + }); + await publishSite(); + return { hall, event }; + } + /** A blob store that counts gets by key. */ + const counting = () => { + const gets: string[] = []; + const wrapped: BlobStore = new Proxy(blobs, { + get(target, prop, receiver) { + if (prop === "get") + return (key: string) => { + gets.push(key); + return target.get(key); + }; + const v = Reflect.get(target, prop, receiver); + return typeof v === "function" ? v.bind(target) : v; + }, + }); + return { gets, blobs: wrapped }; + }; + const capture = + (plans: CachePurgePlan[], answer: "tags" | "everything" = "tags") => + () => + async (_siteId: string, plan: CachePurgePlan) => { + plans.push(plan); + return answer; + }; + + it("re-plans the purge at run time: a referrer published after the pin is purged (RFC-0045)", async () => { + const { hall } = await venueSite(); + // the release retitles the hall; a referrer goes live after the pin, by an ordinary publish + await store.updateContentEntry({ + ...((await store.getContentEntry(hall.id)) as ContentEntry), + title: "Main hall", + }); + await scheduled([hall.id]); + const late = await story({ collection: "events", slug: "late", fields: { venue: hall.id } }); + await publishSite(); + const plans: CachePurgePlan[] = []; + expect((await tick(T, { purge: capture(plans) })).ran).toBe(1); + const tags = plans[0]?.mode === "tags" ? plans[0].tags : []; + expect(tags).toContain(`cloudhal-node-${hall.id}`); + expect(tags).toContain(`cloudhal-node-${late.id}`); + }); + + it("a site without references sends the pin-time plan, reading only the published site node", async () => { + // the same release twice: without a purger (Node: the executor sends nothing), then with + const first = await story(); + await scheduled([first.id]); + const quiet = counting(); + expect((await tick(T, { blobs: quiet.blobs })).ran).toBe(1); + const second = await story({ slug: "second-story", title: "Second story" }); + const { release } = await scheduled([second.id], { at: T + HOUR }); + const stored = await svc.schedules.releasePurgePlan(release.id); + const c = counting(); + const plans: CachePurgePlan[] = []; + expect((await tick(T + HOUR, { blobs: c.blobs, purge: capture(plans) })).ran).toBe(1); + expect(plans).toEqual([stored]); + expect(c.gets.filter((k) => k.startsWith("bundles/"))).toEqual([]); + // one read more than a run that purges nothing: the published site node, for the gate + expect(c.gets).toHaveLength(quiet.gets.length + 1); + }); + + it("a run-time re-plan that fails still purges everything, never drops the purge", async () => { + const { hall } = await venueSite(); + await store.updateContentEntry({ + ...((await store.getContentEntry(hall.id)) as ContentEntry), + title: "Main hall", + }); + await scheduled([hall.id]); + // after the release's batch, every snapshot read fails (a transient D1 error) + let committed = false; + const failing: SqlDriver = { + ...sql, + get: (q, p) => { + if (committed && /FROM snapshots/.test(q)) throw new Error("D1_ERROR: connection lost"); + return sql.get(q, p); + }, + all: (q, p) => sql.all(q, p), + run: (q, p) => sql.run(q, p), + exec: (q) => sql.exec(q), + close: () => sql.close(), + batch: async (statements) => { + const out = await sql.batch(statements); + if (statements.some((st) => /INSERT INTO snapshots/.test(st.sql))) committed = true; + return out; + }, + }; + const plans: CachePurgePlan[] = []; + const report = await tick(T, { sql: failing, purge: capture(plans, "everything") }); + expect(report.ran).toBe(1); + expect(committed).toBe(true); + expect(plans.map((p) => p.mode)).toEqual(["everything"]); + const r = (await svc.list(site.id))[0] as Release; + expect(r.outcome).toMatchObject({ kind: "published", purge: "everything" }); + }); + + const retitle = async (id: string, title: string) => + store.updateContentEntry({ ...((await store.getContentEntry(id)) as ContentEntry), title }); + const tagsOf = (p: CachePurgePlan | undefined) => (p?.mode === "tags" ? p.tags : []); + /** Writes the working site row directly (a definition change the store would refuse). */ + async function rawSite(change: (s: Site) => Site) { + const s = await fresh(); + const next = { ...change(s), rev: s.rev + 1 }; + await sql.run("UPDATE nodes SET rev = ?, hash = ?, body = ? WHERE id = ?", [ + next.rev, + await nodeHash(next), + canonicalJson(next), + s.id, + ]); + } + + it("back-to-back releases on a reference site: a purge each, within the allowance", async () => { + const { hall, event } = await venueSite(); + const releaseBlobs = { left: 350 }; + const plans: CachePurgePlan[] = []; + await retitle(hall.id, "Main hall"); + await scheduled([hall.id]); + await tick(T, { purge: capture(plans), releaseBlobs }); + // the second run's `from` is the first's scheduled snapshot: no bundle, nobody read it + await retitle(hall.id, "Great hall"); + await scheduled([hall.id], { at: T + HOUR }); + await tick(T + HOUR, { purge: capture(plans), releaseBlobs }); + expect(plans).toHaveLength(2); + for (const p of plans) { + expect(["tags", "everything"]).toContain(p.mode); + if (p.mode === "tags") expect(p.tags).toContain(`cloudhal-node-${event.id}`); + } + expect(releaseBlobs.left).toBeGreaterThanOrEqual(0); + expect(releaseBlobs.left).toBeLessThan(350); + }); + + it("gates on the published site node: a draft that drops the last reference field still re-plans", async () => { + const { hall, event } = await venueSite(); + // the working copy drops `events.venue`, unpublished + await rawSite((s) => ({ + ...s, + collections: { ...s.collections, events: {} }, + })); + await retitle(hall.id, "Main hall"); + await scheduled([hall.id]); + const plans: CachePurgePlan[] = []; + await tick(T, { purge: capture(plans) }); + expect(tagsOf(plans[0])).toContain(`cloudhal-node-${event.id}`); + }); + + it("gates on the published site node: a draft that adds a first reference field sends the stored plan", async () => { + const first = await story(); + await scheduled([first.id]); + const quiet = counting(); + await tick(T, { blobs: quiet.blobs }); + // the working copy gains a reference field, unpublished + const s = await fresh(); + const venue: FieldDefinition = { + key: "venue", + type: "reference", + labels: { en: { label: "Venue" } }, + localized: false, + collections: ["posts"], + }; + site = await store.updateSite({ + ...s, + collections: { ...s.collections, events: { fields: [venue] } }, + }); + const second = await story({ slug: "second-story", title: "Second story" }); + const { release } = await scheduled([second.id], { at: T + HOUR }); + const stored = await svc.schedules.releasePurgePlan(release.id); + const c = counting(); + const plans: CachePurgePlan[] = []; + await tick(T + HOUR, { blobs: c.blobs, purge: capture(plans) }); + expect(plans).toEqual([stored]); + // one read more than a run that purges nothing: the published site node, for the gate + expect(c.gets).toHaveLength(quiet.gets.length + 1); + }); + + it("Paid: a tick of 13 reference-site releases purges each by its tags", async () => { + const { hall, event } = await venueSite(); + const ids: string[] = [hall.id]; + for (let i = 0; i < 12; i++) + ids.push( + (await story({ collection: "places", slug: `place-${i}`, title: `Place ${i}` })).id, + ); + await publishSite(); + for (const [i, id] of ids.entries()) { + await retitle(id, `Retitled ${i}`); + await scheduled([id]); + } + const plans: CachePurgePlan[] = []; + const report = await tick(T, { purge: capture(plans), budget: TICK_BUDGETS.paid }); + expect(report.ran).toBe(13); + expect(plans.map((p) => p.mode)).toEqual(Array(13).fill("tags")); + expect(plans.some((p) => tagsOf(p).includes(`cloudhal-node-${event.id}`))).toBe(true); + }); + + it("Paid: a reference site with more than 250 terms gets tags for a one-entry release", async () => { + const { hall, event } = await venueSite(); + for (let i = 0; i < 260; i++) + await store.createTerm({ + siteId: site.id, + taxonomyId: category, + slug: `topic-${i}`, + status: "active", + visibility: "public", + labels: { en: { name: `Topic ${i}` } }, + }); + await publishSite(); + await retitle(hall.id, "Main hall"); + await scheduled([hall.id]); + const plans: CachePurgePlan[] = []; + await tick(T, { purge: capture(plans), budget: TICK_BUDGETS.paid }); + expect(tagsOf(plans[0])).toContain(`cloudhal-node-${event.id}`); + }); + + it("two ticks of one invocation share the allowance, charged by actual calls", async () => { + const { hall } = await venueSite(); + const releaseBlobs = { left: 1000 }; + await retitle(hall.id, "Main hall"); + await scheduled([hall.id]); + await tick(T, { purge: capture([]), releaseBlobs }); + const afterFirst = releaseBlobs.left; + expect(afterFirst).toBeLessThan(1000); + await retitle(hall.id, "Great hall"); + await scheduled([hall.id], { at: T + HOUR }); + await tick(T + HOUR, { purge: capture([]), releaseBlobs }); + expect(releaseBlobs.left).toBeLessThan(afterFirst); + // spent: the next run purges everything without reading + releaseBlobs.left = 0; + await retitle(hall.id, "Grand hall"); + await scheduled([hall.id], { at: T + 2 * HOUR }); + const plans: CachePurgePlan[] = []; + await tick(T + 2 * HOUR, { purge: capture(plans), releaseBlobs }); + expect(plans.map((p) => p.mode)).toEqual(["everything"]); + // the run's own calls (search, label, the gate) are counted, never refused: below 0 + expect(releaseBlobs.left).toBeLessThan(0); + }); + + it("sends the purge re-planned at run time and records a deferred purge", async () => { const s = await story(); await scheduled([s.id]); const plans: CachePurgePlan[] = []; diff --git a/packages/core-ts/test/references-purge.test.ts b/packages/core-ts/test/references-purge.test.ts new file mode 100644 index 00000000..4167c52d --- /dev/null +++ b/packages/core-ts/test/references-purge.test.ts @@ -0,0 +1,535 @@ +/** + * RFC-0045 step 3b (§2 "Purges", Test plan "Renderer"): the purge planner's referrer closure. + * Pages carry only the node tags they already carry, so a theme that emits no new tags is purged: + * every case asserts `cloudhal-node-` tags of referrers, "used by" targets and their language + * versions. The closure reads only the `from`/`to` snapshots' bodies (bundles first), never the + * working copy, so edits after a snapshot never hide a referrer. The types stay refused in + * production; this store enables them (`fieldTypes`), as the shared suites do. + */ +import type { + ContentEntry, + FieldDefinition, + NodeRef, + Site, + Snapshot, + Term, +} from "@cloudhal/core-contracts"; +import { MemoryBlobStore } from "@cloudhal/storage"; +import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; +import { beforeEach, describe, expect, it } from "vitest"; +import { GATES_ENFORCED } from "../src/availability.ts"; +import { type BundleReadOptions, bundleKey, bundleShardCount } from "../src/bundles.ts"; +import { type CachePurgeOptions, planCachePurge, type SnapshotReader } from "../src/cache-purge.ts"; +import { nodeCacheTag, siteCacheTag } from "../src/cache-tags.ts"; +import { continueFieldOperation, runFieldOperation } from "../src/fields-operations/index.ts"; +import { canonicalJson, nodeHash } from "../src/hash.ts"; +import { PNG_1X1 } from "../src/media/fixtures.ts"; +import { sendReleasePurge } from "../src/references/release-purge.ts"; +import { seedStarterSite } from "../src/seed.ts"; +import { type PublishChange, SiteGraphStore } from "../src/store.ts"; + +const human = { kind: "human" as const, id: "user_refpurge001", name: "Editor" }; +const L = (label: string) => ({ en: { label } }); +const VENUE: FieldDefinition = { + key: "venue", + type: "reference", + labels: L("Venue"), + localized: false, + collections: ["places", "events"], +}; +const RELATED: FieldDefinition = { + key: "related", + type: "reference", + labels: L("Related"), + localized: false, + multiple: true, + collections: ["events"], +}; +const NOTES: FieldDefinition = { key: "notes", type: "text", labels: L("Notes"), localized: true }; + +describe("planCachePurge: the referrer closure (RFC-0045 §2)", () => { + let store: SiteGraphStore; + let site: Site; + let terms: ContentEntry["terms"]; + const changes: PublishChange[] = []; + /** What the planner read: node reads, bundle reads. */ + const counts = { nodes: 0, bundles: 0 }; + const reader: SnapshotReader = { + getSnapshot: (id) => store.getSnapshot(id), + readSnapshotNode: (ref: NodeRef) => { + counts.nodes++; + return store.readSnapshotNode(ref); + }, + readSnapshotBundle: (s: Snapshot, o?: BundleReadOptions) => { + counts.bundles++; + return store.readSnapshotBundle(s, o); + }, + }; + + const fresh = async () => (await store.getSite(site.id)) as Site; + const get = async (e: ContentEntry) => (await store.getContentEntry(e.id)) as ContentEntry; + const snap = (reason: "publish" | "manual") => + store.createSnapshot(site.id, { actor: human, reason }); + /** Moves the pointer to `id` (a publish, an approved request, a restore). */ + const point = async (id: string): Promise => { + site = await store.publish(await fresh(), id); + return changes.at(-1) as PublishChange; + }; + const publish = async () => point((await snap("publish")).id); + const plan = (change: PublishChange, o: CachePurgeOptions = {}) => { + counts.nodes = 0; + counts.bundles = 0; + return planCachePurge(reader, change.from, change.to, o); + }; + const tagsOf = async (change: PublishChange, o: CachePurgeOptions = {}) => { + const p = await plan(change, o); + if (p.mode !== "tags") throw new Error(`expected tags, got everything: ${p.reason}`); + return new Set(p.tags); + }; + const node = (e: { id: string }) => nodeCacheTag(e.id); + const make = (collection: string, slug: string, over: Partial = {}) => + store.createContentEntry({ + siteId: site.id, + collection, + slug, + locale: "en", + status: "published", + title: slug, + body: { format: "markdown", value: `About ${slug}.` }, + fields: {}, + tags: [], + ...(terms ? { terms } : {}), + author: human, + ...over, + } as never) as Promise; + const translate = async (of: ContentEntry, slug: string, over: Partial = {}) => + make(of.collection, slug, { + locale: "tr", + translation: { + group: of.id, + method: "human", + source: { id: of.id, locale: of.locale, rev: of.rev, hash: await nodeHash(of) }, + }, + ...over, + }); + const edit = async (e: ContentEntry, over: Partial) => + store.updateContentEntry({ ...(await get(e)), ...over }); + + beforeEach(async () => { + changes.length = 0; + store = new SiteGraphStore({ + sql: await BetterSqlite3Driver.open(":memory:"), + blobs: new MemoryBlobStore(), + gates: GATES_ENFORCED, + fieldTypes: { supported: () => true }, + onPublished: async (_site, change) => { + changes.push(change); + }, + }); + await store.init(); + site = await seedStarterSite(store, { name: "Refs", slug: "refs" }); + site = await store.updateSite({ + ...site, + locales: ["en", "tr"], + collections: { + ...site.collections, + places: {}, + events: { fields: [VENUE, RELATED, NOTES] }, + }, + }); + terms = (await store.listContentEntries(site.id, "posts"))[0]?.terms; + await publish(); + }); + + /** hall, an event naming it, and its Turkish translation (inheriting the venue). */ + async function world() { + const hall = await make("places", "hall"); + const event = await make("events", "launch", { fields: { venue: hall.id } }); + const eventTr = await translate(event, "lansman"); + const other = await make("events", "other"); + await publish(); + return { hall, event, eventTr, other }; + } + + it("a target retitled, unpublished or newly published purges its referrers' pages", async () => { + const { hall, event, eventTr, other } = await world(); + await edit(hall, { title: "Main hall" }); + const retitled = await tagsOf(await publish()); + for (const e of [hall, event, eventTr]) expect(retitled.has(node(e))).toBe(true); + expect(retitled.has(node(other))).toBe(false); + expect(retitled.has(siteCacheTag(site.id))).toBe(false); + await edit(hall, { status: "draft" }); + const unpublished = await tagsOf(await publish()); + for (const e of [hall, event, eventTr]) expect(unpublished.has(node(e))).toBe(true); + await edit(hall, { status: "published" }); + const republished = await tagsOf(await publish()); + for (const e of [hall, event, eventTr]) expect(republished.has(node(e))).toBe(true); + }); + + it("a target newly translated, or with a changed cover, purges its referrers", async () => { + const { hall, event, eventTr } = await world(); + await translate(hall, "salon"); + const translated = await tagsOf(await publish()); + for (const e of [event, eventTr]) expect(translated.has(node(e))).toBe(true); + const { asset } = await store.ingestAsset({ + siteId: site.id, + bytes: PNG_1X1, + filename: "hall.png", + uploadedBy: human, + source: { kind: "upload" }, + fields: { decorative: true }, + }); + await edit(hall, { coverImage: { assetId: asset.id, alt: "" } as never }); + await publish(); + // only the asset changes: the asset rule reaches hall, the closure its referrers + await store.updateAsset({ ...asset, slug: "hall-renamed" }); + const cover = await tagsOf(await publish()); + for (const e of [hall, event, eventTr]) expect(cover.has(node(e))).toBe(true); + }); + + it("depth 2: A → B → C, with C unpublished, purges A", async () => { + const c = await make("events", "c"); + const b = await make("events", "b", { fields: { related: [c.id] } }); + const a = await make("events", "a", { fields: { related: [b.id] } }); + const far = await make("events", "far", { fields: { related: [a.id] } }); + await publish(); + await edit(c, { status: "draft" }); + const tags = await tagsOf(await publish()); + for (const e of [c, b, a]) expect(tags.has(node(e)), e.slug).toBe(true); + // three hops away: its page shows a's card, not a's fields + expect(tags.has(node(far))).toBe(false); + }); + + it('a "used by" page follows referrers that appear or go, a translation that only inherits too', async () => { + const { hall, event } = await world(); + const salon = await translate(hall, "salon"); + await publish(); + const gig = await make("events", "gig", { fields: { venue: hall.id } }); + const appeared = await tagsOf(await publish()); + for (const e of [hall, salon]) expect(appeared.has(node(e))).toBe(true); + // a translation that only inherits the venue appears on its own + const gigTr = await translate(gig, "konser"); + const inherited = await tagsOf(await publish()); + expect(inherited.has(node(gigTr))).toBe(true); + for (const e of [hall, salon]) expect(inherited.has(node(e))).toBe(true); + await store.deleteContentEntry(await get(gigTr)); + await store.deleteContentEntry(await get(gig)); + const gone = await tagsOf(await publish()); + for (const e of [hall, salon]) expect(gone.has(node(e))).toBe(true); + // editing a referrer away from its venue purges the venue it left + await edit(event, { fields: {} }); + const left = await tagsOf(await publish()); + for (const e of [hall, salon]) expect(left.has(node(e))).toBe(true); + }); + + it("a rollback whose referrer was since edited to draft still purges the referrer", async () => { + const { hall, event } = await world(); + const before = site.publishedSnapshotId as string; + await edit(hall, { title: "Main hall" }); + await publish(); + await edit(event, { status: "draft" }); + const { site: back } = await store.rollback(await fresh(), before, human); + site = back; + const tags = await tagsOf(changes.at(-1) as PublishChange); + expect(tags.has(node(event))).toBe(true); + }); + + it("an approved request publishing an older snapshot, after the referrer was edited and the target unpublished, purges the referrer", async () => { + const { hall, event, eventTr } = await world(); + // the request's snapshot: hall unpublished, the event still naming it + await edit(hall, { status: "draft" }); + const request = await snap("manual"); + // meanwhile the referrer stops naming the target (only in the working copy) + await edit(event, { fields: {} }); + const tags = await tagsOf(await point(request.id)); + for (const e of [hall, event, eventTr]) expect(tags.has(node(e))).toBe(true); + }); + + it("a referrer edited away or deleted between snapshot and purge is still purged from to's bodies", async () => { + const { hall, event, eventTr } = await world(); + await edit(hall, { title: "Main hall" }); + const s = await snap("publish"); + await edit(event, { fields: {} }); + await store.deleteContentEntry(await get(eventTr)); + const tags = await tagsOf(await point(s.id)); + for (const e of [event, eventTr]) expect(tags.has(node(e))).toBe(true); + // and deleted outright + await edit(hall, { title: "Great hall" }); + const s2 = await snap("publish"); + await store.deleteContentEntry(await get(event)); + expect((await tagsOf(await point(s2.id))).has(node(event))).toBe(true); + }); + + it("a translation in to that only inherits, deleted since, is still purged", async () => { + const { hall, eventTr } = await world(); + await edit(hall, { title: "Main hall" }); + const s = await snap("publish"); + await store.deleteContentEntry(await get(eventTr)); + expect((await tagsOf(await point(s.id))).has(node(eventTr))).toBe(true); + }); + + it("a snapshot without a bundle costs one bundle plus its items, never purge-everything", async () => { + const { hall, event } = await world(); + for (let i = 0; i < 30; i++) await make("events", `filler-${i}`); + await publish(); + // a release run's `scheduled` snapshot and an approval's manual one have no bundle + await edit(hall, { title: "Main hall" }); + const manual = await snap("manual"); + const tags = await tagsOf(await point(manual.id), { maxReads: 20 }); + expect(tags.has(node(event))).toBe(true); + // `to` has none, `from` has one: two bundle attempts, and node reads only for the diff + expect(counts.bundles).toBe(2); + expect(counts.nodes).toBeLessThan(10); + }); + + it("a site without reference definitions reads no bundle at all", async () => { + const plain = await seedStarterSite(store, { name: "Plain", slug: "plain" }); + const s1 = await store.createSnapshot(plain.id, { actor: human, reason: "publish" }); + let p = await store.publish(plain, s1.id); + const post = (await store.listContentEntries(plain.id, "posts"))[0] as ContentEntry; + await store.updateContentEntry({ ...post, title: "Retitled" }); + const s2 = await store.createSnapshot(plain.id, { actor: human, reason: "publish" }); + p = await store.publish((await store.getSite(plain.id)) as Site, s2.id); + expect(p.publishedSnapshotId).toBe(s2.id); + const out = await plan({ from: s1.id, to: s2.id }); + expect(out.mode).toBe("tags"); + expect(counts.bundles).toBe(0); + }); + + it("the byte, node, read and tag budgets fall back to purge-everything", async () => { + const { hall } = await world(); + await edit(hall, { title: "Main hall" }); + const change = await publish(); + const reason = async (o: CachePurgeOptions) => { + const p = await plan(change, o); + return p.mode === "everything" ? p.reason : "tags"; + }; + expect(await reason({ maxBodyBytes: 10 })).toMatch(/too large/); + expect(await reason({ maxNodes: 2 })).toMatch(/too large/); + expect(await reason({ maxTags: 1 })).toMatch(/tags/); + expect(await reason({})).toBe("tags"); + // without bundles every entry body is a node read + await edit(hall, { title: "Great hall" }); + const manual = await snap("manual"); + await edit(hall, { title: "Grand hall" }); + const manual2 = await snap("manual"); + const p = await plan({ from: manual.id, to: manual2.id }, { maxReads: 3 }); + expect(p).toMatchObject({ mode: "everything" }); + expect(p.mode === "everything" ? p.reason : "").toMatch(/node reads/); + }); + + it("any field-operation marker on from or to purges everything", async () => { + const { hall, event } = await world(); + await edit(event, { fields: { venue: hall.id, notes: "Doors at 7" } }); + await make("events", "second", { fields: { notes: "Late" } }); + await publish(); + // a rename of a text field, stopped by its budget: the marker stays on the site node + const part = await runFieldOperation( + store, + site.id, + { op: "rename_field", collection: "events", from: "notes", to: "remarks" }, + { actor: human, budget: { entries: 1 } }, + ); + expect(part.done).toBe(false); + // purging everything: the whole site's tag, or mode `everything` + const all = (p: Awaited>) => + p.mode === "everything" || (p.tags.length === 1 && p.tags[0] === siteCacheTag(site.id)); + await edit(hall, { title: "Main hall" }); + // the marker's own publish changed the site node: the site tag already + expect(all(await plan(await publish()))).toBe(true); + // the marker on both snapshots, the site node unchanged: the closure purges everything + await edit(hall, { title: "Great hall" }); + const onBoth = await plan(await publish()); + expect(onBoth.mode === "everything" ? onBoth.reason : "").toMatch(/field operation/); + // on `from` only: the operation finished (the site node changed again) + const done = await continueFieldOperation(store, site.id, part.operationId, { actor: human }); + expect(done.done).toBe(true); + expect(all(await plan(await publish()))).toBe(true); + }); + + it("a publish that changes no entry and no shown asset reads nothing: a term's tags only", async () => { + await world(); + const news = (await store.getTermBySlug(site.id, "category", "news")) as Term; + await store.updateTerm({ ...news, labels: { en: { name: "Latest news" } } }); + const p = await plan(await publish()); + expect(p.mode).toBe("tags"); + expect(p.mode === "tags" ? p.tags : []).toContain(`cloudhal-term-${news.id}`); + expect(counts.bundles).toBe(0); + }); + + it("without bundles the closure's own reads keep to the byte budget", async () => { + const { hall } = await world(); + await edit(hall, { title: "Great hall" }); + const a = await snap("manual"); + await edit(hall, { title: "Grand hall" }); + const b = await snap("manual"); + const p = await plan({ from: a.id, to: b.id }, { maxBodyBytes: 300 }); + expect(p.mode === "everything" ? p.reason : "tags").toMatch(/too large/); + expect((await plan({ from: a.id, to: b.id })).mode).toBe("tags"); + }); + + it("the closure's own reads count against maxReads (not just the planner's)", async () => { + const { hall, event } = await world(); + for (let i = 0; i < 20; i++) await make("events", `filler-${i}`); + await publish(); + // a bundled `from` and a manual `to`: the closure spends shard reads only (the changed + // entry's bodies are the planner's, every other one is in `from`'s bundle) + const from = (await fresh()).publishedSnapshotId as string; + await edit(hall, { title: "Great hall" }); + const to = await snap("manual"); + const change = { from, to: to.id }; + await plan(change, { references: false }); + const own = counts.nodes; + const snapshots = await Promise.all([store.getSnapshot(from), store.getSnapshot(to.id)]); + const shards = snapshots.reduce((n, s) => n + bundleShardCount(s as Snapshot), 0); + expect((await tagsOf(change, { maxReads: own + shards })).has(node(event))).toBe(true); + expect(counts.nodes).toBe(own); + const tight = await plan(change, { maxReads: own + shards - 1 }); + expect(tight.mode === "everything" ? tight.reason : "tags").toMatch(/node reads/); + }); + + it("a discarded partial bundle's bytes do not count against the byte budget", async () => { + const { hall, event } = await world(); + const from = (await fresh()).publishedSnapshotId as string; + const fromSnap = (await store.getSnapshot(from)) as Snapshot; + const fromBytes = (await store.blobs.get(bundleKey(site.id, from, 0)))?.byteLength ?? 0; + await edit(hall, { title: "Great hall" }); + const to = await snap("publish"); + // `to`'s shard is replaced by a mismatching one: within the budget alone, past it together + // with `from`'s, read and then discarded + const junk = JSON.stringify({ + v: 1, + snapshotId: "snap_other0001", + pad: "x".repeat(Math.floor(fromBytes * 0.6)), + }); + await store.blobs.put(bundleKey(site.id, to.id, 0), new TextEncoder().encode(junk)); + expect(bundleShardCount(fromSnap)).toBe(1); + const tags = await tagsOf(await point(to.id), { maxBodyBytes: Math.floor(fromBytes * 1.4) }); + expect(tags.has(node(event))).toBe(true); + }); + + it("spends no shard reads on a snapshot known to have no bundle", async () => { + const { hall } = await world(); + await edit(hall, { title: "Main hall" }); + const s = await snap("manual"); + await tagsOf(await point(s.id), { unbundled: new Set([s.id]) }); + expect(counts.bundles).toBe(1); + }); + + it("a conversion from references counts as references: its marker purges everything", async () => { + const { hall } = await world(); + // `change_field_type` reference → text, phase A: the definition is text with the marker, and + // no reference field is left (written directly: the operation tools need rollout step 4) + const s = await fresh(); + const events = s.collections?.events; + const fields = (events?.fields ?? []).flatMap((d): FieldDefinition[] => + d.key === "venue" + ? [ + { + key: "venue", + type: "text", + labels: L("Venue"), + localized: false, + migration: { id: "fop_01HZX8K3Q9CD", fromType: "reference" }, + }, + ] + : d.type === "reference" + ? [] + : [d], + ); + const next = { + ...s, + rev: s.rev + 1, + collections: { ...s.collections, events: { ...events, fields } }, + } as unknown as Site; + await store.sql.run("UPDATE nodes SET rev = ?, hash = ?, body = ? WHERE id = ?", [ + next.rev, + await nodeHash(next), + canonicalJson(next), + s.id, + ]); + await publish(); + // a retitled target, both snapshots converting: its referrers' pages show its title + await edit(hall, { title: "Main hall" }); + const p = await plan(await publish()); + expect(p.mode === "everything" ? p.reason : "tags").toMatch(/field operation/); + }); + + describe("a release run's purge (sendReleasePurge)", () => { + const stored = { mode: "tags", tags: ["cloudhal-node-entry_STORED0001"], why: [] } as const; + const siteRefOf = (s: Snapshot) => s.nodes.find((n) => n.kind === "site") as NodeRef; + const run = async (o: { + graph?: SiteGraphStore; + siteId?: string; + left: number; + from: string; + to: string; + }) => { + const graph = o.graph ?? store; + const sent: unknown[] = []; + const allowance = { left: o.left }; + const toSnap = (await graph.getSnapshot(o.to)) as Snapshot; + const outcome = await sendReleasePurge({ + graph, + purge: async (_siteId, plan) => { + sent.push(plan); + return plan.mode === "everything" ? "everything" : "tags"; + }, + stored, + site: (await graph.getSite(o.siteId ?? site.id)) as Site, + publishedSite: { ref: siteRefOf(toSnap), current: false }, + from: o.from, + snapshotId: o.to, + allowance, + log: () => {}, + }); + return { sent, allowance, outcome }; + }; + + it("a site without references sends the stored plan with the allowance spent, or the gate read failing", async () => { + const plain = await seedStarterSite(store, { name: "Plain", slug: "plain-release" }); + const a = await store.createSnapshot(plain.id, { actor: human, reason: "publish" }); + await store.publish(plain, a.id); + const b = await store.createSnapshot(plain.id, { actor: human, reason: "manual" }); + const spent = await run({ siteId: plain.id, left: 0, from: a.id, to: b.id }); + expect(spent.sent).toEqual([stored]); + const failing = new SiteGraphStore({ + sql: store.sql, + blobs: new Proxy(store.blobs, { + get(target, prop, receiver) { + const v = Reflect.get(target, prop, receiver); + if (prop === "get") return async () => Promise.reject(new Error("R2 went away")); + return typeof v === "function" ? v.bind(target) : v; + }, + }), + }); + const broken = await run({ + graph: failing, + siteId: plain.id, + left: 100, + from: a.id, + to: b.id, + }); + expect(broken.sent).toEqual([stored]); + }); + + it("a re-plan with more terms than reads left purges everything before reading anything", async () => { + const { hall, event } = await world(); + const from = (await fresh()).publishedSnapshotId as string; + await edit(hall, { title: "Main hall" }); + const to = await snap("manual"); + const s = (await store.getSnapshot(to.id)) as Snapshot; + const terms = new Set(s.nodes.filter((n) => n.kind === "term").map((n) => n.hash)).size; + expect(terms).toBeGreaterThan(1); + // the gate read is one call; then fewer reads are left than there are terms + const { sent, allowance } = await run({ left: terms, from, to: to.id }); + expect(sent).toHaveLength(1); + expect((sent[0] as { reason?: string }).reason).toMatch(/terms/); + expect(allowance.left).toBe(terms - 1); + // enough left: tags + const ok = await run({ left: 2000, from, to: to.id }); + expect(ok.sent[0]).toMatchObject({ mode: "tags" }); + expect((ok.sent[0] as { tags: string[] }).tags).toContain(node(event)); + }); + }); +}); diff --git a/packages/mcp-server/src/schedules-deps.ts b/packages/mcp-server/src/schedules-deps.ts index 8a103105..264e5120 100644 --- a/packages/mcp-server/src/schedules-deps.ts +++ b/packages/mcp-server/src/schedules-deps.ts @@ -31,6 +31,8 @@ export interface SchedulesDeps { readonly maxNewBodies?: number | undefined; /** After "Publish now" commits a release (the Node server rebuilds its static site, #150). */ readonly onReleasePublished?: ReleasePublishedHook | undefined; + /** RFC-0045: the deployment's tick budget, which sizes "Publish now"'s purge (default: Free's). */ + readonly tickBudget?: { readonly maxQueries: number } | undefined; } /** True when the deployment serves schedules. */ @@ -50,6 +52,7 @@ export function releaseService( ...(deps.purge ? { purge: deps.purge } : {}), ...(deps.maxNewBodies !== undefined ? { maxNewBodies: deps.maxNewBodies } : {}), ...(deps.onReleasePublished ? { onReleasePublished: deps.onReleasePublished } : {}), + ...(deps.tickBudget ? { tickBudget: deps.tickBudget } : {}), }); }