test: stabilise the Miniflare D1 suites (#190)

.github/workflows/d1-stress.yml

@@ -0,0 +1,73 @@
+name: D1 stress
+
+# #190: runs the Miniflare halves (D1, Durable Object SQLite, R2) of every package that tests with
+# the pinned Miniflare, many times on fresh runners, under the same load as CI
+# (`turbo --concurrency=2`). Run it by hand after a Miniflare bump or a change to its patch
+# (ADR-0012, addendum 2026-11-10); pull requests that touch the patch run it on their own.
+
+on:
+  workflow_dispatch:
+    inputs:
+      runs:
+        description: Runs per Node version
+        type: number
+        default: 10
+      hogs:
+        description: Busy loops beside the tests (0 = CI's own load)
+        type: number
+        default: 4
+  pull_request:
+    paths:
+      - patches/miniflare@*.patch
+      - .github/workflows/d1-stress.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: d1-stress-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  plan:
+    runs-on: ubuntu-latest
+    outputs:
+      runs: ${{ steps.runs.outputs.runs }}
+    steps:
+      - id: runs
+        env:
+          RUNS: ${{ inputs.runs || 10 }}
+        run: |
+          echo "runs=$(jq -nc --argjson n "$RUNS" '[range(1; ([$n, 50] | min) + 1)]')" >> "$GITHUB_OUTPUT"
+
+  d1:
+    name: D1 halves (Node ${{ matrix.node }}, run ${{ matrix.run }})
+    needs: plan
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        node: [24, 26]
+        run: ${{ fromJSON(needs.plan.outputs.runs) }}
+    steps:
+      - uses: actions/checkout@v7
+      - uses: pnpm/action-setup@v6
+      - uses: actions/setup-node@v7
+        with:
+          node-version: ${{ matrix.node }}
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      # CI's test step (`turbo --concurrency=2`), plus one busy loop per vCPU: the proxy race needs a
+      # preempted thread, and an oversubscribed runner preempts far more often than a quiet one.
+      # `-t` keeps only the suites labelled "(Miniflare)".
+      - name: Miniflare halves on an oversubscribed runner
+        env:
+          HOGS: ${{ inputs.hogs || 4 }}
+        run: |
+          for i in $(seq "$HOGS"); do sh -c 'while :; do :; done' & done
+          trap 'kill $(jobs -p) 2>/dev/null' EXIT
+          pnpm turbo run test --concurrency=2 \
+            --filter=@cloudhal/storage --filter=@cloudhal/core-ts --filter=@cloudhal/api \
+            --filter=@cloudhal/schedules --filter=@cloudhal/comments --filter=@cloudhal/analytics \
+            -- -t '\(Miniflare\)'

AGENTS.md

@@ -38,7 +38,7 @@ pnpm i18n:export / pnpm i18n:import [--locale xx]  # translations as nested ICU
 - `packages/api` — Hono app (`createApi`), Better Auth (`src/auth.ts`), error mapping; shared suite `src/testing.ts` on sqlite + D1. Build chat is `build-turn.ts`, its model eval `build-eval.ts`; `design-pass.ts` is RFC-0021 21c (Build chat looks at its render, the eval's vision-judged score).
 - `packages/ui` — `export * from "@cloudflare/kumo"` + blocks; `styles.css` is the only Tailwind entry for admin UIs. Heavy kits have lazy subpaths: `./charts` (ECharts), `./editor` (the Tiptap markdown body editor, RFC-0014).
 - `packages/messages` — RFC-0032 / ADR-0032: the ICU MessageFormat 1 subset of Studio's catalogs (`plural`, `select`, `selectordinal`, `number`). `./compile` (build-time only: the Vite plugin `./vite`, the type generator, tests; `bundle-budget.ts` keeps it out of every chunk) and `./format` (Worker-safe runtime, cached `Intl.PluralRules`/`NumberFormat`), which is also Studio's `Intl` layer: `formatNumber`, `formatDate`, `formatRelative`, `formatList` and cached `numberFormat`/`dateTimeFormat`/`relativeTimeFormat`/`displayNames`/`collator` — screens and blocks never build `Intl` formatters themselves (lint `i18n-format`). Catalogs compile to data, never code.
-- `patches/` — `pnpm patch` files (`patchedDependencies` in `pnpm-workspace.yaml`). `@[email protected]` is **generated** by `node scripts/kumo-patch.ts`: the hand-written changes in `patches/kumo/*.patch` (`tooltip-timestamp.patch` adds `TimeseriesChart`'s `tooltipTimestampFormat`, DESIGN.md §2v "Kumo locale") plus every physical utility in Kumo's `dist` made logical (ADR-0033 A1). Never edit it; on a Kumo upgrade rename it and the `patchedDependencies` entry, re-run the script, `pnpm install`, re-check the gaps in `docs/design/kumo-upstream-notes.md`. `packages/ui/test/kumo-rtl.test.tsx` fails when the installed Kumo is not patched.
+- `patches/` — `pnpm patch` files (`patchedDependencies` in `pnpm-workspace.yaml`). `@[email protected]` is **generated** by `node scripts/kumo-patch.ts`: the hand-written changes in `patches/kumo/*.patch` (`tooltip-timestamp.patch` adds `TimeseriesChart`'s `tooltipTimestampFormat`, DESIGN.md §2v "Kumo locale") plus every physical utility in Kumo's `dist` made logical (ADR-0033 A1). Never edit it; on a Kumo upgrade rename it and the `patchedDependencies` entry, re-run the script, `pnpm install`, re-check the gaps in `docs/design/kumo-upstream-notes.md`. `packages/ui/test/kumo-rtl.test.tsx` fails when the installed Kumo is not patched. `[email protected]` backports Miniflare 5's sync binding-proxy fix (workers-sdk#15552) that the D1 harness needs (#190, ADR-0012); `packages/storage/test/miniflare-patch.test.ts` fails without it.
 - `apps/studio` — SPA; routes in `src/router.tsx`, screens in `src/routes`, typed API client `src/lib/api.ts` (a barrel over `src/lib/api/<area>.ts`), i18n in `src/i18n` (messages are ICU MF1, compiled at build time, with parameter types generated into `src/i18n/generated/params.ts` by `pnpm gen` — never edit it; languages registered in `locales.ts`; add keys to `en.ts` and every `<locale>.ts`, or to a lazy namespace `src/i18n/lazy/<ns>.<locale>.ts` (the list and how to add one: `src/i18n/namespaces.ts`; DESIGN.md §1); only the active language of the core catalogs loads, `tr.ts` through `loadLocale`, and each other language's lazy-namespace loaders come with it from the generated `src/i18n/loaders/<locale>.ts`; Arabic and Persian format with Western digits through the registry's `intl` tag).
 - `docs/adr/` decisions (with sources) · `docs/rfcs/` proposals · `docs/design/DESIGN.md` the binding admin design system (Kumo).
 - `.agents/skills/` — skills for you: `contract-change`, `add-capability`, `write-rfc`, `studio-ui`, `translate`, and Cloudflare's official **`kumo-design`** (installed from `cloudflare/kumo`, tracked in `skills-lock.json`; update with `npx skills update`). Any admin-UI change loads `kumo-design` + `studio-ui` and follows `docs/design/DESIGN.md`.
@@ -59,7 +59,7 @@ pnpm i18n:export / pnpm i18n:import [--locale xx]  # translations as nested ICU
 13. **Regexes over external input** (request paths and bodies, agent or editor markdown, uploads, fetched feeds) must be linear (#168, #174): no unanchored or end-anchored quantified runs (`/-+$/`, `/\s+$/`, `/\/+$/`), no lazy scans that rescan to the end when the closer is missing (`/<x>[\s\S]*?<\/x>/`, `/<[^>]*>/`), no adjacent quantifiers over overlapping classes (`\s+[^:]+`, `\s*(.*)`). Use `trimChar`/`trimEndChar` (`@cloudhal/core-ts/names`), `trimEnd()`, `indexOf`/`lastIndexOf` scanners, and check lengths before patterns. A rewrite gets an equivalence test against the old regex and a 100k-character adversarial case.
 
 ## Versions (pinned via pnpm catalog / Cargo workspace)
-Node 24 · pnpm 12 · TypeScript 7.0 · Biome 2.6 · Vitest 5 · Turborepo 2.11 · Rust 2024 edition · json-schema-to-typescript 16.0.0 · cargo-typify 0.8.0 · ajv 8.20 · jsonschema 0.60 · better-sqlite3 13 · miniflare 4.20260730 (last stable 4.x; its sharp overridden to 0.35.5; see ADR-0012) · wrangler 4.158 · @astrojs/cloudflare 14.4 · Astro 7.4 · Tailwind 4.3 · marked 18 · Hono 4.13 · Better Auth 1.7.12 · Kumo 2.14 · React 19.3 · TanStack Router 1.170 / Query 5.104 · Vite 8.3.
+Node 24 · pnpm 12 · TypeScript 7.0 · Biome 2.6 · Vitest 5 · Turborepo 2.11 · Rust 2024 edition · json-schema-to-typescript 16.0.0 · cargo-typify 0.8.0 · ajv 8.20 · jsonschema 0.60 · better-sqlite3 13 · miniflare 4.20260730 (last stable 4.x, patched for #190; its sharp overridden to 0.35.5; see ADR-0012) · wrangler 4.158 · @astrojs/cloudflare 14.4 · Astro 7.4 · Tailwind 4.3 · marked 18 · Hono 4.13 · Better Auth 1.7.12 · Kumo 2.14 · React 19.3 · TanStack Router 1.170 / Query 5.104 · Vite 8.3.
 
 ## When unsure
 Read `docs/adr/README.md` and the relevant ADR; each ADR links its primary sources. If the change alters a contract or a public interface, write an RFC first (`.agents/skills/write-rfc`).

docs/adr/0012-cloudflare-deployment.md

@@ -53,3 +53,15 @@ Decision 6 pins the direct `miniflare` devDependency (the D1, R2 and Durable Obj
 - **undici.** The undici versions Miniflare pins (7.28.0 in 4.20260730.0, 7.29.1 in the 5.x alpha) are accepted for local development and tests: Miniflare uses undici to reach its own workerd on localhost and, outside tests, to fetch Cloudflare's `cf.json` from workers.cloudflare.com, a trusted endpoint (set `CLOUDFLARE_CF_FETCH_ENABLED=false` to turn that off). The fixed 7.30.1 is a minor move inside `dispatchFetch`, which the D1 harness and Miniflare's binding proxy depend on (the Durable Object harness bypasses it since #116), so an override waits for its own serial test run. *Update 2026-11-08:* that run happened. #159 (Dependabot alerts #5–#32) added the override `"undici@<7.30.1": 7.30.2` to `pnpm-workspace.yaml`, so Miniflare's pinned 7.28.0 resolves to 7.30.2 in monorepo installs; the D1 and Durable Object harnesses passed on it. Like the sharp override it is not published, and it goes when Miniflare 4 is no longer pinned.
 
 Sources: https://github.com/advisories/GHSA-wq5f-xc86-pv6w · https://github.com/advisories/GHSA-rgj7-g3m4-5g8c · https://pnpm.io/settings#overrides · `npm view miniflare dist-tags`, `npm view sharp time`, `npm view wrangler time` (checked 2026-10-25) · Miniflare 4.20260730.0 `dist/src/index.js` (`imagesLocalFetcher`, `cfImageLocalFetcher`, `setupCf`).
+
+## Addendum (2026-11-10): patching the pinned Miniflare's binding proxy (#190)
+
+The D1 suites failed now and then on CI, never on sqlite or Durable Object SQLite, and passed on rerun. Both failures in #190 were the same assertion, `message?.id === id` at `miniflare/src/plugins/core/proxy/fetch-sync.ts:147`, reached from `D1Database#prepare`/`bind`, which Miniflare's binding proxy runs as synchronous calls. The "statement budget" test in `core-ts` was not a counting or timing failure: its stack ends in the same assertion.
+
+- **Cause.** `SynchronousFetcher` blocks the caller on a shared 0/1 flag: the worker thread posts the reply, stores 1 and notifies. If the worker is preempted between its store and its notify while the caller has already read the reply (the flag was 1 when it reached `Atomics.wait`), the late notify wakes the *next* call before its reply exists. `receiveMessageOnPort` returns `undefined`, the assert throws, and every later call reads the previous call's reply, which is the cascade of 1–3 ms failures in `roles.test.ts`. A loaded runner makes the preemption likely: `turbo --concurrency=2` runs two packages of up to three Vitest workers each (two in `@cloudhal/api`), and every D1 or Durable Object file runs its own workerd process plus the proxy's worker thread, on four vCPUs. The race reproduces on the real Miniflare once both preemptions are forced (see the test below), and in a standalone model of the handshake (the shipped protocol desyncs at call 3, the fixed one runs 5,000 calls). It does not show up on demand without them: 20 unpatched runs of the D1 halves passed on CI's own load, and 20 more with one busy loop per vCPU. Upstream: cloudflare/workers-sdk#15552, merged 2026-09-16 into Miniflare 5 only.
+- **Decision.** There is no newer Miniflare 4.x (4.20260730.0 is still `latest`; 5.x is beta), so the pin stays and `patches/[email protected]` backports #15552. It also closes the synchronous proxy's idle sockets after 1 s instead of undici's 4 s, well before workerd's 5 s idle timeout (#16445, part A). That pool lives on the proxy's own worker thread, which only waits for requests, so its timers fire even while a test blocks the main thread. The patch is applied through `patchedDependencies` in `pnpm-workspace.yaml`, so it covers monorepo installs only. Projects that use `@cloudhal/storage/testing` with their own Miniflare 4 keep the bug unless they apply the same patch. `packages/storage/test/miniflare-patch.test.ts` fails if the patch is missing: a `node -r` preload (Node loads it in worker threads too) stalls the proxy thread between its store and its notify and the caller before its wait, on every other call, and 300 `prepare`/`bind` calls must stay in step. Unpatched, the first stalled call fails the CI assertion; patched, all pass. No retries were added and D1 files still run in parallel.
+- **On a Miniflare upgrade.** The patch is hand-written against `dist/src/index.js` (no generator, unlike Kumo's). If the new release carries #15552, delete the patch and its `patchedDependencies` entry. Otherwise `pnpm patch miniflare@<new>`, re-apply the two changes in the edit directory (in `SynchronousFetcher`'s worker script: store `(id + 1) | 0` instead of 1 and set `keepAliveTimeout`/`keepAliveMaxTimeout` to 1000 on its `Pool`; in `fetch()`: drop the reset to 0 and loop `Atomics.wait` until the generation is seen), `pnpm patch-commit`, rename the `patchedDependencies` entry, and run `packages/storage/test/miniflare-patch.test.ts`, which fails until the patch applies.
+- **Not taken: keep-alive `dispatchFetch`.** Every asynchronous proxy call (`first`, `all`, `run`, `batch`) goes through `dispatchFetch`, which sets undici's `reset`: one connection and one port in `TIME_WAIT` per query (2,000 queries left 2,002 `TIME_WAIT` sockets on macOS, where they last 30 s). On a Mac a single `@cloudhal/core-ts` run fills the whole ephemeral range (16,340 of 16,384 ports in `TIME_WAIT`, also at half the workers) and fails with `EADDRNOTAVAIL`; Linux CI has not shown it. cloudflare/workers-sdk#15781 keeps those connections alive in Miniflare 5, and it was backported here first: core-ts then passed locally, but one `roles.test.ts` run in 20 failed with `ECONNRESET` on two kept-alive sockets in a row, which cannot be retried safely because a write may have run. A main-thread stall past workerd's 5 s idle timeout reproduces that error (the client's idle timers cannot fire during it), but a normal run stalls at most about 0.5 s, so the cause is not settled. The backport was dropped from #190, which is about CI; the local port shortage is older than this change and is a follow-up.
+- **Not patched:** #16445 part B (a `FREE` racing an in-flight `batch([prepare()])`). It needs a garbage collection during a workerd stall of over 100 ms, and it has not been seen in CI. When Miniflare 5 is stable, remove the patch together with the sharp and undici overrides.
+
+Sources: https://github.com/cloudflare/workers-sdk/pull/15552 · https://github.com/cloudflare/workers-sdk/pull/15781 · https://github.com/cloudflare/workers-sdk/issues/16445 · `npm view miniflare dist-tags` (checked 2026-11-10) · Miniflare 4.20260730.0 and 5.20261006.0-alpha `dist/src/index.js` (`SynchronousFetcher`, `DispatchFetchDispatcher`) · CI runs 42789152142 and 42793831919 (first attempts).

packages/storage/test/fixtures/preempt-sync-proxy.cjs

@@ -0,0 +1,26 @@
+// Preloaded with `node -r` (#190, test/miniflare-patch.test.ts). Node loads `-r` modules in worker
+// threads too, so this reaches Miniflare's synchronous binding proxy thread. On every other call
+// it stalls the caller between posting its request and waiting, and the proxy thread between
+// storing its "done" flag and notifying: the two preemptions behind cloudflare/workers-sdk#15552.
+const { isMainThread, MessagePort } = require("node:worker_threads");
+
+const spin = (ms) => {
+  const end = process.hrtime.bigint() + BigInt(ms * 1_000_000);
+  while (process.hrtime.bigint() < end);
+};
+
+if (isMainThread) {
+  const post = MessagePort.prototype.postMessage;
+  MessagePort.prototype.postMessage = function (message, ...rest) {
+    const result = post.call(this, message, ...rest);
+    if (process.env.PREEMPT && typeof message?.id === "number" && message.id % 2 === 0) spin(3);
+    return result;
+  };
+} else {
+  const notify = Atomics.notify;
+  let calls = 0;
+  Atomics.notify = (...args) => {
+    if (calls++ % 2 === 0) spin(6);
+    return notify(...args);
+  };
+}

packages/storage/test/fixtures/sync-proxy-calls.cjs

@@ -0,0 +1,24 @@
+// Run with `node -r ./preempt-sync-proxy.cjs` (#190, test/miniflare-patch.test.ts): 300
+// synchronous D1 proxy calls (`prepare`/`bind`) under the simulated preemption. Prints "ok", or
+// the first failure.
+const { Miniflare } = require("miniflare");
+
+(async () => {
+  const mf = new Miniflare({
+    modules: true,
+    script: "export default { fetch() { return new Response('ok'); } }",
+    d1Databases: ["DB"],
+  });
+  const db = await mf.getD1Database("DB");
+  process.env.PREEMPT = "1";
+  let i = 0;
+  try {
+    for (; i < 300; i++) db.prepare("SELECT ?1").bind(i);
+    console.log("ok");
+  } catch (e) {
+    console.log(`call ${i}: ${String(e?.message).split("\n").slice(0, 3).join(" ")}`);
+  } finally {
+    delete process.env.PREEMPT;
+    await mf.dispose();
+  }
+})();

packages/storage/test/miniflare-patch.test.ts

@@ -0,0 +1,28 @@
+/**
+ * #190: `patches/[email protected]` backports cloudflare/workers-sdk#15552, the fix for
+ * the synchronous binding proxy's desync that the D1 harness hit on CI (ADR-0012, addendum
+ * 2026-11-10). This file fails when a Miniflare change arrives without it, or the patch did not
+ * apply: re-check the patch against the new release (drop it once the release carries #15552)
+ * and `pnpm install`.
+ */
+import { execFile } from "node:child_process";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { describe, expect, it } from "vitest";
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const fixture = (name: string) => path.join(here, "fixtures", name);
+
+describe("Miniflare carries the #190 fix (ADR-0012)", () => {
+  it("synchronous D1 calls stay in step when the proxy thread is preempted (workers-sdk#15552)", async () => {
+    // A preload stalls both sides of the handshake on every other call, as a busy runner does
+    // now and then. Unpatched, the first stalled call fails `message?.id === id`.
+    const { stdout } = await promisify(execFile)(
+      process.execPath,
+      ["-r", fixture("preempt-sync-proxy.cjs"), fixture("sync-proxy-calls.cjs")],
+      { timeout: 60_000 },
+    );
+    expect(stdout.trim()).toBe("ok");
+  });
+});