diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2ac82239..53065b9d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,7 @@ jobs: outputs: desktop_smoke: ${{ github.event_name != 'pull_request' || steps.filter.outputs.desktop_smoke == 'true' }} selfhost_docker_smoke: ${{ github.event_name != 'pull_request' || steps.filter.outputs.selfhost_docker_smoke == 'true' }} + cloud_closure: ${{ github.event_name != 'pull_request' || steps.filter.outputs.cloud_closure == 'true' }} steps: - uses: actions/checkout@v4 @@ -41,6 +42,14 @@ jobs: - "packages/kernel/runtime-quickjs/**" - "packages/plugins/**" - "packages/react/**" + cloud_closure: + - ".github/workflows/**" + - "bun.lock" + - "package.json" + - "turbo.json" + - "apps/cloud/**" + - "packages/**" + selfhost_docker_smoke: - ".github/workflows/**" - ".dockerignore" @@ -302,6 +311,50 @@ jobs: path: e2e/runs/ retention-days: 7 + cloud-closure: + name: Cloud evaluated closure + needs: changes + if: needs.changes.outputs.cloud_closure == 'true' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 20 + env: + TURBO_API: ${{ vars.TURBO_API }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }} + # Every cold isolate evaluates this closure before it can answer a + # request, and it only ever grows by accident - a barrel export or a new + # module-scope import quietly pulls megabytes into the server graph. The + # budget is a ratchet just above today's size, not a discovered limit: + # when it trips, make the new dependency lazy rather than raising it. + START_CLOSURE_BUDGET_MB: 13.5 + steps: + - uses: actions/checkout@v4 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.11 + + - name: Cache Bun package cache + uses: actions/cache@v4 + with: + path: ~/.bun/install/cache + key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun-1.3.11- + + - uses: actions/setup-node@v4 + with: + node-version: 24 + + - run: bun install --frozen-lockfile + + - run: bun run build + working-directory: apps/cloud + + - run: node scripts/start-closure.mjs + working-directory: apps/cloud + desktop-smoke: name: Desktop smoke build needs: changes diff --git a/apps/cloud/scripts/start-closure.mjs b/apps/cloud/scripts/start-closure.mjs new file mode 100644 index 000000000..0b45c22f1 --- /dev/null +++ b/apps/cloud/scripts/start-closure.mjs @@ -0,0 +1,115 @@ +// --------------------------------------------------------------------------- +// Measure the Worker's *evaluated* module closures from the build output. +// --------------------------------------------------------------------------- +// +// Upload size does not predict cold-isolate cost: a Worker can ship megabytes +// that never load, and the bytes that matter are the ones an isolate must +// evaluate before it can answer. Rollup already separates static from dynamic +// edges, so we can compute that split directly instead of reading totals. +// +// Two closures matter: +// startup - statically reachable from the Worker entry. Evaluated on every +// cold isolate before any request is served. +// start - the TanStack Start server graph, reached through the lazy +// `loadEntries` dynamic imports. Evaluated on the first request +// that enters the app. +// +// Anything reachable only through a dynamic import is not counted: making a +// heavy dependency lazy is exactly the outcome this rewards. +// +// Scope note: this measures bytes, which is a proxy for cold-start cost, not a +// proven cause of any particular regression. The Aug 2026 MCP incident was +// NOT explained by this number - reverting the offending packages moved the +// evaluated closure by 0.02 MB while restoring production latency, so +// module-scope execution cost, not size, drove that one. Treat a budget breach +// as "this will make cold starts worse", not as "this is why the site is slow". +import { readFileSync, readdirSync, statSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; + +const DIST = resolve(process.argv[2] ?? "dist/server"); +const ENTRY = join(DIST, "index.js"); + +// Rollup emits `from "./x.js"`, bare `import "./x.js"`, `export ... from "./x.js"` +// (all static) and `import("./x.js")` (dynamic). Matching the emitted output +// rather than source means we see the graph as the runtime sees it. +const STATIC_RE = /(?:from|import)\s*["'](\.[^"']+)["']/g; +const DYNAMIC_RE = /import\(\s*["'](\.[^"']+)["']\s*\)/g; + +const listChunks = (dir) => + readdirSync(dir, { withFileTypes: true }).flatMap((e) => + e.isDirectory() + ? listChunks(join(dir, e.name)) + : e.name.endsWith(".js") + ? [join(dir, e.name)] + : [], + ); + +const graph = new Map(); +for (const file of listChunks(DIST)) { + const code = readFileSync(file, "utf8"); + const dynamic = new Set([...code.matchAll(DYNAMIC_RE)].map((m) => resolve(dirname(file), m[1]))); + // A specifier inside `import(...)` also matches STATIC_RE's `import` branch, + // so subtract the dynamic set rather than trusting the static matches alone. + const staticDeps = new Set( + [...code.matchAll(STATIC_RE)] + .map((m) => resolve(dirname(file), m[1])) + .filter((p) => !dynamic.has(p)), + ); + graph.set(file, { size: statSync(file).size, static: staticDeps, dynamic }); +} + +/** Bytes evaluated when `roots` are loaded, following static edges only. */ +const closure = (roots) => { + const seen = new Set(); + const queue = [...roots]; + while (queue.length > 0) { + const file = queue.pop(); + if (seen.has(file) || !graph.has(file)) continue; + seen.add(file); + queue.push(...graph.get(file).static); + } + return seen; +}; + +const bytes = (files) => [...files].reduce((sum, f) => sum + (graph.get(f)?.size ?? 0), 0); +const mb = (n) => `${(n / 1024 / 1024).toFixed(2)} MB`; +const name = (f) => relative(DIST, f); + +const startup = closure([ENTRY]); +// The lazy server-graph entries Start pulls on first request. +const startRoots = [...graph.get(ENTRY).dynamic].filter((f) => + /(start|router|tanstack)/.test(name(f)), +); +const start = closure(startRoots); +const evaluated = new Set([...startup, ...start]); + +const report = (label, files) => { + console.log(`\n${label}: ${mb(bytes(files))} (${files.size} chunks)`); + const own = [...files].filter((f) => !startup.has(f) || label === "startup"); + for (const f of own.sort((a, b) => graph.get(b).size - graph.get(a).size).slice(0, 12)) { + console.log(` ${(graph.get(f).size / 1024).toFixed(0).padStart(6)} KB ${name(f)}`); + } +}; + +report("startup", startup); +report("start", start); +console.log(`\ntotal evaluated on a warm-path request: ${mb(bytes(evaluated))}`); +const lazyOnly = [...graph.keys()].filter((f) => !evaluated.has(f)); +console.log( + `deferred behind dynamic import: ${mb(bytes(lazyOnly))} (${lazyOnly.length} chunks)`, +); + +const budget = Number(process.env.START_CLOSURE_BUDGET_MB ?? 0); +if (budget > 0) { + const actual = bytes(evaluated) / 1024 / 1024; + console.log(`\nbudget ${budget} MB — actual ${actual.toFixed(2)} MB`); + if (actual > budget) { + console.error( + `\nFAIL: evaluated closure ${actual.toFixed(2)} MB exceeds the ${budget} MB budget.\n` + + `Every cold isolate pays to evaluate this closure before it can answer. Move the\n` + + `new weight behind a dynamic import rather than raising the budget; run this\n` + + `script with no budget set to see the biggest members and what is already lazy.`, + ); + process.exit(1); + } +} diff --git a/apps/cloud/src/server.ts b/apps/cloud/src/server.ts index d7bc00702..30624f3b4 100644 --- a/apps/cloud/src/server.ts +++ b/apps/cloud/src/server.ts @@ -211,9 +211,34 @@ const mcpAgentHandler = makeCloudMcpAgentHandler(); // That startup is invisible to every span we emit, and it sits on top of the // ~3.1s `loadEntries` import — together the 3-5s a signed-in page costs. // -// Both counters are cheap: two increments and no I/O. +// executor.isolate.id - identifies the isolate itself, so reuse can +// be counted directly instead of inferred. +// executor.isolate.age_ms - ms since this isolate served its first +// request. +// +// The last two exist because the Aug 2026 hunt inferred "isolates stopped being +// reused" from a latency cutoff (requests slower than 1s were called cold) and +// then built a size-based theory on top of that proxy. The theory was wrong: +// reverting the offending packages restored production while moving the +// evaluated module closure by 0.02 MB (see scripts/start-closure.mjs). Grouping +// by isolate id answers "how many requests did this isolate serve, and were the +// slow ones its first?" directly, which no latency threshold can. +// +// All of it is cheap: two increments, one lazy uuid, and no I/O. let isolateRequestSeq = 0; let startGraphEntered = false; +// Minted on first request rather than at module scope: Workers reject random +// number generation during global-scope evaluation. +let isolateId: string | undefined; +let isolateFirstSeenAt = 0; + +const identifyIsolate = (): { readonly id: string; readonly ageMs: number } => { + if (isolateId === undefined) { + isolateId = crypto.randomUUID(); + isolateFirstSeenAt = Date.now(); + } + return { id: isolateId, ageMs: Date.now() - isolateFirstSeenAt }; +}; const markStartGraphEntered = (): void => { startGraphEntered = true; @@ -300,8 +325,11 @@ const cloudflareHandler: ExportedHandler = { async (span) => { span.setAttribute(ATTR_HTTP_REQUEST_METHOD, request.method); span.setAttribute(ATTR_URL_PATH, url.pathname); + const isolate = identifyIsolate(); span.setAttribute("executor.isolate.request_seq", isolateRequestSeq); span.setAttribute("executor.start_graph.entered", startGraphEntered); + span.setAttribute("executor.isolate.id", isolate.id); + span.setAttribute("executor.isolate.age_ms", isolate.ageMs); // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary; observe response/error for span status, keep the flush alive past the response try { const response = await fetchHandler(