diff --git a/docs/CLOUD_ACCEPTANCE_READINESS.md b/docs/CLOUD_ACCEPTANCE_READINESS.md new file mode 100644 index 0000000..e1f69f4 --- /dev/null +++ b/docs/CLOUD_ACCEPTANCE_READINESS.md @@ -0,0 +1,39 @@ +# Cloud acceptance readiness — milestone 1 + +The owner approved the seven delivery milestones on 29 September 2026. This +change implements the first runtime prerequisite, not all seven milestones. + +`GET /readyz` is an operator-only, effect-free endpoint, authenticated with the +existing WORKER_TICK_TOKEN. A configuration request proves exact deployed +version, source tag and required binding presence without querying the database. +A database request checks only the three existing metadata RPCs and their +required capabilities. No scheduler, tenant record, model, provider, job claim +or completion path is invoked. A readiness response never authorises execution. + +The caller supplies X-OCPF-Expected-Version (UUID), X-OCPF-Expected-Sha (40 hex), +X-OCPF-Readiness-Nonce (32–64 hex), and X-OCPF-Readiness-Mode (configuration or +database). Responses are non-cacheable, echo the challenge and identify the +actual version. Failed authentication stays opaque. Errors contain fixed codes, +not credentials, origins, database responses or tenant data. Only hosted +Supabase origins, or a temporary tunnel in explicit staging, are admitted. + +This addresses the previous acceptance design defect: `/healthz` liveness was +followed immediately by a business tick before authentication, version and +backend readiness were distinguished. It is not yet proof of the cause of every +previous HTTP 404/500, or evidence that hosted acceptance has passed. + +The app-owned harness must verify the active deployment and exact version, +perform bounded configuration/database probes, then execute each acceptance +job once. Only effect-free probes may be retried. An ambiguous job/tick failure +must stop the experiment. Preserve every trial, including failures and cleanup. + +Production controls, cron configuration, provider capability gates and existing +ledgers are unchanged. No new database branch, subscription, image service or +always-running compute is required. Cloud execution still needs capacity/budget +verification and independent teardown. The app repository retains private +schema source; do not copy it into this public repository for CI savings. + +Local validation on the operator's isolated coding environment: TypeScript +check and 63 synthetic readiness tests passed. Hosted/full current-head CI is a +separate result and must be read before promotion. No production deployment was +performed by this source change. diff --git a/package.json b/package.json index 320bbd4..7c9b1cc 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "node --import tsx scripts/build.ts", "typecheck": "tsc --noEmit --project tsconfig.json", - "test": "npm run build && node dist/test/security-hardening.test.js && node dist/test/source-ssrf.test.js && node dist/test/browser-collector-ingest.test.js && node dist/test/slot-scheduler.test.js && node dist/test/daily-inventory-planner.test.js && node dist/test/refresh-queue-finalization.test.js && node dist/test/publish-summary.test.js && node dist/test/threads-refresh.test.js && node dist/test/linkedin-refresh.test.js && node dist/test/instagram-image-timeout.test.js && node dist/test/cost-control.test.js && node dist/test/recovery-scheduler.test.js && node dist/test/social-connector.test.js && node dist/test/meta-publication-boundary.test.js && node dist/test/cloudflare-health.test.js && node dist/test/canary-policy.test.js && node dist/test/exclusive-run-gate.test.js && node dist/test/runtime-scope.test.js && node dist/test/tenant-platform-policy.test.js && node dist/test/supabase-client-retry.test.js && node dist/test/worker-claims.test.js && node dist/test/publication-ledger.test.js && node dist/test/publication-outcome.test.js && node dist/test/publication-executor.test.js && node dist/test/provider-single-dispatch.test.js && node dist/test/publication-receipts.test.js && node dist/test/legacy-revision-hold.test.js && node dist/test/http-cancellation.test.js && node dist/test/scheduler-isolation.test.js && node dist/test/agent-jobs.test.js", + "test": "npm run build && node dist/test/security-hardening.test.js && node dist/test/source-ssrf.test.js && node dist/test/browser-collector-ingest.test.js && node dist/test/slot-scheduler.test.js && node dist/test/daily-inventory-planner.test.js && node dist/test/refresh-queue-finalization.test.js && node dist/test/publish-summary.test.js && node dist/test/threads-refresh.test.js && node dist/test/linkedin-refresh.test.js && node dist/test/instagram-image-timeout.test.js && node dist/test/cost-control.test.js && node dist/test/recovery-scheduler.test.js && node dist/test/social-connector.test.js && node dist/test/meta-publication-boundary.test.js && node dist/test/cloudflare-health.test.js && node dist/test/canary-policy.test.js && node dist/test/exclusive-run-gate.test.js && node dist/test/runtime-scope.test.js && node dist/test/tenant-platform-policy.test.js && node dist/test/supabase-client-retry.test.js && node dist/test/worker-claims.test.js && node dist/test/publication-ledger.test.js && node dist/test/publication-outcome.test.js && node dist/test/publication-executor.test.js && node dist/test/provider-single-dispatch.test.js && node dist/test/publication-receipts.test.js && node dist/test/legacy-revision-hold.test.js && node dist/test/http-cancellation.test.js && node dist/test/scheduler-isolation.test.js && node dist/test/agent-jobs.test.js && node dist/test/cloudflare-readiness.test.js", "smoke:dist": "node dist/src/cli.js status", "ci": "npm run typecheck && npm test && npm run smoke:dist", "dev": "tsx src/agent.ts", diff --git a/src/cloudflare-readiness.ts b/src/cloudflare-readiness.ts new file mode 100644 index 0000000..15c3957 --- /dev/null +++ b/src/cloudflare-readiness.ts @@ -0,0 +1,151 @@ +// Operator-only, effect-free readiness. Never import the scheduler, tenant +// credentials, global config, paid models or provider adapters in this module. +export interface ReadinessEnv { + NODE_ENV?: string; + WORKER_TICK_TOKEN?: string; + SUPABASE_URL?: string; + SUPABASE_SERVICE_ROLE_KEY?: string; + SUPABASE_SECRET_KEY?: string; + SERVICE_ROLE_KEY?: string; + CREDENTIAL_ENCRYPTION_KEY?: string; + CF_VERSION_METADATA?: { id: string; tag?: string; timestamp: string }; +} + +const SHA = /^[a-f0-9]{40}$/; +const UUID = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/; +const NONCE = /^[a-f0-9]{32,64}$/; +const MAX_BYTES = 16384; +const HEADERS = { 'Cache-Control': 'no-store, max-age=0', Vary: 'Authorization' }; + +export const READINESS_CONTRACTS = [ + { rpc: 'get_worker_schema_contract', contract: 'worker-claims-v1', capabilities: [ + 'source-targeted-claim-v1', 'angle-targeted-claim-v1', 'angle-exhaust-fenced-v1', + 'source-angle-atomic-commit-v1', 'angle-queue-atomic-commit-v1', + 'queue-angle-identity-v1', 'legacy-queue-revision-hold-v1', + ] }, + { rpc: 'get_publication_schema_contract', contract: 'publication-ledger-v1', capabilities: [ + 'publication-intent-claim-v1', 'publication-dispatch-boundary-v1', + 'publication-attempt-outcome-v1', 'publication-unknown-reconciliation-v1', + 'publication-exact-history-receipt-v1', 'publication-queue-compatibility-fence-v1', + 'publication-provenance-snapshot-v1', 'publication-legacy-queue-hold-v1', + 'publication-queue-lock-order-v1', + ] }, + { rpc: 'get_agent_jobs_contract', contract: 'agent-jobs-v1', capabilities: [ + 'durable-enqueue-v1', 'fenced-terminal-v1', 'reconciliation-only-recovery-v1', + 'rpc-only-job-writes-v1', + ] }, +] as const; + +type JsonObject = Record; +function object(value: unknown): value is JsonObject { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +async function boundedJson(response: Response): Promise { + const declared = response.headers.get('Content-Length'); + if (declared !== null && (!/^\d+$/.test(declared) || Number(declared) > MAX_BYTES)) { + await response.body?.cancel(); + throw new Error('invalid_response'); + } + const reader = response.body?.getReader(); + if (!reader) throw new Error('invalid_response'); + let size = 0; + const chunks: Uint8Array[] = []; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.length; + if (size > MAX_BYTES) throw new Error('invalid_response'); + chunks.push(value); + } + const joined = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { joined.set(chunk, offset); offset += chunk.length; } + return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(joined)); + } finally { + await reader.cancel().catch(() => {}); + reader.releaseLock(); + } +} + +// Only fixed, existing metadata RPCs are consulted. POST is their transport +// convention, not permission to run a job. Response data is never passed through. +export async function handleReadinessRequest( + request: Request, + env: ReadinessEnv, + fetchImpl: typeof fetch = globalThis.fetch, +): Promise { + const token = env.WORKER_TICK_TOKEN; + if (request.method !== 'GET' || !token || request.headers.get('Authorization') !== `Bearer ${token}`) { + return new Response('Not found', { status: 404, headers: HEADERS }); + } + const nonce = request.headers.get('X-OCPF-Readiness-Nonce') || ''; + const expectedVersion = request.headers.get('X-OCPF-Expected-Version') || ''; + const expectedSha = request.headers.get('X-OCPF-Expected-Sha') || ''; + const mode = request.headers.get('X-OCPF-Readiness-Mode') || ''; + if (!NONCE.test(nonce) || !UUID.test(expectedVersion) || !SHA.test(expectedSha) + || !['configuration', 'database'].includes(mode)) { + return Response.json({ ok: false, code: 'readiness_request_invalid' }, { status: 400, headers: HEADERS }); + } + const metadata = env.CF_VERSION_METADATA; + const version = metadata && UUID.test(metadata.id) ? metadata.id : null; + const sha = metadata?.tag && SHA.test(metadata.tag) ? metadata.tag : null; + const base = { + schema: 'ocpf.readiness.v1', nonce, mode, workerVersionId: version, gitSha: sha, + effectFree: true, authorisesExecution: false, + }; + const fail = (code: string, status = 503) => Response.json( + { ...base, ok: false, code }, { status, headers: HEADERS }, + ); + if (version !== expectedVersion || sha !== expectedSha) return fail('release_identity_mismatch', 409); + const serviceKey = env.SUPABASE_SERVICE_ROLE_KEY || env.SUPABASE_SECRET_KEY || env.SERVICE_ROLE_KEY; + if (!serviceKey?.trim() || !env.CREDENTIAL_ENCRYPTION_KEY?.trim()) return fail('runtime_bindings_incomplete'); + let origin: URL; + try { + origin = new URL(env.SUPABASE_URL || ''); + if (origin.protocol !== 'https:' || origin.username || origin.password || origin.port + || origin.pathname !== '/' || origin.search || origin.hash) throw new Error('invalid_origin'); + const hosted = /^[a-z0-9]{20}\.supabase\.co$/.test(origin.hostname); + const temporary = env.NODE_ENV === 'staging' && /^[a-z0-9-]+\.trycloudflare\.com$/.test(origin.hostname); + if (!hosted && !temporary) throw new Error('invalid_origin'); + } catch { return fail('database_origin_invalid'); } + if (mode === 'configuration') { + return Response.json({ ...base, ok: true, code: 'configuration_ready', databaseEvaluated: false }, + { status: 200, headers: HEADERS }); + } + + const signal = AbortSignal.any([request.signal, AbortSignal.timeout(12000)]); + for (const required of READINESS_CONTRACTS) { + let response: Response; + let body: unknown; + try { + signal.throwIfAborted(); + response = await fetchImpl(`${origin.origin}/rest/v1/rpc/${required.rpc}`, { + method: 'POST', redirect: 'error', cache: 'no-store', signal, + headers: { Authorization: `Bearer ${serviceKey}`, apikey: serviceKey, 'Content-Type': 'application/json' }, + body: '{}', + }); + if (!response.ok) { + await response.body?.cancel().catch(() => {}); + if ([401, 403].includes(response.status)) return fail('database_auth_rejected'); + if (response.status === 404) return fail('database_contract_unavailable'); + return fail('database_transport_unavailable'); + } + body = await boundedJson(response); + signal.throwIfAborted(); + } catch { return fail('database_transport_unavailable'); } + if (!object(body) || body.contract !== required.contract || !Array.isArray(body.capabilities) + || required.capabilities.some(c => !(body.capabilities as unknown[]).includes(c))) { + return fail('database_contract_mismatch'); + } + if (required.contract === 'publication-ledger-v1' + && (body.migration !== '20260907054000' || body.lock_order_migration !== '20260913061000')) { + return fail('database_contract_mismatch'); + } + } + return Response.json({ + ...base, ok: true, code: 'database_ready', databaseEvaluated: true, + contracts: READINESS_CONTRACTS.map(c => c.contract), + }, { status: 200, headers: HEADERS }); +} diff --git a/src/cloudflare-worker.ts b/src/cloudflare-worker.ts index b9b0bb7..ccf97c3 100644 --- a/src/cloudflare-worker.ts +++ b/src/cloudflare-worker.ts @@ -1,3 +1,4 @@ +import { handleReadinessRequest } from './cloudflare-readiness'; import { createExclusiveRunGate } from './exclusive-run-gate'; import { installScopedConfig, runWithRuntimeScope } from './runtime-scope'; @@ -193,6 +194,9 @@ export default { async fetch(request: Request, env: Env): Promise { const url = new URL(request.url); + if (url.pathname === '/readyz') { + return handleReadinessRequest(request, env); + } if (url.pathname === '/healthz') { applyCloudflareEnv(env); return Response.json(healthPayload(env)); diff --git a/test/cloudflare-readiness.test.ts b/test/cloudflare-readiness.test.ts new file mode 100644 index 0000000..ee480db --- /dev/null +++ b/test/cloudflare-readiness.test.ts @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { readFileSync } from 'node:fs'; +import { handleReadinessRequest, READINESS_CONTRACTS, type ReadinessEnv } from '../src/cloudflare-readiness'; +const SHA = 'a'.repeat(40); +const VERSION = '11111111-1111-4111-8111-111111111111'; +const NONCE = 'b'.repeat(32); +const env: ReadinessEnv = { + NODE_ENV: 'staging', WORKER_TICK_TOKEN: 'test-operator-secret', + SUPABASE_URL: 'https://abcdefghijklmnopqrst.supabase.co', + SUPABASE_SERVICE_ROLE_KEY: 'test-database-secret', CREDENTIAL_ENCRYPTION_KEY: 'test-encryption-secret', + CF_VERSION_METADATA: { id: VERSION, tag: SHA, timestamp: '2026-09-29T00:00:00Z' }, +}; +function request(mode = 'database', headers: Record = {}, method = 'GET', signal?: AbortSignal) { + return new Request('https://worker.example/readyz', { method, signal, headers: { + Authorization: `Bearer ${env.WORKER_TICK_TOKEN}`, 'X-OCPF-Readiness-Nonce': NONCE, + 'X-OCPF-Expected-Version': VERSION, 'X-OCPF-Expected-Sha': SHA, 'X-OCPF-Readiness-Mode': mode, ...headers, + } }); +} +function fixture(change?: (body: Record, index: number) => void) { + const calls: Array<{ url: string; init: RequestInit }> = []; + const fetchImpl = (async (input: string | URL | Request, init: RequestInit = {}) => { + const i = calls.length; + const req = READINESS_CONTRACTS[i]; + calls.push({ url: String(input), init }); + assert.ok(req, 'a probe must never loop or retry internally'); + const body: Record = { + contract: req.contract, capabilities: [...req.capabilities], + ...(req.contract === 'publication-ledger-v1' ? { migration: '20260907054000', lock_order_migration: '20260913061000' } : {}), + }; + change?.(body, i); + return Response.json(body); + }) as typeof fetch; + return { calls, fetchImpl }; +} +async function check(e: ReadinessEnv = env, r = request(), change?: (body: Record, index: number) => void) { + const f = fixture(change); + const response = await handleReadinessRequest(r, e, f.fetchImpl); + return { response, body: await response.json() as Record, calls: f.calls }; +} + +test('configuration readiness proves exact identity and auth without database work', async () => { + const { response, body, calls } = await check(env, request('configuration')); + assert.equal(response.status, 200); assert.equal(body.code, 'configuration_ready'); + assert.equal(body.workerVersionId, VERSION); assert.equal(body.gitSha, SHA); assert.equal(body.nonce, NONCE); + assert.equal(body.databaseEvaluated, false); assert.equal(body.authorisesExecution, false); assert.equal(calls.length, 0); + assert.match(response.headers.get('Cache-Control') || '', /no-store/); +}); + +test('database readiness performs only three fixed metadata RPCs', async () => { + const { response, body, calls } = await check(); + assert.equal(response.status, 200); assert.equal(body.code, 'database_ready'); + assert.equal(body.databaseEvaluated, true); assert.equal(body.effectFree, true); + assert.equal(body.authorisesExecution, false); assert.equal(calls.length, 3); + for (const [i, call] of calls.entries()) { + assert.equal(call.url, `${env.SUPABASE_URL}/rest/v1/rpc/${READINESS_CONTRACTS[i].rpc}`); + assert.equal(call.init.method, 'POST'); assert.equal(call.init.body, '{}'); + assert.equal(call.init.redirect, 'error'); assert.equal(call.init.cache, 'no-store'); + assert.ok(call.init.signal instanceof AbortSignal); + } + const text = JSON.stringify(body); + for (const secret of [env.WORKER_TICK_TOKEN, env.SUPABASE_SERVICE_ROLE_KEY, env.CREDENTIAL_ENCRYPTION_KEY, env.SUPABASE_URL]) assert.ok(!text.includes(secret!)); +}); + +for (const auth of ['', 'Bearer wrong-secret', 'Basic test']) test(`unauthorised request stays opaque: ${auth}`, async () => { + const f = fixture(); const response = await handleReadinessRequest(request('database', { Authorization: auth }), env, f.fetchImpl); + assert.equal(response.status, 404); assert.equal(await response.text(), 'Not found'); assert.equal(f.calls.length, 0); +}); +for (const method of ['POST', 'PUT', 'DELETE']) test(`readiness rejects ${method}`, async () => { + const f = fixture(); const response = await handleReadinessRequest(request('database', {}, method), env, f.fetchImpl); + assert.equal(response.status, 404); assert.equal(f.calls.length, 0); +}); +test('absent operator credential denies all probes', async () => { + const f = fixture(); const response = await handleReadinessRequest(request(), { ...env, WORKER_TICK_TOKEN: '' }, f.fetchImpl); + assert.equal(response.status, 404); assert.equal(f.calls.length, 0); +}); +for (const [header, value] of [ + ['X-OCPF-Readiness-Nonce', ''], ['X-OCPF-Readiness-Nonce', 'z'.repeat(33)], + ['X-OCPF-Expected-Sha', 'main'], ['X-OCPF-Expected-Version', '../wrong'], + ['X-OCPF-Readiness-Mode', 'publish'], +]) test(`invalid ${header} is rejected before requests`, async () => { + const { response, calls } = await check(env, request('database', { [header]: value })); + assert.equal(response.status, 400); assert.equal(calls.length, 0); +}); +for (const metadata of [undefined, { id: VERSION, tag: 'c'.repeat(40), timestamp: '' }, { id: '22222222-2222-4222-8222-222222222222', tag: SHA, timestamp: '' }]) { + test(`stale or absent version cannot become readiness: ${metadata?.id}`, async () => { + const { response, body, calls } = await check({ ...env, CF_VERSION_METADATA: metadata }); + assert.equal(response.status, 409); assert.equal(body.code, 'release_identity_mismatch'); assert.equal(calls.length, 0); + }); +} +for (const key of ['SUPABASE_SERVICE_ROLE_KEY', 'CREDENTIAL_ENCRYPTION_KEY']) test(`missing ${key} fails closed`, async () => { + const { body, calls } = await check({ ...env, [key]: '' }); + assert.equal(body.code, 'runtime_bindings_incomplete'); assert.equal(calls.length, 0); +}); +for (const url of ['http://127.0.0.1', 'https://user:pass@abcdefghijklmnopqrst.supabase.co', 'https://abcdefghijklmnopqrst.supabase.co/path', 'https://evil.example', 'https://abcdefghijklmnopqrst.supabase.co?x=1']) { + test(`unapproved database origin rejected: ${url}`, async () => { + const { body, calls } = await check({ ...env, SUPABASE_URL: url }); + assert.equal(body.code, 'database_origin_invalid'); assert.equal(calls.length, 0); + }); +} +test('temporary tunnel is permitted only in explicit staging', async () => { + const url = 'https://temporary-fixture.trycloudflare.com'; + const ok = await check({ ...env, SUPABASE_URL: url }, request('configuration')); + assert.equal(ok.response.status, 200); + const no = await check({ ...env, NODE_ENV: 'production', SUPABASE_URL: url }); + assert.equal(no.body.code, 'database_origin_invalid'); assert.equal(no.calls.length, 0); +}); +for (const [index, contract] of READINESS_CONTRACTS.entries()) { + test(`${contract.contract} mismatch fails without leaking its body`, async () => { + const { response, body } = await check(env, request(), (b, i) => { if (i === index) { b.contract = 'incorrect'; b.secret = 'private-row'; } }); + assert.equal(response.status, 503); assert.equal(body.code, 'database_contract_mismatch'); assert.ok(!JSON.stringify(body).includes('private-row')); + }); + for (const cap of contract.capabilities) test(`missing capability ${cap} blocks readiness`, async () => { + const { body } = await check(env, request(), (b, i) => { if (i === index) b.capabilities = contract.capabilities.filter(c => c !== cap); }); + assert.equal(body.code, 'database_contract_mismatch'); + }); +} +test('feature-introducing migration is not replaced with the latest migration head', async () => { + const { body } = await check(env, request(), b => { if (b.contract === 'publication-ledger-v1') b.lock_order_migration = '20260928220000'; }); + assert.equal(body.code, 'database_contract_mismatch'); +}); +for (const status of [301, 401, 403, 404, 429, 500, 503]) test(`HTTP ${status} is classified without forwarding secrets`, async () => { + let calls = 0; + const response = await handleReadinessRequest(request(), env, (async () => { calls++; return new Response('test-database-secret', { status }); }) as typeof fetch); + const text = await response.text(); assert.equal(response.status, 503); assert.equal(calls, 1); assert.ok(!text.includes('test-database-secret')); +}); +for (const text of ['provider error', 'a'.repeat(17000), '{"contract":null}', '[]']) test(`invalid response fails closed (${text.length} bytes)`, async () => { + const response = await handleReadinessRequest(request(), env, (async () => new Response(text)) as typeof fetch); + assert.equal(response.status, 503); assert.ok(!(await response.text()).includes('')); +}); +test('network exceptions are redacted and not retried', async () => { + let calls = 0; + const response = await handleReadinessRequest(request(), env, (async () => { calls++; throw new Error('credential=secret'); }) as typeof fetch); + assert.equal(response.status, 503); assert.equal(calls, 1); assert.ok(!(await response.text()).includes('credential=')); +}); +test('cancelled request cannot touch the database', async () => { + const controller = new AbortController(); controller.abort(); + const { response, calls } = await check(env, request('database', {}, 'GET', controller.signal)); + assert.equal(response.status, 503); assert.equal(calls.length, 0); +}); +test('entrypoint routes readiness before any config/scheduler side effects', () => { + const src = readFileSync('src/cloudflare-worker.ts', 'utf8'); + assert.match(src, /if \(url\.pathname === '\/readyz'\) \{\s+return handleReadinessRequest\(request, env\);\s+\}/); + const readiness = readFileSync('src/cloudflare-readiness.ts', 'utf8'); + assert.doesNotMatch(readiness, /from ['"].*(?:supabase-worker|config|tenant-credentials|publish|ai)['"]/); + assert.doesNotMatch(readiness, /process\.env|runScheduledTick|enqueue_agent_job|claim_agent_job/); +});