diff --git a/CHANGELOG.md b/CHANGELOG.md index 1945a56..0b0a888 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable project changes are recorded here. Dates use ISO 8601 (`YYYY-MM-DD`) ## [Unreleased] ### Added +- `verify` and `gate` stay on the host unless `--isolated` is passed. That mode runs command, red-green and metric checks with no network, a fresh HOME, and the repository read-only except paths the check declares. The image digest is `env.image` on the contract under test. Without it, `--isolated` refuses and names `env.image`. The receipt records `isolated`, `env_digest` and `heuristic`. grep, wiring and the forbidden scan are one layer, not a boundary. - Every subcommand refuses a flag it does not accept, exits 2, and names the unknown flag and the accepted ones. Nothing runs after that refusal (#33). - The pre-push hook leaves out a commit already on the server default branch. It reads that head with `git ls-remote` of the push URL at send time, for an existing branch and for a new branch. A local remote-tracking ref does not count. If the server cannot be read, every commit in the range is gated (#31, #35). - The pre-push hook gates the commit being pushed, at that sha, in a temporary worktree. It reads the local sha from stdin and does not gate the checked-out HEAD. The base is the most recent `proofline next` pin for the task, or the parent of that commit, and a refusal prints the base it used. `proofline next --base ` records a newer pin. A worktree with no evidence still gates, and stale evidence in the pushing worktree cannot block a commit that passes at its own sha (#21, #22, #23). diff --git a/README.md b/README.md index dfdbd0d..c14150e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ A coding agent reports that a task is finished. Proofline does not take the report: it holds the definition of done as a contract a machine can check, hands the agent one task at a time, re-runs every check itself when the agent reports, and lets a commit through only when the evidence says PASS. The agent gets a corrective prompt that names the file and the line to fix next, never a verdict it can argue with. Zero dependencies, Node 22+, Apache-2.0. +`proofline verify --isolated` and `proofline gate --isolated` run command, red-green and metric checks in a container. Local use stays off unless that flag is passed. The image digest is `env.image` on the contract under test. Without it, `--isolated` refuses and names `env.image`. + ```mermaid flowchart LR C["proofline.json
tasks, criteria, scope,
forbidden actions"] --> N["proofline next
one task, its brief,
the base revision pinned"] diff --git a/docs/receipt.md b/docs/receipt.md index 97b5844..40bfa05 100644 --- a/docs/receipt.md +++ b/docs/receipt.md @@ -21,3 +21,5 @@ The MCP tools `receipt` and `verify_receipt` return that same JSON. `tools/list` When `GET /v1/state/digest` answers within 2 seconds, an L1 receipt gains `memory.vault_digest` and `memory.frontier`. An L2 receipt in ci mode gains only `memory_commitment`, the sha256 of the vault id, the digest and the frontier JSON, so a replica id is not in the receipt. No answer leaves the receipt unchanged. `--vault` writes `work/receipts/--.md` with origin `proofline`. The weakening check is a warning beside the owner signature. It names a loosened threshold, a removed criterion, or a changed command. Once `guard/owner.pub` is pinned, a missing, altered or wrongly-signed contract is refused with `contract not owner-signed`. + +grep, wiring and the forbidden scan are marked `heuristic` in the receipt. They are one layer, not a boundary. An isolated run records `isolated` and `env_digest`. A local run records `isolated` false. diff --git a/fixtures/evasions/benchmark-code-edited/note.txt b/fixtures/evasions/benchmark-code-edited/note.txt new file mode 100644 index 0000000..62e07b8 --- /dev/null +++ b/fixtures/evasions/benchmark-code-edited/note.txt @@ -0,0 +1 @@ +benchmark-code-edited is named in coverage_limits. The static checks do not catch it. diff --git a/fixtures/evasions/check-reads-contract/note.txt b/fixtures/evasions/check-reads-contract/note.txt new file mode 100644 index 0000000..11cdf3c --- /dev/null +++ b/fixtures/evasions/check-reads-contract/note.txt @@ -0,0 +1 @@ +check-reads-contract is named in coverage_limits. The static checks do not catch it. diff --git a/fixtures/evasions/computed-dynamic-import/note.txt b/fixtures/evasions/computed-dynamic-import/note.txt new file mode 100644 index 0000000..00da754 --- /dev/null +++ b/fixtures/evasions/computed-dynamic-import/note.txt @@ -0,0 +1 @@ +computed-dynamic-import is named in coverage_limits. The static checks do not catch it. diff --git a/fixtures/evasions/env-dependent-test/note.txt b/fixtures/evasions/env-dependent-test/note.txt new file mode 100644 index 0000000..185ea48 --- /dev/null +++ b/fixtures/evasions/env-dependent-test/note.txt @@ -0,0 +1 @@ +env-dependent-test is named in coverage_limits. The static checks do not catch it. diff --git a/fixtures/evasions/network-dependent-pass/note.txt b/fixtures/evasions/network-dependent-pass/note.txt new file mode 100644 index 0000000..0042b4c --- /dev/null +++ b/fixtures/evasions/network-dependent-pass/note.txt @@ -0,0 +1 @@ +network-dependent-pass is named in coverage_limits. The static checks do not catch it. diff --git a/fixtures/evasions/pre-generated-cache/note.txt b/fixtures/evasions/pre-generated-cache/note.txt new file mode 100644 index 0000000..6a6a661 --- /dev/null +++ b/fixtures/evasions/pre-generated-cache/note.txt @@ -0,0 +1 @@ +pre-generated-cache is named in coverage_limits. The static checks do not catch it. diff --git a/fixtures/evasions/suite-command-wrapper/note.txt b/fixtures/evasions/suite-command-wrapper/note.txt new file mode 100644 index 0000000..b4e3ce9 --- /dev/null +++ b/fixtures/evasions/suite-command-wrapper/note.txt @@ -0,0 +1 @@ +suite-command-wrapper is named in coverage_limits. The static checks do not catch it. diff --git a/fixtures/evasions/test-removed-by-rename/note.txt b/fixtures/evasions/test-removed-by-rename/note.txt new file mode 100644 index 0000000..c7ec13f --- /dev/null +++ b/fixtures/evasions/test-removed-by-rename/note.txt @@ -0,0 +1 @@ +test-removed-by-rename is named in coverage_limits. The static checks do not catch it. diff --git a/fixtures/evasions/test-weakened-by-snapshot/note.txt b/fixtures/evasions/test-weakened-by-snapshot/note.txt new file mode 100644 index 0000000..88a7dbb --- /dev/null +++ b/fixtures/evasions/test-weakened-by-snapshot/note.txt @@ -0,0 +1 @@ +test-weakened-by-snapshot is named in coverage_limits. The static checks do not catch it. diff --git a/fixtures/evasions/timeout-raised-through-config/note.txt b/fixtures/evasions/timeout-raised-through-config/note.txt new file mode 100644 index 0000000..3b8627c --- /dev/null +++ b/fixtures/evasions/timeout-raised-through-config/note.txt @@ -0,0 +1 @@ +timeout-raised-through-config is named in coverage_limits. The static checks do not catch it. diff --git a/fixtures/tb06/contract.json b/fixtures/tb06/contract.json new file mode 100644 index 0000000..73ab0f2 --- /dev/null +++ b/fixtures/tb06/contract.json @@ -0,0 +1,6 @@ +{ + "image_name": "node:22.11.0", + "env": { + "image": "node:22.11.0@sha256:ec878c763e9fad09d22aae86e2edcb7a05b397dfe8411c16e2b90158d595e2ce" + } +} diff --git a/schema/receipt-v1.json b/schema/receipt-v1.json new file mode 100644 index 0000000..7aee4ac --- /dev/null +++ b/schema/receipt-v1.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://proofline.dev/receipt/v1", + "type": "object", + "properties": { + "isolated": { "type": "boolean" }, + "env_digest": { "type": ["string", "null"] }, + "heuristic": { + "type": "boolean", + "description": "grep, wiring and the forbidden scan are one layer, not a boundary" + }, + "coverage_limits": { + "type": "array", + "items": { "type": "string" } + } + }, + "required": ["isolated", "heuristic"] +} diff --git a/src/checks.mjs b/src/checks.mjs index 77c6d59..00c7018 100644 --- a/src/checks.mjs +++ b/src/checks.mjs @@ -58,8 +58,10 @@ export function diffHunks(root, base) { return files; } -function runCounted(cmd, root, v) { - const r = run(cmd, { cwd: root, timeoutS: v.timeout_s || 900, env: v.env }); +function runCounted(cmd, root, v, ctx) { + const r = ctx?.sandbox + ? ctx.sandbox(cmd, { cwd: root, timeoutS: v.timeout_s || 900, env: v.env, writable: v.writable }) + : run(cmd, { cwd: root, timeoutS: v.timeout_s || 900, env: v.env }); const txt = r.stdout + '\n' + r.stderr; const num = (k) => { const m = txt.match(new RegExp(`^[#ℹ]\\s*${k}\\s+(\\d+)`, 'm')); return m ? +m[1] : undefined; }; r.counts = { tests: num('tests'), pass: num('pass'), fail: num('fail'), skipped: num('skipped'), todo: num('todo'), cancelled: num('cancelled') }; @@ -103,7 +105,7 @@ const named = (e) => { export const verifiers = { command(v, ctx) { const expect = v.expect_exit ?? 0; - let r = runCounted(v.cmd, ctx.root, v); + let r = runCounted(v.cmd, ctx.root, v, ctx); const first = ev(r), firstExit = r.exit; if (r.exit === 127) return { status: 'error', detail: `command not found: ${v.cmd}`, evidence: first }; const tooMany = (r2) => { @@ -117,7 +119,7 @@ export const verifiers = { if (r.exit === expect) return { status: 'fail', detail: [...tooMany(r), named(first)].filter(Boolean).join('; '), evidence: first }; // Both runs keep their failing names: on a flaky result the first run's list is the tests that flapped. for (let i = 0; i < (ctx.flakyReruns ?? 1); i++) { - r = runCounted(v.cmd, ctx.root, v); + r = runCounted(v.cmd, ctx.root, v, ctx); if (r.exit === expect && !tooMany(r).length) return { status: 'flaky', detail: `failed (exit ${firstExit}${named(first) ? `; ${named(first)}` : ''}) then passed on re-run ${i + 1}: flaky, not a pass. Root-cause the non-determinism.`, evidence: { first, rerun: ev(r) } }; } const rerun = ev(r), n = named(rerun.failing.length ? rerun : first); @@ -150,8 +152,8 @@ export const verifiers = { } if (v.setup) run(v.setup, { cwd: wt, timeoutS: 900 }); const cmd = sub(v.cmd, { test: v.test_file, name: v.test_name || '' }); - const red = runCounted(cmd, wt, v); - const green = runCounted(cmd, ctx.root, v); + const red = runCounted(cmd, wt, v, ctx); + const green = runCounted(cmd, ctx.root, v, ctx); const evd = { base: ctx.base, red: ev(red), green: ev(green) }; if (red.exit === 127 || green.exit === 127) return { status: 'error', detail: 'test command not found', evidence: evd }; if (red.exit === 0) return { status: 'fail', detail: `test PASSES on base ${ctx.base.slice(0, 8)} without the change: it does not prove the change (tests how, not what?)`, evidence: evd }; @@ -174,7 +176,10 @@ export const verifiers = { const m = (ctx.machine || readMachine)(); const machine = { load1: m.load1, cpus: m.cpus, load_per_cpu: +(m.load1 / Math.max(1, m.cpus)).toFixed(2), node: nodeVersion(ctx.root, v.env) }; if (typeof max === 'number' && m.load1 / Math.max(1, m.cpus) > max) return { r: null, machine }; - return { r: run(v.cmd, { cwd: ctx.root, timeoutS: v.timeout_s || 1800, env: v.env }), machine }; + const exec = ctx.sandbox + ? (cmd, o) => ctx.sandbox(cmd, { ...o, writable: v.writable }) + : (cmd, o) => run(cmd, o); + return { r: exec(v.cmd, { cwd: ctx.root, timeoutS: v.timeout_s || 1800, env: v.env }), machine }; })(); const on = `load ${machine.load1.toFixed(2)} / ${machine.cpus} CPUs, node ${machine.node || 'not found'}`; if (!r) return { status: 'blocked', detail: `machine busy: load ${machine.load1.toFixed(2)} over ${machine.cpus} CPUs = ${machine.load_per_cpu.toFixed(2)} per CPU > max_load_per_cpu ${max}; the bench was not run. Re-run when the machine is idle.`, evidence: { cmd: v.cmd, machine } }; diff --git a/src/engine.mjs b/src/engine.mjs index 599907a..6acba86 100644 --- a/src/engine.mjs +++ b/src/engine.mjs @@ -8,6 +8,7 @@ import { scanForbidden } from './forbidden.mjs'; import { parseClaims, checkClaim } from './claims.mjs'; import { record, readLog, verifyChain } from './evidence.mjs'; import * as RC from './cache.mjs'; +import { resolveIsolation, COVERAGE_LIMITS } from './isolate.mjs'; export const KNOWN_LIMITS = [ 'Proofline runs on the same machine as the agent: an agent with shell access could edit .proofline/ or proofline.json. The hash chain makes edits detectable, contract weakening vs the base revision is flagged, and CI re-verifies from a clean checkout; protect proofline.json with CODEOWNERS/branch protection for a hard boundary.', @@ -93,31 +94,39 @@ export function baseBrief(root, c, t, explicit) { return L.join('\n'); } -export function verifyTask(root, taskId, { claims, only, skipTypes = [], base: explicitBase, machine, fresh = false } = {}) { +export function verifyTask(root, taskId, opts = {}) { + const { claims, only, skipTypes = [], base: explicitBase, machine, fresh = false } = opts; const c = C.load(root); const t = taskById(c, taskId); + const iso = resolveIsolation(c, opts); const errs = C.validate(c); // Base is pinned when the task starts, so pushing the work does not move the goalposts. const { ref: baseRef, source: baseSource, base, problems: baseProblems } = taskBase(root, c, t, explicitBase); - const ctx = { root, contract: c, task: t, base, changed: base ? changedFiles(root, base) : new Map(), measured: {}, machine, baseline: c.baseline?.metrics, tolerancePct: c.defaults?.regression_tolerance_pct, flakyReruns: c.defaults?.flaky_reruns ?? 1 }; + const ctx = { root, contract: c, task: t, base, changed: base ? changedFiles(root, base) : new Map(), measured: {}, machine, baseline: c.baseline?.metrics, tolerancePct: c.defaults?.regression_tolerance_pct, flakyReruns: c.defaults?.flaky_reruns ?? 1, sandbox: iso.sandbox }; const results = []; const reuse = RC.settings(c); for (const a of t.criteria) { if (only && !only.includes(a.id)) continue; if (skipTypes.includes(a.verifier.type)) continue; let r; - // Reuse (opt-in): a command or metric PASS recorded on this HEAD with a clean tree is not run again; --fresh runs it. - const k = reuse.on && RC.REUSABLE.includes(a.verifier.type) ? RC.keyFor(root, a.verifier, ctx) : null; - const hit = k && !fresh ? RC.lookup(root, k.key, reuse) : null; - if (hit) { - r = RC.reusedResult(hit); - if (a.verifier.type === 'metric') ctx.measured[a.verifier.name || a.verifier.key] = hit.data.evidence.measured; + const sandboxed = ['command', 'test_red_green', 'metric'].includes(a.verifier.type); + if (iso.block && sandboxed) { + r = { status: 'error', detail: iso.block }; } else { - try { r = verifiers[a.verifier.type](a.verifier, ctx); } catch (e) { r = { status: 'error', detail: `verifier crashed: ${e.message}` }; } - // Only a PASS is stored, and only if the tree is still the one the check started on and the log is intact. - if (k && r.status === 'pass' && RC.keyFor(root, a.verifier, ctx)?.key === k.key && verifyChain(root).ok) RC.store(root, k, t.id, a, r); + // Reuse (opt-in): a command or metric PASS recorded on this HEAD with a clean tree is not run again; --fresh runs it. + const k = reuse.on && RC.REUSABLE.includes(a.verifier.type) ? RC.keyFor(root, a.verifier, ctx) : null; + const hit = k && !fresh ? RC.lookup(root, k.key, reuse) : null; + if (hit) { + r = RC.reusedResult(hit); + if (a.verifier.type === 'metric') ctx.measured[a.verifier.name || a.verifier.key] = hit.data.evidence.measured; + } else { + try { r = verifiers[a.verifier.type](a.verifier, ctx); } catch (e) { r = { status: 'error', detail: `verifier crashed: ${e.message}` }; } + // Only a PASS is stored, and only if the tree is still the one the check started on and the log is intact. + if (k && r.status === 'pass' && RC.keyFor(root, a.verifier, ctx)?.key === k.key && verifyChain(root).ok) RC.store(root, k, t.id, a, r); + } } - results.push({ id: a.id, statement: a.statement, mandatory: a.mandatory !== false, type: a.verifier.type, ...r }); + const heuristic = a.verifier.type === 'grep' || a.verifier.type === 'wiring'; + results.push({ id: a.id, statement: a.statement, mandatory: a.mandatory !== false, type: a.verifier.type, ...(heuristic ? { heuristic: true } : {}), ...r }); } const rules = { ...C.DEFAULT_FORBIDDEN, ...(c.forbidden || {}) }; const forbidden = scanForbidden(root, base, rules); @@ -170,7 +179,7 @@ export function verifyTask(root, taskId, { claims, only, skipTypes = [], base: e const pushed = pr ? pr.status === 'pass' : !!hi.head && !git(['status', '--porcelain'], root).out.split('\n').some((l) => l && !l.includes('.proofline/')) && git(['rev-parse', '--verify', '--quiet', `refs/remotes/${tgt.remote}/${tgt.branch}^{commit}`], root).out === hi.head; const reused = results.filter((r) => r.reused).map((r) => ({ id: r.id, type: r.type, ts: r.reused.ts, entry: r.reused.entry })); - return { pushed, task: t.id, title: t.title, phase: t.phase, decision, base, base_ref: baseRef, base_source: baseSource, ...hi, contract_hash: C.contractHash(c), results, reuse: !reuse.on ? 'off' : fresh ? 'fresh' : 'on', reused, forbidden, weakenings: weak, contract_change: contractChange, marker, blocked, failing, measured: ctx.measured, claims: claimResults, changed: [...ctx.changed.keys()], ts: new Date().toISOString() }; + return { isolated: iso.isolated, env_digest: iso.env_digest, heuristic: true, coverage_limits: COVERAGE_LIMITS, pushed, task: t.id, title: t.title, phase: t.phase, decision, base, base_ref: baseRef, base_source: baseSource, ...hi, contract_hash: C.contractHash(c), results, reuse: !reuse.on ? 'off' : fresh ? 'fresh' : 'on', reused, forbidden, weakenings: weak, contract_change: contractChange, marker, blocked, failing, measured: ctx.measured, claims: claimResults, changed: [...ctx.changed.keys()], ts: new Date().toISOString() }; } /** Corrective prompt tied to each failed criterion. */ diff --git a/src/isolate.mjs b/src/isolate.mjs new file mode 100644 index 0000000..8201f21 --- /dev/null +++ b/src/isolate.mjs @@ -0,0 +1,123 @@ +// Opt-in isolation for command, red-green and metric checks. +// Off unless the caller passes isolated, or the process was started with --isolated. +// The image digest lives in the contract under test (a fixture in the TB-06 test), never invented here. +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +const DIGEST = /^node:22\.11\.0@sha256:[0-9a-f]{64}$/; + +/** Evasions the static checks do not catch. Each fixture directory uses one of these names. */ +export const COVERAGE_LIMITS = [ + 'computed-dynamic-import', + 'env-dependent-test', + 'benchmark-code-edited', + 'timeout-raised-through-config', + 'test-removed-by-rename', + 'test-weakened-by-snapshot', + 'suite-command-wrapper', + 'network-dependent-pass', + 'pre-generated-cache', + 'check-reads-contract', +]; + +export function imageDigest(contract) { + const image = contract?.env?.image; + return typeof image === 'string' && DIGEST.test(image) ? image : null; +} + +function hasBin(bin) { + const r = spawnSync(bin, ['--version'], { encoding: 'utf8', timeout: 5000 }); + return r.status === 0; +} + +export function runtimeName() { + if (hasBin('docker')) return 'docker'; + if (hasBin('podman')) return 'podman'; + if (hasBin('bwrap')) return 'bwrap'; + return null; +} + +function result(cmd, cwd, started, child) { + const ms = Math.round(performance.now() - started); + const timedOut = child.error?.code === 'ETIMEDOUT'; + return { + cmd, cwd, ms, timedOut, + exit: timedOut ? 124 : (child.status ?? 1), + stdout: child.stdout || '', + stderr: (child.stderr || '') + (child.error && !timedOut ? String(child.error) : ''), + }; +} + +function bwrapArgs(cmd, cwd, extra, writable) { + const args = ['--unshare-net', '--die-with-parent', '--ro-bind', '/', '/', '--tmpfs', '/tmp', '--clearenv', '--setenv', 'HOME', '/tmp', '--setenv', 'PATH', '/usr/local/bin:/usr/bin:/bin']; + if (cwd) { + args.push('--ro-bind', cwd, '/work', '--chdir', '/work'); + for (const rel of writable) args.push('--bind', path.resolve(cwd, rel), path.posix.join('/work', rel)); + } else { + args.push('--chdir', '/tmp'); + } + for (const [k, val] of extra) args.push('--setenv', k, String(val)); + args.push('--', 'sh', '-c', cmd); + return args; +} + +function dockerArgs(cmd, cwd, image, extra, writable) { + const args = ['run', '--rm', '--network', 'none', '--read-only', '--tmpfs', '/tmp', '-e', 'HOME=/tmp']; + if (cwd) { + args.push('-v', `${cwd}:/work:ro`, '-w', '/work'); + for (const rel of writable) args.push('-v', `${path.resolve(cwd, rel)}:/work/${rel}:rw`); + } + for (const [k, val] of extra) args.push('-e', `${k}=${val}`); + args.push('--entrypoint', 'sh', image, '-c', cmd); + return args; +} + +/** One copy under the home directory, which a desktop runtime can mount when the work tree is not shared. */ +function stageCopy(cwd) { + const parent = path.join(os.homedir(), '.cache', 'proofline-isolate'); + fs.mkdirSync(parent, { recursive: true }); + const dest = fs.mkdtempSync(path.join(parent, 'work-')); + fs.cpSync(cwd, dest, { recursive: true, dereference: false }); + return dest; +} + +/** Run cmd with no network, a fresh HOME, and the repo read-only except declared paths. */ +export function runInSandbox(cmd, { cwd, timeoutS = 600, env, image, runtime, writable = [] } = {}) { + const started = performance.now(); + const extra = Object.entries(env || {}).filter(([k]) => k !== 'NODE_TEST_CONTEXT'); + const argsFor = (dir) => runtime === 'bwrap' + ? bwrapArgs(cmd, dir, extra, writable) + : dockerArgs(cmd, dir, image, extra, writable); + const spawn = (args) => spawnSync(runtime, args, { encoding: 'utf8', timeout: timeoutS * 1000, maxBuffer: 64 << 20 }); + let child = spawn(argsFor(cwd)); + if (cwd && runtime !== 'bwrap' && /Mounts denied/.test(child.stderr || '')) { + const staged = stageCopy(cwd); + try { + child = spawn(argsFor(staged)); + } finally { + fs.rmSync(staged, { recursive: true, force: true }); + } + } + return result(cmd, cwd, started, child); +} + +/** + * Isolation for one verify. Missing env.image refuses and names that item. + * A runtime is probed only after the caller asked for isolation and the digest is present. + */ +export function resolveIsolation(contract, opts = {}) { + const requested = opts.isolated === true || (opts.isolated !== false && process.argv.includes('--isolated')); + if (!requested) return { isolated: false, env_digest: null, block: null, sandbox: null }; + const image = imageDigest(contract); + if (!image) return { isolated: false, env_digest: null, block: 'env.image is required for --isolated', sandbox: null }; + const runtime = runtimeName(); + if (!runtime) return { isolated: false, env_digest: null, block: 'no container runtime for --isolated', sandbox: null }; + const sandbox = (cmd, o) => runInSandbox(cmd, { ...o, image, runtime, writable: o.writable || [] }); + // The version probe does not need the work tree. Mounting it fails where the desktop runtime cannot see that path. + const ver = sandbox('node --version', { timeoutS: 60 }); + const nodeVer = (ver.stdout || '').trim().split('\n')[0] || 'unknown'; + const env_digest = `${image} runtime=${runtime} node=${nodeVer}`; + return { isolated: true, env_digest, block: null, sandbox }; +} diff --git a/test/evasions.test.mjs b/test/evasions.test.mjs new file mode 100644 index 0000000..928077f --- /dev/null +++ b/test/evasions.test.mjs @@ -0,0 +1,14 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { COVERAGE_LIMITS } from '../src/isolate.mjs'; + +test('each evasion fixture is named in coverage_limits', () => { + const root = path.resolve(import.meta.dirname, '../fixtures/evasions'); + const dirs = fs.readdirSync(root, { withFileTypes: true }).filter((d) => d.isDirectory()).map((d) => d.name).sort(); + assert.ok(dirs.length >= 10, `found ${dirs.length}`); + for (const name of dirs) { + assert.ok(COVERAGE_LIMITS.includes(name), `${name} is not named in coverage_limits`); + } +}); diff --git a/test/isolated.test.mjs b/test/isolated.test.mjs new file mode 100644 index 0000000..11b542e --- /dev/null +++ b/test/isolated.test.mjs @@ -0,0 +1,46 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import * as E from '../src/engine.mjs'; +import * as P from '../src/index.mjs'; +import { repo, addTask, commitAll } from './helpers.mjs'; + +function withTask(criteria) { + const r = repo(); + addTask(r, { id: 'T', title: 'Isolation', criteria }); + commitAll(r, 'task'); + return r; +} + +test('a local verify records isolated false', () => { + const r = withTask([{ id: 'T-1', statement: 'file', verifier: { type: 'grep', file: 'package.json', pattern: 'module', must: 'present' } }]); + const out = E.verifyTask(r, 'T', { base: 'HEAD~1' }); + assert.equal(out.isolated, false); + assert.equal(out.env_digest, null); + assert.equal(out.heuristic, true); + assert.equal(out.decision, 'PASS'); +}); + +test('--isolated without env.image names that item', () => { + const r = withTask([{ id: 'T-1', statement: 'cmd', verifier: { type: 'command', cmd: 'echo ok', expect_exit: 0 } }]); + const out = E.verifyTask(r, 'T', { base: 'HEAD~1', isolated: true }); + assert.equal(out.decision, 'BLOCKED'); + assert.match(out.blocked.join('\n'), /env\.image/); + assert.equal(out.isolated, false); +}); + +test('the fixture pins the image digest and an isolated command records it', () => { + const fixture = JSON.parse(fs.readFileSync(path.resolve(import.meta.dirname, '../fixtures/tb06/contract.json'), 'utf8')); + const image = fixture.env?.image || ''; + assert.match(image, /^node:22\.11\.0@sha256:[0-9a-f]{64}$/); + const r = withTask([{ id: 'T-1', statement: 'home and read-only', verifier: { type: 'command', cmd: '[ "$HOME" = /tmp ] && ! touch probe-write', expect_exit: 0 } }]); + const c = P.C.load(r); + c.env = { image }; + P.C.save(r, c); + const out = E.verifyTask(r, 'T', { base: 'HEAD~1', isolated: true }); + assert.equal(out.isolated, true, out.blocked?.join('\n') || out.failing?.join('\n')); + assert.ok(out.env_digest?.includes(image)); + assert.match(out.env_digest || '', /node=v?\d/); + assert.equal(out.decision, 'PASS'); +});