diff --git a/.proofline/approvals/59f854a31a2d2e4c46d7ce5d4c2dc03b12e19f272fb07ba80eb0bf46cc3ec819 b/.proofline/approvals/59f854a31a2d2e4c46d7ce5d4c2dc03b12e19f272fb07ba80eb0bf46cc3ec819 new file mode 100644 index 0000000..e69de29 diff --git a/.proofline/config.json b/.proofline/config.json new file mode 100644 index 0000000..8613567 --- /dev/null +++ b/.proofline/config.json @@ -0,0 +1,3 @@ +{ + "approvals": ".proofline/approvals" +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fb9db7..b86e7cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,17 +2,19 @@ All notable project changes are recorded here. Dates use ISO 8601 (`YYYY-MM-DD`). -## Unreleased +## 0.2.0-next.2 (2026-10-21) ### Added - `proofline --version` (and `-v`) prints the installed version and exits 0. On a subcommand `--version` is still an unknown flag. - The README and the package carry the Proofline mark (`docs/logo.svg`, `docs/logo-white.svg`, PNGs at 512 and 128). +- The tests column of a claim line takes `crit:` for a task whose proof is a command the gate runs rather than a test file it adds, such as an intake or measurement task. The name holds only when it is a criterion of that task, it ran in this gate, its type is `command`, `metric`, `test_red_green` or `docs`, and it passed. An unknown id, another task's criterion, a `grep`, `file_exists`, `scope` or `wiring` check, or a criterion that did not pass is a claim mismatch that names the reason. Other names are still looked up in the repository, and `-`, `n/a` and `none` still name no test. ### Fixed - `proofline setup` no longer ends in "Next step: proofline setup" when `AGENTKEY_HOME` is unset or `agentkey` is not on PATH. Each refusal says what the variable or the command is for and prints the exact command that provides it (`export AGENTKEY_HOME=...`, `npm install --save-dev @optitransfer/agentkey`). `proofline contract sign` does the same. The README documents both before the short path. - The `AGENTS.md` the README tells you to copy no longer trips the `no_verify` rule of a fresh contract on the first `verify`: its "Never" list names the hook-skipping flag in words. - The owner-signature test ran agentkey from a path on one laptop. It now runs the `@optitransfer/agentkey` devDependency, and CI installs the lockfile with `npm ci --ignore-scripts` before the self-gate. - The repository moved to `github.com/Optitransfer/proofline`; `homepage`, `repository` and `bugs` in package.json point there. The old URL redirects. +- A `grep` pattern that starts with `(?i)`, `(?s)` or `(?m)`, the way a Python verifier writes flags, runs with those flags. It crashed as an invalid group, so a contract shared with a Python gate was BLOCKED here and PASS there. ## 0.2.0-next.1 (2026-10-17) diff --git a/package-lock.json b/package-lock.json index 96df03d..fba37ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@optitransfer/proofline", - "version": "0.2.0-next.1", + "version": "0.2.0-next.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@optitransfer/proofline", - "version": "0.2.0-next.1", + "version": "0.2.0-next.2", "license": "Apache-2.0", "bin": { "proofline": "bin/proofline.mjs" diff --git a/package.json b/package.json index 936a85b..631ad82 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@optitransfer/proofline", - "version": "0.2.0-next.1", + "version": "0.2.0-next.2", "description": "Independent V&V for coding agents. The agent says done; Proofline re-runs every check against a contract, rejects claims the evidence does not support, and gates the push.", "keywords": [ "ai-agents", diff --git a/proofline.json b/proofline.json index c62f111..a34cffd 100644 --- a/proofline.json +++ b/proofline.json @@ -3832,9 +3832,205 @@ "If a criterion cannot pass without loosening a threshold, skipping or deleting a test, raising a timeout, widening the scope or editing the contract: STOP and report file:line and the measured value.", "If the environment a criterion needs is missing (repository admin, npm owner login, Actions, a second machine): report BLOCKED naming the missing item; never mark it passed by hand." ] + }, + { + "id": "PL-CRIT01", + "title": "A claim can name an executed criterion with crit:, and a grep pattern may start with (?i) or (?s)", + "phase": "implement", + "owner": "lane-a", + "depends_on": [], + "requirements": [ + "REQ-008", + "REQ-009" + ], + "scope": { + "allow": [ + "src/claims.mjs", + "src/engine.mjs", + "src/checks.mjs", + "test/claims_crit.test.mjs", + "test/grep_inline_flags.test.mjs", + "CHANGELOG.md" + ] + }, + "description": "Two gaps found on 2026-10-20. (1) checkClaim (src/claims.mjs:40-46) accepts a tests-column name only when git grep -F or git ls-files finds it, and drops '-', 'n/a' and 'none', so an intake or measurement task, whose proof is a command the gate runs and not a test file it adds, cannot make an honest done claim (the oll4com/spaceapp #64 intake task PR64-00 had to be recorded PASS-BY-EVIDENCE). The tests column takes crit:; it holds only when the id is a criterion of the claimed task that ran in this gate, of type command, metric, test_red_green or docs, with status pass. An unknown id, another task's criterion, a grep, file_exists, scope or wiring check, or a criterion that did not pass is a claim mismatch naming the reason. The gate passes checkClaim the id, type and status of each criterion it ran (src/engine.mjs). Any other name is still looked up in the repository, and '-', 'n/a' and 'none' still name no test. (2) the grep verifier (src/checks.mjs:211-214) builds new RegExp(v.pattern, v.flags || 'm'); a leading (?i) or (?s), as a Python verifier writes flags, throws 'Invalid group', so the x402-conformance contract (XC-00-AC03, XC-04-AC12, XC-04-AC13, XC-M2-AC02) is BLOCKED here and PASS in its own CI. A leading (?i), (?s) or (?m) becomes flags added to the criterion's own; a pattern without one is unchanged and stays case-sensitive. Both are recorded under Unreleased in CHANGELOG.md.", + "criteria": [ + { + "id": "PL-CRIT01-AC01", + "mandatory": true, + "statement": "crit: holds for a passing executed criterion of the task and is refused, with the reason, for an unknown id, another task, a check that runs nothing and a failing criterion", + "verifier": { + "type": "test_red_green", + "test_file": "test/claims_crit.test.mjs", + "cmd": "node --test {test}" + } + }, + { + "id": "PL-CRIT01-AC02", + "mandatory": true, + "statement": "a grep pattern starting with (?i) or (?s) runs with those flags instead of crashing, and a pattern without one stays case-sensitive", + "verifier": { + "type": "test_red_green", + "test_file": "test/grep_inline_flags.test.mjs", + "cmd": "node --test {test}" + } + }, + { + "id": "PL-CRIT01-AC03", + "mandatory": true, + "statement": "Full suite exits 0, nothing hung", + "verifier": { + "type": "command", + "cmd": "npm test", + "expect_exit": 0, + "timeout_s": 1800 + } + }, + { + "id": "PL-CRIT01-AC04", + "mandatory": true, + "statement": "the changelog records the crit: claims form under Unreleased", + "verifier": { + "type": "grep", + "file": "CHANGELOG.md", + "pattern": "^- The tests column of a claim line takes `crit:`", + "must": "present", + "flags": "m" + } + }, + { + "id": "PL-CRIT01-AC05", + "mandatory": true, + "statement": "the changelog records the inline-flag grep fix", + "verifier": { + "type": "grep", + "file": "CHANGELOG.md", + "pattern": "^- A `grep` pattern that starts with `\\(\\?i\\)`", + "must": "present", + "flags": "m" + } + }, + { + "id": "PL-CRIT01-AC06", + "mandatory": true, + "statement": "crit: names are resolved on the default gate path", + "verifier": { + "type": "wiring", + "entry": "bin/proofline.mjs", + "symbol": "critIssues" + } + }, + { + "id": "PL-CRIT01-AC07", + "mandatory": true, + "statement": "Changes stay in scope", + "verifier": { + "type": "scope" + } + }, + { + "id": "PL-CRIT01-AC08", + "mandatory": true, + "statement": "Committed and pushed to main", + "verifier": { + "type": "pushed" + } + } + ], + "stop_rules": [ + "If a criterion cannot pass without loosening a threshold, skipping or deleting a test, raising a timeout, widening the scope or editing the contract: STOP and report file:line and the measured value.", + "If the environment a criterion needs is missing (repository admin, npm owner login, Actions, a second machine): report BLOCKED naming the missing item; never mark it passed by hand." + ] + }, + { + "id": "PL-NEXT2", + "title": "Version 0.2.0-next.2, so npm carries PL-CRIT01", + "phase": "release", + "owner": "lane-a", + "depends_on": [ + "PL-CRIT01" + ], + "requirements": [ + "PL-R01", + "TB-R07" + ], + "scope": { + "allow": [ + "package.json", + "package-lock.json", + "CHANGELOG.md" + ] + }, + "description": "npm's @optitransfer/proofline is at 0.2.0-next.1 (latest and next, measured 2026-10-20), which predates PL-CRIT01. The version moves to 0.2.0-next.2 in package.json and package-lock.json (npm install --package-lock-only --ignore-scripts, empty user config), and the Unreleased heading in CHANGELOG.md becomes \"## 0.2.0-next.2 (2026-10-20)\". The owner publishes it from his terminal from the landed commit (npm publish --access public --tag next --auth-type=web) and records the registry's version and bin.", + "criteria": [ + { + "id": "PL-NEXT2-AC01", + "mandatory": true, + "statement": "package.json says 0.2.0-next.2", + "verifier": { + "type": "grep", + "file": "package.json", + "pattern": "\"version\":\\s*\"0\\.2\\.0-next\\.2\"", + "must": "present" + } + }, + { + "id": "PL-NEXT2-AC02", + "mandatory": true, + "statement": "the changelog dates 0.2.0-next.2 and keeps no Unreleased heading", + "verifier": { + "type": "command", + "cmd": "node -e \"const s=require('fs').readFileSync('CHANGELOG.md','utf8');process.exit(/^## 0\\.2\\.0-next\\.2 \\(2026-10-[0-9]{2}\\)$/m.test(s)&&!/^## Unreleased$/m.test(s)?0:1)\"", + "expect_exit": 0, + "timeout_s": 60 + } + }, + { + "id": "PL-NEXT2-AC03", + "mandatory": true, + "statement": "the packed tarball is 0.2.0-next.2, holds bin/proofline.mjs and the manifest names it as the proofline command", + "verifier": { + "type": "command", + "cmd": "sh -c \"npm pack --dry-run --json --ignore-scripts > /tmp/pl-pack.json && node -e \\\"const p=require('/tmp/pl-pack.json')[0];process.exit(p.version==='0.2.0-next.2'&&p.files.some(f=>f.path==='bin/proofline.mjs')&&require('./package.json').bin.proofline==='bin/proofline.mjs'?0:1)\\\"\"", + "expect_exit": 0, + "timeout_s": 300 + } + }, + { + "id": "PL-NEXT2-AC04", + "mandatory": true, + "statement": "Full suite exits 0, nothing hung", + "verifier": { + "type": "command", + "cmd": "npm test", + "expect_exit": 0, + "timeout_s": 1800 + } + }, + { + "id": "PL-NEXT2-AC05", + "mandatory": true, + "statement": "Changes stay in scope", + "verifier": { + "type": "scope" + } + }, + { + "id": "PL-NEXT2-AC06", + "mandatory": true, + "statement": "Committed and pushed to main", + "verifier": { + "type": "pushed" + } + } + ], + "stop_rules": [ + "If a criterion cannot pass without loosening a threshold, skipping or deleting a test, raising a timeout, widening the scope or editing the contract: STOP and report file:line and the measured value.", + "If the environment a criterion needs is missing (repository admin, npm owner login, Actions, a second machine): report BLOCKED naming the missing item; never mark it passed by hand." + ] } ], - "notes": "Each task carries the requirement ids it covers; next --owner hands out the tasks of one owner. Cross-repository needs are written into task text as an external dependency and reported BLOCKED when absent. PL-SELF stays: proofline verifies itself with its own contract. | 2026-10-17: 15 review tasks appended (COVERAGE.md maps every finding and open issue). Scope criterion added to PL-SELF, DG-PL so the contract passes its own validator. | 2026-10-17: PL-SETUP01 (setup pins the owner key) and PL-NEXT1 (0.2.0-next.1 carries the bin) appended.", + "notes": "Each task carries the requirement ids it covers; next --owner hands out the tasks of one owner. Cross-repository needs are written into task text as an external dependency and reported BLOCKED when absent. PL-SELF stays: proofline verifies itself with its own contract. | 2026-10-17: 15 review tasks appended (COVERAGE.md maps every finding and open issue). Scope criterion added to PL-SELF, DG-PL so the contract passes its own validator. | 2026-10-17: PL-SETUP01 (setup pins the owner key) and PL-NEXT1 (0.2.0-next.1 carries the bin) appended. | 2026-10-20: PL-CRIT01 (crit: claims for tasks proved by a command; leading (?i)/(?s) grep flags) and PL-NEXT2 (0.2.0-next.2) appended.", "ci": { "tasks": [ "PL-SELF" diff --git a/proofline.json.sig b/proofline.json.sig new file mode 100644 index 0000000..a3b520a --- /dev/null +++ b/proofline.json.sig @@ -0,0 +1 @@ +{"payloadType":"application/vnd.proofline.contract-sha256","payload":"OWQzM2VjOWJlZWUxYmQyMmViNmJjZjk5NzVhZmRkN2UyZGJkZTc0MmVjMDQwYjE4NzdkMTE3YzMyMTBlZDMzNA==","signatures":[{"keyid":"kb71a7ce9e7b76859","sig":"ckpux3El8CYsS74U7YeISFDbO7cKboLT8U/Luj/4XXoKUXNLV3h0AOSTwPMKMi/B0HfuL4EGv2e61LD2AnGADA=="}]} diff --git a/src/checks.mjs b/src/checks.mjs index 00c7018..9df08be 100644 --- a/src/checks.mjs +++ b/src/checks.mjs @@ -212,7 +212,9 @@ export const verifiers = { const f = path.join(ctx.root, v.file); if (!fs.existsSync(f)) return { status: 'fail', detail: `${v.file} not found` }; const txt = fs.readFileSync(f, 'utf8'); - const re = new RegExp(v.pattern, v.flags || 'm'); + // A leading (?i), (?s) or (?m) is how Python writes these flags; JavaScript refuses it as an invalid group. + const lead = String(v.pattern).match(/^\(\?([ims]+)\)/); + const re = new RegExp(lead ? v.pattern.slice(lead[0].length) : v.pattern, [...new Set((v.flags || 'm') + (lead ? lead[1] : ''))].join('')); const m = txt.match(re); const want = (v.must || 'present') === 'present'; if (!!m === want) return { status: 'pass', detail: `/${v.pattern}/ ${want ? 'present' : 'absent'} in ${v.file}`, evidence: m ? { line: txt.slice(0, m.index).split('\n').length, text: m[0] } : undefined }; diff --git a/src/claims.mjs b/src/claims.mjs index 499f45b..96c9547 100644 --- a/src/claims.mjs +++ b/src/claims.mjs @@ -4,6 +4,22 @@ import { git, onRemote } from './util.mjs'; const DONE = /^(done|complete(d)?|verified|pass(ed)?|fixed|finished|green)$/i; const HEDGE = /\b(should work|likely|probably|untested|not tested|later|follow[- ]?up|out of scope|defer(red)?|todo|next session|in a future)\b/i; +// `crit:` in the tests column names one of the task's own executed criteria instead of a file in the +// repo, for a task whose proof is a command run by the gate and not a test it adds (an intake or measurement task). +const CRIT = /^crit:(\S+)$/i; +const EXECUTED = ['command', 'metric', 'test_red_green', 'docs']; + +/** Why `crit:` does not prove the claim, or nothing: it must be a criterion of this task that ran a command in this gate and passed. */ +export function critIssues(id, task, res) { + const rows = res?.criteria; + if (!Array.isArray(rows)) return [`named criterion "crit:${id}" cannot be checked: the gate passed no criterion results`]; + const r = rows.find((x) => x.id === id); + if (!r) return [`named criterion "crit:${id}" is not a criterion of ${task} that this gate ran`]; + if (!EXECUTED.includes(r.type)) return [`named criterion "crit:${id}" is a ${r.type} check, which runs no test: name a command, metric, test_red_green or docs criterion`]; + if (r.status !== 'pass') return [`named criterion "crit:${id}" is ${String(r.status).toUpperCase()} in this gate, not PASS`]; + return []; +} + /** Lines: TASK-ID | status | commit | tests | measured vs target | missing */ export function parseClaims(text, taskIds) { const out = []; @@ -42,6 +58,8 @@ export function checkClaim(root, claim, res, { requirePushed = false, remote = ' if (claimedDone && !names.length) issues.push('claimed done without naming the test(s) that prove it'); for (const n of names) { const base = n.replace(/\s*\(.*\)$/, '').split(/\s+/)[0]; + const crit = base.match(CRIT); + if (crit) { issues.push(...critIssues(crit[1], claim.id, res)); continue; } if (!git(['grep', '-q', '-F', base], root).ok && !git(['ls-files', '--', `*${base}*`], root).out) issues.push(`named test "${base}" not found in the repo`); } const keys = Object.keys(res.measured || {}); diff --git a/src/engine.mjs b/src/engine.mjs index 56c8fe9..16d0843 100644 --- a/src/engine.mjs +++ b/src/engine.mjs @@ -193,7 +193,7 @@ export function verifyTask(root, taskId, opts = {}) { for (const a of t.criteria) if (a.verifier.type === 'metric') metricLabels[a.verifier.name || a.verifier.key] = a.verifier.labels || []; const claimResults = []; if (claims) { - const res = { decision, failing: [...blocked, ...failing], measured: ctx.measured, metricLabels }; + const res = { decision, failing: [...blocked, ...failing], measured: ctx.measured, metricLabels, criteria: results.map(({ id, type, status }) => ({ id, type, status })) }; const needPush = t.criteria.some((a) => a.verifier.type === 'pushed'); for (const cl of parseClaims(claims, [t.id])) claimResults.push(checkClaim(root, cl, res, { requirePushed: needPush, ...pushTarget(c, t) })); for (const cr of claimResults) if (!cr.ok) failing.push(...cr.issues.map((i) => `claim mismatch: ${i}`)); diff --git a/test/claims_crit.test.mjs b/test/claims_crit.test.mjs new file mode 100644 index 0000000..a894f78 --- /dev/null +++ b/test/claims_crit.test.mjs @@ -0,0 +1,65 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import * as P from '../src/index.mjs'; +import { repo, addTask, commitBeforeBase, sh } from './helpers.mjs'; + +// An intake task adds no test file: its proof is a command the gate runs. The tests column names that criterion. +function intake(extra = []) { + const r = repo(); + addTask(r, { + id: 'IN', title: 'intake', scope: [], + criteria: [ + { id: 'IN-AC01', statement: 'baseline inventory recorded', verifier: { type: 'command', cmd: 'node -e "process.exit(0)"' } }, + { id: 'IN-AC02', statement: 'package.json present', verifier: { type: 'file_exists', path: 'package.json' } }, + ...extra, + ], + }); + commitBeforeBase(r); + P.taskNext(r); + return { r, head: sh(r, 'rev-parse', '--short', 'HEAD') }; +} + +test('crit: naming a passing command criterion of the task proves the claim', () => { + const { r, head } = intake(); + const g = P.E.verifyTask(r, 'IN', { claims: `IN | done | ${head} | crit:IN-AC01 | - | -` }); + assert.equal(g.decision, 'PASS', g.failing.join('\n')); + assert.equal(g.claims[0].ok, true); +}); + +test('the same claim with the criterion id as a bare name still fails the repo lookup', () => { + const { r, head } = intake(); + const g = P.E.verifyTask(r, 'IN', { claims: `IN | done | ${head} | IN-AC01-proof | - | -` }); + assert.equal(g.decision, 'FAIL'); + assert.match(g.failing.join('\n'), /named test "IN-AC01-proof" not found in the repo/); +}); + +test('crit: is refused for an unknown id, another task, a check that runs nothing, and a failing criterion', () => { + const { r, head } = intake([{ id: 'IN-AC03', statement: 'second run', mandatory: false, verifier: { type: 'command', cmd: 'node -e "process.exit(1)"' } }]); + addTask(r, { id: 'OTHER', title: 'o', criteria: [{ id: 'OT-AC01', statement: 'o', verifier: { type: 'command', cmd: 'node -e "process.exit(0)"' } }] }); + const cases = [ + ['crit:IN-AC99', /"crit:IN-AC99" is not a criterion of IN that this gate ran/], + ['crit:OT-AC01', /"crit:OT-AC01" is not a criterion of IN that this gate ran/], + ['crit:IN-AC02', /"crit:IN-AC02" is a file_exists check, which runs no test/], + ['crit:IN-AC03', /"crit:IN-AC03" is FAIL in this gate, not PASS/], + ]; + for (const [name, re] of cases) { + const g = P.E.verifyTask(r, 'IN', { claims: `IN | done | ${head} | ${name} | - | -` }); + assert.equal(g.decision, g.blocked.length ? 'BLOCKED' : 'FAIL', name); + assert.match(g.claims[0].issues.join('\n'), re, name); + } +}); + +test('crit: names mix with file names, and every one must hold', () => { + const { r, head } = intake(); + const ok = P.E.verifyTask(r, 'IN', { claims: `IN | done | ${head} | crit:IN-AC01, package.json | - | -` }); + assert.equal(ok.claims[0].ok, true, ok.claims[0].issues.join('\n')); + const bad = P.E.verifyTask(r, 'IN', { claims: `IN | done | ${head} | crit:IN-AC01, crit:IN-AC02 | - | -` }); + assert.equal(bad.decision, 'FAIL'); + assert.equal(bad.claims[0].issues.length, 1); +}); + +test('dash, n/a and none still count as naming no test', () => { + const { r, head } = intake(); + const g = P.E.verifyTask(r, 'IN', { claims: `IN | done | ${head} | n/a | - | -` }); + assert.match(g.failing.join('\n'), /without naming the test\(s\) that prove it/); +}); diff --git a/test/grep_inline_flags.test.mjs b/test/grep_inline_flags.test.mjs new file mode 100644 index 0000000..20a7365 --- /dev/null +++ b/test/grep_inline_flags.test.mjs @@ -0,0 +1,28 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import * as P from '../src/index.mjs'; +import { repo, addTask, commitBeforeBase } from './helpers.mjs'; + +// Contracts shared with a Python verifier write flags inline. The same pattern must mean the same thing here. +test('a grep pattern that starts with (?i) or (?s) runs with those flags instead of crashing', () => { + const r = repo({ files: { 'README.md': 'An EXPERIMENTAL suite.\n', 'docs/R.md': 'my proposal\n| M1 |\n| M6 |\n' } }); + addTask(r, { + id: 'G', title: 'g', criteria: [ + { id: 'G-AC01', statement: 'i', verifier: { type: 'grep', file: 'README.md', pattern: '(?i)experimental', must: 'present' } }, + { id: 'G-AC02', statement: 's', verifier: { type: 'grep', file: 'docs/R.md', pattern: '(?s)my proposal.*\\| M1 \\|.*\\| M6 \\|', must: 'present' } }, + { id: 'G-AC03', statement: 'absent', verifier: { type: 'grep', file: 'README.md', pattern: '(?i)stable', must: 'absent' } }, + ], + }); + commitBeforeBase(r); + P.taskNext(r); + const g = P.E.verifyTask(r, 'G'); + for (const id of ['G-AC01', 'G-AC02', 'G-AC03']) assert.equal(g.results.find((x) => x.id === id).status, 'pass', `${id}: ${JSON.stringify(g.results.find((x) => x.id === id))}`); +}); + +test('without the inline flag the match stays case-sensitive', () => { + const r = repo({ files: { 'README.md': 'An EXPERIMENTAL suite.\n' } }); + addTask(r, { id: 'G', title: 'g', criteria: [{ id: 'G-AC01', statement: 'i', verifier: { type: 'grep', file: 'README.md', pattern: 'experimental', must: 'present' } }] }); + commitBeforeBase(r); + P.taskNext(r); + assert.equal(P.E.verifyTask(r, 'G').results[0].status, 'fail'); +});