From 3c75e1b8559c632ecb3265119dfa5e5750ca40e9 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Tue, 4 Aug 2026 18:07:17 +0100 Subject: [PATCH 1/2] fix: stop spawn stamping live registry entries into the templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `empty_body()` implemented the spec's EMPTY action as "keep line 1 of the live file" rather than "keep the header line". `planned.md` and `ideas.md` carry no H1 at all, so their first line is a registry entry — and spawn stamped it into the public fresh-slate templates, violating the privacy invariant in docs/pyautobrain/spawn_spec.md. The `ideas.md` line reached PyAutoMind-template in the 2026-07-27 sync (3424dba1); `planned.md`'s `## rhayes-...` slug would have shipped on the next regenerate. The canary scan reported clean throughout because CANARY_TOKENS held dataset names only — the spec's own example list names `Nightingale`, but no name token was ever implemented. - empty_body() now generates its header and never opens the source, so no future edit to a live registry file can change what a template ships. Named files use EMPTY_TITLES; glob-matched bibliography files get a generated comment header (spec rule 2). An unmapped EMPTY file raises, same doctrine as UNMATCHED. - CANARY_TOKENS gains `nightingale` and `rhayes`, with a narrow per-file, per-token CANARY_EXEMPT for LICENSE (copyright attribution) and for the two files that define/exercise the tokens. - New tests/ + spec rule 1b (KEEP) so the privacy test travels with the generator it guards. - spawn_drift.yml runs the invariant on PRs touching spawn; the drift job is skipped on PRs since it diffs published templates against main. A heading-shape test would not have caught this: `## rhayes-audit-validation-phases-2-4` is a valid `##` heading. Refs #118 Co-Authored-By: Claude Opus 5 --- .github/workflows/spawn_drift.yml | 28 +++++ docs/pyautobrain/spawn_spec.md | 3 +- scripts/spawn.py | 97 ++++++++++++--- tests/test_spawn_privacy.py | 192 ++++++++++++++++++++++++++++++ 4 files changed, 304 insertions(+), 16 deletions(-) create mode 100644 tests/test_spawn_privacy.py diff --git a/.github/workflows/spawn_drift.yml b/.github/workflows/spawn_drift.yml index c0089bce..baa243f7 100644 --- a/.github/workflows/spawn_drift.yml +++ b/.github/workflows/spawn_drift.yml @@ -10,13 +10,41 @@ name: Spawn Drift on: schedule: - cron: "17 6 * * 1" # weekly, Monday 06:17 UTC + pull_request: + paths: + - "scripts/spawn.py" + - "tests/test_spawn_privacy.py" + - "docs/pyautobrain/spawn_spec.md" + - ".github/workflows/spawn_drift.yml" workflow_dispatch: permissions: contents: read jobs: + # The privacy invariant (spawn_spec.md) is a property of the GENERATOR, so it + # is checked on every PR that touches spawn, not only on the weekly drift run. + # Hermetic — synthetic inputs only, no clones needed. + privacy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install pytest + - name: spawn privacy invariant + run: python3 -m pytest tests/test_spawn_privacy.py -q + drift: + # A drift run that regenerates from a leaking generator would "fix" the + # templates by publishing instance content — so the invariant gates it. + needs: privacy + # PR runs exist to exercise the privacy job. Drift compares the PUBLISHED + # templates against Mind's main (this job clones main, not the PR head), so + # on a PR it would report unrelated pre-existing drift and go red on work + # that did not cause it. + if: github.event_name != 'pull_request' runs-on: ubuntu-latest steps: - name: Checkout the live repos + published templates diff --git a/docs/pyautobrain/spawn_spec.md b/docs/pyautobrain/spawn_spec.md index dd29a6a3..c479dcbb 100644 --- a/docs/pyautobrain/spawn_spec.md +++ b/docs/pyautobrain/spawn_spec.md @@ -32,11 +32,12 @@ deliberately, never silently shipped into a template. | # | Pattern | Action | |---|---------|--------| | 1 | `scripts/**` | KEEP verbatim (generic tooling: repos_sync, prompt_sync, status, spawn itself) | +| 1b | `tests/**` | KEEP verbatim (generator machinery, same class as rule 1; Memory's table already does this). The privacy test in particular must travel with the generator it guards — a spawned org that inherits `spawn.py` without `test_spawn_privacy.py` can reintroduce the rule-5 leak (issue #118) silently | | 2 | `REFERENCE.md`, `AGENTS.md`, `CLAUDE.md`, `LICENSE`, `ROUTING.md`, `.gitignore` | KEEP verbatim | | 3 | `README.md` | KEEP verbatim (already generic post-Phase-1) | | 3b | `AI_POLICY.md`, `CONTRIBUTING.md` | KEEP with owner substitution — both are org-wide pointer docs: generic prose that names the owning org and links the canonical copy inside that org's `PyAutoScientist`. They take the same `PyAutoLabs` → `YOURORG` substitution as rule 9 rather than a verbatim KEEP; verbatim would stamp a literal "Contributing to PyAutoLabs" into a template spawned for another org | | 4 | `repos.yaml` | SUBSTITUTE → the **template body map**: the five organ rows kept with `github:` owner replaced by `YOURORG`; all live satellite rows replaced by the PyAutoProject family rows (`PyAutoProject` category `library`, `autoproject_workspace` category `workspace`, `autoproject_workspace_test` category `workspace_test`) + a commented-out `autoproject_assistant` row ("uncomment when the clone agent seeds it") | -| 5 | `active.md`, `planned.md`, `parked.md`, `condemned.md`, `ideas.md`, `queue.md`, `autonomy_log.md` | EMPTY → header line + schema pointer comment only (e.g. `# Active Tasks` + ``); autonomy_log keeps its schema header rows | +| 5 | `active.md`, `planned.md`, `parked.md`, `condemned.md`, `ideas.md`, `queue.md`, `autonomy_log.md` | EMPTY → header line + schema pointer comment only (e.g. `# Active Tasks` + ``); autonomy_log keeps its schema header rows. **The header is GENERATED, never read from the live file** — `planned.md` and `ideas.md` carry no H1 at all, so "keep line 1" yields a registry entry (issue #118), and a heading-shape test cannot save it either: `## rhayes-audit-validation-phases-2-4` is a valid `##` heading. Each file's title lives in `EMPTY_TITLES`; an EMPTY-ruled file with no entry is an UNMATCHED-class human decision, not a guess | | 6 | `draft/**` (the work-type dirs `feature/ bug/ refactor/ docs/ test/ release/ maintenance/ research/ experiment/ triage/` now live under `draft/`) | SKELETON → keep a single `draft/.gitkeep`; drop all draft prompts and their work-type/target subdirs (a fresh Mind starts with an empty `draft/`; intake recreates the work-type subdirs on demand) | | 6b | `complete/AGENTS.md` | KEEP verbatim (the finished-work archive **schema** is template content; matched before rule 7's `complete/*` DROP) | | 7 | `active/ complete/ z_features/ z_vault/ autoprompt/` + instance reference docs (`docs/**` now holds only reference material like `spawn_spec.md`) + instance root docs (`dashboard.md`, `overview.md`) + legacy pre-migration prompt dirs (`autolens/`) | DROP (lifecycle records + instance content) | diff --git a/scripts/spawn.py b/scripts/spawn.py index 2952968f..362999b1 100644 --- a/scripts/spawn.py +++ b/scripts/spawn.py @@ -42,6 +42,10 @@ MIND_RULES = [ ("scripts/*", "KEEP"), + # Generator machinery, same class as scripts/ (spec rule 1b; MEMORY_RULES + # already keeps tests/). The privacy test must travel with the generator it + # guards, or a spawned org can reintroduce the rule-5 leak (#118) silently. + ("tests/*", "KEEP"), ("REFERENCE.md", "KEEP"), ("AGENTS.md", "KEEP"), ("CLAUDE.md", "KEEP"), ("LICENSE", "KEEP"), ("ROUTING.md", "KEEP"), (".gitignore", "KEEP"), @@ -99,7 +103,37 @@ # Chosen to be absent from every KEEP-verbatim file (verified at run time — # the scan covers the whole output tree, so a canary in a kept file fails # the run and forces the list or the rules to be reconsidered). -CANARY_TOKENS = ("slacs", "b1938", "cosmos_web_ring", "smbh_binary", "arctic") +# Dataset names catch leaked science content; person names catch leaked task +# slugs and prompt lines (the spec's example list names `Nightingale`). +CANARY_TOKENS = ( + "slacs", "b1938", "cosmos_web_ring", "smbh_binary", "arctic", + "nightingale", "rhayes", +) + +# Titles for EMPTY-ruled files, keyed by basename (spawn_spec.md rules 5 + 6: +# "header line + schema pointer comment only"). These are GENERATED, never read +# from the live file: `planned.md` and `ideas.md` carry no H1 at all, so their +# first line is a registry entry, and a heading-shape test cannot save us +# either — `## rhayes-audit-validation-phases-2-4` is a valid `##` heading. +# Copying any source byte here breaks the privacy invariant (spawn_spec.md). +EMPTY_TITLES = { + "active.md": "# Active Tasks", + "planned.md": "# Planned", + "parked.md": "# Parked tasks", + "condemned.md": "# Condemned material", + "ideas.md": "# Ideas", + "queue.md": "# Queue", + "reading-queue.md": "# Reading queue", +} + +# Generated header comments for EMPTY files matched by a glob rather than by +# name (Memory's `bibliography/*` — arbitrary filenames, so no title map). +# spawn_spec.md rule 2 already specifies a generated header comment here. +EMPTY_COMMENTS = { + ".bib": "% Canonical BibTeX metadata — populated by your literature.", + ".yaml": "# Populated by your literature.", + ".yml": "# Populated by your literature.", +} # -------------------------------------------------------------------------- # Generated assets @@ -336,15 +370,32 @@ def match_rule(rel, rules): def empty_body(src): - try: - first = src.read_text(errors="replace").splitlines()[0] - except IndexError: - first = "" - if src.suffix in {".yaml", ".yml"}: - # YAML consumers parse every non-# line — an HTML comment would read - # as content (e.g. a bibkey alias with a missing target). - return first + "\n\n# emptied by spawn; schema: REFERENCE.md\n" - return first + "\n\n\n" + """Generate the EMPTY body for `src` WITHOUT reading its contents. + + The source is never opened: an EMPTY output is a generated title plus a + schema pointer, so no live registry entry, idea line or bibliography entry + can reach a template (the spawn_spec.md privacy invariant). + """ + header = EMPTY_TITLES.get(src.name) + if header is None: + header = EMPTY_COMMENTS.get(src.suffix) + if header is None: + # Same doctrine as UNMATCHED: a new EMPTY file class is a human + # decision — add it to the spec's tables and mirror it here. Guessing + # a header from the live file is what leaked instance content before. + raise SystemExit( + f"spawn: EMPTY file '{src.name}' has no generated header.\n" + f" Add it to EMPTY_TITLES (named registry files) or EMPTY_COMMENTS\n" + f" (glob-matched files), updating docs/pyautobrain/spawn_spec.md first." + ) + if src.suffix in {".yaml", ".yml", ".bib"}: + # YAML/BibTeX consumers parse every non-comment line — an HTML comment + # would read as content (e.g. a bibkey alias with a missing target). + marker = "# emptied by spawn; schema: REFERENCE.md" + if src.suffix == ".bib": + marker = "% emptied by spawn; schema: REFERENCE.md" + return header + "\n\n" + marker + "\n" + return header + "\n\n\n" def autonomy_log_body(src): @@ -427,19 +478,35 @@ def generate_memory(memory_root, out_dir): return warns +# Paths where a specific canary token is legitimate rather than leaked. +# Deliberately narrow: each entry names the exact file AND the exact tokens +# excused there, so a new leak elsewhere still fails the scan. +CANARY_EXEMPT = { + # spawn.py DEFINES CANARY_TOKENS; its list is generator machinery. + "scripts/spawn.py": set(CANARY_TOKENS), + # The privacy test EXERCISES the tokens — it must feed hostile, leak-shaped + # content to the generator to prove it is stripped. Those fixtures are the + # test's whole point, not leaked instance content. + "tests/test_spawn_privacy.py": set(CANARY_TOKENS), + # The licence attributes copyright to a named human — that is the point of + # a licence, not leaked instance content. + "LICENSE": {"nightingale"}, +} + + def canary_scan(out_dir): hits = [] for path in sorted(out_dir.rglob("*")): if not path.is_file(): continue - # spawn.py itself defines CANARY_TOKENS; its token list is generator - # machinery, not instance content, so exclude it from its own scan. - if path.relative_to(out_dir).as_posix() == "scripts/spawn.py": - continue + rel = path.relative_to(out_dir).as_posix() + exempt = CANARY_EXEMPT.get(rel, set()) text = path.read_text(errors="replace").lower() for token in CANARY_TOKENS: + if token in exempt: + continue if token in text: - hits.append(f"{path.relative_to(out_dir)}: '{token}'") + hits.append(f"{rel}: '{token}'") return hits diff --git a/tests/test_spawn_privacy.py b/tests/test_spawn_privacy.py new file mode 100644 index 00000000..fb65cf78 --- /dev/null +++ b/tests/test_spawn_privacy.py @@ -0,0 +1,192 @@ +"""The privacy invariant required by docs/pyautobrain/spawn_spec.md. + + "no live wiki page, bibliography entry, reading-queue line, prompt, or + registry entry may ever appear in a template output. The implementation + must include a test asserting the generated tree contains none of a + canary list of live-content markers." + +The regression these lock down (issue #118): `empty_body()` used to return +line 1 of the live file. `planned.md` and `ideas.md` carry no H1 at all, so +their first line is a registry entry, and spawn stamped it into the public +fresh-slate templates. A heading-shape test would NOT have caught it — +`## rhayes-audit-validation-phases-2-4` is a valid `##` heading. + +Hermetic: everything below runs on synthetic inputs, so the suite needs no +live PyAutoMind/PyAutoMemory checkout. +""" + +import importlib.util +from pathlib import Path + +import pytest + +SPAWN_PY = Path(__file__).resolve().parents[1] / "scripts" / "spawn.py" + +_spec = importlib.util.spec_from_file_location("spawn", SPAWN_PY) +spawn = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(spawn) + + +# Content shaped like the real leaks: a registry H2 slug, a raw idea bullet, +# and an instance-flavoured H1 that a heading test would have waved through. +HOSTILE_BODIES = [ + "## rhayes-audit-validation-phases-2-4\n- issue: https://example/1\n", + "- lens_calc_zero_contour_jax autolens workspace guide.\n", + "# Pytree variant queue\n\n1. slacs0946 refit\n", + "", +] + + +# Read lazily so the suite still COLLECTS against a spawn.py that predates +# EMPTY_TITLES. Otherwise the decisive regression tests below never run — they +# would be masked by a collection error, which proves only that the API moved. +EMPTY_TITLES = getattr(spawn, "EMPTY_TITLES", {}) + + +def test_planned_md_never_carries_a_registry_entry(tmp_path): + """The exact issue #118 regression, in the smallest form that catches it. + + Depends on nothing but `empty_body`, so it fails with a clean assertion + against the pre-fix implementation rather than erroring at import. + """ + src = tmp_path / "planned.md" + src.write_text("## rhayes-audit-validation-phases-2-4\n- issue: https://x/1\n") + assert "rhayes" not in spawn.empty_body(src) + + +def test_ideas_md_never_carries_an_idea_line(tmp_path): + """The leak that actually reached the published template.""" + src = tmp_path / "ideas.md" + src.write_text("- lens_calc_zero_contour_jax autolens workspace guide.\n") + assert "lens_calc_zero_contour_jax" not in spawn.empty_body(src) + + +@pytest.mark.parametrize("name,title", sorted(EMPTY_TITLES.items())) +@pytest.mark.parametrize("body", HOSTILE_BODIES) +def test_empty_output_contains_no_source_bytes(tmp_path, name, title, body): + """Every EMPTY output is exactly the generated title + marker.""" + src = tmp_path / name + src.write_text(body) + + out = spawn.empty_body(src) + + assert out.startswith(title + "\n\n"), f"{name} lost its generated title" + # The only lines are the title, a blank, and the schema-pointer marker. + lines = [ln for ln in out.splitlines() if ln.strip()] + assert len(lines) == 2, f"{name} emitted unexpected lines: {lines!r}" + assert lines[0] == title + + # Nothing from the source survived. Checked line-wise so a title that + # legitimately shares a word with the body cannot mask a real leak. + for src_line in body.splitlines(): + if src_line.strip() and src_line.strip() != title: + assert src_line not in out, f"{name} leaked source line: {src_line!r}" + + +@pytest.mark.parametrize("name", sorted(EMPTY_TITLES)) +def test_empty_body_never_opens_the_source(tmp_path, name): + """The source need not even be readable — EMPTY is purely generated. + + Stronger than diffing output: if the file is never opened, no future edit + to a live registry file can change what a template ships. + """ + missing = tmp_path / name # deliberately not created + assert spawn.empty_body(missing).startswith(EMPTY_TITLES[name]) + + +def test_unmapped_empty_file_is_a_human_decision(tmp_path): + """A new EMPTY file class must fail loudly, never be guessed at.""" + src = tmp_path / "brand_new_ledger.md" + src.write_text("## some-live-task\n") + with pytest.raises(SystemExit) as excinfo: + spawn.empty_body(src) + assert "brand_new_ledger.md" in str(excinfo.value) + + +@pytest.mark.parametrize( + "filename,lead", + [("pyautomemory.bib", "%"), ("bibkey_aliases.yaml", "#"), ("x.yml", "#")], +) +def test_glob_matched_empty_files_get_comment_headers(tmp_path, filename, lead): + """spawn_spec.md rule 2: bibliography files keep a generated header comment. + + The header and marker must both be comments — a YAML/BibTeX consumer parses + every non-comment line, so an HTML comment would read as content. + """ + src = tmp_path / filename + src.write_text("@article{Nightingale2018, title={SLACS lens}}\n") + + out = spawn.empty_body(src) + + assert out.startswith(lead) + for line in [ln for ln in out.splitlines() if ln.strip()]: + assert line.startswith(lead), f"non-comment line in {filename}: {line!r}" + assert "Nightingale2018" not in out + assert "SLACS" not in out + + +def test_canary_scan_catches_a_leaked_registry_slug(tmp_path): + """The scan must flag a task slug, not just a science dataset name.""" + (tmp_path / "planned.md").write_text("## rhayes-audit-validation-phases-2-4\n") + hits = spawn.canary_scan(tmp_path) + assert any("rhayes" in h for h in hits), hits + + +def test_canary_scan_catches_dataset_tokens(tmp_path): + (tmp_path / "notes.md").write_text("refit of SLACS0946 with arctic clocking\n") + hits = spawn.canary_scan(tmp_path) + assert any("slacs" in h for h in hits) + assert any("arctic" in h for h in hits) + + +def test_licence_may_name_its_copyright_holder(tmp_path): + """Attribution in a licence is the point of a licence, not a leak.""" + (tmp_path / "LICENSE").write_text("MIT\n (James Nightingale / Jammy2211).\n") + assert spawn.canary_scan(tmp_path) == [] + + +def test_the_licence_exemption_is_narrow(tmp_path): + """The exemption is per-file AND per-token — it must not become a hole.""" + # Same name, different file: still a leak. + (tmp_path / "AGENTS.md").write_text("ask James Nightingale about this\n") + assert any("nightingale" in h for h in spawn.canary_scan(tmp_path)) + + # Same file, different token: still a leak. + (tmp_path / "AGENTS.md").unlink() + (tmp_path / "LICENSE").write_text("MIT (James Nightingale) — slacs0946\n") + hits = spawn.canary_scan(tmp_path) + assert any("slacs" in h for h in hits) + assert not any("nightingale" in h for h in hits) + + +def test_spawn_py_is_exempt_from_its_own_token_list(tmp_path): + """spawn.py DEFINES the tokens; that definition is not leaked content.""" + scripts = tmp_path / "scripts" + scripts.mkdir() + (scripts / "spawn.py").write_text(f"CANARY_TOKENS = {spawn.CANARY_TOKENS!r}\n") + assert spawn.canary_scan(tmp_path) == [] + + +def test_every_empty_rule_is_covered_by_a_generated_header(): + """No EMPTY rule may exist that empty_body() cannot serve. + + Guards the seam between the partition tables and the header maps: adding + an EMPTY rule without a title is caught here rather than at spawn time. + """ + empty_globs = [ + pattern + for pattern, action in (spawn.MIND_RULES + spawn.MEMORY_RULES) + if action == "EMPTY" + ] + assert empty_globs, "no EMPTY rules found — did the tables move?" + + for pattern in empty_globs: + if "*" in pattern: + # Glob rules are served by suffix; every suffix they can match + # must have a generated comment header. + continue + name = Path(pattern).name + assert name in spawn.EMPTY_TITLES, ( + f"EMPTY rule '{pattern}' has no EMPTY_TITLES entry — " + f"spawn would abort on it" + ) From cb2f60c9de2570899dfb9d624ef2fc420491b390 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Tue, 4 Aug 2026 18:22:17 +0100 Subject: [PATCH 2/2] fix: stop the privacy test itself leaking into the public template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent review (Codex) caught that the first pass reintroduced the exact leak it removed, through a different door. `tests/**` and `scripts/spawn.py` are both KEEP-copied verbatim into the public template. The test file quoted the real live `planned.md` slug and `ideas.md` line as fixtures, and a spawn.py comment quoted the slug too — so the generated template still shipped that content. The canary scan reported clean only because the first pass had exempted those two files wholesale. Verified against a generated tree: the strings were present in gen/PyAutoMind-template/{tests/test_spawn_privacy.py,scripts/spawn.py}. - Every fixture is now fictional, and every canary token is derived from CANARY_TOKENS at run time instead of spelled out, so the test file scans clean on its own merits. Its wholesale exemption is removed; only scripts/spawn.py keeps one, because the token list has to live somewhere. - New test_only_spawn_py_is_exempt_wholesale + test_this_test_file_is_not_exempt so the exemption map cannot quietly grow that hole back. - New end-to-end tests drive generate_all() over a synthetic Mind/Memory and assert no live marker reaches the tree. The old suite unit-tested empty_body() alone, so flipping a rule to KEEP or swapping the dispatcher for copy2 would have stayed green — which is exactly how the above slipped through. - EMPTY_TITLES is keyed by repo-relative path, not basename, so a glob-matched file sharing a root ledger's name (e.g. bibliography/active.md) takes the unmapped-file SystemExit instead of inheriting that ledger's title. - test_every_empty_rule_is_covered checked suffix coverage instead of `continue`-ing past every glob, which made it vacuous. Refs #118 Co-Authored-By: Claude Opus 5 --- scripts/spawn.py | 47 +++--- tests/test_spawn_privacy.py | 294 ++++++++++++++++++++++++++++-------- 2 files changed, 259 insertions(+), 82 deletions(-) diff --git a/scripts/spawn.py b/scripts/spawn.py index 362999b1..50d5fb72 100644 --- a/scripts/spawn.py +++ b/scripts/spawn.py @@ -110,12 +110,17 @@ "nightingale", "rhayes", ) -# Titles for EMPTY-ruled files, keyed by basename (spawn_spec.md rules 5 + 6: -# "header line + schema pointer comment only"). These are GENERATED, never read -# from the live file: `planned.md` and `ideas.md` carry no H1 at all, so their -# first line is a registry entry, and a heading-shape test cannot save us -# either — `## rhayes-audit-validation-phases-2-4` is a valid `##` heading. -# Copying any source byte here breaks the privacy invariant (spawn_spec.md). +# Titles for EMPTY-ruled files, keyed by their path RELATIVE TO THE REPO ROOT +# (spawn_spec.md rules 5 + 6: "header line + schema pointer comment only"). +# These are GENERATED, never read from the live file: some registry files carry +# no H1 at all, so their first line is a live registry entry, and a +# heading-shape test cannot save us either — a task slug written as an H2 is a +# structurally valid heading. Copying any source byte here breaks the privacy +# invariant (spawn_spec.md). +# +# Keyed by relative path, not basename, so a glob-matched file that merely +# SHARES a name (e.g. Memory's `bibliography/active.md`, caught by the broad +# `bibliography/*` EMPTY rule) does not silently inherit a root file's title. EMPTY_TITLES = { "active.md": "# Active Tasks", "planned.md": "# Planned", @@ -369,14 +374,19 @@ def match_rule(rel, rules): return None, None -def empty_body(src): - """Generate the EMPTY body for `src` WITHOUT reading its contents. +def empty_body(src, rel=None): + """Generate the EMPTY body for `rel` WITHOUT reading the source. The source is never opened: an EMPTY output is a generated title plus a schema pointer, so no live registry entry, idea line or bibliography entry can reach a template (the spawn_spec.md privacy invariant). + + `rel` is the path relative to the repo root and is what EMPTY_TITLES is + keyed on. It defaults to the basename only so direct callers (tests) stay + ergonomic; the generators always pass the real relative path. """ - header = EMPTY_TITLES.get(src.name) + key = Path(rel).as_posix() if rel is not None else src.name + header = EMPTY_TITLES.get(key) if header is None: header = EMPTY_COMMENTS.get(src.suffix) if header is None: @@ -384,7 +394,7 @@ def empty_body(src): # decision — add it to the spec's tables and mirror it here. Guessing # a header from the live file is what leaked instance content before. raise SystemExit( - f"spawn: EMPTY file '{src.name}' has no generated header.\n" + f"spawn: EMPTY file '{key}' has no generated header.\n" f" Add it to EMPTY_TITLES (named registry files) or EMPTY_COMMENTS\n" f" (glob-matched files), updating docs/pyautobrain/spawn_spec.md first." ) @@ -441,7 +451,7 @@ def generate_mind(mind_root, out_dir): elif action == "KEEP_SUB": dest.write_text(substitute_owner(src.read_text(errors="replace"))) elif action == "EMPTY": - dest.write_text(empty_body(src)) + dest.write_text(empty_body(src, rel)) elif action == "SPECIAL:autonomy_log": dest.write_text(autonomy_log_body(src)) elif action == "SPECIAL:body_map": @@ -464,7 +474,7 @@ def generate_memory(memory_root, out_dir): elif action == "KEEP_SUB": dest.write_text(substitute_owner(src.read_text(errors="replace"))) elif action == "EMPTY": - dest.write_text(empty_body(src)) + dest.write_text(empty_body(src, rel)) elif action == "SPECIAL:memory_index": dest.write_text(MEMORY_INDEX_TEMPLATE) elif action == "SPECIAL:memory_readme": @@ -482,12 +492,15 @@ def generate_memory(memory_root, out_dir): # Deliberately narrow: each entry names the exact file AND the exact tokens # excused there, so a new leak elsewhere still fails the scan. CANARY_EXEMPT = { - # spawn.py DEFINES CANARY_TOKENS; its list is generator machinery. + # spawn.py DEFINES CANARY_TOKENS; that literal list is generator machinery. + # This is the ONLY unavoidable exemption: the token list has to exist + # somewhere, and this is where. Everything else must earn its place — + # `tests/` deliberately has NO entry here. The privacy test derives every + # token from CANARY_TOKENS at run time and uses fictional fixtures, so it + # scans clean on its own merits. Exempting it instead would let the test + # smuggle the very strings it exists to keep out (both spawn.py and + # tests/ are KEEP-copied verbatim into the public template). "scripts/spawn.py": set(CANARY_TOKENS), - # The privacy test EXERCISES the tokens — it must feed hostile, leak-shaped - # content to the generator to prove it is stripped. Those fixtures are the - # test's whole point, not leaked instance content. - "tests/test_spawn_privacy.py": set(CANARY_TOKENS), # The licence attributes copyright to a named human — that is the point of # a licence, not leaked instance content. "LICENSE": {"nightingale"}, diff --git a/tests/test_spawn_privacy.py b/tests/test_spawn_privacy.py index fb65cf78..090734aa 100644 --- a/tests/test_spawn_privacy.py +++ b/tests/test_spawn_privacy.py @@ -6,16 +6,29 @@ canary list of live-content markers." The regression these lock down (issue #118): `empty_body()` used to return -line 1 of the live file. `planned.md` and `ideas.md` carry no H1 at all, so -their first line is a registry entry, and spawn stamped it into the public -fresh-slate templates. A heading-shape test would NOT have caught it — -`## rhayes-audit-validation-phases-2-4` is a valid `##` heading. - -Hermetic: everything below runs on synthetic inputs, so the suite needs no -live PyAutoMind/PyAutoMemory checkout. +line 1 of the live file. Some registry ledgers carry no H1 at all, so their +first line is a live registry entry, and spawn stamped it into the public +fresh-slate templates. A heading-shape test would NOT have caught it — a task +slug written as an H2 is a structurally valid heading. + +TWO RULES FOR THIS FILE, both learned the hard way: + +1. NO REAL LIVE STRINGS. `tests/**` is KEEP-copied verbatim into the public + template, so a fixture quoting a real registry slug republishes the very + content this suite exists to keep out. Every fixture below is fictional, + and every canary token is derived from `spawn.CANARY_TOKENS` at run time + rather than spelled out. This file must scan CLEAN with no exemption. +2. TEST THE GENERATED TREE, not just the helpers. Unit-testing `empty_body()` + alone would stay green if a rule were flipped to KEEP or the dispatcher + swapped for `copy2` — the leak would simply move. `test_generated_tree_*` + below drives the real generators end to end. + +Hermetic: everything runs on a synthetic Mind/Memory built in tmp_path, so the +suite needs no live checkout. """ import importlib.util +import subprocess from pathlib import Path import pytest @@ -27,38 +40,42 @@ _spec.loader.exec_module(spawn) -# Content shaped like the real leaks: a registry H2 slug, a raw idea bullet, -# and an instance-flavoured H1 that a heading test would have waved through. +# Derived, never spelled out — see rule 1 in the module docstring. +A_TOKEN = spawn.CANARY_TOKENS[0] +NAME_TOKEN = spawn.CANARY_TOKENS[-1] + +# Fictional content shaped like the real leaks: a registry H2 slug, a raw idea +# bullet, and an instance-flavoured H1 that a heading test would wave through. HOSTILE_BODIES = [ - "## rhayes-audit-validation-phases-2-4\n- issue: https://example/1\n", - "- lens_calc_zero_contour_jax autolens workspace guide.\n", - "# Pytree variant queue\n\n1. slacs0946 refit\n", + "## example-task-slug-phases-2-4\n- issue: https://example.invalid/1\n", + "- example_module_feature workspace guide.\n", + "# Some Instance-Specific Ledger Name\n\n1. a live entry\n", "", ] - # Read lazily so the suite still COLLECTS against a spawn.py that predates # EMPTY_TITLES. Otherwise the decisive regression tests below never run — they # would be masked by a collection error, which proves only that the API moved. EMPTY_TITLES = getattr(spawn, "EMPTY_TITLES", {}) -def test_planned_md_never_carries_a_registry_entry(tmp_path): - """The exact issue #118 regression, in the smallest form that catches it. +# -------------------------------------------------------------------------- +# The issue #118 regression, in the smallest forms that catch it +# -------------------------------------------------------------------------- - Depends on nothing but `empty_body`, so it fails with a clean assertion - against the pre-fix implementation rather than erroring at import. - """ + +def test_registry_ledger_never_carries_its_first_entry(tmp_path): + """Depends on nothing but `empty_body`, so it fails with a clean assertion + against the pre-fix implementation rather than erroring at import.""" src = tmp_path / "planned.md" - src.write_text("## rhayes-audit-validation-phases-2-4\n- issue: https://x/1\n") - assert "rhayes" not in spawn.empty_body(src) + src.write_text("## example-task-slug-phases-2-4\n- issue: https://example.invalid/1\n") + assert "example-task-slug" not in spawn.empty_body(src) -def test_ideas_md_never_carries_an_idea_line(tmp_path): - """The leak that actually reached the published template.""" +def test_ideas_ledger_never_carries_an_idea_line(tmp_path): src = tmp_path / "ideas.md" - src.write_text("- lens_calc_zero_contour_jax autolens workspace guide.\n") - assert "lens_calc_zero_contour_jax" not in spawn.empty_body(src) + src.write_text("- example_module_feature workspace guide.\n") + assert "example_module_feature" not in spawn.empty_body(src) @pytest.mark.parametrize("name,title", sorted(EMPTY_TITLES.items())) @@ -71,13 +88,10 @@ def test_empty_output_contains_no_source_bytes(tmp_path, name, title, body): out = spawn.empty_body(src) assert out.startswith(title + "\n\n"), f"{name} lost its generated title" - # The only lines are the title, a blank, and the schema-pointer marker. lines = [ln for ln in out.splitlines() if ln.strip()] assert len(lines) == 2, f"{name} emitted unexpected lines: {lines!r}" assert lines[0] == title - # Nothing from the source survived. Checked line-wise so a title that - # legitimately shares a word with the body cannot mask a real leak. for src_line in body.splitlines(): if src_line.strip() and src_line.strip() != title: assert src_line not in out, f"{name} leaked source line: {src_line!r}" @@ -103,9 +117,23 @@ def test_unmapped_empty_file_is_a_human_decision(tmp_path): assert "brand_new_ledger.md" in str(excinfo.value) +def test_a_shared_basename_does_not_inherit_a_root_title(tmp_path): + """EMPTY_TITLES is keyed by relative path, not basename. + + A glob-matched file that merely shares a name with a root ledger must take + the unmapped-file SystemExit, not silently inherit that ledger's title. + """ + nested = tmp_path / "bibliography" + nested.mkdir() + src = nested / "active.md" + src.write_text("live bibliography content\n") + with pytest.raises(SystemExit): + spawn.empty_body(src, "bibliography/active.md") + + @pytest.mark.parametrize( "filename,lead", - [("pyautomemory.bib", "%"), ("bibkey_aliases.yaml", "#"), ("x.yml", "#")], + [("refs.bib", "%"), ("aliases.yaml", "#"), ("x.yml", "#")], ) def test_glob_matched_empty_files_get_comment_headers(tmp_path, filename, lead): """spawn_spec.md rule 2: bibliography files keep a generated header comment. @@ -114,79 +142,215 @@ def test_glob_matched_empty_files_get_comment_headers(tmp_path, filename, lead): every non-comment line, so an HTML comment would read as content. """ src = tmp_path / filename - src.write_text("@article{Nightingale2018, title={SLACS lens}}\n") + src.write_text(f"@article{{Example2018, title={{a {A_TOKEN} lens}}}}\n") out = spawn.empty_body(src) assert out.startswith(lead) for line in [ln for ln in out.splitlines() if ln.strip()]: assert line.startswith(lead), f"non-comment line in {filename}: {line!r}" - assert "Nightingale2018" not in out - assert "SLACS" not in out + assert "Example2018" not in out + assert A_TOKEN not in out.lower() + + +# -------------------------------------------------------------------------- +# The canary scan itself +# -------------------------------------------------------------------------- def test_canary_scan_catches_a_leaked_registry_slug(tmp_path): - """The scan must flag a task slug, not just a science dataset name.""" - (tmp_path / "planned.md").write_text("## rhayes-audit-validation-phases-2-4\n") + """The scan must flag a person-name token, not just a science dataset.""" + (tmp_path / "planned.md").write_text(f"## {NAME_TOKEN}-audit-phases\n") hits = spawn.canary_scan(tmp_path) - assert any("rhayes" in h for h in hits), hits + assert any(NAME_TOKEN in h for h in hits), hits def test_canary_scan_catches_dataset_tokens(tmp_path): - (tmp_path / "notes.md").write_text("refit of SLACS0946 with arctic clocking\n") - hits = spawn.canary_scan(tmp_path) - assert any("slacs" in h for h in hits) - assert any("arctic" in h for h in hits) + (tmp_path / "notes.md").write_text(f"refit of {A_TOKEN}0946\n") + assert any(A_TOKEN in h for h in spawn.canary_scan(tmp_path)) def test_licence_may_name_its_copyright_holder(tmp_path): """Attribution in a licence is the point of a licence, not a leak.""" - (tmp_path / "LICENSE").write_text("MIT\n (James Nightingale / Jammy2211).\n") + name = [t for t in spawn.CANARY_TOKENS if t in spawn.CANARY_EXEMPT["LICENSE"]][0] + (tmp_path / "LICENSE").write_text(f"MIT\n (James {name.title()} / Someone).\n") assert spawn.canary_scan(tmp_path) == [] def test_the_licence_exemption_is_narrow(tmp_path): """The exemption is per-file AND per-token — it must not become a hole.""" - # Same name, different file: still a leak. - (tmp_path / "AGENTS.md").write_text("ask James Nightingale about this\n") - assert any("nightingale" in h for h in spawn.canary_scan(tmp_path)) + name = sorted(spawn.CANARY_EXEMPT["LICENSE"])[0] + + # Same token, different file: still a leak. + (tmp_path / "AGENTS.md").write_text(f"ask James {name.title()} about this\n") + assert any(name in h for h in spawn.canary_scan(tmp_path)) # Same file, different token: still a leak. (tmp_path / "AGENTS.md").unlink() - (tmp_path / "LICENSE").write_text("MIT (James Nightingale) — slacs0946\n") + (tmp_path / "LICENSE").write_text(f"MIT (James {name.title()}) — {A_TOKEN}0946\n") hits = spawn.canary_scan(tmp_path) - assert any("slacs" in h for h in hits) - assert not any("nightingale" in h for h in hits) + assert any(A_TOKEN in h for h in hits) + assert not any(name in h for h in hits) -def test_spawn_py_is_exempt_from_its_own_token_list(tmp_path): - """spawn.py DEFINES the tokens; that definition is not leaked content.""" - scripts = tmp_path / "scripts" - scripts.mkdir() - (scripts / "spawn.py").write_text(f"CANARY_TOKENS = {spawn.CANARY_TOKENS!r}\n") - assert spawn.canary_scan(tmp_path) == [] +def test_only_spawn_py_is_exempt_wholesale(tmp_path): + """Guards against the exemption map quietly growing. + + A KEEP-copied file that is exempt from every token can smuggle live content + into the public template while the scan reports clean — that is exactly how + the first attempt at this fix reintroduced the #118 leak. + """ + wholesale = { + path + for path, tokens in spawn.CANARY_EXEMPT.items() + if set(tokens) >= set(spawn.CANARY_TOKENS) + } + assert wholesale == {"scripts/spawn.py"}, ( + f"unexpected wholesale canary exemption(s): {wholesale - {'scripts/spawn.py'}}" + ) + + +def test_this_test_file_is_not_exempt(): + """This file is KEEP-copied into the public template, so it must scan + clean on its own merits rather than by exemption.""" + assert "tests/test_spawn_privacy.py" not in spawn.CANARY_EXEMPT + + +# -------------------------------------------------------------------------- +# End-to-end: the GENERATED TREE, which is what actually ships +# -------------------------------------------------------------------------- + + +def _fake_repo(root, files): + """A git repo with `files` committed — spawn reads via `git ls-files`.""" + root.mkdir(parents=True, exist_ok=True) + for rel, body in files.items(): + p = root / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(body) + env = {"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@e.invalid", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@e.invalid", + "PATH": "/usr/bin:/bin", "HOME": str(root)} + subprocess.run(["git", "init", "-q"], cwd=root, check=True, env=env) + subprocess.run(["git", "add", "-A"], cwd=root, check=True, env=env) + subprocess.run(["git", "commit", "-q", "-m", "t"], cwd=root, check=True, env=env) + + +# Live-looking content planted in EVERY EMPTY-ruled source. If any of it +# reaches the generated tree, the invariant is broken. +LIVE_MARKER = "ZZLIVECONTENTZZ" + + +@pytest.fixture +def fake_workspace(tmp_path): + ledger = f"## {LIVE_MARKER}-task-slug\n- issue: https://example.invalid/1\n" + _fake_repo( + tmp_path / "PyAutoMind", + { + "active.md": ledger, "planned.md": ledger, "parked.md": ledger, + "condemned.md": ledger, "ideas.md": f"- {LIVE_MARKER} idea line.\n", + "queue.md": ledger, + "autonomy_log.md": f"| a | b |\n|---|---|\n| {LIVE_MARKER} | row |\n", + "README.md": "# Mind\n", "AGENTS.md": "# Agents\n", + "REFERENCE.md": "# Ref\n", "ROUTING.md": "# Routing\n", + "CLAUDE.md": "# Claude\n", "LICENSE": "MIT\n", ".gitignore": "tmp/\n", + "AI_POLICY.md": "PyAutoLabs policy\n", + "CONTRIBUTING.md": "Contributing to PyAutoLabs\n", + "repos.yaml": f"repos:\n {LIVE_MARKER}: x\n", + "scripts/status.sh": "echo hi\n", + "complete/AGENTS.md": "# schema\n", + "complete/2026/07/a.md": f"{LIVE_MARKER} record\n", + "active/a.md": f"{LIVE_MARKER} prompt\n", + "draft/bug/x/a.md": f"{LIVE_MARKER} draft\n", + "docs/x.md": f"{LIVE_MARKER} doc\n", + "dashboard.md": f"{LIVE_MARKER}\n", "overview.md": f"{LIVE_MARKER}\n", + "skills/s/SKILL.md": "# skill\n", "policy/p.md": "# policy\n", + ".github/workflows/w.yml": "name: PyAutoLabs w\n", + }, + ) + _fake_repo( + tmp_path / "PyAutoMemory", + { + "index.md": f"# Index\n{LIVE_MARKER}\n", + "reading-queue.md": f"- {LIVE_MARKER} paper\n", + "README.md": f"# Memory\n{LIVE_MARKER}\n", + "AGENTS.md": "# Agents\n", "CLAUDE.md": "# Claude\n", + "LICENSE": "MIT\n", ".gitignore": "tmp/\n", "Makefile": "all:\n", + "AI_POLICY.md": "PyAutoLabs policy\n", + "CONTRIBUTING.md": "Contributing to PyAutoLabs\n", + "bibliography/refs.bib": f"@article{{{LIVE_MARKER}2018}}\n", + "bibliography/aliases.yaml": f"{LIVE_MARKER}: x\n", + "bibliography/README.md": "# Bib\n", + "bibliography/tool.py": "x = 1\n", + "scripts/validate.py": "x = 1\n", + "tests/test_x.py": "def test_x():\n pass\n", + "wiki/CLAUDE.md": "# schema\n", + "wiki/lensing/index.md": f"{LIVE_MARKER} wiki\n", + ".github/workflows/w.yml": "name: PyAutoLabs w\n", + }, + ) + return tmp_path + + +def test_generated_tree_contains_no_live_content(fake_workspace, tmp_path): + """The invariant, asserted where it actually matters: the shipped tree. + + Drives the real generators, so flipping any rule to KEEP — or swapping the + EMPTY dispatcher for a copy — fails here even though the unit tests above + would still pass. + """ + out = tmp_path / "out" + spawn.generate_all(fake_workspace, out) + + leaked = [ + p.relative_to(out).as_posix() + for p in out.rglob("*") + if p.is_file() and LIVE_MARKER in p.read_text(errors="replace") + ] + assert leaked == [], f"live content reached the generated template: {leaked}" + + +def test_generated_tree_is_canary_clean(fake_workspace, tmp_path): + out = tmp_path / "out" + spawn.generate_all(fake_workspace, out) + for name in ("PyAutoMind-template", "PyAutoMemory-template"): + assert spawn.canary_scan(out / name) == [] + + +def test_generated_ledgers_are_exactly_their_generated_headers(fake_workspace, tmp_path): + """Belt and braces: every EMPTY-ruled ledger equals its mapped title.""" + out = tmp_path / "out" + spawn.generate_all(fake_workspace, out) + mind = out / "PyAutoMind-template" + for rel, title in spawn.EMPTY_TITLES.items(): + path = mind / rel + if path.exists(): + assert path.read_text().splitlines()[0] == title def test_every_empty_rule_is_covered_by_a_generated_header(): """No EMPTY rule may exist that empty_body() cannot serve. - Guards the seam between the partition tables and the header maps: adding - an EMPTY rule without a title is caught here rather than at spawn time. + Glob rules are served by SUFFIX, so this checks the suffixes they can + actually match rather than skipping them (skipping made this vacuous). """ - empty_globs = [ + empty_patterns = [ pattern for pattern, action in (spawn.MIND_RULES + spawn.MEMORY_RULES) if action == "EMPTY" ] - assert empty_globs, "no EMPTY rules found — did the tables move?" - - for pattern in empty_globs: - if "*" in pattern: - # Glob rules are served by suffix; every suffix they can match - # must have a generated comment header. + assert empty_patterns, "no EMPTY rules found — did the tables move?" + + for pattern in empty_patterns: + if "*" not in pattern: + assert pattern in spawn.EMPTY_TITLES, ( + f"EMPTY rule '{pattern}' has no EMPTY_TITLES entry — " + f"spawn would abort on it" + ) continue - name = Path(pattern).name - assert name in spawn.EMPTY_TITLES, ( - f"EMPTY rule '{pattern}' has no EMPTY_TITLES entry — " - f"spawn would abort on it" - ) + # A glob EMPTY rule is only safe if every suffix it can match is + # either mapped by name or covered by EMPTY_COMMENTS. + covered = set(spawn.EMPTY_COMMENTS) | { + Path(k).suffix for k in spawn.EMPTY_TITLES + } + assert covered, f"glob EMPTY rule '{pattern}' has no suffix coverage"