From 9326c0c38b7a7684203882ecf882b8a81e5993c7 Mon Sep 17 00:00:00 2001 From: rev Date: Mon, 2 Nov 2026 14:45:11 +0100 Subject: [PATCH 1/4] fix: rank protected Done cards before archive retention cutoff --- docs/coordination/v1-writer-census.json | 2 +- scripts/coord-board-archive.py | 22 +++++++++----- tests/coord-board-archive/test_archive.py | 35 +++++++++++++++++++++++ 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/docs/coordination/v1-writer-census.json b/docs/coordination/v1-writer-census.json index 2beb48e8a..eac22a71a 100644 --- a/docs/coordination/v1-writer-census.json +++ b/docs/coordination/v1-writer-census.json @@ -93,7 +93,7 @@ {"path":"scripts/check-skillmirror-freshness.py","disposition":"guard-only","sinkKinds":["dynamic-process"],"sha256":"2e7a72c0418bc132d8fefaa005ab3739c390c980115dd6462ca67aab22e77911","nonWriterJustification":"Read-only checker with embedded refusal fixture; it does not dispatch a production write."}, {"path":"scripts/check-workflow-permissions.py","disposition":"guard-only","sinkKinds":["dynamic-process"],"sha256":"7ed5fe0ddc9e209222fe8eb136869cfd3a970bae4037363be2930c1b60a5ffec","nonWriterJustification":"Read-only checker with embedded refusal fixture; it does not dispatch a production write."}, {"path":"scripts/collect-v1-receiver-census.py","disposition":"guard-only","sinkKinds":["dynamic-process"],"sha256":"c7c1d06e69fc6ba7a525e5d229f2e113763a1f109d9ea428eaba35473e03bfb5","nonWriterJustification":"Bounded source collector invokes only fixed allowlisted GitHub GET routes; write methods and arbitrary endpoints are rejected before subprocess execution."}, - {"path":"scripts/coord-board-archive.py","disposition":"conditional-remote-writer","sinkKinds":["graphql-mutation"],"sha256":"a7b21a589777295c83339239dac2e3e786d1b8fb7382358351d569aead6fcc9b"}, + {"path":"scripts/coord-board-archive.py","disposition":"conditional-remote-writer","sinkKinds":["graphql-mutation"],"sha256":"76f1d033aaa8298cc9f6366329aabd2c9852f6eb5114eccb274be8243d564a5e"}, {"path":"scripts/gs2-09-7-mint-sandbox-token.py","disposition":"credential-lifecycle","sinkKinds":[],"sha256":"392b084bfbf010d0528895400bc3323eaf9a1c921a233a5297bfe6b042ed466c","nonWriterJustification":"The helper mints and may revoke only a sandbox-scoped installation token; it has no V1 data mutation path."}, {"path":"scripts/learn-dispatch.py","disposition":"conditional-remote-writer","sinkKinds":["dynamic-process"],"sha256":"0e359e4d707bf2beb889fb8c7e1bf9da36ec63463736b9a3824a7c3822398093","writerJustification":"Explicit --record mode posts one classification comment to the owning native Delivery issue and submits its digest-bound telemetry observation; offline preparation has no remote effects. Per-Delivery shared lock, durable possible-send custody, exact comment readback and no uncertain automatic resend preserve partial effects. It never writes board fields, merges or dispatches workers. Operation authority remains separate."}, {"path":"scripts/lib/mutation.py","disposition":"test-only","sinkKinds":["graphql-mutation"],"sha256":"d2b7c58495f996836c7f4405757a0daa914bfe7250263c945ee9a4f22a22f183","nonWriterJustification":"Local gate-mutation library; detected syntax is test data, not a production sink."}, diff --git a/scripts/coord-board-archive.py b/scripts/coord-board-archive.py index 13d336c81..ab69e5e71 100755 --- a/scripts/coord-board-archive.py +++ b/scripts/coord-board-archive.py @@ -32,8 +32,9 @@ read completely or the run refuses. This is the "never a live row's blocker" guard the reconciler design keeps. 2. Newest 200 kept. Candidates are ranked by `closedAt` descending, with ties broken by repository, number and item id, so the selection never depends on connection order. The first `--keep` stay; only the rest are - planned. The reconciler (tend) must use exactly this ranking (closedAt, then repository, number and item - id); a shared test pins the two together when tend lands (#5136). + considered for archiving. Protected cards count in this ranking; older protected cards also stay visible, + without shifting the newest-200 cutoff. The reconciler (tend) must use exactly this ranking (closedAt, then + repository, number and item id); a shared test pins the two together when tend lands (#5136). 3. Complete read or nothing. The scan reads `items(query: "status:Done")` in pages of 100, at most 5 pages, the 500-item board limit. A partial page set, a count mismatch, a GraphQL error or an over-limit board refuses without archiving anything. @@ -85,6 +86,7 @@ BOARD_LIMIT = PAGE * MAX_PAGES TERMINAL = ("CLOSED", "MERGED") CUSTODY_SCHEMA = "fsgg.board-done-retention-custody/1" +LIVE_BLOCKER_REASON = "a live card is still blocked by it: stays visible" SCAN = """ query($owner:String!,$number:Int!,$cursor:String){ @@ -282,10 +284,11 @@ def live_blockers(owner: str, number: int) -> tuple[set[str], int]: def plan(rows: list[dict], keep: int = KEEP, protected: set[str] | None = None) -> tuple[list[dict], list[dict], list[tuple[dict, str]]]: - """Split Done rows into (archive, kept, not-rankable-with-reason). Pure, so every guard is testable. + """Split Done rows into (archive, newest-kept, excluded-with-reason). Pure, so every guard is testable. Ranked: Done, issue or PR, terminal state, parseable `closedAt`, not already archived. Newest first by `closedAt`, ties broken by (repo, number, itemId) so connection order never decides. The first `keep` stay. + Only then exclude live blockers from the older candidates; protection never shifts the retention cutoff. """ if keep < 0: raise ValueError("keep must not be negative") @@ -307,13 +310,17 @@ def plan(rows: list[dict], keep: int = KEEP, protected: set[str] | None = None) unranked.append((row, "closedAt is unreadable: never archived")) elif not isinstance(row.get("number"), int) or not row.get("repo"): unranked.append((row, "unreadable repository or number: never archived")) - elif protected and row["contentId"] in protected: - unranked.append((row, "a live card is still blocked by it: stays visible")) else: ranked.append((closed, str(row["repo"]), int(row["number"]), str(row["itemId"]), row)) ranked.sort(key=lambda entry: (-entry[0].timestamp(), entry[1], entry[2], entry[3])) kept = [entry[4] for entry in ranked[:keep]] - archive = [entry[4] for entry in ranked[keep:]] + archive = [] + for entry in ranked[keep:]: + row = entry[4] + if protected and row["contentId"] in protected: + unranked.append((row, LIVE_BLOCKER_REASON)) + else: + archive.append(row) return archive, kept, unranked @@ -535,8 +542,9 @@ def main(argv: list[str] | None = None) -> int: return 1 archive, kept, unranked = plan(rows, args.keep, protected) + ranked_count = len(archive) + len(kept) + sum(why == LIVE_BLOCKER_REASON for _, why in unranked) print(f"board: {project.get('totalCount')} item(s) (limit {BOARD_LIMIT}); Done: {len(rows)}; " - f"ranked {len(archive) + len(kept)}, keep newest {len(kept)}, plan to archive {len(archive)}") + f"ranked {ranked_count}, keep newest {len(kept)}, plan to archive {len(archive)}") for row, why in unranked: print(f" KEEP {'' if args.counts_only else ref_of(row)}: {why}") if enrolled is None: diff --git a/tests/coord-board-archive/test_archive.py b/tests/coord-board-archive/test_archive.py index 75c3b4ba7..99c9e5e37 100644 --- a/tests/coord-board-archive/test_archive.py +++ b/tests/coord-board-archive/test_archive.py @@ -7,6 +7,7 @@ from __future__ import annotations import contextlib +import datetime as dt import importlib.util import io import json @@ -32,6 +33,12 @@ def row(n: int, closed: str | None = None, **kw) -> dict: return base +def done_history(count: int) -> list[dict]: + """A fixed close-time history; larger issue numbers are newer, independent of the clock.""" + start = dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc) + return [row(n, (start + dt.timedelta(days=n)).isoformat()) for n in range(1, count + 1)] + + def node(r: dict) -> dict: content = {"__typename": r["kind"], "id": r["contentId"]} if r["kind"] in ("Issue", "PullRequest"): @@ -199,6 +206,19 @@ def test_every_incomplete_read_refuses(self): class LiveBlockers(unittest.TestCase): + def test_protection_does_not_shift_the_newest_200_done_cutoff(self): + rows = done_history(203) + newer_protected = {f"I_{n}" for n in range(191, 204)} + for protected in (newer_protected, newer_protected | {"I_2"}): + for ordered in (rows, list(reversed(rows))): + with self.subTest(protected_older="I_2" in protected, reversed=ordered[0]["number"] == 203): + archive, kept, excluded = A.plan(ordered, 200, protected) + self.assertEqual([r["number"] for r in kept], list(range(203, 3, -1))) + self.assertEqual([r["number"] for r in archive], + [n for n in (3, 2, 1) if f"I_{n}" not in protected]) + self.assertEqual([r["number"] for r, _ in excluded], [2] if "I_2" in protected else []) + self.assertTrue(all("live card" in why for _, why in excluded)) + def test_a_done_card_a_live_card_is_blocked_by_stays_visible(self): old = "2020-01-01T00:00:00Z" rows = [row(1, old), row(2, old), row(3, old)] @@ -225,6 +245,21 @@ def test_the_dry_run_reports_a_protected_card(self): self.assertIn("plan to archive 2", out) self.assertIn("KEEP FS-GG/.github#1: a live card is still blocked by it", out) + def test_threshold_dry_run_ranks_protected_rows_and_never_sends(self): + rows = done_history(203) + protected = [f"I_{n}" for n in range(191, 204)] + ["I_2"] + stub = Stub(self, scan=[scan_page(rows[:100], total=203, board=204, has_next=True, cursor="c1"), + scan_page(rows[100:200], total=203, board=204, has_next=True, cursor="c2"), + scan_page(rows[200:], total=203, board=204)], + live=[live_page([protected])]) + code, out, _ = run(["--counts-only"]) + self.assertEqual(code, 0, out) + self.assertIn("Done: 203; ranked 203, keep newest 200, plan to archive 2", out) + self.assertIn("KEEP : a live card is still blocked by it", out) + self.assertIn("DRY RUN", out) + self.assertNotIn("FS-GG/", out) + self.assertEqual((len(stub.calls("scan")), stub.calls("item"), stub.calls("mutation")), (3, [], [])) + class Execute(unittest.TestCase): def setUp(self): From 9c3a598e9ca86a073cc5f84523d81b0dcdbb2cd7 Mon Sep 17 00:00:00 2001 From: rev Date: Mon, 2 Nov 2026 15:34:09 +0100 Subject: [PATCH 2/4] feat: prepare guarded fdev Done-retention schedule (Refs #5170) --- .github/workflows/coord-board-archive.yml | 3 +- .../fdev-board-tend/fsgg-board-archive-run.py | 194 +++++++++++++++ .../fsgg-board-archive.service.in | 24 ++ .../fsgg-board-archive.timer.in | 10 + deployment/fdev-board-tend/install.py | 38 ++- docs/coordination/board-tend-fdev-units.md | 77 +++++- .../fdev-board-tend/test_archive_schedule.py | 223 ++++++++++++++++++ 7 files changed, 564 insertions(+), 5 deletions(-) create mode 100644 deployment/fdev-board-tend/fsgg-board-archive-run.py create mode 100644 deployment/fdev-board-tend/fsgg-board-archive.service.in create mode 100644 deployment/fdev-board-tend/fsgg-board-archive.timer.in create mode 100644 tests/fdev-board-tend/test_archive_schedule.py diff --git a/.github/workflows/coord-board-archive.yml b/.github/workflows/coord-board-archive.yml index a9a07959e..9466320a5 100644 --- a/.github/workflows/coord-board-archive.yml +++ b/.github/workflows/coord-board-archive.yml @@ -9,7 +9,8 @@ name: Done retention dry run for the Coordination board # # THIS WORKFLOW ONLY EVER RUNS A DRY RUN. It reports the board size against the 500-item limit, the Done count and # the cards the policy would archive. A real run is a separate, owner-run invocation with the private enrollment -# binding and an exact project id. The schedule gains --execute only after Root accepts the first real run. +# binding and an exact project id. After Root accepts the first real run, the sibling fdev user timer is the +# scheduled writer (deployment/fdev-board-tend); this hosted job remains read-only and counts-only. # # WHY THE PREVIOUS WORKFLOW AT THIS PATH WAS RETIRED (Root, 2026-10-31) # ===================================================================== diff --git a/deployment/fdev-board-tend/fsgg-board-archive-run.py b/deployment/fdev-board-tend/fsgg-board-archive-run.py new file mode 100644 index 000000000..ed9250337 --- /dev/null +++ b/deployment/fdev-board-tend/fsgg-board-archive-run.py @@ -0,0 +1,194 @@ +#!/usr/bin/python3 -I +"""Private fdev Done-retention adapter (#5170). Never enables itself or retries an unresolved run. + +The existing archiver owns native reads, per-card writer exclusion, intent/send/readback and policy. +This adapter pins its activation inputs and preserves a cross-run latch before invoking it. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import stat +import subprocess +import sys +import tempfile + +PROJECT = "PVT_kwDOEYAWY84Bldpa" +SCHEMA = "fsgg.board-archive-config/1" +FIELDS = {"schema", "enabled", "maxArchive", "script", "gh", "binding", "commitment", "readerVersion", + "readerFiles", "acceptedReceipt", "stateDir", "outDir", "workdir"} + + +class Refused(Exception): + pass + + +def private_file(path: Path) -> bytes: + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o600: + raise Refused("private input must be an owner-owned regular 0600 file") + return path.read_bytes() + + +def absolute(text: object) -> Path: + if not isinstance(text, str) or not Path(text).is_absolute(): + raise Refused("absolute path required") + return Path(text) + + +def pin(value: object) -> tuple[Path, str]: + if not isinstance(value, dict) or set(value) != {"path", "sha256"}: + raise Refused("invalid file pin") + path = absolute(value["path"]) + digest = value["sha256"] + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise Refused("invalid file digest") + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o022: + raise Refused("pinned file must be owner-owned, regular and not group/world writable") + if hashlib.sha256(path.read_bytes()).hexdigest() != digest: + raise Refused("pinned file changed; Root must refresh activation") + return path, digest + + +def private_dir(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True, mode=0o700) + info = path.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise Refused("output/state directory must be owner-owned, regular and 0700") + return path + + +def sync_dir(path: Path) -> None: + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def exclusive_json(path: Path, data: dict) -> None: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as handle: + json.dump(data, handle, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + sync_dir(path.parent) + + +def configuration(path: Path) -> tuple[dict, dict[str, Path]]: + c = json.loads(private_file(path)) + if not isinstance(c, dict) or set(c) != FIELDS or c["schema"] != SCHEMA or c["enabled"] is not True: + raise Refused("missing, disabled or invalid archive activation") + if type(c["maxArchive"]) is not int or not 1 <= c["maxArchive"] <= 500: + raise Refused("Root must supply a positive archive bound at most 500") + version = c["readerVersion"] + if not isinstance(version, str) or not re.fullmatch(r"\d+\.\d+\.\d+", version) or tuple(map(int, version.split("."))) < (0, 114, 1): + raise Refused("qualified archived-membership reader required") + paths = {name: pin(c[name])[0] for name in ("script", "gh", "binding", "commitment", "acceptedReceipt")} + private_file(paths["binding"]) + binding = json.loads(paths["binding"].read_bytes()) + selected = binding.get("selectedIssues") if isinstance(binding, dict) else None + if not isinstance(selected, dict) or not selected: + raise Refused("invalid enrollment binding") + if not isinstance(c["readerFiles"], list) or not c["readerFiles"]: + raise Refused("qualified installed reader file pins required") + for value in c["readerFiles"]: + pin(value) + receipt = json.loads(private_file(paths["acceptedReceipt"])) + expected = {"projectId": PROJECT, "keep": 200, "maxArchive": c["maxArchive"], "readerVersion": version, + "readerFiles": c["readerFiles"], "scriptSha256": c["script"]["sha256"], + "bindingSha256": c["binding"]["sha256"], "commitmentSha256": c["commitment"]["sha256"]} + if (not isinstance(receipt, dict) or receipt.get("schema") != "fsgg.board-archive-accepted/1" + or receipt.get("accepted") is not True or receipt.get("inputs") != expected + or not re.fullmatch(r"https://github\.com/FS-GG/\.github/issues/5170#issuecomment-\d+", str(receipt.get("evidenceUrl")))): + raise Refused("Root's accepted first-real execution must match activation inputs") + for name in ("stateDir", "outDir", "workdir"): + paths[name] = absolute(c[name]) + if Path.cwd().resolve() != paths["workdir"].resolve() or (paths["workdir"] / ".git").exists(): + raise Refused("fixed non-checkout working directory required") + runtime = os.environ.get("XDG_RUNTIME_DIR", "") + if not runtime or not Path(runtime).is_absolute(): + raise Refused("shared XDG_RUNTIME_DIR required") + return c, paths + + +def settled(run: Path) -> bool: + custody = run / "custody" / "custody.jsonl" + if not custody.exists(): + return (run / "stdout").read_text().rstrip().endswith("nothing to archive.") + rows = [json.loads(line) for line in custody.read_text().splitlines()] + pending = set() + for row in rows: + if not isinstance(row, dict): + return False + if row.get("step") == "intent": + pending.add(row.get("itemId")) + elif row.get("step") == "archived": + pending.discard(row.get("itemId")) + elif row.get("step") in ("uncertain", "stopped"): + return False + return bool(rows) and rows[0].get("step") == "run" and rows[-1].get("step") == "finished" and not pending + + +def run(config: Path, credentials: Path, timeout: float = 720) -> int: + try: + c, p = configuration(config) + secrets = json.loads(private_file(credentials)) + token = secrets.get("GH_TOKEN") if isinstance(secrets, dict) else None + if not isinstance(token, str) or not token.strip(): + raise Refused("credential file has no GH_TOKEN") + state = private_dir(p["stateDir"]) + latch = state / "in-progress.json" + if latch.exists() or latch.is_symlink(): + raise Refused("unresolved prior run; Root readback required before another invocation") + out = private_dir(p["outDir"]) + stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ-") + directory = Path(tempfile.mkdtemp(prefix=stamp, dir=out)) + exclusive_json(latch, {"run": str(directory), "configSha256": hashlib.sha256(config.read_bytes()).hexdigest()}) + env = {key: os.environ[key] for key in ("HOME", "LANG", "LC_ALL", "XDG_RUNTIME_DIR", "DOTNET_ROOT") if key in os.environ} + env.update(PATH="/usr/local/bin:/usr/bin:/bin", GH_TOKEN=token.strip(), COORD_ARCHIVE_GH=str(p["gh"])) + command = ["/usr/bin/python3", "-I", str(p["script"]), "--owner", "FS-GG", "--project", "3", + "--expect-project-id", PROJECT, "--keep", "200", "--max-archive", str(c["maxArchive"]), + "--enrollment-binding", str(p["binding"]), "--reader-version", c["readerVersion"], + "--custody-dir", str(directory / "custody"), "--execute"] + with os.fdopen(os.open(directory / "stdout", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as stdout, \ + os.fdopen(os.open(directory / "stderr", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as stderr: + process = subprocess.Popen(command, env=env, stdout=stdout, stderr=stderr, start_new_session=True, umask=0o077) + try: + code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + code = 124 + if code != 0 or not settled(directory): + print("board-archive: stopped; private custody retained, Root readback required", file=sys.stderr) + return 3 + exclusive_json(directory / "completed.json", {"exitCode": code, "settled": True}) + latch.unlink() + sync_dir(state) + print("board-archive: completed; private custody retained") + return 0 + except (Refused, OSError, ValueError, KeyError, TypeError): + # Never emit paths, card identities, tokens or exception text to the journal. + print("board-archive: refused; check private activation inputs or unresolved custody", file=sys.stderr) + return 2 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--credentials", type=Path, required=True) + args = parser.parse_args() + return run(args.config, args.credentials) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deployment/fdev-board-tend/fsgg-board-archive.service.in b/deployment/fdev-board-tend/fsgg-board-archive.service.in new file mode 100644 index 000000000..3e372f197 --- /dev/null +++ b/deployment/fdev-board-tend/fsgg-board-archive.service.in @@ -0,0 +1,24 @@ +# Rendered by install.py archive; installation does not enable or start this unit (#5170). +[Unit] +Description=FS-GG Project3 guarded Done retention + +[Service] +Type=exec +WorkingDirectory=@WORKDIR@ +Environment=XDG_RUNTIME_DIR=%t +Environment=XDG_CONFIG_HOME=%h/.config +Environment=XDG_STATE_HOME=%h/.local/state +RuntimeDirectory=fsgg +RuntimeDirectoryMode=0700 +RuntimeDirectoryPreserve=yes +UMask=0077 +NoNewPrivileges=yes +# No PrivateTmp: the archiver shares the literal /tmp legacy writer locks with the installed engine. +CPUQuota=100% +MemoryMax=1G +MemorySwapMax=0 +RuntimeMaxSec=900 +TimeoutStopSec=30 +KillMode=control-group +# No Restart: an incomplete run retains the adapter's durable latch for Root readback. +ExecStart=/usr/bin/python3 -I @RELEASE_DIR@/fsgg-board-archive-run.py --config @CONFIG@ --credentials @CREDENTIALS@ diff --git a/deployment/fdev-board-tend/fsgg-board-archive.timer.in b/deployment/fdev-board-tend/fsgg-board-archive.timer.in new file mode 100644 index 000000000..e5ec7eeb9 --- /dev/null +++ b/deployment/fdev-board-tend/fsgg-board-archive.timer.in @@ -0,0 +1,10 @@ +[Unit] +Description=Daily FS-GG Project3 guarded Done retention + +[Timer] +OnCalendar=*-*-* 04:43:00 UTC +Persistent=true +Unit=fsgg-board-archive.service + +[Install] +WantedBy=timers.target diff --git a/deployment/fdev-board-tend/install.py b/deployment/fdev-board-tend/install.py index 60005cf33..f53fbf943 100644 --- a/deployment/fdev-board-tend/install.py +++ b/deployment/fdev-board-tend/install.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -I """Install the fdev board-tend units (board-reconciler design section 8, #5136 PR 8). User-level only. -Five subcommands, each writing one kind of thing and nothing else. Every one has --dry-run, which prints what it +Six subcommands, each writing one kind of thing and nothing else. Every one has --dry-run, which prints what it would write and changes nothing. None needs or uses root; every destination must be under $HOME. units Renders fsgg-board-tend.{service,timer} and fsgg-board-tend-daily.{service,timer} into the user @@ -16,6 +16,8 @@ the script with --apply. Without --enable it does not touch systemd. chromium-env Copies deployment/fdev-chromium/fdev-chromium-env to ~/.local/bin (the source of truth for the installed wrapper, which sets the short disk-backed TMPDIR default, .github#5240). + archive Stages an immutable archive adapter/script and two daily user units. Never enables or starts + them; Root's accepted first execution and private activation configuration remain mandatory. The engine and the kit root are pinned to one release: the engine must live in .../coord-/ and the kit root must be named kit-. @@ -286,6 +288,29 @@ def command_enable_file(args: argparse.Namespace) -> int: return 0 +def command_archive(args: argparse.Namespace) -> int: + if os.geteuid() == 0: + raise Refusal("run as the fdev user, not root") + release = user_path(args.release_dir, "--release-dir") + workdir = user_path(args.workdir, "--workdir") + values = {"RELEASE_DIR": str(release), "WORKDIR": str(workdir), + "CONFIG": str(user_path(args.config, "--config")), + "CREDENTIALS": str(user_path(args.credentials, "--credentials"))} + unit_dir = user_path(args.unit_dir, "--unit-dir") + plan = Plan(args.dry_run) + # Release files are immutable: --force can replace rendered units, never a pinned release's bytes. + for name, source in (("fsgg-board-archive-run.py", HERE / "fsgg-board-archive-run.py"), + ("coord-board-archive.py", HERE.parents[1] / "scripts/coord-board-archive.py")): + plan.add(release / name, source.read_bytes(), 0o400, False) + for name in ("fsgg-board-archive.service", "fsgg-board-archive.timer"): + plan.add(unit_dir / name, render((HERE / (name + ".in")).read_text(), values).encode(), 0o644, args.force) + plan.apply() + if not args.dry_run: + workdir.mkdir(parents=True, exist_ok=True, mode=0o700) + print("archive staged disabled; systemd not touched; Root owns first-real acceptance and activation") + return 0 + + def parser() -> argparse.ArgumentParser: h = home() root = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) @@ -324,6 +349,14 @@ def parser() -> argparse.ArgumentParser: chromium.add_argument("--dest", default=str(h / ".local/bin/fdev-chromium-env")) chromium.add_argument("--force", action="store_true", help="replace the installed wrapper when it differs") chromium.add_argument("--dry-run", action="store_true") + archive = sub.add_parser("archive") + archive.add_argument("--release-dir", required=True, help="a new immutable, versioned directory under HOME") + archive.add_argument("--workdir", default=str(h / ".local/share/fs-gg/board-archive")) + archive.add_argument("--config", default=str(h / ".config/fsgg/board-archive.json")) + archive.add_argument("--credentials", default=str(h / ".config/fdev/credentials.json")) + archive.add_argument("--unit-dir", default=str(h / ".config/systemd/user")) + archive.add_argument("--force", action="store_true") + archive.add_argument("--dry-run", action="store_true") return root @@ -331,7 +364,8 @@ def main(argv: list[str] | None = None) -> int: args = parser().parse_args(argv) try: return {"units": command_units, "kit-root": command_kit_root, "enable-file": command_enable_file, - "tmp-cleanup": command_tmp_cleanup, "chromium-env": command_chromium_env}[args.command](args) + "tmp-cleanup": command_tmp_cleanup, "chromium-env": command_chromium_env, + "archive": command_archive}[args.command](args) except Refusal as error: print(f"install.py: refused: {error}", file=sys.stderr) return 2 diff --git a/docs/coordination/board-tend-fdev-units.md b/docs/coordination/board-tend-fdev-units.md index 90f3f957e..947eb7a35 100644 --- a/docs/coordination/board-tend-fdev-units.md +++ b/docs/coordination/board-tend-fdev-units.md @@ -239,6 +239,79 @@ activation/readback, separately from source integration. A local certificate est ## Related user units +### Done-retention writer (#5170) + +The hosted `coord-board-archive` workflow stays a **counts-only read-only observer**. Its public logs and +Projects-read App token cannot host the guarded writer. `fsgg-board-archive.{service,timer}` are a separate daily +fdev user route, staged by `install.py archive`; this command never calls systemd, enables a timer, writes an +activation configuration or changes the hourly tend units. The Python archiver remains the only archive writer; +this does not enable A1 or any tend rule. + +``` +python3 -I deployment/fdev-board-tend/install.py archive \ + --release-dir "$HOME/.local/libexec/fsgg-board-archive/" --dry-run +``` + +After review, omit `--dry-run` to stage the adapter/script (0400) and two units. Release files cannot be replaced, +even with `--force`; changed source needs a new release directory. `--force` only replaces rendered unit files. +No installed path points into a working checkout. Root reviews the exact staged hashes before any activation. + +The service uses the existing fdev credentials file, with its 0600 owner check; it requests no new credentials +or grants. The child receives only the credential file's `GH_TOKEN` and the explicitly pinned `gh` executable, +never an inherited token/test override. `XDG_RUNTIME_DIR=%t`, the preserved `fsgg` runtime directory and the +absence of `PrivateTmp` preserve the archiver's per-card shared and legacy writer locks. The adapter does not +hold those locks outside the child. Type=exec, a 900-second service bound, 720-second child bound, one CPU and +1GiB cap bound the run; no Restart directive retries failures. + +**Activation is separate and remains Root's effect, after the first real archive is accepted.** The service's +`--config` file must be an owner-owned regular 0600 file with exactly these keys: + +| Key | Required value | +|---|---| +| `schema`, `enabled` | `fsgg.board-archive-config/1`, explicitly `true`; no configuration is installed by default | +| `maxArchive` | Root's positive per-run bound, at most 500; keep is fixed 200, owner FS-GG/project 3/exact projectID are fixed in the adapter | +| `script`, `gh`, `binding`, `commitment`, `acceptedReceipt` | each exactly `{"path":"","sha256":"<64 hex>"}` | +| `readerVersion`, `readerFiles` | qualified installed reader version >=0.114.1 and a nonempty list of exact file pins for its executable/CLI/adapter closure | +| `stateDir`, `outDir`, `workdir` | absolute private runtime paths; cwd must equal the fixed non-checkout workdir | + +The binding is the **current authoritative private enrollment binding**, not a discovered or historical file. +`commitment` pins its protected public enrollment commitment. Root refreshes and qualifies the pair immediately +before activation; changes to either identity or any reader file refuse until explicitly rebound. Updating the +global engine symlink does not change or qualify the timer reader. A newer global version is not evidence that +it is board-qualified. Missing reader files, stale hashes, unqualified version, stale accepted input pair or +unreadable private credentials refuse before the child starts. + +The private `acceptedReceipt` is Root's accepted first-real-run evidence, not automatically fabricated by the +installer. Its exact shape is `{"schema":"fsgg.board-archive-accepted/1","accepted":true,"evidenceUrl": +"https://github.com/FS-GG/.github/issues/5170#issuecomment-","inputs":{...}}`. `inputs` contains exactly +`projectId`, `keep`200, `maxArchive`, `readerVersion`, `readerFiles`, `scriptSha256`, `bindingSha256` and +`commitmentSha256`, all equal to the accepted activation inputs. The URL points to Root's native acceptance; +the local file retains the private identity join. The adapter verifies the pinned receipt and join locally; +it does not grant acceptance, reinterpret the comment or make an authorization decision from GitHub text. + +All child stdout/stderr, card identities and fsynced custody stay in exclusive 0700 run directories with 0600 +files. Journal messages contain only a generic result. No child log is published. Before starting a child, the +adapter creates and fsyncs `stateDir/in-progress.json`. A zero exit with complete settled custody (or a completed +empty plan) writes a private completion record before removing the latch. A nonzero exit, timeout, interruption, +malformed custody or pending intent keeps the latch. Every later timer activation then refuses before another +child; Root must resolve the original possible effects by readback before clearing it. Never prune unresolved +custody, clear the latch merely because time passed, or blindly resend. The receipt and latch are local +cooperative safeguards, not a boundary against other processes running as this user. + +Root's remaining sequence is: integrate reviewed ranking source; review the nonzero native plan; admit and +accept one real guarded archive with enrollment-reader readback; qualify this scheduling adapter and exact +activation pair; stage disabled and read back hashes/modes/units; then explicitly daemon-reload and enable +`fsgg-board-archive.timer`. Read back its next trigger and the first scheduled run's private custody/native +effects before accepting #5170's schedule promise. The timer is daily 04:43 UTC, Persistent=true (one catch-up, +no backlog replay). Disable the timer and set `enabled:false` as the kill switch, preserving all custody. + +Preflight is the offline recording-child/archiver fixture: positive exact pins, disabled/stale inputs, +environment/output privacy, actual shared-lock contention, successful settlement and failure/crash/timeout +blocking the next invocation. These directly exercise the effect boundary; no new abstract scheduler model or +browser/CLR run is needed. Run `bash tests/fdev-board-tend/run.sh` and the existing archive fixture under the +ordinary resource budget with disk-backed temporary storage. This is safeguard qualification, with no claim +of measured CI savings or native acceptance. + `fsgg-tmp-cleanup.{service,timer}` (hourly dead-run `/tmp` cleanup, #5240) are templated here and installed by their own `install.py tmp-cleanup` effect; they share nothing with board tend at runtime. See [fdev host hygiene](fdev-host-hygiene.md). @@ -247,6 +320,6 @@ activation/readback, separately from source integration. A local certificate est - The token is the credentials file's `GH_TOKEN`. Whether it carries every scope an eventual `--apply` needs is checked at the first apply admission, not in plan-only. -- The Done-retention archiver is a hosted workflow, outside the lock; until it moves or A1 replaces it, the reconciler - writes nothing to archive-eligible cards (design ยง8). +- The hosted Done-retention observer sends no writes. The sibling fdev writer remains inactive until Root accepts + and activates it; tend still writes nothing to archive-eligible cards until its own design/acceptance changes. - `Persistent=true` catches up one missed hourly run after fdev was down; it does not replay every missed hour. diff --git a/tests/fdev-board-tend/test_archive_schedule.py b/tests/fdev-board-tend/test_archive_schedule.py new file mode 100644 index 000000000..4a317d00e --- /dev/null +++ b/tests/fdev-board-tend/test_archive_schedule.py @@ -0,0 +1,223 @@ +"""Offline archive scheduling controls: fake HOME, pinned child/stub, no systemd or native effects (#5170).""" +from __future__ import annotations + +import contextlib +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +DEPLOY = ROOT / "deployment/fdev-board-tend" + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +A = load("archive_schedule", DEPLOY / "fsgg-board-archive-run.py") +I = load("archive_install", DEPLOY / "install.py") +FIXTURE = load("archive_fixture", ROOT / "tests/coord-board-archive/test_archive.py") +FAKE = '''import json,os,pathlib,sys,time +mode = MODE +p=pathlib.Path(sys.argv[sys.argv.index('--custody-dir')+1]);p.mkdir(mode=0o700) +(p.parent/'child.json').write_text(json.dumps({'argv':sys.argv[1:],'env':dict(os.environ)})) +print('PRIVATE-CARD-IDENTITY',flush=True) +if mode=='timeout': time.sleep(60) +rows=[{'step':'run'},{'step':'intent','itemId':'PRIVATE-ID'}] +if mode=='success': rows += [{'step':'archived','itemId':'PRIVATE-ID'},{'step':'finished'}] +if mode=='unresolved': rows += [{'step':'finished'}] +if mode=='failed': rows += [{'step':'uncertain','itemId':'PRIVATE-ID'}] +(p/'custody.jsonl').write_text(''.join(json.dumps(x)+'\\n' for x in rows)) +sys.exit(3 if mode=='failed' else 0) +''' + + +class ArchiveSchedule(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.home = Path(self.temp.name) + self.env = mock.patch.dict(os.environ, {"HOME": str(self.home), "XDG_RUNTIME_DIR": str(self.home / "runtime"), + "GH_TOKEN": "WRONG", "GITHUB_TOKEN": "WRONG", "TMPDIR": "WRONG", + "COORD_ARCHIVE_GH": "/untrusted/gh"}) + self.env.start(); self.addCleanup(self.env.stop) + self.cwd = Path.cwd(); os.chdir(self.home); self.addCleanup(os.chdir, self.cwd) + self.config = self.home / "config.json" + self.credentials = self.write("credentials.json", {"GH_TOKEN": "FIXTURE-TOKEN"}) + self.script = self.write("archive.py", FAKE.replace("MODE", repr("success"))) + self.gh = self.write("gh", "fixture") + self.binding = self.write("binding.json", {"selectedIssues": {"fixture-node": {}}}) + self.commitment = self.write("commitment.json", {"commitmentSha256": "fixture"}) + self.reader = self.write("reader", "qualified fixture reader") + self.receipt = self.home / "receipt.json" + self.c = {"schema": A.SCHEMA, "enabled": True, "maxArchive": 3, "readerVersion": "0.114.1", + "script": self.pin(self.script), "gh": self.pin(self.gh), "binding": self.pin(self.binding), + "commitment": self.pin(self.commitment), "readerFiles": [self.pin(self.reader)], + "stateDir": str(self.home / "state"), "outDir": str(self.home / "out"), "workdir": str(self.home)} + self.rebind() + + def write(self, name, value): + path = self.home / name + path.write_text(value if isinstance(value, str) else json.dumps(value)) + path.chmod(0o600) + return path + + def pin(self, path): + return {"path": str(path), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + + def rebind(self): + self.c["script"] = self.pin(self.script) + inputs = {"projectId": A.PROJECT, "keep": 200, "maxArchive": self.c["maxArchive"], + "readerVersion": self.c["readerVersion"], "readerFiles": self.c["readerFiles"], + "scriptSha256": self.c["script"]["sha256"], "bindingSha256": self.c["binding"]["sha256"], + "commitmentSha256": self.c["commitment"]["sha256"]} + self.write("receipt.json", {"schema": "fsgg.board-archive-accepted/1", "accepted": True, "inputs": inputs, + "evidenceUrl": "https://github.com/FS-GG/.github/issues/5170#issuecomment-1"}) + self.c["acceptedReceipt"] = self.pin(self.receipt) + self.save() + + def save(self): + self.write("config.json", self.c) + + def invoke(self, timeout=5): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = A.run(self.config, self.credentials, timeout) + self.assertNotIn("PRIVATE", out.getvalue() + err.getvalue()) + self.assertNotIn("FIXTURE-TOKEN", out.getvalue() + err.getvalue()) + return code + + def test_success_pins_policy_environment_and_private_outputs(self): + self.assertEqual(self.invoke(), 0) + self.assertFalse((self.home / "state/in-progress.json").exists()) + run = next((self.home / "out").iterdir()) + child = json.loads((run / "child.json").read_text()) + args, env = child["argv"], child["env"] + for flag, value in (("--keep", "200"), ("--max-archive", "3"), ("--project", "3"), + ("--expect-project-id", A.PROJECT), ("--enrollment-binding", str(self.binding)), + ("--reader-version", "0.114.1")): + self.assertEqual(args[args.index(flag) + 1], value) + self.assertIn("--execute", args) + self.assertEqual(env["GH_TOKEN"], "FIXTURE-TOKEN") + self.assertEqual(env["COORD_ARCHIVE_GH"], str(self.gh)) + self.assertNotIn("GITHUB_TOKEN", env); self.assertNotIn("TMPDIR", env) + self.assertIn("PRIVATE-CARD-IDENTITY", (run / "stdout").read_text()) + for path in (run, self.home / "state", self.home / "out"): + self.assertEqual(path.stat().st_mode & 0o777, 0o700) + for path in run.rglob("*"): + if path.is_file(): self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.invoke(), 0) + + def test_disabled_missing_receipt_and_stale_acceptance_refuse_before_child(self): + self.c["enabled"] = False; self.save(); self.assertEqual(self.invoke(), 2) + self.c["enabled"] = True; self.c["maxArchive"] = 4; self.save(); self.assertEqual(self.invoke(), 2) + self.receipt.unlink(); self.assertEqual(self.invoke(), 2) + self.assertFalse((self.home / "out").exists()) + + def test_each_changed_identity_refuses_before_child(self): + for path in (self.binding, self.commitment, self.reader, self.script, self.gh): + with self.subTest(path=path.name): + before = path.read_bytes(); path.write_bytes(before + b" ") + self.assertEqual(self.invoke(), 2) + path.write_bytes(before) + self.assertFalse((self.home / "out").exists()) + + def test_unaccepted_receipt_and_private_input_symlink_refuse(self): + receipt = json.loads(self.receipt.read_text()); receipt["accepted"] = False + self.receipt.write_text(json.dumps(receipt)); self.c["acceptedReceipt"] = self.pin(self.receipt); self.save() + self.assertEqual(self.invoke(), 2) + self.rebind() + link = self.home / "binding-link"; link.symlink_to(self.binding) + self.c["binding"]["path"] = str(link); self.save() + self.assertEqual(self.invoke(), 2) + self.assertFalse((self.home / "out").exists()) + + def test_completed_empty_plan_releases_latch(self): + self.script.write_text("print('nothing to archive.')\n"); self.rebind() + self.assertEqual(self.invoke(), 0) + self.assertFalse((self.home / "state/in-progress.json").exists()) + + def test_malformed_custody_keeps_latch(self): + self.script.write_text(FAKE.replace("MODE", repr("success")) + .replace("sys.exit(3 if mode=='failed' else 0)", + "(p/'custody.jsonl').write_text('[}');sys.exit(0)")) + self.rebind() + self.assertEqual(self.invoke(), 2) + self.assertTrue((self.home / "state/in-progress.json").exists()) + self.assertEqual(self.invoke(), 2) + + def test_bad_credentials_modes_version_and_runtime_refuse(self): + self.credentials.chmod(0o644); self.assertEqual(self.invoke(), 2) + self.credentials.chmod(0o600) + self.c["readerVersion"] = "0.114.0"; self.rebind(); self.assertEqual(self.invoke(), 2) + self.c["readerVersion"] = "0.114.1"; self.rebind() + with mock.patch.dict(os.environ, {"XDG_RUNTIME_DIR": "relative"}): self.assertEqual(self.invoke(), 2) + + def test_failed_zero_exit_unresolved_and_timeout_block_next_invocation(self): + for mode in ("failed", "unresolved", "timeout"): + with self.subTest(mode=mode): + self.script.write_text(FAKE.replace("MODE", repr(mode))); self.rebind() + self.assertEqual(self.invoke(timeout=0.1), 3) + latch = self.home / "state/in-progress.json" + self.assertTrue(latch.exists()) + runs = list((self.home / "out").iterdir()) + self.assertEqual(self.invoke(), 2) + self.assertEqual(list((self.home / "out").iterdir()), runs) + # Only fixture cleanup: production requires Root's resolved readback. + latch.unlink() + + def test_simulated_crash_latch_blocks_even_with_current_inputs(self): + state = self.home / "state"; state.mkdir(mode=0o700) + A.exclusive_json(state / "in-progress.json", {"run": "interrupted fixture"}) + self.assertEqual(self.invoke(), 2) + self.assertFalse((self.home / "out").exists()) + + def test_actual_archiver_shared_lock_busy_sends_no_mutation(self): + import fcntl + rows = FIXTURE.done_history(203) + stub = FIXTURE.Stub(self, scan=[FIXTURE.scan_page(rows[:100], total=203, board=203, has_next=True), + FIXTURE.scan_page(rows[100:200], total=203, board=203, has_next=True, cursor="c2"), + FIXTURE.scan_page(rows[200:], total=203, board=203)]) + self.script.write_bytes((ROOT / "scripts/coord-board-archive.py").read_bytes()) + self.c["gh"] = self.pin(stub.path); self.rebind() + shared = self.home / "runtime/fsgg"; shared.mkdir(parents=True, mode=0o700) + with (shared / f"board-writer-{A.PROJECT}.lock").open("w") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + self.assertEqual(self.invoke(), 3) + self.assertEqual(stub.calls("mutation"), []) + self.assertEqual(stub.calls("item"), []) + self.assertEqual(self.invoke(), 2) + + def test_installer_dry_run_and_disabled_staging_leave_tend_untouched(self): + release = self.home / "release-source-pin" + args = ["archive", "--release-dir", str(release)] + with mock.patch.object(I.subprocess, "run", side_effect=AssertionError("systemd must not run")): + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(I.main(args + ["--dry-run"]), 0) + self.assertFalse(release.exists()) + self.assertEqual(I.main(args), 0) + units = self.home / ".config/systemd/user" + self.assertEqual({x.name for x in units.iterdir()}, {"fsgg-board-archive.service", "fsgg-board-archive.timer"}) + service = (units / "fsgg-board-archive.service").read_text() + self.assertIn("Type=exec", service); self.assertIn("RuntimeMaxSec=900", service) + self.assertIn("Environment=XDG_RUNTIME_DIR=%t", service) + self.assertIn("CPUQuota=100%", service); self.assertIn("MemoryMax=1G", service) + self.assertNotIn("PrivateTmp=", service); self.assertNotIn("Restart=", service) + self.assertEqual((release / "coord-board-archive.py").stat().st_mode & 0o777, 0o400) + (release / "coord-board-archive.py").chmod(0o600) + (release / "coord-board-archive.py").write_text("changed") + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(I.main(args + ["--force"]), 2) + + +if __name__ == "__main__": + unittest.main() From ca418c7d810f598f5cbc7aed819d6db87c10873c Mon Sep 17 00:00:00 2001 From: rev Date: Mon, 2 Nov 2026 15:44:22 +0100 Subject: [PATCH 3/4] fix: durably anchor archive custody and validate settled effects --- .../fdev-board-tend/fsgg-board-archive-run.py | 89 ++++++++++++-- docs/coordination/board-tend-fdev-units.md | 10 ++ .../fdev-board-tend/test_archive_schedule.py | 113 +++++++++++++++++- 3 files changed, 195 insertions(+), 17 deletions(-) diff --git a/deployment/fdev-board-tend/fsgg-board-archive-run.py b/deployment/fdev-board-tend/fsgg-board-archive-run.py index ed9250337..1c2d9adb2 100644 --- a/deployment/fdev-board-tend/fsgg-board-archive-run.py +++ b/deployment/fdev-board-tend/fsgg-board-archive-run.py @@ -58,7 +58,15 @@ def pin(value: object) -> tuple[Path, str]: def private_dir(path: Path) -> Path: - path.mkdir(parents=True, exist_ok=True, mode=0o700) + # Anchor every ancestor, including entries left by an earlier failed fsync. Creating parents=True and + # syncing only the leaf would not make the leaf's entry in its parent durable after a first-run crash. + for directory in reversed((path, *path.parents)): + directory.mkdir(exist_ok=True, mode=0o700) + if not stat.S_ISDIR(directory.lstat().st_mode): + raise Refused("directory ancestry must not contain symlinks or non-directories") + sync_dir(directory) + if directory.parent != directory: + sync_dir(directory.parent) info = path.lstat() if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: raise Refused("output/state directory must be owner-owned, regular and 0700") @@ -120,22 +128,66 @@ def configuration(path: Path) -> tuple[dict, dict[str, Path]]: return c, paths -def settled(run: Path) -> bool: +def settled(run: Path, max_archive: int) -> bool: custody = run / "custody" / "custody.jsonl" - if not custody.exists(): + if not custody.exists() or custody.stat().st_size == 0: return (run / "stdout").read_text().rstrip().endswith("nothing to archive.") rows = [json.loads(line) for line in custody.read_text().splitlines()] - pending = set() + common = {"schema", "at", "step"} + identity = {"itemId", "contentId", "repo", "number", "closedAt", "enrolled"} + def timestamp(value: object) -> bool: + try: + return isinstance(value, str) and dt.datetime.fromisoformat(value).tzinfo is not None + except ValueError: + return False + def integer(value: object) -> bool: + return type(value) is int and value >= 0 for row in rows: - if not isinstance(row, dict): + if (not isinstance(row, dict) or not common <= row.keys() + or row.get("schema") != "fsgg.board-done-retention-custody/1" + or not timestamp(row.get("at"))): return False - if row.get("step") == "intent": - pending.add(row.get("itemId")) - elif row.get("step") == "archived": - pending.discard(row.get("itemId")) - elif row.get("step") in ("uncertain", "stopped"): + if len(rows) < 2: + return False + start, end = rows[0], rows[-1] + if (set(start) != common | {"project", "board", "done", "keep", "planned", "maxArchive"} + or start["step"] != "run" or start["project"] != PROJECT or start["keep"] != 200 + or type(start["keep"]) is not int or start["maxArchive"] != max_archive + or not all(integer(start[k]) for k in ("board", "done", "planned", "maxArchive")) + or not 0 < start["planned"] <= start["done"] - 200 or not 200 < start["done"] <= start["board"] <= 500 + or not 1 <= max_archive <= 500 + or set(end) != common | {"archived", "skipped"} or end["step"] != "finished" + or not all(integer(end[k]) for k in ("archived", "skipped"))): + return False + pending = None + seen_items, seen_content = set(), set() + counts = {"archived": 0, "skipped": 0} + for row in rows[1:-1]: + step = row["step"] + keys = common | identity | ({"reason"} if step == "skipped" else set()) + if (step not in ("intent", "archived", "skipped") or set(row) != keys + or any(not isinstance(row[k], str) or not row[k].strip() for k in ("itemId", "contentId", "repo")) + or not re.fullmatch(r"[^/\s]+/[^/\s]+", row["repo"]) + or type(row["number"]) is not int or row["number"] < 1 + or type(row["enrolled"]) is not bool or not timestamp(row["closedAt"]) + or (step == "skipped" and (not isinstance(row["reason"], str) or not row["reason"].strip()))): return False - return bool(rows) and rows[0].get("step") == "run" and rows[-1].get("step") == "finished" and not pending + ref = {key: row[key] for key in identity} + if pending is not None: + if step != "archived" or ref != pending: + return False + counts["archived"] += 1 + pending = None + else: + if step == "archived" or row["itemId"] in seen_items or row["contentId"] in seen_content: + return False + seen_items.add(row["itemId"]); seen_content.add(row["contentId"]) + if step == "intent": + pending = ref + else: + counts["skipped"] += 1 + return (pending is None and sum(counts.values()) == min(start["planned"], max_archive) + and all(end[k] == count for k, count in counts.items())) def run(config: Path, credentials: Path, timeout: float = 720) -> int: @@ -152,7 +204,17 @@ def run(config: Path, credentials: Path, timeout: float = 720) -> int: out = private_dir(p["outDir"]) stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ-") directory = Path(tempfile.mkdtemp(prefix=stamp, dir=out)) + sync_dir(directory) + sync_dir(out) exclusive_json(latch, {"run": str(directory), "configSha256": hashlib.sha256(config.read_bytes()).hexdigest()}) + # Preprovision the child's append-only file, including its directory entry, before any possible send. + custody = private_dir(directory / "custody") + fd = os.open(custody / "custody.jsonl", os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + try: + os.fsync(fd) + finally: + os.close(fd) + sync_dir(custody) env = {key: os.environ[key] for key in ("HOME", "LANG", "LC_ALL", "XDG_RUNTIME_DIR", "DOTNET_ROOT") if key in os.environ} env.update(PATH="/usr/local/bin:/usr/bin:/bin", GH_TOKEN=token.strip(), COORD_ARCHIVE_GH=str(p["gh"])) command = ["/usr/bin/python3", "-I", str(p["script"]), "--owner", "FS-GG", "--project", "3", @@ -161,6 +223,9 @@ def run(config: Path, credentials: Path, timeout: float = 720) -> int: "--custody-dir", str(directory / "custody"), "--execute"] with os.fdopen(os.open(directory / "stdout", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as stdout, \ os.fdopen(os.open(directory / "stderr", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as stderr: + os.fsync(stdout.fileno()) + os.fsync(stderr.fileno()) + sync_dir(directory) process = subprocess.Popen(command, env=env, stdout=stdout, stderr=stderr, start_new_session=True, umask=0o077) try: code = process.wait(timeout=timeout) @@ -168,7 +233,7 @@ def run(config: Path, credentials: Path, timeout: float = 720) -> int: os.killpg(process.pid, signal.SIGKILL) process.wait() code = 124 - if code != 0 or not settled(directory): + if code != 0 or not settled(directory, c["maxArchive"]): print("board-archive: stopped; private custody retained, Root readback required", file=sys.stderr) return 3 exclusive_json(directory / "completed.json", {"exitCode": code, "settled": True}) diff --git a/docs/coordination/board-tend-fdev-units.md b/docs/coordination/board-tend-fdev-units.md index 947eb7a35..29af5ad70 100644 --- a/docs/coordination/board-tend-fdev-units.md +++ b/docs/coordination/board-tend-fdev-units.md @@ -298,6 +298,16 @@ child; Root must resolve the original possible effects by readback before cleari custody, clear the latch merely because time passed, or blindly resend. The receipt and latch are local cooperative safeguards, not a boundary against other processes running as this user. +Before child launch the adapter also fsyncs the state/output directory ancestry and each new run entry in its +parent. It preprovisions and fsyncs the child's custody directory/file and private stdout/stderr entries, so +first-run directory creation cannot leave an otherwise durable latch or custody file unreachable after a crash. +Any preparation fsync failure prevents launch; a failure after latch creation preserves that hold. Successful +custody must use the archiver's exact schema and supported sequence: one run header, per-item reasoned skips or +immediately joined intent/archived pairs with complete identical identities, then one finished record. Duplicate, +unknown or malformed records refuse; processed items must equal min(planned, configured bound), and the final +archived/skipped totals must equal the validated effects. An empty preprovisioned custody file is accepted only +after the pinned child exits0 and reports a completed empty plan. + Root's remaining sequence is: integrate reviewed ranking source; review the nonzero native plan; admit and accept one real guarded archive with enrollment-reader readback; qualify this scheduling adapter and exact activation pair; stage disabled and read back hashes/modes/units; then explicitly daemon-reload and enable diff --git a/tests/fdev-board-tend/test_archive_schedule.py b/tests/fdev-board-tend/test_archive_schedule.py index 4a317d00e..e6911b45a 100644 --- a/tests/fdev-board-tend/test_archive_schedule.py +++ b/tests/fdev-board-tend/test_archive_schedule.py @@ -2,6 +2,7 @@ from __future__ import annotations import contextlib +import copy import hashlib import importlib.util import io @@ -28,14 +29,19 @@ def load(name, path): FIXTURE = load("archive_fixture", ROOT / "tests/coord-board-archive/test_archive.py") FAKE = '''import json,os,pathlib,sys,time mode = MODE -p=pathlib.Path(sys.argv[sys.argv.index('--custody-dir')+1]);p.mkdir(mode=0o700) +p=pathlib.Path(sys.argv[sys.argv.index('--custody-dir')+1]);p.mkdir(mode=0o700,exist_ok=True) (p.parent/'child.json').write_text(json.dumps({'argv':sys.argv[1:],'env':dict(os.environ)})) print('PRIVATE-CARD-IDENTITY',flush=True) if mode=='timeout': time.sleep(60) -rows=[{'step':'run'},{'step':'intent','itemId':'PRIVATE-ID'}] -if mode=='success': rows += [{'step':'archived','itemId':'PRIVATE-ID'},{'step':'finished'}] -if mode=='unresolved': rows += [{'step':'finished'}] -if mode=='failed': rows += [{'step':'uncertain','itemId':'PRIVATE-ID'}] +base={'schema':'fsgg.board-done-retention-custody/1','at':'2026-11-02T00:00:00+00:00'} +ref={'itemId':'PRIVATE-ID','contentId':'PRIVATE-CONTENT','repo':'fixture/repo','number':1, + 'closedAt':'2026-01-01T00:00:00Z','enrolled':False} +def record(step,**kw): return dict(base,step=step,**kw) +rows=[record('run',project='PVT_kwDOEYAWY84Bldpa',board=203,done=203,keep=200,planned=1,maxArchive=3), + record('intent',**ref)] +if mode=='success': rows += [record('archived',**ref),record('finished',archived=1,skipped=0)] +if mode=='unresolved': rows += [record('finished',archived=0,skipped=0)] +if mode=='failed': rows += [record('uncertain',reason='fixture failure',**ref)] (p/'custody.jsonl').write_text(''.join(json.dumps(x)+'\\n' for x in rows)) sys.exit(3 if mode=='failed' else 0) ''' @@ -155,6 +161,103 @@ def test_malformed_custody_keeps_latch(self): self.assertTrue((self.home / "state/in-progress.json").exists()) self.assertEqual(self.invoke(), 2) + def test_json_valid_but_impossible_custody_never_settles(self): + run = self.home / "probe"; (run / "custody").mkdir(parents=True) + base = {"schema": "fsgg.board-done-retention-custody/1", "at": "2026-11-02T00:00:00+00:00"} + ref = {"itemId": "card", "contentId": "content", "repo": "fixture/repo", "number": 1, + "closedAt": "2026-01-01T00:00:00Z", "enrolled": False} + record = lambda step, **kw: dict(base, step=step, **kw) + good = [record("run", project=A.PROJECT, board=203, done=203, keep=200, planned=1, maxArchive=3), + record("intent", **ref), record("archived", **ref), record("finished", archived=1, skipped=0)] + bad = {} + bad["review probe: missing identities"] = [{"step": "run"}, {"step": "intent"}, + {"step": "archived"}, {"step": "finished"}] + bad["review probe: imaginary three archives"] = [dict(good[0], planned=3), dict(good[-1], archived=3)] + bad["review probe: unknown effect"] = [good[0], record("unknown-effect"), good[-1]] + for label, mutate in [ + ("missing item", lambda x: x[1].pop("itemId")), + ("missing content", lambda x: x[2].pop("contentId")), + ("wrong schema", lambda x: x[1].update(schema="unknown")), + ("changed terminal identity", lambda x: x[2].update(contentId="different")), + ("archived count mismatch", lambda x: x[-1].update(archived=2)), + ("skipped count mismatch", lambda x: x[-1].update(skipped=1)), + ("planned count mismatch", lambda x: x[0].update(planned=3)), + ("wrong bound", lambda x: x[0].update(maxArchive=4)), + ("wrong project", lambda x: x[0].update(project="other")), + ("empty identity", lambda x: x[1].update(itemId="")), + ("invalid close time", lambda x: x[1].update(closedAt="invalid")), + ("invalid timestamp", lambda x: x[1].update(at="invalid")), + ("boolean count", lambda x: x[-1].update(archived=True)), + ]: + value = copy.deepcopy(good); mutate(value); bad[label] = value + bad["terminal without intent"] = [good[0], good[2], good[3]] + bad["repeated terminal"] = good[:3] + [good[2], good[3]] + bad["repeated item"] = [dict(good[0], planned=2)] + good[1:3] * 2 + [dict(good[-1], archived=2)] + bad["unfinished intent"] = good[:2] + [dict(good[-1], archived=0)] + for label, rows in bad.items(): + with self.subTest(probe=label): + (run / "custody/custody.jsonl").write_text("".join(json.dumps(x) + "\n" for x in rows)) + self.assertFalse(A.settled(run, 3)) + (run / "custody/custody.jsonl").write_text("".join(json.dumps(x) + "\n" for x in good)) + self.assertTrue(A.settled(run, 3)) + skipped = [good[0], record("skipped", reason="preimage changed", **ref), record("finished", archived=0, skipped=1)] + (run / "custody/custody.jsonl").write_text("".join(json.dumps(x) + "\n" for x in skipped)) + self.assertTrue(A.settled(run, 3)) + + def test_actual_archiver_custody_joins_bounded_archives_and_skips(self): + rows = FIXTURE.done_history(203) + first, second = rows[2], rows[1] + stub = FIXTURE.Stub(self, scan=[FIXTURE.scan_page(rows[:100], total=203, board=203, has_next=True), + FIXTURE.scan_page(rows[100:200], total=203, board=203, has_next=True, cursor="c2"), + FIXTURE.scan_page(rows[200:], total=203, board=203)], + item=[FIXTURE.item_read(first), FIXTURE.item_read(first, archived=True), + FIXTURE.item_read(second, status="Ready")], + mutation=[{"data": {"archiveProjectV2Item": {"item": {"id": first["itemId"], "isArchived": True}}}}]) + code, stdout, _ = FIXTURE.run(["--execute", "--expect-project-id", A.PROJECT, "--max-archive", "2", + "--enrollment-binding", str(self.binding), "--reader-version", "0.114.1", + "--custody-dir", str(stub.custody), "--lock-dir", str(stub.locks)]) + self.assertEqual(code, 0, stdout) + (Path(stub.dir.name) / "stdout").write_text(stdout) + self.assertTrue(A.settled(Path(stub.dir.name), 2)) + self.assertFalse(A.settled(Path(stub.dir.name), 3)) + self.assertEqual(len(stub.calls("mutation")), 1) + self.assertEqual(stub.records()[-1]["archived"], 1) + self.assertEqual(stub.records()[-1]["skipped"], 1) + + def test_new_directory_entries_are_synced_before_launch(self): + self.c["stateDir"] = str(self.home / "fresh/state") + self.c["outDir"] = str(self.home / "fresh/output") + self.save() + synced = [] + real_sync, real_popen = A.sync_dir, A.subprocess.Popen + def sync(path): + real_sync(path); synced.append((path, set(path.iterdir()))) + def launch(*args, **kw): + run = next((self.home / "fresh/output").iterdir()) + # The parent must be synced AFTER its child entry exists, not merely earlier in traversal. + entries = [self.home / "fresh", self.home / "fresh/state", self.home / "fresh/output", run, + self.home / "fresh/state/in-progress.json", run / "custody", run / "custody/custody.jsonl", + run / "stdout", run / "stderr"] + for entry in entries: + self.assertTrue(any(parent == entry.parent and entry in children for parent, children in synced), str(entry)) + self.assertTrue((run / "custody/custody.jsonl").exists()) + self.assertEqual((run / "custody/custody.jsonl").stat().st_size, 0) + return real_popen(*args, **kw) + with mock.patch.object(A, "sync_dir", side_effect=sync), mock.patch.object(A.subprocess, "Popen", side_effect=launch): + self.assertEqual(self.invoke(), 0) + + def test_directory_sync_failure_prevents_launch_and_preserves_existing_latch(self): + real_sync = A.sync_dir + def fail_custody(path): + if path.name == "custody": + raise OSError("fixture fsync failure after durable latch") + return real_sync(path) + with mock.patch.object(A, "sync_dir", side_effect=fail_custody), mock.patch.object(A.subprocess, "Popen") as launch: + self.assertEqual(self.invoke(), 2) + launch.assert_not_called() + self.assertTrue((self.home / "state/in-progress.json").exists()) + self.assertEqual(self.invoke(), 2) + def test_bad_credentials_modes_version_and_runtime_refuse(self): self.credentials.chmod(0o644); self.assertEqual(self.invoke(), 2) self.credentials.chmod(0o600) From 3f73cb8369ef9b98eb44acd1b9f0270d13e0a860 Mon Sep 17 00:00:00 2001 From: rev Date: Mon, 2 Nov 2026 20:49:32 +0100 Subject: [PATCH 4/4] fix: refresh gated archive route source evidence --- tests/producer-fence-attacks/external-route-evidence.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/producer-fence-attacks/external-route-evidence.json b/tests/producer-fence-attacks/external-route-evidence.json index d23a0b76c..aee56da7b 100644 --- a/tests/producer-fence-attacks/external-route-evidence.json +++ b/tests/producer-fence-attacks/external-route-evidence.json @@ -165,10 +165,10 @@ }, { "path": "scripts/coord-board-archive.py", - "sha256": "a7b21a589777295c83339239dac2e3e786d1b8fb7382358351d569aead6fcc9b", + "sha256": "76f1d033aaa8298cc9f6366329aabd2c9852f6eb5114eccb274be8243d564a5e", "status": "root-run-gated-writer", "gs2089": "unresolved", - "reason": "Project 3 Done-retention archiver (#5119). Its only provider write, archiveProjectV2Item, runs only under --execute, which a Root operator runs on fdev with --expect-project-id, --max-archive, the private enrollment binding and the shared board-writer flock; the hosted workflow runs a counts-only dry run with a read-only App token and never --execute. Fence owner Root approved this route on 2026-11-01." + "reason": "Project 3 Done-retention archiver (#5119, #5170). Predecessor a7b21a589777295c83339239dac2e3e786d1b8fb7382358351d569aead6fcc9b was approved by fence owner Root on 2026-11-01. This reviewed #5170 ranking/reporting successor ranks all valid terminal Done cards before protecting older live blockers. The sole provider write remains archiveProjectV2Item behind --execute, --expect-project-id, --max-archive, the private enrollment binding, qualified reader guard and shared per-card board-writer flock/preimage/intent/one-send/readback; the Root admission boundary is unchanged. The hosted workflow remains a counts-only dry run with a read-only App token. Scheduled fdev installation, activation and native effects remain separately gated on Root acceptance of the first guarded archive. GS2-08.9 remains unresolved; no V1 retirement, historical-client refusal or protection is claimed." }, { "path": "scripts/learn-dispatch.py",