diff --git a/.github/workflows/coord-board-archive.yml b/.github/workflows/coord-board-archive.yml index a9a07959e..9466320a5 100644 --- a/.github/workflows/coord-board-archive.yml +++ b/.github/workflows/coord-board-archive.yml @@ -9,7 +9,8 @@ name: Done retention dry run for the Coordination board # # THIS WORKFLOW ONLY EVER RUNS A DRY RUN. It reports the board size against the 500-item limit, the Done count and # the cards the policy would archive. A real run is a separate, owner-run invocation with the private enrollment -# binding and an exact project id. The schedule gains --execute only after Root accepts the first real run. +# binding and an exact project id. After Root accepts the first real run, the sibling fdev user timer is the +# scheduled writer (deployment/fdev-board-tend); this hosted job remains read-only and counts-only. # # WHY THE PREVIOUS WORKFLOW AT THIS PATH WAS RETIRED (Root, 2026-10-31) # ===================================================================== diff --git a/deployment/fdev-board-tend/fsgg-board-archive-run.py b/deployment/fdev-board-tend/fsgg-board-archive-run.py new file mode 100644 index 000000000..1c2d9adb2 --- /dev/null +++ b/deployment/fdev-board-tend/fsgg-board-archive-run.py @@ -0,0 +1,259 @@ +#!/usr/bin/python3 -I +"""Private fdev Done-retention adapter (#5170). Never enables itself or retries an unresolved run. + +The existing archiver owns native reads, per-card writer exclusion, intent/send/readback and policy. +This adapter pins its activation inputs and preserves a cross-run latch before invoking it. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import stat +import subprocess +import sys +import tempfile + +PROJECT = "PVT_kwDOEYAWY84Bldpa" +SCHEMA = "fsgg.board-archive-config/1" +FIELDS = {"schema", "enabled", "maxArchive", "script", "gh", "binding", "commitment", "readerVersion", + "readerFiles", "acceptedReceipt", "stateDir", "outDir", "workdir"} + + +class Refused(Exception): + pass + + +def private_file(path: Path) -> bytes: + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o600: + raise Refused("private input must be an owner-owned regular 0600 file") + return path.read_bytes() + + +def absolute(text: object) -> Path: + if not isinstance(text, str) or not Path(text).is_absolute(): + raise Refused("absolute path required") + return Path(text) + + +def pin(value: object) -> tuple[Path, str]: + if not isinstance(value, dict) or set(value) != {"path", "sha256"}: + raise Refused("invalid file pin") + path = absolute(value["path"]) + digest = value["sha256"] + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise Refused("invalid file digest") + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o022: + raise Refused("pinned file must be owner-owned, regular and not group/world writable") + if hashlib.sha256(path.read_bytes()).hexdigest() != digest: + raise Refused("pinned file changed; Root must refresh activation") + return path, digest + + +def private_dir(path: Path) -> Path: + # Anchor every ancestor, including entries left by an earlier failed fsync. Creating parents=True and + # syncing only the leaf would not make the leaf's entry in its parent durable after a first-run crash. + for directory in reversed((path, *path.parents)): + directory.mkdir(exist_ok=True, mode=0o700) + if not stat.S_ISDIR(directory.lstat().st_mode): + raise Refused("directory ancestry must not contain symlinks or non-directories") + sync_dir(directory) + if directory.parent != directory: + sync_dir(directory.parent) + info = path.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise Refused("output/state directory must be owner-owned, regular and 0700") + return path + + +def sync_dir(path: Path) -> None: + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def exclusive_json(path: Path, data: dict) -> None: + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as handle: + json.dump(data, handle, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + sync_dir(path.parent) + + +def configuration(path: Path) -> tuple[dict, dict[str, Path]]: + c = json.loads(private_file(path)) + if not isinstance(c, dict) or set(c) != FIELDS or c["schema"] != SCHEMA or c["enabled"] is not True: + raise Refused("missing, disabled or invalid archive activation") + if type(c["maxArchive"]) is not int or not 1 <= c["maxArchive"] <= 500: + raise Refused("Root must supply a positive archive bound at most 500") + version = c["readerVersion"] + if not isinstance(version, str) or not re.fullmatch(r"\d+\.\d+\.\d+", version) or tuple(map(int, version.split("."))) < (0, 114, 1): + raise Refused("qualified archived-membership reader required") + paths = {name: pin(c[name])[0] for name in ("script", "gh", "binding", "commitment", "acceptedReceipt")} + private_file(paths["binding"]) + binding = json.loads(paths["binding"].read_bytes()) + selected = binding.get("selectedIssues") if isinstance(binding, dict) else None + if not isinstance(selected, dict) or not selected: + raise Refused("invalid enrollment binding") + if not isinstance(c["readerFiles"], list) or not c["readerFiles"]: + raise Refused("qualified installed reader file pins required") + for value in c["readerFiles"]: + pin(value) + receipt = json.loads(private_file(paths["acceptedReceipt"])) + expected = {"projectId": PROJECT, "keep": 200, "maxArchive": c["maxArchive"], "readerVersion": version, + "readerFiles": c["readerFiles"], "scriptSha256": c["script"]["sha256"], + "bindingSha256": c["binding"]["sha256"], "commitmentSha256": c["commitment"]["sha256"]} + if (not isinstance(receipt, dict) or receipt.get("schema") != "fsgg.board-archive-accepted/1" + or receipt.get("accepted") is not True or receipt.get("inputs") != expected + or not re.fullmatch(r"https://github\.com/FS-GG/\.github/issues/5170#issuecomment-\d+", str(receipt.get("evidenceUrl")))): + raise Refused("Root's accepted first-real execution must match activation inputs") + for name in ("stateDir", "outDir", "workdir"): + paths[name] = absolute(c[name]) + if Path.cwd().resolve() != paths["workdir"].resolve() or (paths["workdir"] / ".git").exists(): + raise Refused("fixed non-checkout working directory required") + runtime = os.environ.get("XDG_RUNTIME_DIR", "") + if not runtime or not Path(runtime).is_absolute(): + raise Refused("shared XDG_RUNTIME_DIR required") + return c, paths + + +def settled(run: Path, max_archive: int) -> bool: + custody = run / "custody" / "custody.jsonl" + if not custody.exists() or custody.stat().st_size == 0: + return (run / "stdout").read_text().rstrip().endswith("nothing to archive.") + rows = [json.loads(line) for line in custody.read_text().splitlines()] + common = {"schema", "at", "step"} + identity = {"itemId", "contentId", "repo", "number", "closedAt", "enrolled"} + def timestamp(value: object) -> bool: + try: + return isinstance(value, str) and dt.datetime.fromisoformat(value).tzinfo is not None + except ValueError: + return False + def integer(value: object) -> bool: + return type(value) is int and value >= 0 + for row in rows: + if (not isinstance(row, dict) or not common <= row.keys() + or row.get("schema") != "fsgg.board-done-retention-custody/1" + or not timestamp(row.get("at"))): + return False + if len(rows) < 2: + return False + start, end = rows[0], rows[-1] + if (set(start) != common | {"project", "board", "done", "keep", "planned", "maxArchive"} + or start["step"] != "run" or start["project"] != PROJECT or start["keep"] != 200 + or type(start["keep"]) is not int or start["maxArchive"] != max_archive + or not all(integer(start[k]) for k in ("board", "done", "planned", "maxArchive")) + or not 0 < start["planned"] <= start["done"] - 200 or not 200 < start["done"] <= start["board"] <= 500 + or not 1 <= max_archive <= 500 + or set(end) != common | {"archived", "skipped"} or end["step"] != "finished" + or not all(integer(end[k]) for k in ("archived", "skipped"))): + return False + pending = None + seen_items, seen_content = set(), set() + counts = {"archived": 0, "skipped": 0} + for row in rows[1:-1]: + step = row["step"] + keys = common | identity | ({"reason"} if step == "skipped" else set()) + if (step not in ("intent", "archived", "skipped") or set(row) != keys + or any(not isinstance(row[k], str) or not row[k].strip() for k in ("itemId", "contentId", "repo")) + or not re.fullmatch(r"[^/\s]+/[^/\s]+", row["repo"]) + or type(row["number"]) is not int or row["number"] < 1 + or type(row["enrolled"]) is not bool or not timestamp(row["closedAt"]) + or (step == "skipped" and (not isinstance(row["reason"], str) or not row["reason"].strip()))): + return False + ref = {key: row[key] for key in identity} + if pending is not None: + if step != "archived" or ref != pending: + return False + counts["archived"] += 1 + pending = None + else: + if step == "archived" or row["itemId"] in seen_items or row["contentId"] in seen_content: + return False + seen_items.add(row["itemId"]); seen_content.add(row["contentId"]) + if step == "intent": + pending = ref + else: + counts["skipped"] += 1 + return (pending is None and sum(counts.values()) == min(start["planned"], max_archive) + and all(end[k] == count for k, count in counts.items())) + + +def run(config: Path, credentials: Path, timeout: float = 720) -> int: + try: + c, p = configuration(config) + secrets = json.loads(private_file(credentials)) + token = secrets.get("GH_TOKEN") if isinstance(secrets, dict) else None + if not isinstance(token, str) or not token.strip(): + raise Refused("credential file has no GH_TOKEN") + state = private_dir(p["stateDir"]) + latch = state / "in-progress.json" + if latch.exists() or latch.is_symlink(): + raise Refused("unresolved prior run; Root readback required before another invocation") + out = private_dir(p["outDir"]) + stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ-") + directory = Path(tempfile.mkdtemp(prefix=stamp, dir=out)) + sync_dir(directory) + sync_dir(out) + exclusive_json(latch, {"run": str(directory), "configSha256": hashlib.sha256(config.read_bytes()).hexdigest()}) + # Preprovision the child's append-only file, including its directory entry, before any possible send. + custody = private_dir(directory / "custody") + fd = os.open(custody / "custody.jsonl", os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_NOFOLLOW, 0o600) + try: + os.fsync(fd) + finally: + os.close(fd) + sync_dir(custody) + env = {key: os.environ[key] for key in ("HOME", "LANG", "LC_ALL", "XDG_RUNTIME_DIR", "DOTNET_ROOT") if key in os.environ} + env.update(PATH="/usr/local/bin:/usr/bin:/bin", GH_TOKEN=token.strip(), COORD_ARCHIVE_GH=str(p["gh"])) + command = ["/usr/bin/python3", "-I", str(p["script"]), "--owner", "FS-GG", "--project", "3", + "--expect-project-id", PROJECT, "--keep", "200", "--max-archive", str(c["maxArchive"]), + "--enrollment-binding", str(p["binding"]), "--reader-version", c["readerVersion"], + "--custody-dir", str(directory / "custody"), "--execute"] + with os.fdopen(os.open(directory / "stdout", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as stdout, \ + os.fdopen(os.open(directory / "stderr", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as stderr: + os.fsync(stdout.fileno()) + os.fsync(stderr.fileno()) + sync_dir(directory) + process = subprocess.Popen(command, env=env, stdout=stdout, stderr=stderr, start_new_session=True, umask=0o077) + try: + code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + code = 124 + if code != 0 or not settled(directory, c["maxArchive"]): + print("board-archive: stopped; private custody retained, Root readback required", file=sys.stderr) + return 3 + exclusive_json(directory / "completed.json", {"exitCode": code, "settled": True}) + latch.unlink() + sync_dir(state) + print("board-archive: completed; private custody retained") + return 0 + except (Refused, OSError, ValueError, KeyError, TypeError): + # Never emit paths, card identities, tokens or exception text to the journal. + print("board-archive: refused; check private activation inputs or unresolved custody", file=sys.stderr) + return 2 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--credentials", type=Path, required=True) + args = parser.parse_args() + return run(args.config, args.credentials) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/deployment/fdev-board-tend/fsgg-board-archive.service.in b/deployment/fdev-board-tend/fsgg-board-archive.service.in new file mode 100644 index 000000000..3e372f197 --- /dev/null +++ b/deployment/fdev-board-tend/fsgg-board-archive.service.in @@ -0,0 +1,24 @@ +# Rendered by install.py archive; installation does not enable or start this unit (#5170). +[Unit] +Description=FS-GG Project3 guarded Done retention + +[Service] +Type=exec +WorkingDirectory=@WORKDIR@ +Environment=XDG_RUNTIME_DIR=%t +Environment=XDG_CONFIG_HOME=%h/.config +Environment=XDG_STATE_HOME=%h/.local/state +RuntimeDirectory=fsgg +RuntimeDirectoryMode=0700 +RuntimeDirectoryPreserve=yes +UMask=0077 +NoNewPrivileges=yes +# No PrivateTmp: the archiver shares the literal /tmp legacy writer locks with the installed engine. +CPUQuota=100% +MemoryMax=1G +MemorySwapMax=0 +RuntimeMaxSec=900 +TimeoutStopSec=30 +KillMode=control-group +# No Restart: an incomplete run retains the adapter's durable latch for Root readback. +ExecStart=/usr/bin/python3 -I @RELEASE_DIR@/fsgg-board-archive-run.py --config @CONFIG@ --credentials @CREDENTIALS@ diff --git a/deployment/fdev-board-tend/fsgg-board-archive.timer.in b/deployment/fdev-board-tend/fsgg-board-archive.timer.in new file mode 100644 index 000000000..e5ec7eeb9 --- /dev/null +++ b/deployment/fdev-board-tend/fsgg-board-archive.timer.in @@ -0,0 +1,10 @@ +[Unit] +Description=Daily FS-GG Project3 guarded Done retention + +[Timer] +OnCalendar=*-*-* 04:43:00 UTC +Persistent=true +Unit=fsgg-board-archive.service + +[Install] +WantedBy=timers.target diff --git a/deployment/fdev-board-tend/install.py b/deployment/fdev-board-tend/install.py index 60005cf33..f53fbf943 100644 --- a/deployment/fdev-board-tend/install.py +++ b/deployment/fdev-board-tend/install.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 -I """Install the fdev board-tend units (board-reconciler design section 8, #5136 PR 8). User-level only. -Five subcommands, each writing one kind of thing and nothing else. Every one has --dry-run, which prints what it +Six subcommands, each writing one kind of thing and nothing else. Every one has --dry-run, which prints what it would write and changes nothing. None needs or uses root; every destination must be under $HOME. units Renders fsgg-board-tend.{service,timer} and fsgg-board-tend-daily.{service,timer} into the user @@ -16,6 +16,8 @@ the script with --apply. Without --enable it does not touch systemd. chromium-env Copies deployment/fdev-chromium/fdev-chromium-env to ~/.local/bin (the source of truth for the installed wrapper, which sets the short disk-backed TMPDIR default, .github#5240). + archive Stages an immutable archive adapter/script and two daily user units. Never enables or starts + them; Root's accepted first execution and private activation configuration remain mandatory. The engine and the kit root are pinned to one release: the engine must live in .../coord-/ and the kit root must be named kit-. @@ -286,6 +288,29 @@ def command_enable_file(args: argparse.Namespace) -> int: return 0 +def command_archive(args: argparse.Namespace) -> int: + if os.geteuid() == 0: + raise Refusal("run as the fdev user, not root") + release = user_path(args.release_dir, "--release-dir") + workdir = user_path(args.workdir, "--workdir") + values = {"RELEASE_DIR": str(release), "WORKDIR": str(workdir), + "CONFIG": str(user_path(args.config, "--config")), + "CREDENTIALS": str(user_path(args.credentials, "--credentials"))} + unit_dir = user_path(args.unit_dir, "--unit-dir") + plan = Plan(args.dry_run) + # Release files are immutable: --force can replace rendered units, never a pinned release's bytes. + for name, source in (("fsgg-board-archive-run.py", HERE / "fsgg-board-archive-run.py"), + ("coord-board-archive.py", HERE.parents[1] / "scripts/coord-board-archive.py")): + plan.add(release / name, source.read_bytes(), 0o400, False) + for name in ("fsgg-board-archive.service", "fsgg-board-archive.timer"): + plan.add(unit_dir / name, render((HERE / (name + ".in")).read_text(), values).encode(), 0o644, args.force) + plan.apply() + if not args.dry_run: + workdir.mkdir(parents=True, exist_ok=True, mode=0o700) + print("archive staged disabled; systemd not touched; Root owns first-real acceptance and activation") + return 0 + + def parser() -> argparse.ArgumentParser: h = home() root = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) @@ -324,6 +349,14 @@ def parser() -> argparse.ArgumentParser: chromium.add_argument("--dest", default=str(h / ".local/bin/fdev-chromium-env")) chromium.add_argument("--force", action="store_true", help="replace the installed wrapper when it differs") chromium.add_argument("--dry-run", action="store_true") + archive = sub.add_parser("archive") + archive.add_argument("--release-dir", required=True, help="a new immutable, versioned directory under HOME") + archive.add_argument("--workdir", default=str(h / ".local/share/fs-gg/board-archive")) + archive.add_argument("--config", default=str(h / ".config/fsgg/board-archive.json")) + archive.add_argument("--credentials", default=str(h / ".config/fdev/credentials.json")) + archive.add_argument("--unit-dir", default=str(h / ".config/systemd/user")) + archive.add_argument("--force", action="store_true") + archive.add_argument("--dry-run", action="store_true") return root @@ -331,7 +364,8 @@ def main(argv: list[str] | None = None) -> int: args = parser().parse_args(argv) try: return {"units": command_units, "kit-root": command_kit_root, "enable-file": command_enable_file, - "tmp-cleanup": command_tmp_cleanup, "chromium-env": command_chromium_env}[args.command](args) + "tmp-cleanup": command_tmp_cleanup, "chromium-env": command_chromium_env, + "archive": command_archive}[args.command](args) except Refusal as error: print(f"install.py: refused: {error}", file=sys.stderr) return 2 diff --git a/docs/coordination/board-tend-fdev-units.md b/docs/coordination/board-tend-fdev-units.md index 90f3f957e..29af5ad70 100644 --- a/docs/coordination/board-tend-fdev-units.md +++ b/docs/coordination/board-tend-fdev-units.md @@ -239,6 +239,89 @@ activation/readback, separately from source integration. A local certificate est ## Related user units +### Done-retention writer (#5170) + +The hosted `coord-board-archive` workflow stays a **counts-only read-only observer**. Its public logs and +Projects-read App token cannot host the guarded writer. `fsgg-board-archive.{service,timer}` are a separate daily +fdev user route, staged by `install.py archive`; this command never calls systemd, enables a timer, writes an +activation configuration or changes the hourly tend units. The Python archiver remains the only archive writer; +this does not enable A1 or any tend rule. + +``` +python3 -I deployment/fdev-board-tend/install.py archive \ + --release-dir "$HOME/.local/libexec/fsgg-board-archive/" --dry-run +``` + +After review, omit `--dry-run` to stage the adapter/script (0400) and two units. Release files cannot be replaced, +even with `--force`; changed source needs a new release directory. `--force` only replaces rendered unit files. +No installed path points into a working checkout. Root reviews the exact staged hashes before any activation. + +The service uses the existing fdev credentials file, with its 0600 owner check; it requests no new credentials +or grants. The child receives only the credential file's `GH_TOKEN` and the explicitly pinned `gh` executable, +never an inherited token/test override. `XDG_RUNTIME_DIR=%t`, the preserved `fsgg` runtime directory and the +absence of `PrivateTmp` preserve the archiver's per-card shared and legacy writer locks. The adapter does not +hold those locks outside the child. Type=exec, a 900-second service bound, 720-second child bound, one CPU and +1GiB cap bound the run; no Restart directive retries failures. + +**Activation is separate and remains Root's effect, after the first real archive is accepted.** The service's +`--config` file must be an owner-owned regular 0600 file with exactly these keys: + +| Key | Required value | +|---|---| +| `schema`, `enabled` | `fsgg.board-archive-config/1`, explicitly `true`; no configuration is installed by default | +| `maxArchive` | Root's positive per-run bound, at most 500; keep is fixed 200, owner FS-GG/project 3/exact projectID are fixed in the adapter | +| `script`, `gh`, `binding`, `commitment`, `acceptedReceipt` | each exactly `{"path":"","sha256":"<64 hex>"}` | +| `readerVersion`, `readerFiles` | qualified installed reader version >=0.114.1 and a nonempty list of exact file pins for its executable/CLI/adapter closure | +| `stateDir`, `outDir`, `workdir` | absolute private runtime paths; cwd must equal the fixed non-checkout workdir | + +The binding is the **current authoritative private enrollment binding**, not a discovered or historical file. +`commitment` pins its protected public enrollment commitment. Root refreshes and qualifies the pair immediately +before activation; changes to either identity or any reader file refuse until explicitly rebound. Updating the +global engine symlink does not change or qualify the timer reader. A newer global version is not evidence that +it is board-qualified. Missing reader files, stale hashes, unqualified version, stale accepted input pair or +unreadable private credentials refuse before the child starts. + +The private `acceptedReceipt` is Root's accepted first-real-run evidence, not automatically fabricated by the +installer. Its exact shape is `{"schema":"fsgg.board-archive-accepted/1","accepted":true,"evidenceUrl": +"https://github.com/FS-GG/.github/issues/5170#issuecomment-","inputs":{...}}`. `inputs` contains exactly +`projectId`, `keep`200, `maxArchive`, `readerVersion`, `readerFiles`, `scriptSha256`, `bindingSha256` and +`commitmentSha256`, all equal to the accepted activation inputs. The URL points to Root's native acceptance; +the local file retains the private identity join. The adapter verifies the pinned receipt and join locally; +it does not grant acceptance, reinterpret the comment or make an authorization decision from GitHub text. + +All child stdout/stderr, card identities and fsynced custody stay in exclusive 0700 run directories with 0600 +files. Journal messages contain only a generic result. No child log is published. Before starting a child, the +adapter creates and fsyncs `stateDir/in-progress.json`. A zero exit with complete settled custody (or a completed +empty plan) writes a private completion record before removing the latch. A nonzero exit, timeout, interruption, +malformed custody or pending intent keeps the latch. Every later timer activation then refuses before another +child; Root must resolve the original possible effects by readback before clearing it. Never prune unresolved +custody, clear the latch merely because time passed, or blindly resend. The receipt and latch are local +cooperative safeguards, not a boundary against other processes running as this user. + +Before child launch the adapter also fsyncs the state/output directory ancestry and each new run entry in its +parent. It preprovisions and fsyncs the child's custody directory/file and private stdout/stderr entries, so +first-run directory creation cannot leave an otherwise durable latch or custody file unreachable after a crash. +Any preparation fsync failure prevents launch; a failure after latch creation preserves that hold. Successful +custody must use the archiver's exact schema and supported sequence: one run header, per-item reasoned skips or +immediately joined intent/archived pairs with complete identical identities, then one finished record. Duplicate, +unknown or malformed records refuse; processed items must equal min(planned, configured bound), and the final +archived/skipped totals must equal the validated effects. An empty preprovisioned custody file is accepted only +after the pinned child exits0 and reports a completed empty plan. + +Root's remaining sequence is: integrate reviewed ranking source; review the nonzero native plan; admit and +accept one real guarded archive with enrollment-reader readback; qualify this scheduling adapter and exact +activation pair; stage disabled and read back hashes/modes/units; then explicitly daemon-reload and enable +`fsgg-board-archive.timer`. Read back its next trigger and the first scheduled run's private custody/native +effects before accepting #5170's schedule promise. The timer is daily 04:43 UTC, Persistent=true (one catch-up, +no backlog replay). Disable the timer and set `enabled:false` as the kill switch, preserving all custody. + +Preflight is the offline recording-child/archiver fixture: positive exact pins, disabled/stale inputs, +environment/output privacy, actual shared-lock contention, successful settlement and failure/crash/timeout +blocking the next invocation. These directly exercise the effect boundary; no new abstract scheduler model or +browser/CLR run is needed. Run `bash tests/fdev-board-tend/run.sh` and the existing archive fixture under the +ordinary resource budget with disk-backed temporary storage. This is safeguard qualification, with no claim +of measured CI savings or native acceptance. + `fsgg-tmp-cleanup.{service,timer}` (hourly dead-run `/tmp` cleanup, #5240) are templated here and installed by their own `install.py tmp-cleanup` effect; they share nothing with board tend at runtime. See [fdev host hygiene](fdev-host-hygiene.md). @@ -247,6 +330,6 @@ activation/readback, separately from source integration. A local certificate est - The token is the credentials file's `GH_TOKEN`. Whether it carries every scope an eventual `--apply` needs is checked at the first apply admission, not in plan-only. -- The Done-retention archiver is a hosted workflow, outside the lock; until it moves or A1 replaces it, the reconciler - writes nothing to archive-eligible cards (design ยง8). +- The hosted Done-retention observer sends no writes. The sibling fdev writer remains inactive until Root accepts + and activates it; tend still writes nothing to archive-eligible cards until its own design/acceptance changes. - `Persistent=true` catches up one missed hourly run after fdev was down; it does not replay every missed hour. diff --git a/docs/coordination/v1-writer-census.json b/docs/coordination/v1-writer-census.json index 2beb48e8a..eac22a71a 100644 --- a/docs/coordination/v1-writer-census.json +++ b/docs/coordination/v1-writer-census.json @@ -93,7 +93,7 @@ {"path":"scripts/check-skillmirror-freshness.py","disposition":"guard-only","sinkKinds":["dynamic-process"],"sha256":"2e7a72c0418bc132d8fefaa005ab3739c390c980115dd6462ca67aab22e77911","nonWriterJustification":"Read-only checker with embedded refusal fixture; it does not dispatch a production write."}, {"path":"scripts/check-workflow-permissions.py","disposition":"guard-only","sinkKinds":["dynamic-process"],"sha256":"7ed5fe0ddc9e209222fe8eb136869cfd3a970bae4037363be2930c1b60a5ffec","nonWriterJustification":"Read-only checker with embedded refusal fixture; it does not dispatch a production write."}, {"path":"scripts/collect-v1-receiver-census.py","disposition":"guard-only","sinkKinds":["dynamic-process"],"sha256":"c7c1d06e69fc6ba7a525e5d229f2e113763a1f109d9ea428eaba35473e03bfb5","nonWriterJustification":"Bounded source collector invokes only fixed allowlisted GitHub GET routes; write methods and arbitrary endpoints are rejected before subprocess execution."}, - {"path":"scripts/coord-board-archive.py","disposition":"conditional-remote-writer","sinkKinds":["graphql-mutation"],"sha256":"a7b21a589777295c83339239dac2e3e786d1b8fb7382358351d569aead6fcc9b"}, + {"path":"scripts/coord-board-archive.py","disposition":"conditional-remote-writer","sinkKinds":["graphql-mutation"],"sha256":"76f1d033aaa8298cc9f6366329aabd2c9852f6eb5114eccb274be8243d564a5e"}, {"path":"scripts/gs2-09-7-mint-sandbox-token.py","disposition":"credential-lifecycle","sinkKinds":[],"sha256":"392b084bfbf010d0528895400bc3323eaf9a1c921a233a5297bfe6b042ed466c","nonWriterJustification":"The helper mints and may revoke only a sandbox-scoped installation token; it has no V1 data mutation path."}, {"path":"scripts/learn-dispatch.py","disposition":"conditional-remote-writer","sinkKinds":["dynamic-process"],"sha256":"0e359e4d707bf2beb889fb8c7e1bf9da36ec63463736b9a3824a7c3822398093","writerJustification":"Explicit --record mode posts one classification comment to the owning native Delivery issue and submits its digest-bound telemetry observation; offline preparation has no remote effects. Per-Delivery shared lock, durable possible-send custody, exact comment readback and no uncertain automatic resend preserve partial effects. It never writes board fields, merges or dispatches workers. Operation authority remains separate."}, {"path":"scripts/lib/mutation.py","disposition":"test-only","sinkKinds":["graphql-mutation"],"sha256":"d2b7c58495f996836c7f4405757a0daa914bfe7250263c945ee9a4f22a22f183","nonWriterJustification":"Local gate-mutation library; detected syntax is test data, not a production sink."}, diff --git a/scripts/coord-board-archive.py b/scripts/coord-board-archive.py index 13d336c81..ab69e5e71 100755 --- a/scripts/coord-board-archive.py +++ b/scripts/coord-board-archive.py @@ -32,8 +32,9 @@ read completely or the run refuses. This is the "never a live row's blocker" guard the reconciler design keeps. 2. Newest 200 kept. Candidates are ranked by `closedAt` descending, with ties broken by repository, number and item id, so the selection never depends on connection order. The first `--keep` stay; only the rest are - planned. The reconciler (tend) must use exactly this ranking (closedAt, then repository, number and item - id); a shared test pins the two together when tend lands (#5136). + considered for archiving. Protected cards count in this ranking; older protected cards also stay visible, + without shifting the newest-200 cutoff. The reconciler (tend) must use exactly this ranking (closedAt, then + repository, number and item id); a shared test pins the two together when tend lands (#5136). 3. Complete read or nothing. The scan reads `items(query: "status:Done")` in pages of 100, at most 5 pages, the 500-item board limit. A partial page set, a count mismatch, a GraphQL error or an over-limit board refuses without archiving anything. @@ -85,6 +86,7 @@ BOARD_LIMIT = PAGE * MAX_PAGES TERMINAL = ("CLOSED", "MERGED") CUSTODY_SCHEMA = "fsgg.board-done-retention-custody/1" +LIVE_BLOCKER_REASON = "a live card is still blocked by it: stays visible" SCAN = """ query($owner:String!,$number:Int!,$cursor:String){ @@ -282,10 +284,11 @@ def live_blockers(owner: str, number: int) -> tuple[set[str], int]: def plan(rows: list[dict], keep: int = KEEP, protected: set[str] | None = None) -> tuple[list[dict], list[dict], list[tuple[dict, str]]]: - """Split Done rows into (archive, kept, not-rankable-with-reason). Pure, so every guard is testable. + """Split Done rows into (archive, newest-kept, excluded-with-reason). Pure, so every guard is testable. Ranked: Done, issue or PR, terminal state, parseable `closedAt`, not already archived. Newest first by `closedAt`, ties broken by (repo, number, itemId) so connection order never decides. The first `keep` stay. + Only then exclude live blockers from the older candidates; protection never shifts the retention cutoff. """ if keep < 0: raise ValueError("keep must not be negative") @@ -307,13 +310,17 @@ def plan(rows: list[dict], keep: int = KEEP, protected: set[str] | None = None) unranked.append((row, "closedAt is unreadable: never archived")) elif not isinstance(row.get("number"), int) or not row.get("repo"): unranked.append((row, "unreadable repository or number: never archived")) - elif protected and row["contentId"] in protected: - unranked.append((row, "a live card is still blocked by it: stays visible")) else: ranked.append((closed, str(row["repo"]), int(row["number"]), str(row["itemId"]), row)) ranked.sort(key=lambda entry: (-entry[0].timestamp(), entry[1], entry[2], entry[3])) kept = [entry[4] for entry in ranked[:keep]] - archive = [entry[4] for entry in ranked[keep:]] + archive = [] + for entry in ranked[keep:]: + row = entry[4] + if protected and row["contentId"] in protected: + unranked.append((row, LIVE_BLOCKER_REASON)) + else: + archive.append(row) return archive, kept, unranked @@ -535,8 +542,9 @@ def main(argv: list[str] | None = None) -> int: return 1 archive, kept, unranked = plan(rows, args.keep, protected) + ranked_count = len(archive) + len(kept) + sum(why == LIVE_BLOCKER_REASON for _, why in unranked) print(f"board: {project.get('totalCount')} item(s) (limit {BOARD_LIMIT}); Done: {len(rows)}; " - f"ranked {len(archive) + len(kept)}, keep newest {len(kept)}, plan to archive {len(archive)}") + f"ranked {ranked_count}, keep newest {len(kept)}, plan to archive {len(archive)}") for row, why in unranked: print(f" KEEP {'' if args.counts_only else ref_of(row)}: {why}") if enrolled is None: diff --git a/tests/coord-board-archive/test_archive.py b/tests/coord-board-archive/test_archive.py index 75c3b4ba7..99c9e5e37 100644 --- a/tests/coord-board-archive/test_archive.py +++ b/tests/coord-board-archive/test_archive.py @@ -7,6 +7,7 @@ from __future__ import annotations import contextlib +import datetime as dt import importlib.util import io import json @@ -32,6 +33,12 @@ def row(n: int, closed: str | None = None, **kw) -> dict: return base +def done_history(count: int) -> list[dict]: + """A fixed close-time history; larger issue numbers are newer, independent of the clock.""" + start = dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc) + return [row(n, (start + dt.timedelta(days=n)).isoformat()) for n in range(1, count + 1)] + + def node(r: dict) -> dict: content = {"__typename": r["kind"], "id": r["contentId"]} if r["kind"] in ("Issue", "PullRequest"): @@ -199,6 +206,19 @@ def test_every_incomplete_read_refuses(self): class LiveBlockers(unittest.TestCase): + def test_protection_does_not_shift_the_newest_200_done_cutoff(self): + rows = done_history(203) + newer_protected = {f"I_{n}" for n in range(191, 204)} + for protected in (newer_protected, newer_protected | {"I_2"}): + for ordered in (rows, list(reversed(rows))): + with self.subTest(protected_older="I_2" in protected, reversed=ordered[0]["number"] == 203): + archive, kept, excluded = A.plan(ordered, 200, protected) + self.assertEqual([r["number"] for r in kept], list(range(203, 3, -1))) + self.assertEqual([r["number"] for r in archive], + [n for n in (3, 2, 1) if f"I_{n}" not in protected]) + self.assertEqual([r["number"] for r, _ in excluded], [2] if "I_2" in protected else []) + self.assertTrue(all("live card" in why for _, why in excluded)) + def test_a_done_card_a_live_card_is_blocked_by_stays_visible(self): old = "2020-01-01T00:00:00Z" rows = [row(1, old), row(2, old), row(3, old)] @@ -225,6 +245,21 @@ def test_the_dry_run_reports_a_protected_card(self): self.assertIn("plan to archive 2", out) self.assertIn("KEEP FS-GG/.github#1: a live card is still blocked by it", out) + def test_threshold_dry_run_ranks_protected_rows_and_never_sends(self): + rows = done_history(203) + protected = [f"I_{n}" for n in range(191, 204)] + ["I_2"] + stub = Stub(self, scan=[scan_page(rows[:100], total=203, board=204, has_next=True, cursor="c1"), + scan_page(rows[100:200], total=203, board=204, has_next=True, cursor="c2"), + scan_page(rows[200:], total=203, board=204)], + live=[live_page([protected])]) + code, out, _ = run(["--counts-only"]) + self.assertEqual(code, 0, out) + self.assertIn("Done: 203; ranked 203, keep newest 200, plan to archive 2", out) + self.assertIn("KEEP : a live card is still blocked by it", out) + self.assertIn("DRY RUN", out) + self.assertNotIn("FS-GG/", out) + self.assertEqual((len(stub.calls("scan")), stub.calls("item"), stub.calls("mutation")), (3, [], [])) + class Execute(unittest.TestCase): def setUp(self): diff --git a/tests/fdev-board-tend/test_archive_schedule.py b/tests/fdev-board-tend/test_archive_schedule.py new file mode 100644 index 000000000..e6911b45a --- /dev/null +++ b/tests/fdev-board-tend/test_archive_schedule.py @@ -0,0 +1,326 @@ +"""Offline archive scheduling controls: fake HOME, pinned child/stub, no systemd or native effects (#5170).""" +from __future__ import annotations + +import contextlib +import copy +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +DEPLOY = ROOT / "deployment/fdev-board-tend" + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +A = load("archive_schedule", DEPLOY / "fsgg-board-archive-run.py") +I = load("archive_install", DEPLOY / "install.py") +FIXTURE = load("archive_fixture", ROOT / "tests/coord-board-archive/test_archive.py") +FAKE = '''import json,os,pathlib,sys,time +mode = MODE +p=pathlib.Path(sys.argv[sys.argv.index('--custody-dir')+1]);p.mkdir(mode=0o700,exist_ok=True) +(p.parent/'child.json').write_text(json.dumps({'argv':sys.argv[1:],'env':dict(os.environ)})) +print('PRIVATE-CARD-IDENTITY',flush=True) +if mode=='timeout': time.sleep(60) +base={'schema':'fsgg.board-done-retention-custody/1','at':'2026-11-02T00:00:00+00:00'} +ref={'itemId':'PRIVATE-ID','contentId':'PRIVATE-CONTENT','repo':'fixture/repo','number':1, + 'closedAt':'2026-01-01T00:00:00Z','enrolled':False} +def record(step,**kw): return dict(base,step=step,**kw) +rows=[record('run',project='PVT_kwDOEYAWY84Bldpa',board=203,done=203,keep=200,planned=1,maxArchive=3), + record('intent',**ref)] +if mode=='success': rows += [record('archived',**ref),record('finished',archived=1,skipped=0)] +if mode=='unresolved': rows += [record('finished',archived=0,skipped=0)] +if mode=='failed': rows += [record('uncertain',reason='fixture failure',**ref)] +(p/'custody.jsonl').write_text(''.join(json.dumps(x)+'\\n' for x in rows)) +sys.exit(3 if mode=='failed' else 0) +''' + + +class ArchiveSchedule(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.home = Path(self.temp.name) + self.env = mock.patch.dict(os.environ, {"HOME": str(self.home), "XDG_RUNTIME_DIR": str(self.home / "runtime"), + "GH_TOKEN": "WRONG", "GITHUB_TOKEN": "WRONG", "TMPDIR": "WRONG", + "COORD_ARCHIVE_GH": "/untrusted/gh"}) + self.env.start(); self.addCleanup(self.env.stop) + self.cwd = Path.cwd(); os.chdir(self.home); self.addCleanup(os.chdir, self.cwd) + self.config = self.home / "config.json" + self.credentials = self.write("credentials.json", {"GH_TOKEN": "FIXTURE-TOKEN"}) + self.script = self.write("archive.py", FAKE.replace("MODE", repr("success"))) + self.gh = self.write("gh", "fixture") + self.binding = self.write("binding.json", {"selectedIssues": {"fixture-node": {}}}) + self.commitment = self.write("commitment.json", {"commitmentSha256": "fixture"}) + self.reader = self.write("reader", "qualified fixture reader") + self.receipt = self.home / "receipt.json" + self.c = {"schema": A.SCHEMA, "enabled": True, "maxArchive": 3, "readerVersion": "0.114.1", + "script": self.pin(self.script), "gh": self.pin(self.gh), "binding": self.pin(self.binding), + "commitment": self.pin(self.commitment), "readerFiles": [self.pin(self.reader)], + "stateDir": str(self.home / "state"), "outDir": str(self.home / "out"), "workdir": str(self.home)} + self.rebind() + + def write(self, name, value): + path = self.home / name + path.write_text(value if isinstance(value, str) else json.dumps(value)) + path.chmod(0o600) + return path + + def pin(self, path): + return {"path": str(path), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + + def rebind(self): + self.c["script"] = self.pin(self.script) + inputs = {"projectId": A.PROJECT, "keep": 200, "maxArchive": self.c["maxArchive"], + "readerVersion": self.c["readerVersion"], "readerFiles": self.c["readerFiles"], + "scriptSha256": self.c["script"]["sha256"], "bindingSha256": self.c["binding"]["sha256"], + "commitmentSha256": self.c["commitment"]["sha256"]} + self.write("receipt.json", {"schema": "fsgg.board-archive-accepted/1", "accepted": True, "inputs": inputs, + "evidenceUrl": "https://github.com/FS-GG/.github/issues/5170#issuecomment-1"}) + self.c["acceptedReceipt"] = self.pin(self.receipt) + self.save() + + def save(self): + self.write("config.json", self.c) + + def invoke(self, timeout=5): + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = A.run(self.config, self.credentials, timeout) + self.assertNotIn("PRIVATE", out.getvalue() + err.getvalue()) + self.assertNotIn("FIXTURE-TOKEN", out.getvalue() + err.getvalue()) + return code + + def test_success_pins_policy_environment_and_private_outputs(self): + self.assertEqual(self.invoke(), 0) + self.assertFalse((self.home / "state/in-progress.json").exists()) + run = next((self.home / "out").iterdir()) + child = json.loads((run / "child.json").read_text()) + args, env = child["argv"], child["env"] + for flag, value in (("--keep", "200"), ("--max-archive", "3"), ("--project", "3"), + ("--expect-project-id", A.PROJECT), ("--enrollment-binding", str(self.binding)), + ("--reader-version", "0.114.1")): + self.assertEqual(args[args.index(flag) + 1], value) + self.assertIn("--execute", args) + self.assertEqual(env["GH_TOKEN"], "FIXTURE-TOKEN") + self.assertEqual(env["COORD_ARCHIVE_GH"], str(self.gh)) + self.assertNotIn("GITHUB_TOKEN", env); self.assertNotIn("TMPDIR", env) + self.assertIn("PRIVATE-CARD-IDENTITY", (run / "stdout").read_text()) + for path in (run, self.home / "state", self.home / "out"): + self.assertEqual(path.stat().st_mode & 0o777, 0o700) + for path in run.rglob("*"): + if path.is_file(): self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.invoke(), 0) + + def test_disabled_missing_receipt_and_stale_acceptance_refuse_before_child(self): + self.c["enabled"] = False; self.save(); self.assertEqual(self.invoke(), 2) + self.c["enabled"] = True; self.c["maxArchive"] = 4; self.save(); self.assertEqual(self.invoke(), 2) + self.receipt.unlink(); self.assertEqual(self.invoke(), 2) + self.assertFalse((self.home / "out").exists()) + + def test_each_changed_identity_refuses_before_child(self): + for path in (self.binding, self.commitment, self.reader, self.script, self.gh): + with self.subTest(path=path.name): + before = path.read_bytes(); path.write_bytes(before + b" ") + self.assertEqual(self.invoke(), 2) + path.write_bytes(before) + self.assertFalse((self.home / "out").exists()) + + def test_unaccepted_receipt_and_private_input_symlink_refuse(self): + receipt = json.loads(self.receipt.read_text()); receipt["accepted"] = False + self.receipt.write_text(json.dumps(receipt)); self.c["acceptedReceipt"] = self.pin(self.receipt); self.save() + self.assertEqual(self.invoke(), 2) + self.rebind() + link = self.home / "binding-link"; link.symlink_to(self.binding) + self.c["binding"]["path"] = str(link); self.save() + self.assertEqual(self.invoke(), 2) + self.assertFalse((self.home / "out").exists()) + + def test_completed_empty_plan_releases_latch(self): + self.script.write_text("print('nothing to archive.')\n"); self.rebind() + self.assertEqual(self.invoke(), 0) + self.assertFalse((self.home / "state/in-progress.json").exists()) + + def test_malformed_custody_keeps_latch(self): + self.script.write_text(FAKE.replace("MODE", repr("success")) + .replace("sys.exit(3 if mode=='failed' else 0)", + "(p/'custody.jsonl').write_text('[}');sys.exit(0)")) + self.rebind() + self.assertEqual(self.invoke(), 2) + self.assertTrue((self.home / "state/in-progress.json").exists()) + self.assertEqual(self.invoke(), 2) + + def test_json_valid_but_impossible_custody_never_settles(self): + run = self.home / "probe"; (run / "custody").mkdir(parents=True) + base = {"schema": "fsgg.board-done-retention-custody/1", "at": "2026-11-02T00:00:00+00:00"} + ref = {"itemId": "card", "contentId": "content", "repo": "fixture/repo", "number": 1, + "closedAt": "2026-01-01T00:00:00Z", "enrolled": False} + record = lambda step, **kw: dict(base, step=step, **kw) + good = [record("run", project=A.PROJECT, board=203, done=203, keep=200, planned=1, maxArchive=3), + record("intent", **ref), record("archived", **ref), record("finished", archived=1, skipped=0)] + bad = {} + bad["review probe: missing identities"] = [{"step": "run"}, {"step": "intent"}, + {"step": "archived"}, {"step": "finished"}] + bad["review probe: imaginary three archives"] = [dict(good[0], planned=3), dict(good[-1], archived=3)] + bad["review probe: unknown effect"] = [good[0], record("unknown-effect"), good[-1]] + for label, mutate in [ + ("missing item", lambda x: x[1].pop("itemId")), + ("missing content", lambda x: x[2].pop("contentId")), + ("wrong schema", lambda x: x[1].update(schema="unknown")), + ("changed terminal identity", lambda x: x[2].update(contentId="different")), + ("archived count mismatch", lambda x: x[-1].update(archived=2)), + ("skipped count mismatch", lambda x: x[-1].update(skipped=1)), + ("planned count mismatch", lambda x: x[0].update(planned=3)), + ("wrong bound", lambda x: x[0].update(maxArchive=4)), + ("wrong project", lambda x: x[0].update(project="other")), + ("empty identity", lambda x: x[1].update(itemId="")), + ("invalid close time", lambda x: x[1].update(closedAt="invalid")), + ("invalid timestamp", lambda x: x[1].update(at="invalid")), + ("boolean count", lambda x: x[-1].update(archived=True)), + ]: + value = copy.deepcopy(good); mutate(value); bad[label] = value + bad["terminal without intent"] = [good[0], good[2], good[3]] + bad["repeated terminal"] = good[:3] + [good[2], good[3]] + bad["repeated item"] = [dict(good[0], planned=2)] + good[1:3] * 2 + [dict(good[-1], archived=2)] + bad["unfinished intent"] = good[:2] + [dict(good[-1], archived=0)] + for label, rows in bad.items(): + with self.subTest(probe=label): + (run / "custody/custody.jsonl").write_text("".join(json.dumps(x) + "\n" for x in rows)) + self.assertFalse(A.settled(run, 3)) + (run / "custody/custody.jsonl").write_text("".join(json.dumps(x) + "\n" for x in good)) + self.assertTrue(A.settled(run, 3)) + skipped = [good[0], record("skipped", reason="preimage changed", **ref), record("finished", archived=0, skipped=1)] + (run / "custody/custody.jsonl").write_text("".join(json.dumps(x) + "\n" for x in skipped)) + self.assertTrue(A.settled(run, 3)) + + def test_actual_archiver_custody_joins_bounded_archives_and_skips(self): + rows = FIXTURE.done_history(203) + first, second = rows[2], rows[1] + stub = FIXTURE.Stub(self, scan=[FIXTURE.scan_page(rows[:100], total=203, board=203, has_next=True), + FIXTURE.scan_page(rows[100:200], total=203, board=203, has_next=True, cursor="c2"), + FIXTURE.scan_page(rows[200:], total=203, board=203)], + item=[FIXTURE.item_read(first), FIXTURE.item_read(first, archived=True), + FIXTURE.item_read(second, status="Ready")], + mutation=[{"data": {"archiveProjectV2Item": {"item": {"id": first["itemId"], "isArchived": True}}}}]) + code, stdout, _ = FIXTURE.run(["--execute", "--expect-project-id", A.PROJECT, "--max-archive", "2", + "--enrollment-binding", str(self.binding), "--reader-version", "0.114.1", + "--custody-dir", str(stub.custody), "--lock-dir", str(stub.locks)]) + self.assertEqual(code, 0, stdout) + (Path(stub.dir.name) / "stdout").write_text(stdout) + self.assertTrue(A.settled(Path(stub.dir.name), 2)) + self.assertFalse(A.settled(Path(stub.dir.name), 3)) + self.assertEqual(len(stub.calls("mutation")), 1) + self.assertEqual(stub.records()[-1]["archived"], 1) + self.assertEqual(stub.records()[-1]["skipped"], 1) + + def test_new_directory_entries_are_synced_before_launch(self): + self.c["stateDir"] = str(self.home / "fresh/state") + self.c["outDir"] = str(self.home / "fresh/output") + self.save() + synced = [] + real_sync, real_popen = A.sync_dir, A.subprocess.Popen + def sync(path): + real_sync(path); synced.append((path, set(path.iterdir()))) + def launch(*args, **kw): + run = next((self.home / "fresh/output").iterdir()) + # The parent must be synced AFTER its child entry exists, not merely earlier in traversal. + entries = [self.home / "fresh", self.home / "fresh/state", self.home / "fresh/output", run, + self.home / "fresh/state/in-progress.json", run / "custody", run / "custody/custody.jsonl", + run / "stdout", run / "stderr"] + for entry in entries: + self.assertTrue(any(parent == entry.parent and entry in children for parent, children in synced), str(entry)) + self.assertTrue((run / "custody/custody.jsonl").exists()) + self.assertEqual((run / "custody/custody.jsonl").stat().st_size, 0) + return real_popen(*args, **kw) + with mock.patch.object(A, "sync_dir", side_effect=sync), mock.patch.object(A.subprocess, "Popen", side_effect=launch): + self.assertEqual(self.invoke(), 0) + + def test_directory_sync_failure_prevents_launch_and_preserves_existing_latch(self): + real_sync = A.sync_dir + def fail_custody(path): + if path.name == "custody": + raise OSError("fixture fsync failure after durable latch") + return real_sync(path) + with mock.patch.object(A, "sync_dir", side_effect=fail_custody), mock.patch.object(A.subprocess, "Popen") as launch: + self.assertEqual(self.invoke(), 2) + launch.assert_not_called() + self.assertTrue((self.home / "state/in-progress.json").exists()) + self.assertEqual(self.invoke(), 2) + + def test_bad_credentials_modes_version_and_runtime_refuse(self): + self.credentials.chmod(0o644); self.assertEqual(self.invoke(), 2) + self.credentials.chmod(0o600) + self.c["readerVersion"] = "0.114.0"; self.rebind(); self.assertEqual(self.invoke(), 2) + self.c["readerVersion"] = "0.114.1"; self.rebind() + with mock.patch.dict(os.environ, {"XDG_RUNTIME_DIR": "relative"}): self.assertEqual(self.invoke(), 2) + + def test_failed_zero_exit_unresolved_and_timeout_block_next_invocation(self): + for mode in ("failed", "unresolved", "timeout"): + with self.subTest(mode=mode): + self.script.write_text(FAKE.replace("MODE", repr(mode))); self.rebind() + self.assertEqual(self.invoke(timeout=0.1), 3) + latch = self.home / "state/in-progress.json" + self.assertTrue(latch.exists()) + runs = list((self.home / "out").iterdir()) + self.assertEqual(self.invoke(), 2) + self.assertEqual(list((self.home / "out").iterdir()), runs) + # Only fixture cleanup: production requires Root's resolved readback. + latch.unlink() + + def test_simulated_crash_latch_blocks_even_with_current_inputs(self): + state = self.home / "state"; state.mkdir(mode=0o700) + A.exclusive_json(state / "in-progress.json", {"run": "interrupted fixture"}) + self.assertEqual(self.invoke(), 2) + self.assertFalse((self.home / "out").exists()) + + def test_actual_archiver_shared_lock_busy_sends_no_mutation(self): + import fcntl + rows = FIXTURE.done_history(203) + stub = FIXTURE.Stub(self, scan=[FIXTURE.scan_page(rows[:100], total=203, board=203, has_next=True), + FIXTURE.scan_page(rows[100:200], total=203, board=203, has_next=True, cursor="c2"), + FIXTURE.scan_page(rows[200:], total=203, board=203)]) + self.script.write_bytes((ROOT / "scripts/coord-board-archive.py").read_bytes()) + self.c["gh"] = self.pin(stub.path); self.rebind() + shared = self.home / "runtime/fsgg"; shared.mkdir(parents=True, mode=0o700) + with (shared / f"board-writer-{A.PROJECT}.lock").open("w") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + self.assertEqual(self.invoke(), 3) + self.assertEqual(stub.calls("mutation"), []) + self.assertEqual(stub.calls("item"), []) + self.assertEqual(self.invoke(), 2) + + def test_installer_dry_run_and_disabled_staging_leave_tend_untouched(self): + release = self.home / "release-source-pin" + args = ["archive", "--release-dir", str(release)] + with mock.patch.object(I.subprocess, "run", side_effect=AssertionError("systemd must not run")): + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(I.main(args + ["--dry-run"]), 0) + self.assertFalse(release.exists()) + self.assertEqual(I.main(args), 0) + units = self.home / ".config/systemd/user" + self.assertEqual({x.name for x in units.iterdir()}, {"fsgg-board-archive.service", "fsgg-board-archive.timer"}) + service = (units / "fsgg-board-archive.service").read_text() + self.assertIn("Type=exec", service); self.assertIn("RuntimeMaxSec=900", service) + self.assertIn("Environment=XDG_RUNTIME_DIR=%t", service) + self.assertIn("CPUQuota=100%", service); self.assertIn("MemoryMax=1G", service) + self.assertNotIn("PrivateTmp=", service); self.assertNotIn("Restart=", service) + self.assertEqual((release / "coord-board-archive.py").stat().st_mode & 0o777, 0o400) + (release / "coord-board-archive.py").chmod(0o600) + (release / "coord-board-archive.py").write_text("changed") + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(I.main(args + ["--force"]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/producer-fence-attacks/external-route-evidence.json b/tests/producer-fence-attacks/external-route-evidence.json index d23a0b76c..aee56da7b 100644 --- a/tests/producer-fence-attacks/external-route-evidence.json +++ b/tests/producer-fence-attacks/external-route-evidence.json @@ -165,10 +165,10 @@ }, { "path": "scripts/coord-board-archive.py", - "sha256": "a7b21a589777295c83339239dac2e3e786d1b8fb7382358351d569aead6fcc9b", + "sha256": "76f1d033aaa8298cc9f6366329aabd2c9852f6eb5114eccb274be8243d564a5e", "status": "root-run-gated-writer", "gs2089": "unresolved", - "reason": "Project 3 Done-retention archiver (#5119). Its only provider write, archiveProjectV2Item, runs only under --execute, which a Root operator runs on fdev with --expect-project-id, --max-archive, the private enrollment binding and the shared board-writer flock; the hosted workflow runs a counts-only dry run with a read-only App token and never --execute. Fence owner Root approved this route on 2026-11-01." + "reason": "Project 3 Done-retention archiver (#5119, #5170). Predecessor a7b21a589777295c83339239dac2e3e786d1b8fb7382358351d569aead6fcc9b was approved by fence owner Root on 2026-11-01. This reviewed #5170 ranking/reporting successor ranks all valid terminal Done cards before protecting older live blockers. The sole provider write remains archiveProjectV2Item behind --execute, --expect-project-id, --max-archive, the private enrollment binding, qualified reader guard and shared per-card board-writer flock/preimage/intent/one-send/readback; the Root admission boundary is unchanged. The hosted workflow remains a counts-only dry run with a read-only App token. Scheduled fdev installation, activation and native effects remain separately gated on Root acceptance of the first guarded archive. GS2-08.9 remains unresolved; no V1 retirement, historical-client refusal or protection is claimed." }, { "path": "scripts/learn-dispatch.py",