From 54e691457483f2cea712dd6d44f489a45c30d338 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 23 Aug 2026 17:59:51 +0000 Subject: [PATCH] =?UTF-8?q?brain:=20hygiene=20runs=20in=20the=20cloud=20?= =?UTF-8?q?=E2=80=94=20the=20board=20needs=20no=20machine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The human's call: as much of the morning as possible in the cloud. Community, overnight, readiness, versions, resume, autonomy and the trend already were; hygiene's content scans never needed a machine either โ€” they need checkouts. brain_board.yml now clones the body-map scan set (libraries + organs + workspaces) blobless with a sparse text/code checkout (big workspace datasets never reach the runner; a failed clone just lowers repos_present โ€” hygiene reports coverage honestly) and the render runs the conductor's own --json pre-scan (BOARD_HYGIENE_SCAN=1, opt-in so terminal digests stay instant). The board gains a ๐Ÿงผ Hygiene section (rows verbatim, chips = each row's own delegate door, honest N/M coverage header); the Dev box section keeps only what the cloud cannot see โ€” worktree state โ€” its hygiene rows rendering only when no cloud scan ran. Hardening the contract the render now depends on: a crashing hygiene helper used to emit an empty row and corrupt the whole --json document (',,'); helper-backed rows now degrade to a per-mode error row, pinned by a nonexistent-root test. 426 tests pass; firewall gate green; real-conductor smoke on a partial workspace (4/18 repos) renders honest partial rows. Co-Authored-By: Claude --- .github/workflows/brain_board.yml | 39 +++++++++- agents/conductors/hygiene/hygiene.sh | 39 +++++----- board/AGENTS.md | 3 +- board/_board.py | 102 +++++++++++++++++++++++---- tests/test_board.py | 45 +++++++++++- tests/test_hygiene_conductor.py | 13 ++++ 6 files changed, 203 insertions(+), 38 deletions(-) diff --git a/.github/workflows/brain_board.yml b/.github/workflows/brain_board.yml index 9b3a998..6c20188 100644 --- a/.github/workflows/brain_board.yml +++ b/.github/workflows/brain_board.yml @@ -65,11 +65,48 @@ jobs: with: repository: ${{ github.repository_owner }}/PyAutoMind path: PyAutoMind + # Hygiene runs IN THE CLOUD: check out the conductor's scan set + # (libraries + organs + workspaces, from the body map) so its --json + # pre-scan needs no dev box at all. Blobless clones + a sparse checkout + # of text/code files keep the big workspace datasets off the runner; + # a repo that fails to clone is simply absent โ€” hygiene reports + # repos_present honestly rather than pretending it scanned everything. + - name: check out the hygiene scan set (blobless + sparse) + run: | + python3 -m pip install --quiet pyyaml + python3 - <<'EOF' + import subprocess, yaml + from pathlib import Path + repos = yaml.safe_load(Path("PyAutoMind/repos.yaml").read_text())["repos"] + for name, meta in sorted(repos.items()): + if meta.get("category") not in ("library", "organ", "workspace"): + continue + if Path(name).exists(): + continue + home = meta.get("github") + if not home: + continue + url = f"https://github.com/{home}" + try: + subprocess.run(["git", "clone", "--quiet", "--depth", "1", + "--filter=blob:none", "--no-checkout", + url, name], check=True, timeout=300) + subprocess.run(["git", "-C", name, "sparse-checkout", "set", + "--no-cone", "*.py", "*.ipynb", "*.rst", + "*.md", "*.txt", "*.yaml", "*.yml", "*.toml", + "*.cfg", "*.ini", "*.sh", ".gitattributes"], + check=True, timeout=60) + subprocess.run(["git", "-C", name, "checkout", "--quiet"], + check=True, timeout=600) + except (subprocess.CalledProcessError, + subprocess.TimeoutExpired) as e: + print(f"::warning::scan checkout skipped {name}: {e}") + EOF - name: render the board env: PYAUTO_ROOT: ${{ github.workspace }} + BOARD_HYGIENE_SCAN: "1" run: | - python3 -m pip install --quiet pyyaml python3 PyAutoBrain/board/_board.py --apply --out _site # enablement: true creates the Pages site on first run where the token # may (Mind/Heart precedent); where it may not (the Hands hit "Resource diff --git a/agents/conductors/hygiene/hygiene.sh b/agents/conductors/hygiene/hygiene.sh index 141b7e0..946c71a 100755 --- a/agents/conductors/hygiene/hygiene.sh +++ b/agents/conductors/hygiene/hygiene.sh @@ -627,26 +627,25 @@ emit_json_row() { # mode "$m" "${MODE_KIND[$m]}" "${UNSCANNED_REASON//\"/\\\"}" "${MODE_DELEGATE[$m]}" return fi - if [[ "$m" == "docstrings" ]]; then - python3 "$HERE/_hygiene_docstrings.py" --root "$ROOT" --json-row - return - fi - if [[ "$m" == "escapes" ]]; then - python3 "$HERE/_hygiene_escapes.py" --root "$ROOT" --json-row - return - fi - if [[ "$m" == "refs" ]]; then - python3 "$HERE/_hygiene_refs.py" --root "$ROOT" --json-row - return - fi - if [[ "$m" == "optdeps" ]]; then - python3 "$HERE/_hygiene_optdeps.py" --root "$ROOT" --json-row - return - fi - if [[ "$m" == "extras" ]]; then - python3 "$HERE/_hygiene_extras.py" --root "$ROOT" --json-row - return - fi + # Helper-backed rows: a crashing helper must degrade to an error ROW, never + # corrupt the enclosing document โ€” the Brain board's cloud render parses + # this output, and one bad row would blank the whole hygiene section. + helper_row() { # helper-module mode + local out + if out="$(python3 "$HERE/$1" --root "$ROOT" --json-row 2>/dev/null)" \ + && [[ -n "$out" ]] \ + && python3 -c 'import json,sys; json.loads(sys.argv[1])' "$out" 2>/dev/null; then + printf '%s' "$out" + else + printf '{"mode":"%s","kind":"%s","status":"error","count":null,"summary":"helper failed โ€” run: pyauto-brain hygiene %s","delegate":"%s"}' \ + "$2" "${MODE_KIND[$2]}" "$2" "${MODE_DELEGATE[$2]}" + fi + } + if [[ "$m" == "docstrings" ]]; then helper_row _hygiene_docstrings.py docstrings; return; fi + if [[ "$m" == "escapes" ]]; then helper_row _hygiene_escapes.py escapes; return; fi + if [[ "$m" == "refs" ]]; then helper_row _hygiene_refs.py refs; return; fi + if [[ "$m" == "optdeps" ]]; then helper_row _hygiene_optdeps.py optdeps; return; fi + if [[ "$m" == "extras" ]]; then helper_row _hygiene_extras.py extras; return; fi local res count summary kind status res="$(prescan "$m")"; count="${res%%|*}"; summary="${res#*|}"; kind="${MODE_KIND[$m]}" if [[ "$kind" == "advisory" || "$count" == "-1" ]]; then status="advisory" diff --git a/board/AGENTS.md b/board/AGENTS.md index f18cbb5..918eaf5 100644 --- a/board/AGENTS.md +++ b/board/AGENTS.md @@ -22,7 +22,8 @@ copy-for-Claude payload: | ๐Ÿ’ฌ Community | the Ears (`community scan`, reused wholesale) โ€” every open conversation gets a row | `/community`, `/community triage ` | | ๐Ÿ”„ Resume | the Mind's registry + generated counts; pending-release PRs | `/start_dev โ€ฆ`, `/prm ` | | ๐Ÿงน Upkeep | open-issue count; the cleanup doors | `/issue_cleanup`, `/hygiene`, `/repo_cleanup` | -| ๐Ÿ–ฅ๏ธ Dev box | `state/devbox_board.json` โ€” hygiene pre-scan rows + worktree state, pushed by `board publish` (morning.sh's last step); age-stamped, stale at 48h, dropped after 7d | each hygiene row's own delegate door | +| ๐Ÿงผ Hygiene | the hygiene conductor's own `--json` pre-scan, run IN this render (BOARD_HYGIENE_SCAN=1; brain_board.yml checks out the body-map scan set blobless+sparse first) โ€” no machine involved | each row's own delegate door | +| ๐Ÿ–ฅ๏ธ Dev box | `state/devbox_board.json` โ€” worktree state (unpushed/dirty/stashes โ€” the one thing only the dev box can see; its hygiene rows render only when no cloud scan ran), pushed by `board publish` (morning.sh's last step); age-stamped, stale at 48h, dropped after 7d | โ€” | | ๐Ÿค– Autonomous runs | the tail of the Mind's `autonomy_log.md`, verbatim | โ€” | | ๐Ÿšช All doors | `bin/pyauto-brain`'s own registry (never a second copy) | `/` | diff --git a/board/_board.py b/board/_board.py index 7c8e38e..ffa1828 100755 --- a/board/_board.py +++ b/board/_board.py @@ -360,6 +360,46 @@ def collect_open_issues(org, degraded): return res.get("total_count") +# The board is cloud-first: with BOARD_HYGIENE_SCAN=1 (set by brain_board.yml, +# whose checkout step clones the body-map scan set) the collect runs the +# hygiene conductor's own fast pre-scan right here, so hygiene needs no +# machine at all. Opt-in by env because a terminal `pyauto-brain board` +# digest should stay instant. +HYGIENE_CMD = os.environ.get( + "BOARD_HYGIENE_CMD", + str(BRAIN_HOME / "agents" / "conductors" / "hygiene" / "hygiene.sh")) + +# Rows in these states carry nothing actionable for the morning glance. +HYGIENE_QUIET_STATUSES = ("clean", "unscanned", "deferred", "advisory") + + +def collect_hygiene(degraded): + """The hygiene conductor's --json pre-scan, run in THIS render (cloud or + local โ€” wherever the scan set is checked out). None when not enabled.""" + if os.environ.get("BOARD_HYGIENE_SCAN") != "1": + return None + try: + r = subprocess.run(["bash", HYGIENE_CMD, "--json"], + capture_output=True, text=True, timeout=900) + decision = json.loads(r.stdout) if r.returncode == 0 else None + except (OSError, subprocess.TimeoutExpired, json.JSONDecodeError): + decision = None + if decision is None: + degraded.append("hygiene: pre-scan failed โ€” rows unavailable this render") + return None + return { + "repos_present": decision.get("repos_present"), + "repos_declared": decision.get("repos_declared"), + "rows": [{ + "mode": row.get("mode"), + "status": row.get("status"), + "count": row.get("count"), + "summary": str(row.get("summary", ""))[:200], + "delegate": row.get("delegate"), + } for row in decision.get("rows", [])], + } + + # Dev-box observations (state/devbox_board.json, pushed by `board publish` โ€” # usually via bin/morning.sh) are honest only with an age: fresh under 48h, # shown stale up to 7d, then dropped with a re-run hint. @@ -501,6 +541,7 @@ def collect(): "community": community, "resume": resume, "open_issues": open_issues, + "hygiene": collect_hygiene(degraded), "devbox": collect_devbox(), "autonomy": collect_autonomy(), "doors": collect_doors(), @@ -690,20 +731,37 @@ def render_md(data): if data["open_issues"] is not None: L.append(f"- {data['open_issues']} open issue(s) org-wide โ€” reconcile " "via `/issue_cleanup` (closing stays confirmation-gated)") - L.append("- `/hygiene` โ€” code-quality debt sweep (local)") + L.append("- `/hygiene` โ€” code-quality debt sweep") L.append("- `/repo_cleanup` โ€” stale branches / stashes / dirty checkouts (local)") L.append("") + hygiene = data.get("hygiene") + if hygiene: + L.append(f"## ๐Ÿงผ Hygiene (scanned this render โ€” " + f"{hygiene.get('repos_present')}/{hygiene.get('repos_declared')} " + "repos present)") + flagged = [r for r in hygiene["rows"] + if r.get("status") not in HYGIENE_QUIET_STATUSES] + if flagged: + for row in flagged: + summary = str(row.get("summary", ""))[:140] + L.append(f"- {row.get('mode')}: {summary} โ†’ `{row.get('delegate')}`") + else: + L.append("- nothing flagged") + L.append("") devbox = data.get("devbox") if devbox: stale = " โ€” STALE, re-run `bash PyAutoBrain/bin/morning.sh`" \ if devbox.get("stale") else "" L.append(f"## ๐Ÿ–ฅ๏ธ Dev box (observed {age_label(devbox['age_h'])} " f"ago via morning.sh{stale})") - for row in devbox.get("hygiene", {}).get("rows", []): - if row.get("status") in ("clean", "unscanned", "deferred", "advisory"): - continue - summary = str(row.get("summary", ""))[:140] - L.append(f"- {row.get('mode')}: {summary} โ†’ `{row.get('delegate')}`") + # The cloud scan supersedes the dev box's hygiene rows; only the + # worktree state (unknowable from the cloud) still needs this vantage. + if not hygiene: + for row in devbox.get("hygiene", {}).get("rows", []): + if row.get("status") in HYGIENE_QUIET_STATUSES: + continue + summary = str(row.get("summary", ""))[:140] + L.append(f"- {row.get('mode')}: {summary} โ†’ `{row.get('delegate')}`") for wt in devbox.get("worktrees", []): bits = [wt.get("branch") or "?"] if wt.get("ahead"): @@ -970,10 +1028,25 @@ def community_row(e, note_html): H.append(_row(f"{issue_note}reconcile the trackers (closing stays " "confirmation-gated).", "/issue_cleanup")) H.append(_row("Code-quality debt sweep โ€” slow tests, CLI noise, dep-cap " - "drift (runs locally).", "/hygiene")) + "drift (the Hygiene section below is its scan).", "/hygiene")) H.append(_row("Stale branches, stashes, dirty checkouts (runs locally).", "/repo_cleanup")) + hygiene = data.get("hygiene") + if hygiene: + H.append(f'

๐Ÿงผ Hygiene (scanned this render โ€” ' + f'{hygiene.get("repos_present")}/{hygiene.get("repos_declared")} ' + "repos present)

") + flagged = [r for r in hygiene["rows"] + if r.get("status") not in HYGIENE_QUIET_STATUSES] + if flagged: + for row in flagged: + summary = esc(str(row.get("summary", ""))[:140]) + H.append(_row(f'{esc(str(row.get("mode")))}: {summary}', + str(row.get("delegate") or "/hygiene"))) + else: + H.append(_plain('โœ“ nothing flagged')) + devbox = data.get("devbox") if devbox: stale = (' โ€” STALE' @@ -984,12 +1057,15 @@ def community_row(e, note_html): H.append(_row("Refresh the dev-box observation โ€” run in a " "terminal at the workspace root.", MORNING_CMD, term=True)) - for row in devbox.get("hygiene", {}).get("rows", []): - if row.get("status") in ("clean", "unscanned", "deferred", "advisory"): - continue - summary = esc(str(row.get("summary", ""))[:140]) - H.append(_row(f'{esc(str(row.get("mode")))}: {summary}', - str(row.get("delegate") or "/hygiene"))) + # Cloud hygiene supersedes the dev box's hygiene rows; the worktree + # state below is the one thing only this vantage can see. + if not hygiene: + for row in devbox.get("hygiene", {}).get("rows", []): + if row.get("status") in HYGIENE_QUIET_STATUSES: + continue + summary = esc(str(row.get("summary", ""))[:140]) + H.append(_row(f'{esc(str(row.get("mode")))}: {summary}', + str(row.get("delegate") or "/hygiene"))) for wt in devbox.get("worktrees", []): bits = [esc(str(wt.get("branch") or "?"))] if wt.get("ahead"): diff --git a/tests/test_board.py b/tests/test_board.py index 906ce46..7e277c7 100644 --- a/tests/test_board.py +++ b/tests/test_board.py @@ -27,8 +27,8 @@ SURFACE_KEYS = { "generated", "org", "overnight", "heart", "heart_blockers", "hands", - "versions", "community", "resume", "open_issues", "devbox", "autonomy", - "doors", "boards", "degraded", "history", + "versions", "community", "resume", "open_issues", "hygiene", "devbox", + "autonomy", "doors", "boards", "degraded", "history", } AUTONOMY_LOG = """\ @@ -181,7 +181,7 @@ def _fabricate(tmp_path, fixtures): return stub -def _run(args, tmp_path, stub): +def _run(args, tmp_path, stub, env_extra=None): env = { **os.environ, "PYAUTO_ROOT": str(tmp_path), @@ -192,6 +192,8 @@ def _run(args, tmp_path, stub): "COMMUNITY_GH": str(stub), "COMMUNITY_SEARCH_PAUSE": "0", } + env.pop("BOARD_HYGIENE_SCAN", None) # scans are opt-in per test + env.update(env_extra or {}) return subprocess.run( [str(BRAIN), "board", *args], capture_output=True, text=True, env=env, cwd=tmp_path, @@ -428,6 +430,43 @@ def test_devbox_observation_renders_age_stamped(tmp_path): assert "feature/x" in page and "2 unpushed" in page +def _hygiene_stub(tmp_path): + hyg = tmp_path / "hygiene_stub.sh" + decision = {"decision": "HygieneDecision", "repos_declared": 14, + "repos_present": 12, "rows": [ + {"mode": "deps", "kind": "surface", "status": "debris", + "count": 3, "summary": "3 dependency caps trail the floor", + "delegate": "/bug"}, + {"mode": "crlf", "kind": "debris", "status": "clean", + "count": 0, "summary": "clean", "delegate": "/refactor"}]} + hyg.write_text("#!/usr/bin/env bash\ncat <<'EOF'\n" + + json.dumps(decision) + "\nEOF\n") + hyg.chmod(hyg.stat().st_mode | stat.S_IEXEC) + return hyg + + +def test_cloud_hygiene_scan_renders_and_supersedes_devbox_rows(tmp_path): + stub = _fabricate(tmp_path, _default_fixtures()) + fresh = (datetime.now(timezone.utc) - timedelta(hours=2)).strftime( + "%Y-%m-%dT%H:%M:%SZ") + (tmp_path / "devbox_board.json").write_text( + json.dumps(_devbox_payload(fresh))) + env = {"BOARD_HYGIENE_SCAN": "1", + "BOARD_HYGIENE_CMD": str(_hygiene_stub(tmp_path))} + page = _run(["--html"], tmp_path, stub, env_extra=env).stdout + # The cloud scan is its own section, honest about coverage, chips = the + # rows' own delegate doors; clean rows stay quiet. + assert "Hygiene" in page and "12/14" in page + assert "dependency caps trail the floor" in page + assert 'data-cmd="/bug"' in page + # The dev-box section keeps only what the cloud cannot see: worktrees. + assert "packaging leftovers" not in page + assert "feature/x" in page + # Without the env, the scan never runs (terminal digests stay instant). + s = json.loads(_run(["--json"], tmp_path, stub).stdout) + assert s["hygiene"] is None + + def test_expired_devbox_observation_is_dropped(tmp_path): stub = _fabricate(tmp_path, _default_fixtures()) old = (datetime.now(timezone.utc) - timedelta(days=9)).strftime( diff --git a/tests/test_hygiene_conductor.py b/tests/test_hygiene_conductor.py index 909a527..9a08878 100644 --- a/tests/test_hygiene_conductor.py +++ b/tests/test_hygiene_conductor.py @@ -49,6 +49,19 @@ def _run(args, root, extra=None): ) +def test_json_stays_parseable_when_a_helper_crashes(tmp_path): + """A NONEXISTENT scan root crashes the helper-backed pre-scans (listdir). + That must degrade to per-mode `error` rows โ€” the Brain board's cloud + render parses this document, so one crashed helper may never corrupt it + (it did: bare helper failures emitted empty rows, leaving `,,`).""" + r = _run(["--json"], tmp_path / "does_not_exist") + assert r.returncode == 0, r.stderr + doc = json.loads(r.stdout) # the contract under test + by_mode = {row["mode"]: row for row in doc["rows"]} + assert by_mode["refs"]["status"] == "error" + assert "pyauto-brain hygiene refs" in by_mode["refs"]["summary"] + + def test_default_json_is_a_hygiene_decision_with_all_modes(tmp_path): r = _run(["--json"], tmp_path) assert r.returncode == 0, r.stderr