From 21cc82731b0b31abfa5f500c0a49bdb34ac37e45 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:32:19 -0400 Subject: [PATCH 01/29] face-lift: README restructure, conventions, ledger, check-knowledge harness, lint CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit knowledge.md and lessons/ untouched (connector-managed). verified/ sheet got whitespace-only lint fixes and one bare URL wrapped. harness/check-knowledge.py proves the files keep the shape readers and the connector rely on (RESULT: PASS — 9 pass, 0 warn, 0 fail; negative test FAILs as intended). CI: markdownlint + check-knowledge on push and PR. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- .github/PULL_REQUEST_TEMPLATE.md | 16 +++ .github/workflows/lint.yml | 31 ++++ .gitignore | 24 ++++ .markdownlint.yml | 8 ++ CONTRIBUTING.md | 43 ++++++ LEDGER.md | 14 ++ README.md | 70 ++++++++- harness/check-knowledge.py | 168 ++++++++++++++++++++++ verified/2026-09-26_claude-surfaces_v1.md | 8 +- 9 files changed, 376 insertions(+), 6 deletions(-) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/lint.yml create mode 100644 .gitignore create mode 100644 .markdownlint.yml create mode 100644 CONTRIBUTING.md create mode 100644 LEDGER.md create mode 100644 harness/check-knowledge.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..774fae8 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,16 @@ +## What changed + + + +## How it was verified + + + +```text +``` + +## Where the risk lives + +## Ledger + +- [ ] Row added to `LEDGER.md` diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..2c3596f --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,31 @@ +name: Lint + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + markdown: + name: markdownlint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npx --yes markdownlint-cli2 "**/*.md" "#node_modules" + + knowledge: + name: check-knowledge.py + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python3 harness/check-knowledge.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7111378 --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# Secrets and local config — never commit +.env +.env.* +*.pem +*.key +secrets/ + +# OS and editor noise +.DS_Store +Thumbs.db +*.swp +*~ +.idea/ +.vscode/ + +# Toolchains +node_modules/ +__pycache__/ +*.pyc +.venv/ + +# Harness output (paste into the PR, do not commit) +harness/out/ +*.log diff --git a/.markdownlint.yml b/.markdownlint.yml new file mode 100644 index 0000000..5153363 --- /dev/null +++ b/.markdownlint.yml @@ -0,0 +1,8 @@ +# markdownlint-cli2 config. Kept lenient where GitHub renders fine. +default: true +MD013: false # line length — tables and URLs make this noise +MD033: # inline HTML: allow the few tags GitHub renders + allowed_elements: [br, details, summary, img, sub, sup] +MD041: false # first line need not be an H1 (badges, templates) +MD024: + siblings_only: true # duplicate headings allowed in different sections diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..44c270c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,43 @@ +# Conventions + +Two kinds of writer share this repository: the daily-dev-agentic connector, +which appends lessons on a schedule, and Claude sessions or humans, who add +verified fact sheets and fix things. These rules keep them from colliding. + +## What the connector owns + +- `knowledge.md` and `lessons/*.md` are connector-managed. Edit them only to + fix a factual or formatting error in an existing lesson; do not add lessons + by hand (they would not carry a seen-post watermark and could be duplicated + on the next run). +- The connector commits straight to `main`. Do not move these files, rename + the H1, or change the `## YYYY-MM-DD` section format; the connector and + `harness/check-knowledge.py` both depend on it. + +## What sessions and humans own + +- `verified/YYYY-MM-DD_topic_vN.md`: a fact sheet proven in a session with + tool output, not read in an article. Start with an H1, say where the + evidence came from, mark anything unverified as such. A revision is a new + `vN`, never an overwrite. +- `README.md`, `CONTRIBUTING.md`, `LEDGER.md`, `harness/`, `.github/`. + +## Branches and pull requests + +- Hand or session changes go on `claude/` and land through a pull + request. Connector commits are the one exception. +- The PR body says what changed, pastes the `check-knowledge.py` result, and + names the risk. +- Merge only on green CI. + +## Ledger + +Every hand or session change adds one row to `LEDGER.md` (newest first). +Connector runs are not logged there; their trail is the git history of +`knowledge.md`. + +## Lint + +`.markdownlint.yml` is lenient on purpose (no line-length rule, inline HTML +allowed) so connector output passes unchanged. Run +`npx markdownlint-cli2 "**/*.md"` before opening a PR. diff --git a/LEDGER.md b/LEDGER.md new file mode 100644 index 0000000..c7a7f83 --- /dev/null +++ b/LEDGER.md @@ -0,0 +1,14 @@ +# Ledger + +One row per hand-made or session-made change. Newest first. Connector runs are +not logged here; `git log -- knowledge.md` is their trail. + +Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. +**Outcome** is one of `merged`, `closed`, `open`, `direct` (committed to main). +**Evidence** is the commit, PR, or harness result that proves it. + +| Date | Session | Branch | PR | Outcome | Evidence | +| --- | --- | --- | --- | --- | --- | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 9 pass, 0 warn, 0 fail), lint CI, description and topics set | +| 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | +| 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/README.md b/README.md index 25455a9..e82e90d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,69 @@ # agent-knowledge -The readable memory of T's daily-dev-agentic loop. Written by the daily.dev MCP connector (Cloudflare Worker `dailydevplugd`). +[![Lint](https://github.com/lookathim010-source/agent-knowledge/actions/workflows/lint.yml/badge.svg)](https://github.com/lookathim010-source/agent-knowledge/actions/workflows/lint.yml) +![License: MIT](https://img.shields.io/badge/license-MIT-green) +![Written by: daily-dev-agentic connector](https://img.shields.io/badge/written%20by-daily--dev--agentic%20connector-blue) +![Lessons carry a confidence score](https://img.shields.io/badge/lessons-confidence--scored-orange) -- `knowledge.md` — all lessons, newest day first; every lesson links its daily.dev source post. -- `lessons/YYYY-MM-DD.md` — the same lessons as one file per day. -- `verified/YYYY-MM-DD_topic_vN.md` — session-verified fact sheets written by Claude sessions (not daily.dev lessons); hand-maintained. +The readable memory of T's **daily-dev-agentic** loop: what the agent learned +from daily.dev, distilled into lessons that name why each one matters for the +stack (Claude Code, MCP, multi-agent systems, LangGraph, Firebase, Python, +Rust, local LLMs, fintech infrastructure) and what to do about it. -Each lesson is distilled by the agent from community-vetted daily.dev posts on T's stack (Claude Code, MCP, multi-agent systems, LangGraph, Firebase, Python, Rust, local LLMs, fintech infra) and carries a confidence score. Run state (seen-post watermark, counters) lives in Cloudflare KV, not here. Hand edits are safe — the connector only ever inserts and de-duplicates by source link. +Written mostly by a machine. The daily.dev MCP connector (Cloudflare Worker +`dailydevplugd`) appends lessons after each scheduled run; Claude sessions +add verified fact sheets; humans fix typos. Run state (seen-post watermark, +counters) lives in Cloudflare KV, not here. + +## Layout + +| Path | What it holds | Written by | +| --- | --- | --- | +| `knowledge.md` | Every lesson, newest day first. Each links its daily.dev source post and carries a confidence score | Connector (`knowledge_append`) | +| `lessons/YYYY-MM-DD.md` | The same lessons, one file per day | Connector | +| `verified/YYYY-MM-DD_topic_vN.md` | Fact sheets verified live in a Claude session, not daily.dev lessons. Revisions bump `vN` | Claude sessions, hand-maintained | +| `harness/check-knowledge.py` | PASS/FAIL check that the files above still have the shape readers rely on | Run by CI and by sessions | +| `LEDGER.md` | One row per hand-made or session-made change (connector runs are not logged here) | Sessions and hand edits | + +## How a lesson is shaped + +```markdown +## 2026-08-24 + +- **One actionable sentence, supported by the article.** + - Why it matters here: the link to T's stack or a live project. + - Do: the concrete next action, or "None now". + - Source: [Title](https://...) — Author (⬆️ upvotes · 💬 comments) · tags: a, b · confidence 80% +``` + +Confidence follows the connector's scale: official docs or maintainer posts +around 80–95, well-upvoted practitioner posts 60–80, single opinions 30–55. +Zero lessons on a quiet day is a valid result; the loop is told never to pad. + +## Reading it + +- Newest first: open `knowledge.md` and read from the top. +- One day: open `lessons/.md`. +- Something proven rather than read: `verified/`. +- Programmatically: the connector's `knowledge_read` tool (`days: N`) returns + the same content without cloning. + +## Checking it + +```bash +python3 harness/check-knowledge.py # PASS/FAIL lines +python3 harness/check-knowledge.py --json # same, as one JSON object +``` + +CI runs this plus markdownlint on every push and pull request +(`.github/workflows/lint.yml`). + +## Hand edits + +Safe. The connector only inserts and de-duplicates by source link; it never +rewrites what is already there. Conventions for edits, fact sheets and the +ledger are in [CONTRIBUTING.md](CONTRIBUTING.md). + +## License + +MIT. See [LICENSE](LICENSE). diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py new file mode 100644 index 0000000..944c719 --- /dev/null +++ b/harness/check-knowledge.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""check-knowledge.py — PASS/FAIL structural check of this repo's knowledge files. + +Proves that what the daily-dev-agentic connector (and hand edits) wrote still +has the shape every reader relies on: + + knowledge.md H1, then `## YYYY-MM-DD` sections newest first; every lesson + is a bold bullet with "Why it matters here", "Do:", and a + "Source:" line carrying a link and a confidence percentage + lessons/ one YYYY-MM-DD.md per day section in knowledge.md, and the + same lesson count per day + verified/ files named YYYY-MM-DD_topic_vN.md that start with an H1 + +Contract: one line per check (PASS|WARN|FAIL name evidence), final RESULT +line, exit 0 only when nothing FAILed. `--json` prints one JSON object. + +Run: python3 harness/check-knowledge.py [--json] +""" +from __future__ import annotations + +import json +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[1] +RESULTS: list[dict] = [] + +DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +SECTION_RE = re.compile(r"^## (\d{4}-\d{2}-\d{2})\s*$") +LESSON_RE = re.compile(r"^- \*\*.+\*\*\s*$") +SOURCE_RE = re.compile(r"^\s+- Source: \[[^\]]+\]\(https?://[^)]+\).*confidence \d{1,3}%") +VERIFIED_NAME_RE = re.compile(r"^\d{4}-\d{2}-\d{2}_[a-z0-9-]+_v\d+\.md$") + + +def record(status: str, name: str, detail: str) -> None: + RESULTS.append({"status": status, "name": name, "detail": detail}) + + +def ok(name: str, detail: str) -> None: + record("PASS", name, detail) + + +def warn(name: str, detail: str) -> None: + record("WARN", name, detail) + + +def fail(name: str, detail: str) -> None: + record("FAIL", name, detail) + + +def parse_lessons(lines: list[str]) -> dict[str, list[dict]]: + """Return {date: [lesson dicts]} for a knowledge-style file.""" + sections: dict[str, list[dict]] = {} + current: str | None = None + lesson: dict | None = None + for ln in lines: + m = SECTION_RE.match(ln) + if m: + current = m.group(1) + sections[current] = [] + lesson = None + continue + if current is None: + continue + if LESSON_RE.match(ln): + lesson = {"why": False, "do": False, "source": False, "title": ln.strip("- *\n")[:60]} + sections[current].append(lesson) + continue + if lesson is not None and ln.startswith(" - "): + if "Why it matters here:" in ln: + lesson["why"] = True + elif ln.strip().startswith("- Do:"): + lesson["do"] = True + elif SOURCE_RE.match(ln): + lesson["source"] = True + return sections + + +# --- knowledge.md ---------------------------------------------------------- +kpath = ROOT / "knowledge.md" +if not kpath.is_file(): + fail("knowledge.md", "missing") + sections: dict[str, list[dict]] = {} +else: + lines = kpath.read_text(encoding="utf-8").splitlines() + if lines and lines[0].startswith("# "): + ok("knowledge:h1", lines[0][:70]) + else: + fail("knowledge:h1", "first line is not an H1") + sections = parse_lessons(lines) + dates = list(sections) + if not dates: + fail("knowledge:sections", "no `## YYYY-MM-DD` sections") + else: + ok("knowledge:sections", f"{len(dates)} day section(s), {sum(len(v) for v in sections.values())} lesson(s)") + if dates == sorted(dates, reverse=True): + ok("knowledge:order", "newest first") + else: + fail("knowledge:order", f"sections not newest-first: {dates}") + if len(set(dates)) != len(dates): + fail("knowledge:unique-dates", "duplicate day sections") + bad = [(d, i, k) for d, ls in sections.items() for i, l in enumerate(ls, 1) for k in ("why", "do", "source") if not l[k]] + if bad: + fail("knowledge:lesson-shape", f"{len(bad)} missing field(s): " + ", ".join(f"{d}#{i}:{k}" for d, i, k in bad[:6])) + elif sections: + ok("knowledge:lesson-shape", "every lesson has why / do / source+confidence") + empty = [d for d, ls in sections.items() if not ls] + if empty: + warn("knowledge:empty-days", f"day sections with no lessons: {empty}") + +# --- lessons/ -------------------------------------------------------------- +ldir = ROOT / "lessons" +if not ldir.is_dir(): + warn("lessons/", "directory missing") +else: + files = sorted(p.name for p in ldir.glob("*.md")) + stray = [f for f in files if not DATE_RE.match(f[:-3])] + if stray: + fail("lessons:names", f"not YYYY-MM-DD.md: {stray}") + else: + ok("lessons:names", f"{len(files)} file(s), all dated") + missing = [d for d in sections if f"{d}.md" not in files] + extra = [f[:-3] for f in files if f[:-3] not in sections] + if missing: + fail("lessons:coverage", f"days in knowledge.md without a lessons file: {missing}") + elif sections: + ok("lessons:coverage", "every day section has a lessons file") + if extra: + warn("lessons:orphans", f"lessons files with no knowledge.md section: {extra}") + for d in sections: + p = ldir / f"{d}.md" + if p.is_file(): + n_day = sum(1 for ln in p.read_text(encoding="utf-8").splitlines() if LESSON_RE.match(ln)) + n_k = len(sections[d]) + if n_day == n_k: + ok(f"lessons:{d}", f"{n_day} lesson(s) match knowledge.md") + else: + fail(f"lessons:{d}", f"{n_day} lesson(s) here vs {n_k} in knowledge.md") + +# --- verified/ ------------------------------------------------------------- +vdir = ROOT / "verified" +if not vdir.is_dir(): + warn("verified/", "directory missing") +else: + vfiles = sorted(p for p in vdir.glob("*.md")) + badnames = [p.name for p in vfiles if not VERIFIED_NAME_RE.match(p.name)] + if badnames: + fail("verified:names", f"not YYYY-MM-DD_topic_vN.md: {badnames}") + else: + ok("verified:names", f"{len(vfiles)} sheet(s), all dated and versioned") + noh1 = [p.name for p in vfiles if not p.read_text(encoding="utf-8").lstrip().startswith("# ")] + if noh1: + fail("verified:h1", f"no H1: {noh1}") + elif vfiles: + ok("verified:h1", "every sheet starts with an H1") + +# --- summary --------------------------------------------------------------- +counts = {s: sum(r["status"] == s for r in RESULTS) for s in ("PASS", "WARN", "FAIL")} +passed = counts["FAIL"] == 0 +if "--json" in sys.argv: + print(json.dumps({"ok": passed, **{k.lower(): v for k, v in counts.items()}, "checks": RESULTS})) +else: + for r in RESULTS: + print(f"{r['status']:<4} {r['name']:<24} {r['detail']}") + print("----") + print(f"RESULT: {'PASS' if passed else 'FAIL'} — {counts['PASS']} pass, {counts['WARN']} warn, {counts['FAIL']} fail") +sys.exit(0 if passed else 1) diff --git a/verified/2026-09-26_claude-surfaces_v1.md b/verified/2026-09-26_claude-surfaces_v1.md index d912922..2058943 100644 --- a/verified/2026-09-26_claude-surfaces_v1.md +++ b/verified/2026-09-26_claude-surfaces_v1.md @@ -1,11 +1,13 @@ # Verified: Claude surfaces — Code tab cloud sessions vs chat/Cowork (2026-09-26) -Source: live tool runs in cloud session https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK plus the current Claude Code docs (claude-code-on-the-web, routines, claude-projects, remote-control) and the Cowork scheduled-tasks help article. Not a daily.dev lesson — a session-verified fact sheet. +Source: live tool runs in cloud session plus the current Claude Code docs (claude-code-on-the-web, routines, claude-projects, remote-control) and the Cowork scheduled-tasks help article. Not a daily.dev lesson — a session-verified fact sheet. ## Core finding + "Chat/Cowork is file-out, Code tab is repo-in/repo-out" is false. It is one cloud harness (Claude Code Remote) with many front doors: claude.ai/code, mobile Code tab, desktop app "Cloud", `claude --cloud`, routines, Claude Tag. Pick the door by what the session must be handed at birth: GitHub push credential + PR tooling → cloud session; a folder on the PC → Cowork/desktop. Routines, connectors, sub-agents and artifacts travel with all of them. ## Verified live (Code tab cloud session) + - `add_repo(push)` → shallow clone → signed commit → `git push`: native. Credentials never enter the VM; Anthropic's GitHub proxy injects them. - PR creation via `POST api.github.com/repos///pulls` works with the injected credential but REQUIRES `Content-Type: application/json` (proxy returns HTTP 415 without it). `gh` CLI is not installed in the cloud VM. - Opening a PR auto-subscribes the session (`subscription.created` ~1 s later) under a drive-to-green contract: CI red, merge conflicts and non-optional review findings are work now; never skip/disable a test to get green; re-request reviewers after pushing. @@ -16,18 +18,22 @@ Source: live tool runs in cloud session https://claude.ai/code/session_01T7N2Vsx - The Cowork session of 2026-08-24 committed to GitHub (this repo, cfd032a; daily-dev-connector with CI) via a pushbot workaround. Native push from the chat side: UNVERIFIED (~35%). ## From the docs (fetched 2026-09-26) + - Remote Control connects claude.ai to a session running on your machine — a local feature viewed remotely. Teleport pulls a cloud session into a local terminal (clean git state, same repo checkout). - Routines: created from web, desktop app or CLI, all into one cloud account. Desktop "Local" routine = Desktop scheduled task on the machine. Cowork scheduled tasks "can't be tied to a folder on your computer". - Code-tab Projects (parallel-thread coordinator): public beta on Pro/Max, rolling out first to accounts with cloud-session history and no existing chat/Cowork projects → eligibility for this account UNVERIFIED. - Auto-fix PRs requires the Claude GitHub App installed on the repo. ## Gotchas + 1. After a shallow `add_repo` clone, a pushed branch has no `origin/` ref (single-branch fetch refspec); the stop-hook falsely reports "unpushed commits / no remote branch". Fix: `git config --add remote.origin.fetch '+refs/heads/X:refs/remotes/origin/X' && git fetch origin X && git branch -u origin/X`. 2. The PC's `memory` knowledge-graph MCP fails from the cloud side (outputSchema declares JSON-Schema draft-07; the cloud validator supports 2020-12 only) — read and write both blocked as of 2026-09-26. 3. The daily-dev-agentic routine reports SUCCEEDED every day, yet this repo has no lesson commits after 2026-08-24: the run exits cleanly on the "connector not attached" branch of its prompt. Green run status ≠ task success. ## Bots seen on the repos + GitGuardian Security Checks · Kilo Code Review (glm-5.3) · Sourcery (no access on private repos; upsell comment only). ## Proof artifact + lookathim010-source/Claude-Code-Android-App-FULL-ACCESS-ENV-Anthropic-cloud — branch `claude-chat-capability-proof`, commit 9ee5237, PR #1 (merged by T). From 0faaee8e66cac4404b1365b4c79084fea01fb78e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:36:17 -0400 Subject: [PATCH 02/29] harness: detect duplicate day sections, shape-check lessons/ files, fail on orphans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the three Sourcery findings on PR #1: - parse_lessons returns duplicates instead of overwriting; knowledge:unique-dates FAILs on a repeated date - lessons/.md gets the same shape check as knowledge.md plus an ordered title comparison - orphan lessons files are a FAIL, not a WARN Positive run: RESULT: PASS — 10 pass, 0 warn, 0 fail. Each finding reproduced as a FAIL in a negative test before this commit was cut. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- harness/check-knowledge.py | 71 +++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 944c719..aab2f84 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -7,8 +7,8 @@ knowledge.md H1, then `## YYYY-MM-DD` sections newest first; every lesson is a bold bullet with "Why it matters here", "Do:", and a "Source:" line carrying a link and a confidence percentage - lessons/ one YYYY-MM-DD.md per day section in knowledge.md, and the - same lesson count per day + lessons/ one YYYY-MM-DD.md per day section in knowledge.md, no + orphans, and the same lessons (titles, order, shape) per day verified/ files named YYYY-MM-DD_topic_vN.md that start with an H1 Contract: one line per check (PASS|WARN|FAIL name evidence), final RESULT @@ -49,22 +49,30 @@ def fail(name: str, detail: str) -> None: record("FAIL", name, detail) -def parse_lessons(lines: list[str]) -> dict[str, list[dict]]: - """Return {date: [lesson dicts]} for a knowledge-style file.""" +def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: + """Return ({date: [lesson dicts]}, [duplicate dates]) for a knowledge-style file. + + A repeated `## YYYY-MM-DD` heading is recorded as a duplicate and its + lessons are appended to the first occurrence, so nothing is silently lost. + """ sections: dict[str, list[dict]] = {} + duplicates: list[str] = [] current: str | None = None lesson: dict | None = None for ln in lines: m = SECTION_RE.match(ln) if m: current = m.group(1) - sections[current] = [] + if current in sections: + duplicates.append(current) + else: + sections[current] = [] lesson = None continue if current is None: continue if LESSON_RE.match(ln): - lesson = {"why": False, "do": False, "source": False, "title": ln.strip("- *\n")[:60]} + lesson = {"why": False, "do": False, "source": False, "title": ln.strip().strip("-").strip().strip("*").strip()} sections[current].append(lesson) continue if lesson is not None and ln.startswith(" - "): @@ -74,7 +82,12 @@ def parse_lessons(lines: list[str]) -> dict[str, list[dict]]: lesson["do"] = True elif SOURCE_RE.match(ln): lesson["source"] = True - return sections + return sections, duplicates + + +def shape_errors(sections: dict[str, list[dict]]) -> list[str]: + """Names of missing lesson fields as 'date#n:field'.""" + return [f"{d}#{i}:{k}" for d, ls in sections.items() for i, l in enumerate(ls, 1) for k in ("why", "do", "source") if not l[k]] # --- knowledge.md ---------------------------------------------------------- @@ -88,7 +101,7 @@ def parse_lessons(lines: list[str]) -> dict[str, list[dict]]: ok("knowledge:h1", lines[0][:70]) else: fail("knowledge:h1", "first line is not an H1") - sections = parse_lessons(lines) + sections, dup_dates = parse_lessons(lines) dates = list(sections) if not dates: fail("knowledge:sections", "no `## YYYY-MM-DD` sections") @@ -98,11 +111,13 @@ def parse_lessons(lines: list[str]) -> dict[str, list[dict]]: ok("knowledge:order", "newest first") else: fail("knowledge:order", f"sections not newest-first: {dates}") - if len(set(dates)) != len(dates): - fail("knowledge:unique-dates", "duplicate day sections") - bad = [(d, i, k) for d, ls in sections.items() for i, l in enumerate(ls, 1) for k in ("why", "do", "source") if not l[k]] + if dup_dates: + fail("knowledge:unique-dates", f"day section repeated: {sorted(set(dup_dates))}") + else: + ok("knowledge:unique-dates", "no repeated day sections") + bad = shape_errors(sections) if bad: - fail("knowledge:lesson-shape", f"{len(bad)} missing field(s): " + ", ".join(f"{d}#{i}:{k}" for d, i, k in bad[:6])) + fail("knowledge:lesson-shape", f"{len(bad)} missing field(s): " + ", ".join(bad[:6])) elif sections: ok("knowledge:lesson-shape", "every lesson has why / do / source+confidence") empty = [d for d, ls in sections.items() if not ls] @@ -127,16 +142,32 @@ def parse_lessons(lines: list[str]) -> dict[str, list[dict]]: elif sections: ok("lessons:coverage", "every day section has a lessons file") if extra: - warn("lessons:orphans", f"lessons files with no knowledge.md section: {extra}") + fail("lessons:orphans", f"lessons files with no knowledge.md section: {extra}") for d in sections: p = ldir / f"{d}.md" - if p.is_file(): - n_day = sum(1 for ln in p.read_text(encoding="utf-8").splitlines() if LESSON_RE.match(ln)) - n_k = len(sections[d]) - if n_day == n_k: - ok(f"lessons:{d}", f"{n_day} lesson(s) match knowledge.md") - else: - fail(f"lessons:{d}", f"{n_day} lesson(s) here vs {n_k} in knowledge.md") + if not p.is_file(): + continue + day_lines = p.read_text(encoding="utf-8").splitlines() + # A per-day file has no `## date` heading of its own: parse it as that day's section. + day_sections, day_dups = parse_lessons([f"## {d}"] + [ln for ln in day_lines if not SECTION_RE.match(ln)]) + day_lessons = day_sections.get(d, []) + problems: list[str] = [] + bad_day = shape_errors({d: day_lessons}) + if bad_day: + problems.append(f"missing field(s) {bad_day[:4]}") + titles_k = [l["title"] for l in sections[d]] + titles_d = [l["title"] for l in day_lessons] + if titles_d != titles_k: + only_k = [t[:40] for t in titles_k if t not in titles_d] + only_d = [t[:40] for t in titles_d if t not in titles_k] + problems.append(f"{len(titles_d)} lesson(s) here vs {len(titles_k)} in knowledge.md" + + (f"; only in knowledge.md: {only_k}" if only_k else "") + + (f"; only here: {only_d}" if only_d else "") + + (("; same set, different order" if len(titles_d) == len(titles_k) else "; a title repeats") if not only_k and not only_d else "")) + if problems: + fail(f"lessons:{d}", "; ".join(problems)) + else: + ok(f"lessons:{d}", f"{len(day_lessons)} lesson(s), same titles and shape as knowledge.md") # --- verified/ ------------------------------------------------------------- vdir = ROOT / "verified" From f25beab09dcd2879da53ba3ef43ec3b3e5b7c278 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:41:35 -0400 Subject: [PATCH 03/29] harness: fail on missing lessons/, malformed bullets, bad confidence, impossible dates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the remaining Codex findings on PR #1 (duplicate-date finding was fixed in 0faaee8). Positive run: RESULT: PASS — 11 pass, 0 warn, 0 fail; each finding reproduced as a FAIL in a negative test before this commit was cut. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 66 ++++++++++++++++++++++++++++++-------- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index c7a7f83..7a58ce1 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -9,6 +9,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 9 pass, 0 warn, 0 fail), lint CI, description and topics set | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 11 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 5 Codex findings | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index aab2f84..14b39dc 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -4,9 +4,10 @@ Proves that what the daily-dev-agentic connector (and hand edits) wrote still has the shape every reader relies on: - knowledge.md H1, then `## YYYY-MM-DD` sections newest first; every lesson - is a bold bullet with "Why it matters here", "Do:", and a - "Source:" line carrying a link and a confidence percentage + knowledge.md H1, then `## YYYY-MM-DD` sections (real calendar dates, no + repeats) newest first; every top-level bullet in a section is + a bold lesson with "Why it matters here", "Do:", and a + "Source:" line carrying a link and a 0-100 confidence lessons/ one YYYY-MM-DD.md per day section in knowledge.md, no orphans, and the same lessons (titles, order, shape) per day verified/ files named YYYY-MM-DD_topic_vN.md that start with an H1 @@ -18,6 +19,7 @@ """ from __future__ import annotations +import datetime import json import pathlib import re @@ -29,7 +31,17 @@ DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") SECTION_RE = re.compile(r"^## (\d{4}-\d{2}-\d{2})\s*$") LESSON_RE = re.compile(r"^- \*\*.+\*\*\s*$") -SOURCE_RE = re.compile(r"^\s+- Source: \[[^\]]+\]\(https?://[^)]+\).*confidence \d{1,3}%") +SOURCE_RE = re.compile(r"^\s+- Source: \[[^\]]+\]\(https?://[^)]+\).*confidence (\d{1,3})%") +PLAIN_BULLET_RE = re.compile(r"^- (?!\*\*)\S") # a top-level bullet that is not a bold lesson + + +def valid_date(s: str) -> bool: + """True only for a real ISO calendar date (rejects 2026-99-99).""" + try: + datetime.date.fromisoformat(s) + return True + except ValueError: + return False VERIFIED_NAME_RE = re.compile(r"^\d{4}-\d{2}-\d{2}_[a-z0-9-]+_v\d+\.md$") @@ -54,6 +66,9 @@ def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: A repeated `## YYYY-MM-DD` heading is recorded as a duplicate and its lessons are appended to the first occurrence, so nothing is silently lost. + A top-level bullet inside a day section that is not a bold lesson is + recorded as a malformed lesson (why/do/source all False) so it FAILs the + shape check instead of vanishing from the count. """ sections: dict[str, list[dict]] = {} duplicates: list[str] = [] @@ -75,19 +90,36 @@ def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: lesson = {"why": False, "do": False, "source": False, "title": ln.strip().strip("-").strip().strip("*").strip()} sections[current].append(lesson) continue + if PLAIN_BULLET_RE.match(ln): + lesson = {"why": False, "do": False, "source": False, "malformed": True, "title": "MALFORMED: " + ln[2:].strip()[:50]} + sections[current].append(lesson) + continue if lesson is not None and ln.startswith(" - "): if "Why it matters here:" in ln: lesson["why"] = True elif ln.strip().startswith("- Do:"): lesson["do"] = True - elif SOURCE_RE.match(ln): - lesson["source"] = True + else: + sm = SOURCE_RE.match(ln) + if sm: + lesson["source"] = 0 <= int(sm.group(1)) <= 100 + if not lesson["source"]: + lesson["confidence_out_of_range"] = int(sm.group(1)) return sections, duplicates def shape_errors(sections: dict[str, list[dict]]) -> list[str]: - """Names of missing lesson fields as 'date#n:field'.""" - return [f"{d}#{i}:{k}" for d, ls in sections.items() for i, l in enumerate(ls, 1) for k in ("why", "do", "source") if not l[k]] + """Names of lesson-shape problems as 'date#n:field'.""" + out: list[str] = [] + for d, ls in sections.items(): + for i, l in enumerate(ls, 1): + if l.get("malformed"): + out.append(f"{d}#{i}:not-a-bold-lesson-bullet") + continue + for k in ("why", "do", "source"): + if not l[k]: + out.append(f"{d}#{i}:{k}" + (f"(confidence {l['confidence_out_of_range']}% out of 0-100)" if k == "source" and "confidence_out_of_range" in l else "")) + return out # --- knowledge.md ---------------------------------------------------------- @@ -115,6 +147,11 @@ def shape_errors(sections: dict[str, list[dict]]) -> list[str]: fail("knowledge:unique-dates", f"day section repeated: {sorted(set(dup_dates))}") else: ok("knowledge:unique-dates", "no repeated day sections") + impossible = [d for d in dates if not valid_date(d)] + if impossible: + fail("knowledge:real-dates", f"not calendar dates: {impossible}") + else: + ok("knowledge:real-dates", "every day section is a real calendar date") bad = shape_errors(sections) if bad: fail("knowledge:lesson-shape", f"{len(bad)} missing field(s): " + ", ".join(bad[:6])) @@ -127,12 +164,15 @@ def shape_errors(sections: dict[str, list[dict]]) -> list[str]: # --- lessons/ -------------------------------------------------------------- ldir = ROOT / "lessons" if not ldir.is_dir(): - warn("lessons/", "directory missing") + if sections: + fail("lessons/", f"directory missing but knowledge.md has {len(sections)} day section(s)") + else: + warn("lessons/", "directory missing (no day sections to cover)") else: files = sorted(p.name for p in ldir.glob("*.md")) - stray = [f for f in files if not DATE_RE.match(f[:-3])] + stray = [f for f in files if not DATE_RE.match(f[:-3]) or not valid_date(f[:-3])] if stray: - fail("lessons:names", f"not YYYY-MM-DD.md: {stray}") + fail("lessons:names", f"not a real YYYY-MM-DD.md: {stray}") else: ok("lessons:names", f"{len(files)} file(s), all dated") missing = [d for d in sections if f"{d}.md" not in files] @@ -175,9 +215,9 @@ def shape_errors(sections: dict[str, list[dict]]) -> list[str]: warn("verified/", "directory missing") else: vfiles = sorted(p for p in vdir.glob("*.md")) - badnames = [p.name for p in vfiles if not VERIFIED_NAME_RE.match(p.name)] + badnames = [p.name for p in vfiles if not VERIFIED_NAME_RE.match(p.name) or not valid_date(p.name[:10])] if badnames: - fail("verified:names", f"not YYYY-MM-DD_topic_vN.md: {badnames}") + fail("verified:names", f"not a real YYYY-MM-DD_topic_vN.md: {badnames}") else: ok("verified:names", f"{len(vfiles)} sheet(s), all dated and versioned") noh1 = [p.name for p in vfiles if not p.read_text(encoding="utf-8").lstrip().startswith("# ")] From 868c3d1b69a55d519087b614b6f40ee2a7139a3c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:51:31 -0400 Subject: [PATCH 04/29] harness: content-level comparison, strict fields, exact H1, no stray day headings; pin markdownlint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-3 Codex findings. Parser now keeps field text so lessons/ must match knowledge.md on title, why, do and source in order; any non-bold top-level bullet is malformed; Why/Do need exact prefix + value; H1 must be the connector's exact heading; a day file may not contain a `## date` heading. Positive: RESULT: PASS — 11 pass, 0 warn, 0 fail. Six new negatives FAIL; all earlier negatives still FAIL. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- .github/workflows/lint.yml | 2 +- harness/check-knowledge.py | 177 ++++++++++++++++++++++--------------- 2 files changed, 109 insertions(+), 70 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2c3596f..3954627 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 - - run: npx --yes markdownlint-cli2 "**/*.md" "#node_modules" + - run: npx --yes markdownlint-cli2@0.23.3 "**/*.md" "#node_modules" knowledge: name: check-knowledge.py diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 14b39dc..20b03c0 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -4,13 +4,17 @@ Proves that what the daily-dev-agentic connector (and hand edits) wrote still has the shape every reader relies on: - knowledge.md H1, then `## YYYY-MM-DD` sections (real calendar dates, no - repeats) newest first; every top-level bullet in a section is - a bold lesson with "Why it matters here", "Do:", and a - "Source:" line carrying a link and a 0-100 confidence + knowledge.md the exact connector H1, then `## YYYY-MM-DD` sections (real + calendar dates, no repeats) newest first; every top-level + bullet in a section is a bold lesson with a non-empty + "Why it matters here:", a non-empty "Do:", and a "Source:" + line carrying a link and a 0-100 confidence lessons/ one YYYY-MM-DD.md per day section in knowledge.md, no - orphans, and the same lessons (titles, order, shape) per day - verified/ files named YYYY-MM-DD_topic_vN.md that start with an H1 + orphans, no `## date` headings of its own, and lesson + content identical (title, why, do, source) to that day's + section in knowledge.md, in the same order + verified/ files named YYYY-MM-DD_topic_vN.md (real dates) starting + with an H1 Contract: one line per check (PASS|WARN|FAIL name evidence), final RESULT line, exit 0 only when nothing FAILed. `--json` prints one JSON object. @@ -28,21 +32,17 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] RESULTS: list[dict] = [] +KNOWLEDGE_H1 = "# daily-dev-agentic knowledge — T agent" # the connector depends on this exact line + DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") SECTION_RE = re.compile(r"^## (\d{4}-\d{2}-\d{2})\s*$") -LESSON_RE = re.compile(r"^- \*\*.+\*\*\s*$") -SOURCE_RE = re.compile(r"^\s+- Source: \[[^\]]+\]\(https?://[^)]+\).*confidence (\d{1,3})%") -PLAIN_BULLET_RE = re.compile(r"^- (?!\*\*)\S") # a top-level bullet that is not a bold lesson - - -def valid_date(s: str) -> bool: - """True only for a real ISO calendar date (rejects 2026-99-99).""" - try: - datetime.date.fromisoformat(s) - return True - except ValueError: - return False +LESSON_RE = re.compile(r"^- \*\*(.+?)\*\*\s*$") # well-formed bold lesson bullet +TOP_BULLET_RE = re.compile(r"^- \S") # any top-level bullet +WHY_RE = re.compile(r"^ - Why it matters here: (\S.*)$") +DO_RE = re.compile(r"^ - Do: (\S.*)$") +SOURCE_RE = re.compile(r"^ - Source: \[[^\]]+\]\(https?://[^)\s]+\).*confidence (\d{1,3})%") VERIFIED_NAME_RE = re.compile(r"^\d{4}-\d{2}-\d{2}_[a-z0-9-]+_v\d+\.md$") +FIELDS = ("why", "do", "source") def record(status: str, name: str, detail: str) -> None: @@ -61,19 +61,38 @@ def fail(name: str, detail: str) -> None: record("FAIL", name, detail) +def valid_date(s: str) -> bool: + """True only for a real ISO calendar date (rejects 2026-99-99).""" + try: + datetime.date.fromisoformat(s) + return True + except ValueError: + return False + + +def norm(s: str) -> str: + return " ".join(s.split()) + + def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: - """Return ({date: [lesson dicts]}, [duplicate dates]) for a knowledge-style file. + """Return ({date: [lesson]}, [duplicate dates]) for a knowledge-style file. - A repeated `## YYYY-MM-DD` heading is recorded as a duplicate and its - lessons are appended to the first occurrence, so nothing is silently lost. - A top-level bullet inside a day section that is not a bold lesson is - recorded as a malformed lesson (why/do/source all False) so it FAILs the - shape check instead of vanishing from the count. + A lesson is {"title", "why", "do", "source", "problems"} with the field + TEXT (normalized) so two copies can be compared, and a list of problems: + missing or empty fields, a confidence outside 0-100, or a top-level bullet + that is not a well-formed bold lesson (recorded, never dropped, so a + malformed lesson can't vanish from the count). A repeated `## date` + heading is recorded as a duplicate; its lessons join the first occurrence. """ sections: dict[str, list[dict]] = {} duplicates: list[str] = [] current: str | None = None lesson: dict | None = None + + def new_lesson(title: str, malformed: bool) -> dict: + return {"title": norm(title), "why": "", "do": "", "source": "", + "problems": (["not-a-bold-lesson-bullet"] if malformed else [])} + for ln in lines: m = SECTION_RE.match(ln) if m: @@ -86,53 +105,58 @@ def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: continue if current is None: continue - if LESSON_RE.match(ln): - lesson = {"why": False, "do": False, "source": False, "title": ln.strip().strip("-").strip().strip("*").strip()} + m = LESSON_RE.match(ln) + if m: + lesson = new_lesson(m.group(1), malformed=False) sections[current].append(lesson) continue - if PLAIN_BULLET_RE.match(ln): - lesson = {"why": False, "do": False, "source": False, "malformed": True, "title": "MALFORMED: " + ln[2:].strip()[:50]} + if TOP_BULLET_RE.match(ln): + lesson = new_lesson("MALFORMED: " + ln[2:].strip(), malformed=True) sections[current].append(lesson) continue - if lesson is not None and ln.startswith(" - "): - if "Why it matters here:" in ln: - lesson["why"] = True - elif ln.strip().startswith("- Do:"): - lesson["do"] = True + if lesson is None: + continue + if (m := WHY_RE.match(ln)): + lesson["why"] = norm(m.group(1)) + elif (m := DO_RE.match(ln)): + lesson["do"] = norm(m.group(1)) + elif (m := SOURCE_RE.match(ln)): + conf = int(m.group(1)) + if 0 <= conf <= 100: + lesson["source"] = norm(ln.strip()[2:]) else: - sm = SOURCE_RE.match(ln) - if sm: - lesson["source"] = 0 <= int(sm.group(1)) <= 100 - if not lesson["source"]: - lesson["confidence_out_of_range"] = int(sm.group(1)) + lesson["problems"].append(f"confidence {conf}% out of 0-100") + + for ls in sections.values(): + for l in ls: + if "not-a-bold-lesson-bullet" in l["problems"]: + continue + for k in FIELDS: + if not l[k] and not (k == "source" and any(p.startswith("confidence") for p in l["problems"])): + l["problems"].append(f"missing or empty {k}") return sections, duplicates def shape_errors(sections: dict[str, list[dict]]) -> list[str]: - """Names of lesson-shape problems as 'date#n:field'.""" - out: list[str] = [] - for d, ls in sections.items(): - for i, l in enumerate(ls, 1): - if l.get("malformed"): - out.append(f"{d}#{i}:not-a-bold-lesson-bullet") - continue - for k in ("why", "do", "source"): - if not l[k]: - out.append(f"{d}#{i}:{k}" + (f"(confidence {l['confidence_out_of_range']}% out of 0-100)" if k == "source" and "confidence_out_of_range" in l else "")) - return out + """Problems as 'date#n: problem'.""" + return [f"{d}#{i}: {p}" for d, ls in sections.items() for i, l in enumerate(ls, 1) for p in l["problems"]] + + +def lesson_key(l: dict) -> tuple[str, str, str, str]: + return (l["title"], l["why"], l["do"], l["source"]) # --- knowledge.md ---------------------------------------------------------- kpath = ROOT / "knowledge.md" +sections: dict[str, list[dict]] = {} if not kpath.is_file(): fail("knowledge.md", "missing") - sections: dict[str, list[dict]] = {} else: lines = kpath.read_text(encoding="utf-8").splitlines() - if lines and lines[0].startswith("# "): - ok("knowledge:h1", lines[0][:70]) + if lines and lines[0].rstrip() == KNOWLEDGE_H1: + ok("knowledge:h1", "exact connector heading present") else: - fail("knowledge:h1", "first line is not an H1") + fail("knowledge:h1", f"first line must be {KNOWLEDGE_H1!r}, got {(lines[0] if lines else '')[:70]!r}") sections, dup_dates = parse_lessons(lines) dates = list(sections) if not dates: @@ -154,9 +178,9 @@ def shape_errors(sections: dict[str, list[dict]]) -> list[str]: ok("knowledge:real-dates", "every day section is a real calendar date") bad = shape_errors(sections) if bad: - fail("knowledge:lesson-shape", f"{len(bad)} missing field(s): " + ", ".join(bad[:6])) + fail("knowledge:lesson-shape", f"{len(bad)} problem(s): " + "; ".join(bad[:6])) elif sections: - ok("knowledge:lesson-shape", "every lesson has why / do / source+confidence") + ok("knowledge:lesson-shape", "every lesson is a bold bullet with non-empty why / do / source (confidence 0-100)") empty = [d for d, ls in sections.items() if not ls] if empty: warn("knowledge:empty-days", f"day sections with no lessons: {empty}") @@ -188,26 +212,41 @@ def shape_errors(sections: dict[str, list[dict]]) -> list[str]: if not p.is_file(): continue day_lines = p.read_text(encoding="utf-8").splitlines() - # A per-day file has no `## date` heading of its own: parse it as that day's section. - day_sections, day_dups = parse_lessons([f"## {d}"] + [ln for ln in day_lines if not SECTION_RE.match(ln)]) - day_lessons = day_sections.get(d, []) problems: list[str] = [] + stray_headings = [ln.strip() for ln in day_lines if SECTION_RE.match(ln)] + if stray_headings: + problems.append(f"contains day heading(s) {stray_headings} — a per-day file must not re-date its lessons") + # A per-day file carries no `## date` heading of its own: parse it as that day's section. + day_sections, _ = parse_lessons([f"## {d}"] + [ln for ln in day_lines if not SECTION_RE.match(ln)]) + day_lessons = day_sections.get(d, []) bad_day = shape_errors({d: day_lessons}) if bad_day: - problems.append(f"missing field(s) {bad_day[:4]}") - titles_k = [l["title"] for l in sections[d]] - titles_d = [l["title"] for l in day_lessons] - if titles_d != titles_k: - only_k = [t[:40] for t in titles_k if t not in titles_d] - only_d = [t[:40] for t in titles_d if t not in titles_k] - problems.append(f"{len(titles_d)} lesson(s) here vs {len(titles_k)} in knowledge.md" - + (f"; only in knowledge.md: {only_k}" if only_k else "") - + (f"; only here: {only_d}" if only_d else "") - + (("; same set, different order" if len(titles_d) == len(titles_k) else "; a title repeats") if not only_k and not only_d else "")) + problems.append(f"shape: {bad_day[:4]}") + keys_k = [lesson_key(l) for l in sections[d]] + keys_d = [lesson_key(l) for l in day_lessons] + if keys_d != keys_k: + titles_k = {k[0] for k in keys_k} + titles_d = {k[0] for k in keys_d} + only_k = [k[0][:40] for k in keys_k if k[0] not in titles_d] # title absent from the day file + only_d = [k[0][:40] for k in keys_d if k[0] not in titles_k] # title absent from knowledge.md + diff_fields: list[str] = [] + for a, b in zip(keys_k, keys_d): + if a != b and a[0] == b[0]: + diff_fields.append(a[0][:30] + ": " + ",".join(f for f, x, y in zip(("title",) + FIELDS, a, b) if x != y)) + detail = f"{len(keys_d)} lesson(s) here vs {len(keys_k)} in knowledge.md" + if diff_fields: + detail += f"; same title, different content: {diff_fields[:3]}" + if only_k: + detail += f"; only in knowledge.md: {only_k[:3]}" + if only_d: + detail += f"; only here: {only_d[:3]}" + if not diff_fields and not only_k and not only_d: + detail += "; same set, different order" if len(keys_d) == len(keys_k) else "; a lesson repeats" + problems.append(detail) if problems: fail(f"lessons:{d}", "; ".join(problems)) else: - ok(f"lessons:{d}", f"{len(day_lessons)} lesson(s), same titles and shape as knowledge.md") + ok(f"lessons:{d}", f"{len(day_lessons)} lesson(s), identical content and order to knowledge.md") # --- verified/ ------------------------------------------------------------- vdir = ROOT / "verified" From d178e69d440863d413526daea28375b58e7850cc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 22:57:03 -0400 Subject: [PATCH 05/29] harness: reject non-date H2 headings; coverage counts readable files only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-4 Codex findings. Positive: RESULT: PASS — 12 pass, 0 warn, 0 fail. Negatives: '## Notes' / '## 2026-8-25' → FAIL; dangling symlink day file → FAIL. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 35 ++++++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index 7a58ce1..cc7eeb3 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -9,6 +9,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 11 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 5 Codex findings | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 13 Codex findings | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 20b03c0..028d900 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -4,13 +4,13 @@ Proves that what the daily-dev-agentic connector (and hand edits) wrote still has the shape every reader relies on: - knowledge.md the exact connector H1, then `## YYYY-MM-DD` sections (real - calendar dates, no repeats) newest first; every top-level + knowledge.md the exact connector H1, then only `## YYYY-MM-DD` sections + (real calendar dates, no repeats, no other H2s) newest first; every top-level bullet in a section is a bold lesson with a non-empty "Why it matters here:", a non-empty "Do:", and a "Source:" line carrying a link and a 0-100 confidence - lessons/ one YYYY-MM-DD.md per day section in knowledge.md, no - orphans, no `## date` headings of its own, and lesson + lessons/ one readable YYYY-MM-DD.md per day section in knowledge.md, + no orphans, no `## ` headings of its own, and lesson content identical (title, why, do, source) to that day's section in knowledge.md, in the same order verified/ files named YYYY-MM-DD_topic_vN.md (real dates) starting @@ -43,6 +43,8 @@ SOURCE_RE = re.compile(r"^ - Source: \[[^\]]+\]\(https?://[^)\s]+\).*confidence (\d{1,3})%") VERIFIED_NAME_RE = re.compile(r"^\d{4}-\d{2}-\d{2}_[a-z0-9-]+_v\d+\.md$") FIELDS = ("why", "do", "source") +H2_RE = re.compile(r"^## ") +bad_headings: list[str] = [] # filled by parse_lessons: H2s that are not valid day headings def record(status: str, name: str, detail: str) -> None: @@ -88,6 +90,7 @@ def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: duplicates: list[str] = [] current: str | None = None lesson: dict | None = None + bad_headings.clear() def new_lesson(title: str, malformed: bool) -> dict: return {"title": norm(title), "why": "", "do": "", "source": "", @@ -103,6 +106,11 @@ def new_lesson(title: str, malformed: bool) -> dict: sections[current] = [] lesson = None continue + if H2_RE.match(ln): # an H2 that is not `## YYYY-MM-DD` + bad_headings.append(ln.strip()) + current = None # nothing after it belongs to a day + lesson = None + continue if current is None: continue m = LESSON_RE.match(ln) @@ -158,7 +166,12 @@ def lesson_key(l: dict) -> tuple[str, str, str, str]: else: fail("knowledge:h1", f"first line must be {KNOWLEDGE_H1!r}, got {(lines[0] if lines else '')[:70]!r}") sections, dup_dates = parse_lessons(lines) + k_bad_headings = list(bad_headings) dates = list(sections) + if k_bad_headings: + fail("knowledge:headings", f"H2 headings that are not `## YYYY-MM-DD`: {k_bad_headings[:4]}") + else: + ok("knowledge:headings", "every H2 is a day heading") if not dates: fail("knowledge:sections", "no `## YYYY-MM-DD` sections") else: @@ -193,7 +206,11 @@ def lesson_key(l: dict) -> tuple[str, str, str, str]: else: warn("lessons/", "directory missing (no day sections to cover)") else: - files = sorted(p.name for p in ldir.glob("*.md")) + entries = sorted(ldir.glob("*.md")) + unreadable = [p.name for p in entries if not p.is_file()] # dangling symlinks, directories + if unreadable: + fail("lessons:readable", f"not readable regular files: {unreadable}") + files = [p.name for p in entries if p.is_file()] stray = [f for f in files if not DATE_RE.match(f[:-3]) or not valid_date(f[:-3])] if stray: fail("lessons:names", f"not a real YYYY-MM-DD.md: {stray}") @@ -213,11 +230,11 @@ def lesson_key(l: dict) -> tuple[str, str, str, str]: continue day_lines = p.read_text(encoding="utf-8").splitlines() problems: list[str] = [] - stray_headings = [ln.strip() for ln in day_lines if SECTION_RE.match(ln)] + stray_headings = [ln.strip() for ln in day_lines if H2_RE.match(ln)] if stray_headings: - problems.append(f"contains day heading(s) {stray_headings} — a per-day file must not re-date its lessons") - # A per-day file carries no `## date` heading of its own: parse it as that day's section. - day_sections, _ = parse_lessons([f"## {d}"] + [ln for ln in day_lines if not SECTION_RE.match(ln)]) + problems.append(f"contains H2 heading(s) {stray_headings[:4]} — a per-day file has no `## ` headings") + # A per-day file carries no `## ` heading of its own: parse it as that day's section. + day_sections, _ = parse_lessons([f"## {d}"] + [ln for ln in day_lines if not H2_RE.match(ln)]) day_lessons = day_sections.get(d, []) bad_day = shape_errors({d: day_lessons}) if bad_day: From 67c184185b60c9df7d97164384ce7cd5a0e14151 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 23:05:41 -0400 Subject: [PATCH 06/29] harness: any list marker is a lesson candidate, bare ## rejected, verified/ non-files and indented H1 fail; empty day sections fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-5 Codex findings (5). Positive: RESULT: PASS — 12 pass, 0 warn, 0 fail. Negatives: ordered-list lessons, bare ##, verified/ symlink+dir, 4-space H1 → all FAIL. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- CONTRIBUTING.md | 2 +- LEDGER.md | 2 +- harness/check-knowledge.py | 28 +++++++++++++++++++--------- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44c270c..be1286c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,4 +40,4 @@ Connector runs are not logged there; their trail is the git history of `.markdownlint.yml` is lenient on purpose (no line-length rule, inline HTML allowed) so connector output passes unchanged. Run -`npx markdownlint-cli2 "**/*.md"` before opening a PR. +`npx markdownlint-cli2@0.23.3 "**/*.md"` (the version CI pins) before opening a PR. diff --git a/LEDGER.md b/LEDGER.md index cc7eeb3..21f6bbf 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -9,6 +9,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 13 Codex findings | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 18 Codex findings | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 028d900..4a450e5 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -13,8 +13,8 @@ no orphans, no `## ` headings of its own, and lesson content identical (title, why, do, source) to that day's section in knowledge.md, in the same order - verified/ files named YYYY-MM-DD_topic_vN.md (real dates) starting - with an H1 + verified/ readable files named YYYY-MM-DD_topic_vN.md (real dates) + whose first non-blank line is a real H1 (not indented 4+) Contract: one line per check (PASS|WARN|FAIL name evidence), final RESULT line, exit 0 only when nothing FAILed. `--json` prints one JSON object. @@ -37,13 +37,14 @@ DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") SECTION_RE = re.compile(r"^## (\d{4}-\d{2}-\d{2})\s*$") LESSON_RE = re.compile(r"^- \*\*(.+?)\*\*\s*$") # well-formed bold lesson bullet -TOP_BULLET_RE = re.compile(r"^- \S") # any top-level bullet +TOP_BULLET_RE = re.compile(r"^(?:[-*+]|\d+[.)]) +\S") # any top-level list item, any marker WHY_RE = re.compile(r"^ - Why it matters here: (\S.*)$") DO_RE = re.compile(r"^ - Do: (\S.*)$") SOURCE_RE = re.compile(r"^ - Source: \[[^\]]+\]\(https?://[^)\s]+\).*confidence (\d{1,3})%") VERIFIED_NAME_RE = re.compile(r"^\d{4}-\d{2}-\d{2}_[a-z0-9-]+_v\d+\.md$") +H1_LINE_RE = re.compile(r"^ {0,3}# \S") # a real ATX H1: at most 3 leading spaces (4 = code block) FIELDS = ("why", "do", "source") -H2_RE = re.compile(r"^## ") +H2_RE = re.compile(r"^##(?:\s|$)") # any ATX H2, including a bare `##` bad_headings: list[str] = [] # filled by parse_lessons: H2s that are not valid day headings @@ -119,7 +120,7 @@ def new_lesson(title: str, malformed: bool) -> dict: sections[current].append(lesson) continue if TOP_BULLET_RE.match(ln): - lesson = new_lesson("MALFORMED: " + ln[2:].strip(), malformed=True) + lesson = new_lesson("MALFORMED: " + ln.split(None, 1)[1].strip(), malformed=True) sections[current].append(lesson) continue if lesson is None: @@ -196,7 +197,7 @@ def lesson_key(l: dict) -> tuple[str, str, str, str]: ok("knowledge:lesson-shape", "every lesson is a bold bullet with non-empty why / do / source (confidence 0-100)") empty = [d for d, ls in sections.items() if not ls] if empty: - warn("knowledge:empty-days", f"day sections with no lessons: {empty}") + fail("knowledge:empty-days", f"day sections with no recognizable lessons: {empty}") # --- lessons/ -------------------------------------------------------------- ldir = ROOT / "lessons" @@ -270,15 +271,24 @@ def lesson_key(l: dict) -> tuple[str, str, str, str]: if not vdir.is_dir(): warn("verified/", "directory missing") else: - vfiles = sorted(p for p in vdir.glob("*.md")) + ventries = sorted(vdir.glob("*.md")) + vunreadable = [p.name for p in ventries if not p.is_file()] + if vunreadable: + fail("verified:readable", f"not readable regular files: {vunreadable}") + vfiles = [p for p in ventries if p.is_file()] badnames = [p.name for p in vfiles if not VERIFIED_NAME_RE.match(p.name) or not valid_date(p.name[:10])] if badnames: fail("verified:names", f"not a real YYYY-MM-DD_topic_vN.md: {badnames}") else: ok("verified:names", f"{len(vfiles)} sheet(s), all dated and versioned") - noh1 = [p.name for p in vfiles if not p.read_text(encoding="utf-8").lstrip().startswith("# ")] + def first_nonblank(p: pathlib.Path) -> str: + for ln in p.read_text(encoding="utf-8").splitlines(): + if ln.strip(): + return ln + return "" + noh1 = [p.name for p in vfiles if not H1_LINE_RE.match(first_nonblank(p))] if noh1: - fail("verified:h1", f"no H1: {noh1}") + fail("verified:h1", f"first non-blank line is not an H1 (indent ≤3, `# `): {noh1}") elif vfiles: ok("verified:h1", "every sheet starts with an H1") From c27bc35ff0a7a2b29a56e6479dd55e79bd450921 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 23:14:20 -0400 Subject: [PATCH 07/29] harness: HTML comments never count as content; duplicate lesson fields fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-6 Codex findings (2). Positive: RESULT: PASS — 12 pass, 0 warn, 0 fail. Negatives: duplicate Why → FAIL; fields inside (single- and multi-line) → FAIL. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/LEDGER.md b/LEDGER.md index 21f6bbf..348ae10 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -9,6 +9,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 18 Codex findings | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 20 Codex findings | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 4a450e5..36e8064 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -77,6 +77,37 @@ def norm(s: str) -> str: return " ".join(s.split()) +def strip_html_comments(lines: list[str]) -> list[str]: + """Blank out everything inside so commented-out content never counts as present. + + Line count is preserved (commented text becomes empty) so positions stay meaningful. + """ + out: list[str] = [] + in_comment = False + for ln in lines: + buf = [] + i = 0 + while i < len(ln): + if in_comment: + j = ln.find("-->", i) + if j < 0: + i = len(ln) + else: + in_comment = False + i = j + 3 + else: + j = ln.find(" so commented-out content never counts as present. +FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") +unclosed: list[str] = [] # filled by sanitize: a fence or comment still open at end of file - Line count is preserved (commented text becomes empty) so positions stay meaningful. + +def sanitize(lines: list[str]) -> list[str]: + """Return what a reader sees as prose: fenced code and HTML comments blanked. + + Single pass, CommonMark-shaped: inside a fence, nothing is interpreted (a + ``. Line count is preserved. """ out: list[str] = [] + fence_char: str | None = None + fence_len = 0 in_comment = False + unclosed.clear() for ln in lines: - buf = [] + if fence_char is not None: + m = FENCE_RE.match(ln) + if m and m.group(1)[0] == fence_char and len(m.group(1)) >= fence_len and m.group(2).strip() == "": + fence_char = None + out.append("") + continue + if in_comment: + j = ln.find("-->") + if j < 0: + out.append("") + continue + in_comment = False + ln = ln[j + 3:] # the rest of the line is prose again + m = FENCE_RE.match(ln) + if m: + fence_char, fence_len = m.group(1)[0], len(m.group(1)) + out.append("") + continue + buf: list[str] = [] i = 0 while i < len(ln): if in_comment: @@ -106,36 +137,13 @@ def strip_html_comments(lines: list[str]) -> list[str]: in_comment = True i = j + 4 out.append("".join(buf)) + if fence_char is not None: + unclosed.append(f"fenced code block opened with {fence_char * fence_len} never closes") + if in_comment: + unclosed.append("HTML comment never closes") return out -FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})") - - -def blank_fenced_code(lines: list[str]) -> list[str]: - """Blank out fenced code blocks (``` or ~~~) so their contents never count as content.""" - out: list[str] = [] - fence: str | None = None - for ln in lines: - m = FENCE_RE.match(ln) - if fence is None and m: - fence = m.group(1)[0] # opening fence character - out.append("") - continue - if fence is not None: - out.append("") - if m and m.group(1)[0] == fence: - fence = None - continue - out.append(ln) - return out - - -def sanitize(lines: list[str]) -> list[str]: - """Lines with HTML comments and fenced code blanked; what a reader actually sees as prose.""" - return blank_fenced_code(strip_html_comments(lines)) - - def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: """Return ({date: [lesson]}, [duplicate dates]) for a knowledge-style file. @@ -155,6 +163,7 @@ def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: def new_lesson(title: str, malformed: bool) -> dict: return {"title": norm(title), "why": "", "do": "", "source": "", + "seen": {"why": False, "do": False, "source": False}, "problems": (["not-a-bold-lesson-bullet"] if malformed else [])} for ln in lines: @@ -185,30 +194,34 @@ def new_lesson(title: str, malformed: bool) -> dict: continue if lesson is None: continue - if (m := WHY_RE.match(ln)): - if lesson["why"]: - lesson["problems"].append("duplicate why") - lesson["why"] = norm(m.group(1)) - elif (m := DO_RE.match(ln)): - if lesson["do"]: - lesson["problems"].append("duplicate do") - lesson["do"] = norm(m.group(1)) - elif (m := SOURCE_RE.match(ln)): - if lesson["source"] or any(pr.startswith("confidence") for pr in lesson["problems"]): - lesson["problems"].append("duplicate source") - conf = int(m.group(1)) - if 0 <= conf <= 100: - lesson["source"] = norm(ln.strip()[2:]) + m = FIELD_LABEL_RE.match(ln) + if not m: + continue + label, value = m.group(1), m.group(2) + key = {"Why it matters here": "why", "Do": "do", "Source": "source"}[label] + if lesson["seen"][key]: + lesson["problems"].append(f"duplicate {key}") + lesson["seen"][key] = True + if key == "source": + sm = SOURCE_VALUE_RE.match(value) + if not sm: + lesson["problems"].append("source line is not `[title](https://…) … confidence N%`") + elif not 0 <= int(sm.group(1)) <= 100: + lesson["problems"].append(f"confidence {sm.group(1)}% out of 0-100") else: - lesson["problems"].append(f"confidence {conf}% out of 0-100") + lesson["source"] = norm(value) + elif value.strip(): + lesson[key] = norm(value) + else: + lesson["problems"].append(f"missing or empty {key}") for ls in sections.values(): for l in ls: if "not-a-bold-lesson-bullet" in l["problems"]: continue for k in FIELDS: - if not l[k] and not (k == "source" and any(p.startswith("confidence") for p in l["problems"])): - l["problems"].append(f"missing or empty {k}") + if not l["seen"][k]: + l["problems"].append(f"missing {k}") return sections, duplicates @@ -233,6 +246,9 @@ def lesson_key(l: dict) -> tuple[str, str, str, str]: else: fail("knowledge:h1", f"first line must be {KNOWLEDGE_H1!r}, got {(lines[0] if lines else '')[:70]!r}") sections, dup_dates = parse_lessons(lines) + k_unclosed = list(unclosed) + if k_unclosed: + fail("knowledge:unclosed", "; ".join(k_unclosed) + " — everything after it is hidden from readers") k_bad_headings = list(bad_headings) dates = list(sections) if k_bad_headings: @@ -296,7 +312,13 @@ def lesson_key(l: dict) -> tuple[str, str, str, str]: if not p.is_file(): continue day_lines = sanitize(p.read_text(encoding="utf-8").splitlines()) - problems: list[str] = [] + problems: list[str] = list(unclosed) + first = next((ln for ln in day_lines if ln.strip()), "") + hm = DAY_H1_RE.match(first) + if not hm: + problems.append(f"first line must be `# Lessons — {d}`, got {first[:40]!r}") + elif hm.group(1) != d: + problems.append(f"H1 date {hm.group(1)} does not match filename {d}") stray_headings = [ln.strip() for ln in day_lines if H2_RE.match(ln)] if stray_headings: problems.append(f"contains H2 heading(s) {stray_headings[:4]} — a per-day file has no `## ` headings") From 6e845a6c2165155758745032d7de7be11c76dba4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 23:38:10 -0400 Subject: [PATCH 10/29] harness: CommonMark indentation for H2s/bullets (nesting relative to lesson indent), inline code masks comment markers, invalid UTF-8 is a structured FAIL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-9 Codex findings (4). Positive: PASS — 12/0/0. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 78 +++++++++++++++++++++++++------------- 2 files changed, 53 insertions(+), 27 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index f901641..e7da4d3 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -9,6 +9,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 29 Codex findings | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 33 Codex findings | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 5ea7897..b1d99db 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -37,17 +37,17 @@ KNOWLEDGE_H1 = "# daily-dev-agentic knowledge — T agent" # the connector depends on this exact line DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") -SECTION_RE = re.compile(r"^## (\d{4}-\d{2}-\d{2})\s*$") -LESSON_RE = re.compile(r"^- \*\*(.+?)\*\*\s*$") # well-formed bold lesson bullet -TOP_BULLET_RE = re.compile(r"^(?:[-*+]|\d+[.)]) +\S") # any top-level list item, any marker -FIELD_LABEL_RE = re.compile(r"^ - (Why it matters here|Do|Source):(.*)$") # label first, value validated after +SECTION_RE = re.compile(r"^ {0,3}## (\d{4}-\d{2}-\d{2})\s*$") +LESSON_RE = re.compile(r"^ {0,3}- \*\*(.+?)\*\*\s*$") # well-formed bold lesson bullet (≤3-space indent is still top level) +TOP_BULLET_RE = re.compile(r"^ {0,3}(?:[-*+]|\d+[.)]) +\S") # any top-level list item, any marker, ≤3-space indent +FIELD_LABEL_RE = re.compile(r"^\s*- (Why it matters here|Do|Source):(.*)$") # label first, value validated after (nesting checked by indent) LINK_LABEL = r"\[(?:\\.|[^\]\\])+\]" # allows escaped \] inside the label SOURCE_VALUE_RE = re.compile(r"^ " + LINK_LABEL + r"\(https?://[^)\s]+\).*confidence (\d{1,3})%") DAY_H1_RE = re.compile(r"^# Lessons — (\d{4}-\d{2}-\d{2})\s*$") VERIFIED_NAME_RE = re.compile(r"^\d{4}-\d{2}-\d{2}_[a-z0-9-]+_v\d+\.md$") H1_LINE_RE = re.compile(r"^ {0,3}# \S") # a real ATX H1: at most 3 leading spaces (4 = code block) FIELDS = ("why", "do", "source") -H2_RE = re.compile(r"^##(?:\s|$)") # any ATX H2, including a bare `##` +H2_RE = re.compile(r"^ {0,3}##(?:\s|$)") # any ATX H2 (≤3-space indent), including a bare `##` bad_headings: list[str] = [] # filled by parse_lessons: H2s that are not valid day headings @@ -80,7 +80,17 @@ def norm(s: str) -> str: return " ".join(s.split()) +def read_lines(p: pathlib.Path) -> list[str] | None: + """Lines of a UTF-8 text file, or None (with a FAIL recorded) when it is not valid UTF-8.""" + try: + return p.read_text(encoding="utf-8").splitlines() + except UnicodeDecodeError as exc: + fail(f"utf8:{p.name}", f"not valid UTF-8 at byte {exc.start}: {exc.reason}") + return None + + FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") +CODE_SPAN_RE = re.compile(r"(`+)(?!`)(?:.+?)(? list[str]: fence_char, fence_len = m.group(1)[0], len(m.group(1)) out.append("") continue + # Inline code spans are opaque: a ", i) + j = scan.find("-->", i) if j < 0: - i = len(ln) + i = len(scan) else: in_comment = False i = j + 3 else: - j = ln.find("`. Line count is preserved. + comment cannot open a fence) until `-->`. Fences are recognised relative + to the content column of the innermost open list item, so a code block + nested under a lesson field is code too. Line count is preserved. """ out: list[str] = [] fence_char: str | None = None fence_len = 0 + fence_base = 0 in_comment = False + containers: list[int] = [] # content columns of the open list items, innermost last unclosed.clear() for ln in lines: if fence_char is not None: - m = FENCE_RE.match(ln) + m = fence_match(ln, fence_base) if m and m.group(1)[0] == fence_char and len(m.group(1)) >= fence_len and m.group(2).strip() == "": fence_char = None out.append("") @@ -139,11 +155,23 @@ def sanitize(lines: list[str]) -> list[str]: continue in_comment = False ln = ln[j + 3:] # the rest of the line is prose again - m = FENCE_RE.match(ln) + # Track which list items are still open: a non-blank line shallower than an + # item's content column closes that item (and everything nested in it). + if ln.strip(): + indent = len(ln) - len(ln.lstrip(" ")) + while containers and indent < containers[-1]: + containers.pop() + li = LIST_ITEM_RE.match(ln) + if li: + gap = len(li.group(3)) + col = len(li.group(1)) + len(li.group(2)) + (1 if gap >= 5 else gap) # ≥5 spaces = code inside the item + containers.append(col) + base = containers[-1] if containers else 0 + m = fence_match(ln, base) # CommonMark: a backtick fence cannot open when its info string contains a # backtick (that line is an inline code span, not a fence); tilde fences may. if m and not (m.group(1)[0] == "`" and "`" in m.group(2)): - fence_char, fence_len = m.group(1)[0], len(m.group(1)) + fence_char, fence_len, fence_base = m.group(1)[0], len(m.group(1)), base out.append("") continue # Inline code spans are opaque: a `. Fences are recognised relative to the content column of the innermost open list item, so a code block - nested under a lesson field is code too. Line count is preserved. + nested under a lesson field is code too. The opener line is replaced by + FENCE_MARK at the fence's indent so the parser still sees a block boundary + there (a fence closes an open list item just as a heading does); the rest + of the block is blank. Line count is preserved. """ out: list[str] = [] fence_char: str | None = None @@ -172,7 +176,7 @@ def sanitize(lines: list[str]) -> list[str]: # backtick (that line is an inline code span, not a fence); tilde fences may. if m and not (m.group(1)[0] == "`" and "`" in m.group(2)): fence_char, fence_len, fence_base = m.group(1)[0], len(m.group(1)), base - out.append("") + out.append(" " * (len(ln) - len(ln.lstrip(" "))) + FENCE_MARK) continue # Inline code spans are opaque: a `; an inline `` mid-paragraph still leaves nothing. Leading tabs are expanded at 4-column stops. Thematic breaks never open a list container. - THEMATIC_RE / INTERRUPT_RE require 3+ of the SAME character (`- * -` is not a break); `- - -` and `* * *` are breaks: never a malformed lesson, never "misplaced", and they close an open lesson. - verified:h1 and the per-day first-line check skip sentinel lines, so a sheet that opens with a closed fence or comment then a real H1 passes (the contract says code and comments are skipped). - LESSON_RE: the title must be ONE strong span (unescaped `**` inside → malformed); 1-4 columns of whitespace after `-` are accepted, 5+ means the title is an indented code block → malformed. - Lessons store their CommonMark content column (marker + expanded whitespace) and nesting uses it: `-**T**` has content column 4, so 2-space fields are siblings, not children. - lessons/ and verified/ themselves must not be symlinks. - LEDGER: 46 -> 52 Codex findings Positive run: PASS 12/0/0. Round-14 suite 18/18 (10 cases fail on the previous head); rounds 11-13 suites 10/10, 8/8, 6/6 (round-11 tab guard corrected to 4-space fields per CommonMark). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 125 ++++++++++++++++++++++++++++--------- 2 files changed, 95 insertions(+), 32 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index cc5c80a..06e901c 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -9,6 +9,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 46 Codex findings | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 52 Codex findings | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index f76e0b3..049ddd4 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -39,12 +39,13 @@ DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") SECTION_RE = re.compile(r"^ {0,3}## (\d{4}-\d{2}-\d{2})\s*$") -LESSON_RE = re.compile(r"^ {0,3}-[ \t]\*\*(.+?)\*\*\s*$") # well-formed bold lesson bullet (≤3-space indent is still top level; space or tab after `-`) +LESSON_RE = re.compile(r"^ {0,3}-([ \t]+)\*\*(.+?)\*\*\s*$") # bold lesson bullet: `-`, 1-4 columns of whitespace (checked after), **title** TOP_BULLET_RE = re.compile(r"^ {0,3}(?:[-*+]|\d+[.)])(?:[ \t]+\S|\s*$)") # any top-level list item incl. an EMPTY one, any marker, space or tab after it FIELD_LABEL_RE = re.compile(r"^\s*-[ \t](Why it matters here|Do|Source):(.*)$") # label first, value validated after (nesting checked by indent) # Blocks that interrupt a paragraph (CommonMark): an ATX heading, a block quote, a thematic break. # A line like this at top level is never lazy continuation of the lesson above it. -INTERRUPT_RE = re.compile(r"^ {0,3}(?:#{1,6}(?:\s|$)|>|(?:[-*_][ \t]*){3,}$)") +THEMATIC_BREAK = r"(?:(?:-[ \t]*){3,}|(?:\*[ \t]*){3,}|(?:_[ \t]*){3,})$" # 3+ of the SAME character (`- * -` is not a break) +INTERRUPT_RE = re.compile(r"^ {0,3}(?:#{1,6}(?:\s|$)|>|" + THEMATIC_BREAK + ")") LINK_LABEL = r"\[(?:\\.|[^\]\\])+\]" # allows escaped \] inside the label SOURCE_VALUE_RE = re.compile(r"^ " + LINK_LABEL + r"\(https?://[^)\s]+\).*confidence (\d{1,3})%") DAY_H1_RE = re.compile(r"^# Lessons — (\d{4}-\d{2}-\d{2})\s*$") @@ -107,12 +108,48 @@ def read_lines(p: pathlib.Path) -> list[str] | None: FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") -FENCE_MARK = "\x00fence" # left by sanitize() where a fenced block opened, at its indent: a block boundary, never text -LIST_ITEM_RE = re.compile(r"^( *)([-*+]|\d{1,9}[.)])( +)\S") # marker + 1..n spaces + content (for container tracking) +FENCE_MARK = "\x00fence" # left by sanitize() where a fenced block opened, at its indent: a block boundary, never text +COMMENT_MARK = "\x00comment" # same, where an HTML comment BLOCK (`` mid-paragraph leaves nothing, it is just + raw inline HTML. The rest of each block is blank. Leading tabs are expanded + to spaces at 4-column stops. Line count is preserved. """ out: list[str] = [] fence_char: str | None = None fence_len = 0 fence_base = 0 in_comment = False + block_comment = False # the open comment began a line: the whole closing line is HTML, not prose containers: list[int] = [] # content columns of the open list items, innermost last unclosed.clear() for ln in lines: + ln = expand_lead(ln) if fence_char is not None: m = fence_match(ln, fence_base) if m and m.group(1)[0] == fence_char and len(m.group(1)) >= fence_len and m.group(2).strip() == "": @@ -158,6 +200,10 @@ def sanitize(lines: list[str]) -> list[str]: out.append("") continue in_comment = False + if block_comment: + block_comment = False + out.append("") # HTML block type 2 ends WITH the line holding `-->` + continue ln = ln[j + 3:] # the rest of the line is prose again # Track which list items are still open: a non-blank line shallower than an # item's content column closes that item (and everything nested in it). @@ -166,11 +212,16 @@ def sanitize(lines: list[str]) -> list[str]: while containers and indent < containers[-1]: containers.pop() li = LIST_ITEM_RE.match(ln) - if li: - gap = len(li.group(3)) - col = len(li.group(1)) + len(li.group(2)) + (1 if gap >= 5 else gap) # ≥5 spaces = code inside the item - containers.append(col) + if li and not THEMATIC_RE.match(ln): + containers.append(content_col(li)) base = containers[-1] if containers else 0 + # An HTML comment block (`" not in ln[ln.find("` still do - parse_source(): a backslash-escaped ASCII punctuation char in the destination is text, so `https://x.y/\(` balances - LEDGER: 63 -> 67 Codex findings Positive run: PASS 12/0/0. Round-17 suite 6/6 (5 cases fail on the previous head); rounds 11-16 suites 10/10, 8/8, 6/6, 18/18, 11/11, 12/12. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 27 ++++++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index d3ec9e5..0ed5f28 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -9,6 +9,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 63 Codex findings | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 67 Codex findings | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index c6b6569..73a0e18 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -56,6 +56,7 @@ LINK_LABEL = r"\[(?:\\.|[^\]\\])+\]" # allows escaped \] inside the label LINK_OPEN_RE = re.compile(r"^" + LINK_LABEL + r"\(") # `[label](` — the destination is scanned by hand CONFIDENCE_RE = re.compile(r"confidence (\d{1,3})%") +ASCII_PUNCT = set(r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""") # a backslash escapes exactly these (CommonMark) def parse_source(text: str) -> tuple[str, int] | None: @@ -73,6 +74,9 @@ def parse_source(text: str) -> tuple[str, int] | None: c = text[i] if c.isspace(): return None + if c == "\\" and i + 1 < len(text) and text[i + 1] in ASCII_PUNCT: + i += 2 # `\(` is destination text, not a delimiter + continue if c == "(": depth += 1 elif c == ")": @@ -153,9 +157,10 @@ def regular_file(p: pathlib.Path) -> bool: def read_lines(p: pathlib.Path) -> list[str] | None: - """Lines of a UTF-8 text file, or None (with a FAIL recorded) when it is not valid UTF-8.""" + """Lines of a UTF-8 text file (a leading BOM is encoding metadata, not content, and is + dropped), or None (with a FAIL recorded) when it is not valid UTF-8.""" try: - return p.read_text(encoding="utf-8").splitlines() + return p.read_text(encoding="utf-8-sig").splitlines() except UnicodeDecodeError as exc: fail(f"utf8:{p.name}", f"not valid UTF-8 at byte {exc.start}: {exc.reason}") return None @@ -325,6 +330,7 @@ def sanitize(lines: list[str]) -> list[str]: scan, open_run = mask_code(ln, open_run) buf: list[str] = [] i = 0 + comment_here = False while i < len(scan): if in_comment: j = scan.find("-->", i) @@ -340,9 +346,15 @@ def sanitize(lines: list[str]) -> list[str]: i = len(scan) else: buf.append(ln[i:j]) - in_comment = True + in_comment = comment_here = True i = j + 4 - out.append("".join(buf)) + prose = "".join(buf) + # Spaces that sat before a comment which ran to the end of the line (`text ` + # or a comment that continues onto the next line) were never line-ending spaces, so + # they must not read as a hard break once the comment is gone. Spaces AFTER `-->` stay. + if comment_here and (in_comment or scan.endswith("-->")): + prose = prose.rstrip() + out.append(prose) if fence_char is not None: unclosed.append(f"fenced code block opened with {fence_char * fence_len} never closes") if in_comment: @@ -427,9 +439,10 @@ def absorb(lesson: dict, ln: str) -> None: continue m = LESSON_RE.match(ln) li = LIST_ITEM_RE.match(ln) - # `- ** **` is not bold; `- **a** b **c**` is two strong spans, not one title; - # 5+ columns after the marker make the title an indented code block, not bold. - if (m and li and m.group(2).strip() + # `- ** **` / `- ** x**` / `- **x **` are not bold (a `**` next to whitespace cannot + # open or close); `- **a** b **c**` is two strong spans, not one title; 5+ columns + # after the marker make the title an indented code block, not bold. + if (m and li and m.group(2).strip() and not m.group(2)[0].isspace() and not m.group(2)[-1].isspace() and not INNER_STRONG_RE.search(CODE_SPAN_RE.sub(lambda c: "x" * len(c.group(0)), m.group(2))) # `**` in code is opaque and gap_cols(indent + 1, m.group(1)) <= 4): lesson = new_lesson(m.group(2), malformed=False) From 568a411738d95a8ad952f437f4094fe4db5dbf70 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 01:11:52 -0400 Subject: [PATCH 19/29] harness: round-18 review fixes (escaped closers, escaped backticks, all HTML block types, `\ ` breaks, < > in destinations, Markdown line endings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - a title ending in an odd number of backslashes has an escaped closing star → malformed - mask_code()/CODE_SPAN_RE: a backslash-escaped backtick is a literal, never part of a run - sanitize() now handles every CommonMark HTML block type, not just comments: types 1-5 blank their lines through the line holding the terminator (, -->, ?>, ]]>, >), types 6-7 through the next blank line; each leaves a boundary mark (HTML_MARK / COMMENT_MARK); a type-7 tag never starts inside a paragraph. Headings and fields inside raw HTML are not Markdown. - ends_hard(): a backslash is a hard break only when it is the last character on the line - parse_source(): unescaped < or > in a bare destination → not a link - read_lines(): split on CR, LF, CRLF only (U+2028 etc. stay inside a line) - LEDGER: 67 -> 73 Codex findings Positive run: PASS 12/0/0. Round-18 suite 9/9 (6 cases fail on the previous head); rounds 11-17 suites 10/10, 8/8, 6/6, 18/18, 11/11, 12/12, 6/6. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 119 ++++++++++++++++++++++++++++--------- 2 files changed, 92 insertions(+), 29 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index 0ed5f28..9577fdf 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -9,6 +9,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 67 Codex findings | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 73 Codex findings | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 73a0e18..e5d2003 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -37,7 +37,7 @@ KNOWLEDGE_H1 = "# daily-dev-agentic knowledge — T agent" # the connector depends on this exact line -CODE_SPAN_RE = re.compile(r"(`+)(?!`)(?:.+?)(?|$)|" + if re.match(r"^ {0,3}<\?", rel): + return HTML_MARK, "?>" + if re.match(r"^ {0,3}" + if re.match(r"^ {0,3}" + if re.match(r"^ {0,3}|$)", rel, re.IGNORECASE): + return HTML_MARK, None + if not in_paragraph and _HTML_TYPE7.match(rel): + return HTML_MARK, None + return None INNER_STRONG_RE = re.compile(r"(? tuple[str, int]: so `") if j < 0: out.append("") continue in_comment = False - if block_comment: - block_comment = False - out.append("") # HTML block type 2 ends WITH the line holding `-->` - continue ln = ln[j + 3:] # the rest of the line is prose again # Track which list items are still open: a non-blank line shallower than an # item's content column closes that item (and everything nested in it). @@ -310,12 +358,22 @@ def sanitize(lines: list[str]) -> list[str]: if li and not THEMATIC_RE.match(ln): containers.append(content_col(li)) base = containers[-1] if containers else 0 - # An HTML comment block (`" not in ln[ln.find("` closes on its own line continue m = fence_match(ln, base) # CommonMark: a backtick fence cannot open when its info string contains a @@ -323,6 +381,7 @@ def sanitize(lines: list[str]) -> list[str]: if m and not (m.group(1)[0] == "`" and "`" in m.group(2)): fence_char, fence_len, fence_base = m.group(1)[0], len(m.group(1)), base open_run = 0 + in_paragraph = False out.append(" " * (len(ln) - len(ln.lstrip(" "))) + FENCE_MARK) continue # Inline code spans are opaque: a ` stay. if comment_here and (in_comment or scan.endswith("-->")): prose = prose.rstrip() + in_paragraph = bool(prose.strip()) and not H2_RE.match(prose) and not THEMATIC_RE.match(prose) out.append(prose) if fence_char is not None: unclosed.append(f"fenced code block opened with {fence_char * fence_len} never closes") - if in_comment: + if in_comment or (in_html and html_end == "-->"): unclosed.append("HTML comment never closes") + elif in_html and html_end is not None: + unclosed.append(f"HTML block never closes (no {html_end!r})") return out @@ -443,6 +505,7 @@ def absorb(lesson: dict, ln: str) -> None: # open or close); `- **a** b **c**` is two strong spans, not one title; 5+ columns # after the marker make the title an indented code block, not bold. if (m and li and m.group(2).strip() and not m.group(2)[0].isspace() and not m.group(2)[-1].isspace() + and (len(m.group(2)) - len(m.group(2).rstrip("\\"))) % 2 == 0 # `\**` escapes the first closing star and not INNER_STRONG_RE.search(CODE_SPAN_RE.sub(lambda c: "x" * len(c.group(0)), m.group(2))) # `**` in code is opaque and gap_cols(indent + 1, m.group(1)) <= 4): lesson = new_lesson(m.group(2), malformed=False) From 3c4d616a769508ea3ab21354553e19b1c67f1081 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 01:25:29 -0400 Subject: [PATCH 20/29] harness: round-19 review fixes (ten CommonMark edge cases in sanitize/parse/link parsing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - HTML block type 4 needs an UPPERCASE letter after `` is visible text - HTML block terminators match case-insensitively (`
` … `
`) - a title may not begin or end with an unescaped `*` (`**foo***`, `***foo**` are not one strong span) - link labels may contain balanced, unescaped brackets (`[RFC [draft]](…)`) - sanitize(): a lazy continuation (paragraph text that starts no block, judged against the deepest container its indent satisfies) keeps the open list containers, so a fence at the field's column after it is still a fence - in_paragraph resets after any ATX heading, so a type-7 tag right after the H1 is an HTML block - ATX separators are space/tab only (`##note` is a paragraph) - text after a multi-line inline comment's `-->` is emitted as CONT_MARK paragraph text and can never start a field or list item - link reference definitions are skipped when locating a sheet's first visible H1 - an ordered item not starting at 1 (or an empty item) cannot interrupt a paragraph: lazy text - LEDGER: 73 -> 83 Codex findings Positive run: PASS 12/0/0. Round-19 suite 14/14 (10 cases fail on the previous head); rounds 11-18 suites 10/10, 8/8, 6/6, 18/18, 11/11, 12/12, 6/6, 9/9. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 163 ++++++++++++++++++++++++++----------- 2 files changed, 115 insertions(+), 50 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index 9577fdf..5fef6c0 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -9,6 +9,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 73 Codex findings | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 83 Codex findings | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index e5d2003..9358503 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -46,7 +46,9 @@ # Blocks that interrupt a paragraph (CommonMark): an ATX heading, a block quote, a thematic break. # A line like this at top level is never lazy continuation of the lesson above it. THEMATIC_BREAK = r"(?:(?:-[ \t]*){3,}|(?:\*[ \t]*){3,}|(?:_[ \t]*){3,})$" # 3+ of the SAME character (`- * -` is not a break) -INTERRUPT_RE = re.compile(r"^ {0,3}(?:#{1,6}(?:\s|$)|>|" + THEMATIC_BREAK + ")") +INTERRUPT_RE = re.compile(r"^ {0,3}(?:#{1,6}(?:[ \t]|$)|>|" + THEMATIC_BREAK + ")") +ATX_RE = re.compile(r"^ {0,3}#{1,6}(?:[ \t]|$)") # any ATX heading +LINK_REF_DEF_RE = re.compile(r"^ {0,3}\[(?:\\.|[^\]\\])+\]:[ \t]*\S") # `[label]: dest …` renders nothing LINK_LABEL = r"\[(?:\\.|[^\]\\])+\]" # allows escaped \] inside the label LINK_OPEN_RE = re.compile(r"^" + LINK_LABEL + r"\(") # `[label](` — the destination is scanned by hand CONFIDENCE_RE = re.compile(r"confidence (\d{1,3})%") @@ -60,10 +62,24 @@ def parse_source(text: str) -> tuple[str, int] | None: balance — `https://x.y/(broken)` never closes, so it is not a link at all. """ text = text.strip() - m = LINK_OPEN_RE.match(text) - if not m: + if not text.startswith("["): return None - i, depth = m.end(), 0 + j, bdepth = 1, 0 # the label: balanced, unescaped brackets are allowed inside + while j < len(text): + c = text[j] + if c == "\\" and j + 1 < len(text) and text[j + 1] in ASCII_PUNCT: + j += 2 + continue + if c == "[": + bdepth += 1 + elif c == "]": + if bdepth == 0: + break + bdepth -= 1 + j += 1 + if j >= len(text) or j == 1 or j + 1 >= len(text) or text[j + 1] != "(": + return None + i, depth = j + 2, 0 while i < len(text): c = text[i] if c.isspace() or c in "<>": @@ -80,7 +96,7 @@ def parse_source(text: str) -> tuple[str, int] | None: i += 1 else: return None # ran off the end without closing the destination - dest = text[m.end():i] + dest = text[j + 2:i] if not re.match(r"https?://\S+$", dest): return None cm = CONFIDENCE_RE.search(text[i + 1:]) @@ -89,7 +105,7 @@ def parse_source(text: str) -> tuple[str, int] | None: VERIFIED_NAME_RE = re.compile(r"^\d{4}-\d{2}-\d{2}_[a-z0-9-]+_v\d+\.md$") H1_LINE_RE = re.compile(r"^ {0,3}#[ \t]+\S") # a real ATX H1 with text: ≤3 leading spaces (4 = code), space or tab after `#` FIELDS = ("why", "do", "source") -H2_RE = re.compile(r"^ {0,3}##(?:\s|$)") # any ATX H2 (≤3-space indent), including a bare `##` +H2_RE = re.compile(r"^ {0,3}##(?:[ \t]|$)") # any ATX H2 (≤3-space indent), including a bare `##`; NBSP is not a separator bad_headings: list[str] = [] # filled by parse_lessons: H2s that are not valid day headings misplaced: list[str] = [] # filled by parse_lessons: top-level bullets that sit under no `## YYYY-MM-DD` @@ -169,6 +185,7 @@ def read_lines(p: pathlib.Path) -> list[str] | None: FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") FENCE_MARK = "\x00fence" # left by sanitize() where a fenced block opened, at its indent: a block boundary, never text COMMENT_MARK = "\x00comment" # same, where an HTML comment BLOCK (`` spans from one prose line (`scan` is `ln` with code masked), carrying + an unterminated comment into the module-level `in_comment` flag via the caller.""" + global _in_comment + buf: list[str] = [] + i = 0 + comment_here = False + while i < len(scan): + if _in_comment: + j = scan.find("-->", i) + if j < 0: + i = len(scan) + else: + _in_comment = False + i = j + 3 + else: + j = scan.find("` + # or a comment that continues onto the next line) were never line-ending spaces, so + # they must not read as a hard break once the comment is gone. Spaces AFTER `-->` stay. + if comment_here and (_in_comment or scan.endswith("-->")): + prose = prose.rstrip() + return prose + + +_in_comment = False # an inline `") if j < 0: out.append("") continue - in_comment = False - ln = ln[j + 3:] # the rest of the line is prose again + _in_comment = False + ln = ln[j + 3:] # the rest of the line is prose again — but it did not BEGIN the + cont = True # line, so it can never start a block: it stays in the paragraph # Track which list items are still open: a non-blank line shallower than an # item's content column closes that item (and everything nested in it). - if ln.strip(): + base = containers[-1] if containers else 0 + if ln.strip() and not cont: indent = len(ln) - len(ln.lstrip(" ")) - while containers and indent < containers[-1]: - containers.pop() + # A lazy continuation (paragraph text that is none of the block starts) stays inside + # the open list item however shallow it is, so it must not pop the containers. li = LIST_ITEM_RE.match(ln) - if li and not THEMATIC_RE.match(ln): - containers.append(content_col(li)) + # Judge "does this line start a block?" relative to the deepest container the + # line's indent still satisfies (a `` closes on its own line + in_html = html_end not in rel.lower()[len(html_end) - 1:] # e.g. `` closes on its own line continue m = fence_match(ln, base) # CommonMark: a backtick fence cannot open when its info string contains a @@ -387,37 +467,12 @@ def sanitize(lines: list[str]) -> list[str]: # Inline code spans are opaque: a ", i) - if j < 0: - i = len(scan) - else: - in_comment = False - i = j + 3 - else: - j = scan.find("` - # or a comment that continues onto the next line) were never line-ending spaces, so - # they must not read as a hard break once the comment is gone. Spaces AFTER `-->` stay. - if comment_here and (in_comment or scan.endswith("-->")): - prose = prose.rstrip() - in_paragraph = bool(prose.strip()) and not H2_RE.match(prose) and not THEMATIC_RE.match(prose) + prose = strip_inline_comments(ln, scan) + in_paragraph = bool(prose.strip()) and not ATX_RE.match(prose) and not THEMATIC_RE.match(prose) out.append(prose) if fence_char is not None: unclosed.append(f"fenced code block opened with {fence_char * fence_len} never closes") - if in_comment or (in_html and html_end == "-->"): + if _in_comment or (in_html and html_end == "-->"): unclosed.append("HTML comment never closes") elif in_html and html_end is not None: unclosed.append(f"HTML block never closes (no {html_end!r})") @@ -466,6 +521,10 @@ def absorb(lesson: dict, ln: str) -> None: prev_blank = True for ln in lines: was_prev_blank, prev_blank = prev_blank, not ln.strip() + if ln.startswith(CONT_MARK): # paragraph text that began inside an inline comment + if lesson is not None and current is not None and not was_prev_blank: + absorb(lesson, ln[len(CONT_MARK):]) + continue indent = len(ln) - len(ln.lstrip(" ")) # CommonMark: a line indented at or past the open lesson's content column is INSIDE # that list item — even ` ## 2026-08-23` — so it can never open a day section. @@ -506,6 +565,8 @@ def absorb(lesson: dict, ln: str) -> None: # after the marker make the title an indented code block, not bold. if (m and li and m.group(2).strip() and not m.group(2)[0].isspace() and not m.group(2)[-1].isspace() and (len(m.group(2)) - len(m.group(2).rstrip("\\"))) % 2 == 0 # `\**` escapes the first closing star + and m.group(2)[0] != "*" # `***foo**` → `*` + strong + and not (m.group(2).endswith("*") and (len(m.group(2)) - 1 - len(m.group(2)[:-1].rstrip("\\"))) % 2 == 0) # `**foo***` → strong + `*` and not INNER_STRONG_RE.search(CODE_SPAN_RE.sub(lambda c: "x" * len(c.group(0)), m.group(2))) # `**` in code is opaque and gap_cols(indent + 1, m.group(1)) <= 4): lesson = new_lesson(m.group(2), malformed=False) @@ -513,7 +574,11 @@ def absorb(lesson: dict, ln: str) -> None: lesson["hard"] = ends_hard(ln) sections[current].append(lesson) continue - if TOP_BULLET_RE.match(ln): + li_any = LIST_ITEM_RE.match(ln) + if TOP_BULLET_RE.match(ln) and not (lesson is not None and not was_prev_blank + and (li_any is None or not can_interrupt(li_any))): + # (a `2.` item or an empty item directly under a paragraph cannot start a list: it + # is lazy continuation text and is handled below) parts = ln.strip().split(None, 1) lesson = new_lesson("MALFORMED: " + (parts[1].strip() if len(parts) > 1 else "(empty list item)"), malformed=True) lesson["col"] = content_col(li) if li else indent + 2 @@ -678,7 +743,7 @@ def lesson_key(l: dict) -> tuple[str, str, str, str]: continue day_lines = sanitize(raw) problems: list[str] = list(unclosed) - first = next((ln for ln in day_lines if ln.strip() and not is_mark(ln)), "") # comments and code are skipped, not counted + first = next((ln for ln in day_lines if ln.strip() and not is_mark(ln) and not LINK_REF_DEF_RE.match(ln)), "") # comments, code, ref defs skipped hm = DAY_H1_RE.match(first) if not hm: problems.append(f"first visible line must be `# Lessons — {d}`, got {first[:40]!r}") @@ -742,7 +807,7 @@ def first_nonblank(p: pathlib.Path) -> str | None: if lines is None: return None for ln in sanitize(lines): - if ln.strip() and not is_mark(ln): + if ln.strip() and not is_mark(ln) and not LINK_REF_DEF_RE.match(ln): # a `[x]: url` line renders nothing return ln return "" firsts = {p.name: first_nonblank(p) for p in vfiles} From 3c21aab192a0b12b9482844a095b973df0a4c690 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 01:36:46 -0400 Subject: [PATCH 21/29] harness: round-20 review fixes (inline comments end with the paragraph, code-span reset at block starts, two-line ref defs, ASCII tag whitespace and digits, field marker gap, paragraph breaks) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - an inline `` arrives before the paragraph ends (closes_in_paragraph); otherwise it is literal text and later fields stay visible; a blank line also clears any inline-comment state - a backtick run left open in a paragraph is literal once a new block starts (open_run reset whenever a line is not lazy continuation) - first_visible() skips a link reference definition together with a title on its next line - HTML tag grammar uses ASCII space/tab only (`x>` is text); type-4 in the parser's interrupt regex is uppercase-only too - filename/date/marker/confidence regexes use [0-9], not \d (no Unicode digits) - nested field markers accept 1-4 columns of whitespace after `-`; 5+ is an indented code block - a blank line before a field's continuation is a new paragraph and is kept as a ¶ token - LEDGER: 83 -> 90 Codex findings Positive run: PASS 12/0/0. Round-20 suite 9/9 (7 cases fail on the previous head); rounds 11-19 suites all green (round-16 6c updated: an inline ``). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 101 ++++++++++++++++++++++++++----------- 2 files changed, 72 insertions(+), 31 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index 5fef6c0..5743f22 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -9,6 +9,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 83 Codex findings | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 90 Codex findings | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 9358503..feb2a64 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -38,20 +38,39 @@ KNOWLEDGE_H1 = "# daily-dev-agentic knowledge — T agent" # the connector depends on this exact line CODE_SPAN_RE = re.compile(r"(?|" + THEMATIC_BREAK + ")") ATX_RE = re.compile(r"^ {0,3}#{1,6}(?:[ \t]|$)") # any ATX heading LINK_REF_DEF_RE = re.compile(r"^ {0,3}\[(?:\\.|[^\]\\])+\]:[ \t]*\S") # `[label]: dest …` renders nothing +REF_TITLE_RE = re.compile(r"""^[ \t]*(?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|\((?:[^()\\]|\\.)*\))[ \t]*$""") # a ref def's title alone on the next line + + +def first_visible(lines: list[str]) -> str: + """The first line a reader sees as content: blank lines, block marks, link reference + definitions (and a definition's title on the following line) are skipped.""" + k = 0 + while k < len(lines): + ln = lines[k] + if not ln.strip() or is_mark(ln): + k += 1 + continue + if LINK_REF_DEF_RE.match(ln): + k += 1 + if k < len(lines) and REF_TITLE_RE.match(lines[k]): + k += 1 + continue + return ln + return "" LINK_LABEL = r"\[(?:\\.|[^\]\\])+\]" # allows escaped \] inside the label LINK_OPEN_RE = re.compile(r"^" + LINK_LABEL + r"\(") # `[label](` — the destination is scanned by hand -CONFIDENCE_RE = re.compile(r"confidence (\d{1,3})%") +CONFIDENCE_RE = re.compile(r"confidence ([0-9]{1,3})%") ASCII_PUNCT = set(r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""") # a backslash escapes exactly these (CommonMark) @@ -101,8 +120,8 @@ def parse_source(text: str) -> tuple[str, int] | None: return None cm = CONFIDENCE_RE.search(text[i + 1:]) return (dest, int(cm.group(1))) if cm else None -DAY_H1_RE = re.compile(r"^# Lessons — (\d{4}-\d{2}-\d{2})\s*$") -VERIFIED_NAME_RE = re.compile(r"^\d{4}-\d{2}-\d{2}_[a-z0-9-]+_v\d+\.md$") +DAY_H1_RE = re.compile(r"^# Lessons — ([0-9]{4}-[0-9]{2}-[0-9]{2})\s*$") +VERIFIED_NAME_RE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[a-z0-9-]+_v[0-9]+\.md$") H1_LINE_RE = re.compile(r"^ {0,3}#[ \t]+\S") # a real ATX H1 with text: ≤3 leading spaces (4 = code), space or tab after `#` FIELDS = ("why", "do", "source") H2_RE = re.compile(r"^ {0,3}##(?:[ \t]|$)") # any ATX H2 (≤3-space indent), including a bare `##`; NBSP is not a separator @@ -144,6 +163,7 @@ def norm(s: str) -> str: HARD_BREAK = "⏎" # joins two lines of a field when the first ends in a hard line break (2+ spaces or an odd backslash) +PARA_BREAK = "¶" # joins two paragraphs of the same field (a blank line between the field and its continuation) def ends_hard(ln: str) -> bool: @@ -186,20 +206,20 @@ def read_lines(p: pathlib.Path) -> list[str] | None: FENCE_MARK = "\x00fence" # left by sanitize() where a fenced block opened, at its indent: a block boundary, never text COMMENT_MARK = "\x00comment" # same, where an HTML comment BLOCK (`` appears in the following lines before the paragraph ends (a blank line). + CommonMark: raw HTML is inline, so an unclosed `" in nxt: + return True + return False + + +def strip_inline_comments(ln: str, scan: str, rest: list[str] = ()) -> str: """Remove `` spans from one prose line (`scan` is `ln` with code masked), carrying - an unterminated comment into the module-level `in_comment` flag via the caller.""" + an unterminated comment into the module-level `_in_comment` flag when — and only when — its + closer arrives later in the same paragraph (`rest` = the lines that follow).""" global _in_comment buf: list[str] = [] i = 0 @@ -336,6 +369,9 @@ def strip_inline_comments(ln: str, scan: str) -> str: if j < 0: buf.append(ln[i:]) i = len(scan) + elif "-->" not in scan[j + 4:] and not closes_in_paragraph(rest): + buf.append(ln[i:]) # opener with no closer in this paragraph: literal text + i = len(scan) else: buf.append(ln[i:j]) _in_comment = comment_here = True @@ -381,10 +417,11 @@ def sanitize(lines: list[str]) -> list[str]: open_run = 0 # length of an inline-code backtick run still open from the previous line containers: list[int] = [] # content columns of the open list items, innermost last unclosed.clear() - for ln in lines: + for k, ln in enumerate(lines): ln = expand_lead(ln) if not ln.strip(): open_run = 0 # a code span cannot cross a blank line (the paragraph ends) + _in_comment = False # nor can an inline comment: an unclosed `' lookahead stops at any line that can interrupt the paragraph (heading, thematic break, quote, fence, HTML block, list item), judged against the open containers — shared interrupts_paragraph() now drives sanitize's lazy-continuation test too Found while auditing the same code paths: prose after a nested fence/comment block and lesson-level prose after the fields were silently dropped (copies differing there compared equal) — both are now part of the comparison key ("tail"); wholly-commented lines inside a multi-line inline comment keep the paragraph open (CONT_MARK) instead of reading as blank. Three earlier suite cases (neg16 6c, neg18 2, neg20 2) asserted that an inline . commonmark.js 0.31.2 renders the opener as literal text there (block structure is settled before inline HTML), so the assertions were wrong; rewritten to close the comment inside the same paragraph and proven discriminating against the pre-fix harness (neg18, neg20) or a mutant (neg16). Evidence: 12 suites, 126 cases, 0 failing; check-knowledge.py RESULT: PASS — 12 pass, 0 warn, 0 fail; markdownlint-cli2 0.23.3: 0 issues. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 295 +++++++++++++++++++++++++------------ 2 files changed, 203 insertions(+), 94 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index f16f1a6..ba702ff 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -10,6 +10,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | | 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | correction: `verified/2026-09-26_claude-surfaces_v2.md` (session ran in the chat/Cowork tab, not the Code tab — T's correction 2026-09-27); v1 banner-marked superseded | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 94 Codex findings | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 101 Codex findings (22 rounds; round 22 also rewrote 3 earlier suite cases that CommonMark, checked against commonmark.js 0.31.2, had the other way) | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index b3afb39..b8445ec 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -17,7 +17,8 @@ that day's section in knowledge.md, in the same order verified/ only regular files named YYYY-MM-DD_topic_vN.md (real dates) whose first visible non-blank line is a real H1 (not indented 4+) - Text inside HTML comments or fenced code blocks never counts as content. + Text inside HTML comments, fenced code blocks or indented code blocks never + counts as content; a copy that turns prose into code (or back) is a difference. Contract: one line per check (PASS|WARN|FAIL name evidence), final RESULT line, exit 0 only when nothing FAILed. `--json` prints one JSON object. @@ -50,22 +51,32 @@ ATX_RE = re.compile(r"^ {0,3}#{1,6}(?:[ \t]|$)") # any ATX heading _REF_TITLE = r"""(?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|\((?:[^()\\]|\\.)*\))""" LINK_REF_DEF_RE = re.compile( # a COMPLETE `[label]: destination ["title"]` line — anything else after the destination is prose - r"^ {0,3}\[(?:\\.|[^\]\\])+\]:[ \t]*(?:<[^<>]*>|[^ \t<][^ \t]*)(?:[ \t]+" + _REF_TITLE + r")?[ \t]*$") + r"^ {0,3}\[(?:\\.|[^\]\\])+\]:[ \t]*(?:<[^<>]*>|[^ \t<][^ \t]*)(?:[ \t]+(?P" + _REF_TITLE + r"))?[ \t]*$") REF_TITLE_RE = re.compile(r"^[ \t]*" + _REF_TITLE + r"[ \t]*$") # a ref def's title alone on the next line def first_visible(lines: list[str]) -> str: """The first line a reader sees as content: blank lines, block marks, link reference - definitions (and a definition's title on the following line) are skipped.""" + definitions (and, for a definition that has no title yet, its title on the following + line — a quoted line after a definition that already carries one is a paragraph) are + skipped. A continuation line (the prose after a multi-line inline comment's `-->`) + counts as visible only when it carries text.""" k = 0 while k < len(lines): ln = lines[k] + if ln.startswith(CONT_MARK): + tail = ln[len(CONT_MARK):] + if tail.strip(): + return tail + k += 1 + continue if not ln.strip() or is_mark(ln): k += 1 continue - if LINK_REF_DEF_RE.match(ln): + rd = LINK_REF_DEF_RE.match(ln) + if rd: k += 1 - if k < len(lines) and REF_TITLE_RE.match(lines[k]): + if rd.group("title") is None and k < len(lines) and REF_TITLE_RE.match(lines[k]): k += 1 continue return ln @@ -214,9 +225,6 @@ def read_lines(p: pathlib.Path) -> list[str] | None: _HTML6 = ("address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|" "fieldset|figcaption|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|" "menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul") -HTML_BLOCK_RE = re.compile( # CommonMark HTML block starts that CAN interrupt a paragraph (types 1-6); type 7 cannot - r"^ {0,3}(?:<(?:script|pre|style|textarea)(?:[ \t]|>|$)|<!--|<\?|<![A-Z]|<!\[CDATA\[|</?(?:" + _HTML6 + r")(?:[ \t]|/?>|$))", - re.IGNORECASE) HTML_MARK = "\x00html" # left by sanitize() where an HTML block other than a comment (types 1, 3-7) opened MARKS = (FENCE_MARK, COMMENT_MARK, HTML_MARK) _HTML_TYPE1 = re.compile(r"^ {0,3}<(script|pre|style|textarea)(?:[ \t]|>|$)", re.IGNORECASE) # tag whitespace is space/tab only @@ -246,7 +254,34 @@ def html_block_start(rel: str, in_paragraph: bool) -> tuple[str, str | None] | N if not in_paragraph and _HTML_TYPE7.match(rel): return HTML_MARK, None return None -INNER_STRONG_RE = re.compile(r"(?<!\\)\*\*") # an unescaped ** inside a title: not ONE strong span + + +def has_inner_strong(s: str) -> bool: + """True when `s` (a would-be title, code spans masked) holds a live `**`: one whose run of + preceding backslashes is EVEN (`\\**` is an escaped backslash and then a real delimiter, + `\**` is an escaped star). Every position is tested, so `\***` — an escaped star followed + by a live `**` — is caught too. A live `**` inside the title means it is not ONE strong span.""" + for a in range(len(s) - 1): + if s[a] == "*" and s[a + 1] == "*" and (a - len(s[:a].rstrip("\\"))) % 2 == 0: + return True + return False + + +def interrupts_paragraph(ln: str, eff: int) -> bool: + """CommonMark: can `ln` (leading tabs expanded), read inside the list item whose content + column is `eff`, interrupt a paragraph? An ATX heading, a thematic break, a block quote, + a fence, an HTML block of types 1-6 and a non-empty list item (an ordered one only when it + starts at 1) can; anything indented 4+ columns past `eff` cannot — indented code never + interrupts a paragraph, it is lazy continuation text there.""" + rel = ln[eff:] + if len(rel) - len(rel.lstrip(" ")) > 3: + return False + li = LIST_ITEM_RE.match(rel) + fm = FENCE_RE.match(rel) + return bool(THEMATIC_RE.match(rel) or ATX_RE.match(rel) or rel.lstrip(" ").startswith(">") + or (li and can_interrupt(li)) + or (fm and not (fm.group(1)[0] == "`" and "`" in fm.group(2))) + or html_block_start(rel, True)) unclosed: list[str] = [] # filled by sanitize: a fence or comment still open at end of file @@ -339,22 +374,30 @@ def fence_match(ln: str, base: int) -> re.Match | None: return FENCE_RE.match(ln[base:]) -def closes_in_paragraph(rest: list[str]) -> bool: - """True when `-->` appears in the following lines before the paragraph ends (a blank line). - CommonMark: raw HTML is inline, so an unclosed `<!--` whose closer never comes inside the - paragraph is just literal text, not a comment.""" +def closes_in_paragraph(rest: list[str], containers: list[int] = ()) -> bool: + """True when `-->` arrives on a later line of the SAME paragraph. CommonMark settles block + structure before inline HTML, so the paragraph ends at a blank line or at any line that + starts a block able to interrupt it — a heading, thematic break, block quote, fence, HTML + block, or a list item (a sibling `- Do:` included), judged against the deepest open list + item the line's indent satisfies. An opener whose closer lies beyond that is literal text.""" for nxt in rest: + nxt = expand_lead(nxt) if not nxt.strip(): return False + indent = len(nxt) - len(nxt.lstrip(" ")) + eff = next((c for c in reversed(containers) if c <= indent), 0) + if interrupts_paragraph(nxt, eff): + return False if "-->" in nxt: return True return False -def strip_inline_comments(ln: str, scan: str, rest: list[str] = ()) -> str: +def strip_inline_comments(ln: str, scan: str, rest: list[str] = (), containers: list[int] = ()) -> str: """Remove `<!-- … -->` spans from one prose line (`scan` is `ln` with code masked), carrying an unterminated comment into the module-level `_in_comment` flag when — and only when — its - closer arrives later in the same paragraph (`rest` = the lines that follow).""" + closer arrives later in the same paragraph (`rest` = the lines that follow, `containers` = + the content columns of the list items open at this line).""" global _in_comment buf: list[str] = [] i = 0 @@ -372,7 +415,7 @@ def strip_inline_comments(ln: str, scan: str, rest: list[str] = ()) -> str: if j < 0: buf.append(ln[i:]) i = len(scan) - elif "-->" not in scan[j + 4:] and not closes_in_paragraph(rest): + elif "-->" not in scan[j + 4:] and not closes_in_paragraph(rest, containers): buf.append(ln[i:]) # opener with no closer in this paragraph: literal text i = len(scan) else: @@ -446,37 +489,35 @@ def sanitize(lines: list[str]) -> list[str]: if _in_comment: j = ln.find("-->") if j < 0: - out.append("") + out.append(CONT_MARK) # wholly inside the inline comment: no text, but the paragraph is still open continue _in_comment = False ln = ln[j + 3:] # the rest of the line is prose again — but it did not BEGIN the cont = True # line, so it can never start a block: it stays in the paragraph # Track which list items are still open: a non-blank line shallower than an # item's content column closes that item (and everything nested in it). - base = containers[-1] if containers else 0 + lazy = False if ln.strip() and not cont: indent = len(ln) - len(ln.lstrip(" ")) - # A lazy continuation (paragraph text that is none of the block starts) stays inside - # the open list item however shallow it is, so it must not pop the containers. - li = LIST_ITEM_RE.match(ln) # Judge "does this line start a block?" relative to the deepest container the # line's indent still satisfies (a `<!--` at column 0 under a column-4 field is a - # top-level HTML block, not paragraph text). + # top-level HTML block, not paragraph text). A lazy continuation (paragraph text + # that is none of the block starts) stays inside the open list item however + # shallow it is, so it must not pop the containers. eff = next((c for c in reversed(containers) if c <= indent), 0) - lazy = in_paragraph and not ( - (li and not THEMATIC_RE.match(ln) and can_interrupt(li)) or THEMATIC_RE.match(ln) - or ATX_RE.match(ln) or ln.lstrip(" ").startswith(">") - or fence_match(ln, eff) or html_block_start(ln[eff:], True)) + lazy = in_paragraph and not interrupts_paragraph(ln, eff) if not lazy: open_run = 0 # a new block began: a backtick run left open in the previous paragraph was literal while containers and indent < containers[-1]: containers.pop() - if li and not THEMATIC_RE.match(ln) and (not in_paragraph or can_interrupt(li)): - containers.append(content_col(li)) + base = containers[-1] if containers else 0 + li = LIST_ITEM_RE.match(ln[base:]) # a marker 4+ columns past the container is code, not a list item + if li and len(li.group(1)) <= 3 and not THEMATIC_RE.match(ln[base:]): + containers.append(base + content_col(li)) base = containers[-1] if containers else 0 if cont: scan, open_run = mask_code(ln, open_run) - out.append(CONT_MARK + strip_inline_comments(ln, scan, lines[k + 1:])) + out.append(CONT_MARK + strip_inline_comments(ln, scan, lines[k + 1:], containers)) in_paragraph = True continue # An HTML block (`<!--`, `<pre>`, `<div>`, … first on the line, relative to the open @@ -508,8 +549,12 @@ def sanitize(lines: list[str]) -> list[str]: # Inline code spans are opaque: a <!-- inside `…` is code, not a comment opener. # Mask them (same length, harmless chars) for delimiter scanning; restore text after. scan, open_run = mask_code(ln, open_run) - prose = strip_inline_comments(ln, scan, lines[k + 1:]) - in_paragraph = bool(prose.strip()) and not ATX_RE.match(prose) and not THEMATIC_RE.match(prose) + prose = strip_inline_comments(ln, scan, lines[k + 1:], containers) + # Prose that continues the paragraph keeps it open; otherwise the line opens one only if, + # read inside its container, it is neither a heading, a thematic break nor indented code. + rel = prose[base:] + in_paragraph = lazy or (bool(prose.strip()) and len(rel) - len(rel.lstrip(" ")) <= 3 + and not ATX_RE.match(rel) and not THEMATIC_RE.match(rel)) out.append(prose) if fence_char is not None: unclosed.append(f"fenced code block opened with {fence_char * fence_len} never closes") @@ -539,16 +584,23 @@ def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: lines = sanitize(lines) def new_lesson(title: str, malformed: bool) -> dict: - return {"title": norm(title), "why": "", "do": "", "source": "", "source_raw": "", "col": 2, "last": None, "hard": False, + # col content column of the lesson item; fcol that of the open field item (None = none open) + # last the open field's key; para a paragraph is open (lazy continuation may follow); + # pcol the content column of the container that paragraph lives in (a Setext underline + # must sit there); tail lesson-level prose after the fields — visible, so compared + return {"title": norm(title), "why": "", "do": "", "source": "", "source_raw": "", "tail": "", + "col": 2, "fcol": None, "last": None, "hard": False, "para": False, "pcol": 0, "seen": {"why": False, "do": False, "source": False}, "problems": (["not-a-bold-lesson-bullet"] if malformed else [])} def absorb(lesson: dict, ln: str, para: bool = False) -> None: - """Continuation text belongs to the field above it — or to the title when no field - has started. A hard line break on the previous line is kept as HARD_BREAK, and a - blank line before this one (a new paragraph inside the item) as PARA_BREAK, so two - copies that render differently never compare equal.""" + """Prose belongs to the open field — or to the title when no field has started, or to + the lesson's tail when the fields are done. A hard line break on the previous line is + kept as HARD_BREAK, and a new paragraph inside the item as PARA_BREAK, so two copies + that render differently never compare equal.""" text = ln.strip() + if not text: + return sep = f" {PARA_BREAK} " if para else (f" {HARD_BREAK} " if lesson["hard"] else " ") if lesson["last"] is not None: key = lesson["last"] @@ -558,13 +610,26 @@ def absorb(lesson: dict, ln: str, para: bool = False) -> None: lesson[key] = norm(lesson[key] + sep + text) elif not any(lesson["seen"].values()): lesson["title"] = norm(lesson["title"] + sep + text) + else: + lesson["tail"] = norm(lesson["tail"] + sep + text) lesson["hard"] = ends_hard(ln) - prev_blank = True + def open_para(lesson: dict, col: int) -> None: + lesson["para"], lesson["pcol"] = True, col + + def close_para(lesson: dict) -> None: + lesson["para"], lesson["hard"] = False, False + top_para: str | None = None # the previous line, when it was a top-level paragraph line (a Setext underline may follow) + ref_title_next = False # the previous line was a reference definition without a title: its title may follow for idx, ln in enumerate(lines): - was_prev_blank, prev_blank = prev_blank, not ln.strip() + prev_blank = not ln.strip() prev_top_para, top_para = top_para, None + title_may_follow, ref_title_next = ref_title_next, False + if prev_blank: + if lesson is not None: + close_para(lesson) # a blank line ends any paragraph (the list item stays open) + continue # A dash underline right under a top-level paragraph line turns that line into an H2 # (Setext). It is never a day heading, so it is a bad heading that ends the section. if prev_top_para is not None and SETEXT_H2_RE.match(ln): @@ -573,16 +638,30 @@ def absorb(lesson: dict, ln: str, para: bool = False) -> None: lesson = None continue if ln.startswith(CONT_MARK): # paragraph text that began inside an inline comment - if lesson is not None and current is not None and not was_prev_blank: + if lesson is not None and current is not None and lesson["para"]: absorb(lesson, ln[len(CONT_MARK):]) continue indent = len(ln) - len(ln.lstrip(" ")) - # CommonMark: a line indented at or past the open lesson's content column is INSIDE - # that list item — even ` ## 2026-08-23` — so it can never open a day section. - if lesson is not None and current is not None and indent >= lesson["col"] and H2_RE.match(ln): - lesson["problems"].append(f"heading {ln.strip()[:30]!r} nested inside the lesson (indent {indent})") - lesson["last"] = None - continue + # CommonMark: a line indented at or past the open lesson's content column is INSIDE that + # list item. Blocks there are read relative to the innermost item the line sits in: the + # open field's content column when the line reaches it, else the lesson's. + nested = lesson is not None and current is not None and indent >= lesson["col"] + if nested: + in_field = lesson["fcol"] is not None and indent >= lesson["fcol"] + eff = lesson["fcol"] if in_field else lesson["col"] + rel = ln[eff:] + # An ATX heading inside the item — even ` ## 2026-08-23` — can never open a day section + # (and a heading interrupts any paragraph). An H2 there is a problem; other levels are + # recorded as text so copies that differ in them differ. + if ATX_RE.match(rel): + if not in_field and lesson["fcol"] is not None: + lesson["last"] = lesson["fcol"] = None + if H2_RE.match(rel): + lesson["problems"].append(f"heading {ln.strip()[:30]!r} nested inside the lesson (indent {indent})") + else: + absorb(lesson, ln, para=True) + close_para(lesson) + continue m = SECTION_RE.match(ln) if m: current = m.group(1) @@ -598,20 +677,24 @@ def absorb(lesson: dict, ln: str, para: bool = False) -> None: lesson = None continue if current is None: - # Nothing visible belongs outside a day section except the file's H1 on its first line: - # a bullet, a paragraph, a thematic break, a code block — readers see it, no day file has it. - if ln.strip() and not is_mark(ln) and not (idx == 0 and H1_LINE_RE.match(ln)) and not LINK_REF_DEF_RE.match(ln): + # Nothing visible belongs outside a day section except the file's H1 on its first line + # and link reference definitions (with a title on the next line when the definition has + # none): a bullet, a paragraph, a thematic break, a code block — readers see it, no day file has it. + if title_may_follow and REF_TITLE_RE.match(ln): + continue + rd = LINK_REF_DEF_RE.match(ln) + if rd: + ref_title_next = rd.group("title") is None + continue + if not is_mark(ln) and not (idx == 0 and H1_LINE_RE.match(ln)): misplaced.append(ln.strip()[:60]) if ln.strip() in (FENCE_MARK, HTML_MARK): # a comment renders nothing and is allowed metadata misplaced.append("<code block or raw HTML>") - if ln.strip() and not TOP_BULLET_RE.match(ln) and not INTERRUPT_RE.match(ln) and not is_mark(ln): + if not TOP_BULLET_RE.match(ln) and not INTERRUPT_RE.match(ln) and not is_mark(ln) and (indent <= 3 or prev_top_para is not None): top_para = ln continue - # CommonMark: a line indented at or past the current lesson's content column - # (marker + the whitespace after it, tabs expanded) belongs to that lesson; - # anything shallower is a sibling or a block that closes the list. - nested = lesson is not None and indent >= lesson["col"] if not nested: + # A line shallower than the lesson's content column is a sibling or a block that closes the list. if THEMATIC_RE.match(ln): # `- - -` is a thematic break, never a list item lesson = None continue @@ -624,29 +707,30 @@ def absorb(lesson: dict, ln: str, para: bool = False) -> None: and (len(m.group(2)) - len(m.group(2).rstrip("\\"))) % 2 == 0 # `\**` escapes the first closing star and m.group(2)[0] != "*" # `***foo**` → `*` + strong and not (m.group(2).endswith("*") and (len(m.group(2)) - 1 - len(m.group(2)[:-1].rstrip("\\"))) % 2 == 0) # `**foo***` → strong + `*` - and not INNER_STRONG_RE.search(CODE_SPAN_RE.sub(lambda c: "x" * len(c.group(0)), m.group(2))) # `**` in code is opaque + and not has_inner_strong(CODE_SPAN_RE.sub(lambda c: "x" * len(c.group(0)), m.group(2))) # `**` in code is opaque and gap_cols(indent + 1, m.group(1)) <= 4): lesson = new_lesson(m.group(2), malformed=False) lesson["col"] = content_col(li) lesson["hard"] = ends_hard(ln) + open_para(lesson, lesson["col"]) sections[current].append(lesson) continue - li_any = LIST_ITEM_RE.match(ln) - if TOP_BULLET_RE.match(ln) and not (lesson is not None and not was_prev_blank - and (li_any is None or not can_interrupt(li_any))): + if TOP_BULLET_RE.match(ln) and not (lesson is not None and lesson["para"] + and (li is None or not can_interrupt(li))): # (a `2.` item or an empty item directly under a paragraph cannot start a list: it # is lazy continuation text and is handled below) parts = ln.strip().split(None, 1) lesson = new_lesson("MALFORMED: " + (parts[1].strip() if len(parts) > 1 else "(empty list item)"), malformed=True) lesson["col"] = content_col(li) if li else indent + 2 lesson["hard"] = ends_hard(ln) + open_para(lesson, lesson["col"]) sections[current].append(lesson) continue - if ln.strip() and not is_mark(ln) and not INTERRUPT_RE.match(ln) and not HTML_BLOCK_RE.match(ln) and ( - lesson is None or was_prev_blank): + if not is_mark(ln) and not INTERRUPT_RE.match(ln) and (lesson is None or not lesson["para"]) and ( + indent <= 3 or prev_top_para is not None): top_para = ln # a top-level paragraph line: a `---` next would make it an H2 - if lesson is not None and ln.strip(): - if not was_prev_blank and not INTERRUPT_RE.match(ln) and not HTML_BLOCK_RE.match(ln) and not is_mark(ln): + if lesson is not None: + if lesson["para"] and not INTERRUPT_RE.match(ln) and not is_mark(ln): # Lazy continuation (CommonMark): a non-blank, non-list line directly # under a paragraph — even at indent 0 — is still that paragraph's text. absorb(lesson, ln) @@ -656,39 +740,61 @@ def absorb(lesson: dict, ln: str, para: bool = False) -> None: # fields that follow render as a NEW list, not this lesson's, so they must not reconnect. lesson = None continue - # Nested line. A child list marker is valid only at indent col .. col+3 - # (CommonMark: ≥ content column + 4 is an indented code block, not a list). + # ---- nested line: inside the open lesson item ---- if is_mark(ln): - lesson["last"] = None # a code block or comment block inside the item ends the paragraph; the lesson goes on + # A fenced code block, comment block or HTML block inside the item ends the paragraph; + # sitting beside the field items (shallower than the open field's content) it closes that item. + if not in_field and lesson["fcol"] is not None: + lesson["last"] = lesson["fcol"] = None + close_para(lesson) continue - m = FIELD_LABEL_RE.match(ln) - if m and indent >= lesson["col"] + 4: - lesson["problems"].append(f"field '{m.group(3)}' indented {indent} spaces renders as code, not a nested bullet") - lesson["last"] = None + # A dash underline directly under a paragraph inside the item, at that paragraph's + # container column, makes it a Setext H2 (CommonMark) — a heading hidden in a lesson. + if lesson["para"] and indent >= lesson["pcol"] and SETEXT_H2_RE.match(ln[lesson["pcol"]:]): + lesson["problems"].append(f"Setext H2 (underline {ln.strip()[:12]!r}) nested inside the lesson") + close_para(lesson) continue - if m and gap_cols(indent + 1, m.group(2)) > 4: - lesson["problems"].append(f"field '{m.group(3)}' has {gap_cols(indent + 1, m.group(2))} columns after `-`: its text renders as code, not a field") - lesson["last"] = None + if lesson["para"] and not interrupts_paragraph(ln, eff): + absorb(lesson, ln) # lazy continuation of the open paragraph, wherever the line sits continue - if not m: - # continuation text (indented) belongs to the field above it — or to the title; - # after a blank line it is a NEW paragraph of that field, which is recorded - if ln.strip(): - absorb(lesson, ln, para=was_prev_blank) + if not in_field and lesson["fcol"] is not None: + lesson["last"] = lesson["fcol"] = None # shallower than the open field's content: that item is closed + rel_indent = len(rel) - len(rel.lstrip(" ")) + if rel_indent >= 4: + close_para(lesson) # an indented code block inside the item: code, never prose continue - label, value = m.group(3), m.group(4) - key = {"Why it matters here": "why", "Do": "do", "Source": "source"}[label] - lesson["last"] = key - lesson["hard"] = ends_hard(ln) - if lesson["seen"][key]: - lesson["problems"].append(f"duplicate {key}") - lesson["seen"][key] = True - # The value may continue on the next line(s); emptiness and the source shape are - # judged once the whole field has been assembled (see the finalisation loop). - if key == "source": - lesson["source_raw"] = norm(value) - else: - lesson[key] = norm(value) + fm = FIELD_LABEL_RE.match(ln) + if fm and not in_field: + gap = gap_cols(indent + 1, fm.group(2)) + if gap > 4: + lesson["problems"].append(f"field '{fm.group(3)}' has {gap} columns after `-`: its text renders as code, not a field") + lesson["last"], lesson["fcol"] = None, indent + 2 + close_para(lesson) + continue + label, value = fm.group(3), fm.group(4) + key = {"Why it matters here": "why", "Do": "do", "Source": "source"}[label] + lesson["last"], lesson["fcol"] = key, indent + 1 + gap + open_para(lesson, lesson["fcol"]) + lesson["hard"] = ends_hard(ln) + if lesson["seen"][key]: + lesson["problems"].append(f"duplicate {key}") + lesson["seen"][key] = True + # The value may continue on the next line(s); emptiness and the source shape are + # judged once the whole field has been assembled (see the finalisation loop). + if key == "source": + lesson["source_raw"] = norm(value) + else: + lesson[key] = norm(value) + continue + if THEMATIC_RE.match(rel): # a thematic break inside the item: a block, not text + close_para(lesson) + continue + # Anything else opens a new paragraph inside the item: a list item that is not a lesson + # field (a sub-bullet, an ordered item, a `- Do:` nested inside another field), a block + # quote, or plain text after a blank line, a block or code. Its text is recorded. + li = LIST_ITEM_RE.match(rel) + absorb(lesson, ln, para=True) + open_para(lesson, eff + content_col(li) if li else eff) for ls in sections.values(): for l in ls: @@ -715,8 +821,11 @@ def shape_errors(sections: dict[str, list[dict]]) -> list[str]: return [f"{d}#{i}: {p}" for d, ls in sections.items() for i, l in enumerate(ls, 1) for p in l["problems"]] -def lesson_key(l: dict) -> tuple[str, str, str, str]: - return (l["title"], l["why"], l["do"], l["source"]) +KEY_FIELDS = ("title",) + FIELDS + ("tail",) # what two copies of a lesson are compared on + + +def lesson_key(l: dict) -> tuple[str, str, str, str, str]: + return (l["title"], l["why"], l["do"], l["source"], l["tail"]) # --- knowledge.md ---------------------------------------------------------- @@ -728,7 +837,7 @@ def lesson_key(l: dict) -> tuple[str, str, str, str]: fail("knowledge.md", "missing") else: lines = read_lines(kpath) or [] - if lines and lines[0].rstrip() == KNOWLEDGE_H1: + if lines and lines[0] == KNOWLEDGE_H1: # exact: trailing whitespace would break the connector's anchor too ok("knowledge:h1", "exact connector heading present") else: fail("knowledge:h1", f"first line must be {KNOWLEDGE_H1!r}, got {(lines[0] if lines else '')[:70]!r}") @@ -835,7 +944,7 @@ def lesson_key(l: dict) -> tuple[str, str, str, str]: diff_fields: list[str] = [] for a, b in zip(keys_k, keys_d): if a != b and a[0] == b[0]: - diff_fields.append(a[0][:30] + ": " + ",".join(f for f, x, y in zip(("title",) + FIELDS, a, b) if x != y)) + diff_fields.append(a[0][:30] + ": " + ",".join(f for f, x, y in zip(KEY_FIELDS, a, b) if x != y)) detail = f"{len(keys_d)} lesson(s) here vs {len(keys_k)} in knowledge.md" if diff_fields: detail += f"; same title, different content: {diff_fields[:3]}" From e7edf49a73507ecd0a66dd9cf937efb8620606d5 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sun, 27 Sep 2026 12:13:57 -0400 Subject: [PATCH 25/29] harness: round-23 review fixes (CommonMark whitespace, ref-def destinations, code-span parity, visible nested blocks, standalone confidence, block-quoted headings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex findings on 320a5cc, each reproduced as a failing case first (neg23.sh): - whitespace is space/tab only: SECTION_RE, DAY_H1_RE, LESSON_RE, TOP_BULLET_RE, LIST_ITEM_RE, H1_LINE_RE no longer use \s/\S; blank() replaces strip() in structural tests; norm() collapses only spaces/tabs (a NBSP after `## 2026-08-24` is content and FAILs; a NBSP-only line is prose) - link_ref_def(): a real parser (label without unescaped brackets, `<…>` or a bare destination with balanced/escaped parentheses, optional title, nothing else) replaces LINK_REF_DEF_RE; parse_source() shares scan_destination() - code_spans()/backtick_runs(): line-local, parity-aware span pairing replaces CODE_SPAN_RE in norm() and the title check (`- **a \\`**` b**` is one strong span) - fence/HTML marks carry the block's visible text as a payload; nested blocks join the lesson comparison as ⟨code⟩/⟨html⟩ tokens (a `<div>` present in one copy only now differs) - CONFIDENCE_RE requires a standalone label (`overconfidence 80%` is not one) - quote_depth()/unquote(): ATX and Setext H2s inside block quotes are headings — top level, nested in a lesson, and in the per-day stray-H2 check Found via the `<div>` case and confirmed with commonmark.js 0.31.2: a fence or HTML block inside a list item ends when the item ends (a shallower non-blank line), so a sibling `- Do:` after an unclosed nested fence is a field, not swallowed code. Two earlier suite cases (neg13 5 "code is invisible", neg21 3 "NBSP closer hides Do") encoded the old readings; rewritten and proven discriminating by mutation. Evidence: 13 suites, 148 cases, 0 failing; check-knowledge.py RESULT: PASS — 12 pass, 0 warn, 0 fail; markdownlint-cli2 0.23.3: 0 issues. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 347 ++++++++++++++++++++++++++++--------- 2 files changed, 264 insertions(+), 85 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index ba702ff..f8654eb 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -10,6 +10,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | | 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | correction: `verified/2026-09-26_claude-surfaces_v2.md` (session ran in the chat/Cowork tab, not the Code tab — T's correction 2026-09-27); v1 banner-marked superseded | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 101 Codex findings (22 rounds; round 22 also rewrote 3 earlier suite cases that CommonMark, checked against commonmark.js 0.31.2, had the other way) | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 107 Codex findings (23 rounds; rounds 22-23 also rewrote 5 earlier suite cases that CommonMark, checked against commonmark.js 0.31.2, had the other way) | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index b8445ec..18ceef7 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -38,11 +38,13 @@ KNOWLEDGE_H1 = "# daily-dev-agentic knowledge — T agent" # the connector depends on this exact line -CODE_SPAN_RE = re.compile(r"(?<!\\)(`+)(?!`)(?:.+?)(?<!`)\1(?!`)") # CommonMark-ish: matching, unescaped backtick runs +# CommonMark whitespace is space and tab only: a NBSP (U+00A0) or any other Unicode space is +# CONTENT — `## 2026-08-24<NBSP>` is not the date heading, a NBSP-only line is not blank, `-<NBSP>x` +# is not a list item. Structural checks therefore never use `\s`, `\S` or bare `.strip()`. DATE_RE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$") -SECTION_RE = re.compile(r"^ {0,3}## ([0-9]{4}-[0-9]{2}-[0-9]{2})\s*$") -LESSON_RE = re.compile(r"^ {0,3}-([ \t]+)\*\*(.+?)\*\*\s*$") # bold lesson bullet: `-`, 1-4 columns of whitespace (checked after), **title** -TOP_BULLET_RE = re.compile(r"^ {0,3}(?:[-*+]|[0-9]{1,9}[.)])(?:[ \t]+\S|\s*$)") # any top-level list item incl. an EMPTY one; ordered markers are 1-9 digits (CommonMark) +SECTION_RE = re.compile(r"^ {0,3}## ([0-9]{4}-[0-9]{2}-[0-9]{2})[ \t]*$") +LESSON_RE = re.compile(r"^ {0,3}-([ \t]+)\*\*(.+?)\*\*[ \t]*$") # bold lesson bullet: `-`, 1-4 columns of whitespace (checked after), **title** +TOP_BULLET_RE = re.compile(r"^ {0,3}(?:[-*+]|[0-9]{1,9}[.)])(?:[ \t]+[^ \t]|[ \t]*$)") # any top-level list item incl. an EMPTY one; ordered markers are 1-9 digits (CommonMark) FIELD_LABEL_RE = re.compile(r"^( *)-([ \t]+)(Why it matters here|Do|Source):(.*)$") # label first; 1-4 columns after `-` (5+ = code), nesting checked by indent # Blocks that interrupt a paragraph (CommonMark): an ATX heading, a block quote, a thematic break. # A line like this at top level is never lazy continuation of the lesson above it. @@ -50,9 +52,69 @@ INTERRUPT_RE = re.compile(r"^ {0,3}(?:#{1,6}(?:[ \t]|$)|>|" + THEMATIC_BREAK + ")") ATX_RE = re.compile(r"^ {0,3}#{1,6}(?:[ \t]|$)") # any ATX heading _REF_TITLE = r"""(?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|\((?:[^()\\]|\\.)*\))""" -LINK_REF_DEF_RE = re.compile( # a COMPLETE `[label]: destination ["title"]` line — anything else after the destination is prose - r"^ {0,3}\[(?:\\.|[^\]\\])+\]:[ \t]*(?:<[^<>]*>|[^ \t<][^ \t]*)(?:[ \t]+(?P<title>" + _REF_TITLE + r"))?[ \t]*$") REF_TITLE_RE = re.compile(r"^[ \t]*" + _REF_TITLE + r"[ \t]*$") # a ref def's title alone on the next line +_REF_LABEL_RE = re.compile(r"^ {0,3}\[((?:\\.|[^\[\]\\])+)\]:[ \t]*") # `[label]:` — no unescaped brackets inside the label +_REF_TITLE_TAIL_RE = re.compile(r"^(?:[ \t]+(?P<title>" + _REF_TITLE + r"))?[ \t]*$") + + +def indent_of(ln: str) -> int: + return len(ln) - len(ln.lstrip(" ")) + + +def blank(ln: str) -> bool: + """A blank line the CommonMark way: nothing but spaces and tabs (a NBSP-only line is a paragraph).""" + return ln.strip(" \t") == "" + + +def scan_destination(text: str, i: int) -> int | None: + """Index just past a bare link destination starting at `text[i]`, or None when there is none. + + CommonMark: no spaces or ASCII control characters, no unescaped `<` or `>`, and unescaped + parentheses must balance — `https://x.y/(oops` never closes, so it is not a destination. + """ + depth = 0 + while i < len(text): + c = text[i] + if c == " " or c == "\t" or ord(c) < 32 or c == "\x7f" or c in "<>": + break + if c == "\\" and i + 1 < len(text) and text[i + 1] in ASCII_PUNCT: + i += 2 # `\(` is destination text, not a delimiter + continue + if c == "(": + depth += 1 + elif c == ")": + if depth == 0: + break + depth -= 1 + i += 1 + if i < len(text) and text[i] in "<>": + return None + return i if depth == 0 else None + + +def link_ref_def(ln: str) -> bool | None: + """None unless `ln` is a COMPLETE link reference definition `[label]: destination ["title"]`; + otherwise True when it carries a title, False when a title may still follow on the next line. + The label needs a non-whitespace character and no unescaped brackets; the destination is + `<…>` (no `<`, `>` inside) or a bare one with balanced parentheses; anything else after it + makes the line prose.""" + lm = _REF_LABEL_RE.match(ln) + if not lm or not lm.group(1).strip(" \t"): + return None + i = lm.end() + if i < len(ln) and ln[i] == "<": + j = ln.find(">", i + 1) + if j < 0 or "<" in ln[i + 1:j]: + return None + end = j + 1 + else: + end = scan_destination(ln, i) + if end is None or end == i: + return None + tm = _REF_TITLE_TAIL_RE.match(ln[end:]) + if not tm: + return None + return tm.group("title") is not None def first_visible(lines: list[str]) -> str: @@ -66,24 +128,24 @@ def first_visible(lines: list[str]) -> str: ln = lines[k] if ln.startswith(CONT_MARK): tail = ln[len(CONT_MARK):] - if tail.strip(): + if not blank(tail): return tail k += 1 continue - if not ln.strip() or is_mark(ln): + if blank(ln) or is_mark(ln): k += 1 continue - rd = LINK_REF_DEF_RE.match(ln) - if rd: + rd = link_ref_def(ln) + if rd is not None: k += 1 - if rd.group("title") is None and k < len(lines) and REF_TITLE_RE.match(lines[k]): + if rd is False and k < len(lines) and REF_TITLE_RE.match(lines[k]): k += 1 continue return ln return "" LINK_LABEL = r"\[(?:\\.|[^\]\\])+\]" # allows escaped \] inside the label LINK_OPEN_RE = re.compile(r"^" + LINK_LABEL + r"\(") # `[label](` — the destination is scanned by hand -CONFIDENCE_RE = re.compile(r"confidence ([0-9]{1,3})%") +CONFIDENCE_RE = re.compile(r"(?<![A-Za-z0-9_])confidence ([0-9]{1,3})%(?![0-9%])") # a standalone label: `overconfidence 80%` is not one ASCII_PUNCT = set(r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""") # a backslash escapes exactly these (CommonMark) @@ -93,7 +155,7 @@ def parse_source(text: str) -> tuple[str, int] | None: The destination is read the CommonMark way: no whitespace, and parentheses must balance — `https://x.y/(broken)` never closes, so it is not a link at all. """ - text = text.strip() + text = text.strip(" \t") if not text.startswith("["): return None j, bdepth = 1, 0 # the label: balanced, unescaped brackets are allowed inside @@ -111,31 +173,17 @@ def parse_source(text: str) -> tuple[str, int] | None: j += 1 if j >= len(text) or j == 1 or j + 1 >= len(text) or text[j + 1] != "(": return None - i, depth = j + 2, 0 - while i < len(text): - c = text[i] - if c.isspace() or c in "<>": - return None # a bare destination has no whitespace and no unescaped < > - if c == "\\" and i + 1 < len(text) and text[i + 1] in ASCII_PUNCT: - i += 2 # `\(` is destination text, not a delimiter - continue - if c == "(": - depth += 1 - elif c == ")": - if depth == 0: - break - depth -= 1 - i += 1 - else: - return None # ran off the end without closing the destination + i = scan_destination(text, j + 2) + if i is None or i >= len(text) or text[i] != ")": + return None # no whitespace, no unescaped < >, parentheses balance, `)` closes dest = text[j + 2:i] - if not re.match(r"https?://\S+$", dest): + if not re.match(r"https?://[^ \t]+$", dest): return None cm = CONFIDENCE_RE.search(text[i + 1:]) return (dest, int(cm.group(1))) if cm else None -DAY_H1_RE = re.compile(r"^# Lessons — ([0-9]{4}-[0-9]{2}-[0-9]{2})\s*$") +DAY_H1_RE = re.compile(r"^# Lessons — ([0-9]{4}-[0-9]{2}-[0-9]{2})[ \t]*$") VERIFIED_NAME_RE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[a-z0-9-]+_v[0-9]+\.md$") -H1_LINE_RE = re.compile(r"^ {0,3}#[ \t]+\S") # a real ATX H1 with text: ≤3 leading spaces (4 = code), space or tab after `#` +H1_LINE_RE = re.compile(r"^ {0,3}#[ \t]+[^ \t]") # a real ATX H1 with text: ≤3 leading spaces (4 = code), space or tab after `#` FIELDS = ("why", "do", "source") H2_RE = re.compile(r"^ {0,3}##(?:[ \t]|$)") # any ATX H2 (≤3-space indent), including a bare `##`; NBSP is not a separator bad_headings: list[str] = [] # filled by parse_lessons: H2s that are not valid day headings @@ -169,10 +217,18 @@ def valid_date(s: str) -> bool: def norm(s: str) -> str: - """Collapse prose whitespace, but keep inline code verbatim (`printf 'a b'` means two spaces).""" + """Collapse runs of spaces/tabs in prose (a NBSP is content and stays), but keep inline code + verbatim (`printf 'a b'` means two spaces).""" spans: list[str] = [] - masked = CODE_SPAN_RE.sub(lambda m: (spans.append(m.group(0)), f"\x02{len(spans) - 1}\x02")[1], s) - out = " ".join(masked.split()) + parts: list[str] = [] + i = 0 + for a, b in code_spans(s): + parts.append(s[i:a]) + spans.append(s[a:b]) + parts.append(f"\x02{len(spans) - 1}\x02") + i = b + parts.append(s[i:]) + out = " ".join(t for t in re.split(r"[ \t]+", "".join(parts).strip(" \t")) if t) return re.sub(r"\x02(\d+)\x02", lambda m: spans[int(m.group(1))], out) @@ -220,13 +276,15 @@ def read_lines(p: pathlib.Path) -> list[str] | None: FENCE_MARK = "\x00fence" # left by sanitize() where a fenced block opened, at its indent: a block boundary, never text COMMENT_MARK = "\x00comment" # same, where an HTML comment BLOCK (`<!--` at line start) opened CONT_MARK = "\x00cont" # prefix: this line is paragraph continuation text (it began inside an inline comment) -LIST_ITEM_RE = re.compile(r"^( *)([-*+]|[0-9]{1,9}[.)])([ \t]+)\S") # marker + whitespace + content (for content-column tracking) +LIST_ITEM_RE = re.compile(r"^( *)([-*+]|[0-9]{1,9}[.)])([ \t]+)[^ \t]") # marker + whitespace + content (for content-column tracking) THEMATIC_RE = re.compile(r"^ {0,3}" + THEMATIC_BREAK) # `---`, `- - -`, `***`: a thematic break outranks a list item _HTML6 = ("address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|" "fieldset|figcaption|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|" "menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul") HTML_MARK = "\x00html" # left by sanitize() where an HTML block other than a comment (types 1, 3-7) opened MARKS = (FENCE_MARK, COMMENT_MARK, HTML_MARK) +PAYLOAD_SEP = "\x01" # a fence/HTML mark may carry the block's text after this: `<indent>MARK\x01line⏎line` +BLOCK_SEP = "⏎" _HTML_TYPE1 = re.compile(r"^ {0,3}<(script|pre|style|textarea)(?:[ \t]|>|$)", re.IGNORECASE) # tag whitespace is space/tab only _HTML_TYPE7 = re.compile( # a complete open or close tag alone on the line (cannot interrupt a paragraph) r"^ {0,3}(?:<[A-Za-z][A-Za-z0-9-]*(?:[ \t]+[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \t]*=[ \t]*(?:[^ \t\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)*[ \t]*/?>" @@ -285,8 +343,46 @@ def interrupts_paragraph(ln: str, eff: int) -> bool: unclosed: list[str] = [] # filled by sanitize: a fence or comment still open at end of file +QUOTE_RE = re.compile(r"^ {0,3}> ?") + + +def quote_depth(ln: str) -> tuple[int, str]: + """(number of leading block-quote markers, the text inside them) — CommonMark parses the + quoted text as blocks, so `> ## x` holds a real H2 and `> Foo` / `> ---` a Setext one.""" + depth = 0 + while True: + m = QUOTE_RE.match(ln) + if not m: + return depth, ln + depth += 1 + ln = ln[m.end():] + + +def unquote(ln: str) -> str: + return quote_depth(ln)[1] + + +def is_h2(ln: str) -> bool: + """An ATX H2 on this line, inside block quotes or not.""" + return bool(H2_RE.match(unquote(ln))) + + +def mark_kind(ln: str) -> str | None: + """Which block mark this sanitized line is (its payload ignored), or None for ordinary text.""" + head = ln.lstrip(" ").split(PAYLOAD_SEP, 1)[0] + return head if head in MARKS else None + + +def mark_token(ln: str) -> str: + """A comparison token for a fence/HTML mark: what readers see of the block, spaces made visible + so prose normalisation cannot collapse them (`⟨code⟩print(1)`, `⟨html⟩<div>x</div>`).""" + kind = mark_kind(ln) + payload = ln.split(PAYLOAD_SEP, 1)[1] if PAYLOAD_SEP in ln else "" + return ("⟨code⟩" if kind == FENCE_MARK else "⟨html⟩") + payload.replace(" ", "␣").replace("\t", "⇥") + + def is_mark(ln: str) -> bool: - return ln.strip() in MARKS + return mark_kind(ln) is not None def expand_lead(ln: str) -> str: @@ -326,6 +422,46 @@ def content_col(li: re.Match) -> int: BACKTICK_RUN_RE = re.compile(r"`+") +def backtick_runs(ln: str) -> list[tuple[int, int]]: + """(start, end) of every backtick run that can delimit code. A backtick preceded by an ODD + number of backslashes is an escaped literal and drops out of its run; an even number + (`\\\\``) is an escaped backslash followed by a live backtick.""" + runs: list[tuple[int, int]] = [] + for m in BACKTICK_RUN_RE.finditer(ln): + a, b = m.start(), m.end() + if (a - len(ln[:a].rstrip("\\"))) % 2 == 1: + a += 1 + if a < b: + runs.append((a, b)) + return runs + + +def code_spans(s: str) -> list[tuple[int, int]]: + """Extents (delimiters included) of the code spans that open AND close within `s`, paired the + CommonMark way: a run pairs with the next run of the same length; a run with no partner is + literal text and scanning continues after it.""" + runs = backtick_runs(s) + out: list[tuple[int, int]] = [] + k = 0 + while k < len(runs): + a, b = runs[k] + j = next((idx for idx in range(k + 1, len(runs)) if runs[idx][1] - runs[idx][0] == b - a), None) + if j is None: + k += 1 + continue + out.append((a, runs[j][1])) + k = j + 1 + return out + + +def mask_spans(s: str) -> str: + """`s` with each closed code span replaced by `x`s of the same length (delimiters included).""" + out = list(s) + for a, b in code_spans(s): + out[a:b] = "x" * (b - a) + return "".join(out) + + def mask_code(ln: str, open_run: int) -> tuple[str, int]: """Mask inline code with backticks for delimiter scanning; return (masked, open_run_after). @@ -334,13 +470,7 @@ def mask_code(ln: str, open_run: int) -> tuple[str, int]: so `<!--` on the next line inside that span is code, not a comment. `open_run` is the length of such a run carried over from the previous line (0 = none). """ - runs: list[tuple[int, int]] = [] - for m in BACKTICK_RUN_RE.finditer(ln): - a, b = m.start(), m.end() - if (a - len(ln[:a].rstrip("\\"))) % 2 == 1: # `\`` — the first backtick is a literal, not part of the run - a += 1 - if a < b: - runs.append((a, b)) + runs = backtick_runs(ln) out: list[str] = [] i = k = 0 if open_run: @@ -382,7 +512,7 @@ def closes_in_paragraph(rest: list[str], containers: list[int] = ()) -> bool: item the line's indent satisfies. An opener whose closer lies beyond that is literal text.""" for nxt in rest: nxt = expand_lead(nxt) - if not nxt.strip(): + if blank(nxt): return False indent = len(nxt) - len(nxt.lstrip(" ")) eff = next((c for c in reversed(containers) if c <= indent), 0) @@ -462,27 +592,55 @@ def sanitize(lines: list[str]) -> list[str]: in_paragraph = False # the previous line was prose (a type-7 HTML block cannot start here) open_run = 0 # length of an inline-code backtick run still open from the previous line containers: list[int] = [] # content columns of the open list items, innermost last + block_at = -1 # index in `out` of the open fence/HTML block's mark line + block_base = 0 # content column of the list item that block lives in + block_text: list[str] = [] # the block's lines, container indentation removed (its payload) unclosed.clear() + + def close_block() -> None: + nonlocal block_at + if block_at >= 0: + out[block_at] += PAYLOAD_SEP + BLOCK_SEP.join(block_text) + block_at = -1 + block_text.clear() + for k, ln in enumerate(lines): ln = expand_lead(ln) - if not ln.strip(): + indent = len(ln) - len(ln.lstrip(" ")) + if blank(ln): open_run = 0 # a code span cannot cross a blank line (the paragraph ends) _in_comment = False # nor can an inline comment: an unclosed `<!--` was literal text in_paragraph = False + # A fence or HTML block inside a list item ends with that item (CommonMark: the block runs + # "until the end of the containing block"): a non-blank line shallower than the item's + # content column closes both, and is then read as an ordinary line. + if (fence_char is not None or in_html) and not blank(ln) and indent < block_base: + fence_char = None + in_html = False + close_block() if fence_char is not None: m = fence_match(ln, fence_base) if m and m.group(1)[0] == fence_char and len(m.group(1)) >= fence_len and m.group(2).strip(" \t") == "": # NBSP etc. is content fence_char = None + close_block() # the closing fence is not content + else: + block_text.append(ln[min(indent, block_base):]) out.append("") continue if in_html: # Types 1-5 end WITH the line holding their terminator; types 6-7 end at a blank # line, which is itself not part of the block. if html_end is None: - if not ln.strip(): + if blank(ln): in_html = False - elif html_end in ln.lower(): # `</PRE>` closes a `<pre>` block too - in_html = False + close_block() + else: + block_text.append(ln[min(indent, block_base):]) + else: + block_text.append(ln[min(indent, block_base):]) + if html_end in ln.lower(): # `</PRE>` closes a `<pre>` block too + in_html = False + close_block() out.append("") continue cont = False @@ -497,8 +655,7 @@ def sanitize(lines: list[str]) -> list[str]: # Track which list items are still open: a non-blank line shallower than an # item's content column closes that item (and everything nested in it). lazy = False - if ln.strip() and not cont: - indent = len(ln) - len(ln.lstrip(" ")) + if not blank(ln) and not cont: # Judge "does this line start a block?" relative to the deepest container the # line's indent still satisfies (a `<!--` at column 0 under a column-4 field is a # top-level HTML block, not paragraph text). A lazy continuation (paragraph text @@ -523,10 +680,10 @@ def sanitize(lines: list[str]) -> list[str]: # An HTML block (`<!--`, `<pre>`, `<div>`, … first on the line, relative to the open # item) is a block like a fence: it leaves a boundary mark and its lines are raw HTML, # never Markdown — a `## date` or `- Do:` inside it is not a heading or a field. - hb = html_block_start(ln[base:], in_paragraph) if len(ln) - len(ln.lstrip(" ")) >= base else None + hb = html_block_start(ln[base:], in_paragraph) if indent >= base else None if hb: mark, html_end = hb - out.append(" " * (len(ln) - len(ln.lstrip(" "))) + mark) + out.append(" " * indent + mark) open_run = 0 in_paragraph = False rel = ln[base:].lstrip(" ") @@ -536,6 +693,11 @@ def sanitize(lines: list[str]) -> list[str]: in_html = ">" not in rel[2:] # `<!X … >` may close on its own line else: in_html = html_end not in rel.lower()[len(html_end) - 1:] # e.g. `<!-- x -->` closes on its own line + if mark == HTML_MARK: # a comment renders nothing: no payload to compare + block_at, block_base = len(out) - 1, base + block_text.append(ln[base:]) + if not in_html: + close_block() continue m = fence_match(ln, base) # CommonMark: a backtick fence cannot open when its info string contains a @@ -544,7 +706,9 @@ def sanitize(lines: list[str]) -> list[str]: fence_char, fence_len, fence_base = m.group(1)[0], len(m.group(1)), base open_run = 0 in_paragraph = False - out.append(" " * (len(ln) - len(ln.lstrip(" "))) + FENCE_MARK) + out.append(" " * indent + FENCE_MARK) + block_at, block_base = len(out) - 1, base + block_text.append(m.group(2).strip(" \t")) # what readers see of the opener: its info string continue # Inline code spans are opaque: a <!-- inside `…` is code, not a comment opener. # Mask them (same length, harmless chars) for delimiter scanning; restore text after. @@ -553,15 +717,16 @@ def sanitize(lines: list[str]) -> list[str]: # Prose that continues the paragraph keeps it open; otherwise the line opens one only if, # read inside its container, it is neither a heading, a thematic break nor indented code. rel = prose[base:] - in_paragraph = lazy or (bool(prose.strip()) and len(rel) - len(rel.lstrip(" ")) <= 3 + in_paragraph = lazy or (not blank(prose) and len(rel) - len(rel.lstrip(" ")) <= 3 and not ATX_RE.match(rel) and not THEMATIC_RE.match(rel)) out.append(prose) - if fence_char is not None: + if fence_char is not None: # implicitly closed at end of file, but a hand editor forgot the closer unclosed.append(f"fenced code block opened with {fence_char * fence_len} never closes") if _in_comment or (in_html and html_end == "-->"): unclosed.append("HTML comment never closes") elif in_html and html_end is not None: unclosed.append(f"HTML block never closes (no {html_end!r})") + close_block() # a block still open at end of file keeps its text return out @@ -589,7 +754,7 @@ def new_lesson(title: str, malformed: bool) -> dict: # pcol the content column of the container that paragraph lives in (a Setext underline # must sit there); tail lesson-level prose after the fields — visible, so compared return {"title": norm(title), "why": "", "do": "", "source": "", "source_raw": "", "tail": "", - "col": 2, "fcol": None, "last": None, "hard": False, "para": False, "pcol": 0, + "col": 2, "fcol": None, "last": None, "hard": False, "para": False, "pcol": 0, "pq": 0, "seen": {"why": False, "do": False, "source": False}, "problems": (["not-a-bold-lesson-bullet"] if malformed else [])} @@ -598,7 +763,7 @@ def absorb(lesson: dict, ln: str, para: bool = False) -> None: the lesson's tail when the fields are done. A hard line break on the previous line is kept as HARD_BREAK, and a new paragraph inside the item as PARA_BREAK, so two copies that render differently never compare equal.""" - text = ln.strip() + text = ln.strip(" \t") if not text: return sep = f" {PARA_BREAK} " if para else (f" {HARD_BREAK} " if lesson["hard"] else " ") @@ -614,17 +779,19 @@ def absorb(lesson: dict, ln: str, para: bool = False) -> None: lesson["tail"] = norm(lesson["tail"] + sep + text) lesson["hard"] = ends_hard(ln) - def open_para(lesson: dict, col: int) -> None: - lesson["para"], lesson["pcol"] = True, col + def open_para(lesson: dict, col: int, qdepth: int = 0) -> None: + lesson["para"], lesson["pcol"], lesson["pq"] = True, col, qdepth def close_para(lesson: dict) -> None: lesson["para"], lesson["hard"] = False, False top_para: str | None = None # the previous line, when it was a top-level paragraph line (a Setext underline may follow) + quote_para: tuple[int, str] | None = None # same, for a paragraph line inside a top-level block quote (depth, text) ref_title_next = False # the previous line was a reference definition without a title: its title may follow for idx, ln in enumerate(lines): - prev_blank = not ln.strip() + prev_blank = blank(ln) prev_top_para, top_para = top_para, None + prev_quote_para, quote_para = quote_para, None title_may_follow, ref_title_next = ref_title_next, False if prev_blank: if lesson is not None: @@ -632,11 +799,18 @@ def close_para(lesson: dict) -> None: continue # A dash underline right under a top-level paragraph line turns that line into an H2 # (Setext). It is never a day heading, so it is a bad heading that ends the section. - if prev_top_para is not None and SETEXT_H2_RE.match(ln): - bad_headings.append(prev_top_para.strip()[:60] + " / " + ln.strip()[:20]) + # The same holds inside a block quote when both lines sit at the same quote depth. + qd, qtext = quote_depth(ln) + if (prev_top_para is not None and SETEXT_H2_RE.match(ln)) or ( + prev_quote_para is not None and qd == prev_quote_para[0] and SETEXT_H2_RE.match(qtext)): + para_text = prev_top_para if prev_top_para is not None else prev_quote_para[1] + bad_headings.append(para_text.strip()[:60] + " / " + ln.strip()[:20]) current = None lesson = None continue + if qd and not blank(qtext) and not INTERRUPT_RE.match(qtext) and not TOP_BULLET_RE.match(qtext) and not FENCE_RE.match(qtext) \ + and (lesson is None or indent_of(ln) < lesson["col"]): + quote_para = (qd, qtext) # a paragraph line inside a top-level block quote if ln.startswith(CONT_MARK): # paragraph text that began inside an inline comment if lesson is not None and current is not None and lesson["para"]: absorb(lesson, ln[len(CONT_MARK):]) @@ -650,13 +824,13 @@ def close_para(lesson: dict) -> None: in_field = lesson["fcol"] is not None and indent >= lesson["fcol"] eff = lesson["fcol"] if in_field else lesson["col"] rel = ln[eff:] - # An ATX heading inside the item — even ` ## 2026-08-23` — can never open a day section - # (and a heading interrupts any paragraph). An H2 there is a problem; other levels are - # recorded as text so copies that differ in them differ. - if ATX_RE.match(rel): + # An ATX heading inside the item — even ` ## 2026-08-23`, or ` > ## x` inside a block + # quote — can never open a day section (and a heading interrupts any paragraph). An H2 + # there is a problem; other levels are recorded as text so copies that differ in them differ. + if ATX_RE.match(unquote(rel)): if not in_field and lesson["fcol"] is not None: lesson["last"] = lesson["fcol"] = None - if H2_RE.match(rel): + if is_h2(rel): lesson["problems"].append(f"heading {ln.strip()[:30]!r} nested inside the lesson (indent {indent})") else: absorb(lesson, ln, para=True) @@ -671,8 +845,8 @@ def close_para(lesson: dict) -> None: sections[current] = [] lesson = None continue - if H2_RE.match(ln): # an H2 that is not `## YYYY-MM-DD` - bad_headings.append(ln.strip()) + if is_h2(ln): # an H2 that is not `## YYYY-MM-DD` (block-quoted ones included) + bad_headings.append(ln.strip(" \t")) # keep a trailing NBSP visible in the report current = None # nothing after it belongs to a day lesson = None continue @@ -682,13 +856,13 @@ def close_para(lesson: dict) -> None: # none): a bullet, a paragraph, a thematic break, a code block — readers see it, no day file has it. if title_may_follow and REF_TITLE_RE.match(ln): continue - rd = LINK_REF_DEF_RE.match(ln) - if rd: - ref_title_next = rd.group("title") is None + rd = link_ref_def(ln) + if rd is not None: + ref_title_next = rd is False continue if not is_mark(ln) and not (idx == 0 and H1_LINE_RE.match(ln)): misplaced.append(ln.strip()[:60]) - if ln.strip() in (FENCE_MARK, HTML_MARK): # a comment renders nothing and is allowed metadata + if mark_kind(ln) in (FENCE_MARK, HTML_MARK): # a comment renders nothing and is allowed metadata misplaced.append("<code block or raw HTML>") if not TOP_BULLET_RE.match(ln) and not INTERRUPT_RE.match(ln) and not is_mark(ln) and (indent <= 3 or prev_top_para is not None): top_para = ln @@ -707,7 +881,7 @@ def close_para(lesson: dict) -> None: and (len(m.group(2)) - len(m.group(2).rstrip("\\"))) % 2 == 0 # `\**` escapes the first closing star and m.group(2)[0] != "*" # `***foo**` → `*` + strong and not (m.group(2).endswith("*") and (len(m.group(2)) - 1 - len(m.group(2)[:-1].rstrip("\\"))) % 2 == 0) # `**foo***` → strong + `*` - and not has_inner_strong(CODE_SPAN_RE.sub(lambda c: "x" * len(c.group(0)), m.group(2))) # `**` in code is opaque + and not has_inner_strong(mask_spans(m.group(2))) # `**` inside a code span is opaque and gap_cols(indent + 1, m.group(1)) <= 4): lesson = new_lesson(m.group(2), malformed=False) lesson["col"] = content_col(li) @@ -743,17 +917,22 @@ def close_para(lesson: dict) -> None: # ---- nested line: inside the open lesson item ---- if is_mark(ln): # A fenced code block, comment block or HTML block inside the item ends the paragraph; - # sitting beside the field items (shallower than the open field's content) it closes that item. + # sitting beside the field items (shallower than the open field's content) it closes that + # item. Code and raw HTML are visible, so their text joins the comparison as a token. if not in_field and lesson["fcol"] is not None: lesson["last"] = lesson["fcol"] = None + if mark_kind(ln) != COMMENT_MARK: + absorb(lesson, mark_token(ln), para=True) close_para(lesson) continue # A dash underline directly under a paragraph inside the item, at that paragraph's # container column, makes it a Setext H2 (CommonMark) — a heading hidden in a lesson. - if lesson["para"] and indent >= lesson["pcol"] and SETEXT_H2_RE.match(ln[lesson["pcol"]:]): - lesson["problems"].append(f"Setext H2 (underline {ln.strip()[:12]!r}) nested inside the lesson") - close_para(lesson) - continue + if lesson["para"] and indent >= lesson["pcol"]: + uq, utext = quote_depth(ln[lesson["pcol"]:]) + if uq == lesson["pq"] and SETEXT_H2_RE.match(utext): + lesson["problems"].append(f"Setext H2 (underline {ln.strip()[:12]!r}) nested inside the lesson") + close_para(lesson) + continue if lesson["para"] and not interrupts_paragraph(ln, eff): absorb(lesson, ln) # lazy continuation of the open paragraph, wherever the line sits continue @@ -794,7 +973,7 @@ def close_para(lesson: dict) -> None: # quote, or plain text after a blank line, a block or code. Its text is recorded. li = LIST_ITEM_RE.match(rel) absorb(lesson, ln, para=True) - open_para(lesson, eff + content_col(li) if li else eff) + open_para(lesson, eff + content_col(li) if li else eff, quote_depth(rel)[0]) for ls in sections.values(): for l in ls: @@ -923,11 +1102,11 @@ def lesson_key(l: dict) -> tuple[str, str, str, str, str]: problems.append(f"first visible line must be `# Lessons — {d}`, got {first[:40]!r}") elif hm.group(1) != d: problems.append(f"H1 date {hm.group(1)} does not match filename {d}") - stray_headings = [ln.strip() for ln in day_lines if H2_RE.match(ln)] + stray_headings = [ln.strip() for ln in day_lines if is_h2(ln)] if stray_headings: problems.append(f"contains H2 heading(s) {stray_headings[:4]} — a per-day file has no `## ` headings") # A per-day file carries no `## ` heading of its own: parse it as that day's section. - day_sections, _ = parse_lessons([f"## {d}"] + [ln for ln in day_lines if not H2_RE.match(ln)]) + day_sections, _ = parse_lessons([f"## {d}"] + [ln for ln in day_lines if not is_h2(ln)]) if bad_headings: problems.append(f"contains Setext H2 heading(s) {bad_headings[:4]} — a per-day file has no `## ` headings") day_lessons = day_sections.get(d, []) From 04465230a19599b9ae991ef82dc1bf715d9733a9 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sun, 27 Sep 2026 12:35:24 -0400 Subject: [PATCH 26/29] harness: round-24 review fixes (NBSP before comments, escaped openers, label length, angle-bracket escapes, visible blocks before the H1, empty H1s, title continuation, nested <hr>, single sanitize pass) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex findings on e7edf49, each reproduced as a failing case first (neg24.sh): - strip_inline_comments(): the rstrip before an end-of-line comment takes spaces/tabs only (a NBSP stays), and `\<!--` (odd backslashes) is literal text, not an opener - link_ref_def(): labels over 999 characters are prose; `<…>` destinations end at the first UNESCAPED `>` (`<https://x/a\>b>` is valid) - first_visible(): only comment/declaration blocks are invisible — a fence or raw HTML before the H1 is content, so verified:h1 FAILs; html_block_start() now returns the non-rendering mark for types 3-5 (`<?…?>`, `<!DECL>`, CDATA) as well as comments - h1_text()/is_real_h1(): `# ##` is an H1 with no text (closing sequence stripped), `# Title ##` is "Title", `# \#` keeps its hash - absorb(): text in the title's own paragraph after `**title**` is a shape problem ("title paragraph continues past the bold span"); a separate paragraph before the fields is lesson tail - a thematic break inside a lesson joins the comparison as ⟨hr⟩ - parse_lessons(presanitized=True) for the day file: its block payloads were being re-read as Markdown by a second sanitize pass Pushed back on one finding: `hgroup` is not in the CommonMark 0.31.2 type-6 list (commonmark.js's list matches ours tag for tag) and `<hgroup>` under a title renders as lazy inline HTML — which the new title-continuation check now flags (neg24 H4). One earlier case (neg14 3, fence before the H1 → PASS) encoded the old "code is invisible" reading; rewritten to expect FAIL. Evidence: 14 suites, 167 cases, 0 failing; check-knowledge.py RESULT: PASS — 12 pass, 0 warn, 0 fail; markdownlint-cli2 0.23.3: 0 issues. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 83 ++++++++++++++++++++++++++------------ 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index f8654eb..d84679d 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -10,6 +10,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | | 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | correction: `verified/2026-09-26_claude-surfaces_v2.md` (session ran in the chat/Cowork tab, not the Code tab — T's correction 2026-09-27); v1 banner-marked superseded | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 107 Codex findings (23 rounds; rounds 22-23 also rewrote 5 earlier suite cases that CommonMark, checked against commonmark.js 0.31.2, had the other way) | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 116 Codex findings, 1 pushed back with evidence (24 rounds; rounds 22-24 also rewrote 6 earlier suite cases that CommonMark, checked against commonmark.js 0.31.2, had the other way) | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 18ceef7..3514861 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -16,7 +16,8 @@ (title, why, do, source — continuation lines included) to that day's section in knowledge.md, in the same order verified/ only regular files named YYYY-MM-DD_topic_vN.md (real dates) - whose first visible non-blank line is a real H1 (not indented 4+) + whose first visible non-blank line is a real H1 with text (not indented 4+; + a code block or raw HTML before it is visible, a comment is not) Text inside HTML comments, fenced code blocks or indented code blocks never counts as content; a copy that turns prose into code (or back) is a difference. @@ -99,12 +100,14 @@ def link_ref_def(ln: str) -> bool | None: `<…>` (no `<`, `>` inside) or a bare one with balanced parentheses; anything else after it makes the line prose.""" lm = _REF_LABEL_RE.match(ln) - if not lm or not lm.group(1).strip(" \t"): + if not lm or not lm.group(1).strip(" \t") or len(lm.group(1)) > 999: # CommonMark: at most 999 characters in a label return None i = lm.end() if i < len(ln) and ln[i] == "<": - j = ln.find(">", i + 1) - if j < 0 or "<" in ln[i + 1:j]: + j = i + 1 # `<…>`: ends at the first UNESCAPED `>`; an unescaped `<` inside breaks it + while j < len(ln) and ln[j] not in "<>": + j += 2 if ln[j] == "\\" and j + 1 < len(ln) and ln[j + 1] in ASCII_PUNCT else 1 + if j >= len(ln) or ln[j] != ">": return None end = j + 1 else: @@ -118,7 +121,7 @@ def link_ref_def(ln: str) -> bool | None: def first_visible(lines: list[str]) -> str: - """The first line a reader sees as content: blank lines, block marks, link reference + """The first line a reader sees as content: blank lines, comment blocks, link reference definitions (and, for a definition that has no title yet, its title on the following line — a quoted line after a definition that already carries one is a paragraph) are skipped. A continuation line (the prose after a multi-line inline comment's `-->`) @@ -132,9 +135,11 @@ def first_visible(lines: list[str]) -> str: return tail k += 1 continue - if blank(ln) or is_mark(ln): + if blank(ln) or mark_kind(ln) == COMMENT_MARK: k += 1 continue + if is_mark(ln): + return mark_token(ln) # a code block or raw HTML is visible content rd = link_ref_def(ln) if rd is not None: k += 1 @@ -183,7 +188,24 @@ def parse_source(text: str) -> tuple[str, int] | None: return (dest, int(cm.group(1))) if cm else None DAY_H1_RE = re.compile(r"^# Lessons — ([0-9]{4}-[0-9]{2}-[0-9]{2})[ \t]*$") VERIFIED_NAME_RE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[a-z0-9-]+_v[0-9]+\.md$") -H1_LINE_RE = re.compile(r"^ {0,3}#[ \t]+[^ \t]") # a real ATX H1 with text: ≤3 leading spaces (4 = code), space or tab after `#` +_H1_OPEN_RE = re.compile(r"^ {0,3}#(?:[ \t]+(.*))?$") # a `#` heading line: ≤3 leading spaces (4 = code), space/tab or EOL after `#` + + +def h1_text(ln: str) -> str | None: + """The content of an ATX H1, or None when the line is not one. CommonMark strips the optional + closing sequence (`#`s preceded by a space or tab, or making up the whole content): `# ##` + is an H1 with NO text; `# Title ##` is "Title"; `# \#` keeps its escaped hash.""" + m = _H1_OPEN_RE.match(ln) + if not m: + return None + content = (m.group(1) or "").strip(" \t") + if re.fullmatch(r"#+", content): + return "" + return re.sub(r"[ \t]+#+$", "", content).strip(" \t") + + +def is_real_h1(ln: str) -> bool: + return bool(h1_text(ln)) FIELDS = ("why", "do", "source") H2_RE = re.compile(r"^ {0,3}##(?:[ \t]|$)") # any ATX H2 (≤3-space indent), including a bare `##`; NBSP is not a separator bad_headings: list[str] = [] # filled by parse_lessons: H2s that are not valid day headings @@ -274,14 +296,14 @@ def read_lines(p: pathlib.Path) -> list[str] | None: FENCE_RE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$") FENCE_MARK = "\x00fence" # left by sanitize() where a fenced block opened, at its indent: a block boundary, never text -COMMENT_MARK = "\x00comment" # same, where an HTML comment BLOCK (`<!--` at line start) opened +COMMENT_MARK = "\x00comment" # same, where a NON-RENDERING HTML block opened: comment, `<?…?>`, `<!DECL>`, CDATA (types 2-5) CONT_MARK = "\x00cont" # prefix: this line is paragraph continuation text (it began inside an inline comment) LIST_ITEM_RE = re.compile(r"^( *)([-*+]|[0-9]{1,9}[.)])([ \t]+)[^ \t]") # marker + whitespace + content (for content-column tracking) THEMATIC_RE = re.compile(r"^ {0,3}" + THEMATIC_BREAK) # `---`, `- - -`, `***`: a thematic break outranks a list item _HTML6 = ("address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|" "fieldset|figcaption|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|" "menuitem|nav|noframes|ol|optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|tr|track|ul") -HTML_MARK = "\x00html" # left by sanitize() where an HTML block other than a comment (types 1, 3-7) opened +HTML_MARK = "\x00html" # left by sanitize() where a RENDERING HTML block opened (types 1, 6, 7) MARKS = (FENCE_MARK, COMMENT_MARK, HTML_MARK) PAYLOAD_SEP = "\x01" # a fence/HTML mark may carry the block's text after this: `<indent>MARK\x01line⏎line` BLOCK_SEP = "⏎" @@ -299,14 +321,16 @@ def html_block_start(rel: str, in_paragraph: bool) -> tuple[str, str | None] | N m = _HTML_TYPE1.match(rel) if m: return HTML_MARK, "</" + m.group(1).lower() + ">" + # Types 2-5 (comment, processing instruction, declaration, CDATA) are raw HTML that a browser + # never displays: they leave COMMENT_MARK, the non-rendering mark. Types 1, 6 and 7 render. if re.match(r"^ {0,3}<!--", rel): return COMMENT_MARK, "-->" if re.match(r"^ {0,3}<\?", rel): - return HTML_MARK, "?>" + return COMMENT_MARK, "?>" if re.match(r"^ {0,3}<!\[CDATA\[", rel): - return HTML_MARK, "]]>" + return COMMENT_MARK, "]]>" if re.match(r"^ {0,3}<![A-Z]", rel): # uppercase only (CommonMark type 4); `<!foo>` is text - return HTML_MARK, ">" + return COMMENT_MARK, ">" if re.match(r"^ {0,3}</?(?:" + _HTML6 + r")(?:[ \t]|/?>|$)", rel, re.IGNORECASE): return HTML_MARK, None if not in_paragraph and _HTML_TYPE7.match(rel): @@ -542,6 +566,8 @@ def strip_inline_comments(ln: str, scan: str, rest: list[str] = (), containers: i = j + 3 else: j = scan.find("<!--", i) + while j >= 0 and (j - len(scan[:j].rstrip("\\"))) % 2 == 1: + j = scan.find("<!--", j + 1) # `\<!--`: the `<` is escaped, so this is literal text if j < 0: buf.append(ln[i:]) i = len(scan) @@ -557,7 +583,7 @@ def strip_inline_comments(ln: str, scan: str, rest: list[str] = (), containers: # or a comment that continues onto the next line) were never line-ending spaces, so # they must not read as a hard break once the comment is gone. Spaces AFTER `-->` stay. if comment_here and (_in_comment or scan.endswith("-->")): - prose = prose.rstrip() + prose = prose.rstrip(" \t") # a NBSP there is content and stays return prose @@ -730,7 +756,7 @@ def close_block() -> None: return out -def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: +def parse_lessons(lines: list[str], *, presanitized: bool = False) -> tuple[dict[str, list[dict]], list[str]]: """Return ({date: [lesson]}, [duplicate dates]) for a knowledge-style file. A lesson is {"title", "why", "do", "source", "problems"} with the field @@ -746,7 +772,8 @@ def parse_lessons(lines: list[str]) -> tuple[dict[str, list[dict]], list[str]]: lesson: dict | None = None bad_headings.clear() misplaced.clear() - lines = sanitize(lines) + if not presanitized: # a day file arrives already sanitized: a second pass would read block payloads as Markdown + lines = sanitize(lines) def new_lesson(title: str, malformed: bool) -> dict: # col content column of the lesson item; fcol that of the open field item (None = none open) @@ -759,8 +786,9 @@ def new_lesson(title: str, malformed: bool) -> dict: "problems": (["not-a-bold-lesson-bullet"] if malformed else [])} def absorb(lesson: dict, ln: str, para: bool = False) -> None: - """Prose belongs to the open field — or to the title when no field has started, or to - the lesson's tail when the fields are done. A hard line break on the previous line is + """Prose belongs to the open field — or, before any field, to the title when it continues + the title's own paragraph (a shape problem) and to the lesson's tail when it is a separate + paragraph; after the fields, to the tail too. A hard line break on the previous line is kept as HARD_BREAK, and a new paragraph inside the item as PARA_BREAK, so two copies that render differently never compare equal.""" text = ln.strip(" \t") @@ -773,10 +801,14 @@ def absorb(lesson: dict, ln: str, para: bool = False) -> None: lesson["source_raw"] = norm(lesson["source_raw"] + sep + text) else: lesson[key] = norm(lesson[key] + sep + text) - elif not any(lesson["seen"].values()): + elif not any(lesson["seen"].values()) and not para: + # Same paragraph as the bold title: CommonMark renders this text OUTSIDE the closing `**`, + # so the title paragraph is no longer one strong span. lesson["title"] = norm(lesson["title"] + sep + text) + if "not-a-bold-lesson-bullet" not in lesson["problems"] and "title paragraph continues past the bold span" not in lesson["problems"]: + lesson["problems"].append("title paragraph continues past the bold span") else: - lesson["tail"] = norm(lesson["tail"] + sep + text) + lesson["tail"] = norm(lesson["tail"] + sep + text) # lesson-level prose outside title and fields: visible, so compared lesson["hard"] = ends_hard(ln) def open_para(lesson: dict, col: int, qdepth: int = 0) -> None: @@ -860,7 +892,7 @@ def close_para(lesson: dict) -> None: if rd is not None: ref_title_next = rd is False continue - if not is_mark(ln) and not (idx == 0 and H1_LINE_RE.match(ln)): + if not is_mark(ln) and not (idx == 0 and is_real_h1(ln)): misplaced.append(ln.strip()[:60]) if mark_kind(ln) in (FENCE_MARK, HTML_MARK): # a comment renders nothing and is allowed metadata misplaced.append("<code block or raw HTML>") @@ -965,7 +997,8 @@ def close_para(lesson: dict) -> None: else: lesson[key] = norm(value) continue - if THEMATIC_RE.match(rel): # a thematic break inside the item: a block, not text + if THEMATIC_RE.match(rel): # a thematic break inside the item renders as <hr>: a visible block + absorb(lesson, "⟨hr⟩", para=True) close_para(lesson) continue # Anything else opens a new paragraph inside the item: a list item that is not a lesson @@ -1106,7 +1139,7 @@ def lesson_key(l: dict) -> tuple[str, str, str, str, str]: if stray_headings: problems.append(f"contains H2 heading(s) {stray_headings[:4]} — a per-day file has no `## ` headings") # A per-day file carries no `## ` heading of its own: parse it as that day's section. - day_sections, _ = parse_lessons([f"## {d}"] + [ln for ln in day_lines if not is_h2(ln)]) + day_sections, _ = parse_lessons([f"## {d}"] + [ln for ln in day_lines if not is_h2(ln)], presanitized=True) if bad_headings: problems.append(f"contains Setext H2 heading(s) {bad_headings[:4]} — a per-day file has no `## ` headings") day_lessons = day_sections.get(d, []) @@ -1157,15 +1190,15 @@ def lesson_key(l: dict) -> tuple[str, str, str, str, str]: else: ok("verified:names", f"{len(vfiles)} sheet(s), all dated and versioned") def first_nonblank(p: pathlib.Path) -> str | None: - """First line a reader sees as prose: comments and fenced code are skipped, not counted.""" + """First line a reader sees: comment/declaration blocks are skipped; code and raw HTML count.""" lines = read_lines(p) if lines is None: return None return first_visible(sanitize(lines)) firsts = {p.name: first_nonblank(p) for p in vfiles} - noh1 = [n for n, f in firsts.items() if f is not None and not H1_LINE_RE.match(f)] + noh1 = [n for n, f in firsts.items() if f is not None and not is_real_h1(f)] if noh1: - fail("verified:h1", f"first visible non-blank line is not an H1 (indent ≤3, `# `; comments and code ignored): {noh1}") + fail("verified:h1", f"first visible non-blank line is not an H1 with text (indent ≤3, `# `; comments and declarations ignored, code and raw HTML count): {noh1}") elif vfiles: ok("verified:h1", "every sheet starts with an H1") From 03bb96e3b71e562069b11bb50f2f6773aa241e85 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sun, 27 Sep 2026 12:57:18 -0400 Subject: [PATCH 27/29] harness: round-25 review fixes (unmatched backtick runs, fence indentation, indented-code payloads, stray top-level content, field order, quoted fences) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex findings on 0446523, each reproduced as a failing case first (neg25.sh): - mask_code() looks ahead in the paragraph (closer_ahead) before carrying an unmatched backtick run: with no same-length closer anywhere the run is literal and a later <!-- … --> is a comment - fenced payloads drop up to the OPENER's indentation from each content line (fence_indent), so a fence at 2 vs 3 columns with the same 4-column content differs, as it renders - indented code inside a lesson is collected (blank lines included) and joins the comparison as one ⟨code⟩ token, like fences and raw HTML - knowledge:stray — any visible top-level block inside a day section that is not a lesson (paragraph, heading, thematic break, code, raw HTML) is a FAIL; the same check runs on day files. Comment blocks and reference definitions render nothing and are exempt - the sequence of fields (order) is part of the lesson key: Why/Do swapped in one copy differs - fence and HTML-block openers are read inside block-quote markers (block_quote depth); their lines must carry the same markers, and a shallower line ends the block Also: an HTML block that needed a terminator (-->, </pre>, …) and was ended by its container is still reported as unclosed — CommonMark closes the block, but the OUTPUT holds unterminated raw HTML that a browser reads past (neg16 6b). Eight earlier cases used "top-level content inside a section passes" as the vehicle for a different assertion (a 10-digit marker is a paragraph, `- - -` is a rule, …); they now assert the stray FAIL fires and the wrong diagnosis stays absent (stray_only helper). Evidence: 15 suites, 181 cases, 0 failing; check-knowledge.py RESULT: PASS — 12 pass, 0 warn, 0 fail; markdownlint-cli2 0.23.3: 0 issues. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 197 ++++++++++++++++++++++++++----------- 2 files changed, 139 insertions(+), 60 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index d84679d..52406ab 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -10,6 +10,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | | 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | correction: `verified/2026-09-26_claude-surfaces_v2.md` (session ran in the chat/Cowork tab, not the Code tab — T's correction 2026-09-27); v1 banner-marked superseded | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 116 Codex findings, 1 pushed back with evidence (24 rounds; rounds 22-24 also rewrote 6 earlier suite cases that CommonMark, checked against commonmark.js 0.31.2, had the other way) | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 122 Codex findings, 1 pushed back with evidence (25 rounds; rounds 22-25 also rewrote 14 earlier suite cases whose vehicle or CommonMark reading no longer held, each checked against commonmark.js 0.31.2 or by mutation) | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 3514861..1e981ae 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -9,7 +9,9 @@ bullet in a section is a bold lesson with a non-empty "Why it matters here:", a non-empty "Do:", and a "Source:" line carrying a link and a 0-100 confidence; a bullet - outside every day section is a FAIL (readers never see it) + outside every day section is a FAIL (readers never see it), and so + is any other visible top-level block inside a day section (a + paragraph, heading, rule, code or raw HTML — no day file carries it) lessons/ one regular (non-symlink) YYYY-MM-DD.md per day section in knowledge.md, headed `# Lessons — <that date>`, no orphans, no `## ` headings of its own, and lesson content identical @@ -211,6 +213,7 @@ def is_real_h1(ln: str) -> bool: bad_headings: list[str] = [] # filled by parse_lessons: H2s that are not valid day headings SETEXT_H2_RE = re.compile(r"^ {0,3}-+[ \t]*$") # a dash underline directly under a top-level paragraph line makes it an H2 misplaced: list[str] = [] # filled by parse_lessons: top-level bullets that sit under no `## YYYY-MM-DD` +stray: list[str] = [] # filled by parse_lessons: visible top-level blocks inside a day section that are not lessons def record(status: str, name: str, detail: str) -> None: @@ -397,12 +400,17 @@ def mark_kind(ln: str) -> str | None: return head if head in MARKS else None +def visible_ws(s: str) -> str: + """Spaces and tabs made visible so prose normalisation cannot collapse code whitespace.""" + return s.replace(" ", "␣").replace("\t", "⇥") + + def mark_token(ln: str) -> str: """A comparison token for a fence/HTML mark: what readers see of the block, spaces made visible so prose normalisation cannot collapse them (`⟨code⟩print(1)`, `⟨html⟩<div>x</div>`).""" kind = mark_kind(ln) payload = ln.split(PAYLOAD_SEP, 1)[1] if PAYLOAD_SEP in ln else "" - return ("⟨code⟩" if kind == FENCE_MARK else "⟨html⟩") + payload.replace(" ", "␣").replace("\t", "⇥") + return ("⟨code⟩" if kind == FENCE_MARK else "⟨html⟩") + visible_ws(payload) def is_mark(ln: str) -> bool: @@ -486,13 +494,30 @@ def mask_spans(s: str) -> str: return "".join(out) -def mask_code(ln: str, open_run: int) -> tuple[str, int]: +def closer_ahead(run_len: int, rest: list[str], containers: list[int]) -> bool: + """True when a later line of the SAME paragraph (before a blank line or an interrupting block) + carries a backtick run of exactly `run_len` — the closer of a span opened on this line.""" + for nxt in rest: + nxt = expand_lead(nxt) + if blank(nxt): + return False + indent = len(nxt) - len(nxt.lstrip(" ")) + eff = next((c for c in reversed(containers) if c <= indent), 0) + if interrupts_paragraph(nxt, eff): + return False + if any(b - a == run_len for a, b in backtick_runs(nxt)): + return True + return False + + +def mask_code(ln: str, open_run: int, rest: list[str] = (), containers: list[int] = ()) -> tuple[str, int]: """Mask inline code with backticks for delimiter scanning; return (masked, open_run_after). - A backtick run with no same-length closer on its line opens a span that continues - on the following lines of the paragraph (CommonMark treats the newline as a space), - so `<!--` on the next line inside that span is code, not a comment. `open_run` is - the length of such a run carried over from the previous line (0 = none). + A backtick run with no same-length closer on its line opens a span that continues on the + following lines of the paragraph (CommonMark treats the newline as a space) — but only when + such a closer exists later in the paragraph (`rest`, judged against `containers`); with no + closer anywhere the run is literal text and scanning goes on after it. `open_run` is the + length of a run carried over from the previous line (0 = none). """ runs = backtick_runs(ln) out: list[str] = [] @@ -506,10 +531,13 @@ def mask_code(ln: str, open_run: int) -> tuple[str, int]: while k < len(runs): a, b = runs[k] j = next((idx for idx in range(k + 1, len(runs)) if runs[idx][1] - runs[idx][0] == b - a), None) - if j is None: # opens a span that may close on a later line - out.append(ln[i:a]) - out.append("`" * (len(ln) - a)) - return "".join(out), b - a + if j is None: + if closer_ahead(b - a, rest, containers): # opens a span that closes on a later line + out.append(ln[i:a]) + out.append("`" * (len(ln) - a)) + return "".join(out), b - a + k += 1 # no closer anywhere: literal backticks + continue out.append(ln[i:a]) out.append("`" * (runs[j][1] - a)) i, k = runs[j][1], j + 1 @@ -517,17 +545,6 @@ def mask_code(ln: str, open_run: int) -> tuple[str, int]: return "".join(out), 0 -def fence_match(ln: str, base: int) -> re.Match | None: - """FENCE_RE applied relative to the innermost open list item's content column. - - CommonMark strips a list item's content indentation before block parsing, so a - fence nested under ` - Do:` (content column 4) is a fence at 4..7 spaces. - """ - if len(ln) - len(ln.lstrip(" ")) < base: - return None - return FENCE_RE.match(ln[base:]) - - def closes_in_paragraph(rest: list[str], containers: list[int] = ()) -> bool: """True when `-->` arrives on a later line of the SAME paragraph. CommonMark settles block structure before inline HTML, so the paragraph ends at a blank line or at any line that @@ -610,7 +627,6 @@ def sanitize(lines: list[str]) -> list[str]: out: list[str] = [] fence_char: str | None = None fence_len = 0 - fence_base = 0 global _in_comment _in_comment = False # an INLINE `<!-- …` (mid-paragraph) still open from a previous line html_end: str | None = None # inside an HTML block: the string that closes it, or None = closes at a blank line @@ -620,9 +636,24 @@ def sanitize(lines: list[str]) -> list[str]: containers: list[int] = [] # content columns of the open list items, innermost last block_at = -1 # index in `out` of the open fence/HTML block's mark line block_base = 0 # content column of the list item that block lives in - block_text: list[str] = [] # the block's lines, container indentation removed (its payload) + block_quote = 0 # block-quote depth the block was opened at (its lines carry that many `>`) + fence_indent = 0 # indentation of the opening fence inside its container (0-3): removed from content + block_text: list[str] = [] # the block's lines, container and quote markers removed (its payload) unclosed.clear() + def block_inner(ln: str) -> str | None: + """The text of `ln` inside the open block's container and block quotes, or None when the + line does not reach them (the block ends there).""" + if len(ln) - len(ln.lstrip(" ")) < block_base: + return None + inner = ln[block_base:] + for _ in range(block_quote): + qm = QUOTE_RE.match(inner) + if not qm: + return None + inner = inner[qm.end():] + return inner + def close_block() -> None: nonlocal block_at if block_at >= 0: @@ -640,17 +671,25 @@ def close_block() -> None: # A fence or HTML block inside a list item ends with that item (CommonMark: the block runs # "until the end of the containing block"): a non-blank line shallower than the item's # content column closes both, and is then read as an ordinary line. - if (fence_char is not None or in_html) and not blank(ln) and indent < block_base: + inner = block_inner(ln) if (fence_char is not None or in_html) else None + if (fence_char is not None or in_html) and not blank(ln) and inner is None: + # The line leaves the item or the block quote, so CommonMark ends the block there too. Raw + # HTML that needed a terminator (`-->`, `</pre>`, …) is still unterminated in the OUTPUT, + # and a browser reads on past it — so it is reported like an unclosed block. + if in_html and html_end is not None: + unclosed.append("HTML comment never closes" if html_end == "-->" else f"HTML block never closes (no {html_end!r})") fence_char = None in_html = False close_block() if fence_char is not None: - m = fence_match(ln, fence_base) + m = FENCE_RE.match(inner) if inner is not None else None if m and m.group(1)[0] == fence_char and len(m.group(1)) >= fence_len and m.group(2).strip(" \t") == "": # NBSP etc. is content fence_char = None close_block() # the closing fence is not content else: - block_text.append(ln[min(indent, block_base):]) + text = inner if inner is not None else "" # a blank line inside the fence + lead = len(text) - len(text.lstrip(" ")) + block_text.append(text[min(lead, fence_indent):]) # CommonMark: up to the opener's indentation is removed out.append("") continue if in_html: @@ -661,10 +700,10 @@ def close_block() -> None: in_html = False close_block() else: - block_text.append(ln[min(indent, block_base):]) + block_text.append(inner) else: - block_text.append(ln[min(indent, block_base):]) - if html_end in ln.lower(): # `</PRE>` closes a `<pre>` block too + block_text.append(inner if inner is not None else "") + if inner is not None and html_end in inner.lower(): # `</PRE>` closes a `<pre>` block too in_html = False close_block() out.append("") @@ -699,46 +738,51 @@ def close_block() -> None: containers.append(base + content_col(li)) base = containers[-1] if containers else 0 if cont: - scan, open_run = mask_code(ln, open_run) + scan, open_run = mask_code(ln, open_run, lines[k + 1:], containers) out.append(CONT_MARK + strip_inline_comments(ln, scan, lines[k + 1:], containers)) in_paragraph = True continue # An HTML block (`<!--`, `<pre>`, `<div>`, … first on the line, relative to the open # item) is a block like a fence: it leaves a boundary mark and its lines are raw HTML, # never Markdown — a `## date` or `- Do:` inside it is not a heading or a field. - hb = html_block_start(ln[base:], in_paragraph) if indent >= base else None + # Openers are read inside the container AND inside any block-quote markers (`> ```` opens a + # fence inside the quote); the block then lives at that quote depth. + qd, qinner = quote_depth(ln[base:]) if indent >= base else (0, "") + hb = html_block_start(qinner, in_paragraph) if indent >= base else None if hb: mark, html_end = hb out.append(" " * indent + mark) open_run = 0 in_paragraph = False - rel = ln[base:].lstrip(" ") + rel = qinner.lstrip(" ") if html_end is None: in_html = True # until a blank line elif html_end == ">": in_html = ">" not in rel[2:] # `<!X … >` may close on its own line else: in_html = html_end not in rel.lower()[len(html_end) - 1:] # e.g. `<!-- x -->` closes on its own line + block_base, block_quote = base, qd if mark == HTML_MARK: # a comment renders nothing: no payload to compare - block_at, block_base = len(out) - 1, base - block_text.append(ln[base:]) + block_at = len(out) - 1 + block_text.append(qinner) if not in_html: close_block() continue - m = fence_match(ln, base) + m = FENCE_RE.match(qinner) if indent >= base else None # CommonMark: a backtick fence cannot open when its info string contains a # backtick (that line is an inline code span, not a fence); tilde fences may. if m and not (m.group(1)[0] == "`" and "`" in m.group(2)): - fence_char, fence_len, fence_base = m.group(1)[0], len(m.group(1)), base + fence_char, fence_len = m.group(1)[0], len(m.group(1)) + fence_indent = len(qinner) - len(qinner.lstrip(" ")) open_run = 0 in_paragraph = False out.append(" " * indent + FENCE_MARK) - block_at, block_base = len(out) - 1, base + block_at, block_base, block_quote = len(out) - 1, base, qd block_text.append(m.group(2).strip(" \t")) # what readers see of the opener: its info string continue # Inline code spans are opaque: a <!-- inside `…` is code, not a comment opener. # Mask them (same length, harmless chars) for delimiter scanning; restore text after. - scan, open_run = mask_code(ln, open_run) + scan, open_run = mask_code(ln, open_run, lines[k + 1:], containers) prose = strip_inline_comments(ln, scan, lines[k + 1:], containers) # Prose that continues the paragraph keeps it open; otherwise the line opens one only if, # read inside its container, it is neither a heading, a thematic break nor indented code. @@ -772,6 +816,7 @@ def parse_lessons(lines: list[str], *, presanitized: bool = False) -> tuple[dict lesson: dict | None = None bad_headings.clear() misplaced.clear() + stray.clear() if not presanitized: # a day file arrives already sanitized: a second pass would read block payloads as Markdown lines = sanitize(lines) @@ -780,8 +825,8 @@ def new_lesson(title: str, malformed: bool) -> dict: # last the open field's key; para a paragraph is open (lazy continuation may follow); # pcol the content column of the container that paragraph lives in (a Setext underline # must sit there); tail lesson-level prose after the fields — visible, so compared - return {"title": norm(title), "why": "", "do": "", "source": "", "source_raw": "", "tail": "", - "col": 2, "fcol": None, "last": None, "hard": False, "para": False, "pcol": 0, "pq": 0, + return {"title": norm(title), "why": "", "do": "", "source": "", "source_raw": "", "tail": "", "order": [], + "col": 2, "fcol": None, "last": None, "hard": False, "para": False, "pcol": 0, "pq": 0, "code": None, "seen": {"why": False, "do": False, "source": False}, "problems": (["not-a-bold-lesson-bullet"] if malformed else [])} @@ -817,6 +862,16 @@ def open_para(lesson: dict, col: int, qdepth: int = 0) -> None: def close_para(lesson: dict) -> None: lesson["para"], lesson["hard"] = False, False + def flush_code(lesson: dict) -> None: + """An indented code block inside the item is visible: its lines (trailing blank ones dropped) + join the comparison as one ⟨code⟩ token, like a fence's payload.""" + code = lesson["code"] + if code is not None: + while code and code[-1] == "": + code.pop() + absorb(lesson, "⟨code⟩" + visible_ws(BLOCK_SEP.join(code)), para=True) + lesson["code"] = None + top_para: str | None = None # the previous line, when it was a top-level paragraph line (a Setext underline may follow) quote_para: tuple[int, str] | None = None # same, for a paragraph line inside a top-level block quote (depth, text) ref_title_next = False # the previous line was a reference definition without a title: its title may follow @@ -828,6 +883,8 @@ def close_para(lesson: dict) -> None: if prev_blank: if lesson is not None: close_para(lesson) # a blank line ends any paragraph (the list item stays open) + if lesson["code"] is not None: + lesson["code"].append("") # inside an indented code block a blank line is a blank code line continue # A dash underline right under a top-level paragraph line turns that line into an H2 # (Setext). It is never a day heading, so it is a bad heading that ends the section. @@ -852,10 +909,16 @@ def close_para(lesson: dict) -> None: # list item. Blocks there are read relative to the innermost item the line sits in: the # open field's content column when the line reaches it, else the lesson's. nested = lesson is not None and current is not None and indent >= lesson["col"] + is_code = False if nested: in_field = lesson["fcol"] is not None and indent >= lesson["fcol"] eff = lesson["fcol"] if in_field else lesson["col"] rel = ln[eff:] + rel_indent = len(rel) - len(rel.lstrip(" ")) + is_code = rel_indent >= 4 and not lesson["para"] # indented code (a lazy paragraph line may be deeper) + if lesson is not None and lesson["code"] is not None and not is_code: + flush_code(lesson) # the indented code block ended before this line + if nested: # An ATX heading inside the item — even ` ## 2026-08-23`, or ` > ## x` inside a block # quote — can never open a day section (and a heading interrupts any paragraph). An H2 # there is a problem; other levels are recorded as text so copies that differ in them differ. @@ -901,7 +964,8 @@ def close_para(lesson: dict) -> None: continue if not nested: # A line shallower than the lesson's content column is a sibling or a block that closes the list. - if THEMATIC_RE.match(ln): # `- - -` is a thematic break, never a list item + if THEMATIC_RE.match(ln): # `- - -` is a thematic break, never a list item — and a visible <hr> + stray.append(ln.strip()[:60]) lesson = None continue m = LESSON_RE.match(ln) @@ -935,16 +999,18 @@ def close_para(lesson: dict) -> None: if not is_mark(ln) and not INTERRUPT_RE.match(ln) and (lesson is None or not lesson["para"]) and ( indent <= 3 or prev_top_para is not None): top_para = ln # a top-level paragraph line: a `---` next would make it an H2 - if lesson is not None: - if lesson["para"] and not INTERRUPT_RE.match(ln) and not is_mark(ln): - # Lazy continuation (CommonMark): a non-blank, non-list line directly - # under a paragraph — even at indent 0 — is still that paragraph's text. - absorb(lesson, ln) - else: - # A heading, thematic break, block quote, fenced code block, HTML block - # (`<div>`, `<!--`, …), or a paragraph after a blank line closes the list: - # fields that follow render as a NEW list, not this lesson's, so they must not reconnect. - lesson = None + if lesson is not None and lesson["para"] and not INTERRUPT_RE.match(ln) and not is_mark(ln): + # Lazy continuation (CommonMark): a non-blank, non-list line directly + # under a paragraph — even at indent 0 — is still that paragraph's text. + absorb(lesson, ln) + continue + # A heading, thematic break, block quote, fenced code block, HTML block (`<div>`, …) or a + # paragraph after a blank line closes the list: fields that follow render as a NEW list, + # not this lesson's, so they must not reconnect. Unless it renders nothing (a comment block + # or a reference definition) it is visible content that no lesson — and no day file — holds. + lesson = None + if mark_kind(ln) != COMMENT_MARK and link_ref_def(ln) is None: + stray.append(mark_token(ln)[:60] if is_mark(ln) else ln.strip()[:60]) continue # ---- nested line: inside the open lesson item ---- if is_mark(ln): @@ -970,9 +1036,11 @@ def close_para(lesson: dict) -> None: continue if not in_field and lesson["fcol"] is not None: lesson["last"] = lesson["fcol"] = None # shallower than the open field's content: that item is closed - rel_indent = len(rel) - len(rel.lstrip(" ")) - if rel_indent >= 4: - close_para(lesson) # an indented code block inside the item: code, never prose + if rel_indent >= 4: # an indented code block inside the item: collected, flushed as one token + if lesson["code"] is None: + lesson["code"] = [] + lesson["code"].append(rel[4:]) # CommonMark removes four columns of indentation + close_para(lesson) continue fm = FIELD_LABEL_RE.match(ln) if fm and not in_field: @@ -985,6 +1053,7 @@ def close_para(lesson: dict) -> None: label, value = fm.group(3), fm.group(4) key = {"Why it matters here": "why", "Do": "do", "Source": "source"}[label] lesson["last"], lesson["fcol"] = key, indent + 1 + gap + lesson["order"].append(key) open_para(lesson, lesson["fcol"]) lesson["hard"] = ends_hard(ln) if lesson["seen"][key]: @@ -1008,6 +1077,8 @@ def close_para(lesson: dict) -> None: absorb(lesson, ln, para=True) open_para(lesson, eff + content_col(li) if li else eff, quote_depth(rel)[0]) + if lesson is not None and lesson["code"] is not None: + flush_code(lesson) for ls in sections.values(): for l in ls: if "not-a-bold-lesson-bullet" in l["problems"]: @@ -1033,11 +1104,11 @@ def shape_errors(sections: dict[str, list[dict]]) -> list[str]: return [f"{d}#{i}: {p}" for d, ls in sections.items() for i, l in enumerate(ls, 1) for p in l["problems"]] -KEY_FIELDS = ("title",) + FIELDS + ("tail",) # what two copies of a lesson are compared on +KEY_FIELDS = ("title",) + FIELDS + ("tail", "order") # what two copies of a lesson are compared on -def lesson_key(l: dict) -> tuple[str, str, str, str, str]: - return (l["title"], l["why"], l["do"], l["source"], l["tail"]) +def lesson_key(l: dict) -> tuple[str, str, str, str, str, str]: + return (l["title"], l["why"], l["do"], l["source"], l["tail"], " ".join(l["order"])) # --- knowledge.md ---------------------------------------------------------- @@ -1059,6 +1130,7 @@ def lesson_key(l: dict) -> tuple[str, str, str, str, str]: fail("knowledge:unclosed", "; ".join(k_unclosed) + " — everything after it is hidden from readers") k_bad_headings = list(bad_headings) k_misplaced = list(misplaced) + k_stray = list(stray) dates = list(sections) if k_bad_headings: fail("knowledge:headings", f"H2 headings that are not `## YYYY-MM-DD`: {k_bad_headings[:4]}") @@ -1066,6 +1138,8 @@ def lesson_key(l: dict) -> tuple[str, str, str, str, str]: ok("knowledge:headings", "every H2 is a day heading") if k_misplaced: fail("knowledge:misplaced", f"{len(k_misplaced)} visible block(s) outside any `## YYYY-MM-DD` section: {k_misplaced[:3]}") + if k_stray: + fail("knowledge:stray", f"{len(k_stray)} visible block(s) inside a day section that are not lessons (no day file carries them): {k_stray[:3]}") if not dates: fail("knowledge:sections", "no `## YYYY-MM-DD` sections") else: @@ -1139,9 +1213,14 @@ def lesson_key(l: dict) -> tuple[str, str, str, str, str]: if stray_headings: problems.append(f"contains H2 heading(s) {stray_headings[:4]} — a per-day file has no `## ` headings") # A per-day file carries no `## ` heading of its own: parse it as that day's section. - day_sections, _ = parse_lessons([f"## {d}"] + [ln for ln in day_lines if not is_h2(ln)], presanitized=True) + body = [ln for ln in day_lines if not is_h2(ln)] + if hm: + body.remove(first) # the day's own H1 (the first occurrence only) + day_sections, _ = parse_lessons([f"## {d}"] + body, presanitized=True) if bad_headings: problems.append(f"contains Setext H2 heading(s) {bad_headings[:4]} — a per-day file has no `## ` headings") + if stray: + problems.append(f"visible block(s) that are not lessons: {stray[:3]}") day_lessons = day_sections.get(d, []) bad_day = shape_errors({d: day_lessons}) if bad_day: From a0fa435230fb13bdc6f0a509e0a894c6553ee37d Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sun, 27 Sep 2026 13:05:40 -0400 Subject: [PATCH 28/29] harness: round-26 review fixes (single normalisation pass, HTML opener indentation, ref-def titles inside sections) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex findings on 03bb96e, each reproduced as a failing case first (neg26.sh): - prose is gathered raw and normalised once per assembled paragraph in the finalisation loop, so a code span that opens on one field line and closes on the next is seen whole and keeps its inner spaces (two vs three spaces inside it now differ; plain prose still collapses) - a raw-HTML opener's 0-3 spaces of indentation inside its container are not payload - a two-line reference definition inside a day section (definition, then its title on the next line) renders nothing and is no longer reported as stray; a quoted line after a definition that already carries a title still is Evidence: 16 suites, 188 cases, 0 failing; check-knowledge.py RESULT: PASS — 12 pass, 0 warn, 0 fail; markdownlint-cli2 0.23.3: 0 issues. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 31 +++++++++++++++++++++---------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index 52406ab..ae3da0d 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -10,6 +10,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | | 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | correction: `verified/2026-09-26_claude-surfaces_v2.md` (session ran in the chat/Cowork tab, not the Code tab — T's correction 2026-09-27); v1 banner-marked superseded | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 122 Codex findings, 1 pushed back with evidence (25 rounds; rounds 22-25 also rewrote 14 earlier suite cases whose vehicle or CommonMark reading no longer held, each checked against commonmark.js 0.31.2 or by mutation) | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 125 Codex findings, 1 pushed back with evidence (26 rounds; rounds 22-25 also rewrote 14 earlier suite cases whose vehicle or CommonMark reading no longer held, each checked against commonmark.js 0.31.2 or by mutation) | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 1e981ae..0a83f34 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -764,7 +764,7 @@ def close_block() -> None: block_base, block_quote = base, qd if mark == HTML_MARK: # a comment renders nothing: no payload to compare block_at = len(out) - 1 - block_text.append(qinner) + block_text.append(qinner.lstrip(" ")) # the opener's 0-3 spaces of indentation are not content if not in_html: close_block() continue @@ -825,13 +825,14 @@ def new_lesson(title: str, malformed: bool) -> dict: # last the open field's key; para a paragraph is open (lazy continuation may follow); # pcol the content column of the container that paragraph lives in (a Setext underline # must sit there); tail lesson-level prose after the fields — visible, so compared - return {"title": norm(title), "why": "", "do": "", "source": "", "source_raw": "", "tail": "", "order": [], + return {"title": title, "why": "", "do": "", "source": "", "source_raw": "", "tail": "", "order": [], "col": 2, "fcol": None, "last": None, "hard": False, "para": False, "pcol": 0, "pq": 0, "code": None, "seen": {"why": False, "do": False, "source": False}, "problems": (["not-a-bold-lesson-bullet"] if malformed else [])} def absorb(lesson: dict, ln: str, para: bool = False) -> None: - """Prose belongs to the open field — or, before any field, to the title when it continues + """Prose is gathered RAW here and normalised once the lesson is complete. It belongs to the + open field — or, before any field, to the title when it continues the title's own paragraph (a shape problem) and to the lesson's tail when it is a separate paragraph; after the fields, to the tail too. A hard line break on the previous line is kept as HARD_BREAK, and a new paragraph inside the item as PARA_BREAK, so two copies @@ -843,17 +844,17 @@ def absorb(lesson: dict, ln: str, para: bool = False) -> None: if lesson["last"] is not None: key = lesson["last"] if key == "source": - lesson["source_raw"] = norm(lesson["source_raw"] + sep + text) + lesson["source_raw"] = lesson["source_raw"] + sep + text else: - lesson[key] = norm(lesson[key] + sep + text) + lesson[key] = lesson[key] + sep + text elif not any(lesson["seen"].values()) and not para: # Same paragraph as the bold title: CommonMark renders this text OUTSIDE the closing `**`, # so the title paragraph is no longer one strong span. - lesson["title"] = norm(lesson["title"] + sep + text) + lesson["title"] = lesson["title"] + sep + text if "not-a-bold-lesson-bullet" not in lesson["problems"] and "title paragraph continues past the bold span" not in lesson["problems"]: lesson["problems"].append("title paragraph continues past the bold span") else: - lesson["tail"] = norm(lesson["tail"] + sep + text) # lesson-level prose outside title and fields: visible, so compared + lesson["tail"] = lesson["tail"] + sep + text # lesson-level prose outside title and fields: visible, so compared lesson["hard"] = ends_hard(ln) def open_para(lesson: dict, col: int, qdepth: int = 0) -> None: @@ -1009,7 +1010,13 @@ def flush_code(lesson: dict) -> None: # not this lesson's, so they must not reconnect. Unless it renders nothing (a comment block # or a reference definition) it is visible content that no lesson — and no day file — holds. lesson = None - if mark_kind(ln) != COMMENT_MARK and link_ref_def(ln) is None: + if title_may_follow and REF_TITLE_RE.match(ln): + continue # the title of the reference definition on the line above + rd = link_ref_def(ln) + if rd is not None: + ref_title_next = rd is False + continue + if mark_kind(ln) != COMMENT_MARK: stray.append(mark_token(ln)[:60] if is_mark(ln) else ln.strip()[:60]) continue # ---- nested line: inside the open lesson item ---- @@ -1062,9 +1069,9 @@ def flush_code(lesson: dict) -> None: # The value may continue on the next line(s); emptiness and the source shape are # judged once the whole field has been assembled (see the finalisation loop). if key == "source": - lesson["source_raw"] = norm(value) + lesson["source_raw"] = value else: - lesson[key] = norm(value) + lesson[key] = value continue if THEMATIC_RE.match(rel): # a thematic break inside the item renders as <hr>: a visible block absorb(lesson, "⟨hr⟩", para=True) @@ -1081,6 +1088,10 @@ def flush_code(lesson: dict) -> None: flush_code(lesson) for ls in sections.values(): for l in ls: + # Text was gathered raw; normalise each assembled paragraph ONCE, so a code span that opens + # on one line and closes on a later one is seen whole and keeps its inner spaces. + for k in ("title", "why", "do", "source_raw", "tail"): + l[k] = norm(l[k]) if "not-a-bold-lesson-bullet" in l["problems"]: continue for k in FIELDS: From 4593c52da0afbeb7770ba268ae62cb685214e638 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sun, 27 Sep 2026 13:22:24 -0400 Subject: [PATCH 29/29] harness: round-27 review fixes (comment-block tails, blocks opening an item's content, reference definitions in the key, hidden confidence attributes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex findings on a0fa435, each reproduced as a failing case first (neg27.sh): - raw text after a non-rendering block's terminator on its closing line (`<!-- x -->VISIBLE`) is emitted whole by CommonMark and is visible: the mark becomes a rendering one carrying that tail (verified:h1 fails on it; inside a lesson it joins the comparison) - a list item's first content begins a new block: in_paragraph resets when the item is pushed, so a complete type-7 tag there (`- <sub>`) opens a raw HTML block that swallows the lines until the next blank one, as commonmark.js renders it. Fixing that exposed that the opener line itself was never examined for block starts (its indent is shallower than the container it just pushed): `- <sub>` and `- ```` now open blocks on the same line (K2, K2+) - link_ref_def() returns (label, destination, title); parse_lessons() collects every definition in the file (first one per label wins, next-line titles included) and each lesson's key carries the resolved definitions of the labels it uses, so a definition that differs or is missing in one copy is a difference - parse_source() masks inline HTML tags before the confidence search: `<br title="confidence 80%">` is not a visible label Evidence: 17 suites, 199 cases, 0 failing; check-knowledge.py RESULT: PASS — 12 pass, 0 warn, 0 fail; markdownlint-cli2 0.23.3: 0 issues. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK --- LEDGER.md | 2 +- harness/check-knowledge.py | 82 ++++++++++++++++++++++++++++---------- 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/LEDGER.md b/LEDGER.md index ae3da0d..088577a 100644 --- a/LEDGER.md +++ b/LEDGER.md @@ -10,6 +10,6 @@ Columns: **Date** is UTC. **Session** is the claude.ai/code link or `hand`. | Date | Session | Branch | PR | Outcome | Evidence | | --- | --- | --- | --- | --- | --- | | 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | correction: `verified/2026-09-26_claude-surfaces_v2.md` (session ran in the chat/Cowork tab, not the Code tab — T's correction 2026-09-27); v1 banner-marked superseded | -| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 125 Codex findings, 1 pushed back with evidence (26 rounds; rounds 22-25 also rewrote 14 earlier suite cases whose vehicle or CommonMark reading no longer held, each checked against commonmark.js 0.31.2 or by mutation) | +| 2026-09-27 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `claude/face-lift-v1` | #1 | open | README restructure, CONTRIBUTING, ledger, `harness/check-knowledge.py` (RESULT: PASS — 12 pass, 0 warn, 0 fail after review fixes), lint CI, description + topics set via `gh` on the linked PC; follow-up pushes fix 3 Sourcery + 129 Codex findings, 1 pushed back with evidence (27 rounds; rounds 22-25 also rewrote 14 earlier suite cases whose vehicle or CommonMark reading no longer held, each checked against commonmark.js 0.31.2 or by mutation) | | 2026-09-26 | [session_01T7N2VsxDrRZPwWM7zdFnuK](https://claude.ai/code/session_01T7N2VsxDrRZPwWM7zdFnuK) | `main` | — | direct | commit `ef0c3ac`: `verified/2026-09-26_claude-surfaces_v1.md` | | 2026-08-24 | Cowork session | `main` | — | direct | commits `27aa529`…`0763bf4`: repo bootstrap, first 5 lessons (`cfd032a`), README | diff --git a/harness/check-knowledge.py b/harness/check-knowledge.py index 0a83f34..b50fb41 100644 --- a/harness/check-knowledge.py +++ b/harness/check-knowledge.py @@ -95,9 +95,10 @@ def scan_destination(text: str, i: int) -> int | None: return i if depth == 0 else None -def link_ref_def(ln: str) -> bool | None: +def link_ref_def(ln: str) -> tuple[str, str, str | None] | None: """None unless `ln` is a COMPLETE link reference definition `[label]: destination ["title"]`; - otherwise True when it carries a title, False when a title may still follow on the next line. + otherwise (normalised label, destination, title) — title None when one may still follow on + the next line. The label needs a non-whitespace character and no unescaped brackets; the destination is `<…>` (no `<`, `>` inside) or a bare one with balanced parentheses; anything else after it makes the line prose.""" @@ -119,7 +120,13 @@ def link_ref_def(ln: str) -> bool | None: tm = _REF_TITLE_TAIL_RE.match(ln[end:]) if not tm: return None - return tm.group("title") is not None + dest = ln[i + 1:end - 1] if ln[i] == "<" else ln[i:end] + return (ref_label(lm.group(1)), dest, tm.group("title")) + + +def ref_label(label: str) -> str: + """A link label the way CommonMark matches it: case-folded, inner whitespace collapsed.""" + return " ".join(label.casefold().split()) def first_visible(lines: list[str]) -> str: @@ -145,13 +152,17 @@ def first_visible(lines: list[str]) -> str: rd = link_ref_def(ln) if rd is not None: k += 1 - if rd is False and k < len(lines) and REF_TITLE_RE.match(lines[k]): + if rd[2] is None and k < len(lines) and REF_TITLE_RE.match(lines[k]): k += 1 continue return ln return "" LINK_LABEL = r"\[(?:\\.|[^\]\\])+\]" # allows escaped \] inside the label +LINK_LABEL_RE = re.compile(r"\[((?:\\.|[^\[\]\\])+)\]") # every bracketed label candidate in prose LINK_OPEN_RE = re.compile(r"^" + LINK_LABEL + r"\(") # `[label](` — the destination is scanned by hand +INLINE_TAG_RE = re.compile( # a CommonMark inline open or closing tag (attributes included): raw HTML, not prose + r"<[A-Za-z][A-Za-z0-9-]*(?:[ \t]+[A-Za-z_:][A-Za-z0-9_.:-]*(?:[ \t]*=[ \t]*(?:[^ \t\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)*[ \t]*/?>" + r"|</[A-Za-z][A-Za-z0-9-]*[ \t]*>") CONFIDENCE_RE = re.compile(r"(?<![A-Za-z0-9_])confidence ([0-9]{1,3})%(?![0-9%])") # a standalone label: `overconfidence 80%` is not one ASCII_PUNCT = set(r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""") # a backslash escapes exactly these (CommonMark) @@ -186,7 +197,7 @@ def parse_source(text: str) -> tuple[str, int] | None: dest = text[j + 2:i] if not re.match(r"https?://[^ \t]+$", dest): return None - cm = CONFIDENCE_RE.search(text[i + 1:]) + cm = CONFIDENCE_RE.search(INLINE_TAG_RE.sub(" ", text[i + 1:])) # `<br title="confidence 80%">` is not a visible label return (dest, int(cm.group(1))) if cm else None DAY_H1_RE = re.compile(r"^# Lessons — ([0-9]{4}-[0-9]{2}-[0-9]{2})[ \t]*$") VERIFIED_NAME_RE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[a-z0-9-]+_v[0-9]+\.md$") @@ -654,10 +665,18 @@ def block_inner(ln: str) -> str | None: inner = inner[qm.end():] return inner - def close_block() -> None: + def close_block(tail: str = "") -> None: + """Seal the open block's mark. A rendering block gets its text as payload; a non-rendering one + (comment, declaration, …) stays invisible unless raw text follows its terminator on the + closing line — CommonMark emits that line whole, so the tail is visible and the mark becomes + a rendering one carrying it.""" nonlocal block_at if block_at >= 0: - out[block_at] += PAYLOAD_SEP + BLOCK_SEP.join(block_text) + if mark_kind(out[block_at]) == COMMENT_MARK: + if tail.strip(" \t"): + out[block_at] = out[block_at].replace(COMMENT_MARK, HTML_MARK) + PAYLOAD_SEP + tail.strip(" \t") + else: + out[block_at] += PAYLOAD_SEP + BLOCK_SEP.join(block_text) block_at = -1 block_text.clear() @@ -705,7 +724,7 @@ def close_block() -> None: block_text.append(inner if inner is not None else "") if inner is not None and html_end in inner.lower(): # `</PRE>` closes a `<pre>` block too in_html = False - close_block() + close_block(inner[inner.lower().index(html_end) + len(html_end):]) out.append("") continue cont = False @@ -720,6 +739,7 @@ def close_block() -> None: # Track which list items are still open: a non-blank line shallower than an # item's content column closes that item (and everything nested in it). lazy = False + new_item = False # this line opened a list item: its content starts at `base` on this same line if not blank(ln) and not cont: # Judge "does this line start a block?" relative to the deepest container the # line's indent still satisfies (a `<!--` at column 0 under a column-4 field is a @@ -736,6 +756,8 @@ def close_block() -> None: li = LIST_ITEM_RE.match(ln[base:]) # a marker 4+ columns past the container is code, not a list item if li and len(li.group(1)) <= 3 and not THEMATIC_RE.match(ln[base:]): containers.append(base + content_col(li)) + in_paragraph = False # the item's content begins a new block: a type-7 tag there opens an HTML block + new_item = True base = containers[-1] if containers else 0 if cont: scan, open_run = mask_code(ln, open_run, lines[k + 1:], containers) @@ -747,8 +769,9 @@ def close_block() -> None: # never Markdown — a `## date` or `- Do:` inside it is not a heading or a field. # Openers are read inside the container AND inside any block-quote markers (`> ```` opens a # fence inside the quote); the block then lives at that quote depth. - qd, qinner = quote_depth(ln[base:]) if indent >= base else (0, "") - hb = html_block_start(qinner, in_paragraph) if indent >= base else None + in_container = indent >= base or new_item # the opener line's content sits at `base` too (`- <sub>`, `- ````) + qd, qinner = quote_depth(ln[base:]) if in_container else (0, "") + hb = html_block_start(qinner, in_paragraph) if in_container else None if hb: mark, html_end = hb out.append(" " * indent + mark) @@ -762,13 +785,18 @@ def close_block() -> None: else: in_html = html_end not in rel.lower()[len(html_end) - 1:] # e.g. `<!-- x -->` closes on its own line block_base, block_quote = base, qd - if mark == HTML_MARK: # a comment renders nothing: no payload to compare - block_at = len(out) - 1 + block_at = len(out) - 1 + if mark == HTML_MARK: block_text.append(qinner.lstrip(" ")) # the opener's 0-3 spaces of indentation are not content - if not in_html: - close_block() + if not in_html: # closed on its own line: any raw text after the terminator is visible + tail = "" + if html_end is not None: + low = rel.lower() + at = low.find(html_end, len(html_end) - 1 if html_end != ">" else 2) + tail = rel[at + len(html_end):] if at >= 0 else "" + close_block(tail) continue - m = FENCE_RE.match(qinner) if indent >= base else None + m = FENCE_RE.match(qinner) if in_container else None # CommonMark: a backtick fence cannot open when its info string contains a # backtick (that line is an inline code span, not a fence); tilde fences may. if m and not (m.group(1)[0] == "`" and "`" in m.group(2)): @@ -825,7 +853,7 @@ def new_lesson(title: str, malformed: bool) -> dict: # last the open field's key; para a paragraph is open (lazy continuation may follow); # pcol the content column of the container that paragraph lives in (a Setext underline # must sit there); tail lesson-level prose after the fields — visible, so compared - return {"title": title, "why": "", "do": "", "source": "", "source_raw": "", "tail": "", "order": [], + return {"title": title, "why": "", "do": "", "source": "", "source_raw": "", "tail": "", "order": [], "refs": "", "col": 2, "fcol": None, "last": None, "hard": False, "para": False, "pcol": 0, "pq": 0, "code": None, "seen": {"why": False, "do": False, "source": False}, "problems": (["not-a-bold-lesson-bullet"] if malformed else [])} @@ -876,6 +904,8 @@ def flush_code(lesson: dict) -> None: top_para: str | None = None # the previous line, when it was a top-level paragraph line (a Setext underline may follow) quote_para: tuple[int, str] | None = None # same, for a paragraph line inside a top-level block quote (depth, text) ref_title_next = False # the previous line was a reference definition without a title: its title may follow + pending_label = "" # …and this is its label + defs: dict[str, tuple[str, str | None]] = {} # link reference definitions seen in this file: label → (destination, title) for idx, ln in enumerate(lines): prev_blank = blank(ln) prev_top_para, top_para = top_para, None @@ -951,10 +981,12 @@ def flush_code(lesson: dict) -> None: # and link reference definitions (with a title on the next line when the definition has # none): a bullet, a paragraph, a thematic break, a code block — readers see it, no day file has it. if title_may_follow and REF_TITLE_RE.match(ln): + defs[pending_label] = (defs[pending_label][0], ln.strip(" \t")) continue rd = link_ref_def(ln) if rd is not None: - ref_title_next = rd is False + defs.setdefault(rd[0], (rd[1], rd[2])) # CommonMark: the first definition of a label wins + ref_title_next, pending_label = rd[2] is None, rd[0] continue if not is_mark(ln) and not (idx == 0 and is_real_h1(ln)): misplaced.append(ln.strip()[:60]) @@ -1011,10 +1043,12 @@ def flush_code(lesson: dict) -> None: # or a reference definition) it is visible content that no lesson — and no day file — holds. lesson = None if title_may_follow and REF_TITLE_RE.match(ln): + defs[pending_label] = (defs[pending_label][0], ln.strip(" \t")) continue # the title of the reference definition on the line above rd = link_ref_def(ln) if rd is not None: - ref_title_next = rd is False + defs.setdefault(rd[0], (rd[1], rd[2])) + ref_title_next, pending_label = rd[2] is None, rd[0] continue if mark_kind(ln) != COMMENT_MARK: stray.append(mark_token(ln)[:60] if is_mark(ln) else ln.strip()[:60]) @@ -1092,6 +1126,12 @@ def flush_code(lesson: dict) -> None: # on one line and closes on a later one is seen whole and keeps its inner spaces. for k in ("title", "why", "do", "source_raw", "tail"): l[k] = norm(l[k]) + # Reference-style links resolve through the file's definitions, so a label used by the + # lesson carries its destination (and title) into the comparison; a label defined in one + # copy only, or defined differently, is a visible difference. + used = {ref_label(m.group(1)) for k in ("title", "why", "do", "source_raw", "tail") + for m in LINK_LABEL_RE.finditer(l[k])} + l["refs"] = "; ".join(f"{lab}={defs[lab][0]} {defs[lab][1] or ''}".rstrip() for lab in sorted(used) if lab in defs) if "not-a-bold-lesson-bullet" in l["problems"]: continue for k in FIELDS: @@ -1115,11 +1155,11 @@ def shape_errors(sections: dict[str, list[dict]]) -> list[str]: return [f"{d}#{i}: {p}" for d, ls in sections.items() for i, l in enumerate(ls, 1) for p in l["problems"]] -KEY_FIELDS = ("title",) + FIELDS + ("tail", "order") # what two copies of a lesson are compared on +KEY_FIELDS = ("title",) + FIELDS + ("tail", "order", "refs") # what two copies of a lesson are compared on -def lesson_key(l: dict) -> tuple[str, str, str, str, str, str]: - return (l["title"], l["why"], l["do"], l["source"], l["tail"], " ".join(l["order"])) +def lesson_key(l: dict) -> tuple[str, str, str, str, str, str, str]: + return (l["title"], l["why"], l["do"], l["source"], l["tail"], " ".join(l["order"]), l["refs"]) # --- knowledge.md ----------------------------------------------------------