From 245a2d5419021fc198764a0e2396084530d766a7 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 25 Sep 2026 21:30:10 -0700 Subject: [PATCH 1/5] Pin and Decode Git's Quoting in repo_gate.py's ls-files Read tracked() read git ls-files without pinning core.quotePath, so an inherited false emitted a non-UTF-8 name raw and the strict decode raised, while the default quoting yielded an escaped spelling that names no file on disk, dropping it from every check that reads the file. Pin core.quotePath=true on the listing and decode each quoted line back into the real name, the shape prose_lint.py's diff_header_path already uses. sh() decodes with surrogateescape so a non-UTF-8 byte cannot raise. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/actions/repo-gate/repo_gate.py | 36 +++++++++++++++++++++++--- scripts/tests/test_repo_gate.py | 36 ++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index 275cb9da..e45d46a6 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -62,7 +62,12 @@ def sh(*args: str) -> str: return subprocess.run( - args, capture_output=True, text=True, encoding="utf-8", check=False + args, + capture_output=True, + text=True, + encoding="utf-8", + errors="surrogateescape", + check=False, ).stdout @@ -125,8 +130,11 @@ def tracked(root: Path, exclude: list[str] | None = None) -> list[str]: A caller vendoring a subtree it does not author, per GOVERNANCE.md's carry-versus-reach test, can scope every check out of that subtree this way. No check itself needs to change. Additive only: an empty or absent `exclude` scans exactly what it always has. + + Git's quoting is pinned on, so the listing is ASCII whatever a config inherits and each quoted + name reaches `unquote_path` in the one form it decodes. """ - args = ["git", "-C", str(root), "ls-files"] + args = ["git", "-C", str(root), "-c", "core.quotePath=true", "ls-files"] if exclude: args += ["--", *(f":!{pattern}" for pattern in exclude)] result = subprocess.run(args, capture_output=True, text=True, encoding="utf-8", check=False) @@ -136,7 +144,29 @@ def tracked(root: Path, exclude: list[str] | None = None) -> list[str]: reason = result.stderr.strip() or f"exit {result.returncode}, no stderr" print(f"git ls-files failed: {reason}", file=sys.stderr) return [] - return [l for l in result.stdout.split("\n") if l] + return [unquote_path(l) for l in result.stdout.split("\n") if l] + + +def unquote_path(name: str) -> str: + """The real name behind one `git ls-files` line, which git quotes when the name needs it. + + Git quotes a name holding any byte at or above 0x80, and one holding a quote, a backslash, + or a control character, escaping it the way C does. Read as a literal path, the quoted form + names no file on disk, so a check reading the file would pass over it and report clean. + + The caller pins `core.quotePath=true`, which is what makes a quoted line ASCII and so what + this decode assumes. Turning the setting off instead would not do: it stops git quoting the + first of those three routes and leaves the other two quoting a name whose non-ASCII bytes sit + raw inside the quotes, which this decode cannot carry. + + A byte that is not valid UTF-8 comes back as a surrogate escape, the form `Path` and + `resolved_eol` both encode back to the original byte. Latin-1 carries each unescaped byte + through unchanged on the way there. + """ + if name.startswith('"') and name.endswith('"') and len(name) > 1: + unescaped = name[1:-1].encode("latin-1", "backslashreplace").decode("unicode-escape") + return unescaped.encode("latin-1", "surrogateescape").decode("utf-8", "surrogateescape") + return name def workflow_files(files: list[str]) -> list[str]: diff --git a/scripts/tests/test_repo_gate.py b/scripts/tests/test_repo_gate.py index 0ce17d69..867d6769 100755 --- a/scripts/tests/test_repo_gate.py +++ b/scripts/tests/test_repo_gate.py @@ -12,6 +12,7 @@ import contextlib import io +import os import re import shutil import subprocess @@ -467,6 +468,41 @@ def test_the_note_carries_every_count_including_the_zeroes(self) -> None: ) +class TestQuotedNames(GitTreeCase): + """A tracked name git quotes reaches every check as the name on disk. + + The constructed name holds a byte that is not valid UTF-8, the case that both crashed the + listing under `core.quotePath=false` and, at the default, left an escaped spelling no file + answers to. + """ + + NAME = os.fsdecode(b"run-\xff-tool") + + def setUp(self) -> None: + super().setUp() + try: + (self.tmp / self.NAME).write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + except (OSError, UnicodeEncodeError): + self.skipTest("this filesystem refuses a name that is not valid UTF-8") + + def test_an_inherited_quote_path_false_lists_the_real_name_rather_than_raising(self) -> None: + self.git("config", "core.quotePath", "false") + self.git("add", "-A") + self.assertEqual([self.NAME], repo_gate.tracked(self.tmp)) + + def test_a_quoted_shebang_name_is_still_checked(self) -> None: + gitattributes = TestEolCoverage.GITATTRIBUTES.replace("eol=lf", "eol=crlf", 1) + hits = self.coverage(gitattributes, {}) + self.assertIn(self.NAME, repo_gate.tracked(self.tmp)) + self.assertTrue(any(f"{self.NAME}: tracked shebang path" in hit for hit in hits), hits) + + def test_a_name_quoted_for_a_quote_or_backslash_decodes_to_itself(self) -> None: + odd = 'say "hi"\\now' + (self.tmp / odd).write_text("x\n", encoding="utf-8") + self.git("add", "-A") + self.assertIn(odd, repo_gate.tracked(self.tmp)) + + class TestGovernanceCoupling(unittest.TestCase): def test_the_exception_set_matches_what_the_doc_documents(self) -> None: """The doc calls it the one documented exception, so the code must not carry a second.""" From b46638b99c26a7d61c857c8bc10f0abc84db8551 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 25 Sep 2026 21:33:06 -0700 Subject: [PATCH 2/5] Escape a Non-UTF-8 Name in repo_gate.py's Output and Keep Its Tests Importable on Windows A decoded name that is not valid UTF-8 now reaches the finding text, and a strict stdout raised on it, ending the run part way. Reconfigure both streams to backslashreplace, as prose_lint.py already does. Build the test's non-UTF-8 name in setUp so a platform refusing it skips the class instead of failing the module import, and move the quote and backslash case into a class that does not depend on that name. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/actions/repo-gate/repo_gate.py | 16 ++++++++++++++++ scripts/tests/test_repo_gate.py | 23 +++++++++++++++++++---- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index e45d46a6..aefcd22a 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -390,7 +390,23 @@ def check_eol_coverage(root: Path, files: list[str]) -> list[str]: CHECKS = {"sha-pin": check_sha_pin, "eol": check_eol, "eol-coverage": check_eol_coverage} +def report_paths_that_are_not_utf8() -> None: + """Let a path holding a byte that is not UTF-8 print rather than ending the run. + + `unquote_path` decodes such a name with surrogateescape so it opens on disk, which leaves the + lone surrogate in the name to reach this program's own output. Encoding it strictly raises at + the line printing that name, so every check after it is lost along with the run's verdict, and + the exit code becomes a traceback's rather than the gate's. Escaping it costs the reader one + unreadable byte in one name. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is not None: + reconfigure(errors="backslashreplace") + + def main(argv: list[str] | None = None) -> int: + report_paths_that_are_not_utf8() ap = argparse.ArgumentParser() ap.add_argument("--root", default=".") ap.add_argument("--check", action="append", choices=sorted(CHECKS)) diff --git a/scripts/tests/test_repo_gate.py b/scripts/tests/test_repo_gate.py index 867d6769..6cefa8ea 100755 --- a/scripts/tests/test_repo_gate.py +++ b/scripts/tests/test_repo_gate.py @@ -476,13 +476,12 @@ class TestQuotedNames(GitTreeCase): answers to. """ - NAME = os.fsdecode(b"run-\xff-tool") - def setUp(self) -> None: super().setUp() try: + self.NAME = os.fsdecode(b"run-\xff-tool") (self.tmp / self.NAME).write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - except (OSError, UnicodeEncodeError): + except (OSError, ValueError): self.skipTest("this filesystem refuses a name that is not valid UTF-8") def test_an_inherited_quote_path_false_lists_the_real_name_rather_than_raising(self) -> None: @@ -496,9 +495,25 @@ def test_a_quoted_shebang_name_is_still_checked(self) -> None: self.assertIn(self.NAME, repo_gate.tracked(self.tmp)) self.assertTrue(any(f"{self.NAME}: tracked shebang path" in hit for hit in hits), hits) + def test_the_run_prints_such_a_name_under_a_strict_output_encoding(self) -> None: + gitattributes = TestEolCoverage.GITATTRIBUTES.replace("eol=lf", "eol=crlf", 1) + self.coverage(gitattributes, {}) + out = io.TextIOWrapper(io.BytesIO(), encoding="utf-8", errors="strict") + err = io.TextIOWrapper(io.BytesIO(), encoding="utf-8", errors="strict") + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + rc = repo_gate.main(["--root", str(self.tmp), "--check", "eol-coverage"]) + out.flush() + self.assertEqual(1, rc) + self.assertIn(b"run-\\udcff-tool: tracked shebang path", out.buffer.getvalue()) + + +class TestQuotedPlainNames(GitTreeCase): def test_a_name_quoted_for_a_quote_or_backslash_decodes_to_itself(self) -> None: odd = 'say "hi"\\now' - (self.tmp / odd).write_text("x\n", encoding="utf-8") + try: + (self.tmp / odd).write_text("x\n", encoding="utf-8") + except OSError: + self.skipTest("this filesystem refuses a quote or a backslash in a name") self.git("add", "-A") self.assertIn(odd, repo_gate.tracked(self.tmp)) From 7faa64117014cec6ef9305743fbc6404fc962263 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 25 Sep 2026 21:40:32 -0700 Subject: [PATCH 3/5] Tolerate a Non-UTF-8 Byte in repo_gate.py's ls-files stderr Git's error for a root that cannot be entered echoes the path raw, and the strict decode of that stderr raised instead of letting main return 2. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/actions/repo-gate/repo_gate.py | 12 ++++++++++-- scripts/tests/test_repo_gate.py | 7 +++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index aefcd22a..dc39d0a9 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -132,12 +132,20 @@ def tracked(root: Path, exclude: list[str] | None = None) -> list[str]: Additive only: an empty or absent `exclude` scans exactly what it always has. Git's quoting is pinned on, so the listing is ASCII whatever a config inherits and each quoted - name reaches `unquote_path` in the one form it decodes. + name reaches `unquote_path` in the one form it decodes. Git's stderr carries no such quoting, and + a failure naming a root that is not UTF-8 echoes that name raw, so the decode tolerates it. """ args = ["git", "-C", str(root), "-c", "core.quotePath=true", "ls-files"] if exclude: args += ["--", *(f":!{pattern}" for pattern in exclude)] - result = subprocess.run(args, capture_output=True, text=True, encoding="utf-8", check=False) + result = subprocess.run( + args, + capture_output=True, + text=True, + encoding="utf-8", + errors="surrogateescape", + check=False, + ) if result.returncode != 0: # A failed command's stdout is never trusted, even where it is non-empty. # A partial listing read as complete is a scan that missed files and said nothing. diff --git a/scripts/tests/test_repo_gate.py b/scripts/tests/test_repo_gate.py index 6cefa8ea..ff7626f0 100755 --- a/scripts/tests/test_repo_gate.py +++ b/scripts/tests/test_repo_gate.py @@ -506,6 +506,13 @@ def test_the_run_prints_such_a_name_under_a_strict_output_encoding(self) -> None self.assertEqual(1, rc) self.assertIn(b"run-\\udcff-tool: tracked shebang path", out.buffer.getvalue()) + def test_a_missing_root_named_in_bytes_that_are_not_utf8_fails_without_raising(self) -> None: + missing = self.tmp / self.NAME / "gone" + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual([], repo_gate.tracked(missing)) + self.assertIn("git ls-files failed", err.getvalue()) + class TestQuotedPlainNames(GitTreeCase): def test_a_name_quoted_for_a_quote_or_backslash_decodes_to_itself(self) -> None: From f1ba0173195d6443e98e7e2abfd3d6fe6a2cbbee Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 25 Sep 2026 21:47:25 -0700 Subject: [PATCH 4/5] Escape Control Characters in repo_gate.py's Printed Findings A decoded tracked name can hold a newline or an escape sequence, which printed raw forges a line of the gate's output or drives the terminal. Spell each C0 or C1 control and DEL as an escape where a finding or a note is printed. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/actions/repo-gate/repo_gate.py | 16 ++++++++++++++-- scripts/tests/test_repo_gate.py | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index dc39d0a9..eb822536 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -52,6 +52,7 @@ # Reading any of those as absence fails a correct pin, which is the direction that costs most. ABSENT = {"404", "422"} GH_TIMEOUT = 20 +CONTROL = re.compile(r"[\x00-\x1f\x7f-\x9f]") # How a check says it did less than its name. # A gate that quietly degrades to a weaker reading prints the same clean line as one that ran. @@ -398,6 +399,17 @@ def check_eol_coverage(root: Path, files: list[str]) -> list[str]: CHECKS = {"sha-pin": check_sha_pin, "eol": check_eol, "eol-coverage": check_eol_coverage} +def printable(line: str) -> str: + """`line` with each control character spelled as an escape, so one finding prints as one line. + + `unquote_path` hands back a tracked name exactly as it is on disk, and a name may hold a + newline or an escape sequence. Printed raw, such a name forges a line of the gate's own output + or drives the reader's terminal, so each C0 or C1 control and DEL is escaped here, where it is + shown. + """ + return CONTROL.sub(lambda m: f"\\x{ord(m.group()):02x}", line) + + def report_paths_that_are_not_utf8() -> None: """Let a path holding a byte that is not UTF-8 print rather than ending the run. @@ -464,10 +476,10 @@ def main(argv: list[str] | None = None) -> int: status = "FAIL" if hits else "ok" print(f"[{status:4}] {name:12} {len(hits)} issue(s)") for h in hits: - print(f" {h}") + print(f" {printable(h)}") # After the findings and outside the count, since a note is not one. for note in NOTES: - print(f" note: {note}") + print(f" note: {printable(note)}") total += len(hits) return 1 if total else 0 diff --git a/scripts/tests/test_repo_gate.py b/scripts/tests/test_repo_gate.py index ff7626f0..163ad3ee 100755 --- a/scripts/tests/test_repo_gate.py +++ b/scripts/tests/test_repo_gate.py @@ -524,6 +524,22 @@ def test_a_name_quoted_for_a_quote_or_backslash_decodes_to_itself(self) -> None: self.git("add", "-A") self.assertIn(odd, repo_gate.tracked(self.tmp)) + def test_a_control_character_in_a_name_prints_escaped_rather_than_raw(self) -> None: + name = "run\n[ok ] forged\x1b[2J" + gitattributes = TestEolCoverage.GITATTRIBUTES.replace("eol=lf", "eol=crlf", 1) + try: + (self.tmp / name).write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + except OSError: + self.skipTest("this filesystem refuses a control character in a name") + self.coverage(gitattributes, {}) + out = io.StringIO() + with contextlib.redirect_stdout(out): + rc = repo_gate.main(["--root", str(self.tmp), "--check", "eol-coverage"]) + self.assertEqual(1, rc) + self.assertIn("run\\x0a[ok ] forged\\x1b[2J: tracked shebang path", out.getvalue()) + self.assertNotIn("\x1b", out.getvalue()) + self.assertFalse(any(l.startswith("[ok ] forged") for l in out.getvalue().splitlines())) + class TestGovernanceCoupling(unittest.TestCase): def test_the_exception_set_matches_what_the_doc_documents(self) -> None: From 1232c1981f62bea5ed0192b1ac508d58a5162a3b Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Fri, 25 Sep 2026 21:54:48 -0700 Subject: [PATCH 5/5] Print tracked()'s Failure Through printable So It Is Safe on Any Stream git's stderr is now decoded with surrogateescape, so a caller that has not reconfigured stderr could still raise printing it. printable() now also spells a lone surrogate as an escape, and the failure line goes through it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/actions/repo-gate/repo_gate.py | 8 +++++--- scripts/tests/test_repo_gate.py | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/actions/repo-gate/repo_gate.py b/.github/actions/repo-gate/repo_gate.py index eb822536..2f0ab8be 100755 --- a/.github/actions/repo-gate/repo_gate.py +++ b/.github/actions/repo-gate/repo_gate.py @@ -151,7 +151,7 @@ def tracked(root: Path, exclude: list[str] | None = None) -> list[str]: # A failed command's stdout is never trusted, even where it is non-empty. # A partial listing read as complete is a scan that missed files and said nothing. reason = result.stderr.strip() or f"exit {result.returncode}, no stderr" - print(f"git ls-files failed: {reason}", file=sys.stderr) + print(f"git ls-files failed: {printable(reason)}", file=sys.stderr) return [] return [unquote_path(l) for l in result.stdout.split("\n") if l] @@ -405,9 +405,11 @@ def printable(line: str) -> str: `unquote_path` hands back a tracked name exactly as it is on disk, and a name may hold a newline or an escape sequence. Printed raw, such a name forges a line of the gate's own output or drives the reader's terminal, so each C0 or C1 control and DEL is escaped here, where it is - shown. + shown. A byte that is not UTF-8 arrives as a lone surrogate, which a strict stream refuses to + encode, so it is spelled as an escape too, leaving the result safe on any stream. """ - return CONTROL.sub(lambda m: f"\\x{ord(m.group()):02x}", line) + escaped = CONTROL.sub(lambda m: f"\\x{ord(m.group()):02x}", line) + return escaped.encode("utf-8", "backslashreplace").decode("utf-8") def report_paths_that_are_not_utf8() -> None: diff --git a/scripts/tests/test_repo_gate.py b/scripts/tests/test_repo_gate.py index 163ad3ee..be6a25c8 100755 --- a/scripts/tests/test_repo_gate.py +++ b/scripts/tests/test_repo_gate.py @@ -508,10 +508,12 @@ def test_the_run_prints_such_a_name_under_a_strict_output_encoding(self) -> None def test_a_missing_root_named_in_bytes_that_are_not_utf8_fails_without_raising(self) -> None: missing = self.tmp / self.NAME / "gone" - err = io.StringIO() + err = io.TextIOWrapper(io.BytesIO(), encoding="utf-8", errors="strict") with contextlib.redirect_stderr(err): self.assertEqual([], repo_gate.tracked(missing)) - self.assertIn("git ls-files failed", err.getvalue()) + err.flush() + self.assertIn(b"git ls-files failed", err.buffer.getvalue()) + self.assertIn(b"run-\\udcff-tool", err.buffer.getvalue()) class TestQuotedPlainNames(GitTreeCase):