From 34ea46771c4cf3d21d387066603bb5a33d1fc229 Mon Sep 17 00:00:00 2001
From: showxu <10173746+showxu@users.noreply.github.com>
Date: Tue, 6 Oct 2026 14:58:38 +0800
Subject: [PATCH] fix: scan non-UTF-8 Git diffs without losing text checks

---
 .github/workflows/policy.yml | 9 ++++++---
 Scripts/check-policy.py      | 9 ++++++---
 Tests/test_policy.py         | 7 ++++++-
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/policy.yml b/.github/workflows/policy.yml
index 225a1d6..451e9c0 100644
--- a/.github/workflows/policy.yml
+++ b/.github/workflows/policy.yml
@@ -53,8 +53,9 @@ jobs:
           AGENT = re.compile(r"\b(ChatGPT|Codex|Cursor|Claude)\b", re.I)
 
 
-          def git(*arguments):
-              return subprocess.check_output(["git", *arguments], text=True, stdin=subprocess.DEVNULL)
+          def git(*arguments, errors="strict"):
+              return subprocess.check_output(["git", *arguments], encoding="utf-8", errors=errors,
+                                             stdin=subprocess.DEVNULL)
 
 
           def api_commit(repository, sha):
@@ -135,8 +136,10 @@ jobs:
               paths = git("diff", "--name-only", "--no-renames", "-z", comparison, head).split("\0")
               for path in filter(None, paths):
                   findings.extend(content_findings(path, 1, path, allow_terms))
+                  # Git can emit byte fixtures as text when they contain no NUL. Keep
+                  # scanning their readable content without requiring valid UTF-8.
                   patch = git("diff", "--no-ext-diff", "--no-color", "--no-renames", "--unified=0",
-                              comparison, head, "--", path)
+                              comparison, head, "--", path, errors="replace")
                   for number, line in added_lines(patch):
                       findings.extend(content_findings(path, number, line, allow_terms))
               return findings
diff --git a/Scripts/check-policy.py b/Scripts/check-policy.py
index eda6c52..10a2f12 100755
--- a/Scripts/check-policy.py
+++ b/Scripts/check-policy.py
@@ -19,8 +19,9 @@
 AGENT = re.compile(r"\b(ChatGPT|Codex|Cursor|Claude)\b", re.I)
 
 
-def git(*arguments):
-    return subprocess.check_output(["git", *arguments], text=True, stdin=subprocess.DEVNULL)
+def git(*arguments, errors="strict"):
+    return subprocess.check_output(["git", *arguments], encoding="utf-8", errors=errors,
+                                   stdin=subprocess.DEVNULL)
 
 
 def api_commit(repository, sha):
@@ -101,8 +102,10 @@ def check_range(repository, base, head, allow_terms, allow_bots, fetch_commit=ap
     paths = git("diff", "--name-only", "--no-renames", "-z", comparison, head).split("\0")
     for path in filter(None, paths):
         findings.extend(content_findings(path, 1, path, allow_terms))
+        # Git can emit byte fixtures as text when they contain no NUL. Keep
+        # scanning their readable content without requiring valid UTF-8.
         patch = git("diff", "--no-ext-diff", "--no-color", "--no-renames", "--unified=0",
-                    comparison, head, "--", path)
+                    comparison, head, "--", path, errors="replace")
         for number, line in added_lines(patch):
             findings.extend(content_findings(path, number, line, allow_terms))
     return findings
diff --git a/Tests/test_policy.py b/Tests/test_policy.py
index 76ee922..eb92f92 100644
--- a/Tests/test_policy.py
+++ b/Tests/test_policy.py
@@ -60,7 +60,7 @@ def test_real_git_range_and_new_branch(self):
             try:
                 subprocess.run(["git", "init", "-q"], check=True)
                 def commit(text):
-                    Path("sample.txt").write_text(text)
+                    Path("sample.txt").write_bytes(text if isinstance(text, bytes) else text.encode())
                     policy.git("add", "sample.txt")
                     policy.git("-c", "user.name=Fixture", "-c", "user.email=fixture@example.test",
                                "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "commit", "-qm", "fix: fixture")
@@ -70,6 +70,11 @@ def commit(text):
                 head = commit("clean\n/" + "home/fixture/private\n")
                 findings = policy.check_range("example/repo", base, head, set(), set(), lambda *_: record())
                 self.assertEqual(findings, ["sample.txt:2: private execution path"])
+                byte_fixture = commit(b"invalid UTF-8: \xfa\n")
+                self.assertEqual(policy.check_range("example/repo", head, byte_fixture, set(), set(), lambda *_: record()), [])
+                unsafe_fixture = commit(b"invalid UTF-8: \xfa\n/" + b"home/fixture/private\n")
+                self.assertEqual(policy.check_range("example/repo", byte_fixture, unsafe_fixture, set(), set(), lambda *_: record()),
+                                 ["sample.txt:2: private execution path"])
             finally:
                 os.chdir(previous)