From 03fa57825ad9afe4b858b10a359b4d476aaea5ea Mon Sep 17 00:00:00 2001 From: Cameron Brooks <95648474+CameronBrooks11@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:52:16 -0400 Subject: [PATCH 1/3] fix(gitleaks): scan markdown-escaped lines un-escaped Markdown exporters, Google Drive's included, backslash-escape ASCII punctuation, so ghp_... is written as ghp\_... and matches no rule. The Scan step passed a GitHub token committed that way. Add a step that takes every added line carrying a backslash escape, un-escapes all ASCII punctuation (repeated backslashes included), and runs it through gitleaks stdin with the same config. Findings are mapped back to commit and file. The step: - covers merge-only content via --diff-merges=combined - uses byte semantics (LC_ALL=C, awk) so a NUL or invalid UTF-8 byte cannot silently empty the stream - runs even when the Scan step failed - refuses stdin-style .gitleaksignore entries, whose line-based fingerprints would silence the next secret on that line tests/gitleaks-md-escaped.test.sh runs the step's exact run: block against throwaway repos (13 cases). --- .github/workflows/gitleaks.yml | 54 ++++++++++++++++ tests/gitleaks-md-escaped.test.sh | 100 ++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 tests/gitleaks-md-escaped.test.sh diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index 8c7c8dc..a73101e 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -128,3 +128,57 @@ jobs: [ -f .gitleaks.toml ] && cfg="--config .gitleaks.toml" # --redact: never print secret values to the (persisted) Actions log gitleaks git --redact -v $cfg + + # gitleaks matches a token's exact shape, so a Markdown export that writes + # `ghp\_...` for `ghp_...` slips past the Scan above. Unescape only the + # ADDED lines that carry an escape and scan them again. stdin mode has no + # file paths, so the org [allowlist] paths do not apply here, and + # findings carry no commit or file; the index maps them back. + # !cancelled(): still report when the Scan above already failed. + - name: Scan markdown-escaped lines + if: ${{ !cancelled() }} + run: | + set -euo pipefail + # stdin findings are fingerprinted by their line in this stream, which + # shifts with every commit, so a `:rule:line` ignore entry would + # silence whatever secret lands on that line next. + if [ -f .gitleaksignore ] && grep -q '^:' .gitleaksignore; then + echo "::error::.gitleaksignore has a stdin-style (:rule:line) entry; these are not stable" + exit 1 + fi + # Byte semantics. In a UTF-8 locale `.` does not match an invalid byte, so + # a line with a Latin-1 byte before the escape would be silently dropped. + # Under C, [[:punct:]] is exactly the 32 ASCII punctuation characters, + # the set CommonMark lets a backslash escape. The exporter's escape set + # has grown over time (`$`, `:`, `=` appeared later), so take all of it. + export LC_ALL=C + tmp="${RUNNER_TEMP:-$(mktemp -d)}" + # Keep only ADDED content lines carrying a markdown escape, and record which + # commit and file each came from (one index line per emitted line), so a + # finding's StartLine maps back to commit:path. --diff-merges=combined: + # without it a merge shows no diff, and content introduced only by a + # merge resolution (a `++` line) is never seen. + git log -p --all --diff-merges=combined --format='commit %H' \ + | awk -v idx="$tmp/md-escaped.idx" ' + /^commit [0-9a-f]+$/ { c = $2; next } + /^diff --(git|combined) / { hdr = 1; next } + hdr && /^\+\+\+ / { f = substr($0, 5); sub(/^b\//, "", f); sub(/\t$/, "", f); next } + /^@@/ { hdr = 0; next } + !hdr && /^\+.*\\[[:punct:]]/ { print; print c "\t" f > idx } + ' \ + | sed -E 's/\\+([[:punct:]])/\1/g' \ + > "$tmp/md-escaped.txt" + touch "$tmp/md-escaped.idx" + echo "markdown-escaped added lines: $(wc -l < "$tmp/md-escaped.txt")" + rc=0 + gitleaks stdin --redact -v --config .gitleaks.toml \ + --report-format json --report-path "$tmp/md-escaped.json" \ + < "$tmp/md-escaped.txt" || rc=$? + if [ "$rc" -ne 0 ] && [ -s "$tmp/md-escaped.json" ]; then + echo "markdown-escaped findings (commit, file, rule):" + jq -r '.[] | "\(.StartLine)\t\(.RuleID)"' "$tmp/md-escaped.json" \ + | while IFS=$'\t' read -r n rule; do + printf '%s\t%s\n' "$(sed -n "${n}p" "$tmp/md-escaped.idx")" "$rule" + done + fi + exit "$rc" diff --git a/tests/gitleaks-md-escaped.test.sh b/tests/gitleaks-md-escaped.test.sh new file mode 100644 index 0000000..fc2f6b0 --- /dev/null +++ b/tests/gitleaks-md-escaped.test.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Runs the exact "Scan markdown-escaped lines" run: block from gitleaks.yml +# against throwaway git repos, under the shell GitHub uses for `run:`. +# Needs python3 with PyYAML and gitleaks on PATH. +# Usage: bash tests/gitleaks-md-escaped.test.sh [workflow] [org-config] +# +# Token-shaped strings are assembled at runtime and never written to this file: +# the stdin pass cannot apply the org [allowlist] paths, so a literal escaped +# token in tests/ would fail this repo's own scan. +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +WF="${1:-$HERE/../.github/workflows/gitleaks.yml}" +ORG="${2:-$HERE/../.gitleaks.toml}" +T="$(mktemp -d)" +[ -n "${KEEP:-}" ] || trap 'rm -rf "$T"' EXIT; [ -n "${KEEP:-}" ] && echo "T=$T" +python3 -c " +import yaml,sys +steps=yaml.safe_load(open(sys.argv[1]))['jobs']['scan']['steps'] +print(next(s['run'] for s in steps if s.get('name')=='Scan markdown-escaped lines'))" "$WF" > "$T/step.sh" || exit 1 + +rnd() { python3 -c "import secrets,string;print(''.join(secrets.choice(string.ascii_letters+string.digits) for _ in range($1)))"; } +PAT="ghp""_$(rnd 36)" # plain; gitleaks git already catches this +ESC="${PAT/_/\\_}" # what a Drive markdown export writes + +mk() { # dir + git init -q -b main "$1" + git -C "$1" -c user.email=t@example.com -c user.name=t commit -q --allow-empty -m base + cp "$ORG" "$1/.gitleaks.org.toml" +} +put() { # dir path content + mkdir -p "$(dirname "$1/$2")"; printf '%b' "$3" > "$1/$2" + git -C "$1" add -- "$2"; git -C "$1" -c user.email=t@example.com -c user.name=t commit -q -m "add $2" +} +run() { # name dir expect [grep-for-in-output] + local out rc + [ -f "$2/.gitleaks.toml" ] || cp "$2/.gitleaks.org.toml" "$2/.gitleaks.toml" + out=$(cd "$2" && LC_ALL=C.UTF-8 RUNNER_TEMP="$(mktemp -d -p "$T")" bash --noprofile --norc -eo pipefail "$T/step.sh" 2>&1); rc=$? + local ok=1 + { [ "$3" = pass ] && [ $rc -eq 0 ]; } || { [ "$3" = fail ] && [ $rc -ne 0 ]; } || ok=0 + [ -n "${4:-}" ] && ! printf '%s' "$out" | grep -qF -- "$4" && ok=0 + printf '%s' "$out" | grep -qF -- "${PAT#ghp_}" && { ok=0; echo " secret value reached the log"; } + if [ $ok = 1 ]; then echo "ok - $1 (exit $rc)"; else echo "FAIL - $1 (exit $rc, wanted $3${4:+, output naming '$4'})"; echo "$out" | tail -15; FAILED=1; fi +} +FAILED=0 + +mk "$T/a"; put "$T/a" README.md 'nothing escaped here\n' +run "no escaped lines at all passes (grep exit 1 tolerated)" "$T/a" pass + +mk "$T/b"; put "$T/b" README.md 'a\\_b and \\*c\\*\n' +run "escapes without a secret pass" "$T/b" pass + +mk "$T/c"; put "$T/c" "docs/sub dir/notes.md" "PAT: $ESC\n" +run "escaped token fails and names the file" "$T/c" fail "docs/sub dir/notes.md" + +mk "$T/d"; put "$T/d" notes.md "caf\xe9 PAT: $ESC\n" +run "escaped token on a non-UTF-8 line fails" "$T/d" fail notes.md + +# One commit, so the diff order is fixed by path: data.txt (NUL past git's +# 8000-byte binary probe, so it diffs as text) streams before late.md. +mk "$T/e"; printf 'PAT: %s\n' "$ESC" > "$T/e/late.md" +python3 -c "import sys;open(sys.argv[1],'wb').write(b'x\\\\_y\n'*2000+b'nul\x00here\n')" "$T/e/data.txt" +git -C "$T/e" add .; git -C "$T/e" -c user.email=t@example.com -c user.name=t commit -q -m nul +run "a NUL earlier in the stream does not hide a later token" "$T/e" fail late.md + +mk "$T/f"; git -C "$T/f" checkout -q -b side; put "$T/f" side.md "PAT: $ESC\n"; git -C "$T/f" checkout -q main +run "escaped token on a non-HEAD branch fails" "$T/f" fail side.md + +mk "$T/g"; put "$T/g" gone.md "PAT: $ESC\n"; git -C "$T/g" rm -q gone.md +git -C "$T/g" -c user.email=t@example.com -c user.name=t commit -q -m rm +run "escaped token deleted from HEAD still fails" "$T/g" fail gone.md + +mk "$T/h"; printf '[extend]\npath = ".gitleaks.org.toml"\n' > "$T/h/.gitleaks.toml" +put "$T/h" notes.md "PAT: $ESC\n" +run "repo-local config extending the org config still detects" "$T/h" fail notes.md + +mk "$T/i"; printf 'not toml [[[\n' > "$T/i/.gitleaks.toml"; put "$T/i" notes.md 'a\\_b\n' +run "an unloadable config fails closed" "$T/i" fail + +# Escapes outside the original class: Drive writes `\=` and `\:` too. +KEY="$(rnd 32)" +# `\=` must be the line's only escape, or a narrower prefilter still passes it. +mk "$T/j"; put "$T/j" cfg.md "password \\= $KEY\n" +run "escaped assignment (\\=) with a generic secret fails" "$T/j" fail cfg.md + +mk "$T/k"; put "$T/k" notes.md "PAT: ${PAT/_/\\\\\\\\_}\n" +run "double-escaped token fails" "$T/k" fail notes.md + +# A token introduced only by a merge resolution, in neither parent. +mk "$T/l"; put "$T/l" m.md 'base\n' +git -C "$T/l" checkout -q -b side; put "$T/l" s.md 'side\n'; git -C "$T/l" checkout -q main +put "$T/l" n.md 'main\n' +git -C "$T/l" -c user.email=t@example.com -c user.name=t merge -q --no-commit side >/dev/null 2>&1 +printf "PAT: %s\n" "$ESC" >> "$T/l/m.md"; git -C "$T/l" add m.md +git -C "$T/l" -c user.email=t@example.com -c user.name=t commit -q -m "evil merge" +run "escaped token introduced only in a merge fails" "$T/l" fail m.md + +mk "$T/m"; printf ':github-pat:1\n' > "$T/m/.gitleaksignore"; put "$T/m" README.md 'clean\n' +run "a stdin-style .gitleaksignore entry is refused" "$T/m" fail "stdin-style" + +exit $FAILED From 9d07d448bbae12229b99899055f684b62a2b49ec Mon Sep 17 00:00:00 2001 From: Cameron Brooks <95648474+CameronBrooks11@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:52:23 -0400 Subject: [PATCH 2/3] ci: run the workflow tests in the ok check The ok check was an always-pass echo, and tests/commit-email.test.sh was run only by hand. Run both test scripts inside ok, against the gitleaks version read from gitleaks.yml's input default so the two cannot drift. The tests run in ok itself rather than in a job it needs: a required check whose dependency failed reports as skipped, which GitHub counts as passing. --- .github/workflows/ci.yml | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d903ca..2bfc2ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,12 +9,31 @@ permissions: contents: read jobs: - # Standardized org-wide required check. This repo has no automated checks, so - # `ok` is a trivial always-pass gate to keep branch protection uniform. - # See FlowMatrix-AI/eng-governance for the convention. + # Standardized org-wide required check; see FlowMatrix-AI/eng-governance for + # the convention. The tests run inside `ok` itself rather than in a job it + # `needs`: a required check whose dependency failed reports as skipped, and + # GitHub counts a skipped required check as passing. ok: name: ok runs-on: ubuntu-latest steps: - - name: No automated checks defined - run: echo "ok" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + # Same binary version the reusable workflow scans with: read from its + # input default (the one Renovate tracks) rather than pinned twice. + - name: Install gitleaks and PyYAML + run: | + set -euo pipefail + v=$(sed -n 's/^ *default: "\([0-9.]*\)".*/\1/p' .github/workflows/gitleaks.yml) + : "${v:?could not read the gitleaks version from gitleaks.yml}" + curl -fsSL "https://github.com/gitleaks/gitleaks/releases/download/v${v}/gitleaks_${v}_linux_x64.tar.gz" \ + | tar -xz gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks version + python3 -c 'import yaml' 2>/dev/null || sudo apt-get install -y -qq python3-yaml + + - name: Test commit-email check + run: bash tests/commit-email.test.sh + + - name: Test markdown-escaped secret scan + run: bash tests/gitleaks-md-escaped.test.sh From 1a8e50c07da1d2413c755e7e6e460fc662a084c0 Mon Sep 17 00:00:00 2001 From: Cameron Brooks <95648474+CameronBrooks11@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:55:42 -0400 Subject: [PATCH 3/3] ci: run every tests/*.test.sh in the ok check List the test scripts by glob rather than one step each, so a test added later cannot land without running. The loop runs all of them and fails if any failed. --- .github/workflows/ci.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bfc2ad..514827d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,8 +32,14 @@ jobs: gitleaks version python3 -c 'import yaml' 2>/dev/null || sudo apt-get install -y -qq python3-yaml - - name: Test commit-email check - run: bash tests/commit-email.test.sh - - - name: Test markdown-escaped secret scan - run: bash tests/gitleaks-md-escaped.test.sh + # Every tests/*.test.sh, so a new test cannot be added and never run. + - name: Run tests + run: | + set -uo pipefail + rc=0 + for t in tests/*.test.sh; do + echo "::group::$t" + bash "$t" || rc=1 + echo "::endgroup::" + done + exit "$rc"