diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d903ca..514827d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,12 +9,37 @@ permissions: contents: read jobs: - # Standardized org-wide required check. This repo has no automated checks, so - # `ok` is a trivial always-pass gate to keep branch protection uniform. - # See FlowMatrix-AI/eng-governance for the convention. + # Standardized org-wide required check; see FlowMatrix-AI/eng-governance for + # the convention. The tests run inside `ok` itself rather than in a job it + # `needs`: a required check whose dependency failed reports as skipped, and + # GitHub counts a skipped required check as passing. ok: name: ok runs-on: ubuntu-latest steps: - - name: No automated checks defined - run: echo "ok" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + # Same binary version the reusable workflow scans with: read from its + # input default (the one Renovate tracks) rather than pinned twice. + - name: Install gitleaks and PyYAML + run: | + set -euo pipefail + v=$(sed -n 's/^ *default: "\([0-9.]*\)".*/\1/p' .github/workflows/gitleaks.yml) + : "${v:?could not read the gitleaks version from gitleaks.yml}" + curl -fsSL "https://github.com/gitleaks/gitleaks/releases/download/v${v}/gitleaks_${v}_linux_x64.tar.gz" \ + | tar -xz gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks version + python3 -c 'import yaml' 2>/dev/null || sudo apt-get install -y -qq python3-yaml + + # Every tests/*.test.sh, so a new test cannot be added and never run. + - name: Run tests + run: | + set -uo pipefail + rc=0 + for t in tests/*.test.sh; do + echo "::group::$t" + bash "$t" || rc=1 + echo "::endgroup::" + done + exit "$rc" diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index 98263ae..b0b2fa0 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -133,3 +133,57 @@ jobs: # secret introduced only in a merge resolution is scanned. gitleaks git --redact -v $cfg \ --log-opts="--all --full-history --diff-merges=first-parent --diff-filter=tuxdb" + + # gitleaks matches a token's exact shape, so a Markdown export that writes + # `ghp\_...` for `ghp_...` slips past the Scan above. Unescape only the + # ADDED lines that carry an escape and scan them again. stdin mode has no + # file paths, so the org [allowlist] paths do not apply here, and + # findings carry no commit or file; the index maps them back. + # !cancelled(): still report when the Scan above already failed. + - name: Scan markdown-escaped lines + if: ${{ !cancelled() }} + run: | + set -euo pipefail + # stdin findings are fingerprinted by their line in this stream, which + # shifts with every commit, so a `:rule:line` ignore entry would + # silence whatever secret lands on that line next. + if [ -f .gitleaksignore ] && grep -q '^:' .gitleaksignore; then + echo "::error::.gitleaksignore has a stdin-style (:rule:line) entry; these are not stable" + exit 1 + fi + # Byte semantics. In a UTF-8 locale `.` does not match an invalid byte, so + # a line with a Latin-1 byte before the escape would be silently dropped. + # Under C, [[:punct:]] is exactly the 32 ASCII punctuation characters, + # the set CommonMark lets a backslash escape. The exporter's escape set + # has grown over time (`$`, `:`, `=` appeared later), so take all of it. + export LC_ALL=C + tmp="${RUNNER_TEMP:-$(mktemp -d)}" + # Keep only ADDED content lines carrying a markdown escape, and record which + # commit and file each came from (one index line per emitted line), so a + # finding's StartLine maps back to commit:path. --diff-merges=combined: + # without it a merge shows no diff, and content introduced only by a + # merge resolution (a `++` line) is never seen. + git log -p --all --diff-merges=combined --format='commit %H' \ + | awk -v idx="$tmp/md-escaped.idx" ' + /^commit [0-9a-f]+$/ { c = $2; next } + /^diff --(git|combined) / { hdr = 1; next } + hdr && /^\+\+\+ / { f = substr($0, 5); sub(/^b\//, "", f); sub(/\t$/, "", f); next } + /^@@/ { hdr = 0; next } + !hdr && /^\+.*\\[[:punct:]]/ { print; print c "\t" f > idx } + ' \ + | sed -E 's/\\+([[:punct:]])/\1/g' \ + > "$tmp/md-escaped.txt" + touch "$tmp/md-escaped.idx" + echo "markdown-escaped added lines: $(wc -l < "$tmp/md-escaped.txt")" + rc=0 + gitleaks stdin --redact -v --config .gitleaks.toml \ + --report-format json --report-path "$tmp/md-escaped.json" \ + < "$tmp/md-escaped.txt" || rc=$? + if [ "$rc" -ne 0 ] && [ -s "$tmp/md-escaped.json" ]; then + echo "markdown-escaped findings (commit, file, rule):" + jq -r '.[] | "\(.StartLine)\t\(.RuleID)"' "$tmp/md-escaped.json" \ + | while IFS=$'\t' read -r n rule; do + printf '%s\t%s\n' "$(sed -n "${n}p" "$tmp/md-escaped.idx")" "$rule" + done + fi + exit "$rc" diff --git a/tests/gitleaks-md-escaped.test.sh b/tests/gitleaks-md-escaped.test.sh new file mode 100644 index 0000000..fc2f6b0 --- /dev/null +++ b/tests/gitleaks-md-escaped.test.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Runs the exact "Scan markdown-escaped lines" run: block from gitleaks.yml +# against throwaway git repos, under the shell GitHub uses for `run:`. +# Needs python3 with PyYAML and gitleaks on PATH. +# Usage: bash tests/gitleaks-md-escaped.test.sh [workflow] [org-config] +# +# Token-shaped strings are assembled at runtime and never written to this file: +# the stdin pass cannot apply the org [allowlist] paths, so a literal escaped +# token in tests/ would fail this repo's own scan. +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +WF="${1:-$HERE/../.github/workflows/gitleaks.yml}" +ORG="${2:-$HERE/../.gitleaks.toml}" +T="$(mktemp -d)" +[ -n "${KEEP:-}" ] || trap 'rm -rf "$T"' EXIT; [ -n "${KEEP:-}" ] && echo "T=$T" +python3 -c " +import yaml,sys +steps=yaml.safe_load(open(sys.argv[1]))['jobs']['scan']['steps'] +print(next(s['run'] for s in steps if s.get('name')=='Scan markdown-escaped lines'))" "$WF" > "$T/step.sh" || exit 1 + +rnd() { python3 -c "import secrets,string;print(''.join(secrets.choice(string.ascii_letters+string.digits) for _ in range($1)))"; } +PAT="ghp""_$(rnd 36)" # plain; gitleaks git already catches this +ESC="${PAT/_/\\_}" # what a Drive markdown export writes + +mk() { # dir + git init -q -b main "$1" + git -C "$1" -c user.email=t@example.com -c user.name=t commit -q --allow-empty -m base + cp "$ORG" "$1/.gitleaks.org.toml" +} +put() { # dir path content + mkdir -p "$(dirname "$1/$2")"; printf '%b' "$3" > "$1/$2" + git -C "$1" add -- "$2"; git -C "$1" -c user.email=t@example.com -c user.name=t commit -q -m "add $2" +} +run() { # name dir expect [grep-for-in-output] + local out rc + [ -f "$2/.gitleaks.toml" ] || cp "$2/.gitleaks.org.toml" "$2/.gitleaks.toml" + out=$(cd "$2" && LC_ALL=C.UTF-8 RUNNER_TEMP="$(mktemp -d -p "$T")" bash --noprofile --norc -eo pipefail "$T/step.sh" 2>&1); rc=$? + local ok=1 + { [ "$3" = pass ] && [ $rc -eq 0 ]; } || { [ "$3" = fail ] && [ $rc -ne 0 ]; } || ok=0 + [ -n "${4:-}" ] && ! printf '%s' "$out" | grep -qF -- "$4" && ok=0 + printf '%s' "$out" | grep -qF -- "${PAT#ghp_}" && { ok=0; echo " secret value reached the log"; } + if [ $ok = 1 ]; then echo "ok - $1 (exit $rc)"; else echo "FAIL - $1 (exit $rc, wanted $3${4:+, output naming '$4'})"; echo "$out" | tail -15; FAILED=1; fi +} +FAILED=0 + +mk "$T/a"; put "$T/a" README.md 'nothing escaped here\n' +run "no escaped lines at all passes (grep exit 1 tolerated)" "$T/a" pass + +mk "$T/b"; put "$T/b" README.md 'a\\_b and \\*c\\*\n' +run "escapes without a secret pass" "$T/b" pass + +mk "$T/c"; put "$T/c" "docs/sub dir/notes.md" "PAT: $ESC\n" +run "escaped token fails and names the file" "$T/c" fail "docs/sub dir/notes.md" + +mk "$T/d"; put "$T/d" notes.md "caf\xe9 PAT: $ESC\n" +run "escaped token on a non-UTF-8 line fails" "$T/d" fail notes.md + +# One commit, so the diff order is fixed by path: data.txt (NUL past git's +# 8000-byte binary probe, so it diffs as text) streams before late.md. +mk "$T/e"; printf 'PAT: %s\n' "$ESC" > "$T/e/late.md" +python3 -c "import sys;open(sys.argv[1],'wb').write(b'x\\\\_y\n'*2000+b'nul\x00here\n')" "$T/e/data.txt" +git -C "$T/e" add .; git -C "$T/e" -c user.email=t@example.com -c user.name=t commit -q -m nul +run "a NUL earlier in the stream does not hide a later token" "$T/e" fail late.md + +mk "$T/f"; git -C "$T/f" checkout -q -b side; put "$T/f" side.md "PAT: $ESC\n"; git -C "$T/f" checkout -q main +run "escaped token on a non-HEAD branch fails" "$T/f" fail side.md + +mk "$T/g"; put "$T/g" gone.md "PAT: $ESC\n"; git -C "$T/g" rm -q gone.md +git -C "$T/g" -c user.email=t@example.com -c user.name=t commit -q -m rm +run "escaped token deleted from HEAD still fails" "$T/g" fail gone.md + +mk "$T/h"; printf '[extend]\npath = ".gitleaks.org.toml"\n' > "$T/h/.gitleaks.toml" +put "$T/h" notes.md "PAT: $ESC\n" +run "repo-local config extending the org config still detects" "$T/h" fail notes.md + +mk "$T/i"; printf 'not toml [[[\n' > "$T/i/.gitleaks.toml"; put "$T/i" notes.md 'a\\_b\n' +run "an unloadable config fails closed" "$T/i" fail + +# Escapes outside the original class: Drive writes `\=` and `\:` too. +KEY="$(rnd 32)" +# `\=` must be the line's only escape, or a narrower prefilter still passes it. +mk "$T/j"; put "$T/j" cfg.md "password \\= $KEY\n" +run "escaped assignment (\\=) with a generic secret fails" "$T/j" fail cfg.md + +mk "$T/k"; put "$T/k" notes.md "PAT: ${PAT/_/\\\\\\\\_}\n" +run "double-escaped token fails" "$T/k" fail notes.md + +# A token introduced only by a merge resolution, in neither parent. +mk "$T/l"; put "$T/l" m.md 'base\n' +git -C "$T/l" checkout -q -b side; put "$T/l" s.md 'side\n'; git -C "$T/l" checkout -q main +put "$T/l" n.md 'main\n' +git -C "$T/l" -c user.email=t@example.com -c user.name=t merge -q --no-commit side >/dev/null 2>&1 +printf "PAT: %s\n" "$ESC" >> "$T/l/m.md"; git -C "$T/l" add m.md +git -C "$T/l" -c user.email=t@example.com -c user.name=t commit -q -m "evil merge" +run "escaped token introduced only in a merge fails" "$T/l" fail m.md + +mk "$T/m"; printf ':github-pat:1\n' > "$T/m/.gitleaksignore"; put "$T/m" README.md 'clean\n' +run "a stdin-style .gitleaksignore entry is refused" "$T/m" fail "stdin-style" + +exit $FAILED