From 95538a40caa8a83a14646151feb41ddd571b5d63 Mon Sep 17 00:00:00 2001 From: nasser1941 <9349840+nasser1941@users.noreply.github.com> Date: Sat, 10 Oct 2026 17:45:51 +0200 Subject: [PATCH] Check for secrets before commit and push Before the model runs git commit or git push, scan what it would add to the history: the lines a commit adds (including what a git add in the same command stages) or the commits a push sends. Passwords, keys and tokens in code or configuration, credentials in URLs, private keys, known token forms and .env files are reported with file and line. In ask mode the user decides; in auto-edit and yolo the command doesn't run and the model is told to move the secret out of the code. /commit and /pr show the findings before confirming, and the GitHub Action doesn't commit at all. "lcode: allow-secret" in a comment allows a line; secret_check = false turns it off. Closes #94 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 ++ docs/automation.md | 2 + docs/configuration.md | 1 + docs/usage.md | 33 ++++ src/lcode/action.py | 9 +- src/lcode/agent.py | 1 + src/lcode/api.py | 1 + src/lcode/config.py | 1 + src/lcode/gitflow.py | 15 ++ src/lcode/secretscan.py | 315 +++++++++++++++++++++++++++++++++++++++ src/lcode/tools.py | 27 +++- tests/conftest.py | 40 +++++ tests/test_gitflow.py | 41 +---- tests/test_secretscan.py | 163 ++++++++++++++++++++ 14 files changed, 617 insertions(+), 43 deletions(-) create mode 100644 src/lcode/secretscan.py create mode 100644 tests/test_secretscan.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fe5f0f6..de46b36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ All notable changes to lcode are documented here. The format follows ## [Unreleased] +### Added + +- lcode checks for secrets before the model runs `git commit` or `git push`: passwords, keys and + tokens in code or configuration, credentials in URLs (`rtsp://user:pass@…`), private keys, + tokens with a known form, and `.env` files, in the lines a commit adds or the commits a push + sends. In `ask` mode you decide; in `auto-edit` and `yolo` the command doesn't run, and the model + is told to move the secret out of the code. `/commit` and `/pr` show the findings before you + confirm, and the GitHub Action doesn't commit at all. `lcode: allow-secret` in a comment allows a + line; `secret_check = false` turns the check off. See + [Secrets in commits](https://nasser1941.github.io/lcode/usage/#secrets-in-commits). + ## [0.25.0] - 2026-10-10 ### Changed diff --git a/docs/automation.md b/docs/automation.md index fd80c81..9aa3dc6 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -90,6 +90,8 @@ with Session("path/to/repo", permission_mode="auto-edit") as session: - **Events while it runs:** `session.stream(prompt)` yields the events above, ending with the result. - **Permissions:** pass `approve` to decide permission requests. It gets `{"kind": "bash", "title": "…", "target": "pytest -q"}`; without it, every request is refused. + A `git commit` or `git push` that would add [what looks like a secret](usage.md#secrets-in-commits) + comes as `{"kind": "secret", …}` in `ask` mode, and doesn't run in the other modes. - **Options:** `model`, `context`, `permission_mode`, `allowed_tools`, `max_steps`, `think`, `web`, `memory`, `mcp` and `sandbox`. Leave one out to use your [configuration](configuration.md). diff --git a/docs/configuration.md b/docs/configuration.md index acf8e4a..ee56c30 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -43,6 +43,7 @@ lcode config path # print the file location | `notify` | `true` | A desktop [notification](usage.md#notifications) when a long request is done or waits for your answer | | `notify_after` | `30` | Seconds a request runs before it notifies | | `repeat_limit` | `3` | Identical tool calls with identical results before lcode [steps in](how-it-works.md#when-the-model-repeats-itself); `0` turns it off | +| `secret_check` | `true` | Before `git commit` and `git push`, look for passwords, keys and tokens they'd add ([details](usage.md#secrets-in-commits)) | | `checkpoints` | `true` | Save a checkpoint before the model changes files, so [`/undo`](usage.md#undo-and-checkpoints) can restore them | Besides these settings, `config.toml` can hold [hooks](hooks.md) (`[[hooks]]`) and diff --git a/docs/usage.md b/docs/usage.md index 10e0912..2d308fa 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -213,6 +213,39 @@ deny lists, per user or per repository; deny always wins, even in `yolo`. Text a With the [sandbox](sandbox.md) on, `auto-edit` mode runs shell commands without asking too: they can only reach the project, and `/undo` can take their changes back. +### Secrets in commits + +Before the model runs `git commit` or `git push`, lcode checks what would go into the repository's +history: the lines a commit adds (including files a `git add` in the same command stages), or every +commit a push would send. It looks for: + +- passwords, keys and tokens written as values in code or configuration, like + `"password": "…"` or `API_KEY=…`; +- credentials in addresses, like `rtsp://admin:…@192.168.1.10`; +- private keys and tokens with a known form (GitHub, GitLab, AWS, Slack, OpenAI, Anthropic, xAI, + Google, Hugging Face, Stripe); +- `.env` files. + +Placeholders such as `CHANGE_ME`, `your_password`, `${PASSWORD}` or `os.environ["PASSWORD"]` don't +count. + +```text +This commit would add what looks like a secret to the repository's history: +- camera_wall.py:42: a password or key: "password": "Xk…" +``` + +- **In `ask` mode,** you decide: lcode shows the findings and the command. +- **In `auto-edit` and `yolo`,** the command doesn't run. The model is told to read the secret from + an environment variable or a configuration file that git ignores, and to ask you if it isn't one. +- **`/commit` and `/pr`** show the same findings before you confirm. +- **The [GitHub Action](automation.md#github-action)** doesn't commit or push at all when it finds + one. + +If a line only looks like a secret, add `lcode: allow-secret` (or `gitleaks:allow`) in a comment +on it. `lcode config set secret_check false` turns the check off. It's a safety net, not a full +scanner: for a public repository, also run a tool such as [gitleaks](https://github.com/gitleaks/gitleaks) +before you push. + ## Plan mode Local models are slower than cloud models and make more mistakes, so on a bigger change it pays to diff --git a/src/lcode/action.py b/src/lcode/action.py index 9ee2374..80a80ca 100644 --- a/src/lcode/action.py +++ b/src/lcode/action.py @@ -302,7 +302,14 @@ def answer(hub: GitHub, task: Task, inputs: Inputs, cwd: Path) -> None: reply = result.text.strip() or "(no answer)" changed = git(cwd, "status", "--porcelain").strip() if changed: - from lcode import gitflow + from lcode import gitflow, secretscan + + found = secretscan.in_files(cwd, ["."], staged=False) + if found: + raise ActionError( + secretscan.report("commit", found, excerpts=False) + "\n\nSo lcode didn't commit or push its " + "changes. Secrets belong in the repository's secrets or in configuration that git ignores." + ) data = gitflow.ask( session.agent, diff --git a/src/lcode/agent.py b/src/lcode/agent.py index 9a84892..693bf90 100644 --- a/src/lcode/agent.py +++ b/src/lcode/agent.py @@ -233,6 +233,7 @@ class Settings: notify: bool = False # desktop notifications after long requests (lcode.notify) notify_after: int = 30 # seconds: a request this long notifies when it's done or waits for an answer max_parallel_agents: int = 1 + secret_check: bool = True # scan what git commit and git push would add (lcode.secretscan) repeat_limit: int = 3 # identical calls with identical results before the model is told (lcode.repeats) diff --git a/src/lcode/api.py b/src/lcode/api.py index b61b731..c48a8e3 100644 --- a/src/lcode/api.py +++ b/src/lcode/api.py @@ -194,6 +194,7 @@ def open_agent(cwd: Path, options: Options, console: Console, hw: Hardware | Non embed_model=cfg["embed_model"], notify=cfg["notify"], notify_after=cfg["notify_after"], + secret_check=cfg["secret_check"], repeat_limit=cfg["repeat_limit"], ) agent = Agent(ollama, settings, cwd, console=console) diff --git a/src/lcode/config.py b/src/lcode/config.py index 94b3e5a..01830db 100644 --- a/src/lcode/config.py +++ b/src/lcode/config.py @@ -60,6 +60,7 @@ "notify": (True, bool, "desktop notification when a long request is done or waits for you"), "notify_after": (30, int, "seconds a request runs before it notifies (with notify on)"), "repeat_limit": (3, int, "identical tool calls with identical results before lcode steps in (0: never)"), + "secret_check": (True, bool, "before git commit and git push, look for passwords, keys and tokens they'd add"), "max_parallel_agents": (1, int, "subagents that may run at the same time (more needs OLLAMA_NUM_PARALLEL)"), } ENV_OVERRIDES = { diff --git a/src/lcode/gitflow.py b/src/lcode/gitflow.py index 047a329..d0b6965 100644 --- a/src/lcode/gitflow.py +++ b/src/lcode/gitflow.py @@ -24,6 +24,7 @@ from rich.markup import escape from rich.panel import Panel +from rich.text import Text if TYPE_CHECKING: from lcode.agent import Agent @@ -316,6 +317,9 @@ def commit_command(agent: Agent, hint: str = "") -> None: chosen = [f for f in answer.get("files") or [] if f in paths] files = chosen or paths left_out = [p for p in paths if p not in files] + from lcode import secretscan + + secrets_found = secretscan.in_files(root, files, bool(staged)) if agent.settings.secret_check else [] while True: c.print(Panel(escape(message), title="Commit message", title_align="left", border_style="cyan")) c.print(f" [bold]Files[/] {escape(', '.join(files))}") @@ -323,6 +327,9 @@ def commit_command(agent: Agent, hint: str = "") -> None: c.print(f" [dim]Left out (unrelated, the model thinks): {escape(', '.join(left_out))}[/]") if risky: c.print(f" [dim]Left out (may hold secrets; stage them yourself if needed): {escape(', '.join(risky))}[/]") + if secrets_found: + c.print(Text(secretscan.report("commit", secrets_found), style="red")) + c.print(" [dim]Commit them only if they aren't secrets; otherwise move them out of the code first.[/]") choice = confirm(agent, " [y] commit · [e] edit the message · [n] cancel: ", message) if choice == "e": from lcode.planning import edit_text @@ -552,6 +559,14 @@ def pr_command(agent: Agent, arg: str = "") -> None: def push(agent: Agent, root: Path, remote: str, branch: str) -> bool: + from lcode import secretscan + + found = secretscan.outgoing(root, branch) if agent.settings.secret_check else [] + if found: + agent.console.print(Text(secretscan.report("push", found), style="red")) + if confirm(agent, " Push anyway? [y]es / [n]o: ") != "y": + agent.console.print("Not pushed.") + return False try: with agent.console.status(f"Pushing {escape(branch)}…", spinner="dots"): git(root, "push", "-q", "-u", remote, branch, timeout=300) diff --git a/src/lcode/secretscan.py b/src/lcode/secretscan.py new file mode 100644 index 0000000..ef34b33 --- /dev/null +++ b/src/lcode/secretscan.py @@ -0,0 +1,315 @@ +"""Stop secrets before they're committed or pushed. + +Before the model runs `git commit` or `git push`, lcode scans what that would add to the repository's +history: the lines a commit adds (including files a `git add` in the same command stages), or every commit +a push would send. Passwords and tokens in code, private keys, credentials in URLs (`rtsp://user:pass@…`) +and `.env` files are reported with their file and line. In `ask` mode the user decides; in the other modes +the command doesn't run, and the model is told to move the secret out of the code. + +A line that holds a value which only looks like a secret can carry `lcode: allow-secret` (or +`gitleaks:allow`) in a comment, and is skipped. +""" + +from __future__ import annotations + +import re +import shlex +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from lcode.memory import SECRET_PATTERNS + +ALLOW = ("lcode: allow-secret", "lcode:allow-secret", "gitleaks:allow") +MAX_FINDINGS = 10 +GIT_TIMEOUT = 20 +MAX_FILE = 1_000_000 # bytes of an untracked file that are scanned + +# Values that only stand in for a secret. +PLACEHOLDER = re.compile( + r"^(?:change[_-]?me\w*|your[_-]?\w*|x+|\*+|\.+|…+|<[^>]*>" + r"|(?:dummy|example|placeholder|redacted|secret|password|passwd|pass|pwd|token|test|changeit|none|null" + r"|true|false|required|optional|hidden|masked)(?:[_-]?here)?)$", + re.I, +) +# Words that only appear in made-up values: "change-me-in-production", "test-token", "example-key"… +FAKE_WORDS = ( + "example", + "sample", + "dummy", + "fake", + "changeme", + "change-me", + "change_me", + "placeholder", + "redacted", + "test", +) +# A password, key or token written in code or configuration as a literal value. +ASSIGNED = re.compile( + r"""(?P\b[\w.-]*(?:password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|private[_-]?key""" + r"""|client[_-]?secret)[\w.-]*)['"]?\s*(?::|=|:=|=>)\s*(?P['"]?)(?P[^\s'"`,;)}\]]{6,})""" + r"""(?P=quote)""", + re.I, +) +# Names that hold something about a secret (where it is, how it looks), not the secret itself. +ABOUT = re.compile( + r"(?:url|uri|endpoint|file|path|dir|env|var|variable|name|field|header|type|label|prompt|hint|length|len|min|" + r"max|policy|pattern|regex|format|count|expires?|expiry|ttl|timeout|limit|required|enabled|hash|salt_rounds)$", + re.I, +) +EXTRA = [ + ("an xAI API key", re.compile(r"\bxai-[A-Za-z0-9]{20,}")), + ("a Hugging Face token", re.compile(r"\bhf_[A-Za-z0-9]{30,}")), + ("a Stripe key", re.compile(r"\b(?:sk|rk)_live_[A-Za-z0-9]{20,}")), +] +CONFIG_FILES = re.compile(r"(?:\.(?:env|ini|cfg|conf|config|properties|ya?ml|toml)|(?:^|/)\.env[\w.-]*)$", re.I) +URL_PASSWORD = re.compile(r"\b[a-z][a-z0-9+.\-]*://(?P[^\s/:@'\"]+):(?P[^\s/@'\"]+)@", re.I) +STRONG = [p for p in SECRET_PATTERNS if p[0] not in ("a password in a URL", "a password or key")] + EXTRA +ENV_FILE = re.compile(r"(?:^|/)\.env(?:\.[\w-]+)?$") +ENV_EXAMPLES = re.compile(r"\.(?:example|sample|template|dist|defaults?)$") + + +@dataclass +class Finding: + path: str + line: int # 0: the whole file + label: str # what it looks like, e.g. "a password" + excerpt: str # the line, with the secret masked + commit: str = "" # for a push: the commit that adds it + + def describe(self, excerpt: bool = True) -> str: + where = f"{self.path}:{self.line}" if self.line else self.path + if self.commit: + where += f" (commit {self.commit})" + return f"{where}: {self.label}" + (f": {self.excerpt}" if self.excerpt and excerpt else "") + + +def mask(value: str) -> str: + return value[:2] + "…" if len(value) > 4 else "…" + + +def placeholder(value: str) -> bool: + """A value that stands in for a secret: CHANGE_ME, , ${PASSWORD}, a path, a web address…""" + if PLACEHOLDER.match(value) or any(c in value for c in "{}$%<>"): + return True + if any(word in value.lower() for word in FAKE_WORDS): + return True + return value.startswith(("/", "./", "~/", "http:", "https:", "os.", "process.env", "env.")) + + +def scan_line(text: str, config: bool = False) -> tuple[str, str] | None: + """What kind of secret the line holds, and the line with it masked. In configuration files (`config`) a + value counts without quotes too, as in `PASSWORD=hunter22` or `password: hunter22`.""" + if any(marker in text for marker in ALLOW): + return None + shown = text.strip()[:160] + for label, pattern in STRONG: + found = pattern.search(text) + if found: + return label, shown.replace(found.group(0), mask(found.group(0))) + found = URL_PASSWORD.search(text) + if found and not placeholder(found.group("value")) and not placeholder(found.group("user")): + return "a password in a URL", shown.replace(found.group("value"), mask(found.group("value"))) + for found in ASSIGNED.finditer(text): + value = found.group("value") + if placeholder(value) or ABOUT.search(found.group("name")): + continue + if not found.group("quote") and not (config and re.match(r"^\s*(?:export\s+)?[\w.-]+\s*[:=]", text)): + continue # in code, only a quoted literal is a value: `password = args.password` isn't one + return "a password or key", shown.replace(value, mask(value)) + return None + + +def scan_file(path: str, lines: list[tuple[int, str]], commit: str = "") -> list[Finding]: + if ENV_FILE.search(path) and not ENV_EXAMPLES.search(path): + return [Finding(path, 0, "an environment file (.env), which usually holds secrets", "", commit)] + findings = [] + config = bool(CONFIG_FILES.search(path)) + for number, text in lines: + hit = scan_line(text, config) + if hit: + findings.append(Finding(path, number, hit[0], hit[1], commit)) + return findings + + +def scan_diff(diff: str) -> list[Finding]: + """The secrets in the lines a unified diff adds (`git diff`, or `git log -p` with commit: lines).""" + findings: list[Finding] = [] + path, number, commit = "", 0, "" + added: list[tuple[int, str]] = [] + + def flush() -> None: + if path and (added or ENV_FILE.search(path)): + findings.extend(scan_file(path, added, commit)) + added.clear() + + for line in diff.splitlines(): + if line.startswith("commit:"): + flush() + commit, path = line.split(":", 1)[1].strip(), "" + elif line.startswith("+++ "): + flush() + target = line[4:].strip() + path = "" if target == "/dev/null" else target.removeprefix("b/") + elif line.startswith("@@"): + hunk = re.search(r"\+(\d+)", line) + number = int(hunk.group(1)) if hunk else 0 + elif line.startswith("+") and path: + added.append((number, line[1:])) + number += 1 + elif line.startswith(" "): + number += 1 + flush() + return findings + + +def git(cwd: Path, *args: str) -> str: + try: + done = subprocess.run( + ["git", *args], cwd=cwd, capture_output=True, text=True, errors="replace", timeout=GIT_TIMEOUT + ) + except (OSError, subprocess.SubprocessError): + return "" + return done.stdout if done.returncode == 0 else "" + + +GIT_VERB = re.compile(r"\bgit\b(?:\s+-[-\w=.]+(?:\s+[^\s'\"-]\S*)?)*\s+(commit|push)\b") + + +@dataclass +class GitSteps: + adds: list[list[str]] # the arguments of each `git add` + commit: list[str] | None # the arguments of `git commit` + push: list[str] | None # the arguments of `git push` + + +def git_steps(command: str) -> GitSteps | None: + """The git add, commit and push a shell command runs, or None when it commits and pushes nothing.""" + steps = GitSteps([], None, None) + for segment in re.split(r"&&|\|\||;|\n|\|", command): + try: + words = shlex.split(segment) + except ValueError: + words = segment.split() + while words and re.fullmatch(r"\w+=\S*", words[0]): + words = words[1:] # VAR=value in front + if len(words) < 2 or words[0] != "git": + continue + i = 1 + while i < len(words) and words[i].startswith("-"): + i += 2 if words[i] in ("-C", "-c") else 1 # git -C dir / -c key=value + if i >= len(words): + continue + verb, rest = words[i].strip("'\""), words[i + 1 :] + if verb == "add": + steps.adds.append(rest) + elif verb == "commit": + steps.commit = rest + elif verb == "push": + steps.push = rest + # Inside quotes, as in bash -c "git add . && git commit …", sh -c or eval, what gets staged can't be told + # from the words: assume everything. + nested = [verb for m in re.finditer(r"(['\"])(.*?)\1", command, re.S) for verb in GIT_VERB.findall(m.group(2))] + if steps.commit is None and steps.push is None: + nested += GIT_VERB.findall(command) + if "commit" in nested: + steps.commit, steps.adds = ["--all"], [["."]] + if "push" in nested and steps.push is None: + steps.push = [] + return steps if steps.commit is not None or steps.push is not None else None + + +def untracked(cwd: Path, paths: list[str]) -> list[Finding]: + findings = [] + for name in git(cwd, "ls-files", "--others", "--exclude-standard", "--", *paths).splitlines(): + file = cwd / name + try: + if file.stat().st_size > MAX_FILE: + continue + text = file.read_text(errors="strict") + except (OSError, UnicodeDecodeError): + continue # binary or unreadable + findings.extend(scan_file(name, list(enumerate(text.splitlines(), 1)))) + return findings + + +def to_commit(cwd: Path, steps: GitSteps) -> list[Finding]: + """What `git add` and `git commit` in the command would commit: the staged changes, plus what it stages.""" + args = steps.commit or [] + everything = any(a in ("-a", "--all") or re.fullmatch(r"-[a-zA-Z]*a[a-zA-Z]*", a) for a in args if a != "--amend") + findings = scan_diff(git(cwd, "diff", "--cached", "--no-color", "--no-ext-diff", "-U0")) + if everything: + findings += scan_diff(git(cwd, "diff", "--no-color", "--no-ext-diff", "-U0")) + for add in steps.adds: + options = [a for a in add if a.startswith("-")] + paths = [a for a in add if not a.startswith("-")] or (["."] if {"-A", "--all"} & set(options) else []) + if not paths and "-u" not in options and "--update" not in options: + continue + paths = paths or ["."] + findings += scan_diff(git(cwd, "diff", "--no-color", "--no-ext-diff", "-U0", "--", *paths)) + if "-u" not in options and "--update" not in options: + findings += untracked(cwd, paths) + return findings + + +def to_push(cwd: Path, steps: GitSteps) -> list[Finding]: + """The secrets in the commits a push would send that no remote has yet.""" + refs = [a for a in (steps.push or []) if not a.startswith("-")] + branches = [r.split(":", 1)[0].lstrip("+") for r in refs[1:]] or ["HEAD"] + findings: list[Finding] = [] + for branch in branches: + if not branch or not git(cwd, "rev-parse", "--verify", "--quiet", f"{branch}^{{commit}}").strip(): + continue + log = git( + cwd, "log", "-p", "--no-color", "--no-ext-diff", "-U0", "--format=commit:%h", branch, "--not", "--remotes" + ) + findings += scan_diff(log) + return findings + + +def check(cwd: Path, command: str) -> tuple[str, list[Finding]]: + """('commit' or 'push', the secrets it would add to the history) for a shell command.""" + steps = git_steps(command) + if steps is None or not git(cwd, "rev-parse", "--git-dir").strip(): + return "", [] + findings: list[Finding] = [] + if steps.commit is not None: + findings += to_commit(cwd, steps) + if steps.push is not None: + findings += to_push(cwd, steps) + unique = list({(f.path, f.line, f.label): f for f in findings}.values()) + return ("push" if steps.push is not None and steps.commit is None else "commit"), unique + + +def in_files(cwd: Path, paths: list[str], staged: bool) -> list[Finding]: + """For lcode's /commit: the secrets in the staged changes, or in the changes to these files.""" + if staged: + return scan_diff(git(cwd, "diff", "--cached", "--no-color", "--no-ext-diff", "-U0")) + if not paths: + return [] + diffs = [ + git(cwd, "diff", *staged_flag, "--no-color", "--no-ext-diff", "-U0", "--", *paths) + for staged_flag in ([], ["--cached"]) + ] + return scan_diff("\n".join(diffs)) + untracked(cwd, paths) + + +def outgoing(cwd: Path, branch: str = "HEAD") -> list[Finding]: + """For lcode's /pr: the secrets in the commits of `branch` that no remote has yet.""" + return to_push(cwd, GitSteps([], None, ["origin", branch])) + + +def report(action: str, findings: list[Finding], excerpts: bool = True) -> str: + """The findings, one per line; without `excerpts` for places others read, like a public comment.""" + lines = [f"- {f.describe(excerpts)}" for f in findings[:MAX_FINDINGS]] + if len(findings) > MAX_FINDINGS: + lines.append(f"- … and {len(findings) - MAX_FINDINGS} more") + return f"This {action} would add what looks like a secret to the repository's history:\n" + "\n".join(lines) + + +ADVICE = ( + "lcode didn't run the command. Don't commit secrets: read them from an environment variable or from a " + "configuration file that git ignores (and commit an example with placeholders instead). If one of these " + "isn't a secret, tell the user: they decide." +) diff --git a/src/lcode/tools.py b/src/lcode/tools.py index dc22c88..47bf4d3 100644 --- a/src/lcode/tools.py +++ b/src/lcode/tools.py @@ -4,6 +4,7 @@ import difflib import fnmatch +import hashlib import inspect import json import os @@ -19,11 +20,12 @@ from pathlib import Path from typing import TYPE_CHECKING +from rich.console import Group from rich.panel import Panel from rich.syntax import Syntax from rich.text import Text -from lcode import web +from lcode import secretscan, web from lcode.context import shorten from lcode.jobs import JobError, describe, stop_leftovers from lcode.permissions import bash_key, is_read_only @@ -823,7 +825,28 @@ def t_bash(self, command: str, timeout: int = 180, background: bool = False) -> verdict = self.agent.perms.rule("bash", command) if verdict is not None and not verdict[0]: # deny rules apply to read-only commands too return verdict[1] - if not is_read_only(command) and not (sandbox and self.agent.perms.mode == "auto-edit") and verdict is None: + approved = False + if self.agent.settings.secret_check and not is_read_only(command): + action, found = secretscan.check(self.agent.cwd, command) + if found: + text = secretscan.report(action, found) + if self.agent.perms.mode != "ask": # nobody looks at each command: it doesn't run + self.console.print(Text(f" ⎿ {text}", style="red")) + return f"Error: lcode's secret check stopped this command. {text}\n\n{secretscan.ADVICE}" + body = Group(Text(text + "\n", style="red"), Syntax(command, "bash", theme="monokai", word_wrap=True)) + key = "secret:" + hashlib.sha256(text.encode()).hexdigest()[:16] + ok, feedback = self.agent.perms.request( + key, "secret", f"This {action} looks like it holds a secret", body, command + ) + if not ok: + return f"{feedback}\n\n{text}\n{secretscan.ADVICE}" + approved = True + if ( + not approved + and not is_read_only(command) + and not (sandbox and self.agent.perms.mode == "auto-edit") + and verdict is None + ): body = Syntax(command, "bash", theme="monokai", word_wrap=True) where = "in the sandbox" if sandbox else f"in {self.agent.cwd}" ok, feedback = self.agent.perms.request(bash_key(command), "bash", f"Run command ({where})", body, command) diff --git a/tests/conftest.py b/tests/conftest.py index b70be73..dca051a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ from __future__ import annotations import io +import os +import subprocess from pathlib import Path import pytest @@ -106,3 +108,41 @@ def agent(make_agent) -> Agent: def output(agent: Agent) -> str: return agent.console.file.getvalue() + + +def run(cwd, *args) -> str: + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout + + +@pytest.fixture +def git_repo(repo, tmp_path_factory, monkeypatch): + """The demo repository under git, with one commit on main and git isolated from the user's settings.""" + home = tmp_path_factory.mktemp("home") + (home / "gitconfig").write_text("[user]\n\tname = Test\n\temail = test@example.com\n") + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(home / "gitconfig")) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + (repo / ".gitignore").write_text("node_modules/\n") + run(repo, "init", "-q", "-b", "main") + run(repo, "add", "-A") + run(repo, "commit", "-q", "-m", "fix: start the demo") + return repo + + +@pytest.fixture +def github(git_repo, tmp_path_factory, monkeypatch): + """A bare repository as origin, and a stand-in gh that logs its arguments.""" + remote = tmp_path_factory.mktemp("remote") / "demo.git" + run(remote.parent, "init", "-q", "--bare", "-b", "main", str(remote)) + run(git_repo, "remote", "add", "origin", str(remote)) + run(git_repo, "push", "-q", "-u", "origin", "main") + bin_dir = tmp_path_factory.mktemp("bin") + log = bin_dir / "gh.log" + gh = bin_dir / "gh" + gh.write_text( + f'#!/bin/sh\nprintf "%s\\n" "$*" >> {log}\n' + 'if [ "$1 $2" = "pr view" ]; then exit 1; fi\n' + 'if [ "$1 $2" = "pr create" ]; then echo https://github.com/acme/demo/pull/7; fi\n' + ) + gh.chmod(0o755) + monkeypatch.setenv("PATH", f"{bin_dir}:{os.environ['PATH']}") + return remote, log diff --git a/tests/test_gitflow.py b/tests/test_gitflow.py index e3640af..8b86d67 100644 --- a/tests/test_gitflow.py +++ b/tests/test_gitflow.py @@ -1,9 +1,8 @@ import json -import subprocess import pytest -from conftest import output, reply +from conftest import output, reply, run from lcode import gitflow from lcode.agent import git_info from lcode.cli import build_parser @@ -13,24 +12,6 @@ HW = Hardware("linux", "x", 31, "GPU", 12) -def run(cwd, *args) -> str: - return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout - - -@pytest.fixture -def git_repo(repo, tmp_path_factory, monkeypatch): - """The demo repository under git, with one commit on main and git isolated from the user's settings.""" - home = tmp_path_factory.mktemp("home") - (home / "gitconfig").write_text("[user]\n\tname = Test\n\temail = test@example.com\n") - monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(home / "gitconfig")) - monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") - (repo / ".gitignore").write_text("node_modules/\n") - run(repo, "init", "-q", "-b", "main") - run(repo, "add", "-A") - run(repo, "commit", "-q", "-m", "fix: start the demo") - return repo - - def answers(monkeypatch, *values): queue = list(values) monkeypatch.setattr("builtins.input", lambda prompt="": queue.pop(0)) @@ -161,26 +142,6 @@ def test_a_command_of_your_own_replaces_review(make_agent, git_repo): # ----------------------------------------------------------------------------- /pr -@pytest.fixture -def github(git_repo, tmp_path_factory, monkeypatch): - """A bare repository as origin, and a stand-in gh that logs its arguments.""" - remote = tmp_path_factory.mktemp("remote") / "demo.git" - run(remote.parent, "init", "-q", "--bare", "-b", "main", str(remote)) - run(git_repo, "remote", "add", "origin", str(remote)) - run(git_repo, "push", "-q", "-u", "origin", "main") - bin_dir = tmp_path_factory.mktemp("bin") - log = bin_dir / "gh.log" - gh = bin_dir / "gh" - gh.write_text( - f'#!/bin/sh\nprintf "%s\\n" "$*" >> {log}\n' - 'if [ "$1 $2" = "pr view" ]; then exit 1; fi\n' - 'if [ "$1 $2" = "pr create" ]; then echo https://github.com/acme/demo/pull/7; fi\n' - ) - gh.chmod(0o755) - monkeypatch.setenv("PATH", f"{bin_dir}:{__import__('os').environ['PATH']}") - return remote, log - - def test_pr_moves_commits_on_main_to_a_new_branch_and_opens_it(make_agent, git_repo, github, monkeypatch): remote, log = github agent = make_agent() diff --git a/tests/test_secretscan.py b/tests/test_secretscan.py new file mode 100644 index 0000000..ad5c855 --- /dev/null +++ b/tests/test_secretscan.py @@ -0,0 +1,163 @@ +import pytest + +from conftest import call, output, reply, run +from lcode import gitflow, secretscan +from lcode.secretscan import git_steps, scan_diff, scan_line +from test_gitflow import answers + +PASSWORD = "Xk9pq72z" # not a real one; lcode: allow-secret +CONFIG = f'DEFAULTS = {{"ip": "192.168.1.10", "username": "admin", "password": "{PASSWORD}"}}\n' + + +@pytest.mark.parametrize( + ("text", "config", "label"), + [ + (CONFIG, False, "a password or key"), + (f'url = "rtsp://admin:{PASSWORD}@192.168.1.10:554/cam"', False, "a password in a URL"), + (f'connect(password="{PASSWORD}")', False, "a password or key"), + (f"PASSWORD={PASSWORD}", True, "a password or key"), + (f'client_secret: "{PASSWORD}"', True, "a password or key"), + ("XAI_API_KEY=xai-" + "Ab1" * 8, True, "an xAI API key"), # made up, built so scanners skip it + ('key = "ghp_' + "a" * 36 + '"', False, "a GitHub token"), + ("-----BEGIN OPENSSH PRIVATE KEY-----", False, "a private key"), # lcode: allow-secret + ], +) +def test_secrets_are_found(text, config, label): + found = scan_line(text, config) + assert found is not None and found[0] == label + assert PASSWORD not in found[1] and "ghp_" + "a" * 36 not in found[1] + + +@pytest.mark.parametrize( + "text", + [ + 'url = f"rtsp://{user}:{password}@{ip}:554/cam"', + '"password": "CHANGE_ME",', + '"password": "your_password",', + 'password = os.environ["CAM_PASSWORD"]', + 'password = config["password"]', + "password = args.password", + 'token_url = "https://example.com/oauth/token"', + 'password_file = "/run/secrets/db"', + "max_password_length = 128", + "postgresql://user:password@localhost/db", + "rtsp://admin:…@192.168.1.10", + 'secret_key: "change-me-to-something-random"', + 'api_key = "test-key-0123456789"', + f'password = "{PASSWORD}" # lcode: allow-secret', + f'password = "{PASSWORD}" # gitleaks:allow', + ], +) +def test_placeholders_and_references_are_not_secrets(text): + assert scan_line(text) is None + + +def test_unquoted_values_count_only_in_configuration(): + assert scan_line(f"password = {PASSWORD}") is None # code: a variable + assert scan_line(f"password = {PASSWORD}", config=True) is not None + assert scan_line("password: ${DB_PASSWORD}", config=True) is None + + +def test_a_diff_gives_files_and_lines(): + diff = ( + "commit:abc1234\n" + "diff --git a/cam.py b/cam.py\n--- a/cam.py\n+++ b/cam.py\n@@ -10,0 +11,2 @@\n" + f"+import os\n+{CONFIG}" + "diff --git a/.env b/.env\nnew file mode 100644\n--- /dev/null\n+++ b/.env\n@@ -0,0 +1 @@\n+DEBUG=1\n" + "diff --git a/.env.example b/.env.example\n--- /dev/null\n+++ b/.env.example\n@@ -0,0 +1 @@\n+PASSWORD=\n" + ) + found = scan_diff(diff) + assert [(f.path, f.line, f.commit) for f in found] == [("cam.py", 12, "abc1234"), (".env", 0, "abc1234")] + assert found[0].describe().startswith("cam.py:12 (commit abc1234): a password or key: DEFAULTS") + + +def test_which_commands_commit_or_push(): + steps = git_steps('cd /src/app && git add cam.py README.md && git commit -m "fix: fit the screen"') + assert steps.adds == [["cam.py", "README.md"]] and steps.commit == ["-m", "fix: fit the screen"] + assert git_steps("git -C app push -u origin feature").push == ["-u", "origin", "feature"] + assert git_steps("GIT_AUTHOR_NAME=x git commit -am wip").commit == ["-am", "wip"] + assert git_steps("git status && git diff | grep password") is None + assert git_steps("echo commit the git changes") is None + nested = git_steps("bash -c 'git add . && git commit -m wip'") + assert nested.commit == ["--all"] and nested.adds == [["."]] and nested.push is None + assert git_steps('sh -c "git -C app push origin main"').push == [] + assert git_steps("bash -c 'git add . && git commit -m wip && git push'").push == [] + + +def commit_count(repo): + return int(run(repo, "rev-list", "--count", "HEAD")) + + +def test_what_a_commit_would_add_is_scanned(git_repo): + (git_repo / "cam.py").write_text("import os\n" + CONFIG) + action, found = secretscan.check(git_repo, "git add cam.py && git commit -m cam") + assert action == "commit" and [(f.path, f.line) for f in found] == [("cam.py", 2)] + assert secretscan.check(git_repo, "git add README.md && git commit -m docs")[1] == [] + assert secretscan.check(git_repo, "git add . && git commit -m all")[1] != [] + run(git_repo, "add", "cam.py") + assert secretscan.check(git_repo, "git commit -m staged")[1] != [] + run(git_repo, "commit", "-qm", "cam") + (git_repo / "cam.py").write_text("import os\n" + CONFIG.replace(PASSWORD, "Zq81mn44")) + assert secretscan.check(git_repo, "git commit -m nothing-staged")[1] == [] + assert secretscan.check(git_repo, "git commit -am everything")[1] != [] + + +def test_what_a_push_would_send_is_scanned(git_repo, github): + (git_repo / "cam.py").write_text(CONFIG) + run(git_repo, "add", "cam.py") + run(git_repo, "commit", "-qm", "cam") + action, found = secretscan.check(git_repo, "git push origin main") + sha = run(git_repo, "rev-parse", "--short", "HEAD").strip() + assert action == "push" and found[0].commit == sha and found[0].path == "cam.py" + run(git_repo, "push", "-q", "origin", "main") + assert secretscan.check(git_repo, "git push")[1] == [] # already there + + +def test_the_model_cannot_commit_a_secret_without_asking(make_agent, git_repo): + (git_repo / "cam.py").write_text(CONFIG) + commit = call("bash", command="git add cam.py && git commit -qm 'add camera defaults'") + agent = make_agent([reply(tool_calls=[commit]), reply("ok")], mode="yolo") + agent.run_turn("commit it") + result = next(m["content"] for m in agent.messages if m["role"] == "tool") + assert result.startswith("Error: lcode's secret check stopped this command. This commit would add") + assert "cam.py:1: a password or key" in result and PASSWORD not in result + assert "environment variable" in result and commit_count(git_repo) == 1 + assert "cam.py:1" in output(agent) + + +def test_in_ask_mode_the_user_decides(make_agent, git_repo, monkeypatch): + (git_repo / "cam.py").write_text(CONFIG) + asked = [] + commit = call("bash", command="git add cam.py && git commit -qm 'add camera defaults'") + agent = make_agent([reply(tool_calls=[commit]), reply("ok")], mode="ask") + agent.perms.approve = lambda request: asked.append(request) or True + agent.run_turn("commit it") + assert [r["kind"] for r in asked] == ["secret"] and commit_count(git_repo) == 2 + assert asked[0]["title"] == "This commit looks like it holds a secret" + + +def test_the_check_can_be_turned_off(make_agent, git_repo): + (git_repo / "cam.py").write_text(CONFIG) + commit = call("bash", command="git add cam.py && git commit -qm 'add camera defaults'") + agent = make_agent([reply(tool_calls=[commit]), reply("ok")], mode="yolo", secret_check=False) + agent.run_turn("commit it") + assert commit_count(git_repo) == 2 + + +def test_pr_asks_before_pushing_a_secret(make_agent, git_repo, github, monkeypatch): + remote, _ = github + (git_repo / "cam.py").write_text(CONFIG) + run(git_repo, "switch", "-q", "-c", "cams") + run(git_repo, "add", "cam.py") + run(git_repo, "commit", "-qm", "Add cameras") + agent = make_agent() + answers(monkeypatch, "n") + assert gitflow.push(agent, git_repo, "origin", "cams") is False + assert "This push would add what looks like a secret" in output(agent) and "cams" not in run(remote, "branch") + + +def test_the_action_does_not_commit_a_secret(git_repo): + (git_repo / "cam.py").write_text(CONFIG) + found = secretscan.in_files(git_repo, ["."], staged=False) + text = secretscan.report("commit", found, excerpts=False) + assert "cam.py:1: a password or key" in text and "DEFAULTS" not in text