diff --git a/scripts/inspect-detail-shape.py b/scripts/inspect-detail-shape.py index cd3ca43..52cd2b3 100755 --- a/scripts/inspect-detail-shape.py +++ b/scripts/inspect-detail-shape.py @@ -23,6 +23,7 @@ import asyncio import json +import os import sys import httpx @@ -138,7 +139,7 @@ async def survey(slugs: list[str]) -> None: ) -async def dump_content(slugs: list[str], max_chars: int = 4000) -> None: +async def dump_content(slugs: list[str], max_chars: int | None = None) -> None: """Print each skill's SKILL.md so a verdict can be checked against the file. An LLM second opinion agreeing with a rule is not ground truth; both can be @@ -149,6 +150,12 @@ async def dump_content(slugs: list[str], max_chars: int = 4000) -> None: The registry is public, so this prints content that is already published. Truncated per skill to keep a cohort readable in one job log. """ + # Truncation hid the evidence once already: a skill was flagged on a line + # that fell outside the excerpt, so the visible portion produced no match + # and the finding looked unexplainable. Default to the whole file and let + # the caller trim deliberately. + if max_chars is None: + max_chars = int(os.environ.get("MAX_CHARS", "0")) or 10**9 async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client: for slug in slugs: print(f"\n{'=' * 70}\n{slug}\n{'=' * 70}", flush=True) diff --git a/src/malwar/detectors/rule_engine/rules/permission_scope.py b/src/malwar/detectors/rule_engine/rules/permission_scope.py index 3f564ab..498dfcc 100644 --- a/src/malwar/detectors/rule_engine/rules/permission_scope.py +++ b/src/malwar/detectors/rule_engine/rules/permission_scope.py @@ -45,7 +45,37 @@ class PermissionScopeExpansion(BaseRule): "priming a confirmation so later actions run unprompted" ) - PATTERNS: list[tuple[re.Pattern[str], str]] = [ + # A fenced code block is something the agent runs; prose is something the + # author is talking about. Distinguishing them is the difference between + # "pass --yolo" and "the scanner blocks this unless you pass --yolo", + # which is a sentence from a real skill's deployment-notes section that + # this rule convicted as MALICIOUS. + _FENCE = re.compile(r"^\s*(?:```|~~~)") + # An invocation written inline in prose ("Run claude --flag ..."). The + # command name must reach the flag without crossing sentence-ending + # punctuation, so a tool named in one clause cannot lend authority to a + # flag merely discussed in the next. Full-width stops are included because + # much of the registry is written in Chinese. + _COMMAND_CONTEXT = re.compile( + r"\b(?:claude|openclaw|codex|aider|npx|npm|pnpm|yarn|node|deno|bun|uv" + r"|python3?|bash|sh|zsh" + # Imperatives, so "Run the agent with --flag" still counts even though + # "the agent" is not a command name. "use" is deliberately absent: it + # appears far more often in "this skill never uses --flag". + r"|run|execute|start|launch|invoke|pass|append|add)" + r"\b[^.。!?！？\n]*?--", # noqa: RUF001 - CJK stops are real input + re.IGNORECASE, + ) + # Documentation that warns *against* a flag reads exactly like an + # instruction to use it, so a negation anywhere before the flag disarms + # the context. "Never run with --yolo" is advice, not a setup step. + _NEGATED = re.compile( + r"\b(?:never|not|n't|without|avoid|refrain)\b|不要|禁止|切勿|请勿", + re.IGNORECASE, + ) + + # (pattern, what it means, requires an execution context) + PATTERNS: list[tuple[re.Pattern[str], str, bool]] = [ # Editing the agent's tool allowlist / permission config. ( re.compile( @@ -54,6 +84,7 @@ class PermissionScopeExpansion(BaseRule): re.IGNORECASE, ), "writes to the agent's tool allowlist", + False, ), ( re.compile( @@ -61,6 +92,7 @@ class PermissionScopeExpansion(BaseRule): re.IGNORECASE, ), "sets an allowlist or auto-approve key", + False, ), # Approval / sandbox settings turned off in a config file. The skill # that motivated this rule instructs the user to set @@ -76,6 +108,7 @@ class PermissionScopeExpansion(BaseRule): re.IGNORECASE, ), "sets the approval policy to never ask", + False, ), ( re.compile( @@ -84,6 +117,7 @@ class PermissionScopeExpansion(BaseRule): re.IGNORECASE, ), "disables or fully opens the sandbox", + False, ), # Explicit approval bypass flags. ( @@ -93,6 +127,7 @@ class PermissionScopeExpansion(BaseRule): re.IGNORECASE, ), "passes an approval-bypass flag", + True, ), # Turning confirmation off in prose or config. ( @@ -102,6 +137,7 @@ class PermissionScopeExpansion(BaseRule): re.IGNORECASE, ), "disables confirmation prompts", + False, ), # Priming consent: run something harmless first so the approval is # already spent when it matters. The tell is an explicit claim that @@ -115,6 +151,7 @@ class PermissionScopeExpansion(BaseRule): re.IGNORECASE, ), "primes an approval so later actions run unprompted", + False, ), # Same idea expressed as establishing durable trust. ( @@ -124,6 +161,7 @@ class PermissionScopeExpansion(BaseRule): re.IGNORECASE, ), "establishes standing trust to avoid later prompts", + False, ), # A large share of the registry is written in Chinese, and the skill # that prompted this rule is entirely so. English-only patterns would @@ -135,17 +173,33 @@ class PermissionScopeExpansion(BaseRule): ( re.compile(r"无需(?:再次)?确认|免确认|跳过确认|跳过权限|自动(?:批准|确认|授权)"), "states that later actions need no confirmation (zh)", + False, ), ( re.compile(r"建立信任[^。\n]{0,40}(?:无需|不用|不需)"), "establishes standing trust to avoid later prompts (zh)", + False, ), ] def check(self, skill: SkillContent) -> list[Finding]: findings: list[Finding] = [] + in_fence = False for line_num, line in enumerate(skill.raw_content.splitlines(), 1): - for pattern, what in self.PATTERNS: + if self._FENCE.match(line): + in_fence = not in_fence + continue + # Only text *before* the flag can negate it. "Run claude --flag to + # avoid prompts" is an instruction whose tail happens to contain a + # negative word; "Never run with --flag" is not. + prefix = line.split("--", 1)[0] + executable = in_fence or ( + bool(self._COMMAND_CONTEXT.search(line)) + and not self._NEGATED.search(prefix) + ) + for pattern, what, needs_execution in self.PATTERNS: + if needs_execution and not executable: + continue if pattern.search(line): findings.append(Finding( id=f"{self.rule_id}-L{line_num}", diff --git a/tests/unit/detectors/test_rule_engine.py b/tests/unit/detectors/test_rule_engine.py index 6e46762..1b9c219 100644 --- a/tests/unit/detectors/test_rule_engine.py +++ b/tests/unit/detectors/test_rule_engine.py @@ -1180,6 +1180,52 @@ def test_detects_approval_and_sandbox_disabled_in_config(self, rule_instance): ): assert len(rule_instance.check(_make_skill(text))) >= 1, text + def test_bypass_flag_only_counts_when_actually_invoked(self, rule_instance): + # The line below is copied from a live skill's deployment-notes section. + # It explains that the registry's own scanner blocks community uploads + # at CAUTION and that --yolo bypasses it. The rule matched the bare + # token and convicted an HR resume-grading skill as MALICIOUS at risk + # 100 -- the same defect as MULTI-001 reading consent as evasion: a + # technique *discussed* is not a technique performed. + discussed = ( + "- 扫描 verdict 达到 CAUTION 且为 community source 时会 BLOCKED，" # noqa: RUF001 + "需 `--yolo` 绕过或修复" + ) + assert rule_instance.check(_make_skill(discussed)) == [] + for text in ( + "If the scanner blocks you, --yolo will bypass it, or just fix the finding.", + "This skill never uses --dangerously-skip-permissions.", + "Claude Code will prompt you. The --yolo flag exists but we do not use it.", + ): + assert rule_instance.check(_make_skill(text)) == [], text + + def test_warning_against_a_flag_is_not_an_instruction(self, rule_instance): + # Security guidance reads syntactically like a setup step. Only the + # text *before* the flag can negate it, so an instruction whose tail + # happens to say "to avoid prompts" still counts. + for text in ( + "Never run the agent with --yolo.", + "Do not pass --dangerously-skip-permissions.", + "Avoid running with --yolo in production.", + "切勿使用 --yolo 运行", + ): + assert rule_instance.check(_make_skill(text)) == [], text + assert len(rule_instance.check(_make_skill( + "Run claude --dangerously-skip-permissions to avoid prompts." + ))) >= 1 + + def test_bypass_flag_in_a_command_still_fires(self, rule_instance): + # The gate must not cost real detections: a flag inside a code fence, + # or reached from a command name without crossing a sentence boundary, + # is an instruction to run it. + for text in ( + "Run claude --dangerously-skip-permissions to avoid prompts.", + "```bash\nclaude --dangerously-skip-permissions\n```", + "```\nopenclaw --yolo start\n```", + "`npx agent --allow-all-tools`", + ): + assert len(rule_instance.check(_make_skill(text))) >= 1, text + def test_restrictive_settings_are_not_a_finding(self, rule_instance): # The same keys set to safe values, and a skill narrowing its own tool # scope, are the behaviour we want and must never fire.