From ffba6c3a5d50858504598e4cbd410dd61139ab1d Mon Sep 17 00:00:00 2001 From: sunerpy Date: Tue, 14 Jul 2026 21:45:39 +0800 Subject: [PATCH] feat(prompt-hook): gate context injection by confidence tier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports the non-telemetry prompt-hook improvements from upstream colby 1.2.0..1.4.1. `codegraph prompt-hook` was a bare forwarder that ran codegraph_explore on every prompt; it now decides how much context to inject by a three-tier confidence gate, so ordinary chatter no longer triggers a full explore and non-code prompts stay silent. - HIGH — a structural question (multilingual gate covering ~29 languages, with inflections and multi-word phrases, and the call/trace/affect/connect stems bounded on the right so callus/Connecticut/affectionate don't fire) OR a graph-verified code token (inner camelCase, flanked underscore, call form name(, or member access x.name; a bare PascalCase word is not a token) → full explore in a block. - MEDIUM — plain prose words that match indexed symbol-name segments (one name covering >=2 prompt words, or a rare >=4-char segment shared across >=2 names) → a short pointer naming the symbols for the agent to query itself, never a full explore. - otherwise silent. The segment matching is derived at query time from the existing indexed node names, so no schema change is needed (CURRENT_SCHEMA_VERSION stays 7 and the golden .schema is byte-identical). The Claude UserPromptSubmit JSON payload {prompt,cwd} is now parsed (raw-string invocation still works), and CODEGRAPH_NO_PROMPT_HOOK / CODEGRAPH_PROMPT_HOOK=0 disable the hook. No telemetry, counters, or tracking of any kind are added. The only new dependency is unicode-normalization (NFD folding so accented prose matches ASCII identifier segments); regex was already present transitively. Golden fixtures and .schema unchanged (no regen). --- Cargo.lock | 45 +- crates/codegraph-cli/Cargo.toml | 2 + crates/codegraph-cli/src/main.rs | 156 ++++- crates/codegraph-cli/src/segment_match.rs | 192 +++++ crates/codegraph-cli/src/segments.rs | 484 +++++++++++++ crates/codegraph-cli/src/structural_gate.rs | 656 ++++++++++++++++++ crates/codegraph-cli/tests/cli_commands.rs | 22 +- .../prompt_hook_medium/src/checkout.ts | 24 + crates/codegraph-cli/tests/prompt_hook.rs | 317 ++++++++- crates/codegraph-mcp/src/engine.rs | 12 + crates/codegraph-store/src/queries.rs | 38 + docs/mcp.md | 37 + 12 files changed, 1916 insertions(+), 69 deletions(-) create mode 100644 crates/codegraph-cli/src/segment_match.rs create mode 100644 crates/codegraph-cli/src/segments.rs create mode 100644 crates/codegraph-cli/src/structural_gate.rs create mode 100644 crates/codegraph-cli/tests/fixtures/prompt_hook_medium/src/checkout.ts diff --git a/Cargo.lock b/Cargo.lock index e2d9ff19..acdc1245 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -432,7 +432,7 @@ dependencies = [ [[package]] name = "codegraph-bench" -version = "0.28.2" +version = "0.31.1" dependencies = [ "anyhow", "clap", @@ -447,7 +447,7 @@ dependencies = [ [[package]] name = "codegraph-core" -version = "0.28.2" +version = "0.31.1" dependencies = [ "anyhow", "serde", @@ -463,7 +463,7 @@ dependencies = [ [[package]] name = "codegraph-daemon" -version = "0.28.2" +version = "0.31.1" dependencies = [ "anyhow", "codegraph-core", @@ -483,7 +483,7 @@ dependencies = [ [[package]] name = "codegraph-extract" -version = "0.28.2" +version = "0.31.1" dependencies = [ "anyhow", "codegraph-core", @@ -525,7 +525,7 @@ dependencies = [ [[package]] name = "codegraph-graph" -version = "0.28.2" +version = "0.31.1" dependencies = [ "anyhow", "codegraph-core", @@ -539,7 +539,7 @@ dependencies = [ [[package]] name = "codegraph-mcp" -version = "0.28.2" +version = "0.31.1" dependencies = [ "anyhow", "axum", @@ -562,7 +562,7 @@ dependencies = [ [[package]] name = "codegraph-resolve" -version = "0.28.2" +version = "0.31.1" dependencies = [ "anyhow", "codegraph-core", @@ -576,7 +576,7 @@ dependencies = [ [[package]] name = "codegraph-rs" -version = "0.28.2" +version = "0.31.1" dependencies = [ "anyhow", "assert_cmd", @@ -597,6 +597,7 @@ dependencies = [ "jsonc-parser", "predicates", "rayon", + "regex", "rstest", "self_update", "serde", @@ -606,11 +607,12 @@ dependencies = [ "toml", "tracing", "tracing-subscriber", + "unicode-normalization", ] [[package]] name = "codegraph-store" -version = "0.28.2" +version = "0.31.1" dependencies = [ "anyhow", "codegraph-core", @@ -622,7 +624,7 @@ dependencies = [ [[package]] name = "codegraph-watch" -version = "0.28.2" +version = "0.31.1" dependencies = [ "anyhow", "codegraph-core", @@ -1024,7 +1026,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1939,7 +1941,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2184,7 +2186,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2536,7 +2538,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2595,7 +2597,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3016,7 +3018,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3713,6 +3715,15 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-width" version = "0.2.2" @@ -3994,7 +4005,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/codegraph-cli/Cargo.toml b/crates/codegraph-cli/Cargo.toml index 58f6ea59..1a3292c2 100644 --- a/crates/codegraph-cli/Cargo.toml +++ b/crates/codegraph-cli/Cargo.toml @@ -28,6 +28,8 @@ codegraph-watch = { path = "../codegraph-watch" } serde = { workspace = true } serde_json = { workspace = true } sha1_smol = { workspace = true } +regex = "1.10" +unicode-normalization = "0.1" jsonc-parser = { workspace = true } self_update = { workspace = true } toml = { workspace = true } diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index b64a43ff..e0ba5669 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -35,6 +35,9 @@ use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; mod installer; +mod segment_match; +mod segments; +mod structural_gate; const VERSION: &str = env!("CARGO_PKG_VERSION"); const EXTRACTION_VERSION: i64 = 1; @@ -970,45 +973,162 @@ fn cmd_self_update(check: bool, force: bool, tag: Option) -> Result<()> Ok(()) } +/// Claude Code `UserPromptSubmit` payload piped on stdin: `{"prompt", "cwd"}` +/// (upstream `bin/codegraph.ts:1199-1201`). Permissive: extra fields are +/// ignored and a non-JSON body is handled by the raw-string fallback in +/// [`cmd_prompt_hook`], so this never fails the hook. +#[derive(Debug, Default, serde::Deserialize)] +struct PromptHookPayload { + #[serde(default)] + prompt: Option, + #[serde(default)] + cwd: Option, +} + +/// Cap on the HIGH-tier explore injection so a large-repo explore can't flood +/// the prompt (upstream `MAX = 16000`). +const PROMPT_HOOK_MAX_INJECT: usize = 16000; + +/// `codegraph prompt-hook` — the Claude `UserPromptSubmit` hook entry point, +/// now a confidence-tiered gate (upstream #1126 + #1136, telemetry EXCLUDED): +/// +/// - HIGH — a structural keyword (any of ~29 covered languages) OR a code-shaped +/// token verified in the index → full `codegraph_explore` injection. +/// - MEDIUM — no keyword/token, but prose words match indexed symbol-name +/// segments → a short symbol-pointer hint; the agent writes the explore query. +/// - silent — nothing verified → zero-cost no-op. +/// +/// Input: an explicit `--query`/positional arg (raw-string path) wins; else +/// stdin is parsed as the `{prompt,cwd}` JSON payload, falling back to treating +/// the raw stdin as the literal query. Project resolution: payload `.cwd` → +/// `--path` → cwd. Honors the `CODEGRAPH_NO_PROMPT_HOOK`/`CODEGRAPH_PROMPT_HOOK` +/// kill-switch. Degradable by contract: every failure path exits 0 silently. fn cmd_prompt_hook(path: Option, query: Option) -> Result<()> { - let query = match query { - Some(q) if !q.trim().is_empty() => q, + if matches!(std::env::var("CODEGRAPH_NO_PROMPT_HOOK"), Ok(v) if v == "1") + || matches!(std::env::var("CODEGRAPH_PROMPT_HOOK"), Ok(v) if v == "0") + { + return Ok(()); + } + + // Resolve the query and the payload-supplied cwd. + let (query, payload_cwd) = match query { + Some(q) if !q.trim().is_empty() => (q, None), _ => { let mut buf = String::new(); io::stdin().read_to_string(&mut buf).ok(); - buf + match serde_json::from_str::(&buf) { + Ok(payload) => match payload.prompt { + Some(p) if !p.trim().is_empty() => (p, payload.cwd), + _ => (buf, None), + }, + Err(_) => (buf, None), + } } }; let query = query.trim(); if query.is_empty() { - println!("[codegraph] No query provided — nothing to explore."); return Ok(()); } - let start = absolute_path(path.unwrap_or_else(|| PathBuf::from("."))); + // Gate BEFORE opening the index: a prompt that clears none of the three + // tiers is a zero-cost no-op with no filesystem work. + let keyworded = structural_gate::has_structural_keyword(query); + let code_tokens = if keyworded { + Vec::new() + } else { + structural_gate::extract_code_tokens(query) + }; + let prose_words = if keyworded { + Vec::new() + } else { + segments::extract_prose_candidates(query) + }; + if !keyworded && code_tokens.is_empty() && prose_words.is_empty() { + return Ok(()); + } + + // Project resolution: payload .cwd → --path → cwd. + let start = match payload_cwd { + Some(c) if !c.trim().is_empty() => absolute_path(PathBuf::from(c)), + _ => absolute_path(path.unwrap_or_else(|| PathBuf::from("."))), + }; let project = resolve_project_path_optional(&start); if !is_initialized(&project) { - println!( - "[codegraph] No .codegraph index found near {} — run `codegraph init` to enable context.", - start.display() - ); return Ok(()); } let engine = match codegraph_mcp::CodeGraphEngine::open(&project) { Ok(engine) => engine, - Err(err) => { - println!( - "[codegraph] Could not open the index at {}: {err}", - project.display() - ); + Err(_) => return Ok(()), + }; + + // HIGH: structural keyword, or a code token verified as a real symbol here. + let token_verified = !keyworded + && code_tokens + .iter() + .any(|t| matches!(engine.store_nodes_by_name(t), Ok(nodes) if !nodes.is_empty())); + + if keyworded || token_verified { + let result = engine.execute("codegraph_explore", &json!({ "query": query })); + if result.is_error == Some(true) { return Ok(()); } - }; - let result = engine.execute("codegraph_explore", &json!({ "query": query })); - for content in &result.content { - println!("{}", content.text); + let text = result + .content + .iter() + .map(|c| c.text.as_str()) + .collect::>() + .join("\n"); + if text.trim().is_empty() { + return Ok(()); + } + let body = if text.len() > PROMPT_HOOK_MAX_INJECT { + let mut cut = PROMPT_HOOK_MAX_INJECT; + while !text.is_char_boundary(cut) { + cut -= 1; + } + format!( + "{}\n\u{2026}(truncated; call codegraph_explore for the rest)", + &text[..cut] + ) + } else { + text + }; + println!( + "\n{body}\n" + ); + return Ok(()); + } + + // MEDIUM: prose words → indexed symbol-name segments. Name the matching + // symbols and let the agent write the explore query — never run explore, so + // a fuzzy match can't inject a full explore of the wrong feature. + let related = segment_match::get_segment_matches(engine.store(), &prose_words, 6); + if related.is_empty() { + return Ok(()); } + let lines = related + .iter() + .map(|m| { + format!( + " - {} ({} \u{2014} {}:{})", + m.name, + m.kind.as_str(), + m.file_path, + m.start_line + ) + }) + .collect::>() + .join("\n"); + let example_query = related + .iter() + .take(3) + .map(|m| m.name.as_str()) + .collect::>() + .join(" "); + println!( + "\nThis project's CodeGraph index contains symbols matching this request:\n{lines}\nCall codegraph_explore ONCE with the relevant names in one query (e.g. \"{example_query}\") to get their source, call paths, and blast radius \u{2014} cheaper and more complete than Read/Grep.\n" + ); Ok(()) } diff --git a/crates/codegraph-cli/src/segment_match.rs b/crates/codegraph-cli/src/segment_match.rs new file mode 100644 index 00000000..6a1faf9a --- /dev/null +++ b/crates/codegraph-cli/src/segment_match.rs @@ -0,0 +1,192 @@ +//! Query-time graph-derived segment matching for the prompt-hook MEDIUM tier. +//! +//! Rust port of upstream `CodeGraph.getSegmentMatches` (`src/index.ts:908-1030`, +//! #1136 `e699ee9` + #1144/#1145/#1146 hardening from `35611b9`). Instead of +//! reading an index-time `name_segment_vocab` table (which would break this +//! project's golden `.schema` parity), the segment / co-occurrence / rarity map +//! is derived on demand from the already-indexed node names +//! ([`Store::distinct_non_file_node_names`]). The tier is a pure function of the +//! prompt prose words + the current node-name set, and its output is +//! deterministically ordered (matching this project's determinism invariant). + +use std::collections::BTreeMap; + +use codegraph_core::types::NodeKind; +use codegraph_store::Store; + +use crate::segments::{segment_lookup_variants, split_identifier_segments}; + +/// A single word matching hundreds of names in a big repo is noise, not signal: +/// the single-word (Tier B) rarity ceiling. Co-occurrence (Tier A) is exempt — +/// two words on one name is already discriminative. Mirrors upstream +/// `SEGMENT_RARITY_CEILING = 25`. +const SEGMENT_RARITY_CEILING: usize = 25; + +/// A symbol whose name-segments match prose words from a prompt — the +/// graph-derived signal behind the front-load hook's MEDIUM tier. Always +/// verified to exist in `nodes` at the time it is returned. Mirrors upstream +/// `SegmentMatch` (`types.ts`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SegmentMatch { + pub name: String, + pub kind: NodeKind, + pub file_path: String, + pub start_line: i64, + pub matched_words: Vec, +} + +/// Which indexed symbols do these prose `words` name? "state machine des +/// commandes" → `OrderStateMachine`, in any human language whose technical +/// nouns are Latin script — no keyword list involved. +/// +/// Precision comes from the repo's own naming statistics, not a vocabulary: +/// - CO-OCCURRENCE (Tier A): ≥2 DISTINCT prompt words that are segments of the +/// SAME name is strong evidence and always qualifies. Variants of one word +/// are folded back to that word (#1146), so a plural pair can't tie a genuine +/// two-word match. +/// - RARITY (Tier B, only if Tier A empty): a single matched word qualifies +/// only when it is ≥5 chars, its segment clusters across ≥2 and +/// ≤[`SEGMENT_RARITY_CEILING`] distinct names, and the candidate name has ≥2 +/// segments. +/// +/// Every candidate is re-verified against `nodes` (via [`Store::nodes_by_name`]) +/// and a non-file/non-import representative is picked (#1144), so a returned +/// symbol is guaranteed to exist right now. The result is sorted by coverage +/// (desc) then name length (asc) then name (asc) for determinism. +pub fn get_segment_matches(store: &Store, words: &[String], limit: usize) -> Vec { + if words.is_empty() { + return Vec::new(); + } + + // variant → original word (plural folding), for coverage accounting. First + // writer wins so a variant maps to a single stable original word. + let mut variant_to_word: BTreeMap = BTreeMap::new(); + for word in words { + for variant in segment_lookup_variants(word) { + variant_to_word + .entry(variant) + .or_insert_with(|| word.clone()); + } + } + + let names = match store.distinct_non_file_node_names() { + Ok(names) => names, + Err(_) => return Vec::new(), + }; + + // segment → set of names (deterministic insertion via the sorted name feed); + // name → its segment count (for the Tier-B ≥2-segment rule). + let mut segment_to_names: BTreeMap> = BTreeMap::new(); + let mut name_segment_count: BTreeMap = BTreeMap::new(); + for (name, _kind) in &names { + let segs = split_identifier_segments(name); + name_segment_count.insert(name.clone(), segs.len()); + for seg in segs { + let entry = segment_to_names.entry(seg).or_default(); + if !entry.contains(name) { + entry.push(name.clone()); + } + } + } + + // Candidates: (name, matched original words). + let mut candidates: Vec<(String, Vec)> = Vec::new(); + + // Tier A: co-occurrence — a name covering ≥2 DISTINCT prompt words. + for name in name_segment_count.keys() { + let matched = words_matching_name(name, &variant_to_word); + if matched.len() >= 2 { + candidates.push((name.clone(), matched)); + } + } + + // Tier B: single rare word — only if co-occurrence found nothing. + if candidates.is_empty() { + let single_word_variants: Vec<&String> = variant_to_word + .keys() + .filter(|v| { + variant_to_word + .get(*v) + .is_some_and(|w| w.chars().count() >= 5) + }) + .collect(); + + // (variant, distinct-name count) for the eligible variants. + let mut rare: Vec<(String, usize)> = single_word_variants + .iter() + .filter_map(|v| segment_to_names.get(*v).map(|ns| ((*v).clone(), ns.len()))) + .filter(|(_, n)| *n >= 2 && *n <= SEGMENT_RARITY_CEILING) + .collect(); + // Rarest first, then variant name for a stable tie-break; take 2. + rare.sort_by(|a, b| a.1.cmp(&b.1).then_with(|| a.0.cmp(&b.0))); + rare.truncate(2); + + for (variant, _count) in rare { + let word = variant_to_word.get(&variant).cloned().unwrap_or_default(); + if let Some(candidate_names) = segment_to_names.get(&variant) { + // Names for a segment: shortest name first, capped at 12 (upstream). + let mut sorted = candidate_names.clone(); + sorted.sort_by(|a, b| a.len().cmp(&b.len()).then_with(|| a.cmp(b))); + for name in sorted.into_iter().take(12) { + if name_segment_count.get(&name).copied().unwrap_or(0) < 2 { + continue; + } + candidates.push((name, vec![word.clone()])); + } + } + } + } + + // Sort by coverage (desc), then name length (asc), then name (asc). + candidates.sort_by(|a, b| { + b.1.len() + .cmp(&a.1.len()) + .then_with(|| a.0.len().cmp(&b.0.len())) + .then_with(|| a.0.cmp(&b.0)) + }); + + // Verify against nodes (the honesty gate) and pick a representative. + let mut out: Vec = Vec::new(); + let mut seen: Vec = Vec::new(); + for (name, matched) in candidates { + if out.len() >= limit { + break; + } + if seen.contains(&name) { + continue; + } + seen.push(name.clone()); + let nodes = match store.nodes_by_name(&name) { + Ok(nodes) => nodes, + Err(_) => continue, + }; + let rep = nodes + .iter() + .find(|n| n.kind != NodeKind::File && n.kind != NodeKind::Import); + let Some(rep) = rep else { + continue; + }; + let mut matched_sorted = matched; + matched_sorted.sort(); + out.push(SegmentMatch { + name, + kind: rep.kind, + file_path: rep.file_path.clone(), + start_line: rep.start_line, + matched_words: matched_sorted, + }); + } + out +} + +/// Which of the prompt's original words match `name`'s segments (via variants). +fn words_matching_name(name: &str, variant_to_word: &BTreeMap) -> Vec { + let segments = split_identifier_segments(name); + let mut matched: Vec = Vec::new(); + for (variant, word) in variant_to_word { + if segments.iter().any(|s| s == variant) && !matched.contains(word) { + matched.push(word.clone()); + } + } + matched +} diff --git a/crates/codegraph-cli/src/segments.rs b/crates/codegraph-cli/src/segments.rs new file mode 100644 index 00000000..0e361cfd --- /dev/null +++ b/crates/codegraph-cli/src/segments.rs @@ -0,0 +1,484 @@ +//! Identifier-segment utilities for the prompt hook's graph-derived MEDIUM tier. +//! +//! Rust port of the PURE functions from upstream `src/search/identifier-segments.ts` +//! (#1136 `e699ee9` + #1145 plural-folding correctness from `35611b9`). NO +//! DB table: this project derives the MEDIUM-tier matches at QUERY time from the +//! existing indexed node names, so the golden `.schema` stays byte-identical +//! (no `name_segment_vocab` table, no migration). +//! +//! Symbol names are split into the words a human would use for them in prose, +//! and prompt prose is normalized into candidate words to look those segments +//! up with. "OrderStateMachine" → order / state / machine — so a prose prompt +//! naming the concept can be verified against the graph without a keyword list +//! ever knowing the words. + +use std::collections::BTreeSet; + +use regex::Regex; +use std::sync::LazyLock; +use unicode_normalization::UnicodeNormalization; + +/// Bounds keep degenerate identifiers (minified names, hashes) from bloating +/// the segment set: segments outside them carry no prose signal anyway. +const MIN_SEGMENT_CHARS: usize = 2; +const MAX_SEGMENT_CHARS: usize = 32; +const MAX_SEGMENTS_PER_NAME: usize = 12; + +/// Candidate cap + prose-word length bounds. +const MAX_PROSE_CANDIDATES: usize = 16; +const MIN_PROSE_CHARS: usize = 4; // "the"/"des"/"une"/"fix" out; "auth"/"flow"/"path" in +const MAX_PROSE_CHARS: usize = 24; // an unsegmented-script sentence is one giant run — skip it + +/// Letter/digit run (Unicode). +static WORD_RUN_RE: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{L}\p{N}]+").expect("word-run regex is valid")); + +/// Digit-only check. +static ALL_DIGITS_RE: LazyLock = + LazyLock::new(|| Regex::new(r"^\p{N}+$").expect("all-digits regex is valid")); + +/// Split a symbol or file name into lowercase word segments. +/// +/// Handles camelCase / PascalCase (inner lower→Upper), acronym runs +/// ("HTMLParser" → html/parser), snake_case / kebab-case / dotted file names +/// (non-alphanumerics separate), and keeps digits glued to their word +/// ("base64Encode" → base64/encode). Digit-only fragments are dropped. +/// +/// Upstream uses lookbehind/lookahead splits +/// (`(?<=[\p{Ll}\p{N}])(?=\p{Lu})|(?<=\p{Lu})(?=\p{Lu}\p{Ll})`). Rust `regex` +/// has no lookaround, so the same boundaries are computed by a manual +/// char-class scan over each `[\p{L}\p{N}]+` run. +pub fn split_identifier_segments(name: &str) -> Vec { + if name.is_empty() { + return Vec::new(); + } + let mut out: Vec = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + for run_match in WORD_RUN_RE.find_iter(name) { + for part in split_run(run_match.as_str()) { + if out.len() >= MAX_SEGMENTS_PER_NAME { + return out; + } + let seg = part.to_lowercase(); + let len = seg.chars().count(); + if !(MIN_SEGMENT_CHARS..=MAX_SEGMENT_CHARS).contains(&len) { + continue; + } + if ALL_DIGITS_RE.is_match(&seg) { + continue; + } + if seen.insert(seg.clone()) { + out.push(seg); + } + } + } + out +} + +/// Split one alphanumeric run at camelCase humps and acronym→word boundaries, +/// emulating upstream's two lookaround alternatives: +/// - `(?<=[\p{Ll}\p{N}])(?=\p{Lu})` — a lowercase/digit followed by an uppercase +/// (camelCase hump: `orderState` → order|State). +/// - `(?<=\p{Lu})(?=\p{Lu}\p{Ll})` — an uppercase followed by uppercase+lowercase +/// (last acronym letter starts a new word: `HTMLParser` → HTML|Parser). +fn split_run(run: &str) -> Vec { + let chars: Vec = run.chars().collect(); + if chars.is_empty() { + return Vec::new(); + } + let mut parts: Vec = Vec::new(); + let mut current = String::new(); + for i in 0..chars.len() { + if i > 0 { + let prev = chars[i - 1]; + let cur = chars[i]; + let hump = (prev.is_lowercase() || prev.is_numeric()) && cur.is_uppercase(); + let acronym = prev.is_uppercase() + && cur.is_uppercase() + && chars.get(i + 1).is_some_and(|n| n.is_lowercase()); + if hump || acronym { + parts.push(std::mem::take(&mut current)); + } + } + current.push(chars[i]); + } + if !current.is_empty() { + parts.push(current); + } + parts +} + +/// Combining-mark stripper (regex `\p{M}+`) — applied after NFD. +static COMBINING_MARK_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\p{M}+").expect("combining-mark regex is valid")); + +/// Normalize a prose word for segment lookup: NFD + strip combining marks + +/// lowercase, so "références" matches the segment "references" and "résolution" +/// matches "resolution". Identifier segments are overwhelmingly ASCII, so this +/// is what buys Latin-script languages their cross-lingual reach on loanwords. +pub fn normalize_prose_word(word: &str) -> String { + let decomposed: String = word.nfd().collect(); + let stripped = COMBINING_MARK_RE.replace_all(&decomposed, ""); + stripped.to_lowercase() +} + +/// English prompt words that are never evidence a symbol was NAMED, however +/// rare their segment happens to be in a given repo: function words, filler, +/// hyper-common dev verbs, and words ABOUT code rather than OF it. Measured FPs +/// that motivated this: "fix THIS typo" matched `resolveDeferredThisMemberRefs`, +/// "WRITE a haiku" matched `writeConfig`. +/// +/// English-only ON PURPOSE — identifiers are written in English, so only English +/// prose words can accidentally collide with segments. Domain nouns ("state", +/// "checkout", "order") stay OUT — they are exactly the signal. Verbatim from +/// `identifier-segments.ts` `ENGLISH_PROSE_STOPWORDS`. +const ENGLISH_PROSE_STOPWORDS: &[&str] = &[ + "about", + "above", + "actually", + "after", + "again", + "against", + "almost", + "along", + "also", + "always", + "another", + "anything", + "around", + "away", + "back", + "because", + "been", + "before", + "behind", + "being", + "below", + "best", + "better", + "between", + "both", + "cannot", + "come", + "could", + "does", + "doing", + "done", + "down", + "each", + "either", + "else", + "even", + "ever", + "every", + "everything", + "fine", + "first", + "from", + "getting", + "give", + "goes", + "going", + "gone", + "good", + "great", + "have", + "having", + "help", + "here", + "inside", + "instead", + "into", + "just", + "keep", + "know", + "last", + "least", + "less", + "like", + "likely", + "little", + "look", + "looking", + "made", + "make", + "making", + "many", + "maybe", + "mind", + "more", + "most", + "much", + "must", + "need", + "needs", + "never", + "next", + "nice", + "none", + "nothing", + "okay", + "only", + "onto", + "other", + "otherwise", + "over", + "please", + "pretty", + "probably", + "quite", + "rather", + "really", + "right", + "same", + "seem", + "seems", + "should", + "show", + "since", + "some", + "someone", + "something", + "somewhere", + "soon", + "still", + "such", + "sure", + "take", + "than", + "thank", + "thanks", + "that", + "their", + "them", + "then", + "there", + "these", + "they", + "thing", + "things", + "think", + "this", + "those", + "though", + "tried", + "tries", + "trying", + "under", + "until", + "upon", + "very", + "want", + "wants", + "well", + "went", + "were", + "what", + "when", + "which", + "while", + "will", + "wish", + "with", + "within", + "without", + "would", + "wrong", + "your", + "yours", + // words ABOUT code, not OF it — present in a huge share of prompts while + // almost never naming the symbol the user means + "again", + "change", + "changes", + "check", + "class", + "classes", + "code", + "detail", + "details", + "directory", + "error", + "errors", + "example", + "examples", + "file", + "files", + "folder", + "function", + "functions", + "issue", + "issues", + "line", + "lines", + "method", + "methods", + "name", + "names", + "problem", + "problems", + "project", + "question", + "questions", + "rename", + "test", + "tests", + "type", + "types", + "update", + "value", + "values", + "warning", + "warnings", + "work", + "working", + "write", + "writing", +]; + +fn is_stopword(word: &str) -> bool { + ENGLISH_PROSE_STOPWORDS.contains(&word) +} + +/// Candidate words from a prompt for segment lookup, in order of appearance: +/// Unicode letter/digit runs, normalized via [`normalize_prose_word`], +/// length-bounded, digit-only dropped, stopwords dropped, deduped, capped. +pub fn extract_prose_candidates(prompt: &str) -> Vec { + if prompt.is_empty() { + return Vec::new(); + } + let mut out: Vec = Vec::new(); + let mut seen: BTreeSet = BTreeSet::new(); + for run_match in WORD_RUN_RE.find_iter(prompt) { + if out.len() >= MAX_PROSE_CANDIDATES { + break; + } + let run = run_match.as_str(); + if run.chars().count() > MAX_PROSE_CHARS { + continue; + } + let w = normalize_prose_word(run); + let len = w.chars().count(); + if !(MIN_PROSE_CHARS..=MAX_PROSE_CHARS).contains(&len) { + continue; + } + if ALL_DIGITS_RE.is_match(&w) { + continue; + } + if is_stopword(&w) { + continue; + } + if seen.insert(w.clone()) { + out.push(w); + } + } + out +} + +/// Lookup variants for a prose word: the word itself plus light English-plural +/// folding, so common plurals still hit their singular segment. Keyed on +/// English plural spelling (#1145), in three classes: +/// - UNAMBIGUOUS `-es` (after x/sh/ss/zz: boxes, hashes, classes, quizzes) — +/// strip 2 only. Stripping 1 minted a bogus sibling ("classes" → classe). +/// - AMBIGUOUS endings (`-ches`/`-ses`/`-zes`/`-oes`): spelling alone can't +/// split patches(+es) from caches(+s) — emit BOTH candidate keys. +/// - Everything else ending in `-s` — a bare `-s` plural (services, machines): +/// strip 1 only. Stripping 2 minted "services" → servic. +/// +/// A trailing `-ss` is a singular (class, process), not a plural: no strip. +pub fn segment_lookup_variants(word: &str) -> Vec { + let mut variants = vec![word.to_string()]; + let len = word.chars().count(); + let can_strip2 = len >= MIN_PROSE_CHARS + 2; + let can_strip1 = len > MIN_PROSE_CHARS; + // Byte-slice is safe: these suffixes are ASCII, so trailing 1/2 chars are + // single-byte; guard on char count above avoids over-stripping short words. + let strip = |n: usize| -> String { + let mut chars: Vec = word.chars().collect(); + chars.truncate(chars.len().saturating_sub(n)); + chars.into_iter().collect() + }; + if ends_with_any(word, &["xes", "shes", "sses", "zzes"]) { + if can_strip2 { + variants.push(strip(2)); + } + } else if ends_with_any(word, &["ches", "ses", "zes", "oes"]) { + if can_strip2 { + variants.push(strip(2)); + } + if can_strip1 { + variants.push(strip(1)); + } + } else if word.ends_with('s') && !word.ends_with("ss") && can_strip1 { + variants.push(strip(1)); + } + variants +} + +fn ends_with_any(word: &str, suffixes: &[&str]) -> bool { + suffixes.iter().any(|s| word.ends_with(s)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn split_camel_and_acronym() { + assert_eq!( + split_identifier_segments("OrderStateMachine"), + vec!["order", "state", "machine"] + ); + assert_eq!( + split_identifier_segments("HTMLParser"), + vec!["html", "parser"] + ); + assert_eq!(split_identifier_segments("get_user"), vec!["get", "user"]); + assert_eq!( + split_identifier_segments("base64Encode"), + vec!["base64", "encode"] + ); + } + + #[test] + fn plural_folding_correctness() { + // bare-s plural folds to singular, not -es sibling + assert!(segment_lookup_variants("services").contains(&"service".to_string())); + assert!(!segment_lookup_variants("services").contains(&"servic".to_string())); + // unambiguous sibilant -es: strip 2 only, no bogus -s sibling + assert!(segment_lookup_variants("classes").contains(&"class".to_string())); + assert!(!segment_lookup_variants("classes").contains(&"classe".to_string())); + // trailing -ss singular: no strip + let class = segment_lookup_variants("class"); + assert_eq!(class, vec!["class".to_string()]); + assert!(!class.contains(&"clas".to_string())); + // ambiguous -ses: both keys + let dbs = segment_lookup_variants("databases"); + assert!(dbs.contains(&"database".to_string())); + assert!(dbs.contains(&"databases".to_string()) || dbs.contains(&"databas".to_string())); + } + + #[test] + fn prose_candidates_drop_stopwords() { + let cands = extract_prose_candidates("fix this typo"); + // "fix" < 4 chars, "this" is a stopword, "typo" survives + assert!(!cands.contains(&"this".to_string())); + assert!(!cands.contains(&"fix".to_string())); + assert!(cands.contains(&"typo".to_string())); + } + + #[test] + fn prose_candidates_normalize_diacritics() { + let cands = extract_prose_candidates("références resolution"); + assert!(cands.contains(&"references".to_string())); + assert!(cands.contains(&"resolution".to_string())); + } + + #[test] + fn prose_candidates_keep_domain_words() { + let cands = extract_prose_candidates("checkout state machine"); + assert!(cands.contains(&"checkout".to_string())); + assert!(cands.contains(&"state".to_string())); + assert!(cands.contains(&"machine".to_string())); + } +} diff --git a/crates/codegraph-cli/src/structural_gate.rs b/crates/codegraph-cli/src/structural_gate.rs new file mode 100644 index 00000000..e884c3f5 --- /dev/null +++ b/crates/codegraph-cli/src/structural_gate.rs @@ -0,0 +1,656 @@ +//! Multilingual structural-question gate for the `prompt-hook` subcommand. +//! +//! Rust port of upstream `src/directory.ts` (#1126 `317e7f4` multilingual gate + +//! #1138 `713ab7a` right-bound `call`/`trace`/`affect`/`connect` stems), and +//! `extractCodeTokens` (`src/directory.ts:479-497`). The gate decides whether a +//! user prompt is a structural / flow / impact / "where-how" question worth +//! front-loading CodeGraph context for — the cheap, graph-free candidate check. +//! +//! ## Boundary emulation (no lookaround) +//! +//! Upstream expresses word boundaries with Unicode lookaround +//! (`(?> = LazyLock::new(|| { + let mut v: Vec = Vec::new(); + // English + the Latin-script languages that share the spelling (French + // architecture/structure/trace/impact, Spanish depende/implementa/impacto, …). + // call/trace/affect/connect are NOT safe as open prefixes — callus, + // calligraphy, Connecticut, connective, affectionate, Tracey are ordinary + // words that would false-fire the full-explore tier (#1138) — so they carry + // an enumerated suffix set + right boundary. "tracing" lives in + // STRUCTURAL_WORDS (the e is dropped, so no trace-prefix form matches it). + for s in [ + "architect", + "structur", + "depend", + "implement", + "impact", + "explain", + ] { + v.push(s.to_string()); + } + v.push(format!( + "call(?:s|ing|ed|ers?|backs?|able|sites?)?{NOT_WORD_AFTER}" + )); + v.push(format!("trace(?:s|d|rs?)?{NOT_WORD_AFTER}")); + v.push(format!("affect(?:s|ed|ing)?{NOT_WORD_AFTER}")); + v.push(format!( + "connect(?:s|ed|ing|ions?|ors?|ivity)?{NOT_WORD_AFTER}" + )); + for s in STRUCTURAL_STEMS_TAIL { + v.push((*s).to_string()); + } + v +}); + +/// The open-prefix stems for all non-English languages (verbatim from +/// `directory.ts:334-406`); appended after the bounded English entries. +const STRUCTURAL_STEMS_TAIL: &[&str] = &[ + // French (appel(le)=call, dépend=depends, implément(e)=implement, + // connex(ion)=connection, expliqu(e)=explain, fonctionn(e/ement)=works) + "appel", + "dépend", + "implément", + "connex", + "expliqu", + "fonctionn", + // Spanish (llamad(a)=call, afect(a)=affect, conect(a)/conexi(ón)=connect, + // arquitec(tura)=architecture, estructur(a)=structure, funcion(a)=works, + // traza(r)=trace, explica=explain) + "llamad", + "afect", + "conect", + "conexi", + "arquitec", + "estructur", + "funcion", + "traza", + "explica", + // Portuguese (chama(da)=call, afeta=affect, arquitet(ura)=architecture, + // estrutur(a)=structure, quebra(do)=breaks) + "chama", + "afeta", + "arquitet", + "estrutur", + "quebra", + // German (abhäng(t)=depend, Auswirkung=impact, beeinfluss(t)=affect, + // verbind(et)=connect, Architektur, Struktur, funktionier(t)=works, + // Aufruf/aufgerufen=call, erklär(t)=explain, verfolg(en)=trace) + "abhäng", + "auswirkung", + "beeinfluss", + "verbind", + "architekt", + "struktur", + "funktionier", + "aufruf", + "aufgerufen", + "erklär", + "verfolg", + // Italian (chiam(a/ata)=call, dipend(e/enza)=depend, impatt(o)=impact, + // connett(e)/conness(ione)=connect, architett(ura), struttur(a), + // funzion(a/amento)=works, tracci(a)=trace, spiega(mi)=explain) + "chiam", + "dipend", + "impatt", + "connett", + "conness", + "architett", + "struttur", + "funzion", + "tracci", + "spiega", + // Russian (вызыва(ет)=calls, завис(ит)=depends, влия(ет)=affects, + // реализ(ация)=implementation, структур(а), архитектур(а), + // трассир(овка)=trace, лома(ет)=breaks, объясн(и)=explain, поток=flow) + "вызыва", + "завис", + "влия", + "реализ", + "структур", + "архитектур", + "трассир", + "лома", + "объясн", + "поток", + // Ukrainian — і/и spellings diverge from Russian (виклика(є)=calls, + // залеж(ить)=depends, вплива(є)=affects, архітектур(а), реаліз(ація), + // поясн(и)=explain, шлях(у)=path; структур(а) is shared with Russian) + "виклика", + "залеж", + "вплива", + "архітектур", + "реаліз", + "поясн", + "шлях", + // Dutch (aanroep(en)=call, afhankelijk(heid)=depends, beïnvloed(t)=affects, + // structuur — "structur" can't reach the uu; uitleg(gen)=explain) + "aanroep", + "afhankelijk", + "beïnvloed", + "structuur", + "uitleg", + // Polish (wywoł(uje)=calls, zależ(y)=depends, wpływ(a)=affects/impact, + // przepływ=flow, ścieżk(a)=path, działa(nie)=works, wyjaśni(j)=explain, + // śledz(enie)=trace; architektura/struktura fire via the German stems) + "wywoł", + "zależ", + "wpływ", + "przepływ", + "ścieżk", + "działa", + "wyjaśni", + "śledz", + // Czech (volá(ní)=calls, závis(í)=depends, ovlivň(uje)=affects, + // funguj(e)=works, vysvětl(i)=explain) + "volá", + "závis", + "ovlivň", + "funguj", + "vysvětl", + // Romanian (apel(ează)=calls, depind(e)=depends — i not e, so "depend" misses + // it; arhitectur(a) — no c; funcțion(ează)=works, explică=explain) + "apel", + "depind", + "arhitectur", + "funcțion", + "explică", + // Hungarian (hív(ja)=calls, függ(őség)=depends, működ(ik)=works, + // struktúr(a) — ú escapes "struktur"; magyaráz(d)=explain; + // architektúra fires via the German stem) + "hív", + "függ", + "működ", + "struktúr", + "magyaráz", + // Turkish — agglutinative, so stems beat exact words (nere(de/ye/den)=where, + // çağır/çağrı=call, bağıml(ı)=depends, bağlant(ı)=connection, akış(ı)=flow, + // etkile(r)/etkisi=affects/impact) + "nere", + "çağır", + "çağrı", + "bağıml", + "bağlant", + "akış", + "etkile", + "etkisi", + // Indonesian/Malay — me-/di-/ber- prefixes block a bare stem, so affixed + // forms are listed too (panggil(an)/memanggil/dipanggil=call, + // bergantung/tergantung=depends, pengaruh/mempengaruhi/memengaruhi=affect, + // arsitektur=architecture, fungsi/berfungsi=works, + // jelaskan/menjelaskan=explain) + "panggil", + "memanggil", + "dipanggil", + "bergantung", + "tergantung", + "pengaruh", + "mempengaruhi", + "memengaruhi", + "arsitektur", + "fungsi", + "berfungsi", + "jelaskan", + "menjelaskan", + // Swedish / Danish / Norwegian (anrop(ar)=calls, påverk(ar)/påvirk(er)=affects, + // afhæng(er)/avheng(er)=depends, förklar(a)/forklar=explain, + // arkitektur — k not ch; funger(ar/er)=works) + "anrop", + "påverk", + "påvirk", + "afhæng", + "avheng", + "förklar", + "forklar", + "arkitektur", + "funger", + // Finnish (kutsu(u)=calls, riippu(u)=depends, arkkitehtuur(i), + // rakente(en)=structure, selit(ä)=explain) + "kutsu", + "riippu", + "arkkitehtuur", + "rakente", + "selit", + // Greek — accented and unaccented stem spellings both occur + // (εξαρτ(άται)=depends, επηρε(άζει)=affects, αρχιτεκτονικ(ή), + // διαδρομ(ή)=path, εξηγ/εξήγ(ησε)=explain) + "εξαρτ", + "επηρε", + "αρχιτεκτονικ", + "διαδρομ", + "εξηγ", + "εξήγ", + // Hindi (समझा(ओ/इए)=explain, आर्किटेक्चर=architecture) + "समझा", + "आर्किटेक्चर", +]; + +/// Structural keywords matched as bare SUBSTRINGS, for languages where a +/// boundary can't be relied on: scripts with no word separators (Chinese — +/// simplified AND traditional; the original #994 set was simplified-only — +/// Japanese, Thai), Korean (spaced, but particles attach directly to the noun: +/// 구조가/구조를), and Arabic / Farsi / Hebrew (spaced, but proclitics attach to +/// the word: وكيف "and-how", והמבנה "and-the-structure"). JS's `\b` can never +/// fire between Han characters, which was issue #994. +/// +/// KNOWN, ACCEPTED false-positive class (#1140): substring matching cannot see +/// homograph compounds — Korean 구조 (structure) also fires inside 구조대 +/// (rescue squad). Verified unfixable at this layer: ICU word segmentation +/// returns 구조대 and the particle form 구조가 (which the gate MUST keep +/// matching) as equally opaque single segments, and a 구조대 denylist would +/// break 구조대로 ("according to the structure"), a legitimate structural +/// prompt. The miss rate this design avoids (silently no-op'ing every prompt in +/// these languages, #994) outweighs the occasional off-domain fire. +/// Verbatim from `directory.ts:405-406`. +const STRUCTURAL_UNSEGMENTED: &str = "如何|怎么|怎麼|在哪|哪里|哪裡|追踪|跟踪|追蹤|追跡|トレース|流程|流向|流れ|路径|路徑|経路|调用|調用|呼び出|依赖|依賴|依存|影响|影響|实现|實現|実装|架构|架構|アーキテクチャ|结构|結構|構造|介绍|介紹|解析|分析|原理|机制|機制|仕組み|説明|動作|どうやって|どのように|어떻게|어디|호출|흐름|경로|의존|영향|구현|구조|아키텍처|추적|동작|작동|설명|كيف|أين|اين|يستدعي|استدعاء|يعتمد|تعتمد|يؤثر|تأثير|معماري|بنية|هيكل|تدفق|مسار|تتبع|يعمل|تعمل|اشرح|شرح|چگونه|چطور|کجا|فراخوان|وابسته|تأثیر|معماری|ساختار|مسیر|توضیح|איך|איפה|קורא|תלוי|משפיע|ארכיטקטור|מבנה|זרימה|נתיב|הסבר|อย่างไร|ยังไง|ที่ไหน|เรียกใช้|ขึ้นอยู่กับ|ผลกระทบ|สถาปัตยกรรม|โครงสร้าง|เส้นทาง|ติดตาม|ทำงาน|อธิบาย"; + +/// Doc/data/asset file extensions — a `name.ext` of this kind is a file +/// reference, not a code symbol, so it must not trip the member-access signal. +/// Verbatim from `directory.ts` `DOC_DATA_EXT`. +static DOC_DATA_EXT: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?i)\.(md|markdown|txt|rst|json|ya?ml|toml|lock|csv|tsv|log|ini|cfg|conf|env|xml|html?|png|jpe?g|gif|svg|pdf)$", + ) + .expect("DOC_DATA_EXT regex is valid") +}); + +/// WORDS matched with a boundary on BOTH sides (lookaround-free): a leading +/// `(?:^|[^\p{L}\p{N}_])` and a trailing `(?:[^\p{L}\p{N}_]|$)` consume the +/// flanking non-word char (or match the string edge). +static STRUCTURAL_WORDS_RE: LazyLock = LazyLock::new(|| { + let body = STRUCTURAL_WORDS.join("|"); + Regex::new(&format!( + r"(?i)(?:^|[^\p{{L}}\p{{N}}_])(?:{body})(?:[^\p{{L}}\p{{N}}_]|$)" + )) + .expect("STRUCTURAL_WORDS regex is valid") +}); + +/// STEMS matched as word PREFIXES (left boundary only). The four #1138 English +/// entries carry their own trailing boundary inside the alternation. +static STRUCTURAL_STEMS_RE: LazyLock = LazyLock::new(|| { + let body = STRUCTURAL_STEMS.join("|"); + Regex::new(&format!(r"(?i)(?:^|[^\p{{L}}\p{{N}}_])(?:{body})")) + .expect("STRUCTURAL_STEMS regex is valid") +}); + +/// UNSEGMENTED — a plain alternation with NO boundaries (separator-less / +/// proclitic scripts), exactly as upstream matches them without `\b`. +static STRUCTURAL_UNSEGMENTED_RE: LazyLock = LazyLock::new(|| { + Regex::new(STRUCTURAL_UNSEGMENTED).expect("STRUCTURAL_UNSEGMENTED regex is valid") +}); + +/// Does `prompt` contain an explicit structural keyword? A keyword is a strong, +/// self-contained signal, so the front-load hook fires on it directly — no +/// graph check needed. (A *code-token* match, by contrast, is only a candidate +/// the hook verifies against the graph first; see [`extract_code_tokens`].) +/// Coverage is multilingual (#994, #1126): the ~29 languages with the largest +/// developer populations, across Latin, Cyrillic, Greek, CJK, Hangul, Arabic, +/// Hebrew, Thai, and Devanagari scripts. Mirrors upstream `hasStructuralKeyword`. +pub fn has_structural_keyword(prompt: &str) -> bool { + !prompt.is_empty() + && (STRUCTURAL_WORDS_RE.is_match(prompt) + || STRUCTURAL_STEMS_RE.is_match(prompt) + || STRUCTURAL_UNSEGMENTED_RE.is_match(prompt)) +} + +/// Identifier-run regex: `[A-Za-z_$][\w$]*` (a token starting with a letter, +/// underscore, or `$`, then word chars or `$`). +static IDENT_RUN_RE: LazyLock = + LazyLock::new(|| Regex::new(r"[A-Za-z_$][\w$]*").expect("ident-run regex is valid")); + +/// Inner camelCase transition `[a-z][A-Z]`. +static INNER_CAMEL_RE: LazyLock = + LazyLock::new(|| Regex::new(r"[a-z][A-Z]").expect("inner-camel regex is valid")); + +/// Flanked underscore `[A-Za-z0-9]_[A-Za-z0-9]`. +static FLANKED_UNDERSCORE_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"[A-Za-z0-9]_[A-Za-z0-9]").expect("flanked-underscore regex is valid") +}); + +/// Call form: an identifier directly before `(`. +static CALL_FORM_RE: LazyLock = + LazyLock::new(|| Regex::new(r"([A-Za-z_$][\w$]*)\(").expect("call-form regex is valid")); + +/// Member access on identifiers: `a.b`. +static MEMBER_ACCESS_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"([A-Za-z_$][\w$]*)\.([A-Za-z_$][\w$]*)").expect("member-access regex is valid") +}); + +/// Identifier-shaped tokens in `prompt` — camelCase / PascalCase-with-inner-cap, +/// snake_case, a `name(` call, or the two sides of an `a.b` member access. +/// Naming a symbol is a code question whatever the surrounding human language. +/// +/// These are *candidates*, not a verdict: a tech brand like `JavaScript` or +/// `GitHub` is identifier-shaped too, so the front-load hook checks each token +/// against the actual index and only fires when one is a real symbol here. A +/// doc/data filename ("README.md") is excluded from the member-access form. +/// +/// A **bare single PascalCase word** like `Counter` matches NONE of these — no +/// inner `lower→Upper`, no underscore, no `(`, no `.` — so it is NOT extracted +/// (faithful to upstream `extractCodeTokens`, `directory.ts:479-497`). Order is +/// deterministic: tokens are returned in first-seen appearance order. +pub fn extract_code_tokens(prompt: &str) -> Vec { + if prompt.is_empty() { + return Vec::new(); + } + let mut out: Vec = Vec::new(); + let push = |s: &str, out: &mut Vec| { + if !out.iter().any(|x| x == s) { + out.push(s.to_string()); + } + }; + + // camelCase / PascalCase-with-inner-cap or snake_case: a whole identifier + // run with an inner lower→upper transition or an underscore flanked by + // alphanumerics. + for m in IDENT_RUN_RE.find_iter(prompt) { + let w = m.as_str(); + if INNER_CAMEL_RE.is_match(w) || FLANKED_UNDERSCORE_RE.is_match(w) { + push(w, &mut out); + } + } + // call form: an identifier directly before '('. + for caps in CALL_FORM_RE.captures_iter(prompt) { + let name = caps.get(1).map_or("", |m| m.as_str()).to_string(); + push(&name, &mut out); + } + // member access on identifiers (user.login) — but not a doc/data filename. + for caps in MEMBER_ACCESS_RE.captures_iter(prompt) { + let whole = caps.get(0).map_or("", |m| m.as_str()); + if !DOC_DATA_EXT.is_match(whole) { + let a = caps.get(1).map_or("", |m| m.as_str()).to_string(); + let b = caps.get(2).map_or("", |m| m.as_str()).to_string(); + push(&a, &mut out); + push(&b, &mut out); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gate_fires_on_english_structural() { + assert!(has_structural_keyword("how does this work")); + assert!(has_structural_keyword("where is the router")); + } + + #[test] + fn gate_fires_on_non_english() { + assert!(has_structural_keyword("comment marche le routeur")); // FR + assert!(has_structural_keyword("这个模块如何工作")); // zh + assert!(has_structural_keyword("이건 어떻게 작동하나요")); // ko + assert!(has_structural_keyword("где находится роутер")); // ru + } + + #[test] + fn gate_fires_on_inflected_and_phrase() { + assert!(has_structural_keyword("what are the data flows here")); + assert!(has_structural_keyword("this reaches the handler")); + assert!(has_structural_keyword("why does the counter reset")); + } + + #[test] + fn gate_fires_on_devanagari() { + assert!(has_structural_keyword("यह कैसे काम करता है")); // how + assert!(has_structural_keyword("इसकी संरचना क्या है")); // structure + } + + #[test] + fn gate_mid_word_exclusion() { + // "flow" inside "flower", "path" inside "pathological" must NOT fire via + // the WORDS class (boundary char-class, not per-run equality). + assert!(!STRUCTURAL_WORDS_RE.is_match("flower shop")); + assert!(!STRUCTURAL_WORDS_RE.is_match("pathological case")); + } + + #[test] + fn gate_fires_on_calls_in_sentence() { + // Branch-local boundaries compose inside the combined STEMS regex. + assert!(has_structural_keyword("what calls Counter")); + assert!(has_structural_keyword("show me the callback")); + assert!(has_structural_keyword("list the call sites")); + assert!(has_structural_keyword("how many callers")); + assert!(has_structural_keyword("trace the traces")); + assert!(has_structural_keyword("what affects this")); + assert!(has_structural_keyword("show connections")); + } + + #[test] + fn gate_no_false_fire_on_ordinary_words() { + // #1138 right-bounding: ordinary words with these prefixes must NOT fire. + assert!(!has_structural_keyword("about Connecticut weather")); + assert!(!has_structural_keyword("i have a callus on my foot")); + assert!(!has_structural_keyword("calligraphy is an art")); + assert!(!has_structural_keyword("this connective tissue")); + assert!(!has_structural_keyword("an affectionate dog")); + assert!(!has_structural_keyword("Tracey went home")); + } + + #[test] + fn gate_silent_on_plain_prose() { + assert!(!has_structural_keyword("please fix this typo")); + assert!(!has_structural_keyword("rename the variable")); + } + + #[test] + fn extract_code_tokens_rules() { + // inner camelCase / PascalCase-with-inner-cap + assert!(extract_code_tokens("look at OrderService").contains(&"OrderService".to_string())); + assert!(extract_code_tokens("the getUserId helper").contains(&"getUserId".to_string())); + // snake_case flanked underscore + assert!(extract_code_tokens("call get_user now").contains(&"get_user".to_string())); + // call form + assert!(extract_code_tokens("Counter() creates one").contains(&"Counter".to_string())); + // member access + let toks = extract_code_tokens("user.login please"); + assert!(toks.contains(&"user".to_string())); + assert!(toks.contains(&"login".to_string())); + } + + #[test] + fn extract_code_tokens_bare_pascalcase_is_not_a_token() { + // A bare single PascalCase word is NOT a token (upstream faithfulness). + assert!(extract_code_tokens("Counter").is_empty()); + assert!(extract_code_tokens("look at Counter here").is_empty()); + } + + #[test] + fn extract_code_tokens_doc_filename_excluded() { + // README.md is a doc filename — not a member-access symbol. + assert!(extract_code_tokens("open README.md").is_empty()); + assert!(extract_code_tokens("edit config.json now").is_empty()); + } + + #[test] + fn extract_code_tokens_plain_prose_none() { + assert!(extract_code_tokens("please fix this typo").is_empty()); + } +} diff --git a/crates/codegraph-cli/tests/cli_commands.rs b/crates/codegraph-cli/tests/cli_commands.rs index a2a2aff6..df35f43b 100644 --- a/crates/codegraph-cli/tests/cli_commands.rs +++ b/crates/codegraph-cli/tests/cli_commands.rs @@ -416,32 +416,40 @@ fn init_on_already_initialized_is_idempotent() { } #[test] -fn prompt_hook_without_index_prints_hint() { +fn prompt_hook_without_index_is_silent() { + // Under the confidence-tiered gate, the hook is degradable-by-contract: an + // unindexed dir (or a non-matching prompt) exits 0 with NO output — the + // upstream silent no-op, not the old "[codegraph] no index" notice. let dir = TestDir::new("prompt-hook"); let empty = dir.path().join("empty"); std::fs::create_dir_all(&empty).unwrap(); let run = run_in( dir.path(), - &["prompt-hook", "-p", empty.to_str().unwrap(), "some query"], + &[ + "prompt-hook", + "-p", + empty.to_str().unwrap(), + "how does this work", + ], ); assert!(run.ok, "prompt-hook must always succeed: {}", run.stderr); assert!( - run.stdout.contains("[codegraph]"), - "prompt-hook must print a codegraph-tagged line: {}", + run.stdout.trim().is_empty(), + "prompt-hook in an unindexed dir must be a silent no-op: {}", run.stdout ); } #[test] -fn prompt_hook_empty_query_reports_nothing_to_explore() { +fn prompt_hook_empty_query_is_silent() { let dir = TestDir::new("prompt-hook-empty"); let project = indexed_project(&dir); let p = project.to_str().unwrap(); let run = run_in(dir.path(), &["prompt-hook", "-p", p, " "]); assert!(run.ok, "prompt-hook must succeed: {}", run.stderr); assert!( - run.stdout.contains("No query provided"), - "blank query must report nothing to explore: {}", + run.stdout.trim().is_empty(), + "blank query must be a silent no-op: {}", run.stdout ); } diff --git a/crates/codegraph-cli/tests/fixtures/prompt_hook_medium/src/checkout.ts b/crates/codegraph-cli/tests/fixtures/prompt_hook_medium/src/checkout.ts new file mode 100644 index 00000000..dfd5ad48 --- /dev/null +++ b/crates/codegraph-cli/tests/fixtures/prompt_hook_medium/src/checkout.ts @@ -0,0 +1,24 @@ +export class CheckoutStateMachine { + private current = "cart"; + + transition(next: string): string { + this.current = next; + return this.current; + } +} + +export class CheckoutService { + private machine = new CheckoutStateMachine(); + + begin(): string { + return this.machine.transition("payment"); + } +} + +export class CheckoutController { + private service = new CheckoutService(); + + handle(): string { + return this.service.begin(); + } +} diff --git a/crates/codegraph-cli/tests/prompt_hook.rs b/crates/codegraph-cli/tests/prompt_hook.rs index 98265172..34d75da8 100644 --- a/crates/codegraph-cli/tests/prompt_hook.rs +++ b/crates/codegraph-cli/tests/prompt_hook.rs @@ -1,13 +1,17 @@ //! End-to-end tests for the hidden `codegraph prompt-hook` subcommand. //! -//! `prompt-hook` emits `codegraph_explore`-equivalent DETERMINISTIC retrieval -//! output (the SAME text the MCP explore tool produces — NO LLM/AI) to stdout -//! for a given query, resolving the nearest `.codegraph/` (monorepo-aware). +//! `prompt-hook` is a confidence-tiered gate (upstream #1126 multilingual gate +//! + #1136 graph-derived MEDIUM tier, telemetry EXCLUDED): +//! - HIGH — a structural keyword (any of ~29 covered languages) OR a code token +//! verified in the index → full `codegraph_explore` injection. +//! - MEDIUM — prose words match indexed symbol-name segments → a short +//! symbol-pointer hint (never runs explore). +//! - silent — nothing verified → zero-cost no-op. +//! //! Mirrors the spawn-the-real-binary pattern of `sync_incremental.rs` / //! `installer.rs` via `CARGO_BIN_EXE_codegraph`. use std::fs; -use std::io::Write; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; @@ -23,6 +27,10 @@ fn mini_fixture() -> PathBuf { workspace_root().join("crates/codegraph-bench/fixtures/mini") } +fn medium_fixture() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/prompt_hook_medium") +} + fn copy_tree(src: &Path, dst: &Path) { fs::create_dir_all(dst).unwrap(); for entry in fs::read_dir(src).unwrap() { @@ -78,8 +86,9 @@ fn cli(args: &[&str]) -> (String, String, bool) { ) } -/// Run `prompt-hook` feeding the query over stdin (no positional/flag arg). +/// Run `prompt-hook` feeding text over stdin (no positional/flag arg). fn cli_stdin(args: &[&str], stdin_text: &str) -> (String, String, bool) { + use std::io::Write; let mut child = Command::new(env!("CARGO_BIN_EXE_codegraph")) .args(args) .stdin(Stdio::piped()) @@ -101,61 +110,296 @@ fn cli_stdin(args: &[&str], stdin_text: &str) -> (String, String, bool) { ) } -fn index_mini(label: &str) -> (TestDir, PathBuf) { +fn index_fixture(fixture: &Path, subdir: &str, label: &str) -> (TestDir, PathBuf) { let dir = TestDir::new(label); - let project = dir.path().join("mini"); - copy_tree(&mini_fixture(), &project); + let project = dir.path().join(subdir); + copy_tree(fixture, &project); let (out, err, ok) = cli(&["init", project.to_str().unwrap()]); assert!(ok, "init failed: stdout={out} stderr={err}"); (dir, project) } +fn index_mini(label: &str) -> (TestDir, PathBuf) { + index_fixture(&mini_fixture(), "mini", label) +} + +fn index_medium(label: &str) -> (TestDir, PathBuf) { + index_fixture(&medium_fixture(), "medium", label) +} + +// === HIGH tier: named-symbol code token (call form) ========================= + #[test] -fn prompt_hook_emits_explore_output() { +fn prompt_hook_high_tier_on_call_form() { let (_dir, project) = index_mini("emit"); - // Query a symbol known to exist in the mini fixture (src/math.ts: - // `export class Counter`). The explore output must reference it. NO LLM is - // involved — this is codegraph's own deterministic retrieval text. + // `Counter()` is a code token (call form) — a bare `Counter` is NOT (no + // inner camelCase / underscore / `(` / `.`), so the HIGH named-symbol path + // must use the call form. The fixture defines `Counter` in src/math.ts. let (out, err, ok) = cli(&[ "prompt-hook", "--path", project.to_str().unwrap(), "--query", - "Counter", + "Counter()", ]); assert!(ok, "prompt-hook failed: stdout={out} stderr={err}"); - assert!( - !out.trim().is_empty(), - "expected non-empty stdout, got empty" - ); assert!( out.contains("Counter"), - "expected explore output referencing the queried symbol, got:\n{out}" + "expected explore output referencing Counter, got:\n{out}" ); - // It must be explore-style content (the explore renderer's headers), proving - // it is the same output the MCP explore tool produces. assert!( out.contains("Exploration:") || out.contains("Source Code"), "expected codegraph_explore-style content, got:\n{out}" ); + assert!( + out.contains(", got:\n{out}" + ); } #[test] -fn prompt_hook_reads_query_from_stdin() { +fn prompt_hook_call_form_from_stdin() { let (_dir, project) = index_mini("stdin"); let (out, err, ok) = cli_stdin( &["prompt-hook", "--path", project.to_str().unwrap()], - "Counter", + "Counter()", ); assert!(ok, "prompt-hook (stdin) failed: stdout={out} stderr={err}"); assert!( out.contains("Counter"), - "expected stdin query to drive explore output, got:\n{out}" + "expected stdin call form to drive explore output, got:\n{out}" + ); +} + +// === HIGH tier: structural keyword path ===================================== + +#[test] +fn gate_fires_on_calls_in_sentence() { + let (_dir, project) = index_mini("calls-sentence"); + // "what calls Counter()" fires HIGH via the STRUCTURAL-KEYWORD path + // (`calls`), independent of the tokenizer (#1138 branch-local boundary). + let (out, err, ok) = cli(&[ + "prompt-hook", + "--path", + project.to_str().unwrap(), + "--query", + "what calls Counter()", + ]); + assert!(ok, "failed: stdout={out} stderr={err}"); + assert!( + out.contains(", got:\n{out}" + ); + assert!( + out.contains("CheckoutStateMachine"), + "MEDIUM must name the matched symbol, got:\n{out}" + ); + assert!( + out.contains("codegraph_explore ONCE"), + "MEDIUM must carry the pointer guidance, got:\n{out}" + ); + // NOT a full explore render. + assert!( + !out.contains("Exploration:") && !out.contains("Source Code"), + "MEDIUM must NOT run full explore, got:\n{out}" + ); +} + +#[test] +fn medium_tier_cooccurrence_matches() { + let (_dir, project) = index_medium("medium-cooccur"); + let (out, err, ok) = cli(&[ + "prompt-hook", + "--path", + project.to_str().unwrap(), + "--query", + "checkout state machine", + ]); + assert!(ok, "failed: stdout={out} stderr={err}"); + assert!( + out.contains("CheckoutStateMachine"), + "Tier-A co-occurrence must surface CheckoutStateMachine, got:\n{out}" + ); +} + +#[test] +fn medium_tier_rare_single_word() { + let (_dir, project) = index_medium("medium-rare"); + // Single non-structural word: `checkout` clusters across ≥2 names + // (CheckoutService / CheckoutController / CheckoutStateMachine) → Tier B. + let (out, err, ok) = cli(&[ + "prompt-hook", + "--path", + project.to_str().unwrap(), + "--query", + "checkout", + ]); + assert!(ok, "failed: stdout={out} stderr={err}"); + assert!( + out.contains("