From 37fdc09845f3f1d44476bb37c9ec81fcf049915d Mon Sep 17 00:00:00 2001 From: sunerpy Date: Fri, 21 Aug 2026 05:43:12 +0800 Subject: [PATCH] fix(explore): render the symbol you asked for, and charge what you spend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four selection and budget defects, all reachable from one code path. A named symbol could vanish. Asking for a symbol by name gave back 615 bytes, no source sections at all, and no sign anything was missing — while `query` found it immediately. Two independent escapes produced that: a file whose whole section is unaffordable was rejected entire, before any windowing existed to narrow it, and a file that fell through to clustering had its first cluster accepted unconditionally and then thrown away for being oversize. Named symbols now carry a protected focus, a whole-file section that cannot be afforded is windowed around it rather than dropped, and a first cluster is bounded instead of trusted. A later cluster that did not fit was skipped whole, so a file could contribute its header and nothing else. Later clusters now shrink to what the remaining room allows, and the bound is applied to the finished section rather than to the cluster bodies alone — the admission test compares the section, so a ceiling on the bodies leaves the frame unpaid and the windowed section is discarded anyway. An ambient declaration file that nothing depends on could take the top slot on a flow query, pushing implementations down the page. Such files are now damped, combined with the generated-file signal by taking the stronger of the two, so a file that is both is penalised once rather than twice. Files whose type the query names are exempt. The budget was wrong in both directions at once: every section was charged a flat 200 characters instead of its real cost, while the epilogue was appended after the loop and charged nothing. Fixing either alone makes overflow more likely, so both land together, with the epilogue's frame reserved up front and its pointer block sized to the room that is actually left. The hard-ceiling cut consequently becomes unreachable — it sits above the budget at every tier — which is why the truncation note no longer eats the very text that says what was omitted. CG-27 is deliberately not ported: upstream measured it dropping delivered characters from 7,539 to 397 and rejected it. --- crates/codegraph-mcp/src/engine.rs | 1538 ++++++++++++++++++++-- crates/codegraph-mcp/tests/golden_mcp.rs | 578 ++++++++ 2 files changed, 1991 insertions(+), 125 deletions(-) diff --git a/crates/codegraph-mcp/src/engine.rs b/crates/codegraph-mcp/src/engine.rs index 9863312a..f5b25abe 100644 --- a/crates/codegraph-mcp/src/engine.rs +++ b/crates/codegraph-mcp/src/engine.rs @@ -37,6 +37,33 @@ const TRAIL_CAP: usize = 8; /// `Read`-tool cap mirrored by file-mode `codegraph_node` (`tools.ts:489`). const FILE_MODE_MAX_LINES: usize = 2000; +/// Upstream's per-file protected-focus cap (`.slice(0, 6)`), so a 40-token query +/// cannot pin a whole file. +const FOCUS_CAP: usize = 6; + +/// Upstream's `POINTER_SYMBOLS` — `name:line` pairs shown per "Not shown above" +/// pointer line before a `+N more` tail. +const POINTER_SYMBOLS: usize = 6; + +/// Upstream's `POINTER_MAX_FILES` — files listed in the "Not shown above" block. +const POINTER_MAX_FILES: usize = 10; + +/// Upstream's `MIN_WINDOW_LINES` — the never-empty floor for a windowed cluster. +/// A file always emits at least this many whole lines around its +/// highest-importance member, even when that exceeds the room it was given; the +/// caller then drops the file whole rather than emitting a stub. +const MIN_WINDOW_LINES: usize = 12; + +/// Fixed cost of `render_explore_file`'s section template around the header, the +/// language tag and the cluster body: `"\n\n"` + ``"```"`` + `"\n"` + `"\n"` + +/// ``"```"`` + `"\n"`. +const SECTION_FENCE_CHARS: usize = 11; + +/// The trailing `"\n\n... (gap) ..."` a section carries when any cluster was +/// dropped or windowed. Reserved unconditionally in `section_frame`, since +/// whether it is emitted is only known after cluster selection. +const SECTION_GAP_TAIL_CHARS: usize = 17; + /// Callable node kinds whose signature (parameter/return) types the #1064 /// change-surface pass follows. Mirrors upstream `CALLABLE_KINDS` /// (`tools.ts:2620`); `constructor` has no Rust NodeKind (ctors are `method`). @@ -985,6 +1012,16 @@ impl CodeGraphEngine { let mut any_file_trimmed = false; let mut excluded_files: Vec<&String> = Vec::new(); let mut rendered_sources: Vec<(String, usize)> = Vec::new(); + let precise_tokens = precise_query_tokens(&query); + let indexed_file_count = self.store.counts().ok().map(|c| c.file_count); + let effective_budget = budget + .max_output_chars + .saturating_sub(fixed_epilogue_reserve( + &budget, + file_order.len(), + max_files, + indexed_file_count, + )); for file_path in &file_order { if files_included >= max_files { @@ -1000,17 +1037,25 @@ impl CodeGraphEngine { let file_lines: Vec<&str> = content.split('\n').collect(); let lang = subgraph.file_language(file_path); - let rendered = self.render_explore_file( - &subgraph, - file_path, - &file_lines, - &lang, - &budget, - possibly_drifted, - ); + let focuses = subgraph.file_focus_lines(file_path, &precise_tokens); + let ctx = RenderCtx { + budget: &budget, + funded_headroom: effective_budget + .saturating_sub(total_chars) + .saturating_sub(1), + focuses: &focuses, + drifted: possibly_drifted, + }; + let rendered = self.render_explore_file(&subgraph, file_path, &file_lines, &lang, &ctx); // Past the total cap an incidental file is skipped whole — the source // section never slices through a method body (`tools.ts:2888-2894`). - if total_chars + rendered.section.len() + 200 > budget.max_output_chars { + // + // A section costs `section.len() + 1`: its own length — which already + // covers the `#### ` header and the fence — plus the one + // `lines.join("\n")` separator it is pushed behind. The same + // expression appears at the accumulator below, so the admission test + // and the running total cannot disagree. + if total_chars + rendered.section.len() + 1 > effective_budget { any_file_trimmed = true; excluded_files.push(file_path); continue; @@ -1025,59 +1070,53 @@ impl CodeGraphEngine { rendered_sources.push(((*file_path).clone(), lines.len())); } lines.push(rendered.section); - total_chars += section_len + 200; + total_chars += section_len + 1; files_included += 1; } // "Additional relevant files (not shown)" — the excluded set, so the // agent can request specifics. Gated by the budget (`tools.ts:2910-2927`). if budget.include_additional_files && !excluded_files.is_empty() { - lines.push("### Not shown above — explore these names for their source".to_string()); + lines.push(POINTER_BLOCK_HEADER.to_string()); lines.push(String::new()); - for file_path in excluded_files.iter().take(10) { - let names = subgraph.file_node_locations(file_path); - lines.push(format!("- {file_path}: {names}")); - } - if excluded_files.len() > 10 { - lines.push(format!( - "- ... and {} more files", - excluded_files.len() - 10 - )); + // The block is ELASTIC: it is fitted to the headroom the sections did + // not use, so however long the pointer list is it cannot push the + // response past its budget. The frame and the tail are already paid + // for by the reserve and so are not charged here. + let candidates: Vec = excluded_files + .iter() + .map(|file_path| { + format!("- {file_path}: {}", subgraph.file_node_locations(file_path)) + }) + .collect(); + let (pointer_lines, unlisted) = + fit_pointer_lines(&candidates, effective_budget.saturating_sub(total_chars)); + lines.extend(pointer_lines); + // The TRUE unlisted count — those dropped by elasticity plus those + // dropped by the file cap — so a file the response did not render is + // never silently absent. + if unlisted > 0 { + lines.push(pointer_unlisted_tail(unlisted)); } lines.push(String::new()); } - // Completeness signal / trim note, gated by the budget - // (`tools.ts:2933-2940`). - if budget.include_completeness_signal { - lines.push("---".to_string()); - lines.push(format!( - "> **Complete source for {files_included} files is included above — do NOT re-read them.** If your question also needs files/symbols listed under \"Not shown above\" (or any area this call didn't cover), make ANOTHER codegraph_explore targeting those names — it returns the same source with line numbers and is cheaper and more complete than reading. Reserve Read for a single specific line range explore can't surface." - )); - lines.push(String::new()); - } else if any_file_trimmed { - lines.push("> Some file sections were trimmed for size. For a specific symbol you still need, run another `codegraph_explore` (or `codegraph_node`) with its exact name — line-numbered source, cheaper and more complete than Read.".to_string()); - lines.push(String::new()); - } - - // Follow-up-call note, gated (`tools.ts:2943-2952`). + // Completeness signal / trim note (`tools.ts:2933-2940`), then the + // follow-up-call note (`tools.ts:2943-2952`), both gated by the budget and + // both built by the SAME function the pre-loop reserve is computed from. // - // Upstream's wording announced a per-project call BUDGET ("N calls for - // this project … Synthesize once you've used N"), but nothing anywhere - // enforces it, so its only effect was agents stopping while their - // question was still uncovered (#1504). The useful half — one more - // explore beats falling back to Read — is kept, the phantom cap is - // dropped, and the absence of a limit is now stated outright rather than - // left to inference. - if budget.include_budget_note - && let Ok(c) = self.store.counts() - { - lines.push(format!( - "> **{} files indexed.** Each call covers ~6 files; if your question spans more, make ANOTHER `codegraph_explore` targeting the uncovered area BEFORE falling back to Read — another explore is cheaper and more complete than reading those files. There is no call limit.", - c.file_count - )); - lines.push(String::new()); - } + // Upstream's follow-up wording announced a per-project call BUDGET ("N + // calls for this project … Synthesize once you've used N"), but nothing + // anywhere enforces it, so its only effect was agents stopping while their + // question was still uncovered (#1504). The useful half — one more explore + // beats falling back to Read — is kept, the phantom cap is dropped, and + // the absence of a limit is stated outright rather than left to inference. + lines.extend(epilogue_note_lines( + &budget, + files_included, + any_file_trimmed, + indexed_file_count, + )); // Final ABSOLUTE inline ceiling — cut at a `#### ` file-section boundary // so trailing whole sections drop rather than slicing a method body @@ -1099,9 +1138,10 @@ impl CodeGraphEngine { file_path: &str, file_lines: &[&str], lang: &str, - budget: &ExploreOutputBudget, - drifted: bool, + ctx: &RenderCtx<'_>, ) -> ExploreFileRender { + let budget = ctx.budget; + let drifted = ctx.drifted; let total_lines = file_lines.len(); let content = file_lines.join("\n"); let body = content.trim_end_matches('\n'); @@ -1137,10 +1177,33 @@ impl CodeGraphEngine { let numbered = number_source_lines_at(&body.split('\n').collect::>(), 1); let names = subgraph.file_header_names_capped(file_path, budget.max_symbols_in_file_header); - return ExploreFileRender { - section: format!("#### {file_path} — {names}\n\n```{lang}\n{numbered}\n```\n"), - source_emitted: true, - }; + let section = format!("#### {file_path} — {names}\n\n```{lang}\n{numbered}\n```\n"); + // The two conditions are ordered AFFORDABILITY FIRST, and that order + // is normative rather than stylistic. + // + // (1) Affordable ⇒ byte-for-byte today's behaviour, with no focus + // logic consulted at all, so every input that is not the defect — + // including every existing golden — is untouched by EVALUATION + // rather than by unreachability. + let section_cost = section.len() + 1; + if section_cost <= ctx.funded_headroom { + return ExploreFileRender { + section, + source_emitted: true, + }; + } + // (2) Unaffordable but owning NO focus ⇒ also today's behaviour: + // return whole and let the caller drop it whole, so "an incidental + // file that doesn't fit is DROPPED whole" stays literally true. + if ctx.focuses.is_empty() { + return ExploreFileRender { + section, + source_emitted: true, + }; + } + // (3) Unaffordable AND owning a focus ⇒ fall through to the clustering + // path, which windows against `funded_headroom` and guarantees a + // window per focus. The ONLY new behaviour. } // Cluster nearby symbol ranges; merge ranges within `gapThreshold` @@ -1164,7 +1227,14 @@ impl CodeGraphEngine { && (n.end_line - n.start_line + 1) as usize > total_lines / 2) }) .map(|n| { - let importance = if subgraph.roots.iter().any(|r| r == &n.id) { + // A protected focus outranks an ordinary root, so a cluster that + // owns one sorts ahead of equally-important siblings. It is + // matched on the STORED definition line: a focus whose line the + // clamp below moves is one the range filters would drop anyway, + // and §3.1.1 conditions the guarantee on surviving them. + let importance = if ctx.focuses.contains(&(n.start_line as usize)) { + 11 + } else if subgraph.roots.iter().any(|r| r == &n.id) { 10 } else { 1 @@ -1212,17 +1282,133 @@ impl CodeGraphEngine { const CONTEXT_PADDING: usize = 3; const GAP_MARKER: &str = "\n\n... (gap) ...\n\n"; - let build_section = |c: &Cluster| -> String { + // A cluster's padded slice bounds, 0-based with an EXCLUSIVE end — the + // exact bounds `build_section` has always used, so a cluster that fits + // its room still renders byte-identically. + // + // `.min(end_idx)` guards the slice start: even if a cluster's start + // slipped past EOF, `start_idx <= end_idx` keeps the range valid. + let span_of = |c: &Cluster| -> (usize, usize) { let end_idx = (c.end + CONTEXT_PADDING).min(total_lines); - // `.min(end_idx)` guards the slice start: even if a cluster's start - // slipped past EOF, `start_idx <= end_idx` keeps the range valid. let start_idx = c .start .saturating_sub(1) .saturating_sub(CONTEXT_PADDING) .min(end_idx); - let slice = &file_lines[start_idx..end_idx]; - number_source_lines_at(slice, start_idx + 1) + (start_idx, end_idx) + }; + let render_range = |start_idx: usize, end_idx: usize| -> String { + number_source_lines_at(&file_lines[start_idx..end_idx], start_idx + 1) + }; + + // `number_source_lines_at` renders each line as `"{n}\t{line}"` and joins + // with "\n", so a range costs the per-line widths plus one separator + // between neighbours. Summed rather than rendered, so growing a window + // one line at a time stays linear instead of re-formatting the slice. + let range_cost = |start_idx: usize, end_idx: usize| -> usize { + if end_idx <= start_idx { + return 0; + } + (start_idx..end_idx) + .map(|i| decimal_width(i + 1) + 1 + file_lines[i].len()) + .sum::() + + (end_idx - start_idx - 1) + }; + let grow_head = |start_idx: usize, end_idx: usize, room: usize| -> (usize, usize) { + let floor = (start_idx + MIN_WINDOW_LINES).min(end_idx); + let mut end = (start_idx + 1).min(end_idx); + while end < end_idx && range_cost(start_idx, end + 1) <= room { + end += 1; + } + (start_idx, end.max(floor)) + }; + // Grow one line up, then one line down, alternating, so the emitted window + // is centred on the focus rather than biased to whichever side was tried + // first. Bounded by the span's own line count, so it always terminates. + let grow_around = + |focus_idx: usize, start_idx: usize, end_idx: usize, room: usize| -> (usize, usize) { + let focus_idx = focus_idx.clamp(start_idx, end_idx - 1); + let (mut lo, mut hi) = (focus_idx, focus_idx + 1); + let mut up = true; + for _ in 0..(end_idx - start_idx) * 2 { + let can_up = lo > start_idx && range_cost(lo - 1, hi) <= room; + let can_down = hi < end_idx && range_cost(lo, hi + 1) <= room; + if !can_up && !can_down { + break; + } + if (up && can_up) || !can_down { + lo -= 1; + } else { + hi += 1; + } + up = !up; + } + let want = MIN_WINDOW_LINES.min(end_idx - start_idx); + if hi - lo < want { + let lo2 = focus_idx + .saturating_sub(want / 2) + .max(start_idx) + .min(end_idx - want); + return (lo2, lo2 + want); + } + (lo, hi) + }; + // Only the focuses inside THIS cluster's span may claim its reserve; + // clamping an out-of-span focus in would spend the reserve re-emitting + // lines the head window already covers. + let focuses_in = |c: &Cluster| -> Vec { + let (start_idx, end_idx) = span_of(c); + ctx.focuses + .iter() + .copied() + .filter(|&line| line > start_idx && line <= end_idx) + .collect() + }; + // CG-30: a cluster too big for its room is emitted as WHOLE-LINE windows + // instead of being taken whole (which then costs the entire file) or + // skipped whole. Returns the rendered body and whether it was windowed, + // so the caller can add the `... (gap) ...` honesty marker. + let window_cluster = |c: &Cluster, room: usize, focuses: &[usize]| -> (String, bool) { + let (start_idx, end_idx) = span_of(c); + if range_cost(start_idx, end_idx) <= room { + return (render_range(start_idx, end_idx), false); + } + let head_room = if focuses.is_empty() { + room + } else { + room * 60 / 100 + }; + let mut windows: Vec<(usize, usize)> = vec![grow_head(start_idx, end_idx, head_room)]; + if !focuses.is_empty() { + // Even shares with carry-forward, never greedy: upstream measured + // that a greedy split let an early focus eat the whole reserve and + // re-lose the target. One integer division, so the same input + // cannot drift by a char across platforms. + let reserve = room - head_room; + let base = reserve / focuses.len(); + let mut carry = reserve - base * focuses.len(); + for &line in focuses { + let allot = base + carry; + let w = grow_around(line.saturating_sub(1), start_idx, end_idx, allot); + carry = allot.saturating_sub(range_cost(w.0, w.1)); + windows.push(w); + } + } + windows.sort_unstable(); + let mut merged: Vec<(usize, usize)> = Vec::new(); + for w in windows { + match merged.last_mut() { + Some(last) if w.0 <= last.1 => last.1 = last.1.max(w.1), + _ => merged.push(w), + } + } + let covers_all = merged.len() == 1 && merged[0] == (start_idx, end_idx); + let body = merged + .iter() + .map(|&(lo, hi)| render_range(lo, hi)) + .collect::>() + .join(GAP_MARKER); + (body, !covers_all) }; // Rank clusters: entry-point importance first, then density, then span @@ -1240,43 +1426,76 @@ impl CodeGraphEngine { .then((span_a as usize).cmp(&(span_b as usize))) }); - let file_budget = budget.max_chars_per_file; - let mut chosen: HashSet = HashSet::new(); + // CG-30: the first cluster is still always taken, but WINDOWED to a + // ceiling rather than accepted at any size. `ceiling` bounds cluster + // BODIES while the caller's admission test and the 1.5x per-file bound + // both compare the finished SECTION, so `section_frame` — the header, the + // language tag, the fence and the gap tail — is subtracted OUTSIDE the + // `min`. Inside it, the section would clear admission yet overshoot + // `round(max_chars_per_file * 1.5)` by the frame. + let section_frame = subgraph + .file_header_len_upper_bound(file_path, budget.max_symbols_in_file_header) + + lang.len() + + SECTION_FENCE_CHARS + + SECTION_GAP_TAIL_CHARS; + let ceiling = ((budget.max_chars_per_file as f64 * 1.5).round() as usize) + .min(ctx.funded_headroom) + .saturating_sub(section_frame); + // Per-cluster render state lives in a Vec indexed parallel to `clusters`, + // never in a HashSet that is then iterated: emission order must come from + // the index, not from hash order, or the output stops being byte-stable. + let mut rendered_clusters: Vec> = vec![None; clusters.len()]; let mut projected = 0usize; + let mut chosen_count = 0usize; + let mut any_cluster_windowed = false; for &idx in &ranked { - let section_len = build_section(&clusters[idx]).len() - + if chosen.is_empty() { - 0 - } else { - GAP_MARKER.len() - }; - // Always take the top-ranked cluster even if oversize, so the file - // section is never empty (`tools.ts:2828-2835`). - if chosen.is_empty() { - chosen.insert(idx); - projected += section_len; + if chosen_count == 0 { + let (body, windowed) = + window_cluster(&clusters[idx], ceiling, &focuses_in(&clusters[idx])); + projected += body.len(); + any_cluster_windowed |= windowed; + rendered_clusters[idx] = Some(body); + chosen_count += 1; continue; } - if projected + section_len > file_budget { + // CG-36: a later cluster that does not fit is SHRUNK into the + // remainder rather than skipped whole. The bound is `ceiling`, not + // `file_budget`, so the whole file obeys ONE stated bound — and + // `saturating_sub` is required rather than stylistic: half A lets the + // first cluster's body legitimately exceed `file_budget`, so a plain + // subtraction wraps to a huge `room` and hands this cluster unlimited + // budget, the exact opposite of the intent. + let room = ceiling.saturating_sub(projected + GAP_MARKER.len()); + // The floor's own cost IS `MIN_CHARS` — the smallest useful window — + // rather than a new magic number. When even that cannot be paid for + // the cluster is still skipped, so shrinking can never overspend into + // the next file's share. + let (start_idx, end_idx) = span_of(&clusters[idx]); + let floor_end = (start_idx + MIN_WINDOW_LINES).min(end_idx); + if room < range_cost(start_idx, floor_end) { continue; } - chosen.insert(idx); - projected += section_len; + let (body, windowed) = + window_cluster(&clusters[idx], room, &focuses_in(&clusters[idx])); + any_cluster_windowed |= windowed; + projected += body.len() + GAP_MARKER.len(); + rendered_clusters[idx] = Some(body); + chosen_count += 1; } let mut file_section = String::new(); let mut symbols: Vec = Vec::new(); for (i, cluster) in clusters.iter().enumerate() { - if !chosen.contains(&i) { + let Some(body) = rendered_clusters[i].as_ref() else { continue; - } + }; if !file_section.is_empty() { file_section.push_str(GAP_MARKER); } - file_section.push_str(&build_section(cluster)); + file_section.push_str(body); symbols.extend(cluster.symbols.iter().cloned()); } - if chosen.len() < clusters.len() { + if chosen_count < clusters.len() || any_cluster_windowed { file_section.push_str("\n\n... (gap) ..."); } @@ -1780,11 +1999,83 @@ impl CodeGraphEngine { candidate_paths.sort_unstable(); candidate_paths.dedup(); sub.generated_files = self.store.generated_paths_among(&candidate_paths)?; + sub.ambient_files = self.ambient_declaration_files(&candidate_paths, query)?; sub.finalize(); Ok(sub) } + /// The ambient declaration files among `candidate_paths`, minus any this query + /// exempts by naming one of their types (ports upstream `9efae0f`'s + /// `ambientDeclarationPredicateFor`). + /// + /// All four conditions are evaluated INDEX-WIDE, never over the subgraph. The + /// distinction is the whole point of condition 4: the subgraph is seeded from + /// the top 8 search hits and expanded one hop from the ROOTS, so a file that + /// depends on a non-root declaration in the shim is normally absent from it + /// entirely — and a subgraph-scoped predicate would then read "nothing depends + /// on this" and damp a type module the repo genuinely imports. + /// + /// Conditions 1 and 2 run first as a cheap pre-filter, so the two edge queries + /// execute only for the handful of declaration-only files. + fn ambient_declaration_files( + &self, + candidate_paths: &[String], + query: &str, + ) -> anyhow::Result> { + let precise = precise_query_tokens(query); + let mut ambient = HashSet::new(); + for path in candidate_paths { + let nodes = self.store.nodes_by_file_path(path)?; + let declared: Vec<&Node> = nodes + .iter() + .filter(|n| !is_ambient_bookkeeping_kind(n.kind)) + .collect(); + // (1) at least one declaration, and (2) every one of them is a type + // declaration — a file holding an implementation is never ambient. + if declared.is_empty() || !declared.iter().all(|n| is_type_declaration_kind(n.kind)) { + continue; + } + // A query naming one of this file's declarations is asking for it, so + // it keeps its rank. Applied while BUILDING the set because `finalize` + // never sees the query. + if declared + .iter() + .any(|n| precise.iter().any(|t| t.eq_ignore_ascii_case(&n.name))) + { + continue; + } + // (3) nothing in the file runs: no node is the source of a Calls or + // Instantiates edge. + let mut runs = false; + for n in &declared { + for kind in [EdgeKind::Calls, EdgeKind::Instantiates] { + if !self + .store + .edges_by_source_kind(&n.id, Some(kind))? + .is_empty() + { + runs = true; + break; + } + } + if runs { + break; + } + } + if runs { + continue; + } + // (4) nothing anywhere in the index depends on it. Consumed only + // through `is_empty`, so the query's SQLite scan order can never reach + // the output. + if self.store.dependent_file_paths(path)?.is_empty() { + ambient.insert(path.clone()); + } + } + Ok(ambient) + } + /// #1064 change-surface rescue (ports `2b256b9` `tools.ts:2585-2860`, adapted /// to this deterministic explore). /// @@ -2293,6 +2584,16 @@ struct ExploreSubgraph { /// reaches it. Empty leaves behavior on the path check alone, which is what /// keeps a pre-#1500 index ranking as it always did. generated_files: HashSet, + /// Paths that are ambient DECLARATION files — every symbol is a type + /// declaration, nothing in them runs, and nothing anywhere in the index + /// depends on them — and that this query does not name a type in. + /// + /// Computed index-wide in `find_relevant_context` before `finalize`, exactly + /// as `generated_files` and `rescued_files` are: the predicate needs the store + /// and the query, and `finalize` has neither. Membership therefore already + /// accounts for the exemption, which is why the same file can be damped for a + /// prose query and undamped for one naming its type. + ambient_files: HashSet, } impl ExploreSubgraph { @@ -2369,21 +2670,39 @@ impl ExploreSubgraph { } else { 0 }; - // Generated-path demotion sits INSIDE the tier, so it only ever - // reorders files of equal query relevance: a protobuf stub loses - // to the implementation it shadows, never falls behind an - // unrelated incidental file, and still ranks first in its tier - // when it is the only match. The sort is descending, so real - // code must carry the HIGHER value. `find_symbol_matches` and - // `find_all_symbols` apply the same signal. - let not_generated = - u8::from(!(is_generated_file(fp) || self.generated_files.contains(fp))); + // Generated-path and ambient-declaration demotion sit INSIDE the + // tier, so they only ever reorder files of equal query relevance: a + // protobuf stub or a shim nothing depends on loses to the + // implementation it shadows, never falls behind an unrelated + // incidental file, and still ranks first in its tier when it is the + // only match. The sort is descending, so real code must carry the + // HIGHER value. `find_symbol_matches` and `find_all_symbols` apply + // the same generated signal. + // + // The two signals combine by `min`, upstream's rule, so a file that + // is BOTH is damped exactly as much as a generated-only one and no + // more. That is why this is one 3-valued integer and not a second + // boolean key: appending `not_damped_ambient` to the tuple would + // rank generated+ambient strictly BELOW generated-only, which is + // the double penalty `min` exists to prevent. For any non-ambient + // file the values are order-isomorphic to the old + // `not_generated` boolean (10 > 3 exactly where 1 > 0), so no + // ranking can move unless the ambient predicate actually fires. + let generated = is_generated_file(fp) || self.generated_files.contains(fp); + let weight = u8::min( + if generated { 3 } else { 10 }, + if self.ambient_files.contains(fp) { + 5 + } else { + 10 + }, + ); let sym_count = self .nodes .iter() .filter(|n| &n.file_path == fp && n.kind != NodeKind::File) .count(); - (fp.clone(), (tier, not_generated, sym_count)) + (fp.clone(), (tier, weight, sym_count)) }) .collect(); self.file_order @@ -2419,7 +2738,65 @@ impl ExploreSubgraph { cap_header_names(&names, cap) } - /// `name:line` locations for the "Not shown above" list (`tools.ts:2920`). + /// Definition lines of this file's PROTECTED FOCUSES: nodes that are query + /// roots AND whose name a shape-precise query token names, + /// case-insensitively. + /// + /// Selection among more than [`FOCUS_CAP`] candidates is by + /// `(start_line, name)`, never by hash order, so the emitted windows are + /// byte-stable across runs. + fn file_focus_lines(&self, file_path: &str, precise: &[String]) -> Vec { + let mut picked: Vec<(usize, &str)> = self + .nodes + .iter() + .filter(|n| { + n.file_path == file_path + && n.start_line > 0 + && self.roots.iter().any(|r| r == &n.id) + && precise.iter().any(|t| t.eq_ignore_ascii_case(&n.name)) + }) + .map(|n| (n.start_line as usize, n.name.as_str())) + .collect(); + picked.sort_unstable(); + picked.dedup(); + picked.truncate(FOCUS_CAP); + picked.into_iter().map(|(line, _)| line).collect() + } + + /// The longest `#### path — names` header any subset of this file's clusters + /// can produce: the `cap` LONGEST candidate labels plus the maximal `+N more` + /// tail. A max over a set, so it is order-independent, and an over-estimate + /// only ever LOWERS the render ceiling — it can never break the bound. + /// + /// Built from the file's SUBGRAPH nodes, the set `explore_file_header` draws + /// from. Computing it over the store's nodes for the same path would + /// over-estimate whenever the subgraph holds fewer of them. + fn file_header_len_upper_bound(&self, file_path: &str, cap: usize) -> usize { + let mut labels: Vec = Vec::new(); + for n in self.nodes.iter().filter(|n| { + n.file_path == file_path && n.kind != NodeKind::Import && n.kind != NodeKind::Export + }) { + let label = format!("{}({})", n.name, n.kind.as_str()); + if !labels.contains(&label) { + labels.push(label); + } + } + let total = labels.len(); + labels.sort_by(|a, b| b.len().cmp(&a.len()).then_with(|| a.cmp(b))); + let shown = labels.into_iter().take(cap).collect::>().join(", "); + let mut header = format!("#### {file_path} — {shown}"); + if total > cap { + header.push_str(&format!(", +{} more", total - cap)); + } + header.len() + } + + /// `name:line` locations for the "Not shown above" list (`tools.ts:2920`), + /// capped at upstream's `POINTER_SYMBOLS` with a `+N more` tail. + /// + /// The cap is what turns each pointer LINE from unbounded into bounded: a + /// 120-method class contributed ~120 pairs on one line, so a single file's + /// pointer could exhaust the whole epilogue budget. fn file_node_locations(&self, file_path: &str) -> String { let mut in_file: Vec<&Node> = self .nodes @@ -2427,12 +2804,42 @@ impl ExploreSubgraph { .filter(|n| n.file_path == file_path) .collect(); in_file.sort_by_key(|n| n.start_line); - in_file + let pairs: Vec = in_file .iter() .map(|n| format!("{}:{}", n.name, n.start_line)) - .collect::>() - .join(", ") + .collect(); + cap_header_names(&pairs, POINTER_SYMBOLS) + } +} + +/// Per-file render state for one iteration of `handle_explore`'s file loop. +/// +/// These cannot live in [`ExploreOutputBudget`]: that struct is `Copy` and built +/// ONCE per response from the project's file count, so putting loop state in it +/// would conflate a static tier constant with a per-file quantity. +struct RenderCtx<'a> { + budget: &'a ExploreOutputBudget, + /// What the total-output budget can still pay for this file, less the + /// `lines.join("\n")` separator the section will be charged. The ceiling and + /// the caller's admission test are computed from the SAME quantity, so they + /// cannot disagree. + funded_headroom: usize, + /// Definition lines of this file's protected focuses (§3.1): roots whose name + /// a shape-precise query token names. Every focus line is guaranteed to land + /// inside some emitted window. + focuses: &'a [usize], + drifted: bool, +} + +/// Decimal digit count, for costing a numbered source line without rendering it. +fn decimal_width(n: usize) -> usize { + let mut width = 1; + let mut rest = n; + while rest >= 10 { + rest /= 10; + width += 1; } + width } /// Cluster source range used when sizing a god-file (`tools.ts:2708-2718`). @@ -3081,6 +3488,51 @@ fn query_terms(query: &str) -> Vec { out } +/// Whether a query token is SHAPE-PRECISE enough to name a symbol rather than +/// describe one (upstream `89c53dd`'s `isPreciseToken`): at least 3 chars, and +/// either carrying a separator, or a lower→UPPER transition, or an initial +/// capital. +/// +/// This is what keeps a prose query from minting protected focuses: +/// `hugeHandler` and `Foo::Bar` qualify, `upload` and `work` do not. +fn is_precise_query_token(token: &str) -> bool { + if token.chars().count() < 3 { + return false; + } + if token.contains(['.', '_', '$', '/']) || token.contains("::") { + return true; + } + let chars: Vec = token.chars().collect(); + if chars[0].is_uppercase() { + return true; + } + chars + .windows(2) + .any(|w| w[0].is_lowercase() && w[1].is_uppercase()) +} + +/// The query's shape-precise tokens, in query order and deduped. +/// +/// Split on WHITESPACE, unlike `query_terms`: that helper lowercases and strips +/// every separator, which would erase the very shape `is_precise_query_token` +/// tests. Only punctuation that cannot belong to an identifier is trimmed, so +/// `work?` reduces to prose while `Foo::Bar,` keeps its separator. +fn precise_query_tokens(query: &str) -> Vec { + let mut out: Vec = Vec::new(); + for tok in query.split_whitespace() { + let t = tok.trim_matches(|c: char| { + c.is_ascii_punctuation() && !matches!(c, '.' | '_' | '$' | '/' | ':') + }); + if !is_precise_query_token(t) { + continue; + } + if !out.iter().any(|e| e == t) { + out.push(t.to_string()); + } + } + out +} + /// Whether the query itself is about tests — keeps the legitimate "explore the /// tests" case (`tools.ts:2228`). fn query_mentions_tests(query: &str) -> bool { @@ -3093,6 +3545,140 @@ fn query_mentions_tests(query: &str) -> bool { }) } +/// Kinds the ambient-declaration predicate ignores: they describe a file's +/// plumbing rather than what it declares (upstream `9efae0f`). +fn is_ambient_bookkeeping_kind(kind: NodeKind) -> bool { + matches!( + kind, + NodeKind::File | NodeKind::Import | NodeKind::Export | NodeKind::Parameter + ) +} + +/// Type-declaration kinds — upstream's exact set. A file whose every +/// non-bookkeeping node is one of these declares types and nothing else. +fn is_type_declaration_kind(kind: NodeKind) -> bool { + matches!( + kind, + NodeKind::Interface + | NodeKind::TypeAlias + | NodeKind::Enum + | NodeKind::EnumMember + | NodeKind::Namespace + ) +} + +/// Header of the "Not shown above" pointer block (`tools.ts:2910-2927`). +const POINTER_BLOCK_HEADER: &str = "### Not shown above — explore these names for their source"; + +/// Horizontal rule opening the completeness block (`tools.ts:2933-2940`). +const COMPLETENESS_RULE: &str = "---"; + +/// The trim note emitted at tiers that gate the completeness signal OFF. It is +/// only reached when a file was actually trimmed, but `any_file_trimmed` is +/// loop-determined, so the pre-loop reserve has to assume it. +const TRIMMED_NOTE: &str = "> Some file sections were trimmed for size. For a specific symbol you still need, run another `codegraph_explore` (or `codegraph_node`) with its exact name — line-numbered source, cheaper and more complete than Read."; + +fn completeness_signal(files_included: usize) -> String { + format!( + "> **Complete source for {files_included} files is included above — do NOT re-read them.** If your question also needs files/symbols listed under \"Not shown above\" (or any area this call didn't cover), make ANOTHER codegraph_explore targeting those names — it returns the same source with line numbers and is cheaper and more complete than reading. Reserve Read for a single specific line range explore can't surface." + ) +} + +fn explore_budget_note(file_count: i64) -> String { + format!( + "> **{file_count} files indexed.** Each call covers ~6 files; if your question spans more, make ANOTHER `codegraph_explore` targeting the uncovered area BEFORE falling back to Read — another explore is cheaper and more complete than reading those files. There is no call limit." + ) +} + +fn pointer_unlisted_tail(unlisted: usize) -> String { + format!("- ... and {unlisted} more files") +} + +/// The epilogue's note lines — everything after the pointer block. +/// +/// The pre-loop reserve and the post-loop emitter both call THIS function, so the +/// reserve is derived from the very strings that get emitted and cannot drift from +/// them. The reserve passes the arguments that maximise the result, which is why +/// it is an upper bound rather than an estimate. +fn epilogue_note_lines( + budget: &ExploreOutputBudget, + files_included: usize, + any_file_trimmed: bool, + file_count: Option, +) -> Vec { + let mut lines = Vec::new(); + if budget.include_completeness_signal { + lines.push(COMPLETENESS_RULE.to_string()); + lines.push(completeness_signal(files_included)); + lines.push(String::new()); + } else if any_file_trimmed { + lines.push(TRIMMED_NOTE.to_string()); + lines.push(String::new()); + } + if budget.include_budget_note + && let Some(count) = file_count + { + lines.push(explore_budget_note(count)); + lines.push(String::new()); + } + lines +} + +/// What a set of pushed lines costs `lines.join("\n")`: its own length plus the +/// one separator each is pushed behind. +fn joined_line_cost(lines: &[String]) -> usize { + lines.iter().map(|l| 1 + l.len()).sum() +} + +/// The epilogue's FIXED reserve, subtracted from `max_output_chars` before any +/// file is admitted so the sections cannot spend what the epilogue will need. +/// +/// Every term is the length of a string the epilogue actually emits, evaluated at +/// its worst case: `max_files` maximises the completeness signal's digit width, +/// `any_file_trimmed = true` selects the larger of the two mutually exclusive +/// notes, and the unlisted tail is reserved at `file_order` length, which is +/// >= any achievable count because every unlisted file came from `file_order`. +fn fixed_epilogue_reserve( + budget: &ExploreOutputBudget, + file_order_len: usize, + max_files: usize, + file_count: Option, +) -> usize { + let mut reserve = 0usize; + if budget.include_additional_files { + reserve += joined_line_cost(&[ + POINTER_BLOCK_HEADER.to_string(), + String::new(), + pointer_unlisted_tail(file_order_len), + String::new(), + ]); + } + reserve += joined_line_cost(&epilogue_note_lines(budget, max_files, true, file_count)); + reserve +} + +/// Fit the pointer lines into the headroom the source sections did not use, +/// stopping at the first that does not fit, and report how many files went +/// unlisted — by `POINTER_MAX_FILES` and by elasticity together. +/// +/// The iteration order IS part of the output contract: `excluded_files` arrives in +/// `file_order` rank order and the loop stops at the first line that does not fit, +/// so the ORDER selects the SET. A best-fit variant would use the budget better +/// and is deliberately rejected — it would make the emitted set depend on +/// arithmetic rather than on rank, which no caller could predict from the ranking. +fn fit_pointer_lines(candidates: &[String], pointer_budget: usize) -> (Vec, usize) { + let mut emitted = Vec::new(); + let mut spent = 0usize; + for line in candidates.iter().take(POINTER_MAX_FILES) { + if spent + 1 + line.len() > pointer_budget { + break; + } + spent += 1 + line.len(); + emitted.push(line.clone()); + } + (emitted.clone(), candidates.len() - emitted.len()) +} + /// Cut `output` at the last `#### ` file-section boundary before `ceiling` so /// trailing whole sections drop rather than slicing a method body; fall back to /// a line boundary in the degenerate single-giant-section case @@ -3173,6 +3759,17 @@ mod tests { } } + /// A direct-call render context with nothing already spent, mirroring what + /// `handle_explore` builds for the FIRST file of a response. + fn render_ctx<'a>(budget: &'a ExploreOutputBudget, focuses: &'a [usize]) -> RenderCtx<'a> { + RenderCtx { + budget, + funded_headroom: budget.max_output_chars.saturating_sub(1), + focuses, + drifted: false, + } + } + fn subgraph_with(nodes: Vec, roots: Vec) -> ExploreSubgraph { let mut sg = ExploreSubgraph::default(); for n in nodes { @@ -3203,7 +3800,7 @@ mod tests { let budget = crate::explore_budget::get_explore_output_budget(200); let out = engine - .render_explore_file(&sg, file, &file_lines, "rust", &budget, false) + .render_explore_file(&sg, file, &file_lines, "rust", &render_ctx(&budget, &[])) .section; assert!( out.contains(file), @@ -3224,7 +3821,7 @@ mod tests { let budget = crate::explore_budget::get_explore_output_budget(200); let out = engine - .render_explore_file(&sg, file, &file_lines, "rust", &budget, false) + .render_explore_file(&sg, file, &file_lines, "rust", &render_ctx(&budget, &[])) .section; assert!(out.contains(file), "should render a header, got: {out:?}"); } @@ -3243,7 +3840,7 @@ mod tests { let budget = crate::explore_budget::get_explore_output_budget(200); let out = engine - .render_explore_file(&sg, file, &file_lines, "rust", &budget, false) + .render_explore_file(&sg, file, &file_lines, "rust", &render_ctx(&budget, &[])) .section; assert!(out.contains("line 10"), "cluster source missing: {out:?}"); assert!(out.contains("line 30"), "cluster source missing: {out:?}"); @@ -3262,7 +3859,7 @@ mod tests { let budget = crate::explore_budget::get_explore_output_budget(200); let out = engine - .render_explore_file(&sg, file, &file_lines, "rust", &budget, false) + .render_explore_file(&sg, file, &file_lines, "rust", &render_ctx(&budget, &[])) .section; let expected = { let numbered = number_source_lines_at(&file_lines, 1); @@ -3271,6 +3868,223 @@ mod tests { assert_eq!(out, expected); } + /// Tier 5 item 12 (CG-38). Given the shape-precise token test, When each of a + /// symbol-like and a prose-like token is checked, Then only the symbol-like + /// ones qualify — so ordinary prose can never mint a protected focus. + #[test] + fn precise_query_token_shape() { + for t in ["hugeHandler", "processPayroll", "Foo::Bar", "a_b"] { + assert!(is_precise_query_token(t), "{t} must be precise"); + } + for t in ["upload", "work", "id"] { + assert!(!is_precise_query_token(t), "{t} must NOT be precise"); + } + } + + /// Tier 5 item 12's affordability gate, INERT paths. Given a small whole-file + /// eligible file, When the funded headroom can pay for it, Then the complete + /// section is returned even though a focus exists; and When the headroom + /// cannot pay for it but the file owns NO focus, Then the complete section is + /// still returned so the caller drops it whole. + /// + /// Case (a) is the guard on gate (1) itself: measured, deleting that gate + /// windows an affordable focus-owning file, fails this test, AND drifts + /// `explore_matches_golden_structural`, whose mini fixture takes this very + /// branch. Merely SWAPPING the two conditions stays green — also measured — + /// because gate (1) still catches the affordable case; what the swap costs is + /// the golden-neutrality ARGUMENT, which then rests on the fixture holding no + /// precise token rather than on the code. That is why the order is asserted + /// structurally rather than here. + #[test] + fn whole_file_section_is_returned_whole_when_it_fits() { + let engine = test_engine(); + let file = "small.ts"; + let owned: Vec = (1..=20) + .map(|i| format!("export function tinyFocus{i}(): number {{ return {i}; }}")) + .collect(); + let file_lines: Vec<&str> = owned.iter().map(String::as_str).collect(); + let n = node("tinyFocus1", file, 1, 1, NodeKind::Function); + let sg = subgraph_with(vec![n.clone()], vec![n.id.clone()]); + let budget = crate::explore_budget::get_explore_output_budget(3); + let whole = { + let numbered = number_source_lines_at(&file_lines, 1); + format!("#### {file} — tinyFocus1(function)\n\n```typescript\n{numbered}\n```\n") + }; + + let affordable_with_focus = engine + .render_explore_file( + &sg, + file, + &file_lines, + "typescript", + &render_ctx(&budget, &[1]), + ) + .section; + assert_eq!( + affordable_with_focus, whole, + "an AFFORDABLE focus-owning file must come back whole" + ); + assert!(!affordable_with_focus.contains("... (gap) ...")); + + let starved = RenderCtx { + budget: &budget, + funded_headroom: whole.len() - 1, + focuses: &[], + drifted: false, + }; + let unaffordable_no_focus = engine + .render_explore_file(&sg, file, &file_lines, "typescript", &starved) + .section; + assert_eq!( + unaffordable_no_focus, whole, + "an unaffordable file owning NO focus must still be returned whole, \ + so the caller drops it whole" + ); + } + + /// Tier 5 item 12 (CG-38). Given NINE qualifying focuses in one file, When the + /// focus list is selected, Then exactly the six LOWEST `start_line`s are + /// protected — and the same twice in one process, so a hash-ordered + /// implementation cannot pass. + #[test] + fn focus_cap_is_six_and_selection_is_line_ordered() { + let file = "many.ts"; + let nodes: Vec = (0..9) + .map(|i| { + node( + &format!("focusTarget{i}"), + file, + 10 + i * 10, + 12 + i * 10, + NodeKind::Function, + ) + }) + .collect(); + let roots: Vec = nodes.iter().map(|n| n.id.clone()).collect(); + let sg = subgraph_with(nodes, roots); + let precise: Vec = (0..9).map(|i| format!("focusTarget{i}")).collect(); + + let first = sg.file_focus_lines(file, &precise); + let second = sg.file_focus_lines(file, &precise); + assert_eq!(first, vec![10, 20, 30, 40, 50, 60], "cap or order wrong"); + assert_eq!( + first, second, + "focus selection is not stable within a process" + ); + } + + /// Tier 5 item 13 half A (CG-30). Given a 600-line file whose single symbol + /// spans all of it, so its only cluster renders ~13x the per-file cap, When + /// `render_explore_file` runs at the tiny budget, Then the returned SECTION — + /// frame included, the same quantity the caller's admission test compares — + /// obeys `round(max_chars_per_file * 1.5)` while still emitting whole source + /// lines. + /// + /// The unit is the SECTION and not the cluster body on purpose: `ceiling` + /// bounds bodies, so a ceiling that does not subtract the header/fence frame + /// satisfies the admission test yet overshoots this bound by the frame. + #[test] + fn render_explore_file_windows_an_oversize_first_cluster() { + let engine = test_engine(); + let file = "god.ts"; + let owned: Vec = (1..=600) + .map(|i| format!(" // padding line {i} inside hugeHandler keeping the body enormous")) + .collect(); + let file_lines: Vec<&str> = owned.iter().map(String::as_str).collect(); + let huge = node("hugeHandler", file, 1, 600, NodeKind::Function); + let sg = subgraph_with(vec![huge.clone()], vec![huge.id.clone()]); + let budget = crate::explore_budget::get_explore_output_budget(3); + + let section = engine + .render_explore_file( + &sg, + file, + &file_lines, + "typescript", + &render_ctx(&budget, &[]), + ) + .section; + + let bound = (budget.max_chars_per_file as f64 * 1.5).round() as usize; + assert!( + section.contains('\t'), + "an oversize first cluster must still emit numbered source: {section:?}" + ); + assert!( + section.len() <= bound, + "the returned section must obey the {bound}-char 1.5x per-file bound, got {}", + section.len() + ); + for line in section.lines() { + let Some((num, body)) = line.split_once('\t') else { + continue; + }; + let Ok(n) = num.parse::() else { + continue; + }; + assert_eq!( + file_lines[n - 1], + body, + "line {n} was sliced mid-line: {section}" + ); + } + } + + /// Tier 5 item 13 half B's guard. Given an OVER-CEILING first cluster — so + /// `projected` legitimately exceeds the room a later cluster could draw on — + /// When a later cluster is considered, Then it is still DROPPED rather than + /// emitted, and nothing panics. + /// + /// This is the input that underflows a plain `ceiling - projected`: in debug + /// it panics, in release it wraps to a huge `room` and hands the later cluster + /// unlimited budget. A merely tight-but-positive `room` does not exercise it. + #[test] + fn render_explore_file_skips_a_later_cluster_with_no_usable_room() { + let engine = test_engine(); + let file = "twin.ts"; + // Lines long enough that the first cluster's 12-line never-empty FLOOR + // alone costs more than the whole ceiling. That is the only way + // `projected` can exceed `ceiling`: half A otherwise bounds the first + // cluster's body by the ceiling, so a fixture with ordinary-length lines + // never reaches the subtraction at all. + let owned: Vec = (1..=900) + .map(|i| format!(" // padding line {i} {}", "-".repeat(600))) + .collect(); + let file_lines: Vec<&str> = owned.iter().map(String::as_str).collect(); + let head = node("headSymbol", file, 1, 400, NodeKind::Function); + let tail = node("tailSymbol", file, 600, 890, NodeKind::Function); + let sg = subgraph_with(vec![head.clone(), tail], vec![head.id.clone()]); + let budget = crate::explore_budget::get_explore_output_budget(3); + + let section = engine + .render_explore_file( + &sg, + file, + &file_lines, + "typescript", + &render_ctx(&budget, &[]), + ) + .section; + + let emitted: Vec = section + .lines() + .filter_map(|l| l.split_once('\t')) + .filter_map(|(n, _)| n.parse::().ok()) + .collect(); + assert!( + !emitted.is_empty(), + "the first cluster must still emit: {section}" + ); + assert!( + emitted.iter().all(|&n| n < 600), + "the later cluster had no usable room and must be dropped, got lines {emitted:?}" + ); + assert!( + section.contains("... (gap) ..."), + "a dropped later cluster must still be signalled: {section}" + ); + } + /// Given a stale index whose node `end_line` (10) is BELOW its `start_line` /// (50) — a malformed/shrunk-file row — When `get_code` slices the on-disk /// file, Then it returns Ok (graceful) instead of panicking with "slice @@ -4124,6 +4938,43 @@ mod tests { ); } + /// Given retained source that itself contains a line mimicking a `#### ` + /// section header for a DIFFERENT file, When that other file is dropped for the + /// budget, Then the spoofed marker must not make the response cite the dropped + /// file as a drifted source. + /// + /// The drop MECHANISM this exercises moved. It used to be the hard-ceiling cut, + /// and that is now unreachable by construction: `hard_ceiling` is + /// `min(max_output_chars * 1.5, 25_000)`, which is strictly GREATER than + /// `max_output_chars` at every tier (19,500 > 13,000; 25,000 > 18,000; + /// 25,000 > 24,000), while the accounting keeps the response inside + /// `max_output_chars`. So a file is dropped by the ADMISSION test instead, and + /// the assertions below pin that. Do not "restore" this test by widening the + /// ceiling. + /// + /// **Exactly what this test is and is not sensitive to, all measured by + /// reverting and re-running rather than reasoned about.** It fails only when the + /// pointer LINE cap and the elastic pointer BLOCK are BOTH absent — which is the + /// genuine pre-fix state, not a contrived mutant, and is why upstream needs both + /// (the cap bounds a line, elasticity bounds the block): + /// + /// | reverted | verdict | len | cut | ptr lines | completeness | + /// | --- | --- | --- | --- | --- | --- | + /// | nothing | pass | 22,266 | no | 10 | present | + /// | 6-symbol cap only | pass | 24,201 | no | 4 | present | + /// | elastic block only | pass | 22,266 | no | 10 | present | + /// | **cap + elastic block** | **FAIL** | 23,710 | **yes** | 4 | **destroyed** | + /// + /// It is NOT sensitive to the `+ 200` → `+ 1` real-cost charge, nor to the + /// pre-loop epilogue reserve: reverting either leaves this output byte-identical, + /// because with the cap in place all ten pointer lines fit the headroom anyway. + /// Those two are pinned elsewhere and deliberately not here — + /// `explore_output_respects_its_stated_budget` (the reserve, and the section + /// count when the real-cost charge is reverted), + /// `file_node_locations_caps_symbols_at_six` (the cap directly), and + /// `epilogue_pointer_block_fits_the_remaining_budget` (the elastic block's four + /// branches). This test is a whole-item regression check on the marker-collision + /// invariant, not per-change budget coverage. #[test] fn ext_explore_banner_ignores_marker_collision_from_retained_source() { let mut engine = test_engine(); @@ -4148,8 +4999,12 @@ mod tests { let dropped_indexed = "fn marker_collision_target_dropped() { indexed(); }\n"; put_indexed_source(&engine, dropped_file, dropped_indexed, Language::Rust, 1); + // Wide enough that its section cannot fit the headroom the retained file + // leaves, so it is rejected by the ADMISSION test by ~13 KB rather than by + // a knife-edge. Still inside the whole-file gates (<=220 lines, + // <=3 * 7000 chars) so it renders as one section when it does fit. let dropped_padding = (0..180) - .map(|i| format!("// dropped {i:04} yyyyy\n")) + .map(|i| format!("// dropped {i:04} {}\n", "y".repeat(90))) .collect::(); write_src( &engine, @@ -4177,35 +5032,65 @@ mod tests { ); put_nodes(&mut engine, &[retained.clone(), dropped.clone()]); - let long_component = "x".repeat(220); + // The caller files must be REAL pointer candidates, and that needs three + // things at once — each of which the fixture previously lacked: + // + // * source on disk. A file `project_source` cannot read is `continue`d + // BEFORE `excluded_files.push`, so it never reaches the pointer block at + // all. With DB rows alone the epilogue had exactly ONE candidate, which + // always fits, leaving the cap, the reserve and the elastic block inert. + // * a path component under the filesystem's 255-byte limit, so the source + // can actually be written. + // * many SUBGRAPH nodes each, since `file_node_locations` lists the + // subgraph's nodes for the file and only nodes with an edge to a root are + // reached. Uncapped, that is what makes one pointer line ~2 KB. + // + // Their sources are also wide enough to fail admission themselves, so they + // stay pointer candidates instead of becoming a second rendered section. + const CALLER_FILES: usize = 10; + const CALLER_SYMS: usize = 60; + let long_component = "x".repeat(200); let mut callers = Vec::new(); let mut edges = Vec::new(); - for i in 0..10 { + for i in 0..CALLER_FILES { let path = format!("callers/{i:02}_{long_component}.rs"); - put_file(&engine, &file_rec(&path, Language::Rust, 1)); - let caller = node_lang( - &format!("marker_collision_caller_{i}"), - &format!("marker_collision_caller_{i}"), - &path, - 1, - 1, - NodeKind::Function, - Language::Rust, - ); + let body = (0..CALLER_SYMS) + .map(|j| { + format!( + "fn bulk_pointer_sym_{i}_{j}() {{ /* {} */ }}\n", + "z".repeat(90) + ) + }) + .collect::(); + put_indexed_source(&engine, &path, &body, Language::Rust, CALLER_SYMS as i64); let target = if i < 5 { &retained } else { &dropped }; - edges.push(mk_edge( - &caller.id, - &target.id, - codegraph_core::types::EdgeKind::Calls, - )); - callers.push(caller); + for j in 0..CALLER_SYMS { + // Named without the query's own terms, so 600 extra nodes cannot + // crowd the two real targets out of the top-8 FTS roots. + let sym = node_lang( + &format!("bulk_pointer_sym_{i}_{j}"), + &format!("bulk_pointer_sym_{i}_{j}"), + &path, + (j + 1) as i64, + (j + 1) as i64, + NodeKind::Function, + Language::Rust, + ); + edges.push(mk_edge( + &sym.id, + &target.id, + codegraph_core::types::EdgeKind::Calls, + )); + callers.push(sym); + } } put_nodes(&mut engine, &callers); put_edges(&mut engine, &edges); - // Cross into the large-project budget tier so excluded caller files and - // completeness metadata push the accepted source sections past the final - // 25K hard ceiling. + // Cross into the large-project budget tier, which turns the epilogue meta + // ON — the "Not shown above" pointer block and the completeness signal. + // That block is what names an admission-dropped file, and it is what the + // hard-ceiling cut used to destroy. for i in 0..5001 { put_file( &engine, @@ -4222,13 +5107,39 @@ mod tests { text.contains(&format!("// #### {dropped_file} — spoofed source marker")), "the retained source must contain the colliding marker: {text}" ); + // The file left by ADMISSION, not by the cut and not by the file cap. + // `maxFiles` is 2, so a cap-drop would still have rendered two sections; + // exactly one section means the budget rejected the second. + let headers: Vec<&str> = text.lines().filter(|l| l.starts_with("#### ")).collect(); + assert_eq!( + headers.len(), + 1, + "maxFiles permits 2, so one section means admission rejected the other: {text}" + ); + assert!( + headers[0].contains(retained_file), + "the surviving section must be the retained file: {headers:?}" + ); + assert!( + !text.contains("output truncated to budget"), + "the response fits the inline ceiling, so the cut must NOT run: {text}" + ); + // ...and the epilogue survives INTACT, naming the dropped file and carrying + // a true count. Uncapped AND non-elastic — the pre-fix state — the cut + // deletes this block along with the section, so the response neither shows + // the file nor says it exists: measured, the completeness signal is gone and + // only 4 of the 10 pointer lines survive. + assert!( + text.contains(&format!("- {dropped_file}:")), + "an admission-dropped file must be named in the pointer block: {text}" + ); assert!( - text.contains("output truncated to budget"), - "the final hard-ceiling cut must run: {text}" + text.contains(&format!("Complete source for {} files", headers.len())), + "the epilogue must survive with a count matching the rendered sections: {text}" ); assert!( !text.contains("marker_collision_target_dropped() { current(); }"), - "the dropped file's source section must not survive the cut: {text}" + "the dropped file's source section must not survive: {text}" ); assert!( text.starts_with("⚠️ Some files referenced below"), @@ -5493,6 +6404,383 @@ mod tests { assert!(h.starts_with("#### f.rs — "), "got: {h}"); } + /// Tier 5 item 14 (CG-28). Given four files — a pure declaration shim, a shim + /// with an outgoing `Calls`, a shim another file imports, and one of mixed + /// kinds — When the explore subgraph is built, Then only the PURE shim is + /// marked ambient. + /// + /// The four conditions are each necessary: a file that runs code, a file + /// something depends on, and a file holding real implementations are all + /// legitimate top-ranked answers. + #[test] + fn ambient_declaration_predicate_four_conditions() { + let mut engine = test_engine(); + let mut nodes = Vec::new(); + let mut edges = Vec::new(); + + for (stem, decl) in [ + ("pure", "AmbientPureAlpha"), + ("caller", "AmbientCallerAlpha"), + ] { + let path = format!("src/types/{stem}.d.ts"); + put_indexed_source( + &engine, + &path, + &format!("export interface {decl} {{\n ambientId: string;\n}}\n"), + Language::TypeScript, + 1, + ); + nodes.push(node_lang( + decl, + decl, + &path, + 1, + 3, + NodeKind::Interface, + Language::TypeScript, + )); + } + // The `caller` shim additionally has a node that CALLS out, so condition 3 + // rejects it: it does not merely declare. + let runner = node_lang( + "ambientRunner", + "ambientRunner", + "src/types/caller.d.ts", + 5, + 7, + NodeKind::Function, + Language::TypeScript, + ); + let sink = node_lang( + "ambientSink", + "ambientSink", + "src/impl/sink.ts", + 1, + 3, + NodeKind::Function, + Language::TypeScript, + ); + put_indexed_source( + &engine, + "src/impl/sink.ts", + "export function ambientSink(): number {\n return 1;\n}\n", + Language::TypeScript, + 1, + ); + edges.push(mk_edge(&runner.id, &sink.id, EdgeKind::Calls)); + nodes.push(runner); + nodes.push(sink); + + // The `imported` shim is pure, but a DIFFERENT file references it, so + // condition 4 rejects it: something depends on it. + put_indexed_source( + &engine, + "src/types/imported.d.ts", + "export interface AmbientImportedAlpha {\n ambientId: string;\n}\n", + Language::TypeScript, + 1, + ); + let imported = node_lang( + "AmbientImportedAlpha", + "AmbientImportedAlpha", + "src/types/imported.d.ts", + 1, + 3, + NodeKind::Interface, + Language::TypeScript, + ); + let consumer = node_lang( + "ambientConsumer", + "ambientConsumer", + "src/impl/consumer.ts", + 1, + 3, + NodeKind::Function, + Language::TypeScript, + ); + put_indexed_source( + &engine, + "src/impl/consumer.ts", + "export function ambientConsumer(): number {\n return 2;\n}\n", + Language::TypeScript, + 1, + ); + edges.push(mk_edge(&consumer.id, &imported.id, EdgeKind::References)); + nodes.push(imported); + nodes.push(consumer); + + // The `mixed` file declares a type AND implements a function, so condition + // 2 rejects it. + put_indexed_source( + &engine, + "src/types/mixed.ts", + "export interface AmbientMixedAlpha {\n ambientId: string;\n}\nexport function ambientMixedRun(): number {\n return 3;\n}\n", + Language::TypeScript, + 2, + ); + nodes.push(node_lang( + "AmbientMixedAlpha", + "AmbientMixedAlpha", + "src/types/mixed.ts", + 1, + 3, + NodeKind::Interface, + Language::TypeScript, + )); + nodes.push(node_lang( + "ambientMixedRun", + "ambientMixedRun", + "src/types/mixed.ts", + 4, + 6, + NodeKind::Function, + Language::TypeScript, + )); + + put_nodes(&mut engine, &nodes); + put_edges(&mut engine, &edges); + + let sub = engine.find_relevant_context("Ambient").unwrap(); + assert!( + sub.ambient_files.contains("src/types/pure.d.ts"), + "the pure shim must be ambient; got {:?}", + sub.ambient_files + ); + for rejected in [ + "src/types/caller.d.ts", + "src/types/imported.d.ts", + "src/types/mixed.ts", + ] { + assert!( + !sub.ambient_files.contains(rejected), + "{rejected} must NOT be ambient; got {:?}", + sub.ambient_files + ); + } + } + + /// Tier 5 item 14 (CG-28), the combination rule. Given four files equal in + /// `tier` AND in `sym_count`, differing only in the generated and ambient + /// signals, When `finalize` ranks them, Then generated-only and + /// generated+ambient produce the SAME key, so their order falls through to the + /// path tie-break. + /// + /// Holding both other components fixed is the whole point: a four-boolean key + /// would rank generated+ambient strictly below generated-only and still satisfy + /// any assertion that let `tier` or `sym_count` vary. No weight may be 0 + /// either, or damping would cross a tier. + #[test] + fn ambient_and_generated_weight_ties_with_generated_only() { + // The two generated paths are named so that PATH order puts the + // generated+ambient one FIRST. That breaks a coincidence: with + // `c_generated` before `d_both`, a four-boolean key produces the same + // order as `min` and the assertion would discriminate nothing. Here the + // two disagree — `min` ties and the path tie-break wins (`c_both` first), + // while a fourth boolean ranks `d_generated` strictly higher. + let mut sg = ExploreSubgraph::default(); + let mut roots = Vec::new(); + // Distinct symbol names, because `node`'s synthetic id is derived from + // kind+name+line and four identical ones would collide into a single + // inserted node. Each file still contributes exactly one, so `sym_count` + // stays equal across all four. + for (i, stem) in ["a_plain", "b_ambient", "c_both", "d_generated"] + .into_iter() + .enumerate() + { + let path = format!("src/{stem}.ts"); + let n = node(&format!("SharedName{i}"), &path, 1, 3, NodeKind::Interface); + roots.push(n.id.clone()); + sg.insert(n); + } + sg.roots = roots; + sg.generated_files.insert("src/c_both.ts".to_string()); + sg.generated_files.insert("src/d_generated.ts".to_string()); + sg.ambient_files.insert("src/b_ambient.ts".to_string()); + sg.ambient_files.insert("src/c_both.ts".to_string()); + sg.finalize(); + + assert_eq!( + sg.file_order, + vec![ + "src/a_plain.ts", + "src/b_ambient.ts", + "src/c_both.ts", + "src/d_generated.ts", + ], + "plain > ambient > {{generated, generated+ambient TIED, ordered by path}}" + ); + } + + /// Tier 5 item 14 (CG-28), SCOPE. Given a shim whose only dependent references + /// a declaration the query does not match — so that dependent is genuinely + /// absent from the subgraph — When the ambient set is computed, Then the shim + /// is NOT damped, because condition 4 is evaluated index-wide. + /// + /// A subgraph-scoped predicate passes assertion 1 and fails assertion 2: it + /// sees zero inbound cross-file edges and damps a hand-written type module + /// other files genuinely import. `get_callers` walks inbound edges from the + /// ROOTS only, so a consumer pointing at a non-root declaration is unreachable + /// at any depth — which is what makes this a scope test and not a duplicate. + #[test] + fn ambient_predicate_sees_a_dependent_outside_the_subgraph() { + let build = |with_consumer: bool| { + let mut engine = test_engine(); + let shim = "src/types/env.d.ts"; + put_indexed_source( + &engine, + shim, + "export interface UploadAlpha {\n id: string;\n}\nexport interface UploadBeta {\n id: string;\n}\nexport interface UploadGamma {\n id: string;\n}\nexport interface LegacyChunkMeta {\n id: string;\n}\n", + Language::TypeScript, + 4, + ); + let mut nodes: Vec = ["UploadAlpha", "UploadBeta", "UploadGamma"] + .iter() + .enumerate() + .map(|(i, n)| { + node_lang( + n, + n, + shim, + 1 + (i as i64) * 3, + 3 + (i as i64) * 3, + NodeKind::Interface, + Language::TypeScript, + ) + }) + .collect(); + let legacy = node_lang( + "LegacyChunkMeta", + "LegacyChunkMeta", + shim, + 10, + 12, + NodeKind::Interface, + Language::TypeScript, + ); + nodes.push(legacy.clone()); + let mut edges = Vec::new(); + if with_consumer { + let consumer_path = "src/unrelated/consumer.ts"; + put_indexed_source( + &engine, + consumer_path, + "export function unrelatedConsumer(): number {\n return 1;\n}\n", + Language::TypeScript, + 1, + ); + let consumer = node_lang( + "unrelatedConsumer", + "unrelatedConsumer", + consumer_path, + 1, + 3, + NodeKind::Function, + Language::TypeScript, + ); + edges.push(mk_edge(&consumer.id, &legacy.id, EdgeKind::References)); + nodes.push(consumer); + } + put_nodes(&mut engine, &nodes); + put_edges(&mut engine, &edges); + engine.find_relevant_context("Upload").unwrap() + }; + + let sub = build(true); + assert!( + !sub.nodes + .iter() + .any(|n| n.file_path == "src/unrelated/consumer.ts"), + "the depending file must be genuinely OUTSIDE the subgraph, or this \ + proves nothing about scope; got {:?}", + sub.nodes.iter().map(|n| &n.file_path).collect::>() + ); + assert!( + !sub.ambient_files.contains("src/types/env.d.ts"), + "condition 4 must be evaluated index-wide; got {:?}", + sub.ambient_files + ); + + let twin = build(false); + assert!( + twin.ambient_files.contains("src/types/env.d.ts"), + "with no dependent anywhere the same shim IS ambient, so the \ + difference is attributable to condition 4 alone; got {:?}", + twin.ambient_files + ); + } + + /// Tier 5 item 15 (CG-26/31), the elastic pointer block. Given a set of + /// candidate pointer lines and the headroom the source sections left, When the + /// block is fitted, Then it never spends more than that headroom and the + /// unlisted count is TRUE — covering both files dropped by elasticity and + /// files dropped by the ten-file cap. + /// + /// A fixed pointer floor and an incremental per-file charge both fail these + /// cases, which is why they were rejected: the floor over-reserves and sheds a + /// source section, while the incremental charge discovers the cost only after + /// the admission it should have blocked. + #[test] + fn epilogue_pointer_block_fits_the_remaining_budget() { + let three: Vec = (0..3).map(|i| format!("- src/f{i}.ts: sym:1")).collect(); + let one_cost = 1 + three[0].len(); + + let (all, unlisted) = fit_pointer_lines(&three, one_cost * 3); + assert_eq!((all.len(), unlisted), (3, 0), "every line fits"); + + let (some, unlisted) = fit_pointer_lines(&three, one_cost * 2); + assert_eq!((some.len(), unlisted), (2, 1), "prefix emitted, 1 unlisted"); + assert!(some.iter().map(|l| 1 + l.len()).sum::() <= one_cost * 2); + + let (none, unlisted) = fit_pointer_lines(&three, one_cost - 1); + assert_eq!( + (none.len(), unlisted), + (0, 3), + "nothing fits ⇒ the tail must account for ALL candidates" + ); + + // Both drop mechanisms at once: 14 candidates, so the ten-file cap hides + // four, and a budget for two hides six more. + let many: Vec = (0..14) + .map(|i| format!("- src/g{i:02}.ts: sym:1")) + .collect(); + let two_cost = (1 + many[0].len()) * 2; + let (fitted, unlisted) = fit_pointer_lines(&many, two_cost); + assert_eq!( + (fitted.len(), unlisted), + (2, 12), + "N must count BOTH the cap's four and elasticity's eight" + ); + assert!(fitted.iter().map(|l| 1 + l.len()).sum::() <= two_cost); + } + + /// Tier 5 item 15 (CG-26/31). Given a file with 120 subgraph nodes, When its + /// "Not shown above" pointer line is built, Then it carries at most six + /// `name:line` pairs plus a `+114 more` tail. + /// + /// Uncapped, one such line reaches ~1,700 bytes and a single file's pointer + /// exhausts any plausible epilogue budget. The cap is asserted HERE rather + /// than on explore's output because the hard-ceiling cut deletes exactly the + /// long lines that would prove it: measured, the longest surviving `- src/…` + /// line in the defective output is 33 bytes. + #[test] + fn file_node_locations_caps_symbols_at_six() { + let nodes: Vec = (1..=120) + .map(|i| node(&format!("entry{i}"), "wide.ts", i, i, NodeKind::Method)) + .collect(); + let sg = subgraph_with(nodes, Vec::new()); + + let line = sg.file_node_locations("wide.ts"); + assert_eq!( + line.matches(':').count(), + 6, + "expected exactly 6 name:line pairs, got: {line}" + ); + assert!( + line.ends_with(", +114 more"), + "expected a +114 more tail, got: {line}" + ); + } + #[test] fn ext_subgraph_locations_and_language() { let n1 = node_lang("a", "a", "f.rs", 5, 6, NodeKind::Function, Language::Rust); diff --git a/crates/codegraph-mcp/tests/golden_mcp.rs b/crates/codegraph-mcp/tests/golden_mcp.rs index 1e7057b0..716d8869 100644 --- a/crates/codegraph-mcp/tests/golden_mcp.rs +++ b/crates/codegraph-mcp/tests/golden_mcp.rs @@ -943,6 +943,584 @@ fn explore_tiny_tier_budget_drops_whole_methods_not_mid_method() { } } +/// The Tier 5 `src/god.ts` shape (plan §2.1): two header comments, a blank, then +/// `hugeHandler` whose body is `pad_count` padding lines long, then +/// `blank_count` blank lines, then a 4-line `processPayroll`. +/// +/// `blank_count` is LOAD-BEARING and decides which defect the fixture exhibits. +/// With `gap_threshold` 7 at the tiny tier: +/// * `god_ts_file(600, 8)` — `hugeHandler` 4..606, `processPayroll` at 615, gap +/// `615 - 606 = 9 > 7` ⇒ **TWO** clusters (the §2.1 split variant). +/// * `god_ts_file(606, 3)` — `hugeHandler` 4..612, `processPayroll` at 616, gap +/// `616 - 612 = 4 <= 7` ⇒ **ONE** merged cluster (the §2.1.1 merge variant), +/// with the named `processPayroll` in the file's last 1% so no later-cluster +/// path can rescue it. +fn god_ts_file(pad_count: usize, blank_count: usize) -> String { + let mut src = String::from("// header line 1\n// header line 2\n\n"); + src.push_str("export function hugeHandler(x: number): number {\n"); + for j in 0..pad_count { + src.push_str(&format!( + " // padding line {j} inside hugeHandler keeping the body enormous\n" + )); + } + src.push_str(" return x;\n}\n"); + for _ in 0..blank_count { + src.push('\n'); + } + src.push_str( + "export function processPayroll(hours: number): number {\n const rate = 42;\n return hours * rate;\n}\n", + ); + src +} + +/// The five one-function filler files that keep a Tier 5 6-file fixture at the +/// tiny tier (plan §2.1). +fn greek_fillers() -> Vec<(String, String)> { + ["alpha", "beta", "gamma", "delta", "epsilon"] + .iter() + .map(|n| { + ( + format!("src/{n}.ts"), + format!("export function {n}Fn(): number {{\n return 1;\n}}\n"), + ) + }) + .collect() +} + +fn index_god_fixture(pad_count: usize, blank_count: usize) -> TestProject { + let god = god_ts_file(pad_count, blank_count); + let fillers = greek_fillers(); + let mut files: Vec<(&str, &str)> = vec![("src/god.ts", god.as_str())]; + for (rel, src) in &fillers { + files.push((rel.as_str(), src.as_str())); + } + index_fixture(&files) +} + +fn explore_text(project: &Path, query: &str) -> String { + let resp = roundtrip( + project, + json!({ + "jsonrpc": "2.0", + "id": 71, + "method": "tools/call", + "params": { + "name": "codegraph_explore", + "arguments": { "query": query, "projectPath": "/placeholder" } + } + }), + ); + resp["result"]["content"][0]["text"] + .as_str() + .expect("explore must return text") + .to_string() +} + +fn section_count(text: &str) -> usize { + text.lines().filter(|l| l.starts_with("#### ")).count() +} + +/// Tier 5 item 13 half A (CG-30). Given a query whose ONLY match is a symbol +/// whose body is 13x the per-file cap, When explore renders, Then the file's +/// top-ranked cluster is WINDOWED to a ceiling instead of accepted whole and +/// then dropped whole by the total-output admission test. +/// +/// Before the fix the whole 617-byte response carried ZERO `#### ` sections and +/// zero lines of source, while `codegraph query hugeHandler` found the symbol +/// fine — 4.7% of a 13,000-char budget spent, and no signal about which file was +/// lost (`includeAdditionalFiles` is gated off at the tiny tier). +#[test] +fn explore_single_oversize_match_still_returns_source() { + let project = index_god_fixture(600, 8); + let text = explore_text(project.path(), "hugeHandler"); + + assert!( + text.contains("#### src/god.ts"), + "the single oversize match must still produce a file section (was ZERO sections):\n{text}" + ); + assert!( + text.contains("export function hugeHandler"), + "the named definition line must render:\n{text}" + ); + // Bounded, not merely present: the response stays under the tiny tier's + // hard ceiling min(13000 * 1.5, 25000). + assert!( + text.len() <= 19_500, + "windowed output must respect the tiny-tier ceiling, got {} chars", + text.len() + ); + assert!( + text.len() > 1_000, + "a windowed section must actually deliver source, got {} chars:\n{text}", + text.len() + ); + // Whole-LINE windowing: every emitted numbered line must be byte-identical + // to the file line it claims, so nothing was sliced mid-line. + let src = god_ts_file(600, 8); + let file_lines: Vec<&str> = src.split('\n').collect(); + let mut emitted = 0usize; + for line in text.lines() { + let Some((num, body)) = line.split_once('\t') else { + continue; + }; + let Ok(n) = num.parse::() else { + continue; + }; + assert_eq!( + file_lines[n - 1], + body, + "line {n} was sliced mid-line:\n{text}" + ); + emitted += 1; + } + assert!(emitted > 0, "no numbered source lines emitted:\n{text}"); +} + +/// Tier 5 item 13 half B (CG-36). Given a query naming TWO symbols in one file +/// that land in separate clusters, and the second-ranked cluster is far bigger +/// than the room left, When explore renders, Then that cluster is SHRUNK into +/// the remainder instead of skipped whole. +/// +/// Before the fix the 848-byte response rendered only `processPayroll` — the +/// denser cluster wins the density tie-break, takes the first-cluster accept, +/// and `hugeHandler`'s ~42 KB cluster is then whole-or-nothing. The file header +/// read `processPayroll(function)` alone, so the agent had no signal that the +/// other symbol it named even existed in that file. +#[test] +fn explore_named_later_cluster_is_shrunk_not_dropped() { + let project = index_god_fixture(600, 8); + let text = explore_text(project.path(), "hugeHandler processPayroll"); + + assert!( + text.contains("export function processPayroll"), + "the denser named cluster must still render:\n{text}" + ); + assert!( + text.contains("export function hugeHandler"), + "the later named cluster must be shrunk in, not dropped whole:\n{text}" + ); + // The header is built from the CHOSEN clusters' symbols, so a fix that + // renders the body but leaves the header fed from the old chosen set would + // still mislabel the file exactly as before. + let header = text + .lines() + .find(|l| l.starts_with("#### src/god.ts")) + .unwrap_or_else(|| panic!("no src/god.ts section:\n{text}")); + assert!( + header.contains("hugeHandler"), + "the file header must LIST the shrunk-in symbol, got {header:?}" + ); +} + +/// Tier 5 item 12 (CG-38), cluster-tail half. Given ONE oversize cluster whose +/// explicitly named `processPayroll` sits in the file's last 1%, When explore +/// renders, Then a window covering that definition is allocated — a head window +/// alone reaches only ~L100. +/// +/// The MERGE shape is required here, not the split shape: with a single cluster, +/// item 13's later-cluster shrink never executes, so this cannot be closed by +/// Group 2 and stays RED for item 12's own reason. On the split shape the same +/// assertion is made green by item 13 and would prove nothing. +#[test] +fn explore_named_definition_line_always_renders() { + let project = index_god_fixture(606, 3); + let text = explore_text(project.path(), "hugeHandler processPayroll"); + + assert!( + text.contains("#### src/god.ts"), + "a section must exist (was 632 bytes, ZERO sections):\n{text}" + ); + assert!( + text.contains("export function processPayroll"), + "the named TAIL definition must render, not just the head window:\n{text}" + ); + assert!( + text.contains("export function hugeHandler"), + "the head focus must not be displaced by the tail one:\n{text}" + ); + // Still ONE cluster: if a future edit widened the blank-line gap past + // `gap_threshold`, this fixture would silently become the split shape and the + // assertion above would be satisfied by item 13's shrink instead. + assert_eq!( + section_count(&text), + 1, + "the merge-shape gap invariant no longer holds:\n{text}" + ); +} + +/// The Tier 5 escape-A fixture (plan §2.1.2): two files that BOTH clear both +/// whole-file gates, so `render_explore_file` returns a COMPLETE section before +/// any ceiling, focus, or window code exists — and the caller then drops the +/// second one whole. +/// +/// `src/aHogOne.ts` (145 lines) wins rank #1 on `sym_count` 9-vs-1 and consumes +/// the headroom; `src/zTarget.ts` (37 lines) owns the named +/// `reconcileTargetZulu` at L36 of 37, so a head-only window cannot rescue it. +/// Both files must stay INSIDE both gates: push either past one and escape A +/// silently becomes escape B, which the clustering ceiling already fixes. +fn index_escape_a_fixture() -> TestProject { + let pad = |tag: &str, i: usize| { + let head = format!(" // pad {i} for {tag} "); + format!("{head}{}", "-".repeat(65 - head.len())) + }; + let mut hog = String::from("export class HogLedgerOne {\n"); + for i in 0..135 { + hog.push_str(&pad("One", i)); + hog.push('\n'); + } + for k in 0..8 { + hog.push_str(&format!( + " hogOneStep{k}(v: number): number {{ return v + {k}; }}\n" + )); + } + hog.push_str("}\n"); + let mut tgt = String::from("export class TargetLedgerZulu {\n"); + for i in 0..32 { + tgt.push_str(&pad("Zulu", i)); + tgt.push('\n'); + } + for k in 0..2 { + tgt.push_str(&format!( + " helperTargetZulu{k}(v: number): number {{ return v + {k}; }}\n" + )); + } + tgt.push_str(" reconcileTargetZulu(v: number): number { return v + 99; }\n}\n"); + let fillers = greek_fillers(); + let mut files: Vec<(&str, &str)> = vec![ + ("src/aHogOne.ts", hog.as_str()), + ("src/zTarget.ts", tgt.as_str()), + ]; + for (rel, src) in fillers.iter().take(4) { + files.push((rel.as_str(), src.as_str())); + } + index_fixture(&files) +} + +/// Tier 5 item 12 (CG-38), WHOLE-FILE half. Given a focus-owning file that is +/// whole-file-eligible but UNAFFORDABLE behind a bigger rank-#1 file, When +/// explore renders, Then it falls through to the clustering path and emits a +/// window carrying the named definition — instead of returning a complete +/// section the caller then discards whole. +/// +/// This is a SECOND red for item 12 because the cluster-tail fixture cannot +/// reach it: that file is 619 lines and clusters, while both files here return at +/// the whole-file branch before any focus code runs. The two item-12 fixtures sit +/// on disjoint branches by construction. +#[test] +fn explore_named_definition_survives_the_whole_file_branch() { + let project = index_escape_a_fixture(); + let text = explore_text(project.path(), "HogLedgerOne reconcileTargetZulu"); + + assert!( + text.contains("#### src/zTarget.ts"), + "the focus owner was absent entirely (10,616 bytes, ONE section):\n{text}" + ); + assert!( + text.contains("reconcileTargetZulu(v: number)"), + "the explicitly named definition must render:\n{text}" + ); + // The response must GAIN a section, not trade one: a fix that admits the + // focus owner by evicting the rank-#1 hog is not the fix. + assert!( + text.contains("#### src/aHogOne.ts"), + "the budget hog must not be evicted to make room:\n{text}" + ); + assert_eq!(section_count(&text), 2, "expected both sections:\n{text}"); + // The tier's STATED budget, not merely the 19,500 hard ceiling. + assert!( + text.len() <= 13_000, + "response must fit max_output_chars, got {} chars", + text.len() + ); + assert!( + !text.contains("output truncated to budget"), + "the hard-ceiling cut must stay unfired:\n{text}" + ); +} + +/// A >=500-file project: `filler_count` one-function files plus `ledgers` +/// `Ledger*` classes of `methods` one-line methods each. The fillers only buy the +/// tier; the `Ledger*` files are what the query selects. +fn index_ledger_fixture(filler_count: usize, ledgers: usize, methods: usize) -> TestProject { + const NAMES: [&str; 12] = [ + "Alpha", "Bravo", "Charlie", "Delta", "Echo", "Foxtrot", "Golf", "Hotel", "India", + "Juliet", "Kilo", "Lima", + ]; + let mut owned: Vec<(String, String)> = Vec::new(); + for i in 1..=filler_count { + owned.push(( + format!("src/filler{i}.ts"), + format!("export function filler{i}(x: number): number {{\n return x + {i};\n}}\n"), + )); + } + for name in NAMES.iter().take(ledgers) { + let mut src = format!("export class Ledger{name} {{\n"); + for k in 1..=methods { + src.push_str(&format!( + " entry{k}(x: number): number {{ return x + {k}; }}\n" + )); + } + src.push_str("}\n"); + owned.push((format!("src/ledger_{name}.ts"), src)); + } + let files: Vec<(&str, &str)> = owned + .iter() + .map(|(rel, src)| (rel.as_str(), src.as_str())) + .collect(); + index_fixture(&files) +} + +/// Tier 5 item 15 (CG-26/31), pointer-list half. Given a >=500-file project whose +/// candidate files each contribute ~120 subgraph nodes, so one uncapped pointer +/// line runs to ~1,700 bytes, When explore renders, Then it delivers the same +/// number of source sections as a control query over the same three files. +/// +/// Before the fix the epilogue — charged NOTHING by the accountant — pushed the +/// response past the hard ceiling, and the cut then discarded a fully rendered, +/// already-charged 6.4 KB section: 2 sections against the control's 3, and +/// 13,442 of a 24,000 budget delivered. The control is asserted at 3 rather than +/// compared loosely, so the criterion cannot pass with the fixture absent. +#[test] +fn explore_epilogue_never_costs_a_rendered_section() { + let project = index_ledger_fixture(512, 12, 120); + let text = explore_text(project.path(), "Ledger"); + let control = explore_text(project.path(), "LedgerAlpha LedgerBravo LedgerCharlie"); + + assert_eq!( + section_count(&control), + 3, + "control must deliver 3 sections:\n{control}" + ); + assert_eq!( + section_count(&text), + section_count(&control), + "the epilogue cost a rendered section (was 2 vs 3):\n{text}" + ); + assert!( + text.len() <= 24_000, + "response must fit its STATED budget, got {} chars", + text.len() + ); + assert!( + !text.contains("output truncated to budget"), + "the hard-ceiling cut must no longer fire:\n{text}" + ); + assert!( + text.contains(&format!( + "Complete source for {} files", + section_count(&text) + )), + "the completeness signal must survive with a TRUE count:\n{text}" + ); +} + +/// The Tier 5 accounting fixture (plan §2.3(d)): 515 fillers plus ten +/// `ledger{Name}.ts` of 108 lines / ~7.0 KB — one class, 102 padding comments of +/// exactly 65 chars, and only FOUR methods. +/// +/// Four constraints hold simultaneously and the band between the middle two is +/// at most 1,000 bytes wide by construction: believed cost <= 24,000 so three +/// files are admitted, real cost > 24,000 so the overrun is observable, real cost +/// <= 25,000 so the hard-ceiling cut stays unfired and cannot mask it, and only 5 +/// subgraph nodes per file so the 6-symbol pointer cap is a verified NO-OP — +/// which is what attributes the failure to the accounting rather than the cap. +fn index_accounting_fixture() -> TestProject { + const NAMES: [&str; 10] = [ + "Alpha", "Bravo", "Charlie", "Delta", "Echo", "Foxtrot", "Golf", "Hotel", "India", "Juliet", + ]; + let mut owned: Vec<(String, String)> = Vec::new(); + for i in 0..515 { + owned.push(( + format!("src/filler{i}.ts"), + format!("export function wobble{i}(): number {{\n return {i};\n}}\n"), + )); + } + for name in NAMES { + let mut src = format!("export class Ledger{name} {{\n"); + for i in 0..102 { + let head = format!(" // pad {i} for {name} "); + src.push_str(&format!("{head}{}\n", "-".repeat(65 - head.len()))); + } + for k in 0..4 { + src.push_str(&format!( + " reconcileLedger{name}Segment{k}(v: number): number {{ return v + {k}; }}\n" + )); + } + src.push_str("}\n"); + owned.push((format!("src/ledger{name}.ts"), src)); + } + let files: Vec<(&str, &str)> = owned + .iter() + .map(|(rel, src)| (rel.as_str(), src.as_str())) + .collect(); + index_fixture(&files) +} + +/// Tier 5 item 15 (CG-26/31), ACCOUNTING half. Given a >=500-file project whose +/// three admitted sections leave the accountant believing it is inside budget, +/// When explore renders, Then the response fits its own `max_output_chars` — +/// without shedding a section to get there. +/// +/// Before the fix it over-delivered 720 bytes past its stated 24,000: each +/// section was charged a phantom flat +200 while the epilogue's 1,600 bytes were +/// charged NOTHING, and here the under-charge dominates. Every pointer line on +/// this fixture carries exactly 5 pairs, so the 6-symbol cap changes nothing and +/// the failure is attributable to the accounting alone. The 3-section assertion +/// is the half that blocks the one plausible lazy fix — shedding source to fit. +#[test] +fn explore_output_respects_its_stated_budget() { + let project = index_accounting_fixture(); + let text = explore_text(project.path(), "Ledger"); + + assert!( + text.len() <= 24_000, + "response must respect its OWN stated budget, got {} chars", + text.len() + ); + assert_eq!( + section_count(&text), + 3, + "must not have got there by dropping source:\n{text}" + ); + assert!( + !text.contains("output truncated to budget"), + "the hard-ceiling cut must stay unfired, so the overrun is unmasked:\n{text}" + ); + assert!( + text.contains("Complete source for 3 files"), + "the completeness signal must be present with a TRUE count:\n{text}" + ); + // The exclusion must still be STATED even when no pointer line fits: five + // candidates are excluded here and the remaining budget cannot hold a + // 166-byte line, so the tail is the only honest way to say so. + assert!( + text.lines().any(|l| l == "- ... and 5 more files"), + "the unlisted count must be stated:\n{text}" + ); +} + +/// The Tier 5 ambient-damping fixture (plan §2.2): four hand-written ambient +/// shims that nothing imports, plus the real implementation. +/// +/// Every declaration name must start with `Upload`: FTS matches `upload` inside +/// `UploadOptions`, and a prefixed variant (`GUploadOptions`) silently deletes the +/// reproduction — the shims never become roots, never reach tier 2, and the +/// inversion disappears. +fn index_ambient_shim_fixture() -> TestProject { + const SHIMS: [(&str, &str); 4] = [ + ( + "globals", + "Options Result Handle Session Chunk Meta Policy Target Cursor Token Limits Retry", + ), + ( + "vendor", + "Alpha Beta Gamma Delta Epsilon Zeta Eta Theta Iota Kappa Lambda Mu", + ), + ( + "platform", + "One Two Three Four Five Six Seven Eight Nine Ten Eleven Twelve", + ), + ( + "env", + "Nu Xi Omicron Pi Rho Sigma Tau Upsilon Phi Chi Psi Omega", + ), + ]; + let mut owned: Vec<(String, String)> = Vec::new(); + for (stem, suffixes) in SHIMS { + let mut src = String::from( + "// ambient shim - hand written, declarations only, nothing depends on it\n", + ); + for suf in suffixes.split_whitespace() { + src.push_str(&format!( + "export interface Upload{suf} {{\n uploadId: string;\n}}\n" + )); + } + owned.push((format!("src/types/{stem}.d.ts"), src)); + } + owned.push(( + "src/upload.ts".to_string(), + "import { readChunk } from './chunker';\nexport function uploadFile(path: string): number {\n const c = readChunk(path);\n return c + 1;\n}\n".to_string(), + )); + owned.push(( + "src/chunker.ts".to_string(), + "export function readChunk(p: string): number {\n return p.length;\n}\n".to_string(), + )); + let files: Vec<(&str, &str)> = owned + .iter() + .map(|(rel, src)| (rel.as_str(), src.as_str())) + .collect(); + index_fixture(&files) +} + +/// Tier 5 item 14 (CG-28). Given a flow query, When explore ranks the files, Then +/// the implementation outranks an ambient shim that nothing depends on — while the +/// delivered SET and every section length stay exactly as they were. +/// +/// Item 14 mutates `finalize`'s ordering key and nothing else, so the assertions +/// are in that dimension: ORDER changes, set and sizes do not. There is +/// deliberately no share assertion — at this fixture's scale nothing is dropped +/// and nothing is resized, so the ambient share is invariant at ~72.5% in all six +/// orderings and any threshold on it would fail identically before and after. +/// `unchanged set` is also what rejects the laziest wrong fix: dropping `.d.ts` +/// from the file order scores a 0% share and would sail through one. +#[test] +fn explore_damps_ambient_shim_below_implementation() { + let project = index_ambient_shim_fixture(); + let text = explore_text(project.path(), "how does upload work"); + + let order: Vec<&str> = text + .lines() + .filter(|l| l.starts_with("#### ")) + .filter_map(|l| l.split_whitespace().nth(1)) + .collect(); + // Measured on this harness pre-fix: `["src/types/globals.d.ts", + // "src/upload.ts"]`. It is TWO files rather than the three a fully-resolved + // index delivers, because `index_fixture` runs no cross-file resolution and + // `chunker.ts` is therefore never reached — which changes the delivered set but + // not the inversion this test measures. WHICH shim FTS puts first is a search + // detail, so the shim is matched by suffix rather than by name. + assert_eq!( + order.len(), + 2, + "the delivered SET moved; damping must reorder, not drop: {order:?}" + ); + assert_eq!( + order[0], "src/upload.ts", + "the implementation must be rank #1 (was the ambient shim): {order:?}" + ); + let shim = order + .iter() + .position(|f| f.ends_with(".d.ts")) + .unwrap_or_else(|| panic!("the shim must still be DELIVERED, only damped: {order:?}")); + let impl_pos = order.iter().position(|f| *f == "src/upload.ts").unwrap(); + assert!(impl_pos < shim, "the shim still outranks it: {order:?}"); + assert!( + text.contains("export function uploadFile"), + "no-regression: the implementation must still be delivered:\n{text}" + ); +} + +/// Tier 5 item 14 (CG-28), EXEMPTION. Given a query that NAMES a type the shim +/// declares, When explore ranks the files, Then the shim keeps rank #1 — asking +/// for a type is not the flow query damping exists to fix. +/// +/// This passes before the fix too, and is labelled a no-regression check rather +/// than a detection: it is what catches OVER-damping. +#[test] +fn explore_named_type_query_exempts_the_ambient_file() { + let project = index_ambient_shim_fixture(); + let text = explore_text(project.path(), "UploadOptions"); + + let first = text + .lines() + .find(|l| l.starts_with("#### ")) + .unwrap_or_else(|| panic!("no section rendered:\n{text}")); + assert!( + first.contains(".d.ts"), + "a query naming the type must keep the shim first, got {first:?}" + ); +} + /// Regression: at a tier that ENABLES `includeAdditionalFiles` (>=5000 files), /// a file dropped for the total budget surfaces in the trailing "Not shown /// above" list so the agent can request it (`tools.ts:2910-2927`). We can't