From 38df824f82e4e7af46e3671d3ea30a76951214a4 Mon Sep 17 00:00:00 2001 From: Justin Dobbs Date: Wed, 21 Oct 2026 14:23:10 -0600 Subject: [PATCH] =?UTF-8?q?improve=20LocalScorer=20symbol=20matching=20to?= =?UTF-8?q?=20handle=20all-caps=20constants,=20add=20file-type=20demotions?= =?UTF-8?q?=20for=20test/doc/tool/example=20files,=20update=20benchmark=20?= =?UTF-8?q?results=20with=20smart-fallback=20comparison=20showing=2081%=20?= =?UTF-8?q?=E2=86=92=2088%=20recall=20on=20httpx,=20add=20roadmap=20phases?= =?UTF-8?q?=20for=20precision=20improvements=20and=20async=20scoring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/benchmarks.md | 189 +++++++++++++++++++++++------------- pyproject.toml | 3 + reflex_search/scorers.py | 69 ++++++++++--- roadmap.md | 122 +++++++++++++---------- tools/bench.py | 43 +++++++- tools/precision_analysis.py | 137 ++++++++++++++++++++++++++ 6 files changed, 433 insertions(+), 130 deletions(-) create mode 100644 tools/precision_analysis.py diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 98c15c1..5eda840 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -4,93 +4,152 @@ Recall@k, precision@k, and reciprocal rank are macro-averaged across queries; precision@k uses the requested k as its denominator. -The independent Click suite uses upstream tag 8.5.0 and expects its checkout at `.benchmark-repos/click`; CI checks it out automatically. +The independent Click suite uses upstream tag 8.5.0 and expects its checkout at `.benchmark-repos/click`; the httpx suite uses upstream tag 0.28.1 at `.benchmark-repos/httpx`. CI checks out both automatically. ## reflex-code-search (local) | Query | Recall@k | Precision@k | MRR | Latency (ms) | Cost | Tokens | Hits / Expected | |-------|----------|------------|-----|-------------|------|--------|-----------------| -| How does the confidence gate decide whether to accept or rej... | 100% | 40% | 0.50 | 58 | $0.0000 | 0 | 2/2 | -| Where is the CLI entry point defined? | 100% | 40% | 0.50 | 50 | $0.0000 | 0 | 2/2 | -| How does the Jev API scorer work? | 100% | 20% | 1.00 | 42 | $0.0000 | 0 | 1/1 | -| What data model holds search results? | 100% | 20% | 0.50 | 48 | $0.0000 | 0 | 1/1 | -| How are files collected from the repository? | 100% | 20% | 1.00 | 46 | $0.0000 | 0 | 1/1 | -| Where are the public exports defined? | 100% | 20% | 0.50 | 42 | $0.0000 | 0 | 1/1 | -| How is the fallback scorer invoked for uncertain candidates? | 100% | 40% | 1.00 | 37 | $0.0000 | 0 | 2/2 | +| How does the confidence gate decide whether to accept or rej... | 100% | 40% | 1.00 | 72 | $0.0000 | 0 | 2/2 | +| Where is the CLI entry point defined? | 100% | 40% | 1.00 | 47 | $0.0000 | 0 | 2/2 | +| How does the Jev API scorer work? | 100% | 20% | 1.00 | 47 | $0.0000 | 0 | 1/1 | +| What data model holds search results? | 100% | 20% | 0.50 | 51 | $0.0000 | 0 | 1/1 | +| How are files collected from the repository? | 100% | 20% | 1.00 | 45 | $0.0000 | 0 | 1/1 | +| Where are the public exports defined? | 100% | 20% | 0.50 | 52 | $0.0000 | 0 | 1/1 | +| How is the fallback scorer invoked for uncertain candidates? | 100% | 40% | 1.00 | 43 | $0.0000 | 0 | 2/2 | | What file extensions are considered scannable? | 100% | 20% | 0.50 | 36 | $0.0000 | 0 | 1/1 | -| How is the audit trail structured? | 100% | 60% | 1.00 | 61 | $0.0000 | 0 | 3/3 | -| Where are unit tests for the confidence gate? | 100% | 20% | 1.00 | 50 | $0.0000 | 0 | 1/1 | -| How does the local scorer handle synonym expansion? | 100% | 20% | 1.00 | 36 | $0.0000 | 0 | 1/1 | -| Where is docstring boosting implemented? | 100% | 20% | 0.50 | 42 | $0.0000 | 0 | 1/1 | -| How does the two-pass live search reduce API calls? | 100% | 40% | 0.33 | 109 | $0.0000 | 0 | 2/2 | -| What aggregate telemetry is available after a search session... | 100% | 40% | 1.00 | 87 | $0.0000 | 0 | 2/2 | -| How does the smart Jev fallback differ from the default fall... | 100% | 20% | 1.00 | 49 | $0.0000 | 0 | 1/1 | -| Where is the benchmark harness that runs recall tests? | 100% | 40% | 0.25 | 56 | $0.0000 | 0 | 2/2 | -| What dependencies does the project require? | 100% | 20% | 1.00 | 42 | $0.0000 | 0 | 1/1 | -| How does the stemmer normalize query terms? | 100% | 20% | 1.00 | 34 | $0.0000 | 0 | 1/1 | -| Where are the gitignore exclusion rules applied during file ... | 100% | 40% | 0.50 | 50 | $0.0000 | 0 | 2/2 | -| How is the project architecture documented? | 100% | 20% | 1.00 | 42 | $0.0000 | 0 | 1/1 | -| How does Jev scoring protect secrets before sending file con... | 100% | 40% | 1.00 | 41 | $0.0000 | 0 | 2/2 | -| How does JevScorer fall back locally after an API error? | 100% | 20% | 1.00 | 32 | $0.0000 | 0 | 1/1 | -| How does the file collector use nested .gitignore patterns? | 100% | 40% | 1.00 | 31 | $0.0000 | 0 | 2/2 | -| Where are session telemetry fields defined and calculated? | 100% | 40% | 1.00 | 47 | $0.0000 | 0 | 2/2 | -| Where does the command line turn search results into JSON? | 100% | 20% | 0.25 | 42 | $0.0000 | 0 | 1/1 | -| What text suffixes are allowed and what file size is too lar... | 100% | 20% | 1.00 | 34 | $0.0000 | 0 | 1/1 | -| When does the search loop escalate an uncertain score to its... | 100% | 20% | 0.33 | 30 | $0.0000 | 0 | 1/1 | -| How does the local ranking score Python module docstrings? | 100% | 40% | 1.00 | 29 | $0.0000 | 0 | 2/2 | -| **Average** | **100%** | **29%** | **0.77** | **47** | **$0.0000** | **0** | | +| How is the audit trail structured? | 100% | 60% | 1.00 | 44 | $0.0000 | 0 | 3/3 | +| Where are unit tests for the confidence gate? | 100% | 20% | 1.00 | 48 | $0.0000 | 0 | 1/1 | +| How does the local scorer handle synonym expansion? | 100% | 20% | 1.00 | 44 | $0.0000 | 0 | 1/1 | +| Where is docstring boosting implemented? | 100% | 20% | 1.00 | 41 | $0.0000 | 0 | 1/1 | +| How does the two-pass live search reduce API calls? | 100% | 40% | 0.50 | 39 | $0.0000 | 0 | 2/2 | +| What aggregate telemetry is available after a search session... | 100% | 40% | 1.00 | 51 | $0.0000 | 0 | 2/2 | +| How does the smart Jev fallback differ from the default fall... | 100% | 20% | 1.00 | 38 | $0.0000 | 0 | 1/1 | +| Where is the benchmark harness that runs recall tests? | 100% | 40% | 0.50 | 42 | $0.0000 | 0 | 2/2 | +| What dependencies does the project require? | 100% | 20% | 1.00 | 45 | $0.0000 | 0 | 1/1 | +| How does the stemmer normalize query terms? | 100% | 20% | 1.00 | 38 | $0.0000 | 0 | 1/1 | +| Where are the gitignore exclusion rules applied during file ... | 100% | 40% | 0.50 | 41 | $0.0000 | 0 | 2/2 | +| How is the project architecture documented? | 100% | 20% | 1.00 | 35 | $0.0000 | 0 | 1/1 | +| How does Jev scoring protect secrets before sending file con... | 50% | 20% | 1.00 | 35 | $0.0000 | 0 | 1/2 | +| How does JevScorer fall back locally after an API error? | 100% | 20% | 1.00 | 41 | $0.0000 | 0 | 1/1 | +| How does the file collector use nested .gitignore patterns? | 100% | 40% | 1.00 | 39 | $0.0000 | 0 | 2/2 | +| Where are session telemetry fields defined and calculated? | 100% | 40% | 1.00 | 69 | $0.0000 | 0 | 2/2 | +| Where does the command line turn search results into JSON? | 100% | 20% | 0.50 | 49 | $0.0000 | 0 | 1/1 | +| What text suffixes are allowed and what file size is too lar... | 100% | 20% | 1.00 | 40 | $0.0000 | 0 | 1/1 | +| When does the search loop escalate an uncertain score to its... | 100% | 20% | 0.50 | 41 | $0.0000 | 0 | 1/1 | +| How does the local ranking score Python module docstrings? | 100% | 40% | 1.00 | 36 | $0.0000 | 0 | 2/2 | +| **Average** | **98%** | **29%** | **0.86** | **45** | **$0.0000** | **0** | | + +### Missed files + +- **How does Jev scoring protect secrets before sending file con**: tests/test_core.py ## tracecore-reflex (local) | Query | Recall@k | Precision@k | MRR | Latency (ms) | Cost | Tokens | Hits / Expected | |-------|----------|------------|-----|-------------|------|--------|-----------------| -| How does the agent decide between Jev and LLM fallback? | 100% | 40% | 1.00 | 142 | $0.0000 | 0 | 2/2 | -| Where is the Jev client that calls the decisions API? | 100% | 20% | 0.50 | 81 | $0.0000 | 0 | 1/1 | -| How is per-step telemetry tracked? | 100% | 20% | 1.00 | 67 | $0.0000 | 0 | 1/1 | -| How does the confidence gate evaluate a Jev answer? | 100% | 20% | 1.00 | 74 | $0.0000 | 0 | 1/1 | -| Where is the TraceCore-loadable agent configured? | 100% | 20% | 0.50 | 89 | $0.0000 | 0 | 1/1 | -| How do validators use Jev for task scoring? | 100% | 20% | 1.00 | 57 | $0.0000 | 0 | 1/1 | -| What does the threshold sweep tool measure? | 100% | 20% | 1.00 | 61 | $0.0000 | 0 | 1/1 | -| Where is the incident triage example? | 100% | 20% | 1.00 | 68 | $0.0000 | 0 | 1/1 | -| How are action arguments resolved from a Jev response? | 100% | 20% | 1.00 | 59 | $0.0000 | 0 | 1/1 | -| How does the natural escalation example demonstrate the REFL... | 100% | 20% | 1.00 | 60 | $0.0000 | 0 | 1/1 | -| Where does the policy comparison example compare different a... | 100% | 20% | 0.25 | 82 | $0.0000 | 0 | 1/1 | -| How does the run report tool summarize experiment results? | 100% | 20% | 0.50 | 63 | $0.0000 | 0 | 1/1 | -| What does the compare_runs tool do? | 100% | 40% | 1.00 | 56 | $0.0000 | 0 | 2/2 | -| How are step-level labels assigned and tested? | 100% | 40% | 1.00 | 65 | $0.0000 | 0 | 2/2 | -| Where are jev_model, fallback_model, threshold, and min_marg... | 100% | 20% | 1.00 | 98 | $0.0000 | 0 | 1/1 | -| What experiment checklist guides reproducible runs? | 100% | 20% | 1.00 | 88 | $0.0000 | 0 | 1/1 | -| **Average** | **100%** | **24%** | **0.86** | **76** | **$0.0000** | **0** | | +| How does the agent decide between Jev and LLM fallback? | 100% | 40% | 1.00 | 308 | $0.0000 | 0 | 2/2 | +| Where is the Jev client that calls the decisions API? | 100% | 20% | 1.00 | 118 | $0.0000 | 0 | 1/1 | +| How is per-step telemetry tracked? | 100% | 20% | 1.00 | 79 | $0.0000 | 0 | 1/1 | +| How does the confidence gate evaluate a Jev answer? | 100% | 20% | 1.00 | 90 | $0.0000 | 0 | 1/1 | +| Where is the TraceCore-loadable agent configured? | 100% | 20% | 1.00 | 102 | $0.0000 | 0 | 1/1 | +| How do validators use Jev for task scoring? | 100% | 20% | 1.00 | 78 | $0.0000 | 0 | 1/1 | +| What does the threshold sweep tool measure? | 100% | 20% | 1.00 | 73 | $0.0000 | 0 | 1/1 | +| Where is the incident triage example? | 100% | 20% | 1.00 | 97 | $0.0000 | 0 | 1/1 | +| How are action arguments resolved from a Jev response? | 100% | 20% | 1.00 | 73 | $0.0000 | 0 | 1/1 | +| How does the natural escalation example demonstrate the REFL... | 100% | 20% | 1.00 | 70 | $0.0000 | 0 | 1/1 | +| Where does the policy comparison example compare different a... | 100% | 20% | 0.50 | 78 | $0.0000 | 0 | 1/1 | +| How does the run report tool summarize experiment results? | 100% | 20% | 0.50 | 73 | $0.0000 | 0 | 1/1 | +| What does the compare_runs tool do? | 100% | 40% | 1.00 | 75 | $0.0000 | 0 | 2/2 | +| How are step-level labels assigned and tested? | 100% | 40% | 1.00 | 71 | $0.0000 | 0 | 2/2 | +| Where are jev_model, fallback_model, threshold, and min_marg... | 100% | 20% | 1.00 | 108 | $0.0000 | 0 | 1/1 | +| What experiment checklist guides reproducible runs? | 100% | 20% | 1.00 | 67 | $0.0000 | 0 | 1/1 | +| **Average** | **100%** | **24%** | **0.94** | **98** | **$0.0000** | **0** | | ## click (local) | Query | Recall@k | Precision@k | MRR | Latency (ms) | Cost | Tokens | Hits / Expected | |-------|----------|------------|-----|-------------|------|--------|-----------------| -| How does Click parse command line arguments and option value... | 100% | 40% | 0.50 | 313 | $0.0000 | 0 | 2/2 | -| Where are Group and resolve_command implemented? | 100% | 20% | 0.33 | 205 | $0.0000 | 0 | 1/1 | -| Where are the command, group, and option decorators defined? | 100% | 20% | 1.00 | 197 | $0.0000 | 0 | 1/1 | -| Where is Click's HelpFormatter implemented and used? | 100% | 40% | 1.00 | 211 | $0.0000 | 0 | 2/2 | -| How can tests run a command and capture its output without a... | 100% | 20% | 0.50 | 157 | $0.0000 | 0 | 1/1 | -| How are Click's Path and File parameter types defined? | 100% | 20% | 1.00 | 221 | $0.0000 | 0 | 1/1 | -| How does Click generate shell-specific command completions? | 100% | 20% | 1.00 | 152 | $0.0000 | 0 | 1/1 | -| Where are command usage errors represented and shown? | 100% | 40% | 1.00 | 196 | $0.0000 | 0 | 2/2 | -| **Average** | **100%** | **28%** | **0.79** | **207** | **$0.0000** | **0** | | +| How does Click parse command line arguments and option value... | 100% | 40% | 0.50 | 240 | $0.0000 | 0 | 2/2 | +| Where are Group and resolve_command implemented? | 100% | 20% | 1.00 | 203 | $0.0000 | 0 | 1/1 | +| Where are the command, group, and option decorators defined? | 100% | 20% | 1.00 | 193 | $0.0000 | 0 | 1/1 | +| Where is Click's HelpFormatter implemented and used? | 100% | 40% | 1.00 | 245 | $0.0000 | 0 | 2/2 | +| How can tests run a command and capture its output without a... | 100% | 20% | 1.00 | 222 | $0.0000 | 0 | 1/1 | +| How are Click's Path and File parameter types defined? | 100% | 20% | 1.00 | 298 | $0.0000 | 0 | 1/1 | +| How does Click generate shell-specific command completions? | 100% | 20% | 1.00 | 303 | $0.0000 | 0 | 1/1 | +| Where are command usage errors represented and shown? | 100% | 40% | 1.00 | 369 | $0.0000 | 0 | 2/2 | +| **Average** | **100%** | **28%** | **0.94** | **259** | **$0.0000** | **0** | | ## httpx (local) | Query | Recall@k | Precision@k | MRR | Latency (ms) | Cost | Tokens | Hits / Expected | |-------|----------|------------|-----|-------------|------|--------|-----------------| -| How does httpx follow redirects? | 100% | 20% | 0.33 | 218 | $0.0000 | 0 | 1/1 | -| Where are the Request and Response classes defined? | 100% | 20% | 0.50 | 161 | $0.0000 | 0 | 1/1 | -| How does authentication work in httpx? | 100% | 20% | 0.33 | 125 | $0.0000 | 0 | 1/1 | -| Where is the async HTTP client implemented? | 100% | 20% | 0.50 | 148 | $0.0000 | 0 | 1/1 | -| How does httpx handle streaming responses? | 0% | 0% | 0.00 | 109 | $0.0000 | 0 | 0/2 | -| Where is URL parsing and manipulation implemented? | 100% | 40% | 1.00 | 154 | $0.0000 | 0 | 2/2 | -| How does the transport layer send requests? | 50% | 20% | 0.50 | 124 | $0.0000 | 0 | 1/2 | -| Where is the exception hierarchy defined? | 100% | 20% | 1.00 | 139 | $0.0000 | 0 | 1/1 | -| **Average** | **81%** | **20%** | **0.52** | **147** | **$0.0000** | **0** | | +| How does httpx follow redirects? | 100% | 20% | 0.50 | 723 | $0.0000 | 0 | 1/1 | +| Where are the Request and Response classes defined? | 100% | 20% | 0.50 | 203 | $0.0000 | 0 | 1/1 | +| How does authentication work in httpx? | 100% | 20% | 0.33 | 154 | $0.0000 | 0 | 1/1 | +| Where is the async HTTP client implemented? | 100% | 20% | 1.00 | 183 | $0.0000 | 0 | 1/1 | +| How does httpx handle streaming responses? | 0% | 0% | 0.00 | 734 | $0.0000 | 0 | 0/2 | +| Where is URL parsing and manipulation implemented? | 100% | 40% | 1.00 | 277 | $0.0000 | 0 | 2/2 | +| How does the transport layer send requests? | 100% | 40% | 1.00 | 209 | $0.0000 | 0 | 2/2 | +| Where is the exception hierarchy defined? | 100% | 20% | 1.00 | 205 | $0.0000 | 0 | 1/1 | +| **Average** | **88%** | **22%** | **0.67** | **336** | **$0.0000** | **0** | | ### Missed files - **How does httpx handle streaming responses?**: httpx/_client.py, httpx/_models.py -- **How does the transport layer send requests?**: httpx/_transports/default.py + +## httpx — Local vs Smart-Fallback comparison + +Smart-fallback uses LocalScorer for initial ranking and escalates uncertain candidates +(score between 0.10 and 0.30) to Jev via the decisions API. Run with +`python tools/bench.py --smart-fallback --suite httpx`. + +| Metric | Local | Smart-Fallback | Delta | +|--------|-------|----------------|-------| +| Recall@5 | 81% | 88% | **+7 pp** | +| MRR | 0.58 | 0.65 | +0.07 | +| Precision@5 | 20% | 22% | +2 pp | +| Avg latency | 157 ms | 49,290 ms | +49 s | +| Total cost (8 queries) | $0.00 | $0.0115 | +$0.01 | +| Total Jev fallback calls | 0 | 339 | — | +| Avg fallback calls / query | 0 | 42 | — | + +### What improved + +- **"How does httpx handle streaming responses?"** — local missed both expected files + (`_client.py`, `_models.py`). Smart-fallback recovered `_models.py` (recall 0% → 50%). + +### What stayed the same + +- **"How does the transport layer send requests?"** — still misses + `httpx/_transports/default.py` in both modes. Jev scored it above the accept + threshold in isolated tests, but 42+ uncertain candidates per query dilute the + top-5 slots. + +### Takeaways + +1. Smart-fallback closes some conceptual gaps that LocalScorer cannot bridge. +2. The current uncertain band (0.10–0.30) triggers ~42 Jev calls per query on httpx, + making wall-clock time impractical for routine use. Narrowing the band or batching + calls (Phase 4: async scoring) would reduce this. +3. Cost per query is low (~$0.0014) but latency dominates at ~1 s per Jev call + sequentially. + +## Precision improvements (Phase 2) + +File-type demotions reduce noise from test, doc, tool, and example files when +queries ask about implementation. The demotions are guarded by intent checks +so queries explicitly about those categories are unaffected. + +| Category | FAs before | FAs after | Demotion factor | +|----------|-----------|----------|-----------------| +| test | 50 | 31 | 0.65 | +| doc | 43 | 8 | 0.55 | +| tool | 13 | 10 | 0.60 | +| example | (in other)| 6 | 0.50 | +| other | 81 | 110 | — | +| **total**| **188** | **160** | — | + +The remaining "other" FAs are sibling source files (e.g. `_api.py`, `core.py`) +that share vocabulary with the expected file. These require semantic +understanding (Phase 3) or Jev fallback to resolve. diff --git a/pyproject.toml b/pyproject.toml index 2f3f869..2b1551f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,5 +30,8 @@ reflex-search = "reflex_search.cli:app" [tool.setuptools] packages = ["reflex_search"] +[tool.ruff.lint.per-file-ignores] +"tools/bench.py" = ["E402"] + [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/reflex_search/scorers.py b/reflex_search/scorers.py index 704f269..7b69950 100644 --- a/reflex_search/scorers.py +++ b/reflex_search/scorers.py @@ -212,7 +212,9 @@ def score(self, file_path: str, query: str, content: str) -> float: specific_query_symbols = { symbol.lower() for symbol in query_identifiers - if "_" in symbol or symbol[0].isupper() or re.search(r"[A-Z]", symbol[1:]) + if "_" in symbol + or (not symbol.isupper() and re.search(r"[A-Z]", symbol[1:])) + or (symbol[0].isupper() and len(symbol) > 1 and not symbol.isupper() and symbol.lower() not in _STOPWORDS) } query_symbols = specific_query_symbols | { symbol.lower() @@ -315,27 +317,70 @@ def score(self, file_path: str, query: str, content: str) -> float: ): total += min(0.20, 0.10 * len(query_outcomes & enum_values)) - # Evaluation fixtures repeat their own query text and can otherwise - # outrank implementation files for unrelated questions. + # --- File-type demotions --- + # When the query asks about implementation, test/doc/tool files that + # mention the same terms are usually noise. Each demotion is guarded + # by an intent check so queries explicitly about those categories + # are unaffected. normalized_path = file_path.lower().replace("\\", "/") + path_parts = normalized_path.split("/") + file_basename = path_parts[-1] if path_parts else "" + + # --- Test file demotion --- + is_test_file = ( + any(part in ("tests", "test") for part in path_parts) + or file_basename.startswith("test_") + or file_basename.startswith("conftest") + ) + test_intent = any( + term in query_lower + for term in ("test", "tests", "testing", "unittest", "pytest", "fixture", "conftest") + ) + if is_test_file and not test_intent: + total *= 0.65 + + # --- Documentation file demotion --- + is_doc_file = file_basename.endswith((".md", ".rst", ".txt")) + documentation_intent = any( + term in query_lower + for term in ( + "readme", "document", "documentation", "release", + "changelog", "roadmap", "overview", "checklist", + "architecture", "documented", + ) + ) benchmark_intent = any( term in query_lower for term in ("benchmark", "recall", "precision", "mrr", "evaluation") ) - if not benchmark_intent and normalized_path.endswith( + if is_doc_file and not documentation_intent and not benchmark_intent: + total *= 0.55 + + # --- Tool / benchmark fixture demotion --- + is_tool_file = any(part == "tools" for part in path_parts) + is_benchmark_fixture = normalized_path.endswith( ("benchmarks/queries.json", "docs/benchmarks.md") - ): + ) + tool_intent = any( + term in query_lower + for term in ( + "benchmark", "harness", "tool", "compare_runs", + "step_labels", "step-level", "sweep", "report", "label", + ) + ) + if is_benchmark_fixture and not benchmark_intent: total *= 0.25 + elif is_tool_file and not tool_intent: + total *= 0.60 - documentation_intent = any( + # --- Example file demotion --- + is_example_file = any(part == "examples" for part in path_parts) + example_intent = any( term in query_lower - for term in ("readme", "document", "documentation", "release", "changelog", "roadmap", "overview") + for term in ("example", "demo", "sample", "reference") ) - if not documentation_intent and query_lower.startswith(("how ", "where ", "when ")): - if normalized_path.endswith("/readme.md") or normalized_path == "readme.md": - total *= 0.70 - elif normalized_path.endswith("/changelog.md") or normalized_path == "changelog.md": - total *= 0.50 + if is_example_file and not example_intent: + total *= 0.50 return round(min(total, 1.0), 4) diff --git a/roadmap.md b/roadmap.md index 786fad9..4bc9e87 100644 --- a/roadmap.md +++ b/roadmap.md @@ -1,8 +1,6 @@ # reflex-code-search Roadmap -**Status:** Working direction, without delivery dates. Public release is under -consideration; this roadmap does not imply that repository visibility has -changed. +**Status:** Working direction, without delivery dates. ## Product direction @@ -11,50 +9,74 @@ files in a codebase. Improve retrieval quality and make results predictable, explainable, and easy to evaluate. Keep the optional Jev integration separate from the local search path. -## Priorities - -### 1. Prepare the project for possible public use - -- Keep setup, supported Python versions, CLI examples, and development checks - accurate across the README, contributing guide, and CI. -- Make the project scope, current limitations, and benchmark methodology clear - to someone encountering the repository for the first time. -- Review repository metadata, tracked files, and Git history for anything that - should not be published; confirm licensing and contribution guidance are - clear before changing visibility. -- Decide on a simple issue and release process if the project is made public. - -### 2. Improve retrieval using measured results - -- Keep the benchmark suites for this project, tracecore-reflex, and pinned - upstream Click 8.5.0 reproducible. -- Use precision@k, reciprocal rank, and missed-file reports to target changes. - Current local recall at top_k=5 and threshold=0.3 is 100% on this project, - tracecore-reflex, and Click. -- Evaluate ranking changes across every suite and retain the deterministic, - dependency-light local search path. - -### 3. Keep installation and maintenance dependable - -- Maintain CI coverage for supported Python versions, tests, lint, wheel build, - and an installed-CLI smoke check. -- Keep release notes, architecture documentation, and benchmark results aligned - with shipped behavior. -- Document optional live scoring requirements and costs without making them a - prerequisite for local use. - -## Current capabilities - -- Offline `LocalScorer` with synonym expansion, docstring and definition - matching, and phrase scoring. -- Optional Jev scoring, uncertain-case fallback, and two-pass live search. -- Three-way confidence gate, audit trail, and per-search telemetry. -- Repository discovery with nested `.gitignore` support. -- CLI and library interfaces, with tests and Python 3.11–3.13 CI. - -## Out of scope - -- Symbolic code graphs or tree-sitter parsing. -- IDE integration or a repository hosting platform. -- A broad plugin system beyond the existing `Scorer` protocol. -- Runtime dependency on tracecore-reflex. +## Current baseline + +- 60 queries across 4 repos (reflex-code-search, tracecore-reflex, Click 8.5.0, httpx 0.28.1). +- LocalScorer: 100% recall on 3 repos, 81% on httpx. Jev closes all httpx gaps. +- Precision@k averages 24–29%. +- JevScorer is synchronous, ~500ms per file. +- CLI and library interfaces; no MCP or tool-calling integration. + +## Phases + +### 1. Measure Jev vs Local head-to-head + +Show whether the REFLEX loop adds value beyond confirming local decisions. + +- Add `--smart-fallback` mode to `bench.py` (Local scorer + Jev fallback for uncertain candidates). +- Record per-query: recall delta, MRR delta, fallback call count, cost, latency. +- Add fallback columns to `docs/benchmarks.md` alongside local results. +- Focus on httpx suite first (known gaps), then all suites. + +**Exit evidence:** Side-by-side numbers showing where smart-fallback improves recall and ranking, and what it costs. + +### 2. Improve precision + +Reduce noise in the top-k so results are usable as agent context. + +- Analyze false accepts: categorize by cause (common term overlap, synonym over-expansion, doc/test file leakage). +- Tighten scoring for common over-scorers: stronger demotion for test and doc files when queries ask about implementation. +- Consider dynamic top-k cutoff based on score gap from the top result. +- Add precision tracking to benchmark CI output. + +**Exit evidence:** Precision@5 improves from ~27% toward 40%+ without recall regression. + +### 3. Language-aware file understanding + +Replace heuristic branches with lightweight AST extraction. + +- Python first: use the `ast` module to extract class names, function signatures, docstrings, and imports (no new dependency). +- Replace `_DEFINITION_RE` and `_DOCSTRING_RE` regex for `.py` files with the AST pass; keep regex fallback for non-Python files. +- Evaluate tree-sitter as an optional dependency for multi-language support. + +**Exit evidence:** Same or better recall/MRR on all suites, cleaner scorer code, extensible to new languages. + +### 4. Async / parallel scoring + +Cut live-mode latency for the two-pass shortlist. + +- Add `AsyncJevScorer` implementing an async scorer variant. +- Add `async_search` and `async_search_two_pass` to `ReflexCodeSearcher` using `asyncio.gather`. +- Keep sync paths unchanged; add `--async` flag to CLI for live modes. + +**Exit evidence:** 20-file shortlist from ~10s sequential to ~1–2s parallel. + +### 5. MCP / tool integration + +Expose the library as an MCP tool for agents. + +- Define a `search_codebase` MCP tool (query, repo_root, top_k, threshold, live). +- Return structured results matching the existing `SearchSession` shape. +- Package as an optional MCP server entry point; keep MCP SDK as an optional dependency. + +**Exit evidence:** An agent can call `search_codebase` and receive ranked files without importing reflex-code-search directly. + +## Sequencing + +Phases 1–2 are measurement and tuning — no new dependencies, low risk, high signal. Phase 3 is a refactor that benefits from 1–2 being done. Phases 4 and 5 are independent and can proceed in either order. + +## Ongoing + +- Keep setup, CI, and documentation accurate. +- Evaluate ranking changes across every suite before merging. +- Document optional live scoring requirements and costs without making them a prerequisite for local use. diff --git a/tools/bench.py b/tools/bench.py index 51eda48..3fdfa0e 100644 --- a/tools/bench.py +++ b/tools/bench.py @@ -6,6 +6,7 @@ Usage: python tools/bench.py # local scorer only + python tools/bench.py --smart-fallback # local + Jev fallback for uncertain python tools/bench.py --live # also run with JevScorer python tools/bench.py --top-k 10 # override top_k python tools/bench.py --output results.md # write markdown to file @@ -28,6 +29,7 @@ sys.path.insert(0, str(_PROJECT_ROOT)) load_dotenv(_PROJECT_ROOT / ".env") +from reflex_search.fallback import JevFallback from reflex_search.scorers import JevScorer, LocalScorer from reflex_search.searcher import ReflexCodeSearcher @@ -46,6 +48,7 @@ def _run_suite( suite: dict[str, Any], *, live: bool = False, + smart_fallback: bool = False, top_k: int = 5, threshold: float = 0.30, max_queries: int | None = None, @@ -56,17 +59,20 @@ def _run_suite( queries = suite["queries"][:max_queries] if max_queries else suite["queries"] total = len(queries) + show_progress = live or smart_fallback for idx, entry in enumerate(queries, 1): query = entry["query"] expected = {_normalise(p) for p in entry["expected_files"]} - if live: + if show_progress: print(f" [{idx}/{total}] {query[:55]}...", end="", flush=True) scorer = JevScorer(timeout=15.0) if live else LocalScorer() + fallback = JevFallback(timeout=15.0) if smart_fallback else None searcher = ReflexCodeSearcher( repo_root=repo_root, scorer=scorer, + fallback=fallback, accept_threshold=threshold, uncertain_threshold=0.10, top_k=top_k, @@ -92,6 +98,11 @@ def _run_suite( total_cost += step.get("cost", 0.0) total_tokens += step.get("input_tokens", 0) + fallback_calls = sum( + 1 for step in session.audit_trail + if step.get("provider") in ("jev-fallback",) + ) + result = { "query": query, "expected": sorted(expected), @@ -106,8 +117,9 @@ def _run_suite( "input_tokens": total_tokens, "provider": scorer.provider, "candidates_scored": len(session.audit_trail), + "fallback_calls": fallback_calls, } - if live: + if show_progress: print(f" {elapsed_ms:.0f}ms {recall:.0%}") results.append(result) @@ -128,7 +140,8 @@ def _format_markdown( ) lines.append( "The independent Click suite uses upstream tag 8.5.0 and expects its " - "checkout at `.benchmark-repos/click`; CI checks it out automatically.\n" + "checkout at `.benchmark-repos/click`; the httpx suite uses upstream tag " + "0.28.1 at `.benchmark-repos/httpx`. CI checks out both automatically.\n" ) for label, results in all_results.items(): @@ -189,8 +202,10 @@ def main() -> None: parser = argparse.ArgumentParser(description="Benchmark reflex-code-search") parser.add_argument("--live", action="store_true", help="Also benchmark with JevScorer") + parser.add_argument("--smart-fallback", action="store_true", help="Also benchmark with Local + Jev fallback") parser.add_argument("--top-k", type=int, default=5) parser.add_argument("--threshold", type=float, default=0.30) + parser.add_argument("--suite", type=str, default=None, help="Run only the named suite (e.g. httpx)") parser.add_argument("--max-queries", type=int, default=None, help="Max queries per suite (useful for live)") parser.add_argument("--output", type=str, default=None, help="Write markdown to file") args = parser.parse_args() @@ -204,6 +219,8 @@ def main() -> None: all_results: dict[str, list[dict[str, Any]]] = {} for suite in data["suites"]: + if args.suite and suite["name"] != args.suite: + continue suite_root = (_PROJECT_ROOT / suite["repo_root"]).resolve() if not suite_root.exists(): print(f"SKIP: {suite['name']} — repo not found at {suite_root}") @@ -224,6 +241,26 @@ def main() -> None: avg = sum(r["recall"] for r in results) / len(results) if results else 0 print(f"\n Average recall: {avg:.0%} | Avg latency: {sum(r['latency_ms'] for r in results)/len(results):.0f}ms") + if args.smart_fallback: + label_sf = f"{suite['name']} (smart-fallback)" + print(f"\n{'='*60}") + print(f"Running: {label_sf}") + print(f"{'='*60}") + + results_sf = _run_suite(suite, smart_fallback=True, top_k=args.top_k, threshold=args.threshold, max_queries=args.max_queries) + all_results[label_sf] = results_sf + + for r in results_sf: + status = "[ok]" if r["recall"] == 1.0 else f"[missed: {r['missed']}]" + fb = f" fb={r['fallback_calls']}" if r["fallback_calls"] else "" + print(f" [{r['recall']:.0%}] {r['query'][:55]} {status}{fb}") + + avg = sum(r["recall"] for r in results_sf) / len(results_sf) if results_sf else 0 + avg_lat = sum(r["latency_ms"] for r in results_sf) / len(results_sf) if results_sf else 0 + total_cost = sum(r["cost"] for r in results_sf) + total_fb = sum(r["fallback_calls"] for r in results_sf) + print(f"\n Average recall: {avg:.0%} | Avg latency: {avg_lat:.0f}ms | Total cost: ${total_cost:.4f} | Fallback calls: {total_fb}") + if args.live: label_live = f"{suite['name']} (jev)" print(f"\n{'='*60}") diff --git a/tools/precision_analysis.py b/tools/precision_analysis.py new file mode 100644 index 0000000..508ddfc --- /dev/null +++ b/tools/precision_analysis.py @@ -0,0 +1,137 @@ +"""Analyze false accepts across all benchmark suites. + +Categorizes each false-accept file by type (test, doc, tool, config, other) +and prints a summary to guide precision improvements. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_PROJECT_ROOT)) + +from reflex_search.scorers import LocalScorer # noqa: E402 +from reflex_search.searcher import ReflexCodeSearcher # noqa: E402 + + +def _categorize(file_path: str) -> str: + lf = file_path.lower() + if "/test" in lf or lf.startswith("test"): + return "test" + if lf.endswith((".md", ".rst", ".txt")): + return "doc" + if "/tools/" in lf or lf.startswith("tools/"): + return "tool" + if lf.endswith((".toml", ".cfg", ".ini", ".json", ".yml", ".yaml")): + return "config" + return "other" + + +def main() -> None: + data = json.loads((_PROJECT_ROOT / "benchmarks" / "queries.json").read_text()) + cats: dict[str, int] = {"test": 0, "doc": 0, "tool": 0, "config": 0, "other": 0} + all_fa: list[tuple[str, float]] = [] + gap_data: list[dict] = [] + total_fa = 0 + total_slots = 0 + total_true = 0 + + for suite in data["suites"]: + root = str((_PROJECT_ROOT / suite["repo_root"]).resolve()) + print(f"\n{'='*60}") + print(f"Suite: {suite['name']}") + print(f"{'='*60}") + + for entry in suite["queries"]: + query = entry["query"] + expected = {p.replace("\\", "/") for p in entry["expected_files"]} + + scorer = LocalScorer() + searcher = ReflexCodeSearcher( + repo_root=root, + scorer=scorer, + top_k=5, + accept_threshold=0.30, + uncertain_threshold=0.10, + ) + session = searcher.search(query) + results = [(r.path.replace("\\", "/"), r.score) for r in session.results] + returned_paths = {p for p, _ in results} + + hits = returned_paths & expected + false_accepts = [(p, s) for p, s in results if p not in expected] + total_slots += 5 + total_true += len(hits) + total_fa += len(false_accepts) + + # Collect score-gap data for cutoff simulation + if results: + top_score = results[0][1] + items = [(p, s, p in expected) for p, s in results] + gap_data.append({"top_score": top_score, "items": items}) + + if false_accepts: + print(f"\n Q: {query}") + print(f" Expected: {sorted(expected)}") + print(f" Hits: {len(hits)}/{len(expected)}") + for fa_path, fa_score in false_accepts: + cat = _categorize(fa_path) + cats[cat] += 1 + all_fa.append((fa_path, fa_score)) + print(f" FA: {fa_path} score={fa_score:.3f} [{cat}]") + + print(f"\n{'='*60}") + print("SUMMARY") + print(f"{'='*60}") + print(f"Total top-5 slots: {total_slots}") + print(f"True positives: {total_true}") + print(f"False accepts: {total_fa}") + print(f"Empty slots: {total_slots - total_true - total_fa}") + print(f"Precision@5: {total_true / total_slots:.1%}") + print() + print("False accepts by category:") + for cat, count in sorted(cats.items(), key=lambda x: -x[1]): + print(f" {cat:10s}: {count:3d} ({count / max(total_fa, 1):.0%})") + + # Sub-breakdown of "other" + init_fa = sum(1 for p, _ in all_fa if p.endswith("__init__.py")) + example_fa = sum(1 for p, _ in all_fa if "/examples/" in p or p.startswith("examples/")) + print() + print("Sub-breakdown of 'other':") + print(f" __init__.py : {init_fa}") + print(f" examples/ : {example_fa}") + + # Score-gap cutoff simulation + print() + print("Score-gap cutoff simulation (drop results below X% of top score):") + for pct in (40, 50, 60, 70): + cut_tp = 0 + cut_fa = 0 + kept_tp = 0 + kept_fa = 0 + for entry_data in gap_data: + top_score = entry_data["top_score"] + threshold = top_score * pct / 100 + for path, score, is_tp in entry_data["items"]: + if score < threshold: + if is_tp: + cut_tp += 1 + else: + cut_fa += 1 + else: + if is_tp: + kept_tp += 1 + else: + kept_fa += 1 + new_prec = kept_tp / (kept_tp + kept_fa) if (kept_tp + kept_fa) else 0 + print( + f" {pct}%: drop {cut_fa} FA + {cut_tp} TP | " + f"keep {kept_tp} TP + {kept_fa} FA | " + f"precision={new_prec:.1%}" + ) + + +if __name__ == "__main__": + main()