From 64faf68b08829f74327691886f88b7137bc10e15 Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Sun, 11 Oct 2026 06:52:27 +0000 Subject: [PATCH 1/3] feat(relevant): record min_score and its filtered-hit count on the span GET /memories/relevant now stamps `lorekit.min_score` (always, 0 included) and `lorekit.min_score_filtered_count` (ranked candidates the threshold removed before diversification). Since 00112 graded relevance, a min_score tuned against the old 0.375 floor can drop hits it used to keep; the span showed that something was filtered but not the threshold or how much. Pinned by a source-scan spec in mcp-core, the rest-auth-tracing pattern, because the Deno edge tree has no test harness of its own. Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- .../relevant-min-score-tracing.spec.ts | 64 +++++++++++++++++++ .../functions/memories/handlers/relevant.ts | 13 ++++ 2 files changed, 77 insertions(+) create mode 100644 packages/mcp-core/src/telemetry/relevant-min-score-tracing.spec.ts diff --git a/packages/mcp-core/src/telemetry/relevant-min-score-tracing.spec.ts b/packages/mcp-core/src/telemetry/relevant-min-score-tracing.spec.ts new file mode 100644 index 00000000..9dbe1c60 --- /dev/null +++ b/packages/mcp-core/src/telemetry/relevant-min-score-tracing.spec.ts @@ -0,0 +1,64 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; + +/** + * Source-scan guard for the `min_score` telemetry on `GET /memories/relevant` + * (`supabase/functions/memories/handlers/relevant.ts`) — the + * `rest-auth-tracing.spec.ts` pattern, because the Deno edge tree has no test + * harness of its own. + * + * Migration 00112 graded relevance, which removed the `(1 + 0.5) / 4 = 0.375` + * floor every matched hit used to sit on. A `min_score` tuned against the old + * behaviour now drops hits it used to keep, and before these two attributes + * the span could not show it: `result_count < min(limit, candidate_count)` + * revealed THAT something was filtered, but neither the threshold nor how many + * hits it removed. + */ + +const here = path.dirname(fileURLToPath(import.meta.url)); // packages/mcp-core/src/telemetry +const repoRoot = path.resolve(here, '../../../..'); +const handlerPath = path.join(repoRoot, 'supabase', 'functions', 'memories', 'handlers', 'relevant.ts'); +const source = readFileSync(handlerPath, 'utf8'); + +/** Executable lines only, so the explanatory comments cannot satisfy an assertion. */ +const COMMENT_PREFIXES = ['//', '*', '/*', '*/']; + +const executable = source + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .filter((line) => !COMMENT_PREFIXES.some((prefix) => line.startsWith(prefix))) + .join('\n'); + +describe('GET /memories/relevant min_score telemetry (supabase/functions/memories/handlers/relevant.ts)', () => { + // Anti-vacuity: the assertions below describe this handler and this filter. + it('still exports handleRelevant, filters the ranked set by min_score, and diversifies what survives', () => { + expect(executable).toContain('export async function handleRelevant('); + expect(executable).toContain('const filtered = ranked.filter((r) => r.score >= params.min_score);'); + expect(executable).toContain('selectDiverse(filtered, params.limit)'); + }); + + it('records min_score from the validated params, unconditionally', () => { + // A whole-line match: a conditional spread such as + // `...(params.min_score ? { 'lorekit.min_score': … } : {})` sits on one + // line with other text and fails it, which is the point — a default of 0 + // must be recorded as 0, not left absent. + expect(executable).toMatch(/^'lorekit\.min_score':\s*params\.min_score,$/m); + }); + + it('records how many ranked candidates the threshold removed', () => { + // `ranked`, not `candidates`: the count is what the filter dropped, so it + // is measured against the set the filter ran over. + expect(executable).toMatch(/'lorekit\.min_score_filtered_count':\s*ranked\.length\s*-\s*filtered\.length/); + }); + + it('sets the filtered count in the same call as result_count and candidate_count', () => { + // The three are read together to tell "nothing matched" from "min_score + // dropped them" from "limit cut the list", so they land on the span at once. + expect(executable).toMatch( + /span\.setAttributes\(\{[^}]*'lorekit\.result_count'[^}]*'lorekit\.candidate_count'[^}]*'lorekit\.min_score_filtered_count'[^}]*\}\)/, + ); + }); +}); diff --git a/supabase/functions/memories/handlers/relevant.ts b/supabase/functions/memories/handlers/relevant.ts index cba61ee3..5db45b0f 100644 --- a/supabase/functions/memories/handlers/relevant.ts +++ b/supabase/functions/memories/handlers/relevant.ts @@ -134,6 +134,10 @@ export async function handleRelevant( ...(params.q ? { 'lorekit.query': params.q } : {}), 'lorekit.limit': params.limit, 'lorekit.scope_count': scopes.length, + // Recorded even at its default of 0, so "the caller set no threshold" is a + // value a query can select rather than an absent attribute. Bounded to + // [0,1] by `RelevantQuerySchema`, so it adds no cardinality worth naming. + 'lorekit.min_score': params.min_score, }); // Early refusal for a NAMED scope outside the key's allowlist (00068/00069), @@ -246,9 +250,18 @@ export async function handleRelevant( updated_at: entry.updated_at ?? null, })); + // `min_score_filtered_count` is how many RANKED candidates `min_score` removed + // before diversification — `ranked`, not `candidates`, because it counts what + // the filter on the line above dropped and nothing else. With the other two it + // separates the three reasons a response comes back short: nothing matched + // (`candidate_count` 0), the threshold dropped hits (this > 0), or `limit` cut + // the shortlist (`result_count` equal to `lorekit.limit`). Since 00112 graded + // relevance, a `min_score` tuned against the old 0.375 floor can drop hits it + // used to keep, and without this count that change is invisible. span.setAttributes({ 'lorekit.result_count': entries.length, 'lorekit.candidate_count': candidates.length, + 'lorekit.min_score_filtered_count': ranked.length - filtered.length, }); // `candidates` is the RANKED population, so it saturates at CANDIDATE_LIMIT — From cce6f46a2511719caad0218cdf03c270f9d259f9 Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Sun, 11 Oct 2026 07:39:52 +0000 Subject: [PATCH 2/3] address review comment: test(relevant): assert same-call membership without pinning key order Addresses the pr-reviewer bot's comment: https://github.com/mthines/lorekit/pull/688#discussion_r4240712042 Refs: https://github.com/mthines/lorekit/pull/688 Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- .../src/telemetry/relevant-min-score-tracing.spec.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/mcp-core/src/telemetry/relevant-min-score-tracing.spec.ts b/packages/mcp-core/src/telemetry/relevant-min-score-tracing.spec.ts index 9dbe1c60..0830ddc8 100644 --- a/packages/mcp-core/src/telemetry/relevant-min-score-tracing.spec.ts +++ b/packages/mcp-core/src/telemetry/relevant-min-score-tracing.spec.ts @@ -57,8 +57,12 @@ describe('GET /memories/relevant min_score telemetry (supabase/functions/memorie it('sets the filtered count in the same call as result_count and candidate_count', () => { // The three are read together to tell "nothing matched" from "min_score // dropped them" from "limit cut the list", so they land on the span at once. - expect(executable).toMatch( - /span\.setAttributes\(\{[^}]*'lorekit\.result_count'[^}]*'lorekit\.candidate_count'[^}]*'lorekit\.min_score_filtered_count'[^}]*\}\)/, - ); + // Find the call by the filtered count, then check membership: key order + // inside the object literal carries no meaning, so the test must not pin it. + const call = + executable.match(/span\.setAttributes\(\{[^}]*'lorekit\.min_score_filtered_count'[^}]*\}\)/)?.[0] ?? ''; + expect(call).not.toBe(''); + expect(call).toContain("'lorekit.result_count'"); + expect(call).toContain("'lorekit.candidate_count'"); }); }); From ba4ef2f18a5e41fbc8ef14fce3b6b40201e70e05 Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Sun, 11 Oct 2026 07:43:02 +0000 Subject: [PATCH 3/3] address review comment: docs(relevant): name the filtered step instead of "the line above" Addresses the pr-reviewer bot's comment: https://github.com/mthines/lorekit/pull/688#discussion_r4240712043 Refs: https://github.com/mthines/lorekit/pull/688 Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- supabase/functions/memories/handlers/relevant.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/supabase/functions/memories/handlers/relevant.ts b/supabase/functions/memories/handlers/relevant.ts index 5db45b0f..ba9b9f3b 100644 --- a/supabase/functions/memories/handlers/relevant.ts +++ b/supabase/functions/memories/handlers/relevant.ts @@ -252,12 +252,13 @@ export async function handleRelevant( // `min_score_filtered_count` is how many RANKED candidates `min_score` removed // before diversification — `ranked`, not `candidates`, because it counts what - // the filter on the line above dropped and nothing else. With the other two it - // separates the three reasons a response comes back short: nothing matched - // (`candidate_count` 0), the threshold dropped hits (this > 0), or `limit` cut - // the shortlist (`result_count` equal to `lorekit.limit`). Since 00112 graded - // relevance, a `min_score` tuned against the old 0.375 floor can drop hits it - // used to keep, and without this count that change is invisible. + // the `filtered` step (`ranked.filter` by `min_score`) dropped and nothing + // else. With the other two it separates the three reasons a response comes + // back short: nothing matched (`candidate_count` 0), the threshold dropped + // hits (this > 0), or `limit` cut the shortlist (`result_count` equal to + // `lorekit.limit`). Since 00112 graded relevance, a `min_score` tuned against + // the old 0.375 floor can drop hits it used to keep, and without this count + // that change is invisible. span.setAttributes({ 'lorekit.result_count': entries.length, 'lorekit.candidate_count': candidates.length,