From 2e85fd0f9430e9062255dd93a2f48d44110a6494 Mon Sep 17 00:00:00 2001 From: Bai Li Date: Wed, 22 Jul 2026 14:58:27 -0700 Subject: [PATCH 1/4] feat(evalboard): scope task trends to one harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trends aggregated the last N pipeline runs regardless of harness, but the nightly now rotates claude-code / codex / antigravity as separate runs — so a single per-task pass rate, status strip, and averages were blended across harnesses with incomparable pass rates and cost profiles, making reliably-passing tasks read as broken. Scope the page to one harness (default claude-code, the daily primary) with a switcher, and fold the tag-rail counts into the cached aggregate so the window loads once instead of twice per view. Co-Authored-By: Claude Opus 4.8 (1M context) --- evalboard/app/_components/harness-badge.tsx | 21 ++++++- .../app/_components/harness-selector.tsx | 44 ++++++++++++++ .../app/trends/__tests__/trends-view.test.tsx | 11 ++++ evalboard/app/trends/actions.ts | 10 +++- evalboard/app/trends/page.tsx | 43 ++++++++++---- evalboard/app/trends/trends-view.tsx | 40 +++++++++---- evalboard/lib/__tests__/overview.test.ts | 57 +++++++++++++++++++ evalboard/lib/overview.ts | 47 ++++++++++++--- evalboard/lib/trends.ts | 50 ++++++++++++---- 9 files changed, 278 insertions(+), 45 deletions(-) create mode 100644 evalboard/app/_components/harness-selector.tsx diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx index a9eca2bf..fa63a928 100644 --- a/evalboard/app/_components/harness-badge.tsx +++ b/evalboard/app/_components/harness-badge.tsx @@ -1,22 +1,37 @@ import Image from "next/image"; -// Vendor logo for a run's harness (RunConfig). Renders the recognizable +// The harnesses (coder-eval AgentKinds) the nightly rotates through, in display +// order. A run whose RunConfig carries no harness is a legacy claude-code run +// (see normalizeHarness in lib/overview.ts), so claude-code leads the list and +// is the default everywhere a harness must be assumed. +export const KNOWN_HARNESSES = ["claude-code", "codex", "antigravity"] as const; +export type KnownHarness = (typeof KNOWN_HARNESSES)[number]; + +// Vendor logo + labels for a run's harness (RunConfig). Renders the recognizable // vendor mark instead of raw "claude-code"/"codex"/"antigravity" text, // mirroring the Slack rollup's vendor emoji. A missing harness defaults to // claude-code (the nightly). This is an internal-only column — the caller // gates it behind isInternal (see lib/edition.ts). -const HARNESS_LOGO: Record = { +const HARNESS_LOGO: Record = { "claude-code": { src: "/harness/claude-code.png", label: "Claude Code · Anthropic", + short: "Claude Code", }, - codex: { src: "/harness/codex.png", label: "Codex · OpenAI" }, + codex: { src: "/harness/codex.png", label: "Codex · OpenAI", short: "Codex" }, antigravity: { src: "/harness/antigravity.png", label: "Antigravity · Google Gemini", + short: "Antigravity", }, }; +// Short human label for a harness id ("Claude Code"), for selectors and prose. +// Unknown ids fall through to the raw id. +export function harnessShortLabel(harness: string): string { + return HARNESS_LOGO[harness]?.short ?? harness; +} + export function HarnessBadge({ harness }: { harness?: string | null }) { const key = harness ?? "claude-code"; const logo = HARNESS_LOGO[key]; diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx new file mode 100644 index 00000000..862dca67 --- /dev/null +++ b/evalboard/app/_components/harness-selector.tsx @@ -0,0 +1,44 @@ +"use client"; + +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { HarnessBadge, harnessShortLabel } from "./harness-badge"; + +// Segmented control for the trends page's harness scope. Sets `?h=` +// while preserving the active q/tag params, mirroring WindowSelector. Each +// segment shows the vendor logo + short label so it reads like the per-run +// harness badge on the runs tables. +export function HarnessSelector({ + current, + harnesses, +}: { + current: string; + harnesses: readonly string[]; +}) { + const router = useRouter(); + const pathname = usePathname(); + const searchParams = useSearchParams(); + const set = (h: string) => { + const p = new URLSearchParams(searchParams.toString()); + p.set("h", h); + router.replace(`${pathname}?${p.toString()}`, { scroll: false }); + }; + return ( +
+ {harnesses.map((h) => { + const active = h === current; + return ( + + ); + })} +
+ ); +} diff --git a/evalboard/app/trends/__tests__/trends-view.test.tsx b/evalboard/app/trends/__tests__/trends-view.test.tsx index 20d60a42..9899b127 100644 --- a/evalboard/app/trends/__tests__/trends-view.test.tsx +++ b/evalboard/app/trends/__tests__/trends-view.test.tsx @@ -1,6 +1,7 @@ import { describe, expect, test, vi } from "vitest"; import { render, screen } from "@testing-library/react"; import type { TaskTrend } from "@/lib/trends"; +import { KNOWN_HARNESSES } from "@/app/_components/harness-badge"; // The view imports the server action for the expandable history rows; stub it // so rendering doesn't pull the blob-backed loader into a jsdom test. @@ -8,6 +9,14 @@ vi.mock("../actions", () => ({ fetchTaskHistoryAction: vi.fn(async () => []), })); +// The header's HarnessSelector reads router/params hooks; stub them so the view +// renders in jsdom without a router provider. +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace: vi.fn() }), + usePathname: () => "/trends", + useSearchParams: () => new URLSearchParams(), +})); + const { TrendsView } = await import("../trends-view"); function trend(overrides: Partial): TaskTrend { @@ -35,6 +44,8 @@ function renderView(tasks: TaskTrend[], runIds: string[]) { runIds={runIds} q={null} activeTag={null} + activeHarness="claude-code" + harnesses={KNOWN_HARNESSES} skills={[]} taskTags={[]} reviewTags={[]} diff --git a/evalboard/app/trends/actions.ts b/evalboard/app/trends/actions.ts index c1a16574..a84e6e75 100644 --- a/evalboard/app/trends/actions.ts +++ b/evalboard/app/trends/actions.ts @@ -1,10 +1,18 @@ "use server"; import { historyForTask, type TaskHistoryEntry } from "@/lib/trends"; +import { TRENDS_RECENT_RUN_COUNT } from "@/lib/trends"; +import { KNOWN_HARNESSES } from "@/app/_components/harness-badge"; export async function fetchTaskHistoryAction( taskId: string, + harness: string, ): Promise { if (typeof taskId !== "string" || !taskId) return []; - return historyForTask(taskId); + // Validate against the known set so a spoofed value can't widen the scan; + // fall back to the default harness (see trends aggregation). + const h = (KNOWN_HARNESSES as readonly string[]).includes(harness) + ? harness + : "claude-code"; + return historyForTask(taskId, TRENDS_RECENT_RUN_COUNT, h); } diff --git a/evalboard/app/trends/page.tsx b/evalboard/app/trends/page.tsx index f927cf62..41333e30 100644 --- a/evalboard/app/trends/page.tsx +++ b/evalboard/app/trends/page.tsx @@ -3,8 +3,8 @@ import { trendMatchesTag, TRENDS_RECENT_RUN_COUNT, } from "@/lib/trends"; -import { aggregateTaskTagCounts, loadRecentRuns } from "@/lib/overview"; import { fmtRunTime, humanizeTaskId } from "@/lib/format"; +import { KNOWN_HARNESSES } from "@/app/_components/harness-badge"; import { TrendsView } from "./trends-view"; export const dynamic = "force-dynamic"; @@ -25,26 +25,40 @@ function parseTag(raw: string | string[] | undefined): string | null { return trimmed.slice(0, 100); } +// The harness whose trend to show. The nightly rotates claude-code / codex / +// antigravity as separate runs; a trend is only meaningful within one harness, +// so the page scopes to one (default claude-code, the daily primary) and offers +// a switcher. An unknown/absent value falls back to the default. +function parseHarness(raw: string | string[] | undefined): string { + const v = Array.isArray(raw) ? raw[0] : raw; + return v && (KNOWN_HARNESSES as readonly string[]).includes(v) + ? v + : "claude-code"; +} + export default async function TrendsPage({ searchParams, }: { - searchParams: Promise<{ q?: string; tag?: string }>; + searchParams: Promise<{ q?: string; tag?: string; h?: string }>; }) { const params = await searchParams; const q = parseQ(params.q); const activeTag = parseTag(params.tag); + const harness = parseHarness(params.h); - const [{ runIds, trends: allTrends }, perRun] = await Promise.all([ - aggregateTaskTrends(TRENDS_RECENT_RUN_COUNT), - loadRecentRuns(TRENDS_RECENT_RUN_COUNT), - ]); - const tagCounts = aggregateTaskTagCounts(perRun); + // One cached load, scoped to the selected harness — yields the trends, the + // run axis, AND the tag-rail counts (all from the same window). + const { + runIds, + trends: allTrends, + tagCounts, + } = await aggregateTaskTrends(TRENDS_RECENT_RUN_COUNT, harness); - // Provenance: surface the actual run count + date span. Mixing runs across - // model/agent regimes into a single pass rate is otherwise invisible — - // showing the window at least makes the user aware of what's collapsed. - // Derived from the same cached aggregate as the table (runIds is - // newest-first), so the label and the strips can't skew apart. + // Provenance: surface the actual run count + date span. Even scoped to one + // harness, the window can straddle model/prompt regimes, so showing the + // span keeps the user aware of what's collapsed. Derived from the same + // cached aggregate as the table (runIds is newest-first), so the label and + // the strips can't skew apart. const provenance = runIds.length === 0 ? null @@ -75,10 +89,15 @@ export default async function TrendsPage({ return ( (cur === taskId ? null : taskId)); if (history[taskId] != null) return; startTransition(async () => { - const entries = await fetchTaskHistoryAction(taskId); + const entries = await fetchTaskHistoryAction(taskId, activeHarness); setHistory((prev) => ({ ...prev, [taskId]: entries })); }); } @@ -599,17 +606,28 @@ export function TrendsView({ return (
-

- Task trends -

-

- Per-task pass rate and averages across recent runs. - Averages cover successful runs only. -

+
+
+

+ Task trends +

+

+ Per-task pass rate and averages across recent{" "} + {harnessShortLabel(activeHarness)} runs. Averages + cover successful runs only. +

+
+ {/* Scope the whole page to one harness — mixing harnesses + blends incomparable pass rates and cost profiles. */} + +
{provenance && (

- Last {provenance.count} runs · {provenance.oldest} →{" "} - {provenance.newest} + Last {provenance.count} {activeHarness} runs ·{" "} + {provenance.oldest} → {provenance.newest}

)} {maturity.matureTasks > 0 && ( @@ -651,7 +669,7 @@ export function TrendsView({ {filterActive ? ( <>No tasks match the current filter. ) : ( - <>No recent runs. + <>No recent {activeHarness} runs. )}
) : ( diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index aed22813..dcccdd43 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test, vi } from "vitest"; import { buildAdhocRows, collectPipelineRuns, + normalizeHarness, summarizeListing, turnBudgetRateForTasks, type PerRun, @@ -302,6 +303,62 @@ describe("collectPipelineRuns", () => { expect(await collectPipelineRuns([], 5, load)).toEqual([]); expect(load).not.toHaveBeenCalled(); }); + + // A usable run tagged with a specific harness. + function runH(id: string, harness: string): PerRun { + const r = run(id); + return { ...r, overview: { ...r.overview!, harness } }; + } + + test("isMatch filters to the matching harness and backfills the rest", async () => { + // Interleaved harnesses; scoping to claude-code must skip the codex + // runs and reach further back to fill the window rather than returning + // short. + const harnesses: Record = { + r6: "claude-code", + r5: "codex", + r4: "claude-code", + r3: "codex", + r2: "claude-code", + r1: "codex", + }; + const load = vi.fn(async (id: string) => runH(id, harnesses[id])); + const out = await collectPipelineRuns( + ["r6", "r5", "r4", "r3", "r2", "r1"], + 3, + load, + (r) => r.overview?.harness === "claude-code", + ); + expect(out.map((r) => r.id)).toEqual(["r6", "r4", "r2"]); + }); + + test("harness scan reaches past RECENT_SCAN_FACTOR to gather a rare harness", async () => { + // limit 1; the only antigravity run sits at index 5 — beyond the + // unfiltered cap (1 × RECENT_SCAN_FACTOR = 3). The wider harness cap + // (1 × HARNESS_SCAN_FACTOR = 8) must reach it. + const ids = ["r6", "r5", "r4", "r3", "r2", "r1"]; + const load = vi.fn(async (id: string) => + runH(id, id === "r1" ? "antigravity" : "claude-code"), + ); + const out = await collectPipelineRuns( + ids, + 1, + load, + (r) => r.overview?.harness === "antigravity", + ); + expect(out.map((r) => r.id)).toEqual(["r1"]); + }); +}); + +describe("normalizeHarness", () => { + test("null/undefined fold to claude-code (legacy pre-stamp runs)", () => { + expect(normalizeHarness(null)).toBe("claude-code"); + expect(normalizeHarness(undefined)).toBe("claude-code"); + }); + test("an explicit harness passes through unchanged", () => { + expect(normalizeHarness("codex")).toBe("codex"); + expect(normalizeHarness("antigravity")).toBe("antigravity"); + }); }); describe("buildAdhocRows", () => { diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 68c81568..ef57940c 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -254,19 +254,40 @@ async function loadWindowDataInner(window: Window): Promise { return mapWithConcurrency(ids, FETCH_CONCURRENCY, cachedLoadPerRun); } +// The harness a run is attributed to. A run whose RunConfig carried no harness +// predates the stamp, and every such run was claude-code (the only nightly +// harness before codex/antigravity were added), so null folds to claude-code. +// Mirrors HarnessBadge's `harness ?? "claude-code"` default. +export const DEFAULT_HARNESS = "claude-code"; +export function normalizeHarness(harness: string | null | undefined): string { + return harness ?? DEFAULT_HARNESS; +} + // Fetch the N most recent runs in PerRun shape. Recency-based (fixed count) -// rather than date-bounded — used by the trends page. -export function loadRecentRuns(limit: number): Promise { - return loadRecentRunsInner(limit); +// rather than date-bounded — used by the trends page. When `harness` is set, +// only runs of that harness count toward N (the nightly now rotates +// claude-code / codex / antigravity, and a trend is only meaningful within one +// harness — mixing them blends incomparable pass rates and cost profiles). +export function loadRecentRuns( + limit: number, + harness?: string, +): Promise { + return loadRecentRunsInner(limit, harness); } -async function loadRecentRunsInner(limit: number): Promise { +async function loadRecentRunsInner( + limit: number, + harness?: string, +): Promise { // Trends is the daily-cadence view: only pipeline runs belong here. Prune // to date-shaped ids BEFORE slicing (cheap, no IO) so ad-hoc runs — whose // ids sort lexically above every `2026-…` daily id and would otherwise // crowd out the real "recent N" — never occupy a slot. const ids = (await listRunIds()).filter((id) => parseRunIdDate(id) != null); - return collectPipelineRuns(ids, limit, cachedLoadPerRun); + const matchesHarness = harness + ? (r: PerRun) => normalizeHarness(r.overview?.harness) === harness + : undefined; + return collectPipelineRuns(ids, limit, cachedLoadPerRun, matchesHarness); } // A loaded run occupies a window slot only when it's usable downstream: @@ -287,6 +308,11 @@ function isUsablePipelineRun(r: PerRun): boolean { // deficit-sized rounds from degenerating into one-id-per-round serial loads // when a single slot stays unfilled. const RECENT_SCAN_FACTOR = 3; +// When filtering to a single harness, that harness holds only a fraction of the +// recent runs (codex/antigravity run a few times a week vs. claude-code daily), +// so the scan has to reach further back to gather `limit` of them. A larger cap +// keeps a rarer harness from coming up short while still bounding the probe. +const HARNESS_SCAN_FACTOR = 8; const MIN_PROBE_BATCH = 5; // Load runs newest-first until `limit` usable pipeline runs are in hand, the @@ -305,8 +331,15 @@ export async function collectPipelineRuns( ids: string[], limit: number, load: (id: string) => Promise, + // Optional extra predicate (AND-ed with usability) — e.g. a harness filter. + // When set, the scan reaches further back since matches are sparser. + isMatch?: (r: PerRun) => boolean, ): Promise { - const maxScan = Math.min(ids.length, limit * RECENT_SCAN_FACTOR); + const scanFactor = isMatch ? HARNESS_SCAN_FACTOR : RECENT_SCAN_FACTOR; + const maxScan = Math.min(ids.length, limit * scanFactor); + const usable = isMatch + ? (r: PerRun) => isUsablePipelineRun(r) && isMatch(r) + : isUsablePipelineRun; const out: PerRun[] = []; let cursor = 0; while (out.length < limit && cursor < maxScan) { @@ -318,7 +351,7 @@ export async function collectPipelineRuns( const batch = ids.slice(cursor, Math.min(cursor + batchSize, maxScan)); cursor += batch.length; const loaded = await mapWithConcurrency(batch, FETCH_CONCURRENCY, load); - out.push(...loaded.filter(isUsablePipelineRun)); + out.push(...loaded.filter(usable)); } if (out.length < limit && cursor >= maxScan && cursor < ids.length) { console.warn( diff --git a/evalboard/lib/trends.ts b/evalboard/lib/trends.ts index 1c8b6787..53e3cdd6 100644 --- a/evalboard/lib/trends.ts +++ b/evalboard/lib/trends.ts @@ -4,7 +4,12 @@ // means "last 10 runs" is a more useful unit than "last 7 days". import { unstable_cache } from "next/cache"; -import { loadRecentRuns, type PerRun } from "./overview"; +import { + aggregateTaskTagCounts, + loadRecentRuns, + type PerRun, + type TagCount, +} from "./overview"; import type { ComponentSha } from "./runs"; export const TRENDS_RECENT_RUN_COUNT = 10; @@ -208,22 +213,41 @@ export function aggregate(perRun: PerRun[]): TrendsData { return { runIds, trends }; } -async function aggregateTaskTrendsInner(limit: number): Promise { - return aggregate(await loadRecentRuns(limit)); +// TrendsData plus the tag-rail counts, both derived from the SAME loaded +// window. Folding the counts in here means the trends page loads the recent-run +// window once (via this cached aggregate) instead of a second uncached +// loadRecentRuns call just for the rail. +export interface TrendsPageData extends TrendsData { + tagCounts: { + skills: TagCount[]; + taskTags: TagCount[]; + reviewTags: TagCount[]; + }; +} + +async function aggregateTaskTrendsInner( + limit: number, + harness: string, +): Promise { + const perRun = await loadRecentRuns(limit, harness); + return { ...aggregate(perRun), tagCounts: aggregateTaskTagCounts(perRun) }; } const cachedAggregate = unstable_cache( aggregateTaskTrendsInner, - // v2: the cached shape changed from TaskTrend[] to TrendsData — the key - // bump keeps a stale pre-deploy array from being served into new code. - ["aggregate-task-trends-v2"], + // v3: the cached shape gained tagCounts and the args gained `harness` — the + // key bump keeps a stale pre-deploy payload from being served into new code. + // (harness is part of the auto-generated key too, so each harness caches + // independently.) + ["aggregate-task-trends-v3"], { revalidate: 300 }, ); export function aggregateTaskTrends( limit: number = TRENDS_RECENT_RUN_COUNT, -): Promise { - return cachedAggregate(limit); + harness: string = "claude-code", +): Promise { + return cachedAggregate(limit, harness); } // Predicate matching getOverview's tag scoping logic, but operating on the @@ -238,8 +262,9 @@ export function trendMatchesTag(trend: TaskTrend, tag: string): boolean { export async function historyForTaskInner( taskId: string, limit: number, + harness: string = "claude-code", ): Promise { - const perRun = await loadRecentRuns(limit); + const perRun = await loadRecentRuns(limit, harness); const out: TaskHistoryEntry[] = []; for (const { id, overview, reviewTagsByTask } of perRun) { if (!overview) continue; @@ -266,13 +291,16 @@ export async function historyForTaskInner( const cachedHistory = unstable_cache( historyForTaskInner, - ["history-for-task"], + // v2: gained the `harness` arg (auto-keyed), so a task's history caches + // per harness rather than serving one harness's rows under another. + ["history-for-task-v2"], { revalidate: 300 }, ); export function historyForTask( taskId: string, limit: number = TRENDS_RECENT_RUN_COUNT, + harness: string = "claude-code", ): Promise { - return cachedHistory(taskId, limit); + return cachedHistory(taskId, limit, harness); } From d639dd2ed618bde269c24d535ebea71dd32f4c9f Mon Sep 17 00:00:00 2001 From: Bai Li Date: Wed, 22 Jul 2026 15:35:31 -0700 Subject: [PATCH 2/4] feat(evalboard): make all pages harness-aware and stream tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend harness scoping beyond trends to the whole board and make the harness set data-driven so a new harness (e.g. delegate) appears on its own: - lib/harness.ts: shared leaf module (constants + normalize + a whitelist-free param parser) so client and server can import without bundle contamination. - listRecentHarnesses(): cached discovery of which harnesses have recent runs; the switcher lists exactly those (plus the active one), no hardcoded set. - Watchlist and path-to-ga: scoped to one harness (default claude-code) with a selector, like trends — their blended pass rates / streaks / charts were mixing incomparable harnesses. - Front page: analytics (success chart + tag rail) scoped to a harness; the run LIST stays all-harness (its job is to show every recent run, and the Harness column already disambiguates each row). - Tag rail preserves the active harness across chip clicks (fixes a reset on trends too). Also stream the trends and watchlist tables behind a Suspense boundary so the header + selector paint immediately instead of blocking on the cold data load. Co-Authored-By: Claude Opus 4.8 (1M context) --- evalboard/app/_components/harness-badge.tsx | 10 +- .../app/_components/harness-selector.tsx | 7 +- evalboard/app/_overview/tag-rail.tsx | 16 ++- evalboard/app/page.tsx | 38 +++++-- evalboard/app/path-to-ga/page.tsx | 29 +++-- evalboard/app/trends/actions.ts | 18 ++-- evalboard/app/trends/page.tsx | 102 ++++++++++++++---- evalboard/app/trends/trends-view.tsx | 1 + .../__tests__/watchlist-view.test.tsx | 37 +++++-- evalboard/app/watchlist/page.tsx | 73 ++++++++++++- evalboard/app/watchlist/watchlist-view.tsx | 29 +++-- evalboard/lib/__tests__/overview.test.ts | 2 +- evalboard/lib/harness.ts | 31 ++++++ evalboard/lib/overview.ts | 60 +++++++++-- 14 files changed, 366 insertions(+), 87 deletions(-) create mode 100644 evalboard/lib/harness.ts diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx index fa63a928..da16b704 100644 --- a/evalboard/app/_components/harness-badge.tsx +++ b/evalboard/app/_components/harness-badge.tsx @@ -1,11 +1,9 @@ import Image from "next/image"; -// The harnesses (coder-eval AgentKinds) the nightly rotates through, in display -// order. A run whose RunConfig carries no harness is a legacy claude-code run -// (see normalizeHarness in lib/overview.ts), so claude-code leads the list and -// is the default everywhere a harness must be assumed. -export const KNOWN_HARNESSES = ["claude-code", "codex", "antigravity"] as const; -export type KnownHarness = (typeof KNOWN_HARNESSES)[number]; +// Canonical harness constants live in the leaf lib/harness module (shared by +// the server data layer); re-exported here so existing badge importers are +// unaffected. +export { KNOWN_HARNESSES, type KnownHarness } from "@/lib/harness"; // Vendor logo + labels for a run's harness (RunConfig). Renders the recognizable // vendor mark instead of raw "claude-code"/"codex"/"antigravity" text, diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx index 862dca67..efbfcaa4 100644 --- a/evalboard/app/_components/harness-selector.tsx +++ b/evalboard/app/_components/harness-selector.tsx @@ -22,9 +22,14 @@ export function HarnessSelector({ p.set("h", h); router.replace(`${pathname}?${p.toString()}`, { scroll: false }); }; + // Always show the active harness, even if it has aged out of the recent + // window (so a deep-linked `?h=` still reads as selected rather than absent). + const opts = (harnesses as readonly string[]).includes(current) + ? harnesses + : [current, ...harnesses]; return (
- {harnesses.map((h) => { + {opts.map((h) => { const active = h === current; return (
- +
+ + +
diff --git a/evalboard/app/path-to-ga/page.tsx b/evalboard/app/path-to-ga/page.tsx index be6496aa..57b8514b 100644 --- a/evalboard/app/path-to-ga/page.tsx +++ b/evalboard/app/path-to-ga/page.tsx @@ -1,7 +1,14 @@ import Link from "next/link"; -import { getOverview, getTagTaskBreakdown } from "@/lib/overview"; +import { + getOverview, + getTagTaskBreakdown, + listRecentHarnesses, +} from "@/lib/overview"; +import { parseHarnessParam } from "@/lib/harness"; import { humanizeTaskId } from "@/lib/format"; import { WindowSelector } from "../_components/window-selector"; +import { HarnessSelector } from "../_components/harness-selector"; +import { harnessShortLabel } from "../_components/harness-badge"; import { WINDOWS, type Window } from "@/lib/reviews-types"; import { DailySuccessChart } from "../_overview/daily-chart"; import { TableScroll } from "../_components/scroll-table"; @@ -26,14 +33,18 @@ function passClass(pct: number | null): string { export default async function PathToGaPage({ searchParams, }: { - searchParams: Promise<{ window?: string }>; + searchParams: Promise<{ window?: string; h?: string }>; }) { const params = await searchParams; const window = parseWindow(params.window); + const harness = parseHarnessParam(params.h); - const [overview, taskRows] = await Promise.all([ - getOverview(window, TAG, null), - getTagTaskBreakdown(window, TAG), + // Scope to one harness — readiness of a task is per-harness, and a blended + // chart/pass-rate mixes incomparable regimes. + const [overview, taskRows, harnesses] = await Promise.all([ + getOverview(window, TAG, null, harness), + getTagTaskBreakdown(window, TAG, harness), + listRecentHarnesses(), ]); const runsInWindow = overview.runs.length; @@ -52,10 +63,14 @@ export default async function PathToGaPage({

Score for every task tagged{" "} - {TAG}. + {TAG} on{" "} + {harnessShortLabel(harness)}.

- +
+ + +
diff --git a/evalboard/app/trends/actions.ts b/evalboard/app/trends/actions.ts index a84e6e75..a629e4d6 100644 --- a/evalboard/app/trends/actions.ts +++ b/evalboard/app/trends/actions.ts @@ -1,18 +1,18 @@ "use server"; -import { historyForTask, type TaskHistoryEntry } from "@/lib/trends"; -import { TRENDS_RECENT_RUN_COUNT } from "@/lib/trends"; -import { KNOWN_HARNESSES } from "@/app/_components/harness-badge"; +import { + historyForTask, + type TaskHistoryEntry, + TRENDS_RECENT_RUN_COUNT, +} from "@/lib/trends"; +import { parseHarnessParam } from "@/lib/harness"; export async function fetchTaskHistoryAction( taskId: string, harness: string, ): Promise { if (typeof taskId !== "string" || !taskId) return []; - // Validate against the known set so a spoofed value can't widen the scan; - // fall back to the default harness (see trends aggregation). - const h = (KNOWN_HARNESSES as readonly string[]).includes(harness) - ? harness - : "claude-code"; - return historyForTask(taskId, TRENDS_RECENT_RUN_COUNT, h); + // Normalize/validate the harness (any plausible id; defaults to the primary + // harness) so the history scope matches the trends table it expands from. + return historyForTask(taskId, TRENDS_RECENT_RUN_COUNT, parseHarnessParam(harness)); } diff --git a/evalboard/app/trends/page.tsx b/evalboard/app/trends/page.tsx index 41333e30..37555662 100644 --- a/evalboard/app/trends/page.tsx +++ b/evalboard/app/trends/page.tsx @@ -1,10 +1,14 @@ +import { Suspense } from "react"; import { aggregateTaskTrends, trendMatchesTag, TRENDS_RECENT_RUN_COUNT, } from "@/lib/trends"; +import { listRecentHarnesses } from "@/lib/overview"; +import { KNOWN_HARNESSES, parseHarnessParam } from "@/lib/harness"; import { fmtRunTime, humanizeTaskId } from "@/lib/format"; -import { KNOWN_HARNESSES } from "@/app/_components/harness-badge"; +import { harnessShortLabel } from "@/app/_components/harness-badge"; +import { HarnessSelector } from "@/app/_components/harness-selector"; import { TrendsView } from "./trends-view"; export const dynamic = "force-dynamic"; @@ -25,17 +29,10 @@ function parseTag(raw: string | string[] | undefined): string | null { return trimmed.slice(0, 100); } -// The harness whose trend to show. The nightly rotates claude-code / codex / -// antigravity as separate runs; a trend is only meaningful within one harness, -// so the page scopes to one (default claude-code, the daily primary) and offers -// a switcher. An unknown/absent value falls back to the default. -function parseHarness(raw: string | string[] | undefined): string { - const v = Array.isArray(raw) ? raw[0] : raw; - return v && (KNOWN_HARNESSES as readonly string[]).includes(v) - ? v - : "claude-code"; -} - +// The page shell only needs the URL params (cheap), so it returns immediately +// and streams the data-backed table in behind a Suspense boundary — the ~20 MB +// cold aggregate load no longer blocks the whole response. The header + +// harness selector paint instantly (and stay usable) via the fallback. export default async function TrendsPage({ searchParams, }: { @@ -44,15 +41,35 @@ export default async function TrendsPage({ const params = await searchParams; const q = parseQ(params.q); const activeTag = parseTag(params.tag); - const harness = parseHarness(params.h); + const harness = parseHarnessParam(params.h); + + return ( + // Key on harness so a switch (a fresh, heavy load) shows the skeleton; + // q/tag changes keep the boundary and re-filter the cached aggregate in + // a transition without flashing the fallback. + }> + + + ); +} +async function TrendsContent({ + q, + activeTag, + harness, +}: { + q: string | null; + activeTag: string | null; + harness: string; +}) { // One cached load, scoped to the selected harness — yields the trends, the - // run axis, AND the tag-rail counts (all from the same window). - const { - runIds, - trends: allTrends, - tagCounts, - } = await aggregateTaskTrends(TRENDS_RECENT_RUN_COUNT, harness); + // run axis, AND the tag-rail counts (all from the same window). The harness + // list (for the switcher) is a separate small cached discovery load. + const [{ runIds, trends: allTrends, tagCounts }, harnesses] = + await Promise.all([ + aggregateTaskTrends(TRENDS_RECENT_RUN_COUNT, harness), + listRecentHarnesses(), + ]); // Provenance: surface the actual run count + date span. Even scoped to one // harness, the window can straddle model/prompt regimes, so showing the @@ -89,15 +106,12 @@ export default async function TrendsPage({ return ( ); } + +// Streamed while the aggregate loads. Mirrors TrendsView's header (so it +// doesn't jump on swap) and keeps the harness selector live so the user can +// re-scope without waiting, then shims the table with pulsing rows. +function TrendsSkeleton({ activeHarness }: { activeHarness: string }) { + return ( +
+
+
+

+ Task trends +

+

+ Per-task pass rate and averages across recent{" "} + {harnessShortLabel(activeHarness)} runs. Averages cover + successful runs only. +

+
+ +
+
+
+
+
+ {Array.from({ length: 8 }).map((_, i) => ( +
+
+
+ ))} +
+
+ ); +} diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx index 520a17e9..e56d0207 100644 --- a/evalboard/app/trends/trends-view.tsx +++ b/evalboard/app/trends/trends-view.tsx @@ -650,6 +650,7 @@ export function TrendsView({ activeTag={activeTag} basePath="/trends" q={q} + harness={activeHarness} limit={24} /> diff --git a/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx b/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx index 9e147173..c19c6c7a 100644 --- a/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx +++ b/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx @@ -1,9 +1,28 @@ -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import { render, screen } from "@testing-library/react"; import type { PerRun } from "@/lib/overview"; import type { RunOverview, RunOverviewTask } from "@/lib/runs"; -import { buildWatchlist } from "@/lib/watchlist"; -import { WatchlistView } from "../watchlist-view"; +import { buildWatchlist, type WatchlistData } from "@/lib/watchlist"; + +// The header's HarnessSelector reads router/params hooks; stub them so the view +// renders in jsdom without a router provider. +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace: vi.fn() }), + usePathname: () => "/watchlist", + useSearchParams: () => new URLSearchParams(), +})); + +const { WatchlistView } = await import("../watchlist-view"); + +function renderView(data: WatchlistData) { + return render( + , + ); +} function task(o: Partial): RunOverviewTask { return { @@ -45,7 +64,7 @@ describe("WatchlistView", () => { task({ taskId: "b", skill: "beta", status: "FAILURE" }), ]), ]); - render(); + renderView(data); expect(screen.getByText("Watchlist")).toBeInTheDocument(); expect(screen.getByText("Needs Attention")).toBeInTheDocument(); @@ -70,7 +89,7 @@ describe("WatchlistView", () => { task({ taskId: "w2", skill: "wobble", status: "SUCCESS" }), ]), ]); - render(); + renderView(data); // Definition is reachable as the ⓘ tooltip's accessible label. expect( @@ -91,9 +110,7 @@ describe("WatchlistView", () => { status: "FAILURE", }), ); - render( - , - ); + renderView(buildWatchlist([run("2026-01-01", tasks)])); // The expander summary appears with the full count + tie callout. expect(screen.getByText(/Show all 12 \(2 more tied\)/)).toBeInTheDocument(); // Beyond-cap rows are in the DOM (inside
), not dropped. @@ -101,8 +118,8 @@ describe("WatchlistView", () => { }); test("empty window renders friendly empty states without throwing", () => { - render(); + renderView(buildWatchlist([])); expect(screen.getByText("Nothing needs attention 🎉")).toBeInTheDocument(); - expect(screen.getByText("last 0 runs")).toBeInTheDocument(); + expect(screen.getByText("last 0 Claude Code runs")).toBeInTheDocument(); }); }); diff --git a/evalboard/app/watchlist/page.tsx b/evalboard/app/watchlist/page.tsx index da78c3c9..83e5326b 100644 --- a/evalboard/app/watchlist/page.tsx +++ b/evalboard/app/watchlist/page.tsx @@ -1,12 +1,77 @@ -import { loadRecentRuns } from "@/lib/overview"; +import { Suspense } from "react"; +import { loadRecentRuns, listRecentHarnesses } from "@/lib/overview"; import { TRENDS_RECENT_RUN_COUNT } from "@/lib/trends"; import { buildWatchlist } from "@/lib/watchlist"; +import { KNOWN_HARNESSES, parseHarnessParam } from "@/lib/harness"; +import { HarnessSelector } from "@/app/_components/harness-selector"; import { WatchlistView } from "./watchlist-view"; export const dynamic = "force-dynamic"; -export default async function WatchlistPage() { - const perRun = await loadRecentRuns(TRENDS_RECENT_RUN_COUNT); +export default async function WatchlistPage({ + searchParams, +}: { + searchParams: Promise<{ h?: string }>; +}) { + const params = await searchParams; + const harness = parseHarnessParam(params.h); + return ( + } + > + + + ); +} + +async function WatchlistContent({ harness }: { harness: string }) { + // Scope the whole watchlist to one harness — the pass rates, streaks, and + // volatility only mean something within a single harness (mixing them makes + // a Claude-reliable skill look flaky on codex/antigravity days). + const [perRun, harnesses] = await Promise.all([ + loadRecentRuns(TRENDS_RECENT_RUN_COUNT, harness), + listRecentHarnesses(), + ]); const data = buildWatchlist(perRun); - return ; + return ( + + ); +} + +function WatchlistSkeleton({ activeHarness }: { activeHarness: string }) { + return ( +
+
+

+ Watchlist +

+
+ +
+
+

+ What leadership should be watching — ranked by where the signal + says to look first, for the selected harness. +

+
+ {Array.from({ length: 5 }).map((_, i) => ( +
+ ))} +
+
+ ); } diff --git a/evalboard/app/watchlist/watchlist-view.tsx b/evalboard/app/watchlist/watchlist-view.tsx index 22f3b00a..17b0c971 100644 --- a/evalboard/app/watchlist/watchlist-view.tsx +++ b/evalboard/app/watchlist/watchlist-view.tsx @@ -6,6 +6,8 @@ import type { ReactNode } from "react"; import Link from "next/link"; import { humanizeTaskId } from "@/lib/format"; +import { HarnessSelector } from "@/app/_components/harness-selector"; +import { harnessShortLabel } from "@/app/_components/harness-badge"; import { FAIL_WEIGHT, REG_WEIGHT, @@ -275,24 +277,39 @@ function HeroHeader() { ); } -export function WatchlistView({ data }: { data: WatchlistData }) { +export function WatchlistView({ + data, + activeHarness, + harnesses, +}: { + data: WatchlistData; + activeHarness: string; + harnesses: readonly string[]; +}) { // Bars are scaled to the worst offender (full track) so score differences // within the list are legible — not to the theoretical max of 100, which // renders every realistic score as a near-identical sliver. const max = Math.max(1, ...data.topAttention.map((r) => r.score)); return (
-
+

Watchlist

- - last {data.windowSize} runs - +
+ + + last {data.windowSize}{" "} + {harnessShortLabel(activeHarness)} runs + +

What leadership should be watching — ranked by where the signal - says to look first. + says to look first, for the selected harness.

{/* HERO */} diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index dcccdd43..eaa935d1 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -2,12 +2,12 @@ import { describe, expect, test, vi } from "vitest"; import { buildAdhocRows, collectPipelineRuns, - normalizeHarness, summarizeListing, turnBudgetRateForTasks, type PerRun, type RunListingRow, } from "../overview"; +import { normalizeHarness } from "../harness"; import type { RunOverviewTask } from "../runs"; function task(overrides: Partial): RunOverviewTask { diff --git a/evalboard/lib/harness.ts b/evalboard/lib/harness.ts new file mode 100644 index 00000000..bf36cb28 --- /dev/null +++ b/evalboard/lib/harness.ts @@ -0,0 +1,31 @@ +// Canonical harness (coder-eval AgentKind) constants. Leaf module with no +// server- or client-only dependencies, so both the client badge/selector and +// the server data layer can import it without pulling node-only code into the +// client bundle or creating an import cycle. + +// The harnesses the nightly rotates through, in preferred display order. This +// is only a display-order hint / default source — the switcher is data-driven +// (see listRecentHarnesses), so a new harness like "delegate" surfaces +// automatically without editing this list. +export const KNOWN_HARNESSES = ["claude-code", "codex", "antigravity"] as const; +export type KnownHarness = (typeof KNOWN_HARNESSES)[number]; + +// A run with no RunConfig harness predates the stamp; every such run was +// claude-code (the only nightly harness before codex/antigravity), so +// null/undefined folds to the default. Mirrors HarnessBadge's default. +export const DEFAULT_HARNESS = "claude-code"; +export function normalizeHarness(harness: string | null | undefined): string { + return harness ?? DEFAULT_HARNESS; +} + +// Accept any syntactically-plausible harness id, defaulting to claude-code when +// absent or malformed. Deliberately NOT whitelisted against KNOWN_HARNESSES so +// a newly-added harness (e.g. "delegate") is selectable the moment its runs +// exist — the value is only ever compared for equality against a run's stamped +// harness, never used in a path or query, so a bounded charset is enough. +export function parseHarnessParam(raw: string | string[] | undefined): string { + const v = Array.isArray(raw) ? raw[0] : raw; + if (typeof v !== "string") return DEFAULT_HARNESS; + const trimmed = v.trim(); + return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : DEFAULT_HARNESS; +} diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index ef57940c..0b32a981 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -15,6 +15,7 @@ import { listRunIdsInWindow, readRunReviewIndex, parseRunIdDate } from "./review import { withinTurnBudget } from "./turns"; import { humanizeTaskId } from "./format"; import { mapWithConcurrency } from "./concurrency"; +import { KNOWN_HARNESSES, normalizeHarness } from "./harness"; import type { Window } from "./reviews-types"; export interface RunPoint { @@ -254,15 +255,6 @@ async function loadWindowDataInner(window: Window): Promise { return mapWithConcurrency(ids, FETCH_CONCURRENCY, cachedLoadPerRun); } -// The harness a run is attributed to. A run whose RunConfig carried no harness -// predates the stamp, and every such run was claude-code (the only nightly -// harness before codex/antigravity were added), so null folds to claude-code. -// Mirrors HarnessBadge's `harness ?? "claude-code"` default. -export const DEFAULT_HARNESS = "claude-code"; -export function normalizeHarness(harness: string | null | undefined): string { - return harness ?? DEFAULT_HARNESS; -} - // Fetch the N most recent runs in PerRun shape. Recency-based (fixed count) // rather than date-bounded — used by the trends page. When `harness` is set, // only runs of that harness count toward N (the nightly now rotates @@ -290,6 +282,37 @@ async function loadRecentRunsInner( return collectPipelineRuns(ids, limit, cachedLoadPerRun, matchesHarness); } +// How many recent runs to scan when discovering which harnesses are active. +// All harnesses that run at least weekly appear within a window this size, so +// the switcher lists them without a hardcoded set (a new harness like +// "delegate" shows up on its own). +const HARNESS_DISCOVERY_COUNT = 12; + +async function listRecentHarnessesInner(): Promise { + const perRun = await loadRecentRuns(HARNESS_DISCOVERY_COUNT); + const seen = new Set(); + for (const r of perRun) { + if (r.overview) seen.add(normalizeHarness(r.overview.harness)); + } + // Known harnesses first (stable display order), then any newcomers + // (alphabetical) so the list is deterministic but self-extending. + const known = KNOWN_HARNESSES.filter((h) => seen.has(h)); + const extras = [...seen] + .filter((h) => !(KNOWN_HARNESSES as readonly string[]).includes(h)) + .sort(); + const ordered = [...known, ...extras]; + // Never hand back an empty list — the default must always be selectable. + return ordered.length > 0 ? ordered : [...KNOWN_HARNESSES.slice(0, 1)]; +} + +// The harnesses present in recent runs, ordered for the switcher. Cached (and +// shares the per-run cache with the aggregates), revalidated every 5 min. +export const listRecentHarnesses = unstable_cache( + listRecentHarnessesInner, + ["recent-harnesses-v1"], + { revalidate: 300 }, +); + // A loaded run occupies a window slot only when it's usable downstream: // pipeline (non-adhoc) AND has a readable overview with at least one task. // Ad-hoc uploads, transiently unreadable runs (loadPerRunForId downgrades @@ -510,11 +533,20 @@ export async function getOverview( window: Window, tag: string | null = null, q: string | null = null, + // When set, scope the chart + rails to one harness. The nightly rotates + // harnesses as separate runs, so an unscoped chart interleaves incomparable + // pass rates into one zigzag line. null = all harnesses (legacy behavior). + harness: string | null = null, ): Promise { // Ad-hoc runs never feed the daily chart or the tag rails — they're not // pipeline cadence. (Non-date-named ones are already pruned upstream by // listRunIdsInWindow; this also drops date-named runs flagged adhoc.) - const perRun = (await loadWindowData(window)).filter((r) => !r.adhoc); + const perRun = (await loadWindowData(window)).filter( + (r) => + !r.adhoc && + (harness == null || + normalizeHarness(r.overview?.harness) === harness), + ); const needle = q?.trim().toLowerCase() || null; // ---- Per-run chart points ---- @@ -592,8 +624,14 @@ export interface TagTaskRow { export async function getTagTaskBreakdown( window: Window, tag: string, + harness: string | null = null, ): Promise { - const perRun = (await loadWindowData(window)).filter((r) => !r.adhoc); + const perRun = (await loadWindowData(window)).filter( + (r) => + !r.adhoc && + (harness == null || + normalizeHarness(r.overview?.harness) === harness), + ); const sorted = [...perRun].sort((a, b) => b.id.localeCompare(a.id)); interface Acc { From b78b5d187547f8536f554c788e3b6670a9e215eb Mon Sep 17 00:00:00 2001 From: Bai Li Date: Wed, 22 Jul 2026 15:40:29 -0700 Subject: [PATCH 3/4] fix(evalboard): match watchlist skeleton header to avoid layout shift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Suspense fallback rendered only the harness selector, while the loaded view adds a "last N … runs" pill in the same right-aligned group, so the selector jumped left when the pill appeared. Mirror the pill (pulsing count) in the skeleton so the header geometry is stable through streaming. Co-Authored-By: Claude Opus 4.8 (1M context) --- evalboard/app/watchlist/page.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/evalboard/app/watchlist/page.tsx b/evalboard/app/watchlist/page.tsx index 83e5326b..7203f41d 100644 --- a/evalboard/app/watchlist/page.tsx +++ b/evalboard/app/watchlist/page.tsx @@ -4,6 +4,7 @@ import { TRENDS_RECENT_RUN_COUNT } from "@/lib/trends"; import { buildWatchlist } from "@/lib/watchlist"; import { KNOWN_HARNESSES, parseHarnessParam } from "@/lib/harness"; import { HarnessSelector } from "@/app/_components/harness-selector"; +import { harnessShortLabel } from "@/app/_components/harness-badge"; import { WatchlistView } from "./watchlist-view"; export const dynamic = "force-dynamic"; @@ -50,11 +51,20 @@ function WatchlistSkeleton({ activeHarness }: { activeHarness: string }) {

Watchlist

-
+ {/* Mirror WatchlistView's header exactly — selector + the + "last N … runs" pill — so nothing shifts horizontally when + the real content streams in and the pill appears. Only the + run count is unknown at skeleton time, so it pulses. */} +
+ + last{" "} + {" "} + {harnessShortLabel(activeHarness)} runs +

From 86315785b1cf88b40cb15e25e9a28e97a5e8e8af Mon Sep 17 00:00:00 2001 From: Bai Li Date: Wed, 22 Jul 2026 16:53:14 -0700 Subject: [PATCH 4/4] refactor(evalboard): address review nits on harness plumbing - Use the DEFAULT_HARNESS constant for the default-harness value in trends.ts and overview.ts's never-empty fallback instead of re-hardcoding the "claude-code" literal, removing a silent drift trap. - Render harnessShortLabel in the trends provenance + empty-state prose so the view no longer shows both "Claude Code" and "claude-code". - Drop a no-op readonly cast in the harness selector. - Add parseHarnessParam tests (array/absent/whitespace/valid/reject/length). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../app/_components/harness-selector.tsx | 2 +- evalboard/app/trends/trends-view.tsx | 6 +-- evalboard/lib/__tests__/harness.test.ts | 49 +++++++++++++++++++ evalboard/lib/overview.ts | 4 +- evalboard/lib/trends.ts | 7 +-- 5 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 evalboard/lib/__tests__/harness.test.ts diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx index efbfcaa4..7a814f0e 100644 --- a/evalboard/app/_components/harness-selector.tsx +++ b/evalboard/app/_components/harness-selector.tsx @@ -24,7 +24,7 @@ export function HarnessSelector({ }; // Always show the active harness, even if it has aged out of the recent // window (so a deep-linked `?h=` still reads as selected rather than absent). - const opts = (harnesses as readonly string[]).includes(current) + const opts = harnesses.includes(current) ? harnesses : [current, ...harnesses]; return ( diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx index e56d0207..67654303 100644 --- a/evalboard/app/trends/trends-view.tsx +++ b/evalboard/app/trends/trends-view.tsx @@ -626,8 +626,8 @@ export function TrendsView({

{provenance && (

- Last {provenance.count} {activeHarness} runs ·{" "} - {provenance.oldest} → {provenance.newest} + Last {provenance.count} {harnessShortLabel(activeHarness)}{" "} + runs · {provenance.oldest} → {provenance.newest}

)} {maturity.matureTasks > 0 && ( @@ -670,7 +670,7 @@ export function TrendsView({ {filterActive ? ( <>No tasks match the current filter. ) : ( - <>No recent {activeHarness} runs. + <>No recent {harnessShortLabel(activeHarness)} runs. )}
) : ( diff --git a/evalboard/lib/__tests__/harness.test.ts b/evalboard/lib/__tests__/harness.test.ts new file mode 100644 index 00000000..8dcb064c --- /dev/null +++ b/evalboard/lib/__tests__/harness.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from "vitest"; +import { DEFAULT_HARNESS, parseHarnessParam } from "../harness"; + +// parseHarnessParam gates the untrusted `?h=` query param at every +// harness-scoped route, and its result decides which runs are counted toward +// the pass-rate/trend numbers — so an over-permissive charset or a wrong +// default would silently mis-scope the board with no other failing assertion. +describe("parseHarnessParam", () => { + test("picks the first element of an array param", () => { + expect(parseHarnessParam(["codex", "antigravity"])).toBe("codex"); + }); + + test("falls back to the default for absent / empty input", () => { + expect(parseHarnessParam(undefined)).toBe(DEFAULT_HARNESS); + expect(parseHarnessParam([])).toBe(DEFAULT_HARNESS); + expect(parseHarnessParam("")).toBe(DEFAULT_HARNESS); + }); + + test("trims surrounding whitespace before validating", () => { + expect(parseHarnessParam(" codex ")).toBe("codex"); + expect(parseHarnessParam([" antigravity "])).toBe("antigravity"); + }); + + test("passes through any syntactically-valid id (not whitelisted)", () => { + // A brand-new harness must be selectable the moment its runs exist, + // so the parser must NOT reject ids outside KNOWN_HARNESSES. + for (const id of [ + "claude-code", + "codex", + "antigravity", + "delegate", + "gpt-5.5", + "some_harness", + ]) { + expect(parseHarnessParam(id)).toBe(id); + } + }); + + test("rejects malformed ids back to the default", () => { + expect(parseHarnessParam("has space")).toBe(DEFAULT_HARNESS); + expect(parseHarnessParam("a/b")).toBe(DEFAULT_HARNESS); + expect(parseHarnessParam("bad;rm")).toBe(DEFAULT_HARNESS); + }); + + test("enforces the 1-64 char length bound", () => { + expect(parseHarnessParam("a".repeat(64))).toBe("a".repeat(64)); + expect(parseHarnessParam("a".repeat(65))).toBe(DEFAULT_HARNESS); + }); +}); diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 0b32a981..4e6cfa6b 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -15,7 +15,7 @@ import { listRunIdsInWindow, readRunReviewIndex, parseRunIdDate } from "./review import { withinTurnBudget } from "./turns"; import { humanizeTaskId } from "./format"; import { mapWithConcurrency } from "./concurrency"; -import { KNOWN_HARNESSES, normalizeHarness } from "./harness"; +import { DEFAULT_HARNESS, KNOWN_HARNESSES, normalizeHarness } from "./harness"; import type { Window } from "./reviews-types"; export interface RunPoint { @@ -302,7 +302,7 @@ async function listRecentHarnessesInner(): Promise { .sort(); const ordered = [...known, ...extras]; // Never hand back an empty list — the default must always be selectable. - return ordered.length > 0 ? ordered : [...KNOWN_HARNESSES.slice(0, 1)]; + return ordered.length > 0 ? ordered : [DEFAULT_HARNESS]; } // The harnesses present in recent runs, ordered for the switcher. Cached (and diff --git a/evalboard/lib/trends.ts b/evalboard/lib/trends.ts index 53e3cdd6..f1b25b72 100644 --- a/evalboard/lib/trends.ts +++ b/evalboard/lib/trends.ts @@ -10,6 +10,7 @@ import { type PerRun, type TagCount, } from "./overview"; +import { DEFAULT_HARNESS } from "./harness"; import type { ComponentSha } from "./runs"; export const TRENDS_RECENT_RUN_COUNT = 10; @@ -245,7 +246,7 @@ const cachedAggregate = unstable_cache( export function aggregateTaskTrends( limit: number = TRENDS_RECENT_RUN_COUNT, - harness: string = "claude-code", + harness: string = DEFAULT_HARNESS, ): Promise { return cachedAggregate(limit, harness); } @@ -262,7 +263,7 @@ export function trendMatchesTag(trend: TaskTrend, tag: string): boolean { export async function historyForTaskInner( taskId: string, limit: number, - harness: string = "claude-code", + harness: string = DEFAULT_HARNESS, ): Promise { const perRun = await loadRecentRuns(limit, harness); const out: TaskHistoryEntry[] = []; @@ -300,7 +301,7 @@ const cachedHistory = unstable_cache( export function historyForTask( taskId: string, limit: number = TRENDS_RECENT_RUN_COUNT, - harness: string = "claude-code", + harness: string = DEFAULT_HARNESS, ): Promise { return cachedHistory(taskId, limit, harness); }