diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx index a9eca2bf..da16b704 100644 --- a/evalboard/app/_components/harness-badge.tsx +++ b/evalboard/app/_components/harness-badge.tsx @@ -1,22 +1,35 @@ import Image from "next/image"; -// Vendor logo for a run's harness (RunConfig). Renders the recognizable +// Canonical harness constants live in the leaf lib/harness module (shared by +// the server data layer); re-exported here so existing badge importers are +// unaffected. +export { KNOWN_HARNESSES, type KnownHarness } from "@/lib/harness"; + +// Vendor logo + labels for a run's harness (RunConfig). Renders the recognizable // vendor mark instead of raw "claude-code"/"codex"/"antigravity" text, // mirroring the Slack rollup's vendor emoji. A missing harness defaults to // claude-code (the nightly). This is an internal-only column — the caller // gates it behind isInternal (see lib/edition.ts). -const HARNESS_LOGO: Record = { +const HARNESS_LOGO: Record = { "claude-code": { src: "/harness/claude-code.png", label: "Claude Code · Anthropic", + short: "Claude Code", }, - codex: { src: "/harness/codex.png", label: "Codex · OpenAI" }, + codex: { src: "/harness/codex.png", label: "Codex · OpenAI", short: "Codex" }, antigravity: { src: "/harness/antigravity.png", label: "Antigravity · Google Gemini", + short: "Antigravity", }, }; +// Short human label for a harness id ("Claude Code"), for selectors and prose. +// Unknown ids fall through to the raw id. +export function harnessShortLabel(harness: string): string { + return HARNESS_LOGO[harness]?.short ?? harness; +} + export function HarnessBadge({ harness }: { harness?: string | null }) { const key = harness ?? "claude-code"; const logo = HARNESS_LOGO[key]; diff --git a/evalboard/app/_components/harness-selector.tsx b/evalboard/app/_components/harness-selector.tsx new file mode 100644 index 00000000..7a814f0e --- /dev/null +++ b/evalboard/app/_components/harness-selector.tsx @@ -0,0 +1,49 @@ +"use client"; + +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { HarnessBadge, harnessShortLabel } from "./harness-badge"; + +// Segmented control for the trends page's harness scope. Sets `?h=` +// while preserving the active q/tag params, mirroring WindowSelector. Each +// segment shows the vendor logo + short label so it reads like the per-run +// harness badge on the runs tables. +export function HarnessSelector({ + current, + harnesses, +}: { + current: string; + harnesses: readonly string[]; +}) { + const router = useRouter(); + const pathname = usePathname(); + const searchParams = useSearchParams(); + const set = (h: string) => { + const p = new URLSearchParams(searchParams.toString()); + p.set("h", h); + router.replace(`${pathname}?${p.toString()}`, { scroll: false }); + }; + // Always show the active harness, even if it has aged out of the recent + // window (so a deep-linked `?h=` still reads as selected rather than absent). + const opts = harnesses.includes(current) + ? harnesses + : [current, ...harnesses]; + return ( +
+ {opts.map((h) => { + const active = h === current; + return ( + + ); + })} +
+ ); +} diff --git a/evalboard/app/_overview/tag-rail.tsx b/evalboard/app/_overview/tag-rail.tsx index 0e3a65b1..2bf1f9be 100644 --- a/evalboard/app/_overview/tag-rail.tsx +++ b/evalboard/app/_overview/tag-rail.tsx @@ -1,5 +1,6 @@ import Link from "next/link"; import type { TagCount } from "@/lib/overview"; +import { DEFAULT_HARNESS } from "@/lib/harness"; import type { Window } from "@/lib/reviews-types"; type Variant = "neutral" | "rose" | "indigo"; @@ -35,11 +36,16 @@ function hrefForTag( tag: string | null, window: Window | null, q: string | null, + harness: string | null, ): string { const params = new URLSearchParams(); if (window) params.set("window", window); if (tag) params.set("tag", tag); if (q) params.set("q", q); + // Preserve the active harness scope across tag clicks (omit the default to + // keep URLs clean). Without this, filtering by a tag would silently reset a + // codex/antigravity view back to claude-code. + if (harness && harness !== DEFAULT_HARNESS) params.set("h", harness); const qs = params.toString(); return qs ? `${basePath}?${qs}` : basePath; } @@ -52,6 +58,7 @@ function TagChip({ basePath, window, q, + harness, }: { tag: string; count: number; @@ -60,11 +67,12 @@ function TagChip({ basePath: string; window: Window | null; q: string | null; + harness: string | null; }) { const s = STYLES[variant]; return ( @@ -111,6 +119,7 @@ export function MergedTagRail({ basePath = "/", window = null, q = null, + harness = null, limit = 24, }: { skills: TagCount[]; @@ -123,6 +132,8 @@ export function MergedTagRail({ // Null on pages that don't expose a window selector (e.g. /trends). window?: Window | null; q?: string | null; + // Active harness scope to preserve in chip links (null = not harness-scoped). + harness?: string | null; limit?: number; }) { const s = pickShown(skills, limit, activeTag); @@ -146,6 +157,7 @@ export function MergedTagRail({ basePath={basePath} window={window} q={q} + harness={harness} /> ))} {r.shown.map((tc) => ( @@ -158,6 +170,7 @@ export function MergedTagRail({ basePath={basePath} window={window} q={q} + harness={harness} /> ))} {t.shown.map((tc) => ( @@ -170,6 +183,7 @@ export function MergedTagRail({ basePath={basePath} window={window} q={q} + harness={harness} /> ))} {totalRemaining > 0 && ( diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index 5cd66b92..d8326a0a 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -3,8 +3,10 @@ import { getAdhocRunListing, getOverview, getRunListing, + listRecentHarnesses, type TagCount, } from "@/lib/overview"; +import { parseHarnessParam, DEFAULT_HARNESS } from "@/lib/harness"; import { fmtDuration, fmtRunTime, fmtTimestamp, passClass } from "@/lib/format"; import { WindowSelector } from "./_components/window-selector"; import { WINDOWS, type Window } from "@/lib/reviews-types"; @@ -15,7 +17,8 @@ import { ChipLegend, MergedTagRail } from "./_overview/tag-rail"; import { TableScroll } from "./_components/scroll-table"; import { CollapsibleRail } from "./_components/collapsible-rail"; import { isInternal } from "@/lib/edition"; -import { HarnessBadge } from "@/app/_components/harness-badge"; +import { HarnessBadge, harnessShortLabel } from "@/app/_components/harness-badge"; +import { HarnessSelector } from "@/app/_components/harness-selector"; export const dynamic = "force-dynamic"; @@ -98,6 +101,7 @@ export default async function Page({ window?: string; tag?: string; q?: string; + h?: string; limit?: string; alimit?: string; }>; @@ -106,14 +110,20 @@ export default async function Page({ const window = parseWindow(params.window); const activeTag = parseTag(params.tag); const q = parseQ(params.q); + const harness = parseHarnessParam(params.h); const limit = parseLimit(params.limit); const adhocLimit = parseAdhocLimit(params.alimit); const isFiltered = activeTag != null || q != null; - const [overview, listing, adhoc] = await Promise.all([ - getOverview(window, activeTag, q), + // The analytics block (chart + rails) is scoped to one harness so the + // success line stops zigzagging across incomparable harnesses. The run + // LIST stays all-harness — seeing every recent run is the page's job, and + // the Harness column already disambiguates each row. + const [overview, listing, adhoc, harnesses] = await Promise.all([ + getOverview(window, activeTag, q, harness), getRunListing(window, activeTag, q, limit), getAdhocRunListing(adhocLimit), + listRecentHarnesses(), ]); const skills = filterTagsByQuery(overview.skills, q); @@ -133,10 +143,14 @@ export default async function Page({ const rawAlimit = Array.isArray(params.alimit) ? params.alimit[0] : params.alimit; + // Omit the default harness from URLs to keep them clean; carry a non-default + // scope through every self-link so it isn't reset by pagination/clear. + const hParam = harness === DEFAULT_HARNESS ? undefined : harness; const base = { window, tag: activeTag, q, + h: hParam, limit: rawLimit, alimit: rawAlimit, }; @@ -146,7 +160,7 @@ export default async function Page({ limit: Math.min(tableTotalLabel, shownCount + DEFAULT_LIMIT), }); const showAllHref = buildHref({ ...base, limit: "all" }); - const clearAllHref = buildHref({ window }); + const clearAllHref = buildHref({ window, h: hParam }); // Ad-hoc section disclosure: rows are filtered (by `q`) then capped to // adhocLimit; offer "Show all" while more match than are shown, and a @@ -212,7 +226,7 @@ export default async function Page({ {overview.runs.length === 1 ? "" : "s"} {" · "} @@ -221,14 +235,21 @@ export default async function Page({ ) : ( <> - Success rate per run across the last{" "} - {window} · {overview.runs.length} run + Success rate per{" "} + {harnessShortLabel(harness)} run across the + last {window} · {overview.runs.length} run {overview.runs.length === 1 ? "" : "s"} )}

- +
+ + +
diff --git a/evalboard/app/path-to-ga/page.tsx b/evalboard/app/path-to-ga/page.tsx index be6496aa..57b8514b 100644 --- a/evalboard/app/path-to-ga/page.tsx +++ b/evalboard/app/path-to-ga/page.tsx @@ -1,7 +1,14 @@ import Link from "next/link"; -import { getOverview, getTagTaskBreakdown } from "@/lib/overview"; +import { + getOverview, + getTagTaskBreakdown, + listRecentHarnesses, +} from "@/lib/overview"; +import { parseHarnessParam } from "@/lib/harness"; import { humanizeTaskId } from "@/lib/format"; import { WindowSelector } from "../_components/window-selector"; +import { HarnessSelector } from "../_components/harness-selector"; +import { harnessShortLabel } from "../_components/harness-badge"; import { WINDOWS, type Window } from "@/lib/reviews-types"; import { DailySuccessChart } from "../_overview/daily-chart"; import { TableScroll } from "../_components/scroll-table"; @@ -26,14 +33,18 @@ function passClass(pct: number | null): string { export default async function PathToGaPage({ searchParams, }: { - searchParams: Promise<{ window?: string }>; + searchParams: Promise<{ window?: string; h?: string }>; }) { const params = await searchParams; const window = parseWindow(params.window); + const harness = parseHarnessParam(params.h); - const [overview, taskRows] = await Promise.all([ - getOverview(window, TAG, null), - getTagTaskBreakdown(window, TAG), + // Scope to one harness — readiness of a task is per-harness, and a blended + // chart/pass-rate mixes incomparable regimes. + const [overview, taskRows, harnesses] = await Promise.all([ + getOverview(window, TAG, null, harness), + getTagTaskBreakdown(window, TAG, harness), + listRecentHarnesses(), ]); const runsInWindow = overview.runs.length; @@ -52,10 +63,14 @@ export default async function PathToGaPage({

Score for every task tagged{" "} - {TAG}. + {TAG} on{" "} + {harnessShortLabel(harness)}.

- +
+ + +
diff --git a/evalboard/app/trends/__tests__/trends-view.test.tsx b/evalboard/app/trends/__tests__/trends-view.test.tsx index 20d60a42..9899b127 100644 --- a/evalboard/app/trends/__tests__/trends-view.test.tsx +++ b/evalboard/app/trends/__tests__/trends-view.test.tsx @@ -1,6 +1,7 @@ import { describe, expect, test, vi } from "vitest"; import { render, screen } from "@testing-library/react"; import type { TaskTrend } from "@/lib/trends"; +import { KNOWN_HARNESSES } from "@/app/_components/harness-badge"; // The view imports the server action for the expandable history rows; stub it // so rendering doesn't pull the blob-backed loader into a jsdom test. @@ -8,6 +9,14 @@ vi.mock("../actions", () => ({ fetchTaskHistoryAction: vi.fn(async () => []), })); +// The header's HarnessSelector reads router/params hooks; stub them so the view +// renders in jsdom without a router provider. +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace: vi.fn() }), + usePathname: () => "/trends", + useSearchParams: () => new URLSearchParams(), +})); + const { TrendsView } = await import("../trends-view"); function trend(overrides: Partial): TaskTrend { @@ -35,6 +44,8 @@ function renderView(tasks: TaskTrend[], runIds: string[]) { runIds={runIds} q={null} activeTag={null} + activeHarness="claude-code" + harnesses={KNOWN_HARNESSES} skills={[]} taskTags={[]} reviewTags={[]} diff --git a/evalboard/app/trends/actions.ts b/evalboard/app/trends/actions.ts index c1a16574..a629e4d6 100644 --- a/evalboard/app/trends/actions.ts +++ b/evalboard/app/trends/actions.ts @@ -1,10 +1,18 @@ "use server"; -import { historyForTask, type TaskHistoryEntry } from "@/lib/trends"; +import { + historyForTask, + type TaskHistoryEntry, + TRENDS_RECENT_RUN_COUNT, +} from "@/lib/trends"; +import { parseHarnessParam } from "@/lib/harness"; export async function fetchTaskHistoryAction( taskId: string, + harness: string, ): Promise { if (typeof taskId !== "string" || !taskId) return []; - return historyForTask(taskId); + // Normalize/validate the harness (any plausible id; defaults to the primary + // harness) so the history scope matches the trends table it expands from. + return historyForTask(taskId, TRENDS_RECENT_RUN_COUNT, parseHarnessParam(harness)); } diff --git a/evalboard/app/trends/page.tsx b/evalboard/app/trends/page.tsx index f927cf62..37555662 100644 --- a/evalboard/app/trends/page.tsx +++ b/evalboard/app/trends/page.tsx @@ -1,10 +1,14 @@ +import { Suspense } from "react"; import { aggregateTaskTrends, trendMatchesTag, TRENDS_RECENT_RUN_COUNT, } from "@/lib/trends"; -import { aggregateTaskTagCounts, loadRecentRuns } from "@/lib/overview"; +import { listRecentHarnesses } from "@/lib/overview"; +import { KNOWN_HARNESSES, parseHarnessParam } from "@/lib/harness"; import { fmtRunTime, humanizeTaskId } from "@/lib/format"; +import { harnessShortLabel } from "@/app/_components/harness-badge"; +import { HarnessSelector } from "@/app/_components/harness-selector"; import { TrendsView } from "./trends-view"; export const dynamic = "force-dynamic"; @@ -25,26 +29,53 @@ function parseTag(raw: string | string[] | undefined): string | null { return trimmed.slice(0, 100); } +// The page shell only needs the URL params (cheap), so it returns immediately +// and streams the data-backed table in behind a Suspense boundary — the ~20 MB +// cold aggregate load no longer blocks the whole response. The header + +// harness selector paint instantly (and stay usable) via the fallback. export default async function TrendsPage({ searchParams, }: { - searchParams: Promise<{ q?: string; tag?: string }>; + searchParams: Promise<{ q?: string; tag?: string; h?: string }>; }) { const params = await searchParams; const q = parseQ(params.q); const activeTag = parseTag(params.tag); + const harness = parseHarnessParam(params.h); - const [{ runIds, trends: allTrends }, perRun] = await Promise.all([ - aggregateTaskTrends(TRENDS_RECENT_RUN_COUNT), - loadRecentRuns(TRENDS_RECENT_RUN_COUNT), - ]); - const tagCounts = aggregateTaskTagCounts(perRun); + return ( + // Key on harness so a switch (a fresh, heavy load) shows the skeleton; + // q/tag changes keep the boundary and re-filter the cached aggregate in + // a transition without flashing the fallback. + }> + + + ); +} + +async function TrendsContent({ + q, + activeTag, + harness, +}: { + q: string | null; + activeTag: string | null; + harness: string; +}) { + // One cached load, scoped to the selected harness — yields the trends, the + // run axis, AND the tag-rail counts (all from the same window). The harness + // list (for the switcher) is a separate small cached discovery load. + const [{ runIds, trends: allTrends, tagCounts }, harnesses] = + await Promise.all([ + aggregateTaskTrends(TRENDS_RECENT_RUN_COUNT, harness), + listRecentHarnesses(), + ]); - // Provenance: surface the actual run count + date span. Mixing runs across - // model/agent regimes into a single pass rate is otherwise invisible — - // showing the window at least makes the user aware of what's collapsed. - // Derived from the same cached aggregate as the table (runIds is - // newest-first), so the label and the strips can't skew apart. + // Provenance: surface the actual run count + date span. Even scoped to one + // harness, the window can straddle model/prompt regimes, so showing the + // span keeps the user aware of what's collapsed. Derived from the same + // cached aggregate as the table (runIds is newest-first), so the label and + // the strips can't skew apart. const provenance = runIds.length === 0 ? null @@ -79,6 +110,8 @@ export default async function TrendsPage({ runIds={runIds} q={q} activeTag={activeTag} + activeHarness={harness} + harnesses={harnesses} skills={tagCounts.skills} taskTags={tagCounts.taskTags} reviewTags={tagCounts.reviewTags} @@ -86,3 +119,45 @@ export default async function TrendsPage({ /> ); } + +// Streamed while the aggregate loads. Mirrors TrendsView's header (so it +// doesn't jump on swap) and keeps the harness selector live so the user can +// re-scope without waiting, then shims the table with pulsing rows. +function TrendsSkeleton({ activeHarness }: { activeHarness: string }) { + return ( +
+
+
+

+ Task trends +

+

+ Per-task pass rate and averages across recent{" "} + {harnessShortLabel(activeHarness)} runs. Averages cover + successful runs only. +

+
+ +
+
+
+
+
+ {Array.from({ length: 8 }).map((_, i) => ( +
+
+
+ ))} +
+
+ ); +} diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx index 89700879..67654303 100644 --- a/evalboard/app/trends/trends-view.tsx +++ b/evalboard/app/trends/trends-view.tsx @@ -21,6 +21,8 @@ import { ChipButton } from "@/app/runs/[id]/chips"; import { TableScroll } from "@/app/_components/scroll-table"; import { CollapsibleRail } from "@/app/_components/collapsible-rail"; import { VersionChip } from "@/app/_components/version-list"; +import { HarnessSelector } from "@/app/_components/harness-selector"; +import { harnessShortLabel } from "@/app/_components/harness-badge"; import { fetchTaskHistoryAction } from "./actions"; function fmtUsd(c: number | null): string { @@ -534,6 +536,8 @@ export function TrendsView({ runIds, q, activeTag, + activeHarness, + harnesses, skills, taskTags, reviewTags, @@ -544,6 +548,9 @@ export function TrendsView({ runIds: string[]; q: string | null; activeTag: string | null; + // The harness this trend is scoped to, and the switchable set. + activeHarness: string; + harnesses: readonly string[]; skills: TagCount[]; taskTags: TagCount[]; reviewTags: TagCount[]; @@ -589,7 +596,7 @@ export function TrendsView({ setOpenTaskId((cur) => (cur === taskId ? null : taskId)); if (history[taskId] != null) return; startTransition(async () => { - const entries = await fetchTaskHistoryAction(taskId); + const entries = await fetchTaskHistoryAction(taskId, activeHarness); setHistory((prev) => ({ ...prev, [taskId]: entries })); }); } @@ -599,17 +606,28 @@ export function TrendsView({ return (
-

- Task trends -

-

- Per-task pass rate and averages across recent runs. - Averages cover successful runs only. -

+
+
+

+ Task trends +

+

+ Per-task pass rate and averages across recent{" "} + {harnessShortLabel(activeHarness)} runs. Averages + cover successful runs only. +

+
+ {/* Scope the whole page to one harness — mixing harnesses + blends incomparable pass rates and cost profiles. */} + +
{provenance && (

- Last {provenance.count} runs · {provenance.oldest} →{" "} - {provenance.newest} + Last {provenance.count} {harnessShortLabel(activeHarness)}{" "} + runs · {provenance.oldest} → {provenance.newest}

)} {maturity.matureTasks > 0 && ( @@ -632,6 +650,7 @@ export function TrendsView({ activeTag={activeTag} basePath="/trends" q={q} + harness={activeHarness} limit={24} /> @@ -651,7 +670,7 @@ export function TrendsView({ {filterActive ? ( <>No tasks match the current filter. ) : ( - <>No recent runs. + <>No recent {harnessShortLabel(activeHarness)} runs. )}
) : ( diff --git a/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx b/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx index 9e147173..c19c6c7a 100644 --- a/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx +++ b/evalboard/app/watchlist/__tests__/watchlist-view.test.tsx @@ -1,9 +1,28 @@ -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import { render, screen } from "@testing-library/react"; import type { PerRun } from "@/lib/overview"; import type { RunOverview, RunOverviewTask } from "@/lib/runs"; -import { buildWatchlist } from "@/lib/watchlist"; -import { WatchlistView } from "../watchlist-view"; +import { buildWatchlist, type WatchlistData } from "@/lib/watchlist"; + +// The header's HarnessSelector reads router/params hooks; stub them so the view +// renders in jsdom without a router provider. +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace: vi.fn() }), + usePathname: () => "/watchlist", + useSearchParams: () => new URLSearchParams(), +})); + +const { WatchlistView } = await import("../watchlist-view"); + +function renderView(data: WatchlistData) { + return render( + , + ); +} function task(o: Partial): RunOverviewTask { return { @@ -45,7 +64,7 @@ describe("WatchlistView", () => { task({ taskId: "b", skill: "beta", status: "FAILURE" }), ]), ]); - render(); + renderView(data); expect(screen.getByText("Watchlist")).toBeInTheDocument(); expect(screen.getByText("Needs Attention")).toBeInTheDocument(); @@ -70,7 +89,7 @@ describe("WatchlistView", () => { task({ taskId: "w2", skill: "wobble", status: "SUCCESS" }), ]), ]); - render(); + renderView(data); // Definition is reachable as the ⓘ tooltip's accessible label. expect( @@ -91,9 +110,7 @@ describe("WatchlistView", () => { status: "FAILURE", }), ); - render( - , - ); + renderView(buildWatchlist([run("2026-01-01", tasks)])); // The expander summary appears with the full count + tie callout. expect(screen.getByText(/Show all 12 \(2 more tied\)/)).toBeInTheDocument(); // Beyond-cap rows are in the DOM (inside
), not dropped. @@ -101,8 +118,8 @@ describe("WatchlistView", () => { }); test("empty window renders friendly empty states without throwing", () => { - render(); + renderView(buildWatchlist([])); expect(screen.getByText("Nothing needs attention 🎉")).toBeInTheDocument(); - expect(screen.getByText("last 0 runs")).toBeInTheDocument(); + expect(screen.getByText("last 0 Claude Code runs")).toBeInTheDocument(); }); }); diff --git a/evalboard/app/watchlist/page.tsx b/evalboard/app/watchlist/page.tsx index da78c3c9..7203f41d 100644 --- a/evalboard/app/watchlist/page.tsx +++ b/evalboard/app/watchlist/page.tsx @@ -1,12 +1,87 @@ -import { loadRecentRuns } from "@/lib/overview"; +import { Suspense } from "react"; +import { loadRecentRuns, listRecentHarnesses } from "@/lib/overview"; import { TRENDS_RECENT_RUN_COUNT } from "@/lib/trends"; import { buildWatchlist } from "@/lib/watchlist"; +import { KNOWN_HARNESSES, parseHarnessParam } from "@/lib/harness"; +import { HarnessSelector } from "@/app/_components/harness-selector"; +import { harnessShortLabel } from "@/app/_components/harness-badge"; import { WatchlistView } from "./watchlist-view"; export const dynamic = "force-dynamic"; -export default async function WatchlistPage() { - const perRun = await loadRecentRuns(TRENDS_RECENT_RUN_COUNT); +export default async function WatchlistPage({ + searchParams, +}: { + searchParams: Promise<{ h?: string }>; +}) { + const params = await searchParams; + const harness = parseHarnessParam(params.h); + return ( + } + > + + + ); +} + +async function WatchlistContent({ harness }: { harness: string }) { + // Scope the whole watchlist to one harness — the pass rates, streaks, and + // volatility only mean something within a single harness (mixing them makes + // a Claude-reliable skill look flaky on codex/antigravity days). + const [perRun, harnesses] = await Promise.all([ + loadRecentRuns(TRENDS_RECENT_RUN_COUNT, harness), + listRecentHarnesses(), + ]); const data = buildWatchlist(perRun); - return ; + return ( + + ); +} + +function WatchlistSkeleton({ activeHarness }: { activeHarness: string }) { + return ( +
+
+

+ Watchlist +

+ {/* Mirror WatchlistView's header exactly — selector + the + "last N … runs" pill — so nothing shifts horizontally when + the real content streams in and the pill appears. Only the + run count is unknown at skeleton time, so it pulses. */} +
+ + + last{" "} + {" "} + {harnessShortLabel(activeHarness)} runs + +
+
+

+ What leadership should be watching — ranked by where the signal + says to look first, for the selected harness. +

+
+ {Array.from({ length: 5 }).map((_, i) => ( +
+ ))} +
+
+ ); } diff --git a/evalboard/app/watchlist/watchlist-view.tsx b/evalboard/app/watchlist/watchlist-view.tsx index 22f3b00a..17b0c971 100644 --- a/evalboard/app/watchlist/watchlist-view.tsx +++ b/evalboard/app/watchlist/watchlist-view.tsx @@ -6,6 +6,8 @@ import type { ReactNode } from "react"; import Link from "next/link"; import { humanizeTaskId } from "@/lib/format"; +import { HarnessSelector } from "@/app/_components/harness-selector"; +import { harnessShortLabel } from "@/app/_components/harness-badge"; import { FAIL_WEIGHT, REG_WEIGHT, @@ -275,24 +277,39 @@ function HeroHeader() { ); } -export function WatchlistView({ data }: { data: WatchlistData }) { +export function WatchlistView({ + data, + activeHarness, + harnesses, +}: { + data: WatchlistData; + activeHarness: string; + harnesses: readonly string[]; +}) { // Bars are scaled to the worst offender (full track) so score differences // within the list are legible — not to the theoretical max of 100, which // renders every realistic score as a near-identical sliver. const max = Math.max(1, ...data.topAttention.map((r) => r.score)); return (
-
+

Watchlist

- - last {data.windowSize} runs - +
+ + + last {data.windowSize}{" "} + {harnessShortLabel(activeHarness)} runs + +

What leadership should be watching — ranked by where the signal - says to look first. + says to look first, for the selected harness.

{/* HERO */} diff --git a/evalboard/lib/__tests__/harness.test.ts b/evalboard/lib/__tests__/harness.test.ts new file mode 100644 index 00000000..8dcb064c --- /dev/null +++ b/evalboard/lib/__tests__/harness.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from "vitest"; +import { DEFAULT_HARNESS, parseHarnessParam } from "../harness"; + +// parseHarnessParam gates the untrusted `?h=` query param at every +// harness-scoped route, and its result decides which runs are counted toward +// the pass-rate/trend numbers — so an over-permissive charset or a wrong +// default would silently mis-scope the board with no other failing assertion. +describe("parseHarnessParam", () => { + test("picks the first element of an array param", () => { + expect(parseHarnessParam(["codex", "antigravity"])).toBe("codex"); + }); + + test("falls back to the default for absent / empty input", () => { + expect(parseHarnessParam(undefined)).toBe(DEFAULT_HARNESS); + expect(parseHarnessParam([])).toBe(DEFAULT_HARNESS); + expect(parseHarnessParam("")).toBe(DEFAULT_HARNESS); + }); + + test("trims surrounding whitespace before validating", () => { + expect(parseHarnessParam(" codex ")).toBe("codex"); + expect(parseHarnessParam([" antigravity "])).toBe("antigravity"); + }); + + test("passes through any syntactically-valid id (not whitelisted)", () => { + // A brand-new harness must be selectable the moment its runs exist, + // so the parser must NOT reject ids outside KNOWN_HARNESSES. + for (const id of [ + "claude-code", + "codex", + "antigravity", + "delegate", + "gpt-5.5", + "some_harness", + ]) { + expect(parseHarnessParam(id)).toBe(id); + } + }); + + test("rejects malformed ids back to the default", () => { + expect(parseHarnessParam("has space")).toBe(DEFAULT_HARNESS); + expect(parseHarnessParam("a/b")).toBe(DEFAULT_HARNESS); + expect(parseHarnessParam("bad;rm")).toBe(DEFAULT_HARNESS); + }); + + test("enforces the 1-64 char length bound", () => { + expect(parseHarnessParam("a".repeat(64))).toBe("a".repeat(64)); + expect(parseHarnessParam("a".repeat(65))).toBe(DEFAULT_HARNESS); + }); +}); diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index aed22813..eaa935d1 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -7,6 +7,7 @@ import { type PerRun, type RunListingRow, } from "../overview"; +import { normalizeHarness } from "../harness"; import type { RunOverviewTask } from "../runs"; function task(overrides: Partial): RunOverviewTask { @@ -302,6 +303,62 @@ describe("collectPipelineRuns", () => { expect(await collectPipelineRuns([], 5, load)).toEqual([]); expect(load).not.toHaveBeenCalled(); }); + + // A usable run tagged with a specific harness. + function runH(id: string, harness: string): PerRun { + const r = run(id); + return { ...r, overview: { ...r.overview!, harness } }; + } + + test("isMatch filters to the matching harness and backfills the rest", async () => { + // Interleaved harnesses; scoping to claude-code must skip the codex + // runs and reach further back to fill the window rather than returning + // short. + const harnesses: Record = { + r6: "claude-code", + r5: "codex", + r4: "claude-code", + r3: "codex", + r2: "claude-code", + r1: "codex", + }; + const load = vi.fn(async (id: string) => runH(id, harnesses[id])); + const out = await collectPipelineRuns( + ["r6", "r5", "r4", "r3", "r2", "r1"], + 3, + load, + (r) => r.overview?.harness === "claude-code", + ); + expect(out.map((r) => r.id)).toEqual(["r6", "r4", "r2"]); + }); + + test("harness scan reaches past RECENT_SCAN_FACTOR to gather a rare harness", async () => { + // limit 1; the only antigravity run sits at index 5 — beyond the + // unfiltered cap (1 × RECENT_SCAN_FACTOR = 3). The wider harness cap + // (1 × HARNESS_SCAN_FACTOR = 8) must reach it. + const ids = ["r6", "r5", "r4", "r3", "r2", "r1"]; + const load = vi.fn(async (id: string) => + runH(id, id === "r1" ? "antigravity" : "claude-code"), + ); + const out = await collectPipelineRuns( + ids, + 1, + load, + (r) => r.overview?.harness === "antigravity", + ); + expect(out.map((r) => r.id)).toEqual(["r1"]); + }); +}); + +describe("normalizeHarness", () => { + test("null/undefined fold to claude-code (legacy pre-stamp runs)", () => { + expect(normalizeHarness(null)).toBe("claude-code"); + expect(normalizeHarness(undefined)).toBe("claude-code"); + }); + test("an explicit harness passes through unchanged", () => { + expect(normalizeHarness("codex")).toBe("codex"); + expect(normalizeHarness("antigravity")).toBe("antigravity"); + }); }); describe("buildAdhocRows", () => { diff --git a/evalboard/lib/harness.ts b/evalboard/lib/harness.ts new file mode 100644 index 00000000..bf36cb28 --- /dev/null +++ b/evalboard/lib/harness.ts @@ -0,0 +1,31 @@ +// Canonical harness (coder-eval AgentKind) constants. Leaf module with no +// server- or client-only dependencies, so both the client badge/selector and +// the server data layer can import it without pulling node-only code into the +// client bundle or creating an import cycle. + +// The harnesses the nightly rotates through, in preferred display order. This +// is only a display-order hint / default source — the switcher is data-driven +// (see listRecentHarnesses), so a new harness like "delegate" surfaces +// automatically without editing this list. +export const KNOWN_HARNESSES = ["claude-code", "codex", "antigravity"] as const; +export type KnownHarness = (typeof KNOWN_HARNESSES)[number]; + +// A run with no RunConfig harness predates the stamp; every such run was +// claude-code (the only nightly harness before codex/antigravity), so +// null/undefined folds to the default. Mirrors HarnessBadge's default. +export const DEFAULT_HARNESS = "claude-code"; +export function normalizeHarness(harness: string | null | undefined): string { + return harness ?? DEFAULT_HARNESS; +} + +// Accept any syntactically-plausible harness id, defaulting to claude-code when +// absent or malformed. Deliberately NOT whitelisted against KNOWN_HARNESSES so +// a newly-added harness (e.g. "delegate") is selectable the moment its runs +// exist — the value is only ever compared for equality against a run's stamped +// harness, never used in a path or query, so a bounded charset is enough. +export function parseHarnessParam(raw: string | string[] | undefined): string { + const v = Array.isArray(raw) ? raw[0] : raw; + if (typeof v !== "string") return DEFAULT_HARNESS; + const trimmed = v.trim(); + return /^[\w.-]{1,64}$/.test(trimmed) ? trimmed : DEFAULT_HARNESS; +} diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index 68c81568..4e6cfa6b 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -15,6 +15,7 @@ import { listRunIdsInWindow, readRunReviewIndex, parseRunIdDate } from "./review import { withinTurnBudget } from "./turns"; import { humanizeTaskId } from "./format"; import { mapWithConcurrency } from "./concurrency"; +import { DEFAULT_HARNESS, KNOWN_HARNESSES, normalizeHarness } from "./harness"; import type { Window } from "./reviews-types"; export interface RunPoint { @@ -255,20 +256,63 @@ async function loadWindowDataInner(window: Window): Promise { } // Fetch the N most recent runs in PerRun shape. Recency-based (fixed count) -// rather than date-bounded — used by the trends page. -export function loadRecentRuns(limit: number): Promise { - return loadRecentRunsInner(limit); +// rather than date-bounded — used by the trends page. When `harness` is set, +// only runs of that harness count toward N (the nightly now rotates +// claude-code / codex / antigravity, and a trend is only meaningful within one +// harness — mixing them blends incomparable pass rates and cost profiles). +export function loadRecentRuns( + limit: number, + harness?: string, +): Promise { + return loadRecentRunsInner(limit, harness); } -async function loadRecentRunsInner(limit: number): Promise { +async function loadRecentRunsInner( + limit: number, + harness?: string, +): Promise { // Trends is the daily-cadence view: only pipeline runs belong here. Prune // to date-shaped ids BEFORE slicing (cheap, no IO) so ad-hoc runs — whose // ids sort lexically above every `2026-…` daily id and would otherwise // crowd out the real "recent N" — never occupy a slot. const ids = (await listRunIds()).filter((id) => parseRunIdDate(id) != null); - return collectPipelineRuns(ids, limit, cachedLoadPerRun); + const matchesHarness = harness + ? (r: PerRun) => normalizeHarness(r.overview?.harness) === harness + : undefined; + return collectPipelineRuns(ids, limit, cachedLoadPerRun, matchesHarness); +} + +// How many recent runs to scan when discovering which harnesses are active. +// All harnesses that run at least weekly appear within a window this size, so +// the switcher lists them without a hardcoded set (a new harness like +// "delegate" shows up on its own). +const HARNESS_DISCOVERY_COUNT = 12; + +async function listRecentHarnessesInner(): Promise { + const perRun = await loadRecentRuns(HARNESS_DISCOVERY_COUNT); + const seen = new Set(); + for (const r of perRun) { + if (r.overview) seen.add(normalizeHarness(r.overview.harness)); + } + // Known harnesses first (stable display order), then any newcomers + // (alphabetical) so the list is deterministic but self-extending. + const known = KNOWN_HARNESSES.filter((h) => seen.has(h)); + const extras = [...seen] + .filter((h) => !(KNOWN_HARNESSES as readonly string[]).includes(h)) + .sort(); + const ordered = [...known, ...extras]; + // Never hand back an empty list — the default must always be selectable. + return ordered.length > 0 ? ordered : [DEFAULT_HARNESS]; } +// The harnesses present in recent runs, ordered for the switcher. Cached (and +// shares the per-run cache with the aggregates), revalidated every 5 min. +export const listRecentHarnesses = unstable_cache( + listRecentHarnessesInner, + ["recent-harnesses-v1"], + { revalidate: 300 }, +); + // A loaded run occupies a window slot only when it's usable downstream: // pipeline (non-adhoc) AND has a readable overview with at least one task. // Ad-hoc uploads, transiently unreadable runs (loadPerRunForId downgrades @@ -287,6 +331,11 @@ function isUsablePipelineRun(r: PerRun): boolean { // deficit-sized rounds from degenerating into one-id-per-round serial loads // when a single slot stays unfilled. const RECENT_SCAN_FACTOR = 3; +// When filtering to a single harness, that harness holds only a fraction of the +// recent runs (codex/antigravity run a few times a week vs. claude-code daily), +// so the scan has to reach further back to gather `limit` of them. A larger cap +// keeps a rarer harness from coming up short while still bounding the probe. +const HARNESS_SCAN_FACTOR = 8; const MIN_PROBE_BATCH = 5; // Load runs newest-first until `limit` usable pipeline runs are in hand, the @@ -305,8 +354,15 @@ export async function collectPipelineRuns( ids: string[], limit: number, load: (id: string) => Promise, + // Optional extra predicate (AND-ed with usability) — e.g. a harness filter. + // When set, the scan reaches further back since matches are sparser. + isMatch?: (r: PerRun) => boolean, ): Promise { - const maxScan = Math.min(ids.length, limit * RECENT_SCAN_FACTOR); + const scanFactor = isMatch ? HARNESS_SCAN_FACTOR : RECENT_SCAN_FACTOR; + const maxScan = Math.min(ids.length, limit * scanFactor); + const usable = isMatch + ? (r: PerRun) => isUsablePipelineRun(r) && isMatch(r) + : isUsablePipelineRun; const out: PerRun[] = []; let cursor = 0; while (out.length < limit && cursor < maxScan) { @@ -318,7 +374,7 @@ export async function collectPipelineRuns( const batch = ids.slice(cursor, Math.min(cursor + batchSize, maxScan)); cursor += batch.length; const loaded = await mapWithConcurrency(batch, FETCH_CONCURRENCY, load); - out.push(...loaded.filter(isUsablePipelineRun)); + out.push(...loaded.filter(usable)); } if (out.length < limit && cursor >= maxScan && cursor < ids.length) { console.warn( @@ -477,11 +533,20 @@ export async function getOverview( window: Window, tag: string | null = null, q: string | null = null, + // When set, scope the chart + rails to one harness. The nightly rotates + // harnesses as separate runs, so an unscoped chart interleaves incomparable + // pass rates into one zigzag line. null = all harnesses (legacy behavior). + harness: string | null = null, ): Promise { // Ad-hoc runs never feed the daily chart or the tag rails — they're not // pipeline cadence. (Non-date-named ones are already pruned upstream by // listRunIdsInWindow; this also drops date-named runs flagged adhoc.) - const perRun = (await loadWindowData(window)).filter((r) => !r.adhoc); + const perRun = (await loadWindowData(window)).filter( + (r) => + !r.adhoc && + (harness == null || + normalizeHarness(r.overview?.harness) === harness), + ); const needle = q?.trim().toLowerCase() || null; // ---- Per-run chart points ---- @@ -559,8 +624,14 @@ export interface TagTaskRow { export async function getTagTaskBreakdown( window: Window, tag: string, + harness: string | null = null, ): Promise { - const perRun = (await loadWindowData(window)).filter((r) => !r.adhoc); + const perRun = (await loadWindowData(window)).filter( + (r) => + !r.adhoc && + (harness == null || + normalizeHarness(r.overview?.harness) === harness), + ); const sorted = [...perRun].sort((a, b) => b.id.localeCompare(a.id)); interface Acc { diff --git a/evalboard/lib/trends.ts b/evalboard/lib/trends.ts index 1c8b6787..f1b25b72 100644 --- a/evalboard/lib/trends.ts +++ b/evalboard/lib/trends.ts @@ -4,7 +4,13 @@ // means "last 10 runs" is a more useful unit than "last 7 days". import { unstable_cache } from "next/cache"; -import { loadRecentRuns, type PerRun } from "./overview"; +import { + aggregateTaskTagCounts, + loadRecentRuns, + type PerRun, + type TagCount, +} from "./overview"; +import { DEFAULT_HARNESS } from "./harness"; import type { ComponentSha } from "./runs"; export const TRENDS_RECENT_RUN_COUNT = 10; @@ -208,22 +214,41 @@ export function aggregate(perRun: PerRun[]): TrendsData { return { runIds, trends }; } -async function aggregateTaskTrendsInner(limit: number): Promise { - return aggregate(await loadRecentRuns(limit)); +// TrendsData plus the tag-rail counts, both derived from the SAME loaded +// window. Folding the counts in here means the trends page loads the recent-run +// window once (via this cached aggregate) instead of a second uncached +// loadRecentRuns call just for the rail. +export interface TrendsPageData extends TrendsData { + tagCounts: { + skills: TagCount[]; + taskTags: TagCount[]; + reviewTags: TagCount[]; + }; +} + +async function aggregateTaskTrendsInner( + limit: number, + harness: string, +): Promise { + const perRun = await loadRecentRuns(limit, harness); + return { ...aggregate(perRun), tagCounts: aggregateTaskTagCounts(perRun) }; } const cachedAggregate = unstable_cache( aggregateTaskTrendsInner, - // v2: the cached shape changed from TaskTrend[] to TrendsData — the key - // bump keeps a stale pre-deploy array from being served into new code. - ["aggregate-task-trends-v2"], + // v3: the cached shape gained tagCounts and the args gained `harness` — the + // key bump keeps a stale pre-deploy payload from being served into new code. + // (harness is part of the auto-generated key too, so each harness caches + // independently.) + ["aggregate-task-trends-v3"], { revalidate: 300 }, ); export function aggregateTaskTrends( limit: number = TRENDS_RECENT_RUN_COUNT, -): Promise { - return cachedAggregate(limit); + harness: string = DEFAULT_HARNESS, +): Promise { + return cachedAggregate(limit, harness); } // Predicate matching getOverview's tag scoping logic, but operating on the @@ -238,8 +263,9 @@ export function trendMatchesTag(trend: TaskTrend, tag: string): boolean { export async function historyForTaskInner( taskId: string, limit: number, + harness: string = DEFAULT_HARNESS, ): Promise { - const perRun = await loadRecentRuns(limit); + const perRun = await loadRecentRuns(limit, harness); const out: TaskHistoryEntry[] = []; for (const { id, overview, reviewTagsByTask } of perRun) { if (!overview) continue; @@ -266,13 +292,16 @@ export async function historyForTaskInner( const cachedHistory = unstable_cache( historyForTaskInner, - ["history-for-task"], + // v2: gained the `harness` arg (auto-keyed), so a task's history caches + // per harness rather than serving one harness's rows under another. + ["history-for-task-v2"], { revalidate: 300 }, ); export function historyForTask( taskId: string, limit: number = TRENDS_RECENT_RUN_COUNT, + harness: string = DEFAULT_HARNESS, ): Promise { - return cachedHistory(taskId, limit); + return cachedHistory(taskId, limit, harness); }