diff --git a/packages/ui/API.md b/packages/ui/API.md index 5ecb190..2ab40cd 100644 --- a/packages/ui/API.md +++ b/packages/ui/API.md @@ -158,11 +158,15 @@ All return `ComparisonResponse`: `columns` (each with the runs it covers), `rows | `GET /api/matrix?workspace=&project=&experiment=` | `MatrixResponse` | Every run in scope (filters as in `/api/runs`; none = all runs; an unknown value is an empty matrix). | A cell is one eval entry × run-matrix row × eval-matrix row (from the task id). Every run of a cell has a `coverage`: -`full` (every expected case settled and at least one has a result; errored cases count as settled), `partial` (some -expected cases never ran or never settled), or `none` (no case has a result). `latest` is the task of the newest run -(by start time) that covered the cell fully or is still running it, else of the newest run: run, experiment, attempt, -task state, coverage, expected case total (`caseTotal`), case counts, exact and judge score. A partial re-run of a few -cases therefore never replaces a full run as the cell's score. `history` lists every run with that cell, oldest first +`full` (every expected case settled and at least one has a result; errored cases count as settled, over the same cases +as the headline run), `other` (every expected case settled, but over other cases than the headline run, told apart by +`identity`: a smoke or limited run, or another case set under the same entry), `partial` (some expected cases never ran or never settled), or `none` (no case has a result). `latest` is the task +of the newest run (by start time) that covered the cell fully or is still running it, else of the newest run: run, +experiment, attempt, task state, coverage, expected case total (`caseTotal`), case counts, the exact score of every case +(`caseScores`), exact and judge score. The headline run is the newest one that settled every case it planned over cases no other run covers a strict +superset of; `caseCount` is its number of cases. A partial re-run or a run over other cases therefore never replaces a full run as the +cell's score. `superseded` is `true` when a newer run ran the same entry and run row with other eval values (the eval +axes changed since). `history` lists every run with that cell, oldest first (one point per run: attempt, coverage, case counts, exact, state, start time). Two tasks of one run in the same cell (several inference executors) keep the last one in queue order. `axes` lists the run-matrix axes, then the eval-matrix axes, each with its values in natural order. Cells are ordered by entry, then run row and eval diff --git a/packages/ui/client/src/caseMetrics.test.ts b/packages/ui/client/src/caseMetrics.test.ts index cacb462..f5e99e8 100644 --- a/packages/ui/client/src/caseMetrics.test.ts +++ b/packages/ui/client/src/caseMetrics.test.ts @@ -12,6 +12,7 @@ function caseNode(index: number, name: string, state: CaseNode['state']): CaseNo // Durations that do not follow the metrics below, so they stay their own signal. durationMs: [5, 1, 4, 2][index]! * 1000, evidenceLinkCount: 0, + identity: caseId, index, key: `task\u0000${caseId}`, logLineCount: 0, diff --git a/packages/ui/client/src/components/ErrorGroupsPanel.test.ts b/packages/ui/client/src/components/ErrorGroupsPanel.test.ts index ab639e6..12dbc46 100644 --- a/packages/ui/client/src/components/ErrorGroupsPanel.test.ts +++ b/packages/ui/client/src/components/ErrorGroupsPanel.test.ts @@ -6,7 +6,7 @@ import { render } from 'vitest-browser-vue' import ErrorGroupsPanel from './ErrorGroupsPanel.vue' function caseOf(caseId: string, state: CaseNode['state'], errorMessage?: string): CaseNode { - return { artifactCount: 0, caseId, errorMessage, errorName: errorMessage == null ? undefined : 'APICallError', evidenceLinkCount: 0, index: 0, key: `task\u0000${caseId}`, logLineCount: 0, metricCount: 0, name: caseId, rounds: [], scores: {}, startedAt: '2026-10-19T06:36:26.000Z', state, taskId: 'task' } + return { artifactCount: 0, caseId, errorMessage, errorName: errorMessage == null ? undefined : 'APICallError', evidenceLinkCount: 0, identity: caseId, index: 0, key: `task\u0000${caseId}`, logLineCount: 0, metricCount: 0, name: caseId, rounds: [], scores: {}, startedAt: '2026-10-19T06:36:26.000Z', state, taskId: 'task' } } describe('errorGroupsPanel', () => { diff --git a/packages/ui/client/src/components/MatrixView.vue b/packages/ui/client/src/components/MatrixView.vue index f26f2e6..0c46400 100644 --- a/packages/ui/client/src/components/MatrixView.vue +++ b/packages/ui/client/src/components/MatrixView.vue @@ -2,8 +2,9 @@ import type { MatrixCell, MatrixResponse } from '../../../src/shared/contract' import { useQuery } from '@pinia/colada' -import { computed, watch } from 'vue' +import { computed, shallowRef, watch } from 'vue' +import ScoreBar from './ScoreBar.vue' import ScoreCard from './ScoreCard.vue' import Sparkline from './Sparkline.vue' import StateView from './StateView.vue' @@ -18,6 +19,9 @@ import { href } from '../router' * Matrix of a project or experiment (`GET /api/matrix`): one grid per eval entry, rows along the run axis with the * most values (for example `model`), columns for the combinations of the other axes (for example `repeat`). A cell * is the newest run that covered that combination fully (see `MatrixCell.latest`), with the trend of every run. + * + * Columns of a configuration the project moved away from (every cell `superseded`, for example an eval axis dropped + * since) are folded until asked for, so the table opens on what the newest runs measure. */ const props = defineProps<{ experiment?: string, project: string, workspace: string }>() @@ -45,12 +49,14 @@ function runOf(cell: MatrixCell) { const axisName = (key: string) => key.replace(/^(?:run|eval)\./, '') interface Grid { - columns: { key: string, label: string }[] + columns: { key: string, label: string, superseded: boolean }[] entryId: string name: string rowAxis: string | undefined /** One row per value of `rowAxis`; `cells` line up with `columns` (`undefined`: combination never ran). */ rows: { cells: (MatrixCell | undefined)[], key: string, label: string }[] + /** Axis values every column has (for example `listener gpt-6-luna`), said once above the table instead of per column. */ + shared: string } function buildGrid(entryId: string, cells: MatrixCell[], response: MatrixResponse): Grid { @@ -68,8 +74,23 @@ function buildGrid(entryId: string, cells: MatrixCell[], response: MatrixRespons } const columnKey = (cell: MatrixCell) => columnAxes.map(key => `${axisName(key)} ${coordinates(cell)[key] ?? '—'}`).join(' · ') const rowKey = (cell: MatrixCell) => (rowAxis == null ? '' : coordinates(cell)[rowAxis] ?? '—') + // With several columns, an axis with one value in every cell says nothing about a column; it moves above the table. + // The other axes name a column by the values it has; an axis a column lacks (it ran before the axis was added, or + // after it was dropped) is left out instead of shown as `asr —`. + const multiColumn = new Set(cells.map(columnKey)).size > 1 + const sharedAxes = multiColumn ? columnAxes.filter(key => cells.every(cell => coordinates(cell)[key] != null && coordinates(cell)[key] === coordinates(cells[0]!)[key])) : [] + const varyingAxes = columnAxes.filter(key => !sharedAxes.includes(key)) + const columnLabel = (cell: MatrixCell) => { + const present = varyingAxes.filter(key => coordinates(cell)[key] != null) + if (present.length > 0) + return present.map(key => `${axisName(key)} ${coordinates(cell)[key]}`).join(' · ') + return varyingAxes.length > 0 ? `without ${varyingAxes.map(axisName).join(', ')}` : 'all' + } - const columns = [...new Set(cells.map(columnKey))].map(key => ({ key, label: key === '' ? 'all' : key })) + const columns = [...new Set(cells.map(columnKey))].map((key) => { + const columnCells = cells.filter(cell => columnKey(cell) === key) + return { key, label: columnLabel(columnCells[0]!), superseded: columnCells.every(cell => cell.superseded) } + }) // Column order follows the axis order of the API (numeric for `repeat`), not the order cells arrived in. columns.sort((a, b) => { const cellA = cells.find(cell => columnKey(cell) === a.key)! @@ -88,7 +109,14 @@ function buildGrid(entryId: string, cells: MatrixCell[], response: MatrixRespons key: value, label: rowAxis == null ? 'all' : value, })) - return { columns, entryId, name: shortTaskId(entryId).split('/').at(-1) ?? entryId, rowAxis: rowAxis == null ? undefined : axisName(rowAxis), rows } + return { + columns, + entryId, + name: shortTaskId(entryId).split('/').at(-1) ?? entryId, + rowAxis: rowAxis == null ? undefined : axisName(rowAxis), + rows, + shared: sharedAxes.map(key => `${axisName(key)} ${coordinates(cells[0]!)[key]}`).join(' · '), + } } const grids = computed(() => { @@ -105,6 +133,25 @@ const grids = computed(() => { } return [...byEntry.entries()].map(([entryId, cells]) => buildGrid(entryId, cells, response)) }) + +const showSuperseded = shallowRef(false) +/** Labels of the folded columns across the grids (one label per configuration, however many entries ran it). */ +const supersededLabels = computed(() => new Set(grids.value.flatMap(grid => grid.columns.filter(column => column.superseded).map(column => column.label)))) +const visibleGrids = computed(() => grids.value.map((grid) => { + if (showSuperseded.value) + return grid + const kept = grid.columns.flatMap((column, index) => (column.superseded ? [] : [index])) + return { ...grid, columns: kept.map(index => grid.columns[index]!), rows: grid.rows.map(row => ({ ...row, cells: kept.map(index => row.cells[index]) })) } +})) + +/** + * Whether a cell is read by its score: every case of its run got a result and none failed an assertion, so the case + * counts are all green and say nothing; how the scores spread does. + */ +function isScored(cell: MatrixCell): boolean { + const { counts, exact, state } = cell.latest + return exact != null && counts.failed === 0 && state !== 'running' && state !== 'queued' +} /** When a point of a cell's history started: its run's start time, else the attempt label. */ function pointTime(point: MatrixCell['history'][number]): string { return point.startedAt == null ? attemptLabel(point.attemptId) : formatShortDateTime(new Date(point.startedAt)) @@ -122,25 +169,30 @@ function scoreDelta(cell: MatrixCell): number | undefined { return previous == null ? undefined : (cell.latest.exact - previous.exact!) * 100 } -const COVERAGE_TEXT = { full: 'every case', none: 'no result', partial: 'some cases' } as const +const COVERAGE_TEXT = { full: 'every case', none: 'no result', other: 'other cases', partial: 'some cases' } as const /** Tooltip of one history point: when, how much it measured, and its score. */ -function pointTitle(point: MatrixCell['history'][number]): string { +function pointTitle(cell: MatrixCell, point: MatrixCell['history'][number]): string { const measured = point.counts.passed + point.counts.failed - return `${pointTime(point)} · ${COVERAGE_TEXT[point.coverage]} (${measured} with a result) · exact ${formatScore(point.exact)}` + return `${pointTime(point)} · ${COVERAGE_TEXT[point.coverage]} (${measured} of ${cell.caseCount} cases with a result) · exact ${formatScore(point.exact)}` } + - {{ cell.latest.startedAt ? formatShortDateTime(new Date(cell.latest.startedAt)) : attemptLabel(cell.latest.attemptId) }} · {{ countOf(cell.history.length, 'run') }} + {{ cell.latest.startedAt ? formatShortDateTime(new Date(cell.latest.startedAt)) : attemptLabel(cell.latest.attemptId) }} · {{ countOf(cell.latest.counts.total, 'case') }} · {{ countOf(cell.history.length, 'run') }} -