From 861aeebe66dccd9b4837225301574dabe8c3fbde Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sat, 7 Nov 2026 01:54:16 +0000 Subject: [PATCH] feat(cli): list findings since a deploy telemetry findings lists the alerts that opened and the checks that failed in a project's window, newest first, through the MCP server's list_findings tool. It exits with 3 when the window has a finding, so a CI step or a coding agent can check a deploy with --since. A deployment without the tool fails with server_unsupported. --- packages/cli/README.md | 24 +++++-- packages/cli/src/cli.ts | 28 ++++++++ packages/cli/src/commands.ts | 118 +++++++++++++++++++++++++++++++++ packages/cli/src/errors.ts | 2 + packages/cli/src/tools.ts | 24 +++++++ packages/cli/tests/cli.test.ts | 60 +++++++++++++++++ packages/cli/tests/fixture.ts | 71 +++++++++++++++++++- 7 files changed, 320 insertions(+), 7 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index 9bc83a67..cc2ba4a2 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -62,19 +62,30 @@ telemetry sessions --range 7d --errors # conversations with a failed trace telemetry logs --level error telemetry usage --range 30d --by user telemetry payload --field output +telemetry findings --since 2h # alerts and failed checks; exits 3 when there are any ``` Run `telemetry --help` to see each command's options and examples. Options go after the command: `telemetry traces --json`, not `telemetry --json traces`. -In human output, `telemetry trace` shows a summary above the span tree. It lists up to 5 of the failed spans where failures began, with their path. It also lists up to 5 of the operations that ran more than once, with their wall time, which counts overlapping calls once. In the span tree, each failed span has its id and error under it, for `telemetry payload`. When they list results, `traces` and `sessions` end with a likely next command on stderr. `trace` ends with the `payload` command that reads the full error, when a span where a failure began has an error message and the sign-in can read payloads. +In human output, `telemetry trace` shows a summary above the span tree. It lists up to 5 of the failed spans where failures began, with their path. It also lists up to 5 of the operations that ran more than once, with their wall time, which counts overlapping calls once. In the span tree, each failed span has its id and error under it, for `telemetry payload`. When they list results, `traces`, `sessions`, and `findings` end with a likely next command on stderr. `trace` ends with the `payload` command that reads the full error, when a span where a failure began has an error message and the sign-in can read payloads. A command reads its project from `--project `. Without that option, it uses `TELEMETRY_DEV_PROJECT`, then the default that you set with `telemetry use`. If the workspace has only one project, the command uses it. List commands return 20 results by default. Use `--limit` to return from 1 to 50 results. When more results are available, the command prints the same command with `--cursor` for the next page. On Windows, when an option holds text that no quoting keeps as typed, such as `%` or `$`, it prints only the `--cursor` to add. Keep the other options as they are when you pass `--cursor`. Only `--limit` may change between pages. +### Findings + +`telemetry findings` lists what telemetry.dev found wrong in the project, newest first: alerts that opened, such as a runaway trace or a tool failure cluster, and failed checks, which are fail scores from an automatic pass/fail evaluator such as an LLM judge. Each finding has a link to its alert page, or to the trace at the span that was checked. Without `--since`, it lists the last 7 days. `--kind alert` or `--kind check` lists one kind. + +The command exits with 3 when the window has at least one finding, also with `--json`, and with 0 when it has none. To check a deploy from CI or a coding agent, pass the time of the deploy: + +```sh +telemetry findings --since 2026-11-06T14:00:00Z --json +``` + ### Time windows -`traces`, `sessions`, and `logs` take `--since` and `--until` to choose the time they search: +`traces`, `sessions`, `logs`, and `findings` take `--since` and `--until` to choose the time they search: ```sh telemetry traces --errors --since 2h @@ -102,15 +113,15 @@ A duration has up to five digits. The CLI sends a date or a time without a zone Add `--json` to any command for machine-readable output on stdout. Messages and progress go to stderr, so `telemetry traces --json > traces.json` writes only data. List commands print the MCP tool's result as returned, with the fields that the [MCP server documentation](https://telemetry.dev/docs/platform/mcp-server) describes. -The results of `traces`, `sessions`, and `logs` include the window that telemetry.dev searched, with both ends resolved to ISO times. `capped_by_retention` is true when the window starts at the start of the project's retention, so older data wasn't searched. That is also the case for `traces` without `--since`, which searches the whole retention: +The results of `traces`, `sessions`, `logs`, and `findings` include the window that telemetry.dev searched, with both ends resolved to ISO times. `capped_by_retention` is true when the window starts at the start of the project's retention, so older data wasn't searched. That is also the case for `traces` without `--since`, which searches the whole retention: ```json "window": { "since": "2026-10-30T12:00:00.000Z", "until": "2026-10-30T14:00:00.000Z", "capped_by_retention": false } ``` -`logs` results also have `next_cursor`, which is `null` on the last page, and `range_days` is `null` when the command used `--since` or `--until`. +`logs` and `findings` results also have `next_cursor`, which is `null` on the last page. In `logs` results, `range_days` is `null` when the command used `--since` or `--until`. -An older telemetry.dev deployment may ignore `--since`, `--until`, the `traces` filters, and `logs --cursor` and return unfiltered results, or the first page of logs again. So when the CLI sends one of them and the result has no `window`, or no `next_cursor` for `logs --cursor`, the command fails with `server_unsupported` and prints no results. A deployment that refuses `--agent` or `--tool` because it doesn't know them also fails with `server_unsupported`; one that refuses another argument fails with `tool_error`. +An older telemetry.dev deployment may ignore `--since`, `--until`, the `traces` filters, and `logs --cursor` and return unfiltered results, or the first page of logs again. So when the CLI sends one of them and the result has no `window`, or no `next_cursor` for `logs --cursor`, the command fails with `server_unsupported` and prints no results. A deployment that refuses `--agent` or `--tool` because it doesn't know them also fails with `server_unsupported`; one that refuses another argument fails with `tool_error`. `findings` on a deployment from before findings also fails with `server_unsupported`. `telemetry trace` and `telemetry payload` read every page of a trace or a captured text, up to a limit: 2,000 spans for a trace, and 1,000,000 characters for each payload field. Their JSON output has `complete: false` when the limit cut it short, and human output says so on stderr. @@ -121,6 +132,7 @@ Captured text comes from your users, models, and tools. telemetry.dev removes AN | 0 | Success | | 1 | The request failed | | 2 | The command or its options were not valid | +| 3 | `findings` found at least one finding in its window | | 4 | You are not signed in, the sign-in expired, or access is denied | | 130 | Interrupted with Ctrl-C | @@ -154,7 +166,7 @@ Branch on `code`, not on `message`, whose wording can change. `hint` is a comman | `server_error` | 1 | telemetry.dev failed to complete the request; try again | | `tool_error` | 1 | telemetry.dev returned another error; `message` says what it was | | `result_too_large` | 1 | The result was too large to read; narrow it with filters or `--limit` | -| `server_unsupported` | 1 | The deployment doesn't support an option used and would have ignored it | +| `server_unsupported` | 1 | The deployment doesn't support the command, or an option it would have ignored | | `request_failed` | 1 | The request failed for another reason | | `failed` | 1 | The command failed for another reason | | `internal` | 1 | An unexpected error, such as a configuration file the CLI couldn't read or write | diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index d756bf6c..04c88038 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -4,6 +4,7 @@ import { parseArgs } from "node:util"; import { VERSION } from "./api.ts"; import { type Context, + findingsCommand, home, jsonText, loginCommand, @@ -234,6 +235,33 @@ const COMMANDS: Readonly> = { ], run: logsCommand, }, + findings: { + summary: "List alerts and failed checks; exit 3 when there are any", + usage: `${BIN} findings [options]`, + options: { + since: { + ...since, + description: + "Start of the window, such as the time of a deploy: 30m, 2h, or 7d ago, a date, or a time (default 7d)", + }, + until, + kind: { + type: "string", + value: "alert|check", + description: "Only alerts, or only failed checks", + }, + limit, + cursor, + project, + json, + }, + examples: [ + `${BIN} findings --since 2h`, + `${BIN} findings --since 2026-11-06T14:00:00Z # since a deploy, in CI or an agent's shell`, + `${BIN} findings --kind check --json`, + ], + run: findingsCommand, + }, usage: { summary: "Summarize requests, errors, latency, tokens, and cost", usage: `${BIN} usage [options]`, diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index d221bf97..c475d0a6 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -37,6 +37,7 @@ import { } from "./terminal.ts"; import { cachedLatestVersion } from "./update.ts"; import type { + FindingPage, LogList, PayloadPage, ProjectList, @@ -1586,6 +1587,123 @@ export async function logsCommand(context: Context) { }); } +const KINDS = { alert: "alert", check: "check" } as const; + +/** + * Lists the window's findings, newest first. Any finding in the window sets exit code 3, so a CI + * step or a coding agent can run it after a deploy and act on what it found. + */ +export async function findingsCommand(context: Context) { + const window = timeWindow(context); + const kind = choice(context, "kind", KINDS); + + await withApi(async (api) => { + const project = await resolveProject(api, context); + + const result = await api + .call( + "list_findings", + { + project_id: project.id, + kind, + ...window.args, + limit: limit(context), + cursor: string(context, "cursor"), + }, + "Loading findings", + ) + .catch((error: unknown) => { + // A deployment from before findings answers with its MCP SDK's protocol error, which has + // no telemetry.dev code. + if (error instanceof CliError && /Tool list_findings not found/.test(error.message)) { + throw new CliError( + `This telemetry.dev deployment doesn't have findings yet. Use ${BIN} traces --errors instead.`, + { code: "server_unsupported" }, + ); + } + + throw error; + }); + + const { counts } = result; + const total = counts.alerts + counts.checks; + + if (total > 0) process.exitCode = EXIT.findings; + + if (json(context)) return printJson(result); + + const count = (n: number, one: string, many: string) => + `${formatInteger(n)} ${n === 1 ? one : many}`; + + // The server counts 0 for the kind that --kind leaves out, so only the asked kinds show. + const byKind = [ + kind !== "check" && + `${count(counts.alerts, "alert", "alerts")} (${formatInteger(counts.open_alerts)} open)`, + kind !== "alert" && count(counts.checks, "failed check", "failed checks"), + ].filter((part) => part !== false); + + out( + style.dim( + [`${count(total, "finding", "findings")} ${windowText(result.window)}`, ...byKind].join( + " · ", + ), + ), + ); + + if (result.findings.length === 0) { + nothing("Nothing found wrong in this window."); + + return noteCapped(context, result.window); + } + + const lines = table( + [ + { header: "WHEN" }, + { header: "KIND" }, + { header: "STATUS" }, + { header: "FINDING", flex: true }, + ], + result.findings.map((finding) => [ + { text: formatTimestamp(finding.at), paint: dim }, + oneLine(finding.kind), + { + text: oneLine(finding.status), + paint: finding.status === "resolved" ? style.green : style.red, + }, + oneLine(finding.summary ? `${finding.title}: ${finding.summary}` : finding.title), + ]), + ).split("\n"); + + // A link goes on its own line under its finding, outside the columns, so it never takes the + // finding's width. Line 0 is the header, so row i is line i + 1. + out(); + out( + lines + .flatMap((line, index) => { + const link = index === 0 ? undefined : dashboardLink(result.findings[index - 1]?.url); + + return link === undefined ? [line] : [line, style.dim(` ${link}`)]; + }) + .join("\n"), + ); + + noteCapped(context, result.window); + + const check = result.findings.find((finding) => finding.trace_id !== null); + + suggest( + context, + check?.trace_id + ? labelled( + "Inspect the newest failed check", + suggestion(context, "trace", [check.trace_id]), + ) + : undefined, + nextPage(context, result.next_cursor), + ); + }); +} + /** The change from the previous range; red or green only where up is worse. */ const change = (current: number, previous: number | undefined, worseWhenUp?: boolean) => { if (previous === undefined) return ""; diff --git a/packages/cli/src/errors.ts b/packages/cli/src/errors.ts index d370c619..2609499a 100644 --- a/packages/cli/src/errors.ts +++ b/packages/cli/src/errors.ts @@ -5,6 +5,8 @@ export const EXIT = { ok: 0, failure: 1, usage: 2, + /** `findings` found at least one finding in its window. */ + findings: 3, auth: 4, interrupted: 130, } as const; diff --git a/packages/cli/src/tools.ts b/packages/cli/src/tools.ts index 3cfbc834..65a781ef 100644 --- a/packages/cli/src/tools.ts +++ b/packages/cli/src/tools.ts @@ -161,3 +161,27 @@ export type UsageSummary = { }[]; breakdown_limit: number; }; + +export type Finding = { + kind: "alert" | "check"; + id: string; + /** When the alert opened or the check failed. */ + at: string; + status: "open" | "resolved" | "failed"; + resolved_at: string | null; + /** The alert rule's kind, or the evaluator's name. */ + source: string; + title: string; + summary: string | null; + /** The alert's page, or the trace at the span that was checked. */ + url: string; + trace_id: string | null; + span_id: string | null; +}; + +export type FindingPage = { + window: Window; + counts: { alerts: number; open_alerts: number; checks: number }; + findings: Finding[]; + next_cursor: string | null; +}; diff --git a/packages/cli/tests/cli.test.ts b/packages/cli/tests/cli.test.ts index f74607f8..54381bf8 100644 --- a/packages/cli/tests/cli.test.ts +++ b/packages/cli/tests/cli.test.ts @@ -26,6 +26,7 @@ import { beforeTool, calls, EMAIL, + findings, logs, PARALLEL_TRACE_ID, PAYLOADS, @@ -92,6 +93,7 @@ beforeEach(async () => { windows: true, agentAndTool: true, empty: false, + findings: true, }); return () => rm(home, { recursive: true, force: true }); @@ -146,6 +148,12 @@ test("human output never passes terminal escapes from captured text, and --json expect(result.stdout + result.stderr).not.toMatch(CONTROL); } + // Alert summaries quote tool and model names that the application chose. + const found = await run("findings"); + expect(found.code).toBe(3); + expect(found.stdout).toContain("lookup_order failed on 40% of calls."); + expect(found.stdout).not.toMatch(CONTROL); + const json = await run("traces", "--json"); expect(json.code).toBe(0); expect(json.stdout).not.toContain("\u001b"); @@ -1816,3 +1824,55 @@ test("the sign-in lock is taken over at once, without lost writes, when its hold expect(await readdir(home)).toEqual([sessionFile(fixture.url)]); }); }); + +test("findings lists what went wrong since a time and exits 3, or 0 when nothing did", async () => { + await signIn(PROJECTS[0]); + + const human = await run("findings", "--since", "2026-11-06T14:00:00Z"); + expect(human.code).toBe(3); + expect(human.stdout).toMatch(/^2 findings from .* · 1 alert \(1 open\) · 1 failed check\n/); + expect(human.stdout).toContain( + "lookup_order fails more often: lookup_order failed on 40% of calls.", + ); + expect(human.stdout).toContain(`/traces/${TRACE_ID}?span=f6`); + expect(human.stderr).toContain(`Inspect the newest failed check: telemetry trace ${TRACE_ID}`); + expect(calls.at(-1)).toMatchObject({ + tool: "list_findings", + args: { project_id: PROJECTS[0]!.id, since: "2026-11-06T14:00:00Z" }, + }); + + // JSON is the tool's result, and the exit code still says something was found. + // The counts of the kind that --kind leaves out are not data, so they don't show. + const checks = await run("findings", "--since", "2h", "--kind", "check"); + expect(checks.stdout).toMatch(/^1 finding from [^·]* · 1 failed check\n/); + + const paged = await run("findings", "--limit", "1"); + expect(paged.code).toBe(3); + expect(paged.stderr).toContain("More: telemetry findings --limit 1 --cursor=page-2"); + + const json = await run("findings", "--since", "2h", "--kind", "check", "--json"); + expect(json.code).toBe(3); + expect(JSON.parse(json.stdout)).toMatchObject({ + findings: findings() + .findings.filter((finding) => finding.kind === "check") + .map(({ id }) => ({ id })), + }); + expect(calls.at(-1)?.args).toMatchObject({ kind: "check", since: "2h" }); + + serverState.empty = true; + const quiet = await run("findings", "--since", "2h"); + expect(quiet).toMatchObject({ code: 0 }); + expect(quiet.stderr).toContain("Nothing found wrong in this window."); + + const bad = await run("findings", "--kind", "alerts"); + expect(bad.code).toBe(2); +}); + +test("findings against a deployment from before findings says so instead of a raw tool error", async () => { + await signIn(PROJECTS[0]); + serverState.findings = false; + + const json = await run("findings", "--json"); + expect(json).toMatchObject({ code: 1, stdout: "" }); + expect(JSON.parse(json.stderr).error).toMatchObject({ code: "server_unsupported" }); +}); diff --git a/packages/cli/tests/fixture.ts b/packages/cli/tests/fixture.ts index 421a4eb8..660d5bb5 100644 --- a/packages/cli/tests/fixture.ts +++ b/packages/cli/tests/fixture.ts @@ -369,6 +369,56 @@ export const logs = () => ({ ], }); +/** The findings of `kind`, as the server counts and lists them; `paged` returns the first of two pages. */ +const ofKind = (kind: unknown, paged: boolean, window: unknown) => { + const all = findings(); + const listed = all.findings.filter((finding) => kind === undefined || finding.kind === kind); + + return { + window, + counts: { + alerts: kind === "check" ? 0 : all.counts.alerts, + open_alerts: kind === "check" ? 0 : all.counts.open_alerts, + checks: kind === "alert" ? 0 : all.counts.checks, + }, + findings: paged ? listed.slice(0, 1) : listed, + next_cursor: paged ? "page-2" : null, + }; +}; + +export const findings = () => ({ + counts: { alerts: 1, open_alerts: 1, checks: 1 }, + findings: [ + { + kind: "check", + id: "5d0f7a1c-2b3e-4f50-8a6b-7c8d9e0f1a2b", + at: minutesAgo(12), + status: "failed", + resolved_at: null, + source: "Claims vs tool results", + title: "Claims vs tool results", + summary: "The reply says the refund went through; no tool call made one.", + url: `https://telemetry.dev/o/org_1/p/proj_1/traces/${TRACE_ID}?span=f6`, + trace_id: TRACE_ID, + span_id: "f6", + }, + { + kind: "alert", + id: "8e1a2b3c-4d5e-4f60-9a7b-8c9d0e1f2a3b", + at: minutesAgo(40), + status: "open", + resolved_at: null, + source: "tool_failure_cluster", + title: "lookup_order fails more often", + summary: "lookup_order failed on 40% of calls.\u001b]52;c;ZXZpbA==\u0007\u001b[2J", + url: "https://telemetry.dev/o/org_1/p/proj_1/alerts/8e1a2b3c-4d5e-4f60-9a7b-8c9d0e1f2a3b", + trace_id: null, + span_id: null, + }, + ], + next_cursor: null, +}); + const totals = (scale: number) => ({ totalRequests: Math.round(12480 * scale), errorCount: Math.round(132 * scale * 1.3), @@ -653,6 +703,23 @@ const server = () => { logs: matching.slice(offset, end), }; }); + if (serverState.findings) { + tool("list_findings", (args) => { + const window = resolveWindow(args, 7 * UNIT_MS.d); + + if (window instanceof ToolFailure) return window; + + return serverState.empty + ? { + window, + counts: { alerts: 0, open_alerts: 0, checks: 0 }, + findings: [], + next_cursor: null, + } + : ofKind(args.kind, args.limit === 1, window); + }); + } + tool("get_usage_summary", () => usage()); tool("get_span_payload", (args) => { // Offsets and counts are in code points, as the server counts them. @@ -694,8 +761,10 @@ export const serverState = { windows: true, /** search_traces takes agent and tool; false acts as a server with windows from before them. */ agentAndTool: true, - /** search_traces and search_sessions find nothing, as in a project nothing was sent to yet. */ + /** search_traces, search_sessions, and list_findings find nothing, as in a project nothing was sent to yet. */ empty: false, + /** The server has list_findings; false acts as a server from before findings. */ + findings: true, }; /**