fix(review-items): Close only the Review items that a thread reply answers

.github/workflows/evals-classifiers.yml

@@ -0,0 +1,128 @@
+name: Evals
+
+permissions:
+  contents: read
+  pull-requests: read
+  actions: read
+  checks: write
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, reopened, synchronize, labeled]
+
+concurrency:
+  group: evals-classifiers-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  select:
+    name: classifiers / select
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    outputs:
+      should_run: ${{ steps.decision.outputs.should_run }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: getsentry/action-filter-paths@98a158469c63115591d1c2952d34450838ab3bc1 # v0.1.0
+        id: changes
+        with:
+          filters: |
+            relevant:
+              - 'packages/junior-evals/src/roach/**'
+              - 'packages/junior-evals/src/recording-rules.ts'
+              - 'packages/junior-evals/src/recording-run.ts'
+              - 'packages/junior-evals/src/recording-setup.ts'
+              - 'packages/junior-evals/src/proxy-dispatcher.ts'
+              - 'packages/junior-evals/evals/classifiers/**'
+              - 'packages/junior-evals/src/review-item-reply-harness.ts'
+              - 'packages/junior-evals/src/eval-ai-gateway-dispatcher.ts'
+              - 'packages/junior-evals/classifiers-global-setup.ts'
+              - 'packages/junior-evals/vitest.evals.classifiers.config.ts'
+              - 'packages/junior-evals/package.json'
+              - 'packages/junior-evals/scripts/report.mjs'
+              - 'packages/junior-evals/scripts/network-jail.sh'
+              - 'packages/junior/src/chat/pi/client.ts'
+              - 'packages/junior/src/chat/review-items/reply-match.ts'
+      - id: decision
+        env:
+          AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
+          VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }}
+          RELEVANT: ${{ steps.changes.outputs.relevant }}
+          PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }}
+        run: |
+          set -euo pipefail
+          gateway_ready=false
+          requested=false
+          [[ -n "${AI_GATEWAY_API_KEY:-}" || -n "${VERCEL_OIDC_TOKEN:-}" ]] && gateway_ready=true
+          IFS=',' read -r -a labels <<< "${PR_LABELS:-}"
+          for label in "${labels[@]}"; do
+            if [[ "$label" == "trigger-evals" || "$label" == "trigger-evals-classifiers" ]]; then
+              requested=true
+            fi
+          done
+          should_run=false
+          [[ "$gateway_ready" == "true" && ( "$RELEVANT" == "true" || "$requested" == "true" ) ]] && should_run=true
+          echo "should_run=$should_run" >> "$GITHUB_OUTPUT"
+          {
+            echo "## Classifier eval selection"
+            echo
+            echo "- relevant_files_changed: $RELEVANT"
+            echo "- requested: $requested"
+            echo "- gateway_ready: $gateway_ready"
+            echo "- will_run: $should_run"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  classifiers:
+    name: classifiers / run
+    needs: select
+    if: needs.select.outputs.should_run == 'true'
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    env:
+      AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
+      VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-node-pnpm
+      - name: Run classifier evals
+        id: run
+        continue-on-error: true
+        env:
+          VITEST_EVALS_OUTPUT_FILE: classifiers-results.json
+          VITEST_EVALS_REPORT_LEVEL: info
+          # The write token of Roach. With it, the run records new requests.
+          ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }}
+        run: packages/junior-evals/scripts/network-jail.sh pnpm --filter @sentry/junior-evals evals:classifiers
+      - name: Require classifier eval results
+        id: results
+        if: steps.run.conclusion != 'skipped'
+        run: |
+          set -euo pipefail
+          result_file="packages/junior-evals/classifiers-results.json"
+          if [[ ! -f "$result_file" ]]; then
+            echo "::error::missing classifier eval results ($result_file). Treat setup/runtime crashes as hard failures."
+            exit 1
+          fi
+      - name: Publish classifier eval summary
+        if: steps.results.conclusion == 'success'
+        uses: getsentry/[email protected]
+        with:
+          results: packages/junior-evals/classifiers-results.json
+          publish-check: true
+          check-name: classifiers / score
+          fail-on-failures: true
+      - name: Upload classifier eval results
+        if: steps.results.conclusion == 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: classifiers-evals
+          path: packages/junior-evals/classifiers-results.json
+          if-no-files-found: error
+          retention-days: 7
+      - name: Report evals to Sentry Evals
+        if: ${{ !cancelled() && steps.results.outcome == 'success' }}
+        env:
+          SENTRY_EVALS_API_KEY: ${{ secrets.SENTRY_EVALS_API_KEY }}
+        run: node packages/junior-evals/scripts/report.mjs classifiers packages/junior-evals/classifiers-results.json

.gitignore

@@ -20,6 +20,7 @@ packages/junior-evals/behavioral-results*.json
 packages/junior-evals/integration-results*.json
 packages/junior-evals/guardian-results*.json
 packages/junior-evals/router-results*.json
+packages/junior-evals/classifiers-results*.json
 packages/junior-evals/eval-results/
 .env*
 test-results/

AGENTS.md

@@ -24,7 +24,7 @@ Use **pnpm**: `pnpm install`, `pnpm dev`, `pnpm test`, `pnpm typecheck`,
 | One test file          | `pnpm --filter @sentry/junior exec vitest run path/to/file.test.ts`              |
 | Eval harness test file | `pnpm --filter @sentry/junior-evals test path/to/file.test.ts`                   |
 | One eval file or case  | `pnpm --filter @sentry/junior-evals evals:<suite> path/to/x.eval.ts [-t "case"]` |
-| Eval suites            | `behavioral`, `integration`, `guardian`, `router`                                |
+| Eval suites            | `behavioral`, `integration`, `guardian`, `router`, `classifiers`                 |
 | Package schema         | `pnpm --filter <package> db:generate`                                            |
 | Dashboard e2e          | `pnpm test:e2e:dashboard`                                                        |
 | Release alignment      | `pnpm release:check`                                                             |

packages/junior-evals/AGENTS.md

@@ -1,8 +1,8 @@
 # Evals
 
 Evals test agent-facing behavior through the real runtime. `policies/evals.md`
-owns the suite policy (integration, behavioral, guardian, router), the rubric
-rules, and the scenario limits. This file adds the harness rules.
+owns the suites, the rubric rules, and the scenario limits. This file adds the
+harness rules.
 
 ## Layer Boundaries
 
@@ -16,6 +16,8 @@ rules, and the scenario limits. This file adds the harness rules.
   builder (`src/guardian-tools.ts`).
 - `evals/router/**`: `describeEval()` with `routerEvals`. Assert the exact
   model profile and reasoning level.
+- `evals/classifiers/**`: `describeEval()` with one harness per classifier,
+  such as `reviewItemReplyEvals`. Assert the exact choice.
 - Slack HTTP payload contracts and isolated logic belong in
   `packages/junior/tests/`, not here. `pnpm lint` enforces this.
 
@@ -114,5 +116,5 @@ Pass paths, `-t`, and `--shard` directly after the script. Do not use
 
 - Do not bind env vars manually. The eval config loads repo env files.
 - Behavioral and integration need Vercel Sandbox and a Quick Tunnel. Guardian
-  and router need only AI Gateway access.
+  router, and classifiers need only AI Gateway access.
 - CI setup and labels: `evals/github-actions.md`.

packages/junior-evals/classifiers-global-setup.ts

@@ -0,0 +1,26 @@
+import { installEvalAiGatewayDispatcher } from "./src/eval-ai-gateway-dispatcher";
+import { startRecordingRun } from "./src/recording-run";
+
+/**
+ * Set up the lightweight classifier eval invocation.
+ *
+ * Classifier cases only need AI Gateway access through Roach.
+ * They intentionally skip Postgres, Redis fixtures, MSW, plugin catalogs, and
+ * sandbox egress.
+ */
+export default async function setup(
+  project: Parameters<typeof startRecordingRun>[0],
+): Promise<() => Promise<void>> {
+  const restoreAiGatewayDispatcher = installEvalAiGatewayDispatcher();
+  const stopRecordings = await startRecordingRun(project);
+  process.stdout.write(
+    "[evals:classifiers] AI Gateway dispatcher ready (no sandbox egress)\n",
+  );
+  return async () => {
+    try {
+      await stopRecordings();
+    } finally {
+      await restoreAiGatewayDispatcher();
+    }
+  };
+}

packages/junior-evals/evals/classifiers/review-item-replies.eval.ts

@@ -0,0 +1,132 @@
+import { describeEval } from "vitest-evals";
+import { reviewItemReplyEvals } from "../../src/review-item-reply-harness";
+
+function shipItem(version: string) {
+  return {
+    id: `ship-${version}`,
+    title: `Ship release ${version}`,
+    body: "The release checks passed.",
+    request: {
+      question: `Ship release ${version}?`,
+      fields: [
+        {
+          id: "decision",
+          kind: "choice" as const,
+          label: "Decision",
+          options: [{ label: "Ship it" }, { label: "Hold" }],
+        },
+      ],
+    },
+  };
+}
+
+const ENTRY_REVIEW = {
+  id: "acme-entry",
+  title: "Review the Acme entry",
+  body: "Acme uses BUSL with a six-year change date.",
+  request: {
+    question: "Approve or reject the Acme entry?",
+    fields: [
+      {
+        id: "decision",
+        kind: "choice" as const,
+        label: "Decision",
+        options: [{ label: "Approve" }, { label: "Reject" }],
+      },
+      { id: "reason", kind: "text" as const, label: "Reason" },
+    ],
+  },
+};
+
+const ROTATION_START = {
+  id: "rotation-start",
+  title: "Start date for the new on-call rotation",
+  body: "The new rotation adds two people.",
+  request: {
+    question: "Which day should the new on-call rotation start?",
+    fields: [{ id: "date", kind: "text" as const, label: "Start date" }],
+  },
+};
+
+describeEval("Review Item Reply Matching", reviewItemReplyEvals, (it) => {
+  it("when the reply picks an option, it answers the question", async ({
+    run,
+  }) => {
+    await run({
+      items: [shipItem("1.2.5")],
+      reply: "ship it",
+      expected: ["ship-1.2.5"],
+    });
+  });
+
+  it("when the reply only acknowledges, it answers no question", async ({
+    run,
+  }) => {
+    await run({
+      items: [shipItem("1.2.5")],
+      reply: "on it, I'll look after lunch",
+      expected: [],
+    });
+  });
+
+  it("when the reply asks a follow-up, it answers no question", async ({
+    run,
+  }) => {
+    await run({
+      items: [shipItem("1.2.5")],
+      reply: "what failed in the last run?",
+      expected: [],
+    });
+  });
+
+  it("when the reply names one of two similar questions, it answers only that one", async ({
+    run,
+  }) => {
+    await run({
+      items: [shipItem("1.2.5"), shipItem("1.2.6")],
+      reply: "Ship 1.2.6",
+      expected: ["ship-1.2.6"],
+    });
+  });
+
+  it("when the reply decides two questions, it answers both", async ({
+    run,
+  }) => {
+    await run({
+      items: [shipItem("1.2.5"), shipItem("1.2.6")],
+      reply: "Ship 1.2.6, hold 1.2.5 until the flaky test is fixed",
+      expected: ["ship-1.2.5", "ship-1.2.6"],
+    });
+  });
+
+  it("when the reply decides the later of two topics, it answers only that one", async ({
+    run,
+  }) => {
+    await run({
+      items: [shipItem("1.2.5"), ENTRY_REVIEW],
+      reply:
+        "Reject the Acme entry, BUSL with that change date isn't fair source",
+      expected: ["acme-entry"],
+    });
+  });
+
+  it("when the reply gives the requested value, it answers the text question", async ({
+    run,
+  }) => {
+    await run({
+      items: [shipItem("1.2.5"), ROTATION_START],
+      reply: "Start it Monday the 9th",
+      expected: ["rotation-start"],
+    });
+  });
+
+  it("when the reply is about something else, it answers no question", async ({
+    run,
+  }) => {
+    await run({
+      items: [shipItem("1.2.5"), ENTRY_REVIEW],
+      reply: "btw the build on main is red again",
+      expected: [],
+    });
+  });
+});

packages/junior-evals/evals/github-actions.md

@@ -27,7 +27,7 @@ is required. A missing key skips uploads, including on fork pull requests.
 
 Each suite uploads one run after execution. Behavioral and integration combine
 all shards. Dataset names are `junior-behavioral`, `junior-integration`,
-`junior-guardian`, and `junior-router`. The job summary links to the run.
+`junior-guardian`, `junior-router`, and `junior-classifiers`. The job summary links to the run.
 Existing score gates and artifacts stay in place, including when tests fail.
 
 `scripts/report.mjs` maps Vitest results to scores, errors, transcripts, duration,
@@ -99,8 +99,9 @@ Four independent workflows run on pull requests. All use the display name `Evals
 - `evals-integration.yml` runs system evals when integration eval files/harness changed or the PR has `trigger-evals-integration` / `trigger-evals`
 - `evals-guardian.yml` runs isolated Guardian snapshots when Guardian eval files/harness changed or the PR has `trigger-evals-guardian` / `trigger-evals`
 - `evals-router.yml` runs isolated turn route snapshots when Router eval files/harness changed or the PR has `trigger-evals-router` / `trigger-evals`
+- `evals-classifiers.yml` runs isolated classifier snapshots when classifier eval files/harnesses or classifier source changed or the PR has `trigger-evals-classifiers` / `trigger-evals`
 
-Suite labels follow `trigger-evals-[domain]`. Adding a trigger label fires immediately. If the label is already on the PR, future `synchronize` events still run the matching suite(s). Product source under `packages/junior/src/**` does not auto-run evals, except Guardian policy and turn router changes.
+Suite labels follow `trigger-evals-[domain]`. Adding a trigger label fires immediately. If the label is already on the PR, future `synchronize` events still run the matching suite(s). Product source under `packages/junior/src/**` does not auto-run evals, except Guardian policy, turn router, and classifier changes.
 
 Guardian and Router evals only need gateway credentials. Behavioral and integration evals still need gateway plus sandbox access.
 

packages/junior-evals/package.json

@@ -11,6 +11,7 @@
     "evals:integration": "vitest run -c vitest.evals.integration.config.ts",
     "evals:guardian": "vitest run -c vitest.evals.guardian.config.ts",
     "evals:router": "vitest run -c vitest.evals.router.config.ts",
+    "evals:classifiers": "vitest run -c vitest.evals.classifiers.config.ts",
     "evals:record": "VITEST_EVALS_REPLAY_MODE=record vitest run -c vitest.evals.behavioral.config.ts"
   },
   "devDependencies": {

packages/junior-evals/scripts/report.mjs

@@ -4,7 +4,13 @@ import { pathToFileURL } from "node:url";
 // Upload completed Vitest artifacts, not live runtime state. Each suite has one
 // run across all shards. Keep CI gates and artifact storage independent.
 const API_URL = "https://evals.sentry.dev";
-const SUITES = ["behavioral", "integration", "guardian", "router"];
+const SUITES = [
+  "behavioral",
+  "integration",
+  "guardian",
+  "router",
+  "classifiers",
+];
 
 // Fixture evals are in `src/fixture/`, so an eval file can be outside `evals/`.
 function scenarioFile(filename) {
@@ -168,7 +174,7 @@ async function post(endpoint, body, apiKey) {
 export async function reportEvals(suite, files, env = process.env) {
   if (!SUITES.includes(suite) || files.length === 0) {
     throw new Error(
-      "Usage: report.mjs <behavioral|integration|guardian|router> <results.json> [...]",
+      "Usage: report.mjs <behavioral|integration|guardian|router|classifiers> <results.json> [...]",
     );
   }
   const apiKey = env.SENTRY_EVALS_API_KEY?.trim();