From 9e0491046c196402ed408411dba7cc7aac41fba4 Mon Sep 17 00:00:00 2001
From: Sergiy Dybskiy <sergiy.dybskiy@sentry.io>
Date: Mon, 5 Oct 2026 23:11:52 -0400
Subject: [PATCH] Return a test summary from run_tests instead of the raw log

Co-authored-by: Claude <claude@anthropic.com>
---
 apps/pr-reviewer/src/agents/review.ts | 32 ++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/apps/pr-reviewer/src/agents/review.ts b/apps/pr-reviewer/src/agents/review.ts
index c7fd4b8..96ab97f 100644
--- a/apps/pr-reviewer/src/agents/review.ts
+++ b/apps/pr-reviewer/src/agents/review.ts
@@ -79,18 +79,34 @@ export function ReviewLead() {
 		description: 'Run the demo-pr test suite and return its output',
 		input: v.object({}),
 		async run({ log }) {
-			let output: string;
+			const reportPath = '/tmp/vitest.json';
 			let exitCode = 0;
 			try {
-				const { stdout, stderr } = await execFileAsync('npm', ['test'], {
-					cwd: 'fixtures/demo-pr',
-					maxBuffer: 64 * 1024 * 1024,
-				});
-				output = `${stdout}\n${stderr}`;
+				await execFileAsync(
+					'npx',
+					['vitest', 'run', '--reporter=json', `--outputFile=${reportPath}`],
+					{ cwd: 'fixtures/demo-pr', maxBuffer: 64 * 1024 * 1024 },
+				);
 			} catch (error) {
-				const failure = error as { code?: number; stdout?: string; stderr?: string };
+				const failure = error as { code?: number };
 				exitCode = typeof failure.code === 'number' ? failure.code : 1;
-				output = `Tests failed (exit ${exitCode})\n${failure.stdout ?? ''}\n${failure.stderr ?? ''}`;
+			}
+			let output: string;
+			try {
+				const report = JSON.parse(await readFile(reportPath, 'utf8'));
+				const failures = report.testResults.flatMap(
+					(file: { name: string; assertionResults: { status: string; fullName: string }[] }) =>
+						file.assertionResults
+							.filter((test) => test.status === 'failed')
+							.map((test) => `${path.basename(file.name)}: ${test.fullName}`),
+				);
+				output = JSON.stringify({
+					passed: report.numPassedTests,
+					failed: report.numFailedTests,
+					failures,
+				});
+			} catch {
+				output = `Tests failed to run (exit ${exitCode})`;
 			}
 			log.info('tests finished', { exitCode, bytes: output.length });
 			return output;