FE-1943: Show where each Brunch exchange's time goes in persona runs

apps/brunch-agent/evals/petrinaut/README.md

@@ -45,6 +45,22 @@ yarn workspace @apps/brunch-agent probe:turn pharma-cold-chain openai/gpt-6.1-so
 
 The arguments are the case, the `provider/id` model and the thinking level; they default to `pharma-cold-chain` and Brunch's default model and thinking level. The provider's API key must be in the environment. Each probe spends one live model turn and requires the same model and spend authorization as a persona run. Output stays under `apps/brunch-agent/.data-wipe-me/turn-probes/` and is not committed. A probe is an observation of one turn, not a quality grade.
 
+## Run packets
+
+Every persona run and probe ends by writing `evidence/packet.json` and `evidence/packet.md` into its directory. The packet accounts for each exchange's time, which is what a change to turn time is judged by:
+
+- active time (wall time less approval waits) and time until the first reply text;
+- model steps, each split into time before first output (hidden reasoning), tool-argument streaming and reply text, and tool phases;
+- each step's shape (its tool categories, in order);
+- the tool mix, skill activations, Ledger commit sizes, net growth, tokens and cost;
+- what it could not account for: exchanges that never settled (left out of the time sums), steps cut off mid-message or mid-tools, and stored records it could not read.
+
+[`run-packet.ts`](./run-packet.ts) reads `conversation.db` and, when present, the server log's `flue.submission chronology` lines, `run.json` and the evidence files. Without the server log, which the probe does not keep, argument-streaming times are unknown. To write or refresh packets for retained runs, which spends no model turns:
+
+```sh
+yarn workspace @apps/brunch-agent packet:run .data-wipe-me/persona-runs/run-XXXXXX [...]
+```
+
 ## Scoring
 
 Each maintained case also keeps a `key.md` for scoring a run by hand: what the person believes, where the pack puts that under pressure, and what Brunch must tell apart, respect or keep conditional. Neither runner loads it. There are no graders or campaign protocols. A persona run or a probe is an observation, not an automatic quality grade or acceptance decision.

apps/brunch-agent/evals/petrinaut/persona/README.md

@@ -50,7 +50,7 @@ The helper speaks a Pi-style JSONL protocol to the bridge. Commands are `{ "id"?
 
 The persona stops by its own rule: when the person has what they came for, when the conversation is going nowhere, or when they run out of patience. It then runs `persona end` and tells you why it stopped. You, or an agent supervising the run, can force a stop at any point, for example after a chosen number of turns, with `<run>/bin/persona end`, or with Ctrl-C in the launcher when the agent runs in its own pane.
 
-**Persona runs have no automatic accounting cutoff.** The launcher disables the campaign accounting wrapper for Brunch even if `BRUNCH_STEP_A_ACCOUNTING` was inherited. There is no built-in turn-count limit. A live-provider run requires explicit model and spend authorization; the existence of this command grants none.
+**Persona runs have no automatic accounting cutoff.** The launcher disables the campaign accounting wrapper for Brunch even if `BRUNCH_STEP_A_ACCOUNTING` was inherited. `--max-messages <n>` stops the run once Brunch has replied to the persona's nth message; the case's opening is not counted, and the run packet is written as on any other stop. A live-provider run requires explicit model and spend authorization; the existence of this command grants none.
 
 ## Guidance provenance
 
@@ -123,6 +123,7 @@ Each launch prints its directory under `apps/brunch-agent/.data-wipe-me/persona-
 - `persona-brief.md`, `resume-brief.md` and `bin/persona`: the agent's private brief, latest resume notice and bridge helper.
 - `bridge-log.jsonl`: every utterance Brunch admitted, with the persona agent settings that wrote each persona utterance (empty when you started the agent yourself), used to reconcile resume; and every destructive edit the launcher allowed, with the items its approval panel listed.
 - `evidence/`: canonical snapshot and derived transcript, tool trace, compiled `ledger.md` with its `ledger.json` commits, and bound `net.json`; refreshed after completed turns and on shutdown.
+- `evidence/packet.json` and `packet.md`: the run packet, written on shutdown (see [Run packets](../README.md#run-packets)).
 - Service logs, only for services this launch started.
 
 The persistent Chrome profile lives outside the checkout; its path is in `run.json`. Older runs may also contain `pi/`, `persona-input.md`, `usage-ledger.json` or `attempt-ledger.md`; leave them untouched as historical evidence.

apps/brunch-agent/evals/petrinaut/persona/launch.ts

@@ -32,6 +32,7 @@ import {
   DEFAULT_CHAT_THINKING,
 } from "../../../src/shared/chat-model.ts";
 import { agentOwnershipHeaders } from "../../../src/shared/conversation/identity.ts";
+import { writeRunPacket } from "../run-packet.ts";
 import { openPersonaBrowserBridge } from "./browser-bridge.ts";
 import {
   PersonaBrowserTurnError,
@@ -286,6 +287,8 @@ type PersonaLaunchOptions = {
   readonly agent?: PersonaAgentSettings;
   /** Wait for Enter before the first message; only possible on a TTY. */
   readonly recordingPause?: boolean;
+  /** Stop the run once Brunch has replied to this many persona messages. */
+  readonly maxMessages?: number;
 };
 
 /**
@@ -302,6 +305,7 @@ const launchPersona = async ({
   roles = resolvePersonaRoleSettings(),
   agent,
   recordingPause: pauseForRecording = true,
+  maxMessages,
 }: PersonaLaunchOptions) => {
   const manifest = resume
     ? await verifyGuidanceResume(resume.config.guidance)
@@ -385,25 +389,46 @@ const launchPersona = async ({
   report(`Run: ${run}`);
   const stop = new AbortController();
   const started: ChildProcess[] = [];
-  let servicesStopRequested = false;
+  /** Whether the signal reached the process group; false once it has gone. */
+  const signalGroup = (pid: number, signal: NodeJS.Signals | 0) => {
+    try {
+      process.kill(-pid, signal);
+      return true;
+    } catch (error) {
+      if (
+        !(error instanceof Error) ||
+        !("code" in error) ||
+        error.code !== "ESRCH"
+      ) {
+        throw error;
+      }
+      return false;
+    }
+  };
+  let servicesStopped: Promise<void> | undefined;
+  /** Ends the owned process groups and settles once every one has exited. */
   const stopStartedServices = () => {
-    if (servicesStopRequested) return;
-    servicesStopRequested = true;
-    for (const child of started) {
-      if (!child.pid || child.exitCode !== null || child.signalCode !== null)
-        continue;
-      try {
-        process.kill(-child.pid, "SIGTERM");
-      } catch (error) {
-        if (
-          !(error instanceof Error) ||
-          !("code" in error) ||
-          error.code !== "ESRCH"
-        ) {
-          throw error;
+    servicesStopped ??= (async () => {
+      const groups = started.flatMap((child) =>
+        child.pid !== undefined &&
+        child.exitCode === null &&
+        child.signalCode === null &&
+        signalGroup(child.pid, "SIGTERM")
+          ? [child.pid]
+          : [],
+      );
+      // Services flush their logs and the conversation store on SIGTERM;
+      // one that has not exited by then is killed so the launcher can finish.
+      const deadline = Date.now() + 10_000;
+      while (groups.some((pid) => signalGroup(pid, 0))) {
+        if (Date.now() > deadline) {
+          for (const pid of groups) signalGroup(pid, "SIGKILL");
+          return;
         }
+        await delay(100);
       }
-    }
+    })();
+    return servicesStopped;
   };
   let browser:
     | Awaited<ReturnType<typeof chromium.launchPersistentContext>>
@@ -427,6 +452,7 @@ const launchPersona = async ({
   };
   process.on("SIGINT", onInterruptKey);
   process.on("SIGTERM", interrupt);
+  process.on("SIGHUP", interrupt);
   try {
     const preflight = await execute(
       process.execPath,
@@ -505,6 +531,11 @@ const launchPersona = async ({
       executablePath: chromeExecutable,
       headless: false,
       viewport: null,
+      // The launcher's own handlers stop the run and close the browser; Playwright's
+      // would kill it and exit before cleanup retains the run.
+      handleSIGINT: false,
+      handleSIGTERM: false,
+      handleSIGHUP: false,
       args: [
         "--remote-debugging-address=127.0.0.1",
         "--remote-debugging-port=0",
@@ -619,6 +650,7 @@ const launchPersona = async ({
       url: opened.session.url,
       headers: agentOwnershipHeaders(opened.session),
     });
+    let repliedMessages = 0;
     bridge = await openPersonaBrowserBridge({
       prompt: async (message, turn) => {
         await verifyGuidanceResume(manifest);
@@ -656,6 +688,13 @@ const launchPersona = async ({
               documentId,
               join(run, "evidence"),
             );
+          repliedMessages += 1;
+          if (maxMessages !== undefined && repliedMessages >= maxMessages)
+            // After the bridge has delivered this reply.
+            setImmediate(() => {
+              report(`Message limit reached: ${maxMessages} persona messages.`);
+              interrupt();
+            });
           return {
             text: result.reply.text,
             submissionIds: result.submissionIds,
@@ -760,8 +799,9 @@ const launchPersona = async ({
         });
         report(`Launcher Stop: ${stopDisposition.settlement}.`);
         // Settlement is durable now; stop services before slower browser cleanup
-        // so terminal Ctrl-C cannot orphan the launcher's process groups.
-        stopStartedServices();
+        // so terminal Ctrl-C cannot orphan the launcher's process groups. The
+        // packet step below awaits the same stop and reports any failure.
+        stopStartedServices().catch(() => undefined);
       }
       await bridge?.close();
       if (agentProcess?.kind === "herdr") {
@@ -787,7 +827,17 @@ const launchPersona = async ({
           await browser?.close();
         }
       } finally {
-        stopStartedServices();
+        await stopStartedServices();
+        // After the services stop, the server log and conversation store are final.
+        await writeRunPacket(run)
+          .then(() => refreshProofManifest(join(run, "evidence")))
+          .then(
+            () => report(`Run packet: ${join(run, "evidence", "packet.md")}`),
+            (error: unknown) =>
+              process.stderr.write(
+                `Could not write the run packet (retry with \`yarn workspace @apps/brunch-agent packet:run ${run}\`): ${String(error)}\n`,
+              ),
+          );
         report(`Retained run: ${run}`);
       }
       if (agentProcess?.kind === "foreground" && !lifecycle.agentExited) {
@@ -797,6 +847,7 @@ const launchPersona = async ({
     } finally {
       process.removeListener("SIGINT", onInterruptKey);
       process.removeListener("SIGTERM", interrupt);
+      process.removeListener("SIGHUP", interrupt);
     }
   }
 };
@@ -820,13 +871,14 @@ if (
       "persona-model": { type: "string" },
       "persona-thinking": { type: "string" },
       "skip-recording-pause": { type: "boolean" },
+      "max-messages": { type: "string" },
       resume: { type: "string" },
       help: { type: "boolean", short: "h" },
     },
   });
   if (values.help) {
     report(
-      `Usage: yarn brunch:persona --case <name-or-directory> [--objective <the person's aim>] [--route </path?search>] [--initial-net <sdcpn.json>] [--interview-length quick|standard|thorough|deep] [--brunch-model <provider/id>] [--brunch-thinking <level>] [agent options]\nDiscover cases: yarn brunch:persona --list-cases\nDefault: empty net on /; optional --initial-net stages a model and is not a from-scratch run. --interview-length turns on the website's Interview length at that level for the whole run; without it the run has none. --objective is a private aim for the person, phrased in their own terms; without it the person's goal comes from the case. --objective is fresh-run-only and is neither retained nor reapplied on resume. Starts owned services, a fresh headed Chrome window and a browser bridge; on an interactive terminal, pauses for Enter before sending anything (--skip-recording-pause skips it). Defaults: Brunch ${DEFAULT_CHAT_MODEL} ${DEFAULT_CHAT_THINKING}. Native usage is retained; ordinary external-agent runs have no automatic budget cutoff. Requires macOS Chrome, unused ${brunchEnv.chatPort}/${brunchEnv.panelPort}, and Brunch's provider API key. The persona stops by its own rule; Ctrl-C here or <run>/bin/persona end stops the run at any point, stopping owned resources and retaining run data.`,
+      `Usage: yarn brunch:persona --case <name-or-directory> [--objective <the person's aim>] [--route </path?search>] [--initial-net <sdcpn.json>] [--interview-length quick|standard|thorough|deep] [--brunch-model <provider/id>] [--brunch-thinking <level>] [agent options]\nDiscover cases: yarn brunch:persona --list-cases\nDefault: empty net on /; optional --initial-net stages a model and is not a from-scratch run. --interview-length turns on the website's Interview length at that level for the whole run; without it the run has none. --objective is a private aim for the person, phrased in their own terms; without it the person's goal comes from the case. --objective is fresh-run-only and is neither retained nor reapplied on resume. Starts owned services, a fresh headed Chrome window and a browser bridge; on an interactive terminal, pauses for Enter before sending anything (--skip-recording-pause skips it). --max-messages <n> stops the run once Brunch has replied to the persona's nth message (the case's opening is not counted). Defaults: Brunch ${DEFAULT_CHAT_MODEL} ${DEFAULT_CHAT_THINKING}. Native usage is retained; ordinary external-agent runs have no automatic budget cutoff. Requires macOS Chrome, unused ${brunchEnv.chatPort}/${brunchEnv.panelPort}, and Brunch's provider API key. The persona stops by its own rule; Ctrl-C here or <run>/bin/persona end stops the run at any point, stopping owned resources and retaining run data.`,
     );
     report(
       `Agent options:\n  --agent ${personaAgentPresets.join("|")}   Start that agent with the persona brief.\n  --agent-command '<shell command with {prompt}>'   Start any other agent; {prompt} becomes the quoted launch prompt.\n  --persona-model <model>   Passed to the --agent preset's own model flag.\n  --persona-thinking <level>   Passed to pi's --thinking (only with --agent pi).\nThe agent runs in a Herdr pane when HERDR_ENV=1, otherwise in this terminal. Without an agent option the launcher prints the launch prompt for you to give any agent. The agent talks to Brunch through <run>/bin/persona (say, transcript, state, end, rpc).`,
@@ -862,6 +914,13 @@ if (
           })
         : undefined;
     const recordingPause = !values["skip-recording-pause"];
+    const maxMessages =
+      values["max-messages"] === undefined
+        ? undefined
+        : Number(values["max-messages"]);
+    const invalidMaxMessages =
+      maxMessages !== undefined &&
+      !(Number.isInteger(maxMessages) && maxMessages > 0);
     const resumeRun = async (resumeDirectory: string) => {
       if (
         values.case ||
@@ -883,6 +942,7 @@ if (
         resume: retained,
         agent: selectedAgent,
         recordingPause,
+        maxMessages,
       });
     };
     const freshRun = async (caseDirectory: string) =>
@@ -903,12 +963,17 @@ if (
         }),
         agent: agent() ?? {},
         recordingPause,
+        maxMessages,
       });
-    const task = values.resume
-      ? resumeRun(values.resume)
-      : directory
-        ? freshRun(directory)
-        : Promise.reject(new Error("Supply --case <name-or-directory>"));
+    const task = invalidMaxMessages
+      ? Promise.reject(
+          new Error("--max-messages takes a positive whole number"),
+        )
+      : values.resume
+        ? resumeRun(values.resume)
+        : directory
+          ? freshRun(directory)
+          : Promise.reject(new Error("Supply --case <name-or-directory>"));
     await task.catch((error: unknown) => {
       process.stderr.write(
         `${error instanceof Error ? error.message : "Persona launch failed"}\n`,

apps/brunch-agent/evals/petrinaut/run-packet-cli.ts

@@ -0,0 +1,30 @@
+/**
+ * Writes `evidence/packet.json` and `evidence/packet.md` for each given
+ * persona run or turn-probe directory. Reads retained files only; spends no
+ * model turns.
+ *
+ *   yarn workspace @apps/brunch-agent packet:run <run-dir> [<run-dir> …]
+ */
+import { resolve } from "node:path";
+
+import { writeRunPacket } from "./run-packet.ts";
+
+const directories = process.argv
+  .slice(2)
+  .map((directory) => resolve(directory));
+if (directories.length === 0) {
+  process.stderr.write("Usage: packet:run <run-dir> [<run-dir> …]\n");
+  process.exit(2);
+}
+const written = await Promise.all(
+  directories.map(async (directory) => ({
+    directory,
+    packet: await writeRunPacket(directory),
+  })),
+);
+for (const { directory, packet } of written) {
+  const { summary } = packet;
+  process.stdout.write(
+    `${packet.label}: ${summary.exchanges} exchanges, median ${Math.round((summary.activePerExchangeMs.median ?? 0) / 1000)} s, ${summary.stepsPerExchange?.toFixed(1) ?? "–"} steps per exchange → ${resolve(directory, "evidence", "packet.md")}\n`,
+  );
+}

apps/brunch-agent/evals/petrinaut/run-packet.ts

@@ -0,0 +1,32 @@
+/**
+ * A run's analysable packet: per-exchange turn time split into hidden
+ * reasoning, argument streaming, reply text and tool phases, with the tool
+ * mix, skill activations, Ledger commit sizes, net growth and cost. Reads a
+ * persona run or turn-probe directory and writes `evidence/packet.json` and
+ * `evidence/packet.md` into it.
+ */
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import {
+  buildRunPacket,
+  type RunPacket,
+} from "./run-packet/build-run-packet.ts";
+import { readRunDirectory } from "./run-packet/read-run-directory.ts";
+import { renderRunPacket } from "./run-packet/render-run-packet.ts";
+
+export type { RunPacket } from "./run-packet/build-run-packet.ts";
+
+export const writeRunPacket = async (directory: string): Promise<RunPacket> => {
+  const packet = buildRunPacket(await readRunDirectory(directory));
+  const evidence = join(directory, "evidence");
+  await mkdir(evidence, { recursive: true });
+  await Promise.all([
+    writeFile(
+      join(evidence, "packet.json"),
+      `${JSON.stringify(packet, null, 2)}\n`,
+    ),
+    writeFile(join(evidence, "packet.md"), renderRunPacket(packet)),
+  ]);
+  return packet;
+};