Report what the app said and did, not just how the page moved

CHANGELOG.md

@@ -11,10 +11,42 @@
 - New Bunosh tasks for feature worktrees: `worktree:create <feature>` opens a new branch off main
   in a sibling directory, `worktree:fetch <branch>` opens an existing branch there, both symlinking
   the main checkout's `node_modules`; `worktree:delete [branch]` removes a worktree when merged.
+
 ## 2026-08-21
 
+### Actions report what the app said and did, not just how the page moved
+
+A click that fired a request the server rejected used to look like a success. The page diff described
+DOM and accessibility-tree movement only, so a toast reading "Operation against a key holding the wrong
+kind of value" was either buried in raw markup or dropped altogether, and the rejected request surfaced
+much later as a session-wide count. Every action now also carries:
+
+- **messages** — text the app put on the page in response: toasts, alerts, banners and inline errors,
+  including ones built without any accessibility markup. An action that navigated reports only what its
+  live regions announced, so the content of the page that opened is not read back as a reply
+- **requests** — the calls the action made to the application, each with its status, capped per action
+  with rejected calls kept ahead of successful ones
+- **consoleErrors** — what the page logged while the action ran
+
+When a request comes back 400 or 500 the result leads with that, and points at the message the user was
+shown, rather than leaving the model to read the click as successful and repeat it.
+
+Elements are no longer compared across two pages. An action that navigated used to report one ARIA diff
+whose halves belonged to different pages — the elements of the page left behind listed as removed next to
+the elements of the page arrived at, with the chrome common to both cancelled out and nothing saying which
+side was which, so elements that no longer exist read as available. Such an action now reports the move
+itself, the message the app announced in transit, and its requests; the page arrived at is described in
+full by the context that follows it.
+
+The Pilot's review reads the same evidence attached to the action that caused it, narrowed to what
+indicates failure: rejected requests, the first two messages, one console error. It used to get a bare
+`POST /api/… → 400` with no page context, which reads as a missing value, and would send the tester back
+to fill in a form that was already filled.
+
 ### Changes
 
+- Console messages from the browser were dropped before they were ever recorded, so console errors always
+  showed as none and a page that logged its own failure reported nothing.
 - AI models that emit channel markers in tool names (e.g. `click<|channel|>commentary`, common with
   gpt-oss and gemma) no longer waste a turn: the provider now repairs the name to the real tool and
   executes it, instead of rejecting the call and telling the AI to retry. In a 24h CI sample this

boat/prima/src/prima.ts

@@ -1043,7 +1043,12 @@ export class Prima {
   private async pageChanges(result: ActionResult, previousState: WebPageState | null, code: string): Promise<string> {
     if (!previousState) return 'no snapshot was captured before this command, so nothing could be compared';
     const toolResult = await result.toToolResult(ActionResult.fromState(previousState), code);
-    return toolResult.pageDiff?.ariaChanges || 'no change';
+    const pageDiff = toolResult.pageDiff;
+    if (!pageDiff?.urlChanged) return pageDiff?.ariaChanges || 'no change';
+
+    const lines = [`left ${previousState.url} for ${result.url}`];
+    for (const message of pageDiff.messages ?? []) lines.push(`- ${message}`);
+    return lines.join('\n');
   }
 
   async status(hash: string): Promise<EnvelopeData> {

boat/prima/tests/prima.test.ts

@@ -75,7 +75,7 @@ function fakePrima(options: Record<string, unknown> = {}) {
     title: 'Dashboard',
     hash: 'dashboard_h1_dashboard',
     getStateHash: () => 'dashboard_h1_dashboard',
-    toToolResult: async () => ({ pageDiff: { urlChanged: true, ariaChanges: 'added:\n  - heading "Dashboard"' } }),
+    toToolResult: async () => ({ pageDiff: { urlChanged: true, messages: ['Signed in as admin'] } }),
   });
   (prima as any).bot = {
     getExplorer: () => ({
@@ -130,7 +130,8 @@ describe('Prima.pw', () => {
   test('reports page changes from the action pipeline diff and writes artifacts', async () => {
     const { prima } = fakePrima();
     const envelope = await prima.pw("({ page }) => page.click('text=Login')");
-    expect(envelope.changes).toContain('heading "Dashboard"');
+    expect(envelope.changes).toContain('left https://app.example.com/login for https://app.example.com/dashboard');
+    expect(envelope.changes).toContain('Signed in as admin');
     expect(envelope.status).toMatch(/^[0-9a-f]{15}$/);
     expect(existsSync(path.join(artifactsRoot, envelope.status!, 'aria.yml'))).toBe(true);
     expect(existsSync(path.join(artifactsRoot, envelope.status!, 'page.html'))).toBe(true);

src/action-result.ts

@@ -4,7 +4,7 @@ import { ConfigParser, type HtmlConfig, outputPath } from './config.ts';
 import type { Link, WebPageState } from './state-manager.ts';
 import { LARGE_ARIA_CHANGE_THRESHOLD, compactAriaSnapshot, diffAriaSnapshots } from './utils/aria.ts';
 import { TTLCache } from './utils/cache.ts';
-import { type HtmlDiffPart, type HtmlDiffResult, htmlDiff } from './utils/html-diff.ts';
+import { type HtmlDiffPart, type HtmlDiffResult, htmlDiff, liveRegionMessages } from './utils/html-diff.ts';
 import { extractHeadings, extractLinks, extractTargetedHtml, htmlCombinedSnapshot, htmlMinimalUISnapshot, htmlTextSnapshot, minifyHtml } from './utils/html.ts';
 import { createDebug } from './utils/logger.ts';
 import { slugify } from './utils/strings.ts';
@@ -27,6 +27,7 @@ interface ActionResultData extends WebPageState {
   h3?: string | undefined;
   h4?: string | undefined;
   browserLogs?: any[];
+  networkRequests?: NetworkCall[];
   iframeSnapshots?: Array<{ src: string; html: string; id?: string }>;
   ariaSnapshot?: string | null;
   ariaSnapshotFile?: string;
@@ -41,6 +42,9 @@ export interface PageDiff {
   currentUrl: string;
   ariaChanges?: string | null;
   ariaChangeCount?: number;
+  messages?: string[];
+  requests?: NetworkCall[];
+  consoleErrors?: string[];
   htmlParts?: HtmlDiffPart[];
   iframes?: string;
 }
@@ -66,6 +70,7 @@ export class ActionResult implements ActionResultData {
   public url = '';
   public fullUrl: string | undefined = undefined;
   public browserLogs: any[] = [];
+  public networkRequests: NetworkCall[] = [];
   public iframeSnapshots: Array<{ src: string; html: string; id?: string }> = [];
   public iframeURL: string | undefined = undefined;
   readonly screenshotFile: string | undefined = undefined;
@@ -91,6 +96,7 @@ export class ActionResult implements ActionResultData {
     this.httpStatus = data.httpStatus;
     this.error = data.error ?? null;
     this.browserLogs = data.browserLogs ?? [];
+    this.networkRequests = data.networkRequests ?? [];
     this.iframeSnapshots = data.iframeSnapshots ?? [];
     this.iframeURL = data.iframeURL;
     this.notes = data.notes ?? [];
@@ -508,23 +514,30 @@ export class ActionResult implements ActionResultData {
       return result;
     }
 
-    const urlChanged = previousState ? !this.isSameUrl({ url: previousState.url }) : true;
+    const pageDiff: PageDiff = {
+      urlChanged: previousState ? !this.isSameUrl({ url: previousState.url }) : true,
+      currentUrl: this.url,
+    };
+    result.pageDiff = pageDiff;
 
-    if (!previousState) {
-      result.pageDiff = {
-        urlChanged: true,
-        currentUrl: this.url,
-      };
-      return result;
+    if (this.networkRequests.length > 0) {
+      pageDiff.requests = this.networkRequests;
+    }
+
+    const consoleErrors = this.consoleErrors();
+    if (consoleErrors.length > 0) {
+      pageDiff.consoleErrors = consoleErrors;
     }
 
+    if (!previousState) return result;
+
+    pageDiff.previousUrl = previousState.url;
+
     const diff = await this.diff(previousState);
 
-    const pageDiff: PageDiff = {
-      urlChanged,
-      previousUrl: previousState.url,
-      currentUrl: this.url,
-    };
+    if (diff.messages.length > 0) {
+      pageDiff.messages = diff.messages;
+    }
 
     if (diff.ariaChanged) {
       pageDiff.ariaChanges = diff.ariaChanged;
@@ -552,11 +565,28 @@ export class ActionResult implements ActionResultData {
       }
     }
 
-    result.pageDiff = pageDiff;
     return result;
   }
+
+  private consoleErrors(): string[] {
+    const errors: string[] = [];
+
+    for (const log of this.browserLogs) {
+      if ((log.type || log.level) !== 'error') continue;
+      const text = String(log.text || log.message || log).trim();
+      if (!text) continue;
+      if (errors.includes(text)) continue;
+      errors.push(text.slice(0, CONSOLE_ERROR_MAX_LENGTH));
+      if (errors.length === CONSOLE_ERROR_LIMIT) break;
+    }
+
+    return errors;
+  }
 }
 
+const CONSOLE_ERROR_MAX_LENGTH = 300;
+const CONSOLE_ERROR_LIMIT = 3;
+
 const HTML_PARTS_TOTAL_BUDGET = 8000;
 const HTML_PARTS_COUNT_LIMIT = 8;
 const HTML_PART_SUBTREE_BUDGET = 2000;
@@ -584,6 +614,7 @@ function collapseHtmlParts(parts: HtmlDiffPart[]): HtmlDiffPart[] {
 
 export class Diff {
   private _htmlDiffResult: HtmlDiffResult | null = null;
+  private _messages: string[] = [];
   private _ariaDiffResult: string | null = null;
   private _ariaChangeCount = 0;
   private _isSameUrl: boolean;
@@ -636,19 +667,33 @@ export class Diff {
     return this._htmlDiffResult;
   }
 
+  get messages(): string[] {
+    return this._messages;
+  }
+
   async calculate(): Promise<void> {
     if (!this.previous) return;
 
-    if (this._isSameUrl) {
-      this._htmlDiffResult = await htmlDiff(this.previous.html, this.current.html, ConfigParser.getInstance().getConfig().html);
+    if (!this._isSameUrl) {
+      this._messages = liveRegionMessages(this.previous.html, this.current.html);
+      return;
     }
 
+    this._htmlDiffResult = await htmlDiff(this.previous.html, this.current.html, ConfigParser.getInstance().getConfig().html);
+    this._messages = this._htmlDiffResult.messages;
+
     const ariaDiff = diffAriaSnapshots(this.previous.ariaSnapshot, this.current.ariaSnapshot);
     this._ariaDiffResult = ariaDiff.text;
     this._ariaChangeCount = ariaDiff.count;
   }
 }
 
+export interface NetworkCall {
+  method: string;
+  path: string;
+  status: number;
+}
+
 export interface FocusedElement {
   role: string;
   name: string;