From d51899328d3ba996932c0cb91cd37ac17db3e985 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:29:17 +0000 Subject: [PATCH 01/21] fix(agent-invocations): Route child resumes to the agent invocation first A child agent working on a watch event yields after the turn deadline. Its resume wake has an empty mailbox. Server conversation work looked for a paused web or event Turn before it looked for an agent invocation. The child Turn has an event Source, so the wake went to the root Event Turn worker. That worker needs a Destination, and a child has none, so every retry failed with "Invalid local conversation id" and the child stayed paused. Server conversation work now uses routeAgentInvocationWork, the same router the CLI uses, so child work reaches its agent invocation before any other worker. The agent invocation tests composed their own router with the correct order, so they never ran the server path. They now use the production conversation work harness. The resume test also covers a child timer event that yields and resumes, and it fails without this fix with the production error. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../src/chat/agent-invocations/README.md | 6 +- .../junior/src/chat/agent-invocations/work.ts | 6 +- .../junior/src/chat/app/conversation-work.ts | 37 +-- .../junior/tests/fixtures/agent-dispatch.ts | 4 +- .../tests/fixtures/conversation-work.ts | 13 + .../agent-dispatch-recovery.test.ts | 8 +- .../integration/agent-dispatch-work.test.ts | 14 +- .../agent-invocation-concurrency.test.ts | 69 ++---- .../integration/agent-invocation-work.test.ts | 226 +++++++++--------- scripts/test-architecture-baseline.json | 4 +- 10 files changed, 197 insertions(+), 190 deletions(-) diff --git a/packages/junior/src/chat/agent-invocations/README.md b/packages/junior/src/chat/agent-invocations/README.md index ad47351bd6..82f301ae33 100644 --- a/packages/junior/src/chat/agent-invocations/README.md +++ b/packages/junior/src/chat/agent-invocations/README.md @@ -54,7 +54,11 @@ conversation tree. 1. Creation writes the invocation before attempting the mailbox append. 2. The idempotent mailbox append sends a normal conversation queue wake. 3. The invocation router recognizes destinationless child work and advances it - through the shared `AgentRunner`. + through the shared `AgentRunner`. `routeAgentInvocationWork` checks for an + agent invocation before any other worker, in the server and in the CLI. A + child Turn can have any Source. A watch event Turn has an event Source, so a + router that picks the worker by Source sends its resume to the root Event + Turn worker. That worker needs a Destination, and a child has none. 4. A cooperative yield keeps the same turn and invocation active for another execution slice. 5. A stranded `running` child is re-parked at its latest durable safe boundary. diff --git a/packages/junior/src/chat/agent-invocations/work.ts b/packages/junior/src/chat/agent-invocations/work.ts index 5c7f8a4416..c8cec273cf 100644 --- a/packages/junior/src/chat/agent-invocations/work.ts +++ b/packages/junior/src/chat/agent-invocations/work.ts @@ -870,7 +870,11 @@ export function createAgentInvocationWorker( }; } -/** Route invocation-backed work before falling through to existing consumers. */ +/** + * Route child work to its agent invocation before any other worker. Server + * conversation work and the local CLI both use this order, because a child + * Turn can have any Source and only its agent invocation can run it. + */ export function routeAgentInvocationWork(options: { fallbackWorker: ( context: ConversationWorkerContext, diff --git a/packages/junior/src/chat/app/conversation-work.ts b/packages/junior/src/chat/app/conversation-work.ts index 4052fa079b..7c78228c81 100644 --- a/packages/junior/src/chat/app/conversation-work.ts +++ b/packages/junior/src/chat/app/conversation-work.ts @@ -18,7 +18,7 @@ import { } from "@/chat/agent-dispatch/work"; import { createAgentInvocationWorker, - resolveAgentInvocationId, + routeAgentInvocationWork, } from "@/chat/agent-invocations/work"; import { enqueueAgentInvocationResult, @@ -176,6 +176,24 @@ export function createConversationWork( `Conversation ${context.conversationId} has a ${destination.platform} destination but no matching conversation worker`, ); }; + // Child work must reach its agent invocation before any root worker. + const routeWork = routeAgentInvocationWork({ + invocationWorker, + fallbackWorker: async (context) => { + const mailboxTurn = await resolveMailboxTurnWork(context); + if (mailboxTurn) { + return await conversationTurnWorker( + context, + await withAgentInvocationResultActors(mailboxTurn), + ); + } + const dispatchId = await resolveAgentDispatchId(context); + if (dispatchId) { + return await dispatchWorker(context, dispatchId); + } + return await destinationWorker(context); + }, + }); const run = async ( context: ConversationWorkerContext, ): Promise => { @@ -194,22 +212,7 @@ export function createConversationWork( if (childInput) { return childInput; } - const mailboxTurn = await resolveMailboxTurnWork(context); - if (mailboxTurn) { - return await conversationTurnWorker( - context, - await withAgentInvocationResultActors(mailboxTurn), - ); - } - const invocationId = await resolveAgentInvocationId(context); - if (invocationId) { - return await invocationWorker(context, invocationId); - } - const dispatchId = await resolveAgentDispatchId(context); - if (dispatchId) { - return await dispatchWorker(context, dispatchId); - } - return await destinationWorker(context); + return await routeWork(context); }; return { conversationStore: options.conversationStore, diff --git a/packages/junior/tests/fixtures/agent-dispatch.ts b/packages/junior/tests/fixtures/agent-dispatch.ts index a15df79766..cb917e680c 100644 --- a/packages/junior/tests/fixtures/agent-dispatch.ts +++ b/packages/junior/tests/fixtures/agent-dispatch.ts @@ -19,8 +19,8 @@ export const agentDispatchTestDestination = { channelId: "C123", } as const; -/** Compose production conversation work for one agent dispatch test. */ -export async function createAgentDispatchWorkHarness(agentRunner: AgentRunner) { +/** Compose production conversation work for one integration test. */ +export async function createConversationWorkHarness(agentRunner: AgentRunner) { const state = getStateAdapter(); await state.connect(); const queue = createConversationWorkQueueTestAdapter(); diff --git a/packages/junior/tests/fixtures/conversation-work.ts b/packages/junior/tests/fixtures/conversation-work.ts index 2e00e975fa..fc7fb47d32 100644 --- a/packages/junior/tests/fixtures/conversation-work.ts +++ b/packages/junior/tests/fixtures/conversation-work.ts @@ -154,6 +154,19 @@ export class ConversationWorkQueueTestAdapter implements ConversationWorkQueue { return entry.message; } + /** Deliver the next payload for one Conversation and keep the others queued. */ + takeMessageFor(conversationId: string): ConversationQueueMessage { + const index = this.#queuedMessages.findIndex( + (entry) => entry.message.conversationId === conversationId, + ); + if (index === -1) { + throw new Error( + `Expected queued conversation work for ${conversationId}`, + ); + } + return this.#queuedMessages.splice(index, 1)[0]!.message; + } + /** Honor queue delays when running evals against the real clock. */ async takeReadyMessage( signal?: AbortSignal, diff --git a/packages/junior/tests/integration/agent-dispatch-recovery.test.ts b/packages/junior/tests/integration/agent-dispatch-recovery.test.ts index 11b6076a07..1e87224fcf 100644 --- a/packages/junior/tests/integration/agent-dispatch-recovery.test.ts +++ b/packages/junior/tests/integration/agent-dispatch-recovery.test.ts @@ -21,7 +21,7 @@ import { createModelStream } from "../fixtures/model-stream"; import { agentDispatchTestDestination as destination, createAgentDispatchTestRecord as createDispatch, - createAgentDispatchWorkHarness, + createConversationWorkHarness, } from "../fixtures/agent-dispatch"; vi.hoisted(() => { @@ -41,7 +41,7 @@ describe("agent dispatch recovery", () => { it("projects the diagnostic from a terminal model failure", async () => { const dispatch = await createDispatch("model-failure-detail"); - const firstRun = await createAgentDispatchWorkHarness( + const firstRun = await createConversationWorkHarness( createModelAgentRunner( createModelStream([ { @@ -74,7 +74,7 @@ describe("agent dispatch recovery", () => { errorMessage: "Model provider quota exhausted", }); - const replay = await createAgentDispatchWorkHarness(neverRunAgentRunner()); + const replay = await createConversationWorkHarness(neverRunAgentRunner()); await enqueueAgentDispatch(dispatch, { queue: replay.queue, state: replay.state, @@ -142,7 +142,7 @@ describe("agent dispatch recovery", () => { ); const runAgent = vi.spyOn(agentRunner, "run"); const { queue, run, state } = - await createAgentDispatchWorkHarness(agentRunner); + await createConversationWorkHarness(agentRunner); await enqueueAgentDispatch(dispatch, { queue, state }); let deliveries = 0; diff --git a/packages/junior/tests/integration/agent-dispatch-work.test.ts b/packages/junior/tests/integration/agent-dispatch-work.test.ts index bcd699d3f4..3284b863db 100644 --- a/packages/junior/tests/integration/agent-dispatch-work.test.ts +++ b/packages/junior/tests/integration/agent-dispatch-work.test.ts @@ -31,7 +31,7 @@ import { import { agentDispatchTestDestination as destination, createAgentDispatchTestRecord as createDispatch, - createAgentDispatchWorkHarness, + createConversationWorkHarness, } from "../fixtures/agent-dispatch"; vi.hoisted(() => { @@ -70,7 +70,7 @@ describe("agent dispatch conversation work", () => { input, ); const modelStream = vi.fn(createModelStream([sendMessage("Done")])); - const { queue, run, state } = await createAgentDispatchWorkHarness( + const { queue, run, state } = await createConversationWorkHarness( createModelAgentRunner(modelStream), ); @@ -148,7 +148,7 @@ describe("agent dispatch conversation work", () => { queue, run: runWork, state, - } = await createAgentDispatchWorkHarness(agentRunner); + } = await createConversationWorkHarness(agentRunner); const titleRequests: TitleModelRequest[] = []; mockTitleModel("Weekly digest for the platform team", { @@ -237,7 +237,7 @@ describe("agent dispatch conversation work", () => { ], { ...destination, threadTs: "1700000000.000200" }, ); - const { queue, run, state } = await createAgentDispatchWorkHarness( + const { queue, run, state } = await createConversationWorkHarness( createModelAgentRunner( createModelStream([sendMessage("Scheduled digest")]), ), @@ -275,7 +275,7 @@ describe("agent dispatch conversation work", () => { }, ], ); - const { queue, run, state } = await createAgentDispatchWorkHarness( + const { queue, run, state } = await createConversationWorkHarness( createModelAgentRunner(createModelStream([sendMessage("Work complete")])), ); @@ -320,7 +320,7 @@ describe("agent dispatch conversation work", () => { ); const runAgent = vi.spyOn(agentRunner, "run"); const { queue, run, state } = - await createAgentDispatchWorkHarness(agentRunner); + await createConversationWorkHarness(agentRunner); const processNext = async (softYieldAfterMs?: number) => await processConversationQueueMessage(queue.takeMessage(), { queue, @@ -396,7 +396,7 @@ describe("agent dispatch conversation work", () => { }, ); await persistConversationMessages({ conversation, conversationId }); - const { queue, run, state } = await createAgentDispatchWorkHarness( + const { queue, run, state } = await createConversationWorkHarness( neverRunAgentRunner(), ); diff --git a/packages/junior/tests/integration/agent-invocation-concurrency.test.ts b/packages/junior/tests/integration/agent-invocation-concurrency.test.ts index 2412071b60..73bcf3ec6d 100644 --- a/packages/junior/tests/integration/agent-invocation-concurrency.test.ts +++ b/packages/junior/tests/integration/agent-invocation-concurrency.test.ts @@ -8,34 +8,28 @@ import { MAX_ACTIVE_AGENT_INVOCATIONS_PER_PARENT, } from "@/chat/agent-invocations/store"; import { - createAgentInvocationWorker, enqueueAgentInvocation, - routeAgentInvocationWork, createAndEnqueueAgentInvocation, } from "@/chat/agent-invocations/work"; import type { CreateAgentInvocationInput } from "@/chat/agent-invocations/types"; -import { startAgentInvocationsForChildInput } from "@/chat/agent-invocations/result"; import { EVENT_SYSTEM_ACTOR } from "@/chat/events/actor"; import { getEventCatalog } from "@/chat/events/runtime-catalog"; import { runTimerWatchHeartbeat } from "@/chat/events/timers"; import { getAutomatedTurnLimitState } from "@/chat/services/automated-turn-limit"; import { createEventTools } from "@/chat/tools/events"; -import { skipInputBeforeConversationStop } from "@/chat/conversations/session-stop"; import { getConversationEventStore } from "@/chat/db"; import { context, execute } from "../fixtures/event-automations"; import { migrateSchema } from "@/chat/conversations/sql/migrations"; import { createSqlStore } from "@/chat/conversations/sql/store"; -import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; +import { disconnectStateAdapter } from "@/chat/state/adapter"; import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; import { createListAgentsTool } from "@/chat/tools/runtime/list-agents"; import { createStopAgentTool } from "@/chat/tools/runtime/stop-agent"; -import { - createConversationWorkQueueTestAdapter, - deferred, -} from "../fixtures/conversation-work"; +import { deferred } from "../fixtures/conversation-work"; import { createConfiguredJuniorSqlFixture } from "../fixtures/sql"; import { createModelAgentRunnerForRun } from "../fixtures/agent-runner"; import { createModelStream } from "../fixtures/model-stream"; +import { createConversationWorkHarness } from "../fixtures/agent-dispatch"; const parentConversationId = "local:test:agent-invocation-concurrency"; const destination = { @@ -70,21 +64,27 @@ async function createHarness(streamForRun: (request: AgentRun) => StreamFn) { nowMs: 1_000, source: "local", }); - const state = getStateAdapter(); - await state.connect(); - const queue = createConversationWorkQueueTestAdapter(); const agentRunner = createModelAgentRunnerForRun(streamForRun); const run = vi.spyOn(agentRunner, "run"); - const route = routeAgentInvocationWork({ - // The same order as production conversation work. - fallbackWorker: async (workContext) => - (await skipInputBeforeConversationStop(workContext)) ?? - (await startAgentInvocationsForChildInput(workContext, { - queue, - state, - })) ?? { status: "completed" as const }, - invocationWorker: createAgentInvocationWorker(agentRunner), - }); + const { + queue, + run: route, + state, + } = await createConversationWorkHarness(agentRunner); + // This harness runs only child work. Each child result also wakes the + // parent, and agent-invocation-work covers that parent Turn. + const processNext = async () => { + const message = queue.takeMessage(); + if (message.conversationId === parentConversationId) { + return; + } + await processConversationQueueMessage(message, { + conversationStore, + queue, + run: route, + state, + }); + }; return { async close() { @@ -93,30 +93,12 @@ async function createHarness(streamForRun: (request: AgentRun) => StreamFn) { async drainQueuedWork() { while (queue.hasQueuedMessages()) { const batch = queue.queuedMessages(); - await Promise.all( - batch.map(async () => { - await processConversationQueueMessage(queue.takeMessage(), { - conversationStore, - queue, - run: route, - state, - }); - }), - ); + await Promise.all(batch.map(processNext)); } }, processQueuedBatch: async () => { const batch = queue.queuedMessages(); - await Promise.all( - batch.map(async () => { - await processConversationQueueMessage(queue.takeMessage(), { - conversationStore, - queue, - run: route, - state, - }); - }), - ); + await Promise.all(batch.map(processNext)); }, /** Send one parent message to an existing child, as messageAgent does. */ message: async (args: { @@ -499,7 +481,8 @@ describe("agent invocation identity and concurrency", () => { { conversation_id: parentConversationId, relation: "parent", - status: "idle", + // Both child results wait in the parent mailbox. + status: "pending", updated_at: expect.any(String), }, { relation: "sibling", ...failingChild }, diff --git a/packages/junior/tests/integration/agent-invocation-work.test.ts b/packages/junior/tests/integration/agent-invocation-work.test.ts index fe5e89c7f7..00bad53385 100644 --- a/packages/junior/tests/integration/agent-invocation-work.test.ts +++ b/packages/junior/tests/integration/agent-invocation-work.test.ts @@ -2,30 +2,28 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { PiMessage } from "@/chat/pi/messages"; import { completeAgentInvocation, + getActiveAgentInvocationForConversation, getAgentInvocation, getAgentInvocationMessageId, getAgentInvocationResultMessageId, getAgentInvocationTurnId, } from "@/chat/agent-invocations/store"; -import { - enqueueAgentInvocationResult, - withAgentInvocationResultActors, -} from "@/chat/agent-invocations/result"; +import { withAgentInvocationResultActors } from "@/chat/agent-invocations/result"; import { EVENT_MESSAGE_AUTHOR, EVENT_SYSTEM_ACTOR } from "@/chat/events/actor"; +import { getEventCatalog } from "@/chat/events/runtime-catalog"; +import { runTimerWatchHeartbeat } from "@/chat/events/timers"; +import { createEventTools } from "@/chat/tools/events"; import { getConversationWorkState } from "@/chat/task-execution/store"; -import { - createAgentInvocationWorker, - routeAgentInvocationWork, - createAndEnqueueAgentInvocation, -} from "@/chat/agent-invocations/work"; +import { createAndEnqueueAgentInvocation } from "@/chat/agent-invocations/work"; import { loadProjection } from "@/chat/conversations/projection"; -import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; +import { disconnectStateAdapter } from "@/chat/state/adapter"; import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; import { getTurnRecord, upsertTurnRecord, } from "@/chat/task-execution/turn-cursor"; -import { createConversationWorkQueueTestAdapter } from "../fixtures/conversation-work"; +import { createConversationWorkHarness } from "../fixtures/agent-dispatch"; +import { context, execute } from "../fixtures/event-automations"; import { createModelAgentRunner, neverRunAgentRunner, @@ -56,9 +54,24 @@ describe("agent invocation conversation work", () => { it("reads the parent Location and visibility without delivering child output", async () => { const { conversationStore, fixture } = await prepareSlackParentConversation(); - const queue = createConversationWorkQueueTestAdapter(); - const state = getStateAdapter(); - await state.connect(); + let observedVisibility: ReturnType; + const agentRunner = createModelAgentRunner( + createModelStream([ + { + type: "text", + text: "Durable child result", + onRequest: () => { + observedVisibility = getCurrentConversationPrivacy(); + }, + }, + ]), + ); + const run = vi.spyOn(agentRunner, "run"); + const { + queue, + run: route, + state, + } = await createConversationWorkHarness(agentRunner); try { const created = await createAndEnqueueAgentInvocation( { @@ -82,29 +95,6 @@ describe("agent invocation conversation work", () => { expect(child).not.toHaveProperty("location"); expect(child).not.toHaveProperty("sessionSource"); expect(child).not.toHaveProperty("visibility"); - let observedVisibility: ReturnType; - const agentRunner = createModelAgentRunner( - createModelStream([ - { - type: "text", - text: "Durable child result", - onRequest: () => { - observedVisibility = getCurrentConversationPrivacy(); - }, - }, - ]), - ); - const run = vi.spyOn(agentRunner, "run"); - const fallbackWorker = vi.fn(async () => ({ - status: "completed" as const, - })); - const route = routeAgentInvocationWork({ - fallbackWorker, - invocationWorker: createAgentInvocationWorker(agentRunner, { - enqueueResult: async (invocation) => - await enqueueAgentInvocationResult(invocation, { queue, state }), - }), - }); const queueMessage = queue.takeMessage(); await expect( @@ -302,7 +292,6 @@ describe("agent invocation conversation work", () => { }); expect(run.mock.calls[0]?.[0].delivery).toBeUndefined(); expect(observedVisibility).toBe("public"); - expect(fallbackWorker).not.toHaveBeenCalled(); } finally { await fixture.close(); } @@ -338,9 +327,13 @@ describe("agent invocation conversation work", () => { terminal, }) => { const { conversationStore, fixture } = await prepareParentConversation(); - const queue = createConversationWorkQueueTestAdapter(); - const state = getStateAdapter(); - await state.connect(); + const { + queue, + run: route, + state, + } = await createConversationWorkHarness( + createModelAgentRunner(createModelStream([output])), + ); try { const created = await createAndEnqueueAgentInvocation( { @@ -349,12 +342,6 @@ describe("agent invocation conversation work", () => { }, { conversationStore, queue, state }, ); - const route = routeAgentInvocationWork({ - fallbackWorker: vi.fn(async () => ({ status: "completed" as const })), - invocationWorker: createAgentInvocationWorker( - createModelAgentRunner(createModelStream([output])), - ), - }); await expect( processConversationQueueMessage(queue.takeMessage(), { @@ -400,57 +387,90 @@ describe("agent invocation conversation work", () => { }, ); - it("resumes a yielded invocation from durable child state", async () => { + // A child watch event runs with an event Source. When its Turn yields, the + // resume wake must still reach the agent invocation. Production once sent + // it to the root Event Turn worker, which needs a Destination that a child + // does not have. + it.each([ + { name: "a parent task", watchEvent: false }, + { name: "a watch event", watchEvent: true }, + ])("resumes a yielded child invocation for $name", async ({ watchEvent }) => { + const clock = vi.spyOn(Date, "now").mockReturnValue(1_800_000_000_000); const { conversationStore, fixture } = await prepareParentConversation(); - const queue = createConversationWorkQueueTestAdapter(); - const state = getStateAdapter(); - await state.connect(); + const agentRunner = createModelAgentRunner( + createModelStream([ + ...(watchEvent ? [{ type: "text" as const, text: "Watching" }] : []), + { type: "toolCall", name: "systemTime", arguments: {} }, + { type: "text", text: "Resumed child result" }, + ]), + ); + const run = vi.spyOn(agentRunner, "run"); + const { + queue, + run: route, + state, + } = await createConversationWorkHarness(agentRunner); try { const created = await createAndEnqueueAgentInvocation( { ...invocationInput, idempotencyKey: "resume-1", }, - { - conversationStore, - nowMs: 2_000, - queue, - state, - }, - ); - const agentRunner = createModelAgentRunner( - createModelStream([ - { type: "toolCall", name: "systemTime", arguments: {} }, - { type: "text", text: "Resumed child result" }, - ]), + { conversationStore, queue, state }, ); - const run = vi.spyOn(agentRunner, "run"); - const invocationWorker = createAgentInvocationWorker(agentRunner); - const route = routeAgentInvocationWork({ - fallbackWorker: vi.fn(async () => ({ status: "completed" as const })), - invocationWorker, - }); + const { childConversationId } = created; + // Child results also wake the parent. This test runs only the child. + const processChild = async (softYieldAfterMs?: number) => + await processConversationQueueMessage( + queue.takeMessageFor(childConversationId), + { + conversationStore, + queue, + run: route, + state, + ...(softYieldAfterMs === undefined + ? undefined + : { softYieldAfterMs }), + }, + ); + if (watchEvent) { + // Finish the parent task, then set a timer on the child. The timer + // event waits for its burst window, then starts an agent invocation + // with the event Source. + await processChild(); + await execute( + createEventTools( + { ...context(), conversationId: childConversationId }, + getEventCatalog(), + ).watchTimer!, + { afterMs: 60_000, intent: "Check the PR checks." }, + "child-timer", + ); + clock.mockReturnValue(clock() + 120_000); + await runTimerWatchHeartbeat({ nowMs: clock(), queue }); + await processChild(); + clock.mockReturnValue(clock() + 120_000); + await processChild(0); + } + await expect(processChild(0)).resolves.toMatchObject({ + status: "yielded", + }); + const target = watchEvent + ? (await getActiveAgentInvocationForConversation(childConversationId))! + : created; await expect( - processConversationQueueMessage(queue.takeMessage(), { - conversationStore, - queue, - run: route, - softYieldAfterMs: 0, - state, - }), - ).resolves.toMatchObject({ status: "yielded" }); - await expect( - getAgentInvocation(created.invocationId), + getAgentInvocation(target.invocationId), ).resolves.toMatchObject({ status: "paused" }); await expect( getTurnRecord( - created.childConversationId, - getAgentInvocationTurnId(created.invocationId), + childConversationId, + getAgentInvocationTurnId(target.invocationId), ), ).resolves.toMatchObject({ state: "paused", resumeReason: "yield", + source: { kind: watchEvent ? "event" : "agent_invocation" }, piMessages: expect.arrayContaining([ expect.objectContaining({ role: "toolResult", @@ -459,23 +479,18 @@ describe("agent invocation conversation work", () => { ]), }); + await expect(processChild()).resolves.toMatchObject({ + status: "completed", + }); + expect(run).toHaveBeenCalledTimes(watchEvent ? 3 : 2); await expect( - processConversationQueueMessage(queue.takeMessage(), { - conversationStore, - queue, - run: route, - state, - }), - ).resolves.toMatchObject({ status: "completed" }); - expect(run).toHaveBeenCalledTimes(2); - await expect( - getAgentInvocation(created.invocationId), + getAgentInvocation(target.invocationId), ).resolves.toMatchObject({ result: "Resumed child result", status: "completed", }); const projection = await loadProjection({ - conversationId: created.childConversationId, + conversationId: childConversationId, }); expect( projection.filter( @@ -483,20 +498,8 @@ describe("agent invocation conversation work", () => { message.role === "toolResult" && message.toolName === "systemTime", ), ).toHaveLength(1); - expect(projection).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - role: "assistant", - content: expect.arrayContaining([ - expect.objectContaining({ - type: "text", - text: "Resumed child result", - }), - ]), - }), - ]), - ); } finally { + clock.mockRestore(); await fixture.close(); } }); @@ -544,9 +547,11 @@ describe("agent invocation conversation work", () => { "recovers a completed child turn that ends with $name through production routing", async ({ expected, tail, turnOutcome }) => { const { conversationStore, fixture } = await prepareParentConversation(); - const queue = createConversationWorkQueueTestAdapter(); - const state = getStateAdapter(); - await state.connect(); + const { + queue, + run: route, + state, + } = await createConversationWorkHarness(neverRunAgentRunner()); try { const created = await createAndEnqueueAgentInvocation( { @@ -592,11 +597,6 @@ describe("agent invocation conversation work", () => { state: "completed", surface: "internal", }); - const route = routeAgentInvocationWork({ - fallbackWorker: vi.fn(async () => ({ status: "completed" as const })), - invocationWorker: createAgentInvocationWorker(neverRunAgentRunner()), - }); - await expect( processConversationQueueMessage(queue.takeMessage(), { conversationStore, diff --git a/scripts/test-architecture-baseline.json b/scripts/test-architecture-baseline.json index bd310b6fc0..5e096d5559 100644 --- a/scripts/test-architecture-baseline.json +++ b/scripts/test-architecture-baseline.json @@ -171,8 +171,8 @@ "packages/junior/tests/integration/acp-http-recovery.test.ts": 2, "packages/junior/tests/integration/agent-dispatch-recovery.test.ts": 2, "packages/junior/tests/integration/agent-dispatch-work.test.ts": 7, - "packages/junior/tests/integration/agent-invocation-concurrency.test.ts": 2, - "packages/junior/tests/integration/agent-invocation-work.test.ts": 6, + "packages/junior/tests/integration/agent-invocation-concurrency.test.ts": 1, + "packages/junior/tests/integration/agent-invocation-work.test.ts": 5, "packages/junior/tests/integration/conversation-turn-work.test.ts": 9, "packages/junior/tests/integration/durable-queue.test.ts": 2, "packages/junior/tests/integration/event-wake-delay.test.ts": 1, From 7a099f15875d34dfb7b20db9ef0374fd5fc20865 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:35:57 +0000 Subject: [PATCH 02/21] fix(agent-invocations): Never route a child wake to a root worker When a child has no active agent invocation, an empty wake fell through to the mailbox Turn and Destination workers. A child agent invocation can end while its event Turn stays paused, and a later wake then crashed with the same "Invalid local conversation id" error. An empty wake now goes to the latest agent invocation of the child, which repeats its idempotent finish. Before this, only a stopped agent invocation got the wake. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../src/chat/agent-invocations/README.md | 7 +- .../junior/src/chat/agent-invocations/work.ts | 9 +- .../integration/agent-invocation-work.test.ts | 240 ++++++++++-------- 3 files changed, 143 insertions(+), 113 deletions(-) diff --git a/packages/junior/src/chat/agent-invocations/README.md b/packages/junior/src/chat/agent-invocations/README.md index 82f301ae33..93d39b2ce1 100644 --- a/packages/junior/src/chat/agent-invocations/README.md +++ b/packages/junior/src/chat/agent-invocations/README.md @@ -55,8 +55,11 @@ conversation tree. 2. The idempotent mailbox append sends a normal conversation queue wake. 3. The invocation router recognizes destinationless child work and advances it through the shared `AgentRunner`. `routeAgentInvocationWork` checks for an - agent invocation before any other worker, in the server and in the CLI. A - child Turn can have any Source. A watch event Turn has an event Source, so a + agent invocation before any other worker, in the server and in the CLI. An + empty wake goes to the active agent invocation of the child. When none is + active, it goes to the latest one, which repeats its idempotent finish. + Child work never falls through to a root worker. A child Turn can have any + Source. A watch event Turn has an event Source, so a router that picks the worker by Source sends its resume to the root Event Turn worker. That worker needs a Destination, and a child has none. 4. A cooperative yield keeps the same turn and invocation active for another diff --git a/packages/junior/src/chat/agent-invocations/work.ts b/packages/junior/src/chat/agent-invocations/work.ts index c8cec273cf..03af9a2924 100644 --- a/packages/junior/src/chat/agent-invocations/work.ts +++ b/packages/junior/src/chat/agent-invocations/work.ts @@ -257,12 +257,15 @@ export async function resolveAgentInvocationId( if (active) { return active.invocationId; } - // A stop request can outlive the run that it stopped and wake the child - // again. The stopped agent invocation handles that wake. + // A child Conversation never runs on a root worker. A wake after its last + // agent invocation ended, such as a late stop request, goes to that agent + // invocation, which only repeats its idempotent finish. A paused Turn of an + // ended agent invocation can have an event Source, and a root worker would + // try to resume it. const latest = await getLatestAgentInvocationForConversation( context.conversationId, ); - return latest?.status === "stopped" ? latest.invocationId : undefined; + return latest?.invocationId; } /** diff --git a/packages/junior/tests/integration/agent-invocation-work.test.ts b/packages/junior/tests/integration/agent-invocation-work.test.ts index 00bad53385..436de3611a 100644 --- a/packages/junior/tests/integration/agent-invocation-work.test.ts +++ b/packages/junior/tests/integration/agent-invocation-work.test.ts @@ -388,121 +388,145 @@ describe("agent invocation conversation work", () => { ); // A child watch event runs with an event Source. When its Turn yields, the - // resume wake must still reach the agent invocation. Production once sent - // it to the root Event Turn worker, which needs a Destination that a child - // does not have. + // next wake must still reach the agent invocation. Production once sent it + // to the root Event Turn worker, which needs a Destination that a child + // does not have. That also holds when the agent invocation ended while its + // Turn stayed paused. + const resumed = { result: "Resumed child result", status: "completed" }; it.each([ - { name: "a parent task", watchEvent: false }, - { name: "a watch event", watchEvent: true }, - ])("resumes a yielded child invocation for $name", async ({ watchEvent }) => { - const clock = vi.spyOn(Date, "now").mockReturnValue(1_800_000_000_000); - const { conversationStore, fixture } = await prepareParentConversation(); - const agentRunner = createModelAgentRunner( - createModelStream([ - ...(watchEvent ? [{ type: "text" as const, text: "Watching" }] : []), - { type: "toolCall", name: "systemTime", arguments: {} }, - { type: "text", text: "Resumed child result" }, - ]), - ); - const run = vi.spyOn(agentRunner, "run"); - const { - queue, - run: route, - state, - } = await createConversationWorkHarness(agentRunner); - try { - const created = await createAndEnqueueAgentInvocation( - { - ...invocationInput, - idempotencyKey: "resume-1", - }, - { conversationStore, queue, state }, + { + name: "a parent task", + watchEvent: false, + endBeforeWake: false, + runs: 2, + expected: resumed, + }, + { + name: "a watch event", + watchEvent: true, + endBeforeWake: false, + runs: 3, + expected: resumed, + }, + { + name: "an ended watch event", + watchEvent: true, + endBeforeWake: true, + runs: 2, + expected: { parentMailboxStatus: "appended", status: "failed" }, + }, + ])( + "gives the wake after a yielded child invocation for $name to that invocation", + async ({ endBeforeWake, expected, runs, watchEvent }) => { + const clock = vi.spyOn(Date, "now").mockReturnValue(1_800_000_000_000); + const { conversationStore, fixture } = await prepareParentConversation(); + const agentRunner = createModelAgentRunner( + createModelStream([ + ...(watchEvent ? [{ type: "text" as const, text: "Watching" }] : []), + { type: "toolCall", name: "systemTime", arguments: {} }, + { type: "text", text: "Resumed child result" }, + ]), ); - const { childConversationId } = created; - // Child results also wake the parent. This test runs only the child. - const processChild = async (softYieldAfterMs?: number) => - await processConversationQueueMessage( - queue.takeMessageFor(childConversationId), + const run = vi.spyOn(agentRunner, "run"); + const { + queue, + run: route, + state, + } = await createConversationWorkHarness(agentRunner); + try { + const created = await createAndEnqueueAgentInvocation( { - conversationStore, - queue, - run: route, - state, - ...(softYieldAfterMs === undefined - ? undefined - : { softYieldAfterMs }), + ...invocationInput, + idempotencyKey: "resume-1", }, + { conversationStore, queue, state }, ); - if (watchEvent) { - // Finish the parent task, then set a timer on the child. The timer - // event waits for its burst window, then starts an agent invocation - // with the event Source. - await processChild(); - await execute( - createEventTools( - { ...context(), conversationId: childConversationId }, - getEventCatalog(), - ).watchTimer!, - { afterMs: 60_000, intent: "Check the PR checks." }, - "child-timer", - ); - clock.mockReturnValue(clock() + 120_000); - await runTimerWatchHeartbeat({ nowMs: clock(), queue }); - await processChild(); - clock.mockReturnValue(clock() + 120_000); - await processChild(0); - } + const { childConversationId } = created; + // Child results also wake the parent. This test runs only the child. + const processChild = async (softYieldAfterMs?: number) => + await processConversationQueueMessage( + queue.takeMessageFor(childConversationId), + { + conversationStore, + queue, + run: route, + state, + ...(softYieldAfterMs === undefined + ? undefined + : { softYieldAfterMs }), + }, + ); + if (watchEvent) { + // Finish the parent task, then set a timer on the child. The timer + // event waits for its burst window, then starts an agent invocation + // with the event Source. + await processChild(); + await execute( + createEventTools( + { ...context(), conversationId: childConversationId }, + getEventCatalog(), + ).watchTimer!, + { afterMs: 60_000, intent: "Check the PR checks." }, + "child-timer", + ); + clock.mockReturnValue(clock() + 120_000); + await runTimerWatchHeartbeat({ nowMs: clock(), queue }); + await processChild(); + clock.mockReturnValue(clock() + 120_000); + await processChild(0); + } - await expect(processChild(0)).resolves.toMatchObject({ - status: "yielded", - }); - const target = watchEvent - ? (await getActiveAgentInvocationForConversation(childConversationId))! - : created; - await expect( - getAgentInvocation(target.invocationId), - ).resolves.toMatchObject({ status: "paused" }); - await expect( - getTurnRecord( - childConversationId, - getAgentInvocationTurnId(target.invocationId), - ), - ).resolves.toMatchObject({ - state: "paused", - resumeReason: "yield", - source: { kind: watchEvent ? "event" : "agent_invocation" }, - piMessages: expect.arrayContaining([ - expect.objectContaining({ - role: "toolResult", - toolName: "systemTime", - }), - ]), - }); + await expect(processChild(0)).resolves.toMatchObject({ + status: "yielded", + }); + const { invocationId } = + (await getActiveAgentInvocationForConversation(childConversationId))!; + await expect( + getTurnRecord( + childConversationId, + getAgentInvocationTurnId(invocationId), + ), + ).resolves.toMatchObject({ + state: "paused", + resumeReason: "yield", + source: { kind: watchEvent ? "event" : "agent_invocation" }, + piMessages: expect.arrayContaining([ + expect.objectContaining({ + role: "toolResult", + toolName: "systemTime", + }), + ]), + }); + if (endBeforeWake) { + await completeAgentInvocation({ + errorMessage: "Agent invocation ended", + invocationId, + status: "failed", + }); + } - await expect(processChild()).resolves.toMatchObject({ - status: "completed", - }); - expect(run).toHaveBeenCalledTimes(watchEvent ? 3 : 2); - await expect( - getAgentInvocation(target.invocationId), - ).resolves.toMatchObject({ - result: "Resumed child result", - status: "completed", - }); - const projection = await loadProjection({ - conversationId: childConversationId, - }); - expect( - projection.filter( - (message) => - message.role === "toolResult" && message.toolName === "systemTime", - ), - ).toHaveLength(1); - } finally { - clock.mockRestore(); - await fixture.close(); - } - }); + await expect(processChild()).resolves.toMatchObject({ + status: "completed", + }); + expect(run).toHaveBeenCalledTimes(runs); + await expect(getAgentInvocation(invocationId)).resolves.toMatchObject( + expected, + ); + const projection = await loadProjection({ + conversationId: childConversationId, + }); + expect( + projection.filter( + (message) => + message.role === "toolResult" && + message.toolName === "systemTime", + ), + ).toHaveLength(1); + } finally { + await fixture.close(); + } + }, + ); const missing = "Write access to getsentry/junior. Ask the user."; it.each([ From 4b966f63fd959cc3a6f0eaca5a21f2b7f6d5b647 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:40:37 +0000 Subject: [PATCH 03/21] refactor(agent-invocations): Use one child worker in the server and CLI The server and the CLI each composed child routing from the same parts in a different order: the child input step, agent invocation lookup, and the agent invocation worker. That drift is how the server got the wrong order. createChildConversationWorker now owns that composition, and both callers run it before any root worker. It returns undefined for root work, so routeAgentInvocationWork and its fallback callback go away. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../src/chat/agent-invocations/README.md | 19 +++---- .../src/chat/agent-invocations/result.ts | 38 ++++++++++++- .../junior/src/chat/agent-invocations/work.ts | 24 --------- .../junior/src/chat/app/conversation-work.ts | 54 +++++++------------ packages/junior/src/cli/chat.ts | 31 +++++------ .../integration/agent-invocation-stop.test.ts | 27 +++++----- 6 files changed, 91 insertions(+), 102 deletions(-) diff --git a/packages/junior/src/chat/agent-invocations/README.md b/packages/junior/src/chat/agent-invocations/README.md index 93d39b2ce1..d6f4032e80 100644 --- a/packages/junior/src/chat/agent-invocations/README.md +++ b/packages/junior/src/chat/agent-invocations/README.md @@ -53,15 +53,16 @@ conversation tree. 1. Creation writes the invocation before attempting the mailbox append. 2. The idempotent mailbox append sends a normal conversation queue wake. -3. The invocation router recognizes destinationless child work and advances it - through the shared `AgentRunner`. `routeAgentInvocationWork` checks for an - agent invocation before any other worker, in the server and in the CLI. An - empty wake goes to the active agent invocation of the child. When none is - active, it goes to the latest one, which repeats its idempotent finish. - Child work never falls through to a root worker. A child Turn can have any - Source. A watch event Turn has an event Source, so a - router that picks the worker by Source sends its resume to the root Event - Turn worker. That worker needs a Destination, and a child has none. +3. `createChildConversationWorker` is the one entry point for child work. The + server and the CLI run it before any root worker. It starts agent + invocations for child input, then runs the agent invocation that owns the + work through the shared `AgentRunner`. An empty wake goes to the active + agent invocation of the child. When none is active, it goes to the latest + one, which repeats its idempotent finish. Child work never falls through to + a root worker. A child Turn can have any Source. A watch event Turn has an + event Source, so a router that picks the worker by Source sends its resume + to the root Event Turn worker. That worker needs a Destination, and a child + has none. 4. A cooperative yield keeps the same turn and invocation active for another execution slice. 5. A stranded `running` child is re-parked at its latest durable safe boundary. diff --git a/packages/junior/src/chat/agent-invocations/result.ts b/packages/junior/src/chat/agent-invocations/result.ts index e3fde0d7fe..d2d225d4fe 100644 --- a/packages/junior/src/chat/agent-invocations/result.ts +++ b/packages/junior/src/chat/agent-invocations/result.ts @@ -2,6 +2,7 @@ import type { StateAdapter } from "chat"; import type { Actor } from "@/chat/actor"; import { credentialContextForActor } from "@/chat/credentials/context"; import { SYSTEM_IDENTITY_ID } from "@/chat/identities/actor"; +import type { AgentRunner } from "@/chat/runtime/agent-runner"; import { EVENT_WAIT_MS, isEventMailboxMetadata, @@ -40,7 +41,11 @@ import type { ChildAgent, TerminalAgentInvocation, } from "./types"; -import { enqueueAgentInvocation } from "./work"; +import { + createAgentInvocationWorker, + enqueueAgentInvocation, + resolveAgentInvocationId, +} from "./work"; const AGENT_INVOCATION_EVENT_NAMESPACE = "junior"; // TODO(dcramer): Add an `agent_invocation.needs_input` Event type when a child @@ -443,6 +448,37 @@ export async function startAgentInvocationsForChildInput( return { status: "completed" }; } +/** + * Build the worker for child Conversation work. Server conversation work and + * the local CLI both run it before any root worker. It starts agent + * invocations for child input, then runs the agent invocation that owns the + * work. A child Turn can have any Source, so only SQL agent invocation state + * can select this worker. Returns undefined for root Conversation work. + */ +export function createChildConversationWorker( + agentRunner: AgentRunner, + options: { + enqueueResult?: (invocation: TerminalAgentInvocation) => Promise; + maxAgentDepth?: number; + queue: ConversationWorkQueue; + state?: StateAdapter; + }, +) { + const invocationWorker = createAgentInvocationWorker(agentRunner, options); + return async ( + context: ConversationWorkerContext, + ): Promise => { + const started = await startAgentInvocationsForChildInput(context, options); + if (started) { + return started; + } + const invocationId = await resolveAgentInvocationId(context); + return invocationId + ? await invocationWorker(context, invocationId) + : undefined; + }; +} + /** * Run each agent invocation result or message as the Actor that triggered the * child, user or system. This is on purpose: the result continues that diff --git a/packages/junior/src/chat/agent-invocations/work.ts b/packages/junior/src/chat/agent-invocations/work.ts index 03af9a2924..b0883494c9 100644 --- a/packages/junior/src/chat/agent-invocations/work.ts +++ b/packages/junior/src/chat/agent-invocations/work.ts @@ -872,27 +872,3 @@ export function createAgentInvocationWorker( return { status: "completed" }; }; } - -/** - * Route child work to its agent invocation before any other worker. Server - * conversation work and the local CLI both use this order, because a child - * Turn can have any Source and only its agent invocation can run it. - */ -export function routeAgentInvocationWork(options: { - fallbackWorker: ( - context: ConversationWorkerContext, - ) => Promise; - invocationWorker: ( - context: ConversationWorkerContext, - invocationId: string, - ) => Promise; -}) { - return async ( - context: ConversationWorkerContext, - ): Promise => { - const invocationId = await resolveAgentInvocationId(context); - return invocationId - ? await options.invocationWorker(context, invocationId) - : await options.fallbackWorker(context); - }; -} diff --git a/packages/junior/src/chat/app/conversation-work.ts b/packages/junior/src/chat/app/conversation-work.ts index 7c78228c81..2125785c1e 100644 --- a/packages/junior/src/chat/app/conversation-work.ts +++ b/packages/junior/src/chat/app/conversation-work.ts @@ -17,12 +17,8 @@ import { resolveAgentDispatchId, } from "@/chat/agent-dispatch/work"; import { - createAgentInvocationWorker, - routeAgentInvocationWork, -} from "@/chat/agent-invocations/work"; -import { + createChildConversationWorker, enqueueAgentInvocationResult, - startAgentInvocationsForChildInput, withAgentInvocationResultActors, } from "@/chat/agent-invocations/result"; import { botConfig } from "@/chat/config"; @@ -143,13 +139,15 @@ export function createConversationWork( }, runTurn: runtime.runDispatchTurn, }); - const invocationWorker = createAgentInvocationWorker(options.agentRunner, { + const childWorker = createChildConversationWorker(options.agentRunner, { enqueueResult: async (invocation) => await enqueueAgentInvocationResult(invocation, { queue: options.queue, state: options.state, }), maxAgentDepth: botConfig.maxAgentDepth, + queue: options.queue, + state: options.state, }); const conversationTurnWorker = createConversationTurnWorker( options.agentRunner, @@ -176,24 +174,6 @@ export function createConversationWork( `Conversation ${context.conversationId} has a ${destination.platform} destination but no matching conversation worker`, ); }; - // Child work must reach its agent invocation before any root worker. - const routeWork = routeAgentInvocationWork({ - invocationWorker, - fallbackWorker: async (context) => { - const mailboxTurn = await resolveMailboxTurnWork(context); - if (mailboxTurn) { - return await conversationTurnWorker( - context, - await withAgentInvocationResultActors(mailboxTurn), - ); - } - const dispatchId = await resolveAgentDispatchId(context); - if (dispatchId) { - return await dispatchWorker(context, dispatchId); - } - return await destinationWorker(context); - }, - }); const run = async ( context: ConversationWorkerContext, ): Promise => { @@ -202,17 +182,23 @@ export function createConversationWork( if (skipped) { return skipped; } - // Mailbox input for a child Conversation (a result, a message, or a watch - // event) starts an agent invocation on that child. The same input for a - // root Conversation runs as an Event Turn below. - const childInput = await startAgentInvocationsForChildInput(context, { - queue: options.queue, - state: options.state, - }); - if (childInput) { - return childInput; + // Child work must reach the child worker before any root worker. + const childWork = await childWorker(context); + if (childWork) { + return childWork; + } + const mailboxTurn = await resolveMailboxTurnWork(context); + if (mailboxTurn) { + return await conversationTurnWorker( + context, + await withAgentInvocationResultActors(mailboxTurn), + ); + } + const dispatchId = await resolveAgentDispatchId(context); + if (dispatchId) { + return await dispatchWorker(context, dispatchId); } - return await routeWork(context); + return await destinationWorker(context); }; return { conversationStore: options.conversationStore, diff --git a/packages/junior/src/cli/chat.ts b/packages/junior/src/cli/chat.ts index 6207bb5f72..29b5366b64 100644 --- a/packages/junior/src/cli/chat.ts +++ b/packages/junior/src/cli/chat.ts @@ -246,9 +246,7 @@ async function prepareLocalChatRun( await import("@/chat/local/sandbox-egress-signals"); const { bindSpawnAgent } = await import("@/chat/agent-invocations/spawn"); const { bindMessageAgent } = await import("@/chat/agent-invocations/message"); - const { createAgentInvocationWorker, routeAgentInvocationWork } = - await import("@/chat/agent-invocations/work"); - const { startAgentInvocationsForChildInput } = + const { createChildConversationWorker } = await import("@/chat/agent-invocations/result"); const { skipInputBeforeConversationStop } = await import("@/chat/conversations/session-stop"); @@ -261,26 +259,21 @@ async function prepareLocalChatRun( if (!agentRunner) { throw new Error("Local agent runner is not ready"); } - const run = routeAgentInvocationWork({ + const childWorker = createChildConversationWorker(agentRunner, { + queue: localConversationWork.queue, + }); + await processConversationWork(message, { + queue: localConversationWork.queue, // The local queue carries only child Conversation work. - fallbackWorker: async (context) => { - const skipped = await skipInputBeforeConversationStop(context); - if (skipped) { - return skipped; - } - const started = await startAgentInvocationsForChildInput(context, { - queue: localConversationWork.queue, - }); - if (!started) { + run: async (context) => { + const result = + (await skipInputBeforeConversationStop(context)) ?? + (await childWorker(context)); + if (!result) { throw new Error("Local child queue received non-invocation work"); } - return started; + return result; }, - invocationWorker: createAgentInvocationWorker(agentRunner), - }); - await processConversationWork(message, { - queue: localConversationWork.queue, - run, }); }); agentRunner = createAgentRunner(executeAgentRun, { diff --git a/packages/junior/tests/integration/agent-invocation-stop.test.ts b/packages/junior/tests/integration/agent-invocation-stop.test.ts index ccafbe9b42..14d7a60500 100644 --- a/packages/junior/tests/integration/agent-invocation-stop.test.ts +++ b/packages/junior/tests/integration/agent-invocation-stop.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { AgentRun } from "@/chat/agent/types"; import { + createChildConversationWorker, enqueueAgentInvocationResult, startAgentInvocationsForChildInput, } from "@/chat/agent-invocations/result"; @@ -20,9 +21,9 @@ import type { import { createAgentInvocationWorker, createAndEnqueueAgentInvocation, - routeAgentInvocationWork, } from "@/chat/agent-invocations/work"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; +import { neverRunAgentRunner } from "../fixtures/agent-runner"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { getConversationWorkState, @@ -266,21 +267,17 @@ describe("stopping child agents", () => { ).resolves.toMatchObject({ status: "stopped" }); // A stop request can outlive its run. The stopped agent invocation - // handles the next empty wake. - const invocationWorker = vi.fn(async () => ({ - status: "completed" as const, - })); + // handles the next empty wake, so it never reaches a root worker. const wake = await harness.context(pending.childConversationId); - await routeAgentInvocationWork({ - fallbackWorker: async () => { - throw new Error("Child work reached the fallback worker"); - }, - invocationWorker, - })({ ...wake, attempt: { ...wake.attempt, messages: [] } }); - expect(invocationWorker).toHaveBeenCalledWith( - expect.anything(), - pending.invocationId, - ); + await expect( + createChildConversationWorker(neverRunAgentRunner(), { + queue: harness.queue, + state: harness.state, + })({ ...wake, attempt: { ...wake.attempt, messages: [] } }), + ).resolves.toEqual({ status: "completed" }); + await expect( + getAgentInvocation(pending.invocationId), + ).resolves.toMatchObject({ status: "stopped" }); // A stop that does not come from stopAgent still reaches the parent, // and it also stops the child agents of the child. From 49babd3efbaf82be6d7e95081675ec47f559f10a Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:10:36 +0000 Subject: [PATCH 04/21] ref(agent-invocations): Select the child worker from the stored parent The server picked the worker for a Conversation by asking a list of questions about the work, in order. A wrong order sent a child resume to a root worker. Now the stored parentConversationId decides once: a child Conversation only reaches the child worker, and work that no agent invocation owns is an error instead of falling through. Root wakes also skip the agent invocation SQL reads. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../src/chat/agent-invocations/result.ts | 20 ++++++++++--------- .../junior/src/chat/app/conversation-work.ts | 15 ++++++++------ packages/junior/src/cli/chat.ts | 12 +++-------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/packages/junior/src/chat/agent-invocations/result.ts b/packages/junior/src/chat/agent-invocations/result.ts index d2d225d4fe..3f36cdf6a0 100644 --- a/packages/junior/src/chat/agent-invocations/result.ts +++ b/packages/junior/src/chat/agent-invocations/result.ts @@ -449,11 +449,10 @@ export async function startAgentInvocationsForChildInput( } /** - * Build the worker for child Conversation work. Server conversation work and - * the local CLI both run it before any root worker. It starts agent - * invocations for child input, then runs the agent invocation that owns the - * work. A child Turn can have any Source, so only SQL agent invocation state - * can select this worker. Returns undefined for root Conversation work. + * Build the worker for all work of a child Conversation. The caller selects + * it from the stored parent of the Conversation, never from a Turn Source. It + * starts agent invocations for child input, then runs the agent invocation + * that owns the work. Work that no agent invocation owns is an error. */ export function createChildConversationWorker( agentRunner: AgentRunner, @@ -467,15 +466,18 @@ export function createChildConversationWorker( const invocationWorker = createAgentInvocationWorker(agentRunner, options); return async ( context: ConversationWorkerContext, - ): Promise => { + ): Promise => { const started = await startAgentInvocationsForChildInput(context, options); if (started) { return started; } const invocationId = await resolveAgentInvocationId(context); - return invocationId - ? await invocationWorker(context, invocationId) - : undefined; + if (!invocationId) { + throw new Error( + `Child conversation ${context.conversationId} has no agent invocation for this work`, + ); + } + return await invocationWorker(context, invocationId); }; } diff --git a/packages/junior/src/chat/app/conversation-work.ts b/packages/junior/src/chat/app/conversation-work.ts index 2125785c1e..fa6986f3d6 100644 --- a/packages/junior/src/chat/app/conversation-work.ts +++ b/packages/junior/src/chat/app/conversation-work.ts @@ -156,8 +156,8 @@ export function createConversationWork( state: options.state, }), ); - // A Slack Destination uses the Slack worker. All other work must have been - // selected above this function. + // A Slack Destination uses the Slack worker. All other root work must have + // been selected above this function. const destinationWorker = async ( context: ConversationWorkerContext, ): Promise => { @@ -182,10 +182,13 @@ export function createConversationWork( if (skipped) { return skipped; } - // Child work must reach the child worker before any root worker. - const childWork = await childWorker(context); - if (childWork) { - return childWork; + // The stored parent decides the kind of Conversation, so child work never + // reaches a root worker. A child Turn can have any Source. + const conversation = await options.conversationStore.get({ + conversationId: context.conversationId, + }); + if (conversation?.parentConversationId) { + return await childWorker(context); } const mailboxTurn = await resolveMailboxTurnWork(context); if (mailboxTurn) { diff --git a/packages/junior/src/cli/chat.ts b/packages/junior/src/cli/chat.ts index 29b5366b64..906c85e878 100644 --- a/packages/junior/src/cli/chat.ts +++ b/packages/junior/src/cli/chat.ts @@ -265,15 +265,9 @@ async function prepareLocalChatRun( await processConversationWork(message, { queue: localConversationWork.queue, // The local queue carries only child Conversation work. - run: async (context) => { - const result = - (await skipInputBeforeConversationStop(context)) ?? - (await childWorker(context)); - if (!result) { - throw new Error("Local child queue received non-invocation work"); - } - return result; - }, + run: async (context) => + (await skipInputBeforeConversationStop(context)) ?? + (await childWorker(context)), }); }); agentRunner = createAgentRunner(executeAgentRun, { From 6afc5beda743e84418ed69cc82d191efc6c9c127 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:10:40 +0000 Subject: [PATCH 05/21] fix(task-execution): Mark the last resume attempt as final isFinalAttempt came only from message delivery counts, so a resume attempt with no messages was never final. When a resumed Turn kept failing, the worker gave up without telling the adapter. A child agent then stayed paused, the parent was never told, and later results of that child were held back as replaced. The retry count now also marks the last attempt as final. On that attempt the agent invocation worker fails the invocation for any error, also a save error without an AgentRunError wrapper, and reports it to the parent. Root Turns and dispatches get the same final attempt. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../src/chat/agent-invocations/README.md | 27 +++++++------ .../junior/src/chat/agent-invocations/work.ts | 7 ++-- .../junior/src/chat/task-execution/README.md | 5 +++ .../junior/src/chat/task-execution/worker.ts | 25 +++++++++++- .../integration/agent-invocation-work.test.ts | 38 +++++++++++++++++-- 5 files changed, 81 insertions(+), 21 deletions(-) diff --git a/packages/junior/src/chat/agent-invocations/README.md b/packages/junior/src/chat/agent-invocations/README.md index d6f4032e80..efe57ccb21 100644 --- a/packages/junior/src/chat/agent-invocations/README.md +++ b/packages/junior/src/chat/agent-invocations/README.md @@ -53,22 +53,25 @@ conversation tree. 1. Creation writes the invocation before attempting the mailbox append. 2. The idempotent mailbox append sends a normal conversation queue wake. -3. `createChildConversationWorker` is the one entry point for child work. The - server and the CLI run it before any root worker. It starts agent - invocations for child input, then runs the agent invocation that owns the - work through the shared `AgentRunner`. An empty wake goes to the active - agent invocation of the child. When none is active, it goes to the latest - one, which repeats its idempotent finish. Child work never falls through to - a root worker. A child Turn can have any Source. A watch event Turn has an - event Source, so a router that picks the worker by Source sends its resume - to the root Event Turn worker. That worker needs a Destination, and a child - has none. +3. `createChildConversationWorker` runs all work of a child Conversation. The + server and the CLI select it from the stored `parentConversationId` of the + Conversation, never from a Turn Source or mailbox metadata. A child Turn can + have any Source. A watch event Turn has an event Source, and the root Event + Turn worker needs a Destination that a child does not have. The worker + starts agent invocations for child input, then runs the agent invocation + that owns the work through the shared `AgentRunner`. An empty wake goes to + the active agent invocation of the child. When none is active, it goes to + the latest one, which repeats its idempotent finish. Work that no agent + invocation owns is an error. It never falls through to a root worker. 4. A cooperative yield keeps the same turn and invocation active for another execution slice. 5. A stranded `running` child is re-parked at its latest durable safe boundary. If no model or resumable boundary remains, the session fails immediately and - projects onto the invocation. Empty resume wakes never become final delivery - attempts, so unrecoverable stranded work must not throw and requeue forever. + projects onto the invocation. The conversation worker marks the last attempt + before it gives up as final, also for an empty resume wake. On that attempt + any error fails the agent invocation and reports it to the parent. Without + this, the work stops and the agent invocation stays open, so the parent is + never told and later results of the child are held back as `replaced`. 6. Completion writes the session record first, then projects the immutable result or error onto the agent invocation. The same SQL write sets `parentMailboxStatus: "pending"`. diff --git a/packages/junior/src/chat/agent-invocations/work.ts b/packages/junior/src/chat/agent-invocations/work.ts index b0883494c9..29fdf49a05 100644 --- a/packages/junior/src/chat/agent-invocations/work.ts +++ b/packages/junior/src/chat/agent-invocations/work.ts @@ -790,10 +790,9 @@ export function createAgentInvocationWorker( }), ); } catch (error) { - if (!(error instanceof AgentRunError)) { - throw error; - } - const runError = error.cause; + // A save error has no AgentRunError wrapper. On the final attempt it + // must also fail the agent invocation, or the child stays open. + const runError = error instanceof AgentRunError ? error.cause : error; if (isInvocationInputCommitLost(runError)) { return { status: "lost_lease" }; } diff --git a/packages/junior/src/chat/task-execution/README.md b/packages/junior/src/chat/task-execution/README.md index 1bf6bf8618..a8b3a84248 100644 --- a/packages/junior/src/chat/task-execution/README.md +++ b/packages/junior/src/chat/task-execution/README.md @@ -139,6 +139,11 @@ new turn starts, `interrupt` delivery is handled before queued `defer` delivery. possible. - Delivery failures leave enough durable state for safe retry and must not mark the turn delivered. +- The worker gives up after its retry limit and drops the mailbox. No code + runs for that work after that. So `isFinalAttempt` is true on the last + attempt that can fail, including a resume attempt with no messages. On that + attempt each worker records its own failure: a fallback reply, a failed + dispatch, or a failed agent invocation that reports to its parent. ## Integration Contract diff --git a/packages/junior/src/chat/task-execution/worker.ts b/packages/junior/src/chat/task-execution/worker.ts index 4c472587e5..5934121602 100644 --- a/packages/junior/src/chat/task-execution/worker.ts +++ b/packages/junior/src/chat/task-execution/worker.ts @@ -71,6 +71,11 @@ export interface InboxAttempt { handle: (messages: InboundMessage[]) => Promise, options?: { deferred?: boolean }, ): Promise; + /** + * True when a failure of this attempt can end the work: the worker gives up + * and drops the mailbox. The adapter must record its own failure first. + * Resume attempts, which have no messages, can also be final. + */ isFinalAttempt: boolean; messages: InboundMessage[]; } @@ -226,6 +231,21 @@ function isTerminalFailure(failure: AttemptFailure): boolean { ); } +/** + * Return whether a failure of this attempt can end the conversation work. The + * retry limit applies to every attempt, also one with no messages, such as a + * resume. A message also ends it at its own delivery limit. + */ +function isLastConversationAttempt( + work: ConversationWorkState, + messages: readonly InboundMessage[], +): boolean { + return ( + (work.execution.retryCount ?? 0) + 1 >= CONVERSATION_WORK_MAX_RETRIES || + messages.some((message) => isFinalAttempt(message)) + ); +} + function logRetryExhausted(args: { error: unknown; retryCount: number; @@ -636,8 +656,9 @@ async function processConversationWorkInContext( conversationId, destination, drain, - isFinalAttempt: attemptMessages.some((message) => - isFinalAttempt(message), + isFinalAttempt: isLastConversationAttempt( + leasedWork, + attemptMessages, ), messages: attemptMessages, }, diff --git a/packages/junior/tests/integration/agent-invocation-work.test.ts b/packages/junior/tests/integration/agent-invocation-work.test.ts index 436de3611a..f2e96335ae 100644 --- a/packages/junior/tests/integration/agent-invocation-work.test.ts +++ b/packages/junior/tests/integration/agent-invocation-work.test.ts @@ -13,7 +13,10 @@ import { EVENT_MESSAGE_AUTHOR, EVENT_SYSTEM_ACTOR } from "@/chat/events/actor"; import { getEventCatalog } from "@/chat/events/runtime-catalog"; import { runTimerWatchHeartbeat } from "@/chat/events/timers"; import { createEventTools } from "@/chat/tools/events"; -import { getConversationWorkState } from "@/chat/task-execution/store"; +import { + CONVERSATION_WORK_MAX_RETRIES, + getConversationWorkState, +} from "@/chat/task-execution/store"; import { createAndEnqueueAgentInvocation } from "@/chat/agent-invocations/work"; import { loadProjection } from "@/chat/conversations/projection"; import { disconnectStateAdapter } from "@/chat/state/adapter"; @@ -391,7 +394,8 @@ describe("agent invocation conversation work", () => { // next wake must still reach the agent invocation. Production once sent it // to the root Event Turn worker, which needs a Destination that a child // does not have. That also holds when the agent invocation ended while its - // Turn stayed paused. + // Turn stayed paused. A resume wake has no messages, but its last try is + // still final, so a resume that keeps failing tells the parent. const resumed = { result: "Resumed child result", status: "completed" }; it.each([ { @@ -415,9 +419,27 @@ describe("agent invocation conversation work", () => { runs: 2, expected: { parentMailboxStatus: "appended", status: "failed" }, }, + { + name: "a resume that keeps failing", + watchEvent: false, + // The last try is final, so it fails the invocation, not the work. + failedResumes: CONVERSATION_WORK_MAX_RETRIES - 1, + runs: CONVERSATION_WORK_MAX_RETRIES + 1, + expected: { + errorMessage: "resume unavailable", + parentMailboxStatus: "appended", + status: "failed", + }, + }, ])( "gives the wake after a yielded child invocation for $name to that invocation", - async ({ endBeforeWake, expected, runs, watchEvent }) => { + async ({ + endBeforeWake, + expected, + failedResumes = 0, + runs, + watchEvent, + }) => { const clock = vi.spyOn(Date, "now").mockReturnValue(1_800_000_000_000); const { conversationStore, fixture } = await prepareParentConversation(); const agentRunner = createModelAgentRunner( @@ -504,6 +526,16 @@ describe("agent invocation conversation work", () => { status: "failed", }); } + if (failedResumes > 0) { + run.mockRejectedValue(new Error("resume unavailable")); + } + for (let retry = 0; retry < failedResumes; retry++) { + await expect(processChild()).resolves.toMatchObject({ + status: "failed", + }); + // Each retry wake has a new idempotency key from the clock. + clock.mockReturnValue(clock() + 1_000); + } await expect(processChild()).resolves.toMatchObject({ status: "completed", From 0b2d66b78071b0511b818a17a324f4f90713c445 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:09:11 +0000 Subject: [PATCH 06/21] test(agent-invocations): Run child routing tests through production work The message, nested, and stop tests built worker contexts by hand and called startAgentInvocationsForChildInput, createAgentInvocationWorker, and createChildConversationWorker directly. They did not run the production conversation work that selects the child worker. They now process queue wakes through createConversationWorkHarness, which moves to fixtures/conversation-work and wraps processConversationQueueMessage. Stops use the real stop request poll and stopConversationTurn instead of a local AbortController. Runners only read or abort runs, because tests in packages/junior/tests must not run the agent. The nested report case in the spawn test duplicated the nested test, so it is removed. The maxAgentDepth check moves to the component worker test, which owns that option. The stopped empty resume wake is now a case of the yielded child wake test. startAgentInvocationsForChildInput has no callers outside result.ts, so it is no longer exported. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../src/chat/agent-invocations/result.ts | 2 +- .../component/agent-invocation-worker.test.ts | 53 ++++- .../junior/tests/fixtures/agent-dispatch.ts | 26 --- .../tests/fixtures/conversation-work.ts | 69 ++++-- .../agent-dispatch-recovery.test.ts | 6 +- .../integration/agent-dispatch-work.test.ts | 2 +- .../agent-invocation-concurrency.test.ts | 2 +- .../agent-invocation-message.test.ts | 99 ++++---- .../agent-invocation-nested.test.ts | 214 ++++++------------ .../agent-invocation-spawn.test.ts | 156 +------------ .../integration/agent-invocation-stop.test.ts | 178 ++++++--------- .../integration/agent-invocation-work.test.ts | 26 ++- scripts/test-architecture-baseline.json | 3 +- 13 files changed, 321 insertions(+), 515 deletions(-) diff --git a/packages/junior/src/chat/agent-invocations/result.ts b/packages/junior/src/chat/agent-invocations/result.ts index 3f36cdf6a0..07c0d2f094 100644 --- a/packages/junior/src/chat/agent-invocations/result.ts +++ b/packages/junior/src/chat/agent-invocations/result.ts @@ -365,7 +365,7 @@ async function getChildAgentInputInvocation( * reports to the child's parent. Returns undefined for work that is not * mailbox input for a child Conversation. */ -export async function startAgentInvocationsForChildInput( +async function startAgentInvocationsForChildInput( context: ConversationWorkerContext, options: { queue: ConversationWorkQueue; state?: StateAdapter }, ): Promise { diff --git a/packages/junior/tests/component/agent-invocation-worker.test.ts b/packages/junior/tests/component/agent-invocation-worker.test.ts index 22513c8ca2..0b491dafc7 100644 --- a/packages/junior/tests/component/agent-invocation-worker.test.ts +++ b/packages/junior/tests/component/agent-invocation-worker.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { PiMessage } from "@/chat/pi/messages"; import { buildSteeringPiMessage } from "@/chat/agent/prompt"; -import type { AgentSteeringMessage } from "@/chat/agent/types"; +import type { AgentRun, AgentSteeringMessage } from "@/chat/agent/types"; import { agentInstructionProvenance } from "@/chat/conversations/provenance"; import { createAgentInvocation, @@ -523,6 +523,57 @@ describe("agent invocation worker", () => { } }); + it("lets child agents spawn only down to maxAgentDepth", async () => { + const fixture = await prepareParentConversation(); + try { + const child = await createAgentInvocation({ + ...INVOCATION_INPUT, + idempotencyKey: "depth-child", + }); + const helper = await createAgentInvocation({ + ...INVOCATION_INPUT, + idempotencyKey: "depth-helper", + parentConversationId: child.childConversationId, + }); + const runs: AgentRun[] = []; + const worker = createAgentInvocationWorker( + { + run: async (run) => { + runs.push(run); + throw new Error("stop after reading the run"); + }, + }, + { maxAgentDepth: 1 }, + ); + // With maxAgentDepth 1, depth 1 can spawn and depth 2 cannot. + for (const invocation of [child, helper]) { + await expect( + worker( + { + attempt: { + ack: vi.fn(), + conversationId: invocation.childConversationId, + drain: vi.fn(), + isFinalAttempt: false, + messages: [buildAgentInvocationInboundMessage(invocation)], + }, + checkIn: vi.fn(), + conversationId: invocation.childConversationId, + shouldYield: () => false, + }, + invocation.invocationId, + ), + ).rejects.toThrow("stop after reading the run"); + } + expect(runs.map((run) => run.disabledFeatures)).toEqual([ + ["handoff", "interactive-auth"], + ["handoff", "interactive-auth", "subagents"], + ]); + } finally { + await fixture.close(); + } + }); + it("keeps unexpected runner failures retryable until the final attempt", async () => { const fixture = await prepareParentConversation(); try { diff --git a/packages/junior/tests/fixtures/agent-dispatch.ts b/packages/junior/tests/fixtures/agent-dispatch.ts index cb917e680c..5c4b6aea97 100644 --- a/packages/junior/tests/fixtures/agent-dispatch.ts +++ b/packages/junior/tests/fixtures/agent-dispatch.ts @@ -5,13 +5,7 @@ import type { } from "@sentry/junior-plugin-api"; import { createOrGetDispatch } from "@/chat/agent-dispatch/store"; import type { BoundDispatchOptions } from "@/chat/agent-dispatch/types"; -import { createConversationWork } from "@/chat/app/conversation-work"; -import { createJuniorSlackAdapter } from "@/chat/slack/adapter"; import type { CredentialSubject } from "@/chat/credentials/context"; -import type { AgentRunner } from "@/chat/runtime/agent-runner"; -import { getConversationStore } from "@/chat/db"; -import { getStateAdapter } from "@/chat/state/adapter"; -import { createConversationWorkQueueTestAdapter } from "./conversation-work"; export const agentDispatchTestDestination = { platform: "slack", @@ -19,26 +13,6 @@ export const agentDispatchTestDestination = { channelId: "C123", } as const; -/** Compose production conversation work for one integration test. */ -export async function createConversationWorkHarness(agentRunner: AgentRunner) { - const state = getStateAdapter(); - await state.connect(); - const queue = createConversationWorkQueueTestAdapter(); - const adapter = createJuniorSlackAdapter({ - botToken: "xoxb-test", - botUserId: "U0BOT", - signingSecret: "test-signing-secret", - }); - const work = createConversationWork({ - agentRunner, - conversationStore: getConversationStore(), - getSlackAdapter: () => adapter, - queue, - state, - }); - return { queue, run: work.run, runtime: work.runtime, state }; -} - /** Create a durable dispatch record for integration tests. */ export async function createAgentDispatchTestRecord( idempotencyKey: string, diff --git a/packages/junior/tests/fixtures/conversation-work.ts b/packages/junior/tests/fixtures/conversation-work.ts index fc7fb47d32..b4cd04b75a 100644 --- a/packages/junior/tests/fixtures/conversation-work.ts +++ b/packages/junior/tests/fixtures/conversation-work.ts @@ -9,6 +9,7 @@ import { } from "@earendil-works/pi-ai/providers/faux"; import type { Destination } from "@sentry/junior-plugin-api"; import { createConversationWork } from "@/chat/app/conversation-work"; +import type { JuniorRuntimeServiceOverrides } from "@/chat/app/services"; import { executeAgentRun } from "@/chat/agent"; import { getConversationStore } from "@/chat/db"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; @@ -448,6 +449,48 @@ export type ConversationWorkSlackHarness = { }) => Promise; }; +/** + * Compose production conversation work once for one test. `process` runs one + * queue wake through the production worker, as the Vercel queue callback does. + * Pass `services` only to fake a model boundary that the work does not own. + */ +export async function createConversationWorkHarness( + agentRunner: AgentRunner, + options: { services?: JuniorRuntimeServiceOverrides } = {}, +) { + const state = getStateAdapter(); + await state.connect(); + const queue = createConversationWorkQueueTestAdapter(); + const adapter = createSlackAdapterFixture(); + const conversationStore = getConversationStore(); + const work = createConversationWork({ + agentRunner, + conversationStore, + getSlackAdapter: () => adapter, + queue, + ...(options.services ? { services: options.services } : undefined), + state, + }); + return { + adapter, + process: async ( + message: ConversationQueueMessage, + processOptions: { softYieldAfterMs?: number } = {}, + ) => + await processConversationQueueMessage(message, { + conversationStore, + queue, + run: work.run, + state, + ...processOptions, + }), + queue, + run: work.run, + runtime: work.runtime, + state, + }; +} + /** * Compose the production Slack ingress → durable queue → worker → agent path. * Fake only model generation at the executeAgentRun stream boundary and Slack @@ -459,19 +502,11 @@ export async function createConversationWorkSlackHarness( modelStream?: StreamFn; } = {}, ): Promise { - const state = getStateAdapter(); - await state.connect(); - const wakes = createConversationWorkQueueTestAdapter(); - const adapter = createSlackAdapterFixture(); const modelStream = options.modelStream ?? streamReplies("Deploy checked."); const agentRunner: AgentRunner = options.agentRunner ?? { run: async (request) => await executeAgentRun(request, modelStream), }; - const work = createConversationWork({ - agentRunner, - conversationStore: getConversationStore(), - getSlackAdapter: () => adapter, - queue: wakes, + const work = await createConversationWorkHarness(agentRunner, { services: { agentRunner, subscribedReplyPolicy: { @@ -483,8 +518,9 @@ export async function createConversationWorkSlackHarness( listThreadReplies: async () => [], }, }, - state, }); + const { adapter, state } = work; + const wakes = work.queue; const services = { getSlackAdapter: () => adapter, @@ -503,23 +539,14 @@ export async function createConversationWorkSlackHarness( .map((call) => call.params.text) .filter((text): text is string => typeof text === "string"), next: async (requestStartedAtMs?: number) => { - const process = async () => - await processConversationQueueMessage(wakes.takeMessage(), { - queue: wakes, - run: work.run, - state, - }); + const process = async () => await work.process(wakes.takeMessage()); return requestStartedAtMs === undefined ? await process() : await runWithTurnRequestDeadline(process, requestStartedAtMs); }, drain: async () => { for (let i = 0; i < 12 && wakes.hasQueuedMessages(); i += 1) { - await processConversationQueueMessage(wakes.takeMessage(), { - queue: wakes, - run: work.run, - state, - }); + await work.process(wakes.takeMessage()); } if (wakes.hasQueuedMessages()) { throw new Error("queue still has work after drain"); diff --git a/packages/junior/tests/integration/agent-dispatch-recovery.test.ts b/packages/junior/tests/integration/agent-dispatch-recovery.test.ts index 1e87224fcf..402723061f 100644 --- a/packages/junior/tests/integration/agent-dispatch-recovery.test.ts +++ b/packages/junior/tests/integration/agent-dispatch-recovery.test.ts @@ -8,7 +8,10 @@ import { import { recoverPendingDispatchMailboxAppends } from "@/chat/agent-dispatch/heartbeat"; import { enqueueAgentDispatch } from "@/chat/agent-dispatch/work"; import { disconnectStateAdapter } from "@/chat/state/adapter"; -import { createConversationWorkQueueTestAdapter } from "../fixtures/conversation-work"; +import { + createConversationWorkHarness, + createConversationWorkQueueTestAdapter, +} from "../fixtures/conversation-work"; import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; import { getTurnRecord } from "@/chat/task-execution/turn-cursor"; import { slackApiOutbox } from "../fixtures/slack-api-outbox"; @@ -21,7 +24,6 @@ import { createModelStream } from "../fixtures/model-stream"; import { agentDispatchTestDestination as destination, createAgentDispatchTestRecord as createDispatch, - createConversationWorkHarness, } from "../fixtures/agent-dispatch"; vi.hoisted(() => { diff --git a/packages/junior/tests/integration/agent-dispatch-work.test.ts b/packages/junior/tests/integration/agent-dispatch-work.test.ts index 3284b863db..ac6dec8ea2 100644 --- a/packages/junior/tests/integration/agent-dispatch-work.test.ts +++ b/packages/junior/tests/integration/agent-dispatch-work.test.ts @@ -31,8 +31,8 @@ import { import { agentDispatchTestDestination as destination, createAgentDispatchTestRecord as createDispatch, - createConversationWorkHarness, } from "../fixtures/agent-dispatch"; +import { createConversationWorkHarness } from "../fixtures/conversation-work"; vi.hoisted(() => { process.env.JUNIOR_STATE_ADAPTER = "memory"; diff --git a/packages/junior/tests/integration/agent-invocation-concurrency.test.ts b/packages/junior/tests/integration/agent-invocation-concurrency.test.ts index 73bcf3ec6d..b5cf833c31 100644 --- a/packages/junior/tests/integration/agent-invocation-concurrency.test.ts +++ b/packages/junior/tests/integration/agent-invocation-concurrency.test.ts @@ -29,7 +29,7 @@ import { deferred } from "../fixtures/conversation-work"; import { createConfiguredJuniorSqlFixture } from "../fixtures/sql"; import { createModelAgentRunnerForRun } from "../fixtures/agent-runner"; import { createModelStream } from "../fixtures/model-stream"; -import { createConversationWorkHarness } from "../fixtures/agent-dispatch"; +import { createConversationWorkHarness } from "../fixtures/conversation-work"; const parentConversationId = "local:test:agent-invocation-concurrency"; const destination = { diff --git a/packages/junior/tests/integration/agent-invocation-message.test.ts b/packages/junior/tests/integration/agent-invocation-message.test.ts index 2ac8ffe347..73cb8c7da5 100644 --- a/packages/junior/tests/integration/agent-invocation-message.test.ts +++ b/packages/junior/tests/integration/agent-invocation-message.test.ts @@ -1,9 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { bindMessageAgent } from "@/chat/agent-invocations/message"; -import { - startAgentInvocationsForChildInput, - withAgentInvocationResultActors, -} from "@/chat/agent-invocations/result"; import { completeAgentInvocation, createAgentInvocation, @@ -14,18 +10,17 @@ import { } from "@/chat/agent-invocations/store"; import type { AgentInvocation } from "@/chat/agent-invocations/types"; import type { AgentRun } from "@/chat/agent/types"; -import { resolveMailboxTurnWork } from "@/chat/task-execution/mailbox-turn"; import { createSlackSource } from "@sentry/junior-plugin-api"; import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { getConversationWorkState, type InboundMessage, } from "@/chat/task-execution/store"; -import type { ConversationWorkerContext } from "@/chat/task-execution/worker"; import { createListAgentsTool } from "@/chat/tools/runtime/list-agents"; import { createMessageAgentTool } from "@/chat/tools/runtime/message-agent"; import { stopChildAgentInvocations } from "@/chat/agent-invocations/stop"; -import { createConversationWorkQueueTestAdapter } from "../fixtures/conversation-work"; +import { neverRunAgentRunner } from "../fixtures/agent-runner"; +import { createConversationWorkHarness } from "../fixtures/conversation-work"; import { prepareSlackParentConversation, slackDestination, @@ -75,21 +70,20 @@ async function eventMessages( return work?.messages.filter((message) => message.source === "event") ?? []; } -function context( - conversationId: string, - messages: InboundMessage[], -): ConversationWorkerContext { +/** + * Integration tests do not run the agent. This runner reads each run and + * stops it, so the work retries. + */ +function readRunsAgentRunner() { + const runs: AgentRun[] = []; return { - attempt: { - ack: async () => undefined, - conversationId, - drain: async () => [], - isFinalAttempt: false, - messages, + agentRunner: { + run: async (run: AgentRun): Promise => { + runs.push(run); + throw new Error("stop after reading the run"); + }, }, - checkIn: async () => true, - conversationId, - shouldYield: () => false, + runs, }; } @@ -101,9 +95,9 @@ describe("messageAgent", () => { it("gives a busy child more work and lets a child message its parent", async () => { const { fixture } = await prepareSlackParentConversation(); - const queue = createConversationWorkQueueTestAdapter(); - const state = getStateAdapter(); - await state.connect(); + const parent = readRunsAgentRunner(); + const harness = await createConversationWorkHarness(parent.agentRunner); + const { queue, state } = harness; const mailbox = async (conversationId: string) => (await getConversationWorkState({ conversationId, state }))?.messages ?? []; @@ -188,15 +182,13 @@ describe("messageAgent", () => { expect(parentMessages[0]!.input.text).toContain( "Found the bug in the retry path.", ); - const turn = await withAgentInvocationResultActors( - (await resolveMailboxTurnWork( - context(slackParentConversationId, parentMessages), - ))!, - ); - expect(turn).toMatchObject({ - batch: [{ actor: parentActor }], - kind: "mailbox", - }); + // The parent Turn runs after the Event burst window. + vi.spyOn(Date, "now").mockReturnValue(Date.now() + 60_000); + await harness.process(queue.takeMessageFor(slackParentConversationId)); + vi.mocked(Date.now).mockRestore(); + expect(parent.runs).toEqual([ + expect.objectContaining({ actor: parentActor }), + ]); // The first work finishes. The queued message reports for both. await expect( @@ -226,9 +218,9 @@ describe("messageAgent", () => { it("starts a new agent invocation when a grandchild messages a child, even after a stop", async () => { const { fixture } = await prepareSlackParentConversation(); - const queue = createConversationWorkQueueTestAdapter(); - const state = getStateAdapter(); - await state.connect(); + const runner = readRunsAgentRunner(); + const harness = await createConversationWorkHarness(runner.agentRunner); + const { queue, state } = harness; try { const child = await createAgentInvocation({ ...slackInvocationInput, @@ -256,25 +248,20 @@ describe("messageAgent", () => { childConversationId: helper.childConversationId, parentConversationId: child.childConversationId, }); - // A retried delivery starts the same agent invocation. - for (let attempt = 0; attempt < 2; attempt++) { - await expect( - startAgentInvocationsForChildInput( - context(child.childConversationId, messages), - { queue, state }, - ), - ).resolves.toEqual({ status: "completed" }); - } - await expect( - getAgentInvocation( - getAgentInvocationIdForMessage(messages[0]!.inboundMessageId), - ), - ).resolves.toMatchObject({ + // The message starts and runs a new agent invocation on the child. + await harness.process(queue.takeMessageFor(child.childConversationId)); + const started = await getAgentInvocation( + getAgentInvocationIdForMessage(messages[0]!.inboundMessageId), + ); + expect(started).toMatchObject({ childConversationId: child.childConversationId, input: expect.stringContaining("Halfway there."), parentConversationId: slackParentConversationId, - status: "pending", + status: "running", }); + expect(runner.runs).toEqual([ + expect.objectContaining({ runId: started!.invocationId }), + ]); } finally { await fixture.close(); } @@ -282,9 +269,8 @@ describe("messageAgent", () => { it("drops a grandchild message when the tree of its parent is stopped", async () => { const { fixture } = await prepareSlackParentConversation(); - const queue = createConversationWorkQueueTestAdapter(); - const state = getStateAdapter(); - await state.connect(); + const harness = await createConversationWorkHarness(neverRunAgentRunner()); + const { queue, state } = harness; try { const child = await createAgentInvocation({ ...slackInvocationInput, @@ -327,11 +313,8 @@ describe("messageAgent", () => { const messages = await eventMessages(child.childConversationId); expect(messages).toHaveLength(1); await expect( - startAgentInvocationsForChildInput( - context(child.childConversationId, messages), - { queue, state }, - ), - ).resolves.toEqual({ status: "completed" }); + harness.process(queue.takeMessageFor(child.childConversationId)), + ).resolves.toMatchObject({ status: "completed" }); await expect( getAgentInvocation( getAgentInvocationIdForMessage(messages[0]!.inboundMessageId), diff --git a/packages/junior/tests/integration/agent-invocation-nested.test.ts b/packages/junior/tests/integration/agent-invocation-nested.test.ts index 963fdabb2a..cc7808aa31 100644 --- a/packages/junior/tests/integration/agent-invocation-nested.test.ts +++ b/packages/junior/tests/integration/agent-invocation-nested.test.ts @@ -6,63 +6,18 @@ import { getAgentInvocationIdForResult, getAgentInvocationResultMessageId, } from "@/chat/agent-invocations/store"; -import { - enqueueAgentInvocationResult, - startAgentInvocationsForChildInput, -} from "@/chat/agent-invocations/result"; +import { enqueueAgentInvocationResult } from "@/chat/agent-invocations/result"; import type { TerminalAgentInvocation } from "@/chat/agent-invocations/types"; -import { createAgentInvocationWorker } from "@/chat/agent-invocations/work"; import type { AgentRun } from "@/chat/agent/types"; -import { migrateSchema } from "@/chat/conversations/sql/migrations"; -import { createSqlStore } from "@/chat/conversations/sql/store"; -import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; -import { - getConversationWorkState, - type InboundMessage, -} from "@/chat/task-execution/store"; -import type { ConversationWorkerContext } from "@/chat/task-execution/worker"; +import { disconnectStateAdapter } from "@/chat/state/adapter"; +import { getConversationWorkState } from "@/chat/task-execution/store"; import { createListAgentsTool } from "@/chat/tools/runtime/list-agents"; -import { createSlackSource } from "@sentry/junior-plugin-api"; -import { createConversationWorkQueueTestAdapter } from "../fixtures/conversation-work"; -import { createConfiguredJuniorSqlFixture } from "../fixtures/sql"; - -const slackParentConversationId = "slack:C123:1712345.0001"; -const slackDestination = { - platform: "slack", - teamId: "T123", - channelId: "C123", -} as const; -const slackInvocationInput = { - actor: { - platform: "slack", - teamId: "T123", - userId: "U123", - } as const, - destination: slackDestination, - input: "Summarize the durable task.", - parentConversationId: slackParentConversationId, - reasoningLevel: "medium" as const, -}; - -async function prepareSlackParentConversation() { - const fixture = createConfiguredJuniorSqlFixture(); - await migrateSchema(fixture.sql); - const conversationStore = createSqlStore(fixture.sql); - await conversationStore.recordActivity({ - conversationId: slackParentConversationId, - destination: slackDestination, - nowMs: 1_000, - sessionSource: createSlackSource({ - channelId: "C123", - teamId: "T123", - threadTs: "1712345.0001", - visibility: "public", - }), - source: "slack", - visibility: "public", - }); - return { fixture }; -} +import { createConversationWorkHarness } from "../fixtures/conversation-work"; +import { + prepareSlackParentConversation, + slackInvocationInput, + slackParentConversationId, +} from "../fixtures/agent-invocations"; async function listAgents(conversationId: string) { return (await createListAgentsTool({ conversationId }).execute!({}, {})) @@ -77,24 +32,16 @@ describe("nested agent invocations", () => { it("sends each nested child's report to its parent once its own children finish", async () => { const { fixture } = await prepareSlackParentConversation(); - const queue = createConversationWorkQueueTestAdapter(); - const state = getStateAdapter(); - await state.connect(); - const context = ( - conversationId: string, - messages: InboundMessage[] = [], - ): ConversationWorkerContext => ({ - attempt: { - ack: async () => undefined, - conversationId, - drain: async () => [], - isFinalAttempt: false, - messages, + // Integration tests do not run the agent. The runner reads each run and + // stops it, so its work retries until the test finishes the invocation. + const runs: AgentRun[] = []; + const harness = await createConversationWorkHarness({ + run: async (run) => { + runs.push(run); + throw new Error("stop after reading the run"); }, - checkIn: async () => true, - conversationId, - shouldYield: () => false, }); + const { queue, state } = harness; const mailbox = async (conversationId: string) => (await getConversationWorkState({ conversationId, state }))?.messages ?? []; @@ -104,29 +51,28 @@ describe("nested agent invocations", () => { result, status: "completed", })) as TerminalAgentInvocation; - // Send one result to its parent child, as the worker does, and return the - // agent invocation that the result starts there. - const sendToChild = async (result: TerminalAgentInvocation) => { - await enqueueAgentInvocationResult(result, { queue, state }); - const messages = (await mailbox(result.parentConversationId)).filter( - (message) => - message.inboundMessageId === - getAgentInvocationResultMessageId(result.invocationId), - ); - expect(messages).toHaveLength(1); - // A retried delivery starts the same agent invocation. - for (let attempt = 0; attempt < 2; attempt++) { - await expect( - startAgentInvocationsForChildInput( - context(result.parentConversationId, messages), - { queue, state }, - ), - ).resolves.toEqual({ status: "completed" }); - } + // Production work gives a result to the parent child. That starts a new + // agent invocation there and runs it. + const deliver = async (result: TerminalAgentInvocation) => { + await expect( + harness.process(queue.takeMessageFor(result.parentConversationId)), + ).resolves.toMatchObject({ status: "failed" }); return (await getAgentInvocation( getAgentInvocationIdForResult(result.invocationId), ))!; }; + // Finish the run of one agent invocation. Its retry wake reports the + // result to the parent. + const finish = async ( + invocation: { childConversationId: string; invocationId: string }, + result: string, + ) => { + const terminal = await complete(invocation.invocationId, result); + await expect( + harness.process(queue.takeMessageFor(invocation.childConversationId)), + ).resolves.toMatchObject({ status: "completed" }); + return terminal; + }; try { const child = await createAgentInvocation({ ...slackInvocationInput, @@ -143,57 +89,21 @@ describe("nested agent invocations", () => { parentConversationId: helper.childConversationId, }); - // With maxAgentDepth 1, depth 1 can spawn and depth 2 cannot. Both read - // the root Location. - const runs: AgentRun[] = []; - const worker = createAgentInvocationWorker( - { - run: async (run) => { - runs.push(run); - throw new Error("stop after reading the run"); - }, - }, - { maxAgentDepth: 1 }, - ); - for (const invocation of [child, helper]) { - await expect( - worker( - context(invocation.childConversationId), - invocation.invocationId, - ), - ).rejects.toThrow("stop after reading the run"); - } - expect(runs.map((run) => run.disabledFeatures)).toEqual([ - ["handoff", "interactive-auth"], - ["handoff", "interactive-auth", "subagents"], - ]); - expect(runs[1]?.location).toMatchObject({ - provider: "slack", - channelId: "C123", - }); - // A child that finishes while its own child runs does not report yet. for (const invocation of [child, helper]) { await expect( complete(invocation.invocationId, "Started a helper."), ).resolves.toMatchObject({ parentMailboxStatus: "replaced" }); } - const grandResult = await complete( grandHelper.invocationId, "Grand helper fact: 42.", ); expect(grandResult).toMatchObject({ parentMailboxStatus: "pending" }); - const helperContinuation = await sendToChild(grandResult); - expect(helperContinuation).toMatchObject({ - actor: slackInvocationInput.actor, - childConversationId: helper.childConversationId, - input: expect.stringContaining("Grand helper fact: 42."), - parentConversationId: child.childConversationId, - status: "pending", - }); + await enqueueAgentInvocationResult(grandResult, { queue, state }); + // listAgents shows the state of open work. A `replaced` child still - // has work, and an agent invocation that receives a result is not a task. + // has work. const listedChild = (childConversationId: string) => ({ conversation_id: childConversationId, relation: "child", @@ -208,20 +118,44 @@ describe("nested agent invocations", () => { listedChild(helper.childConversationId), ]); + // The result runs on the helper as the Actor that started the tree, + // with the root Location. + const helperContinuation = await deliver(grandResult); + expect(helperContinuation).toMatchObject({ + actor: slackInvocationInput.actor, + childConversationId: helper.childConversationId, + input: expect.stringContaining("Grand helper fact: 42."), + parentConversationId: child.childConversationId, + status: "running", + }); + expect(runs).toEqual([ + expect.objectContaining({ + actor: slackInvocationInput.actor, + conversationId: helper.childConversationId, + disabledFeatures: ["handoff", "interactive-auth", "subagents"], + location: expect.objectContaining({ + provider: "slack", + channelId: "C123", + }), + }), + ]); + // An agent invocation that receives a result is not a new task. + await expect(listAgents(child.childConversationId)).resolves.toEqual([ + { ...listedChild(helper.childConversationId), status: "running" }, + ]); + // Each child reports once, from the agent invocation that received the // last result of its own children. - const helperResult = await complete( - helperContinuation.invocationId, - "Helper fact: 42.", - ); - expect(helperResult).toMatchObject({ parentMailboxStatus: "pending" }); - const childContinuation = await sendToChild(helperResult); - const childResult = await complete( - childContinuation.invocationId, - "Final answer: 42.", - ); - expect(childResult).toMatchObject({ parentMailboxStatus: "pending" }); - await enqueueAgentInvocationResult(childResult, { queue, state }); + const helperResult = await finish(helperContinuation, "Helper fact: 42."); + const childContinuation = await deliver(helperResult); + expect(childContinuation).toMatchObject({ + input: expect.stringContaining("Helper fact: 42."), + parentConversationId: slackParentConversationId, + }); + await finish(childContinuation, "Final answer: 42."); + await expect( + getAgentInvocation(childContinuation.invocationId), + ).resolves.toMatchObject({ parentMailboxStatus: "appended" }); expect(await mailbox(slackParentConversationId)).toEqual([ expect.objectContaining({ inboundMessageId: getAgentInvocationResultMessageId( diff --git a/packages/junior/tests/integration/agent-invocation-spawn.test.ts b/packages/junior/tests/integration/agent-invocation-spawn.test.ts index bd64c4d3df..01486b3bff 100644 --- a/packages/junior/tests/integration/agent-invocation-spawn.test.ts +++ b/packages/junior/tests/integration/agent-invocation-spawn.test.ts @@ -4,20 +4,13 @@ import { createAgentInvocation, getAgentInvocation, getAgentInvocationId, - getAgentInvocationIdForResult, getAgentInvocationMessageId, getAgentInvocationResultMessageId, getAgentInvocationTurnId, } from "@/chat/agent-invocations/store"; -import { - enqueueAgentInvocationResult, - startAgentInvocationsForChildInput, -} from "@/chat/agent-invocations/result"; +import { enqueueAgentInvocationResult } from "@/chat/agent-invocations/result"; import type { TerminalAgentInvocation } from "@/chat/agent-invocations/types"; -import type { InboundMessage } from "@/chat/task-execution/store"; -import type { ConversationWorkerContext } from "@/chat/task-execution/worker"; import { getConversationWorkState } from "@/chat/task-execution/store"; -import { createAgentInvocationWorker } from "@/chat/agent-invocations/work"; import { bindSpawnAgent } from "@/chat/agent-invocations/spawn"; import { createSpawnAgentTool } from "@/chat/tools/runtime/spawn-agent"; import type { AgentRun } from "@/chat/agent/types"; @@ -248,153 +241,6 @@ describe("agent invocation spawn and result delivery", () => { } }); - it("sends each nested child's report to its parent once its own children finish", async () => { - const { fixture } = await prepareSlackParentConversation(); - const queue = createConversationWorkQueueTestAdapter(); - const state = getStateAdapter(); - await state.connect(); - const context = ( - conversationId: string, - messages: InboundMessage[] = [], - ): ConversationWorkerContext => ({ - attempt: { - ack: async () => undefined, - conversationId, - drain: async () => [], - isFinalAttempt: false, - messages, - }, - checkIn: async () => true, - conversationId, - shouldYield: () => false, - }); - const mailbox = async (conversationId: string) => - (await getConversationWorkState({ conversationId, state }))?.messages ?? - []; - const complete = async (invocationId: string, result: string) => - (await completeAgentInvocation({ - invocationId, - result, - status: "completed", - })) as TerminalAgentInvocation; - // Send one result to its parent child, as the worker does, and return the - // agent invocation that the result starts there. - const sendToChild = async (result: TerminalAgentInvocation) => { - await enqueueAgentInvocationResult(result, { queue, state }); - const messages = (await mailbox(result.parentConversationId)).filter( - (message) => - message.inboundMessageId === - getAgentInvocationResultMessageId(result.invocationId), - ); - expect(messages).toHaveLength(1); - // A retried delivery starts the same agent invocation. - for (let attempt = 0; attempt < 2; attempt++) { - await expect( - startAgentInvocationsForChildInput( - context(result.parentConversationId, messages), - { queue, state }, - ), - ).resolves.toEqual({ status: "completed" }); - } - return (await getAgentInvocation( - getAgentInvocationIdForResult(result.invocationId), - ))!; - }; - try { - const child = await createAgentInvocation({ - ...slackInvocationInput, - idempotencyKey: "nested-child", - }); - const helper = await createAgentInvocation({ - ...slackInvocationInput, - idempotencyKey: "nested-helper", - parentConversationId: child.childConversationId, - }); - const grandHelper = await createAgentInvocation({ - ...slackInvocationInput, - idempotencyKey: "nested-grand-helper", - parentConversationId: helper.childConversationId, - }); - - // With maxAgentDepth 1, depth 1 can spawn and depth 2 cannot. Both read - // the root Location. - const runs: AgentRun[] = []; - const worker = createAgentInvocationWorker( - { - run: async (run) => { - runs.push(run); - throw new Error("stop after reading the run"); - }, - }, - { maxAgentDepth: 1 }, - ); - for (const invocation of [child, helper]) { - await expect( - worker( - context(invocation.childConversationId), - invocation.invocationId, - ), - ).rejects.toThrow("stop after reading the run"); - } - expect(runs.map((run) => run.disabledFeatures)).toEqual([ - ["handoff", "interactive-auth"], - ["handoff", "interactive-auth", "subagents"], - ]); - expect(runs[1]?.location).toMatchObject({ - provider: "slack", - channelId: "C123", - }); - - // A child that finishes while its own child runs does not report yet. - for (const invocation of [child, helper]) { - await expect( - complete(invocation.invocationId, "Started a helper."), - ).resolves.toMatchObject({ parentMailboxStatus: "replaced" }); - } - - const grandResult = await complete( - grandHelper.invocationId, - "Grand helper fact: 42.", - ); - expect(grandResult).toMatchObject({ parentMailboxStatus: "pending" }); - const helperContinuation = await sendToChild(grandResult); - expect(helperContinuation).toMatchObject({ - actor: slackInvocationInput.actor, - childConversationId: helper.childConversationId, - input: expect.stringContaining("Grand helper fact: 42."), - parentConversationId: child.childConversationId, - status: "pending", - }); - - // Each child reports once, from the agent invocation that received the - // last result of its own children. - const helperResult = await complete( - helperContinuation.invocationId, - "Helper fact: 42.", - ); - expect(helperResult).toMatchObject({ parentMailboxStatus: "pending" }); - const childContinuation = await sendToChild(helperResult); - const childResult = await complete( - childContinuation.invocationId, - "Final answer: 42.", - ); - expect(childResult).toMatchObject({ parentMailboxStatus: "pending" }); - await enqueueAgentInvocationResult(childResult, { queue, state }); - expect(await mailbox(slackParentConversationId)).toEqual([ - expect.objectContaining({ - inboundMessageId: getAgentInvocationResultMessageId( - childContinuation.invocationId, - ), - input: expect.objectContaining({ - text: expect.stringContaining("Final answer: 42."), - }), - }), - ]); - } finally { - await fixture.close(); - } - }); - it("repairs the durable creation-to-mailbox crash gap once", async () => { const { fixture } = await prepareParentConversation(); const queue = createConversationWorkQueueTestAdapter(); diff --git a/packages/junior/tests/integration/agent-invocation-stop.test.ts b/packages/junior/tests/integration/agent-invocation-stop.test.ts index 14d7a60500..34fd121a5b 100644 --- a/packages/junior/tests/integration/agent-invocation-stop.test.ts +++ b/packages/junior/tests/integration/agent-invocation-stop.test.ts @@ -1,10 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { AgentRun } from "@/chat/agent/types"; -import { - createChildConversationWorker, - enqueueAgentInvocationResult, - startAgentInvocationsForChildInput, -} from "@/chat/agent-invocations/result"; +import { enqueueAgentInvocationResult } from "@/chat/agent-invocations/result"; import { completeAgentInvocation, createAgentInvocation, @@ -18,18 +14,15 @@ import type { CreateAgentInvocationInput, TerminalAgentInvocation, } from "@/chat/agent-invocations/types"; -import { - createAgentInvocationWorker, - createAndEnqueueAgentInvocation, -} from "@/chat/agent-invocations/work"; +import { createAndEnqueueAgentInvocation } from "@/chat/agent-invocations/work"; +import { stopConversationTurn } from "@/chat/conversations/stop"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; import { neverRunAgentRunner } from "../fixtures/agent-runner"; -import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; +import { disconnectStateAdapter } from "@/chat/state/adapter"; import { getConversationWorkState, hasConversationStop, } from "@/chat/task-execution/store"; -import type { ConversationWorkerContext } from "@/chat/task-execution/worker"; import { createListAgentsTool } from "@/chat/tools/runtime/list-agents"; import { createStopAgentTool } from "@/chat/tools/runtime/stop-agent"; import { @@ -40,7 +33,7 @@ import { prepareParentConversation, } from "../fixtures/agent-invocations"; import { - createConversationWorkQueueTestAdapter, + createConversationWorkHarness, deferred, } from "../fixtures/conversation-work"; @@ -75,36 +68,24 @@ async function waitForAbort(run: AgentRun): Promise { throw signal.reason; } -async function createHarness() { +/** + * Compose production conversation work for a local parent. Each test runs + * child work from the queue, as the queue callback does. + */ +async function createHarness(agentRunner: AgentRunner) { const { conversationStore, fixture } = await prepareParentConversation(); - const state = getStateAdapter(); - await state.connect(); - const queue = createConversationWorkQueueTestAdapter(); - const mailbox = async (conversationId: string) => - (await getConversationWorkState({ conversationId, state }))?.messages ?? []; + const work = await createConversationWorkHarness(agentRunner); + const { queue, state } = work; return { close: async () => await fixture.close(), - /** Build the worker context for the child's queued input. */ - async context( - conversationId: string, - signal?: AbortSignal, - ): Promise { - const messages = await mailbox(conversationId); - return { - attempt: { - ack: async () => undefined, - conversationId, - drain: async () => [], - isFinalAttempt: false, - messages, - }, - checkIn: async () => true, - conversationId, - shouldYield: () => false, - ...(signal ? { stopSignal: () => signal } : undefined), - }; - }, - mailbox, + conversationStore, + mailbox: async (conversationId: string) => + (await getConversationWorkState({ conversationId, state }))?.messages ?? + [], + process: work.process, + /** Run the next queued wake of one Conversation. */ + processNext: async (conversationId: string) => + await work.process(queue.takeMessageFor(conversationId)), queue, spawn: async ( overrides: Partial & { @@ -116,11 +97,27 @@ async function createHarness() { { conversationStore, queue, state }, ), state, - worker: (agentRunner: AgentRunner) => - createAgentInvocationWorker(agentRunner, { - enqueueResult: async (invocation) => - await enqueueAgentInvocationResult(invocation, { queue, state }), - }), + }; +} + +/** Run each child until a stop aborts it, and report when it starts. */ +function abortableAgentRunner() { + let started = deferred(); + const runs: AgentRun[] = []; + return { + agentRunner: { + run: async (run: AgentRun) => { + runs.push(run); + started.resolve(undefined); + return await waitForAbort(run); + }, + } satisfies AgentRunner, + runs, + /** Wait for the next run to start. */ + started: async () => { + await started.promise; + started = deferred(); + }, }; } @@ -131,26 +128,16 @@ describe("stopping child agents", () => { }); it("stops a running child without a result and lets a message start new work", async () => { - const harness = await createHarness(); + const runner = abortableAgentRunner(); + const harness = await createHarness(runner.agentRunner); try { const running = await harness.spawn({ agentName: "researcher", idempotencyKey: "stop-running", input: "long task", }); - const started = deferred(); - // Stands in for the worker's poll of the Conversation stop request. - const stopRequest = new AbortController(); - const processing = harness.worker({ - run: async (run) => { - started.resolve(undefined); - return await waitForAbort(run); - }, - })( - await harness.context(running.childConversationId, stopRequest.signal), - running.invocationId, - ); - await started.promise; + const processing = harness.processNext(running.childConversationId); + await runner.started(); await expect(stopAgent(running.childConversationId)).resolves.toEqual({ conversation_id: running.childConversationId, @@ -167,8 +154,8 @@ describe("stopping child agents", () => { state: harness.state, }), ).resolves.toBe(true); - stopRequest.abort(new Error("Conversation work stopped")); - await expect(processing).resolves.toEqual({ status: "completed" }); + // The worker sees the stop request and aborts the run. + await expect(processing).resolves.toMatchObject({ status: "completed" }); await expect( getAgentInvocation(running.invocationId), @@ -238,12 +225,8 @@ describe("stopping child agents", () => { }); it("never runs a stopped pending child and reports other stops to the parent", async () => { - const harness = await createHarness(); - const neverRun: AgentRunner = { - run: async () => { - throw new Error("A stopped child must not run"); - }, - }; + const runner = abortableAgentRunner(); + const harness = await createHarness(runner.agentRunner); try { const pending = await harness.spawn({ idempotencyKey: "stop-pending", @@ -256,25 +239,16 @@ describe("stopping child agents", () => { conversation_id: pending.childConversationId, stopped: true, }); - await expect( - harness.worker(neverRun)( - await harness.context(pending.childConversationId), - pending.invocationId, - ), - ).resolves.toEqual({ status: "completed" }); - await expect( - getAgentInvocation(pending.invocationId), - ).resolves.toMatchObject({ status: "stopped" }); - - // A stop request can outlive its run. The stopped agent invocation - // handles the next empty wake, so it never reaches a root worker. - const wake = await harness.context(pending.childConversationId); - await expect( - createChildConversationWorker(neverRunAgentRunner(), { - queue: harness.queue, - state: harness.state, - })({ ...wake, attempt: { ...wake.attempt, messages: [] } }), - ).resolves.toEqual({ status: "completed" }); + // The stopped child takes its wake without a run. A redelivery finds + // no work. + const wake = harness.queue.takeMessageFor(pending.childConversationId); + await expect(harness.process(wake)).resolves.toMatchObject({ + status: "completed", + }); + await expect(harness.process(wake)).resolves.toMatchObject({ + status: "no_work", + }); + expect(runner.runs).toEqual([]); await expect( getAgentInvocation(pending.invocationId), ).resolves.toMatchObject({ status: "stopped" }); @@ -290,17 +264,17 @@ describe("stopping child agents", () => { idempotencyKey: "stop-external-helper", parentConversationId: external.childConversationId, }); - const stopRequest = new AbortController(); - stopRequest.abort(new Error("Conversation work stopped")); + const processing = harness.processNext(external.childConversationId); + await runner.started(); await expect( - harness.worker({ run: waitForAbort })( - await harness.context( - external.childConversationId, - stopRequest.signal, - ), - external.invocationId, - ), - ).resolves.toEqual({ status: "completed" }); + stopConversationTurn({ + conversationId: external.childConversationId, + conversationStore: harness.conversationStore, + queue: harness.queue, + state: harness.state, + }), + ).resolves.toMatchObject({ status: "requested" }); + await expect(processing).resolves.toMatchObject({ status: "completed" }); await expect( getAgentInvocation(external.invocationId), ).resolves.toMatchObject({ @@ -338,7 +312,7 @@ describe("stopping child agents", () => { }); it("stops every child agent below the child and drops their results", async () => { - const harness = await createHarness(); + const harness = await createHarness(neverRunAgentRunner()); try { const child = await createAgentInvocation({ ...invocationInput, @@ -399,14 +373,12 @@ describe("stopping child agents", () => { status: "completed", }); // The child already had the result in its mailbox. It starts no work. - const childContext = await harness.context(child.childConversationId); - expect(childContext.attempt.messages).toHaveLength(1); await expect( - startAgentInvocationsForChildInput(childContext, { - queue: harness.queue, - state: harness.state, - }), - ).resolves.toEqual({ status: "completed" }); + harness.mailbox(child.childConversationId), + ).resolves.toHaveLength(1); + await expect( + harness.processNext(child.childConversationId), + ).resolves.toMatchObject({ status: "completed" }); await expect( getAgentInvocation( getAgentInvocationIdForResult(doneHelper.invocationId), diff --git a/packages/junior/tests/integration/agent-invocation-work.test.ts b/packages/junior/tests/integration/agent-invocation-work.test.ts index f2e96335ae..1bba4c592e 100644 --- a/packages/junior/tests/integration/agent-invocation-work.test.ts +++ b/packages/junior/tests/integration/agent-invocation-work.test.ts @@ -13,6 +13,7 @@ import { EVENT_MESSAGE_AUTHOR, EVENT_SYSTEM_ACTOR } from "@/chat/events/actor"; import { getEventCatalog } from "@/chat/events/runtime-catalog"; import { runTimerWatchHeartbeat } from "@/chat/events/timers"; import { createEventTools } from "@/chat/tools/events"; +import { createStopAgentTool } from "@/chat/tools/runtime/stop-agent"; import { CONVERSATION_WORK_MAX_RETRIES, getConversationWorkState, @@ -25,7 +26,7 @@ import { getTurnRecord, upsertTurnRecord, } from "@/chat/task-execution/turn-cursor"; -import { createConversationWorkHarness } from "../fixtures/agent-dispatch"; +import { createConversationWorkHarness } from "../fixtures/conversation-work"; import { context, execute } from "../fixtures/event-automations"; import { createModelAgentRunner, @@ -393,9 +394,10 @@ describe("agent invocation conversation work", () => { // A child watch event runs with an event Source. When its Turn yields, the // next wake must still reach the agent invocation. Production once sent it // to the root Event Turn worker, which needs a Destination that a child - // does not have. That also holds when the agent invocation ended while its - // Turn stayed paused. A resume wake has no messages, but its last try is - // still final, so a resume that keeps failing tells the parent. + // does not have. That also holds when the agent invocation ended or was + // stopped while its Turn stayed paused. A resume wake has no messages, but + // its last try is still final, so a resume that keeps failing tells the + // parent. const resumed = { result: "Resumed child result", status: "completed" }; it.each([ { @@ -419,6 +421,13 @@ describe("agent invocation conversation work", () => { runs: 2, expected: { parentMailboxStatus: "appended", status: "failed" }, }, + { + name: "a stopped child", + watchEvent: false, + stopBeforeWake: true, + runs: 1, + expected: { parentMailboxStatus: "skipped", status: "stopped" }, + }, { name: "a resume that keeps failing", watchEvent: false, @@ -438,6 +447,7 @@ describe("agent invocation conversation work", () => { expected, failedResumes = 0, runs, + stopBeforeWake = false, watchEvent, }) => { const clock = vi.spyOn(Date, "now").mockReturnValue(1_800_000_000_000); @@ -526,6 +536,14 @@ describe("agent invocation conversation work", () => { status: "failed", }); } + if (stopBeforeWake) { + // The stop request outlives the paused run. The stopped agent + // invocation takes the resume wake and does not run again. + await createStopAgentTool(parentConversationId).execute!( + { conversation_id: childConversationId }, + {}, + ); + } if (failedResumes > 0) { run.mockRejectedValue(new Error("resume unavailable")); } diff --git a/scripts/test-architecture-baseline.json b/scripts/test-architecture-baseline.json index 5e096d5559..ecec90d703 100644 --- a/scripts/test-architecture-baseline.json +++ b/scripts/test-architecture-baseline.json @@ -163,9 +163,8 @@ "packages/junior/tests/component/task-execution/conversation-work.test.ts": 2, "packages/junior/tests/component/task-execution/slack-conversation-work.test.ts": 1, "packages/junior/tests/fixtures/acp-http.ts": 1, - "packages/junior/tests/fixtures/agent-dispatch.ts": 1, "packages/junior/tests/fixtures/chat-runtime.ts": 1, - "packages/junior/tests/fixtures/conversation-work.ts": 3, + "packages/junior/tests/fixtures/conversation-work.ts": 2, "packages/junior/tests/fixtures/conversation.ts": 2, "packages/junior/tests/fixtures/oauth-work.ts": 2, "packages/junior/tests/integration/acp-http-recovery.test.ts": 2, From 785bb293fa91127cc001a8cdac9d7183f84bd547 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:50:30 +0000 Subject: [PATCH 07/21] ref(task-execution): Share the worker Turn lifecycle between root and child work Mailbox work and agent invocation work each had their own copy of the Turn lifecycle: input ack, stop checks, AgentRunError unwrap, lost-lease detection, and the retry-or-fail decision. The child-resume bugs came from drift between these copies. runWorkerTurn in task-execution/worker-turn.ts now owns that lifecycle. Each work owner supplies its Run, its result save, and its finish steps. Behavior changes from aligning the two copies: - An execution limit now fails a root Turn at once, as it already did for a child. A retry cannot pass the limit. - A root setup error on the last attempt now fails the Turn, as it did for a child, instead of leaving it open. - A child setup error while a stop is pending now ends the child as stopped. - A lost lease while a child records a stop returns lost_lease. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../junior/src/chat/agent-invocations/work.ts | 424 +++++++----------- .../junior/src/chat/task-execution/README.md | 6 + .../chat/task-execution/conversation-turn.ts | 394 +++++++--------- .../src/chat/task-execution/worker-turn.ts | 170 +++++++ 4 files changed, 524 insertions(+), 470 deletions(-) create mode 100644 packages/junior/src/chat/task-execution/worker-turn.ts diff --git a/packages/junior/src/chat/agent-invocations/work.ts b/packages/junior/src/chat/agent-invocations/work.ts index 29fdf49a05..8eafd43c6a 100644 --- a/packages/junior/src/chat/agent-invocations/work.ts +++ b/packages/junior/src/chat/agent-invocations/work.ts @@ -18,8 +18,6 @@ import { getConversationEventStore, getConversationStore } from "@/chat/db"; import { SYSTEM_IDENTITY_ID } from "@/chat/identities/actor"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; import { - AgentRunError, - executeTurn, finishTurn, type SavedTurnResult, } from "@/chat/runtime/turn-execution"; @@ -34,11 +32,7 @@ import { getTurnRecord, saveTurnCheckpoint, } from "@/chat/task-execution/checkpoint"; -import { - getConversationTurnBoundaryError, - isTurnInputCommitLostError, - TurnInputCommitLostError, -} from "@/chat/runtime/turn"; +import { getConversationTurnBoundaryError } from "@/chat/runtime/turn"; import { AuthorizationFlowDisabledError } from "@/chat/services/auth-pause"; import { botConfig } from "@/chat/config"; import { @@ -47,10 +41,8 @@ import { } from "@/chat/services/automated-turn-limit"; import { maybePostAutomatedTurnLimitNotice } from "@/chat/slack/automated-turn-limit-notice"; import { PluginCredentialFailureError } from "@/chat/services/plugin-auth-orchestration"; -import { isTurnExecutionLimitExceededError } from "@/chat/services/turn-limit"; import { getAssistantReplyText } from "@/chat/services/assistant-reply"; import { getTerminalAssistantMessages } from "@/chat/pi/transcript"; -import type { PiMessage } from "@/chat/pi/messages"; import type { SandboxRef } from "@/chat/sandbox/ref"; import { agentTypeInstructions, @@ -67,6 +59,7 @@ import type { ConversationWorkerContext, ConversationWorkerResult, } from "@/chat/task-execution/worker"; +import { runWorkerTurn } from "@/chat/task-execution/worker-turn"; import { completeAgentInvocation, createAgentInvocation, @@ -481,11 +474,6 @@ function blockingInvocationError( : undefined; } -function isInvocationInputCommitLost(error: unknown): boolean { - const cause = getConversationTurnBoundaryError(error)?.cause ?? error; - return isTurnInputCommitLostError(cause); -} - /** Save one completed agent result on its child agent invocation. */ async function saveAgentInvocationResult(args: { invocation: AgentInvocation; @@ -600,10 +588,11 @@ export function createAgentInvocationWorker( context: ConversationWorkerContext, invocationId: string, ): Promise => { - let invocation = await getAgentInvocation(invocationId); - if (!invocation) { + const loaded = await getAgentInvocation(invocationId); + if (!loaded) { throw new Error(`Agent invocation is missing for ${invocationId}`); } + let invocation: AgentInvocation = loaded; // Report the result before ack so a failed parent append retries this work. const report = async (terminal: TerminalAgentInvocation): Promise => { await recordSubagentEnded(terminal); @@ -622,252 +611,187 @@ export function createAgentInvocationWorker( await report(terminal); }; - let acknowledged = context.attempt.messages.length === 0; - const acknowledge = async (): Promise => { - if (acknowledged) { - return; - } - try { - await context.attempt.ack(); - } catch { - throw new TurnInputCommitLostError( - `Conversation work lease lost before invocation inbox ack for ${context.conversationId}`, - ); - } - acknowledged = true; - }; - // Read the signal before any early return, so the worker clears a stop // request that this wake handles. const stopSignal = context.stopSignal?.(); - // `stopChildAgent` already marked its work `stopped`. Any other stop is - // recorded here and reported to the parent. - // The stop also stops the child agents of this child, so no result of - // theirs starts this child again. - const finishStopped = async (): Promise => { + const turnId = getAgentInvocationTurnId(invocation.invocationId); + const endInvocation = async ( + status: "blocked" | "failed" | "stopped", + errorMessage: string, + ): Promise => { await finish( await completeAgentInvocation({ invocationId, - errorMessage: "Agent invocation was stopped", - status: "stopped", + errorMessage, + status, }), ); - await stopChildAgentsOf(context.conversationId); - await acknowledge(); - return { status: "completed" }; }; - const turnId = getAgentInvocationTurnId(invocation.invocationId); - const lifecycle = new ConversationTurnLifecycleService( - getConversationEventStore(), - ); - let sandboxRef: SandboxRef | undefined; - let history: PiMessage[]; - let location: Location | undefined; - let canSpawn = false; - try { - if (invocation.childConversationId !== context.conversationId) { - throw new Error( - `Agent invocation ${invocationId} belongs to ${invocation.childConversationId}, not ${context.conversationId}`, - ); - } - if (context.destination) { - throw new Error( - `Agent invocation conversation ${context.conversationId} must not own a provider destination`, - ); - } - if (isTerminalAgentInvocation(invocation)) { - await finish(invocation); - await acknowledge(); - return { status: "completed" }; - } - const projected = await projectTerminalTurn(invocation); - if (projected && isTerminalAgentInvocation(projected)) { - await finish(projected); - await acknowledge(); - return { status: "completed" }; - } - invocation = - (await markAgentInvocationRunning(invocation.invocationId)) ?? - invocation; - // A stop can land between the read above and this write. - if (isTerminalAgentInvocation(invocation)) { - await finish(invocation); - await acknowledge(); - return { status: "completed" }; - } - await lifecycle.start({ - conversationId: invocation.childConversationId, - createdAtMs: invocation.createdAtMs, - inputMessageIds: [getAgentInvocationMessageId(invocation.invocationId)], - surface: "internal", - turnId, - }); - const [persisted, projection, tree] = await Promise.all([ - getPersistedThreadState(invocation.childConversationId), - openConversationProjection({ - conversationId: invocation.childConversationId, - }), - readChildDepthAndRootLocation(invocation.childConversationId), - ]); - sandboxRef = getPersistedSandboxState(persisted); - // Some types start with the parent's history. Later agent invocations - // read it from the child's own Conversation. - history = - agentTypeStartsWithParentHistory(invocation.agentType) && - projection.messages.length === 0 - ? await loadForkHistory(invocation) - : projection.messages; - location = tree.location; - canSpawn = tree.depth <= maxAgentDepth; - } catch (error) { - if (isInvocationInputCommitLost(error)) { - return { status: "lost_lease" }; - } - if (!context.attempt.isFinalAttempt) { - throw error; - } - const terminal = await completeAgentInvocation({ - invocationId: invocation.invocationId, - errorMessage: - error instanceof Error ? error.message : "Agent invocation failed", - status: "failed", - }); - await finish(terminal); - await acknowledge(); - return { status: "completed" }; - } - - let outcome; - try { - outcome = await executeTurn( - agentRunner, - { - conversationId: invocation.childConversationId, - parentConversationId: invocation.parentConversationId, - turnId, - runId: invocation.invocationId, - instruction: { - text: agentInvocationInputText(invocation), - }, - history, - actor: invocation.actor, - credentialContext: invocation.credentialContext, - destination: invocation.destination, - source: invocation.source, - ...(location ? { location } : undefined), - surface: "internal", - ...(stopSignal ? { signal: stopSignal } : undefined), - // TODO(dcramer): Issues #881 and #883 track a path for child runs to - // force interactive auth when a delegated tool requires credentials - // the parent can request. Today background children hard-fail instead - // of pausing for an OAuth link. - ...agentInvocationRunPolicy(invocation, canSpawn), - state: { - sandboxRef, - }, - durability: { - drainSteeringMessages: createAgentMessageSteeringDrain( - context, - invocation, - ), - onInputCommitted: acknowledge, - shouldYield: context.shouldYield, - onSandboxRefChanged: async (nextSandboxRef) => { - sandboxRef = nextSandboxRef; - await persistThreadStateById(invocation.childConversationId, { + return await runWorkerTurn({ + agentRunner, + context, + inputCommitted: context.attempt.messages.length === 0, + stopSignal, + steps: { + prepare: async () => { + if (invocation.childConversationId !== context.conversationId) { + throw new Error( + `Agent invocation ${invocationId} belongs to ${invocation.childConversationId}, not ${context.conversationId}`, + ); + } + if (context.destination) { + throw new Error( + `Agent invocation conversation ${context.conversationId} must not own a provider destination`, + ); + } + if (isTerminalAgentInvocation(invocation)) { + await finish(invocation); + return "finished"; + } + const projected = await projectTerminalTurn(invocation); + if (projected && isTerminalAgentInvocation(projected)) { + await finish(projected); + return "finished"; + } + invocation = + (await markAgentInvocationRunning(invocation.invocationId)) ?? + invocation; + // A stop can land between the read above and this write. + if (isTerminalAgentInvocation(invocation)) { + await finish(invocation); + return "finished"; + } + await new ConversationTurnLifecycleService( + getConversationEventStore(), + ).start({ + conversationId: invocation.childConversationId, + createdAtMs: invocation.createdAtMs, + inputMessageIds: [ + getAgentInvocationMessageId(invocation.invocationId), + ], + surface: "internal", + turnId, + }); + const [persisted, projection, tree] = await Promise.all([ + getPersistedThreadState(invocation.childConversationId), + openConversationProjection({ + conversationId: invocation.childConversationId, + }), + readChildDepthAndRootLocation(invocation.childConversationId), + ]); + let sandboxRef = getPersistedSandboxState(persisted); + return { + run: { + conversationId: invocation.childConversationId, + parentConversationId: invocation.parentConversationId, + turnId, + runId: invocation.invocationId, + instruction: { + text: agentInvocationInputText(invocation), + }, + // Some types start with the parent's history. Later agent + // invocations read it from the child's own Conversation. + history: + agentTypeStartsWithParentHistory(invocation.agentType) && + projection.messages.length === 0 + ? await loadForkHistory(invocation) + : projection.messages, + actor: invocation.actor, + credentialContext: invocation.credentialContext, + destination: invocation.destination, + source: invocation.source, + ...(tree.location ? { location: tree.location } : undefined), + surface: "internal", + // TODO(dcramer): Issues #881 and #883 track a path for child runs + // to force interactive auth when a delegated tool requires + // credentials the parent can request. Today background children + // hard-fail instead of pausing for an OAuth link. + ...agentInvocationRunPolicy( + invocation, + tree.depth <= maxAgentDepth, + ), + state: { sandboxRef, - }); + }, + durability: { + drainSteeringMessages: createAgentMessageSteeringDrain( + context, + invocation, + ), + onSandboxRefChanged: async (nextSandboxRef) => { + sandboxRef = nextSandboxRef; + await persistThreadStateById(invocation.childConversationId, { + sandboxRef, + }); + }, + }, }, - }, + saveResult: async (result) => + await saveAgentInvocationResult({ + invocation: invocation, + result, + sandboxRef, + turnId, + }), + }; }, - async (result) => - await saveAgentInvocationResult({ - invocation, - result, - sandboxRef, - turnId, - }), - ); - } catch (error) { - // A save error has no AgentRunError wrapper. On the final attempt it - // must also fail the agent invocation, or the child stays open. - const runError = error instanceof AgentRunError ? error.cause : error; - if (isInvocationInputCommitLost(runError)) { - return { status: "lost_lease" }; - } - if (stopSignal?.aborted) { - return await finishStopped(); - } - const blocking = blockingInvocationError(runError); - if (blocking) { - const terminal = await completeAgentInvocation({ - invocationId: invocation.invocationId, - errorMessage: blocking.message, - status: "blocked", - }); - await finish(terminal); - await acknowledge(); - return { status: "completed" }; - } - // A retry cannot pass an execution limit, and its Turn record would - // hide the limit as a lost worker. - if ( - !context.attempt.isFinalAttempt && - !isTurnExecutionLimitExceededError(runError) - ) { - throw runError; - } - const terminal = await completeAgentInvocation({ - invocationId: invocation.invocationId, - errorMessage: - runError instanceof Error - ? runError.message - : "Agent invocation failed", - status: "failed", - }); - await finish(terminal); - await acknowledge(); - return { status: "completed" }; - } - - if (outcome.status !== "completed" && stopSignal?.aborted) { - return await finishStopped(); - } - if (outcome.status === "suspended") { - await markAgentInvocationPaused(invocation.invocationId); - return { status: "yielded" }; - } - if (outcome.status === "awaiting_auth") { - const terminal = await completeAgentInvocation({ - invocationId: invocation.invocationId, - errorMessage: `Agent invocation requires ${outcome.providerDisplayName} authorization`, - status: "blocked", - }); - await finish(terminal); - await acknowledge(); - return { status: "completed" }; - } - - // executeTurn already recorded the terminal Turn; only report the result. - let terminal = await getAgentInvocation(invocation.invocationId); - // A stop that comes after the save still stops the child agents of this - // child. Then they cannot report for a `replaced` result, so this result - // reports now. - if (stopSignal?.aborted) { - if ( - terminal && - isTerminalAgentInvocation(terminal) && - terminal.parentMailboxStatus === "replaced" - ) { - terminal = await reportReplacedAgentInvocation(terminal.invocationId); - } - await stopChildAgentsOf(context.conversationId); - } - if (terminal && isTerminalAgentInvocation(terminal)) { - await report(terminal); - } - await acknowledge(); - return { status: "completed" }; + // `stopChildAgent` already marked its work `stopped`. Any other stop is + // recorded here and reported to the parent. The stop also stops the + // child agents of this child, so no result of theirs starts this + // child again. + stop: async () => { + await endInvocation("stopped", "Agent invocation was stopped"); + await stopChildAgentsOf(context.conversationId); + }, + suspend: async () => { + await markAgentInvocationPaused(invocation.invocationId); + }, + awaitAuth: async (outcome) => { + await endInvocation( + "blocked", + `Agent invocation requires ${outcome.providerDisplayName} authorization`, + ); + return "completed"; + }, + complete: async () => { + // executeTurn already recorded the terminal Turn; only report the + // result. + let terminal = await getAgentInvocation(invocation.invocationId); + // A stop that comes after the save still stops the child agents of + // this child. Then they cannot report for a `replaced` result, so + // this result reports now. + if (stopSignal?.aborted) { + if ( + terminal && + isTerminalAgentInvocation(terminal) && + terminal.parentMailboxStatus === "replaced" + ) { + terminal = await reportReplacedAgentInvocation( + terminal.invocationId, + ); + } + await stopChildAgentsOf(context.conversationId); + } + if (terminal && isTerminalAgentInvocation(terminal)) { + await report(terminal); + } + }, + block: async (error) => { + const blocking = blockingInvocationError(error); + if (!blocking) { + return false; + } + await endInvocation("blocked", blocking.message); + return true; + }, + fail: async (error) => { + await endInvocation( + "failed", + error instanceof Error ? error.message : "Agent invocation failed", + ); + }, + }, + }); }; } diff --git a/packages/junior/src/chat/task-execution/README.md b/packages/junior/src/chat/task-execution/README.md index a8b3a84248..d995187dd3 100644 --- a/packages/junior/src/chat/task-execution/README.md +++ b/packages/junior/src/chat/task-execution/README.md @@ -11,6 +11,12 @@ accept the result. `loadTurnCheckpoint` and `saveTurnCheckpoint`. `turn-cursor.ts` stores the checkpoint and is private to this module. +`worker-turn.ts` owns the Turn lifecycle under the lease: the input ack, stop +checks, the Run, lost leases, and the choice to retry or fail on the last +attempt. Mailbox work and agent invocation work call `runWorkerTurn`. Each work +owner supplies its Run, its result save, and its finish steps. The lifecycle +does not check which owner called it. + `turn-wake.ts` wakes paused turns. `paused-turn.ts` runs them under the conversation lease. SQL conversation events store history. The Redis Turn cursor stores only the data that is needed to resume a Turn, including the diff --git a/packages/junior/src/chat/task-execution/conversation-turn.ts b/packages/junior/src/chat/task-execution/conversation-turn.ts index 1a67c9b4d6..2642490464 100644 --- a/packages/junior/src/chat/task-execution/conversation-turn.ts +++ b/packages/junior/src/chat/task-execution/conversation-turn.ts @@ -26,7 +26,6 @@ import { credentialContextForActor } from "@/chat/credentials/context"; import { getConversationEventStore, getConversationStore } from "@/chat/db"; import { logException, setTags, withLogContext } from "@/chat/logging"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; -import { AgentRunError, executeTurn } from "@/chat/runtime/turn-execution"; import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; import { getPersistedSandboxState, @@ -34,12 +33,9 @@ import { persistThreadStateById, } from "@/chat/runtime/thread-state"; import { - getConversationTurnBoundaryError, - isTurnInputCommitLostError, markTurnClosed, markTurnFailed, startActiveTurn, - TurnInputCommitLostError, } from "@/chat/runtime/turn"; import { completeAuthPauseTurn } from "@/chat/runtime/auth-pause-state"; import { getTurnUserMessage } from "@/chat/runtime/turn-user-message"; @@ -70,6 +66,10 @@ import type { ConversationWorkerContext, ConversationWorkerResult, } from "@/chat/task-execution/worker"; +import { + runWorkerTurn, + type PreparedWorkerTurn, +} from "@/chat/task-execution/worker-turn"; import { processPluginTask, scheduleSessionCompletedPluginTasks, @@ -139,13 +139,7 @@ function resumedTimerWatchCredentialSubject( : undefined; } -function hasLostTurnInputCommit(error: unknown): boolean { - const cause = getConversationTurnBoundaryError(error)?.cause ?? error; - return isTurnInputCommitLostError(error) || isTurnInputCommitLostError(cause); -} - async function completeCancelledConversationTurn(args: { - acknowledge(): Promise; conversation: ThreadConversationState; conversationId: string; sandboxRef?: SandboxRef; @@ -189,7 +183,6 @@ async function completeCancelledConversationTurn(args: { outcome: "cancelled", turnId: args.turnId, }); - await args.acknowledge(); } /** @@ -368,97 +361,10 @@ export function createConversationTurnWorker( : undefined), }, async () => { - let acknowledged = isResume || context.attempt.messages.length === 0; - const acknowledge = async (): Promise => { - if (acknowledged) { - return; - } - try { - await context.attempt.ack(); - } catch { - throw new TurnInputCommitLostError( - `Conversation work lease lost before mailbox Message acknowledgement for ${context.conversationId}`, - ); - } - acknowledged = true; - }; - let sandboxRef: SandboxRef | undefined = getPersistedSandboxState(persisted); const initialSandboxRef = sandboxRef; const stopSignal = context.stopSignal?.(); - const completeCancelledTurn = - async (): Promise => { - try { - await completeCancelledConversationTurn({ - acknowledge, - conversation, - conversationId: context.conversationId, - sandboxRef, - turnId, - userMessageId, - }); - } catch (error) { - if (hasLostTurnInputCommit(error)) { - return { status: "lost_lease" }; - } - throw error; - } - return { status: "completed" }; - }; - - if (isResume) { - if (stopSignal?.aborted) { - return await completeCancelledTurn(); - } - } else { - upsertConversationMessage(conversation, { - id: userMessageId, - role: "user", - text: normalizeConversationText(text), - createdAtMs: startedAtMs, - author, - ...(authorIdentityId ? { authorIdentityId } : undefined), - meta: { - ...(attachments.length ? { attachments } : undefined), - explicitMention: true, - replied: false, - ...(source.kind === "web" - ? { source: "web" as const } - : undefined), - ...(event - ? { - eventType: event.eventType, - eventObjectType: event.objectType, - ...(event.trustedSummary - ? { trustedSummary: event.trustedSummary } - : undefined), - ...(batchEvents.length > 1 - ? { events: batchEvents } - : undefined), - } - : undefined), - }, - }); - await persistConversationMessages({ - conversation, - conversationId: context.conversationId, - }); - await lifecycle.start({ - conversationId: context.conversationId, - createdAtMs: Date.now(), - inputMessageIds, - surface, - turnId, - }); - if (stopSignal?.aborted) { - return await completeCancelledTurn(); - } - startActiveTurn({ - conversation, - nextTurnId: turnId, - }); - } let currentRunId: string | undefined; let assistantMessageDelivered = false; @@ -510,10 +416,64 @@ export function createConversationTurnWorker( assistantMessageDelivered = true; failureCode = "agent_run_failed"; }; + const prepare = async (): Promise< + PreparedWorkerTurn | "finished" | "stopped" + > => { + if (isResume) { + if (stopSignal?.aborted) { + return "stopped"; + } + } else { + upsertConversationMessage(conversation, { + id: userMessageId, + role: "user", + text: normalizeConversationText(text), + createdAtMs: startedAtMs, + author, + ...(authorIdentityId ? { authorIdentityId } : undefined), + meta: { + ...(attachments.length ? { attachments } : undefined), + explicitMention: true, + replied: false, + ...(source.kind === "web" + ? { source: "web" as const } + : undefined), + ...(event + ? { + eventType: event.eventType, + eventObjectType: event.objectType, + ...(event.trustedSummary + ? { trustedSummary: event.trustedSummary } + : undefined), + ...(batchEvents.length > 1 + ? { events: batchEvents } + : undefined), + } + : undefined), + }, + }); + await persistConversationMessages({ + conversation, + conversationId: context.conversationId, + }); + await lifecycle.start({ + conversationId: context.conversationId, + createdAtMs: Date.now(), + inputMessageIds, + surface, + turnId, + }); + if (stopSignal?.aborted) { + return "stopped"; + } + startActiveTurn({ + conversation, + nextTurnId: turnId, + }); + } - try { if (stopSignal?.aborted) { - return await completeCancelledTurn(); + return "stopped"; } if (!isResume) { // Match Slack: new input supersedes an auth-paused Turn @@ -532,7 +492,7 @@ export function createConversationTurnWorker( ); for (const parked of authParked) { if (stopSignal?.aborted) { - return await completeCancelledTurn(); + return "stopped"; } await abandonTurnRecord({ conversationId: context.conversationId, @@ -561,7 +521,7 @@ export function createConversationTurnWorker( conversation, }); if (stopSignal?.aborted) { - return await completeCancelledTurn(); + return "stopped"; } const piMessages = await loadProjection({ conversationId: context.conversationId, @@ -570,9 +530,8 @@ export function createConversationTurnWorker( currentRunId = `conversation-run:${stableHex(turnId, String(startedAtMs))}`; setTags({ runId: currentRunId }); - const outcome = await executeTurn( - agentRunner, - { + return { + run: { conversationId: context.conversationId, turnId, runId: currentRunId, @@ -597,7 +556,6 @@ export function createConversationTurnWorker( ? { location: conversationLocation } : undefined), surface, - ...(stopSignal ? { signal: stopSignal } : undefined), ...(webActor ? { authorization: createWebAuthorization({ @@ -623,8 +581,6 @@ export function createConversationTurnWorker( }), } : undefined), - onInputCommitted: acknowledge, - shouldYield: context.shouldYield, onSandboxRefChanged: async (nextSandboxRef) => { sandboxRef = nextSandboxRef; await persistThreadStateById(context.conversationId, { @@ -641,7 +597,7 @@ export function createConversationTurnWorker( }, }, }, - async (result) => { + saveResult: async (result) => { if (stopSignal?.aborted) { throw ( stopSignal.reason ?? @@ -724,123 +680,121 @@ export function createConversationTurnWorker( outcome: "failed", }; }, - ); - - if (outcome.status !== "completed" && stopSignal?.aborted) { - return await completeCancelledTurn(); - } + }; + }; - if (outcome.status === "suspended") { - return { status: "yielded" }; - } - if (outcome.status === "awaiting_auth") { - // Close the live turn the same way Slack does after private-link - // delivery. The turn record stays paused for OAuth resume; only - // the conversation active pointer is cleared. - completeAuthPauseTurn({ - conversation, - sessionId: turnId, - }); - await persistThreadStateById(context.conversationId, { - conversation, - sandboxRef, - }); - await acknowledge(); - return { status: "paused" }; - } - if (completedSuccessfully) { - try { - await scheduleSessionCompletedPluginTasks( - { - conversationId: context.conversationId, + return await runWorkerTurn({ + agentRunner, + context, + inputCommitted: isResume || context.attempt.messages.length === 0, + stopSignal, + steps: { + prepare, + stop: async () => { + await completeCancelledConversationTurn({ + conversation, + conversationId: context.conversationId, + sandboxRef, + turnId, + userMessageId, + }); + }, + awaitAuth: async () => { + // Close the live turn the same way Slack does after private-link + // delivery. The turn record stays paused for OAuth resume; only + // the conversation active pointer is cleared. + completeAuthPauseTurn({ + conversation, + sessionId: turnId, + }); + await persistThreadStateById(context.conversationId, { + conversation, + sandboxRef, + }); + return "paused"; + }, + complete: async () => { + if (completedSuccessfully) { + try { + await scheduleSessionCompletedPluginTasks( + { + conversationId: context.conversationId, + sessionId: turnId, + }, + { + send: async (message) => { + try { + await processPluginTask(message); + } catch (error) { + logException( + error, + "conversation.plugin.session_completion_task.failed", + { + conversationId: context.conversationId, + pluginName: message.plugin, + taskName: message.name, + turnId, + }, + ); + } + }, + }, + ); + } catch (error) { + logException( + error, + "conversation.plugin.session_completion_task.failed", + { + conversationId: context.conversationId, + turnId, + }, + ); + } + } + }, + fail: async (failure) => { + const failureEventId = + modelFailureCaptureAttempted && + failureCode === "agent_run_failed" + ? modelFailureEventId + : captureConversationTurnFailure({ + conversationId: context.conversationId, + error: failure, + failureCode, + runId: currentRunId, + turnId, + }); + try { + markTurnFailed({ + conversation, + nowMs: Date.now(), sessionId: turnId, - }, - { - send: async (message) => { - try { - await processPluginTask(message); - } catch (error) { - logException( - error, - "conversation.plugin.session_completion_task.failed", - { - conversationId: context.conversationId, - pluginName: message.plugin, - taskName: message.name, - turnId, - }, - ); - } - }, - }, - ); - } catch (error) { - logException( - error, - "conversation.plugin.session_completion_task.failed", - { - conversationId: context.conversationId, - turnId, - }, - ); - } - } - - await acknowledge(); - return { status: "completed" }; - } catch (error) { - const failure = error instanceof AgentRunError ? error.cause : error; - if (hasLostTurnInputCommit(failure)) { - return { status: "lost_lease" }; - } - if (stopSignal?.aborted) { - return await completeCancelledTurn(); - } - if (!context.attempt.isFinalAttempt) { - throw failure; - } - - const failureEventId = - modelFailureCaptureAttempted && failureCode === "agent_run_failed" - ? modelFailureEventId - : captureConversationTurnFailure({ + userMessageId, + markConversationMessage, + }); + await persistThreadStateById(context.conversationId, { + conversation, + sandboxRef: initialSandboxRef ?? null, + }); + } catch (persistenceError) { + captureConversationTurnFailure({ conversationId: context.conversationId, - error: failure, - failureCode, + error: persistenceError, + failureCode: "persistence_failed", runId: currentRunId, turnId, }); - try { - markTurnFailed({ - conversation, - nowMs: Date.now(), - sessionId: turnId, - userMessageId, - markConversationMessage, - }); - await persistThreadStateById(context.conversationId, { - conversation, - sandboxRef: initialSandboxRef ?? null, - }); - } catch (persistenceError) { - captureConversationTurnFailure({ - conversationId: context.conversationId, - error: persistenceError, - failureCode: "persistence_failed", - runId: currentRunId, - turnId, - }); - } - await lifecycle.fail({ - conversationId: context.conversationId, - createdAtMs: Date.now(), - ...(failureEventId ? { eventId: failureEventId } : undefined), - failureCode, - turnId, - }); - await acknowledge(); - return { status: "completed" }; - } + } + await lifecycle.fail({ + conversationId: context.conversationId, + createdAtMs: Date.now(), + ...(failureEventId ? { eventId: failureEventId } : undefined), + failureCode, + turnId, + }); + }, + }, + }); }, ); }; diff --git a/packages/junior/src/chat/task-execution/worker-turn.ts b/packages/junior/src/chat/task-execution/worker-turn.ts new file mode 100644 index 0000000000..1e15234d0e --- /dev/null +++ b/packages/junior/src/chat/task-execution/worker-turn.ts @@ -0,0 +1,170 @@ +/** + * Run one Turn under the conversation lease. Root mailbox work and child agent + * invocation work both use this lifecycle, so they ack, stop, retry, and fail + * the same way. Each work owner supplies its own steps. + */ +import type { AgentRun } from "@/chat/agent/types"; +import type { AgentRunOutcome } from "@/chat/runtime/agent-run-outcome"; +import type { AgentRunner } from "@/chat/runtime/agent-runner"; +import { + getConversationTurnBoundaryError, + isTurnInputCommitLostError, + TurnInputCommitLostError, +} from "@/chat/runtime/turn"; +import { + AgentRunError, + executeTurn, + type SavedTurnResult, +} from "@/chat/runtime/turn-execution"; +import { isTurnExecutionLimitExceededError } from "@/chat/services/turn-limit"; +import type { AgentRunResult } from "@/chat/services/turn-result"; +import type { + ConversationWorkerContext, + ConversationWorkerResult, +} from "@/chat/task-execution/worker"; + +/** + * The Run of a prepared Turn. The lifecycle adds the stop signal, the input + * acknowledgement, and the yield check. + */ +export interface PreparedWorkerTurn { + run: AgentRun; + saveResult(result: AgentRunResult): Promise; +} + +/** The steps that one work owner supplies to `runWorkerTurn`. */ +export interface WorkerTurnSteps { + /** + * Start the Turn and read its Run. Return `stopped` when a stop came first. + * Return `finished` when the work already ended and only the ack is left. + */ + prepare(): Promise; + /** Record a stopped Turn. The lifecycle acks after this step. */ + stop(): Promise; + /** Record a Run that paused at a boundary. */ + suspend?(): Promise; + /** Record a Run that waits for authorization, and say how the attempt ends. */ + awaitAuth( + outcome: Extract, + ): Promise<"completed" | "paused">; + /** Finish after `executeTurn` recorded the terminal Turn. */ + complete(): Promise; + /** End the Turn on any attempt for an error that a retry cannot fix. Return true when handled. */ + block?(error: unknown): Promise; + /** Record the failure on the last attempt. */ + fail(error: unknown): Promise; +} + +function isInputCommitLost(error: unknown): boolean { + const cause = getConversationTurnBoundaryError(error)?.cause ?? error; + return isTurnInputCommitLostError(error) || isTurnInputCommitLostError(cause); +} + +/** + * Run one Turn: prepare it, run the agent, and end the attempt. A lost lease + * returns `lost_lease`. A stop ends the Turn as stopped. Other errors retry, + * except on the last attempt or for an execution limit, where the Turn fails. + */ +export async function runWorkerTurn(args: { + agentRunner: AgentRunner; + context: ConversationWorkerContext; + /** True when no attempt input needs an ack, such as on a resume. */ + inputCommitted: boolean; + steps: WorkerTurnSteps; + stopSignal: AbortSignal | undefined; +}): Promise { + const { context, steps, stopSignal } = args; + let acknowledged = args.inputCommitted; + const acknowledge = async (): Promise => { + if (acknowledged) { + return; + } + try { + await context.attempt.ack(); + } catch { + throw new TurnInputCommitLostError( + `Conversation work lease lost before mailbox acknowledgement for ${context.conversationId}`, + ); + } + acknowledged = true; + }; + const end = async ( + status: "completed" | "paused" = "completed", + ): Promise => { + await acknowledge(); + return { status }; + }; + + const advance = async (): Promise => { + const prepared = await steps.prepare(); + if (prepared === "stopped") { + return "stopped"; + } + if (prepared === "finished") { + return await end(); + } + const outcome = await executeTurn( + args.agentRunner, + { + ...prepared.run, + ...(stopSignal ? { signal: stopSignal } : undefined), + durability: { + ...prepared.run.durability, + onInputCommitted: acknowledge, + shouldYield: context.shouldYield, + }, + }, + prepared.saveResult, + ); + if (outcome.status !== "completed" && stopSignal?.aborted) { + return "stopped"; + } + if (outcome.status === "suspended") { + await steps.suspend?.(); + return { status: "yielded" }; + } + if (outcome.status === "awaiting_auth") { + return await end(await steps.awaitAuth(outcome)); + } + await steps.complete(); + return await end(); + }; + + let ended: ConversationWorkerResult | "stopped"; + try { + ended = await advance(); + } catch (error) { + // A save error has no AgentRunError wrapper. It fails the same way. + const failure = error instanceof AgentRunError ? error.cause : error; + if (isInputCommitLost(failure)) { + return { status: "lost_lease" }; + } + if (stopSignal?.aborted) { + ended = "stopped"; + } else if (await steps.block?.(failure)) { + ended = await end(); + } else if ( + // A retry cannot pass an execution limit, and its Turn record would + // hide the limit as a lost worker. + !context.attempt.isFinalAttempt && + !isTurnExecutionLimitExceededError(failure) + ) { + throw failure; + } else { + await steps.fail(failure); + ended = await end(); + } + } + if (ended !== "stopped") { + return ended; + } + try { + await steps.stop(); + return await end(); + } catch (error) { + if (isInputCommitLost(error)) { + return { status: "lost_lease" }; + } + throw error; + } +} From 56dc4e85db417480dfbe70a091ecc1c268039d08 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:01:10 +0000 Subject: [PATCH 08/21] fix(task-execution): Retry a failed finish step after the Turn ends After executeTurn records the terminal Turn, an error in the owner's complete step ran the stop step when a stop was pending, then acked. For a child, that skipped promoting a replaced result, so the parent never got it. Run complete outside the stop and failure handling, so an error there retries the attempt like it did before the shared lifecycle. Reported by Cursor Bugbot on #2499. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../src/chat/task-execution/worker-turn.ts | 18 +++--- .../component/agent-invocation-worker.test.ts | 56 ++++++++++++------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/packages/junior/src/chat/task-execution/worker-turn.ts b/packages/junior/src/chat/task-execution/worker-turn.ts index 1e15234d0e..ce6a2fb5a3 100644 --- a/packages/junior/src/chat/task-execution/worker-turn.ts +++ b/packages/junior/src/chat/task-execution/worker-turn.ts @@ -47,7 +47,7 @@ export interface WorkerTurnSteps { awaitAuth( outcome: Extract, ): Promise<"completed" | "paused">; - /** Finish after `executeTurn` recorded the terminal Turn. */ + /** Finish after `executeTurn` recorded the terminal Turn. An error retries. */ complete(): Promise; /** End the Turn on any attempt for an error that a retry cannot fix. Return true when handled. */ block?(error: unknown): Promise; @@ -95,7 +95,9 @@ export async function runWorkerTurn(args: { return { status }; }; - const advance = async (): Promise => { + const advance = async (): Promise< + ConversationWorkerResult | "completed" | "stopped" + > => { const prepared = await steps.prepare(); if (prepared === "stopped") { return "stopped"; @@ -126,11 +128,10 @@ export async function runWorkerTurn(args: { if (outcome.status === "awaiting_auth") { return await end(await steps.awaitAuth(outcome)); } - await steps.complete(); - return await end(); + return "completed"; }; - let ended: ConversationWorkerResult | "stopped"; + let ended: ConversationWorkerResult | "completed" | "stopped"; try { ended = await advance(); } catch (error) { @@ -155,11 +156,14 @@ export async function runWorkerTurn(args: { ended = await end(); } } - if (ended !== "stopped") { + if (typeof ended !== "string") { return ended; } + // An error after the Turn ends, or while a stop is recorded, retries the + // attempt. It must not run the stop or fail step. Those steps would end + // finished work again and skip its report to the parent. try { - await steps.stop(); + await (ended === "completed" ? steps.complete() : steps.stop()); return await end(); } catch (error) { if (isInputCommitLost(error)) { diff --git a/packages/junior/tests/component/agent-invocation-worker.test.ts b/packages/junior/tests/component/agent-invocation-worker.test.ts index 0b491dafc7..241974d8cc 100644 --- a/packages/junior/tests/component/agent-invocation-worker.test.ts +++ b/packages/junior/tests/component/agent-invocation-worker.test.ts @@ -177,27 +177,33 @@ describe("agent invocation worker", () => { parentConversationId: created.childConversationId, }); const stopRequest = new AbortController(); - const worker = createAgentInvocationWorker({ - run: async () => { - // The stop request arrives after the run finished. - stopRequest.abort(new Error("Conversation work stopped")); - return { - status: "completed" as const, - result: { - diagnostics: { - assistantMessageCount: 1, - modelId: "test-model", - outcome: "success" as const, - toolCalls: [], - toolErrorCount: 0, - toolResultCount: 0, - usedPrimaryText: true, + const enqueueResult = vi + .fn(async () => {}) + .mockRejectedValueOnce(new Error("Parent mailbox unavailable")); + const worker = createAgentInvocationWorker( + { + run: async () => { + // The stop request arrives after the run finished. + stopRequest.abort(new Error("Conversation work stopped")); + return { + status: "completed" as const, + result: { + diagnostics: { + assistantMessageCount: 1, + modelId: "test-model", + outcome: "success" as const, + toolCalls: [], + toolErrorCount: 0, + toolResultCount: 0, + usedPrimaryText: true, + }, + text: "Partial result", }, - text: "Partial result", - }, - }; + }; + }, }, - }); + { enqueueResult }, + ); const context = { attempt: { ack: vi.fn(async () => {}), @@ -212,9 +218,21 @@ describe("agent invocation worker", () => { stopSignal: () => stopRequest.signal, } satisfies ConversationWorkerContext; + // A failed report after the save retries the attempt. The pending stop + // must not end the saved result as stopped. + await expect(worker(context, created.invocationId)).rejects.toThrow( + "Parent mailbox unavailable", + ); + expect(context.attempt.ack).not.toHaveBeenCalled(); await expect(worker(context, created.invocationId)).resolves.toEqual({ status: "completed", }); + expect(enqueueResult).toHaveBeenLastCalledWith( + expect.objectContaining({ + parentMailboxStatus: "pending", + status: "completed", + }), + ); // The open helper would have reported for the child. The stop ends the // helper, so the child's own result reports now. From 8b66a13d5ff3e3f66f50fce609d04afe6ac2894e Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 19:41:19 +0000 Subject: [PATCH 09/21] ref(task-execution): Share core Turn steps between root and child work A child agent invocation is a core Turn with no provider Delivery. Move the steps that root mailbox work and child work both ran into task-execution/core-turn.ts: - createTurnSandbox: read the saved sandbox ref and save each change. - createSteeringDrain: drain pending mailbox input, skip messages that an earlier drain already saved in the Turn, and accept the rest. - saveFinishedTurn: save the sandbox ref, the completed checkpoint, and the automated-turn limit. - recordAutomatedTurnLimit: count an automated Turn, or reset the count for input from the Conversation owner (a person, or the parent agent of a child). Child work also reads history with loadProjection, as root work does. Align three drifts: - A grandchild result now counts toward the automated-turn limit of a child, as a child result counts on a root. - A resumed Slack Turn now records the automated-turn limit, as a Slack Turn that finishes in one slice does. - A completed child checkpoint with no earlier record starts at slice 1, as on a root, instead of failing. The web and event worker now selects Delivery from the input owner (event mailbox input or a saved event Message), not from Source. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../junior/src/chat/agent-invocations/work.ts | 248 +++++++----------- .../junior/src/chat/providers/slack/resume.ts | 6 + .../junior/src/chat/providers/slack/turn.ts | 16 +- .../junior/src/chat/task-execution/README.md | 8 + .../chat/task-execution/conversation-turn.ts | 102 +++---- .../src/chat/task-execution/core-turn.ts | 201 ++++++++++++++ .../src/chat/task-execution/web-steering.ts | 48 ++-- .../agent-invocation-concurrency.test.ts | 3 + .../integration/event-wake-delay.test.ts | 6 + 9 files changed, 369 insertions(+), 269 deletions(-) create mode 100644 packages/junior/src/chat/task-execution/core-turn.ts diff --git a/packages/junior/src/chat/agent-invocations/work.ts b/packages/junior/src/chat/agent-invocations/work.ts index 8eafd43c6a..d21e9fb303 100644 --- a/packages/junior/src/chat/agent-invocations/work.ts +++ b/packages/junior/src/chat/agent-invocations/work.ts @@ -1,10 +1,6 @@ import { isDeepStrictEqual } from "node:util"; import type { StateAdapter } from "chat"; import { z } from "zod"; -import { - buildSteeringPiMessage, - withoutSavedSteeringMessages, -} from "@/chat/agent/prompt"; import type { AgentRun, AgentSteeringMessage } from "@/chat/agent/types"; import type { Location } from "@/chat/conversations/location"; import { @@ -12,7 +8,7 @@ import { sameActorIdentity, } from "@/chat/conversations/provenance"; import type { ConversationStore } from "@/chat/conversations/store"; -import { openConversationProjection } from "@/chat/conversations/projection"; +import { loadProjection } from "@/chat/conversations/projection"; import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { getConversationEventStore, getConversationStore } from "@/chat/db"; import { SYSTEM_IDENTITY_ID } from "@/chat/identities/actor"; @@ -21,29 +17,23 @@ import { finishTurn, type SavedTurnResult, } from "@/chat/runtime/turn-execution"; -import { - getPersistedSandboxState, - getPersistedThreadState, - persistThreadStateById, -} from "@/chat/runtime/thread-state"; +import { getPersistedThreadState } from "@/chat/runtime/thread-state"; import { abandonTurnRecord, failTurnRecord, getTurnRecord, - saveTurnCheckpoint, } from "@/chat/task-execution/checkpoint"; import { getConversationTurnBoundaryError } from "@/chat/runtime/turn"; import { AuthorizationFlowDisabledError } from "@/chat/services/auth-pause"; -import { botConfig } from "@/chat/config"; -import { - recordFinishedTurnForAutomatedLimit, - resetAutomatedTurnLimit, -} from "@/chat/services/automated-turn-limit"; -import { maybePostAutomatedTurnLimitNotice } from "@/chat/slack/automated-turn-limit-notice"; import { PluginCredentialFailureError } from "@/chat/services/plugin-auth-orchestration"; import { getAssistantReplyText } from "@/chat/services/assistant-reply"; import { getTerminalAssistantMessages } from "@/chat/pi/transcript"; -import type { SandboxRef } from "@/chat/sandbox/ref"; +import { + createSteeringDrain, + createTurnSandbox, + saveFinishedTurn, + type TurnSandbox, +} from "@/chat/task-execution/core-turn"; import { agentTypeInstructions, agentTypeStartsWithParentHistory, @@ -272,91 +262,69 @@ function createAgentMessageSteeringDrain( context: ConversationWorkerContext, running: AgentInvocation, ) { - const turnId = getAgentInvocationTurnId(running.invocationId); - return async ( - accept: (messages: AgentSteeringMessage[]) => Promise, - ): Promise => { - const accepted: AgentSteeringMessage[] = []; - await context.attempt.drain( - async (pending) => { - const readIds: string[] = []; - const joining: AgentInvocation[] = []; - for (const message of pending) { - const metadata = agentInvocationMailboxMetadataSchema.safeParse( - message.input.metadata, - ); - if (!metadata.success) { - continue; - } - const candidate = await getAgentInvocation( - metadata.data.agentInvocationId, - ); - // Only a parent task or message steers. A grandchild result or a - // watch event starts its own agent invocation. - if ( - !candidate?.parentToolCallId || - candidate.invocationId === running.invocationId - ) { - continue; - } - if (isTerminalAgentInvocation(candidate)) { - // An earlier drain marked this message but did not acknowledge it. - if (candidate.parentMailboxStatus === "steered") { - readIds.push(message.inboundMessageId); - } - continue; - } - if ( - !sameActorIdentity(candidate.actor, running.actor) || - !isDeepStrictEqual( - candidate.credentialContext, - running.credentialContext, - ) - ) { - break; - } - joining.push(candidate); - readIds.push(message.inboundMessageId); - } - if (joining.length === 0) { - return readIds; + return createSteeringDrain({ + context, + deferred: true, + turnId: getAgentInvocationTurnId(running.invocationId), + select: async (pending) => { + const readIds: string[] = []; + const joining: AgentInvocation[] = []; + for (const message of pending) { + const metadata = agentInvocationMailboxMetadataSchema.safeParse( + message.input.metadata, + ); + if (!metadata.success) { + continue; } - // An earlier drain can save a message in the Turn and then fail - // before it marks the agent invocation. Do not give it to the Turn - // again. - const saved = - (await getTurnRecord(running.childConversationId, turnId)) - ?.piMessages ?? []; - const messages = withoutSavedSteeringMessages( - saved, - joining.map( - (candidate): AgentSteeringMessage => ({ - provenance: agentInstructionProvenance, - text: agentInvocationInputText(candidate), - timestampMs: candidate.createdAtMs, - }), - ), - buildSteeringPiMessage, + const candidate = await getAgentInvocation( + metadata.data.agentInvocationId, ); - // The Turn saves the messages before they are marked. - if (messages.length > 0) { - await accept(messages); - accepted.push(...messages); + // Only a parent task or message steers. A grandchild result or a + // watch event starts its own agent invocation. + if ( + !candidate?.parentToolCallId || + candidate.invocationId === running.invocationId + ) { + continue; } - for (const candidate of joining) { - const terminal = await markAgentInvocationSteered( - candidate.invocationId, - ); - if (terminal && isTerminalAgentInvocation(terminal)) { - await recordSubagentEnded(terminal); + if (isTerminalAgentInvocation(candidate)) { + // An earlier drain marked this message but did not acknowledge it. + if (candidate.parentMailboxStatus === "steered") { + readIds.push(message.inboundMessageId); } + continue; } - return readIds; - }, - { deferred: true }, - ); - return accepted; - }; + if ( + !sameActorIdentity(candidate.actor, running.actor) || + !isDeepStrictEqual( + candidate.credentialContext, + running.credentialContext, + ) + ) { + break; + } + joining.push(candidate); + readIds.push(message.inboundMessageId); + } + return { joining, readIds }; + }, + toSteeringMessage: (candidate): AgentSteeringMessage => ({ + provenance: agentInstructionProvenance, + text: agentInvocationInputText(candidate), + timestampMs: candidate.createdAtMs, + }), + // The Turn saves the messages before they are marked. + afterAccept: async (joining) => { + for (const candidate of joining) { + const terminal = await markAgentInvocationSteered( + candidate.invocationId, + ); + if (terminal && isTerminalAgentInvocation(terminal)) { + await recordSubagentEnded(terminal); + } + } + }, + }); } /** @@ -474,56 +442,30 @@ function blockingInvocationError( : undefined; } -/** Save one completed agent result on its child agent invocation. */ +/** + * Save one completed agent result. The core Turn save comes first. The + * Delivery of a child then saves the result on its agent invocation. + */ async function saveAgentInvocationResult(args: { invocation: AgentInvocation; result: AgentRunResult; - sandboxRef?: SandboxRef; + sandbox: TurnSandbox; turnId: string; }) { const failed = args.result.diagnostics.outcome !== "success"; - await persistThreadStateById(args.invocation.childConversationId, { - sandboxRef: args.result.sandboxRef ?? args.sandboxRef, + await saveFinishedTurn({ + actor: args.invocation.actor, + conversationId: args.invocation.childConversationId, + destination: args.invocation.destination, + // A task or message from the parent resets the automated-turn limit, like + // a person's message on a root. + ownerInput: Boolean(args.invocation.parentToolCallId), + result: args.result, + sandbox: args.sandbox, + source: args.invocation.source, + surface: "internal", + turnId: args.turnId, }); - if (args.result.piMessages?.length) { - await saveTurnCheckpoint({ - mode: "completed", - conversationId: args.invocation.childConversationId, - turnId: args.turnId, - durationMs: args.result.diagnostics.durationMs, - usage: args.result.diagnostics.usage, - destination: args.invocation.destination, - ...(failed - ? { - errorMessage: - args.result.diagnostics.errorMessage ?? "Agent invocation failed", - } - : undefined), - messages: args.result.piMessages, - actor: args.invocation.actor, - source: args.invocation.source, - surface: "internal", - }); - } - // As on a root, a watch event Turn counts toward the automated-turn limit, - // and a task or message from the parent resets it like a user message. - if (args.invocation.parentToolCallId) { - await resetAutomatedTurnLimit({ - conversationId: args.invocation.childConversationId, - }); - } else if (args.invocation.source.kind !== "agent_invocation") { - const maxTurns = botConfig.maxConsecutiveAutomatedTurns; - // A child has no Location, so this skips the notice and clears its claim. - await maybePostAutomatedTurnLimitNotice({ - conversationId: args.invocation.childConversationId, - maxTurns, - update: await recordFinishedTurnForAutomatedLimit({ - conversationId: args.invocation.childConversationId, - maxTurns, - source: args.invocation.source, - }), - }); - } const terminal = await completeAgentInvocation({ invocationId: args.invocation.invocationId, ...(failed @@ -675,12 +617,15 @@ export function createAgentInvocationWorker( }); const [persisted, projection, tree] = await Promise.all([ getPersistedThreadState(invocation.childConversationId), - openConversationProjection({ + loadProjection({ conversationId: invocation.childConversationId, }), readChildDepthAndRootLocation(invocation.childConversationId), ]); - let sandboxRef = getPersistedSandboxState(persisted); + const sandbox = createTurnSandbox({ + conversationId: invocation.childConversationId, + persisted, + }); return { run: { conversationId: invocation.childConversationId, @@ -694,9 +639,9 @@ export function createAgentInvocationWorker( // invocations read it from the child's own Conversation. history: agentTypeStartsWithParentHistory(invocation.agentType) && - projection.messages.length === 0 + projection.length === 0 ? await loadForkHistory(invocation) - : projection.messages, + : projection, actor: invocation.actor, credentialContext: invocation.credentialContext, destination: invocation.destination, @@ -712,26 +657,21 @@ export function createAgentInvocationWorker( tree.depth <= maxAgentDepth, ), state: { - sandboxRef, + sandboxRef: sandbox.ref, }, durability: { drainSteeringMessages: createAgentMessageSteeringDrain( context, invocation, ), - onSandboxRefChanged: async (nextSandboxRef) => { - sandboxRef = nextSandboxRef; - await persistThreadStateById(invocation.childConversationId, { - sandboxRef, - }); - }, + onSandboxRefChanged: sandbox.onSandboxRefChanged, }, }, saveResult: async (result) => await saveAgentInvocationResult({ - invocation: invocation, + invocation, result, - sandboxRef, + sandbox, turnId, }), }; diff --git a/packages/junior/src/chat/providers/slack/resume.ts b/packages/junior/src/chat/providers/slack/resume.ts index cffc7eb333..6c177524aa 100644 --- a/packages/junior/src/chat/providers/slack/resume.ts +++ b/packages/junior/src/chat/providers/slack/resume.ts @@ -49,6 +49,7 @@ import { recordTurnSummary, saveTurnCheckpoint, } from "@/chat/task-execution/checkpoint"; +import { recordAutomatedTurnLimit } from "@/chat/task-execution/core-turn"; import { createSlackWebApiAssistantStatusSession, type AssistantStatusSession, @@ -673,6 +674,11 @@ async function resumeSlackTurnInContext( }); } await runArgs.commitResult?.(reply); + // A resumed Turn counts like a Turn that ran in one slice. + await recordAutomatedTurnLimit({ + conversationId: runArgs.conversationId, + source: run.source, + }); if (reply.diagnostics.outcome === "success") { shouldScheduleCompletedPluginTasks = true; return { diff --git a/packages/junior/src/chat/providers/slack/turn.ts b/packages/junior/src/chat/providers/slack/turn.ts index e4c8c3227a..288d3b87e0 100644 --- a/packages/junior/src/chat/providers/slack/turn.ts +++ b/packages/junior/src/chat/providers/slack/turn.ts @@ -119,8 +119,7 @@ import { getAgentTurnDiagnosticsAttributes, } from "@/chat/services/turn-failure-response"; import { buildAuthPauseResponse } from "@/chat/services/auth-pause-response"; -import { recordFinishedTurnForAutomatedLimit } from "@/chat/services/automated-turn-limit"; -import { maybePostAutomatedTurnLimitNotice } from "@/chat/slack/automated-turn-limit-notice"; +import { recordAutomatedTurnLimit } from "@/chat/task-execution/core-turn"; import { AuthorizationFlowDisabledError } from "@/chat/services/auth-pause"; import { PluginCredentialFailureError } from "@/chat/services/plugin-auth-orchestration"; import { maybeApplyProviderDefaultConfigRequest } from "@/chat/services/provider-default-config"; @@ -1281,18 +1280,7 @@ export function createSlackTurn(deps: SlackTurnDeps) { conversationId, completedState, ); - const automatedTurnLimit = - await recordFinishedTurnForAutomatedLimit({ - conversationId, - maxTurns: botConfig.maxConsecutiveAutomatedTurns, - source, - }); - await maybePostAutomatedTurnLimitNotice({ - conversationId, - maxTurns: botConfig.maxConsecutiveAutomatedTurns, - source, - update: automatedTurnLimit, - }); + await recordAutomatedTurnLimit({ conversationId, source }); } catch (saveError) { // The user already saw the reply. Record the save failure without // posting a second error reply. diff --git a/packages/junior/src/chat/task-execution/README.md b/packages/junior/src/chat/task-execution/README.md index d995187dd3..95144d46ef 100644 --- a/packages/junior/src/chat/task-execution/README.md +++ b/packages/junior/src/chat/task-execution/README.md @@ -17,6 +17,14 @@ attempt. Mailbox work and agent invocation work call `runWorkerTurn`. Each work owner supplies its Run, its result save, and its finish steps. The lifecycle does not check which owner called it. +`core-turn.ts` owns the Turn steps that root and child work share with +different inputs: the sandbox ref save, the steering drain, the completed +Turn save, and the automated-turn limit. A child agent invocation is a core +Turn with no provider Delivery. Its own parts are only its run policy, the +history of a fork, and its Delivery, which saves the result on the agent +invocation and reports it to the parent. Slack Turns use only the +automated-turn limit step today. + `turn-wake.ts` wakes paused turns. `paused-turn.ts` runs them under the conversation lease. SQL conversation events store history. The Redis Turn cursor stores only the data that is needed to resume a Turn, including the diff --git a/packages/junior/src/chat/task-execution/conversation-turn.ts b/packages/junior/src/chat/task-execution/conversation-turn.ts index 2642490464..1a9559c3bb 100644 --- a/packages/junior/src/chat/task-execution/conversation-turn.ts +++ b/packages/junior/src/chat/task-execution/conversation-turn.ts @@ -28,7 +28,6 @@ import { logException, setTags, withLogContext } from "@/chat/logging"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; import { - getPersistedSandboxState, getPersistedThreadState, persistThreadStateById, } from "@/chat/runtime/thread-state"; @@ -40,15 +39,12 @@ import { import { completeAuthPauseTurn } from "@/chat/runtime/auth-pause-state"; import { getTurnUserMessage } from "@/chat/runtime/turn-user-message"; import { getAssistantReplyText } from "@/chat/services/assistant-reply"; -import { botConfig } from "@/chat/config"; import { buildConversationContext, markConversationMessage, normalizeConversationText, upsertConversationMessage, } from "@/chat/services/conversation-memory"; -import { recordFinishedTurnForAutomatedLimit } from "@/chat/services/automated-turn-limit"; -import { maybePostAutomatedTurnLimitNotice } from "@/chat/slack/automated-turn-limit-notice"; import { finalizeFailedTurnReplyWithEvent } from "@/chat/services/turn-failure-response"; import { clearPendingAuth } from "@/chat/services/pending-auth"; import { @@ -60,8 +56,11 @@ import { abandonTurnRecord, getTurnRecord, listTurnSummaries, - saveTurnCheckpoint, } from "@/chat/task-execution/checkpoint"; +import { + createTurnSandbox, + saveFinishedTurn, +} from "@/chat/task-execution/core-turn"; import type { ConversationWorkerContext, ConversationWorkerResult, @@ -199,11 +198,13 @@ function sharedAuthorIdentityId( } /** - * Create the shared mailbox worker for web and event Turns. + * Create the shared mailbox worker for web and event Turns. Web input has no + * provider Delivery, so its replies stay in Junior. `eventDelivery` is the + * Delivery of event input, which sends replies to the Conversation Location. */ export function createConversationTurnWorker( agentRunner: AgentRunner, - deliverMessage?: DeliverMessage, + eventDelivery?: DeliverMessage, ) { return async ( context: ConversationWorkerContext, @@ -233,6 +234,8 @@ export function createConversationTurnWorker( let event: EventMailboxMetadata["event"] | undefined; let batchEvents: ConversationMessageEvent[] = []; let authorIdentityId: string | undefined; + // The input owner selects Delivery. Source must not select it. + let eventInput = false; const storedConversation = await getConversationStore().get({ conversationId: context.conversationId, @@ -253,6 +256,7 @@ export function createConversationTurnWorker( const messages = resolved.batch.map((entry) => entry.message); const firstMetadata = first.message.input.metadata; if (isEventMailboxMetadata(firstMetadata)) { + eventInput = true; text = renderEventBatchText(messages); event = firstMetadata.event; batchEvents = eventBatchMessageEvents(messages); @@ -283,7 +287,6 @@ export function createConversationTurnWorker( const savedTurn = isResume ? await getTurnRecord(context.conversationId, turnId) : undefined; - let savedMessageIsEvent = false; if (isResume) { const userMessage = getTurnUserMessage(conversation, turnId); if (!userMessage) { @@ -291,7 +294,7 @@ export function createConversationTurnWorker( `Unable to locate the persisted user message for Turn "${turnId}"`, ); } - savedMessageIsEvent = isEventConversationMessage(userMessage); + eventInput = isEventConversationMessage(userMessage); // Resume has no new input. Restore Source and Actor from the Turn. // TODO(dcramer): Remove the saved Message fallback after no deployed Turn // cursor can omit Source or Actor. @@ -335,14 +338,7 @@ export function createConversationTurnWorker( }); const webActor = actor.platform === "web" ? actor : undefined; const conversationLocation = storedConversation?.location; - // TODO(dcramer): Remove the saved Message check after every deployed Turn - // cursor stores Event Source. - // TODO(dcramer): Remove this Source-based Delivery choice after the core - // Turn lifecycle stores assistant Messages and web and Event work - // each supplies optional provider Delivery. Source must not select Delivery. - const deliverToProvider = - Boolean(conversationLocation) && - (source.kind === "event" || savedMessageIsEvent); + const deliverToProvider = Boolean(conversationLocation) && eventInput; // TODO(dcramer): Stop deriving surface from Delivery after active Turn // lookup and reporting read Source for web and Event Turns. const surface = deliverToProvider ? ("slack" as const) : ("api" as const); @@ -361,9 +357,11 @@ export function createConversationTurnWorker( : undefined), }, async () => { - let sandboxRef: SandboxRef | undefined = - getPersistedSandboxState(persisted); - const initialSandboxRef = sandboxRef; + const sandbox = createTurnSandbox({ + conversation, + conversationId: context.conversationId, + persisted, + }); const stopSignal = context.stopSignal?.(); let currentRunId: string | undefined; @@ -386,12 +384,12 @@ export function createConversationTurnWorker( const cards = await loadPendingMessageCards(context.conversationId); let deliveryResult: DeliveryResult | undefined; if (deliverToProvider) { - if (!conversationLocation || !deliverMessage) { + if (!conversationLocation || !eventDelivery) { throw new Error( `Conversation ${context.conversationId} cannot deliver to its Location`, ); } - deliveryResult = await deliverMessage({ + deliveryResult = await eventDelivery({ cards, conversationId: context.conversationId, location: conversationLocation, @@ -566,7 +564,7 @@ export function createConversationTurnWorker( : { disabledFeatures: ["interactive-auth"] as const }), state: { pendingAuth: conversation.processing.pendingAuth, - sandboxRef, + sandboxRef: sandbox.ref, }, delivery: deliverAssistantMessage, durability: { @@ -581,18 +579,12 @@ export function createConversationTurnWorker( }), } : undefined), - onSandboxRefChanged: async (nextSandboxRef) => { - sandboxRef = nextSandboxRef; - await persistThreadStateById(context.conversationId, { - conversation, - sandboxRef, - }); - }, + onSandboxRefChanged: sandbox.onSandboxRefChanged, recordPendingAuth: async (pendingAuth) => { conversation.processing.pendingAuth = pendingAuth; await persistThreadStateById(context.conversationId, { conversation, - sandboxRef, + sandboxRef: sandbox.ref, }); }, }, @@ -624,44 +616,16 @@ export function createConversationTurnWorker( sessionId: turnId, userMessageId, }); - await persistThreadStateById(context.conversationId, { + await saveFinishedTurn({ + actor, conversation: completedState.conversation, - sandboxRef: reply.sandboxRef ?? sandboxRef, - }); - if (reply.piMessages?.length) { - // Prefer the live checkpoint slice after yield/resume; first - // completion has no prior record and starts at slice 1. - const latest = await getTurnRecord( - context.conversationId, - turnId, - ); - await saveTurnCheckpoint({ - mode: "completed", - conversationId: context.conversationId, - turnId, - sliceId: latest?.sliceId ?? 1, - messages: reply.piMessages, - durationMs: reply.diagnostics.durationMs, - usage: reply.diagnostics.usage, - // TODO(dcramer): Remove checkpoint Destination after resume - // reads the Conversation Location. - destination, - source, - actor, - surface, - }); - } - const automatedTurnLimit = - await recordFinishedTurnForAutomatedLimit({ - conversationId: context.conversationId, - maxTurns: botConfig.maxConsecutiveAutomatedTurns, - source, - }); - await maybePostAutomatedTurnLimitNotice({ conversationId: context.conversationId, - maxTurns: botConfig.maxConsecutiveAutomatedTurns, + destination, + result: reply, + sandbox, source, - update: automatedTurnLimit, + surface, + turnId, }); completedSuccessfully = reply.diagnostics.outcome === "success"; @@ -694,7 +658,7 @@ export function createConversationTurnWorker( await completeCancelledConversationTurn({ conversation, conversationId: context.conversationId, - sandboxRef, + sandboxRef: sandbox.ref, turnId, userMessageId, }); @@ -709,7 +673,7 @@ export function createConversationTurnWorker( }); await persistThreadStateById(context.conversationId, { conversation, - sandboxRef, + sandboxRef: sandbox.ref, }); return "paused"; }, @@ -774,7 +738,7 @@ export function createConversationTurnWorker( }); await persistThreadStateById(context.conversationId, { conversation, - sandboxRef: initialSandboxRef ?? null, + sandboxRef: sandbox.initial ?? null, }); } catch (persistenceError) { captureConversationTurnFailure({ diff --git a/packages/junior/src/chat/task-execution/core-turn.ts b/packages/junior/src/chat/task-execution/core-turn.ts new file mode 100644 index 0000000000..77439b1398 --- /dev/null +++ b/packages/junior/src/chat/task-execution/core-turn.ts @@ -0,0 +1,201 @@ +/** + * Turn steps that every work owner shares. A root Turn and a child agent + * invocation give different inputs to the same steps. Only the run policy, + * the history of a fork, and Delivery stay with each owner. + */ +import type { Actor, Destination, Source } from "@sentry/junior-plugin-api"; +import { + buildSteeringPiMessage, + withoutSavedSteeringMessages, +} from "@/chat/agent/prompt"; +import type { AgentSteeringMessage } from "@/chat/agent/types"; +import { botConfig } from "@/chat/config"; +import { + getPersistedSandboxState, + persistThreadStateById, +} from "@/chat/runtime/thread-state"; +import type { SandboxRef } from "@/chat/sandbox/ref"; +import { recordFinishedTurnForAutomatedLimit } from "@/chat/services/automated-turn-limit"; +import type { AgentRunResult } from "@/chat/services/turn-result"; +import { maybePostAutomatedTurnLimitNotice } from "@/chat/slack/automated-turn-limit-notice"; +import type { ThreadConversationState } from "@/chat/state/conversation"; +import { + getTurnRecord, + saveTurnCheckpoint, + type AgentTurnSurface, +} from "@/chat/task-execution/checkpoint"; +import type { InboundMessage } from "@/chat/task-execution/store"; +import type { ConversationWorkerContext } from "@/chat/task-execution/worker"; + +/** The sandbox of one Turn. Each change is saved on the Conversation. */ +export interface TurnSandbox { + /** The sandbox ref when the Turn started. */ + readonly initial: SandboxRef | undefined; + /** The latest sandbox ref of the Turn. */ + readonly ref: SandboxRef | undefined; + /** Save a new sandbox ref. Use it as `onSandboxRefChanged` of the Run. */ + onSandboxRefChanged(next: SandboxRef): Promise; +} + +/** + * Read the saved sandbox ref of a Conversation and save each change. When the + * owner keeps transcript state, `conversation` is saved with the ref. + */ +export function createTurnSandbox(args: { + conversation?: ThreadConversationState; + conversationId: string; + persisted: Record; +}): TurnSandbox { + const initial = getPersistedSandboxState(args.persisted); + let ref = initial; + return { + initial, + get ref() { + return ref; + }, + onSandboxRefChanged: async (next) => { + ref = next; + await persistThreadStateById(args.conversationId, { + ...(args.conversation + ? { conversation: args.conversation } + : undefined), + sandboxRef: ref, + }); + }, + }; +} + +/** + * Count one finished Turn toward the automated-turn limit, or reset the count + * when the Turn answers its owner. A root's owner is a person. A child's + * owner is its parent agent. Post the pause notice when the limit is full. + */ +export async function recordAutomatedTurnLimit(args: { + conversationId: string; + /** True when the input came from the owner of the Conversation. */ + ownerInput?: boolean; + source: Source; +}): Promise { + const maxTurns = botConfig.maxConsecutiveAutomatedTurns; + const update = await recordFinishedTurnForAutomatedLimit({ + conversationId: args.conversationId, + maxTurns, + // An owner input resets the count like a person's message. + source: args.ownerInput ? undefined : args.source, + }); + // A Conversation without a Slack Location skips the notice and clears its + // claim. + await maybePostAutomatedTurnLimitNotice({ + conversationId: args.conversationId, + maxTurns, + source: args.source, + update, + }); +} + +/** + * Save a finished Run on its Conversation: the sandbox ref, the completed + * Turn checkpoint, and the automated-turn limit. `conversation` is the + * transcript state to save, when the owner keeps one. + */ +export async function saveFinishedTurn(args: { + actor: Actor; + conversation?: ThreadConversationState; + conversationId: string; + destination?: Destination; + ownerInput?: boolean; + result: AgentRunResult; + sandbox: TurnSandbox; + source: Source; + surface: AgentTurnSurface; + turnId: string; +}): Promise { + const { result } = args; + await persistThreadStateById(args.conversationId, { + ...(args.conversation ? { conversation: args.conversation } : undefined), + sandboxRef: result.sandboxRef ?? args.sandbox.ref, + }); + if (result.piMessages?.length) { + // Continue the slice of the latest checkpoint after a yield or resume. A + // Turn with no checkpoint yet starts at slice 1. + const latest = await getTurnRecord(args.conversationId, args.turnId); + await saveTurnCheckpoint({ + mode: "completed", + conversationId: args.conversationId, + turnId: args.turnId, + sliceId: latest?.sliceId ?? 1, + messages: result.piMessages, + durationMs: result.diagnostics.durationMs, + usage: result.diagnostics.usage, + ...(result.diagnostics.outcome !== "success" + ? { errorMessage: result.diagnostics.errorMessage ?? "Turn failed" } + : undefined), + // TODO(dcramer): Remove checkpoint Destination after resume reads the + // Conversation Location. + destination: args.destination, + source: args.source, + actor: args.actor, + surface: args.surface, + }); + } + await recordAutomatedTurnLimit({ + conversationId: args.conversationId, + ownerInput: args.ownerInput, + source: args.source, + }); +} + +/** + * Give pending mailbox input to the running Turn as steering messages. + * `select` picks the input that joins, and the mailbox ids to mark read. The + * first input that cannot join must stop the scan, so later input waits too. + * A message that a failed earlier drain already saved in the Turn does not + * join again. + */ +export function createSteeringDrain(args: { + context: ConversationWorkerContext; + /** Drain deferred input too, such as messages from a parent agent. */ + deferred?: boolean; + select(pending: InboundMessage[]): Promise<{ + joining: T[]; + readIds: string[]; + }>; + /** Save the joining input before the Turn reads it. */ + beforeAccept?(joining: T[]): Promise; + /** Mark the joining input after the Turn saved it. */ + afterAccept?(joining: T[]): Promise; + toSteeringMessage(item: T): AgentSteeringMessage; + turnId: string; +}) { + const { context } = args; + return async ( + accept: (messages: AgentSteeringMessage[]) => Promise, + ): Promise => { + const accepted: AgentSteeringMessage[] = []; + await context.attempt.drain( + async (pending) => { + const { joining, readIds } = await args.select(pending); + if (joining.length === 0) { + return readIds; + } + await args.beforeAccept?.(joining); + const saved = + (await getTurnRecord(context.conversationId, args.turnId)) + ?.piMessages ?? []; + const messages = withoutSavedSteeringMessages( + saved, + joining.map(args.toSteeringMessage), + buildSteeringPiMessage, + ); + if (messages.length > 0) { + await accept(messages); + accepted.push(...messages); + } + await args.afterAccept?.(joining); + return readIds; + }, + args.deferred ? { deferred: true } : undefined, + ); + return accepted; + }; +} diff --git a/packages/junior/src/chat/task-execution/web-steering.ts b/packages/junior/src/chat/task-execution/web-steering.ts index 8f46e5d577..15571e709b 100644 --- a/packages/junior/src/chat/task-execution/web-steering.ts +++ b/packages/junior/src/chat/task-execution/web-steering.ts @@ -1,9 +1,5 @@ import type { WebActor } from "@/chat/actor"; import type { AgentSteeringMessage } from "@/chat/agent/types"; -import { - buildSteeringPiMessage, - withoutSavedSteeringMessages, -} from "@/chat/agent/prompt"; import { readMessageAttachments } from "@/chat/attachments/input"; import { persistConversationMessages } from "@/chat/conversations/messages"; import { instructionProvenanceFor } from "@/chat/conversations/provenance"; @@ -16,7 +12,7 @@ import type { ConversationAuthor, ThreadConversationState, } from "@/chat/state/conversation"; -import { getTurnRecord } from "@/chat/task-execution/checkpoint"; +import { createSteeringDrain } from "@/chat/task-execution/core-turn"; import type { InboundMessage } from "@/chat/task-execution/store"; import type { ConversationWorkerContext } from "@/chat/task-execution/worker"; @@ -53,11 +49,10 @@ export function createWebSteeringDrain(args: { turnId: string; }) { const { actor, author, context, conversation, turnId } = args; - return async ( - accept: (messages: AgentSteeringMessage[]) => Promise, - ): Promise => { - const accepted: AgentSteeringMessage[] = []; - await context.attempt.drain(async (pending) => { + return createSteeringDrain({ + context, + turnId, + select: async (pending) => { const joining = pending.filter( (message) => message.source === "web" && @@ -65,10 +60,14 @@ export function createWebSteeringDrain(args: { legacyWebMailboxMetadataSchema.safeParse(message.input.metadata) .success, ); - if (joining.length === 0) return []; - - // Save the transcript Message first. A failed write leaves the input - // pending instead of missing from the transcript. + return { + joining, + readIds: joining.map((message) => message.inboundMessageId), + }; + }, + // Save the transcript Message first. A failed write leaves the input + // pending instead of missing from the transcript. + beforeAccept: async (joining) => { for (const message of joining) { const attachments = readMessageAttachments(message.input.attachments); upsertConversationMessage(conversation, { @@ -91,22 +90,7 @@ export function createWebSteeringDrain(args: { conversation, conversationId: context.conversationId, }); - - // An earlier drain can save the messages in the Turn and fail before - // the mailbox ack. Do not give them to the Turn again. - const saved = - (await getTurnRecord(context.conversationId, turnId))?.piMessages ?? []; - const messages = withoutSavedSteeringMessages( - saved, - joining.map((message) => steeringMessageFromWeb(message, actor)), - buildSteeringPiMessage, - ); - if (messages.length > 0) { - await accept(messages); - accepted.push(...messages); - } - return joining.map((message) => message.inboundMessageId); - }); - return accepted; - }; + }, + toSteeringMessage: (message) => steeringMessageFromWeb(message, actor), + }); } diff --git a/packages/junior/tests/integration/agent-invocation-concurrency.test.ts b/packages/junior/tests/integration/agent-invocation-concurrency.test.ts index b5cf833c31..e82e9f14d8 100644 --- a/packages/junior/tests/integration/agent-invocation-concurrency.test.ts +++ b/packages/junior/tests/integration/agent-invocation-concurrency.test.ts @@ -497,10 +497,13 @@ describe("agent invocation identity and concurrency", () => { const clock = vi.spyOn(Date, "now").mockReturnValue(1_800_000_000_000); const harness = await createHarness(taskModel); try { + // As from spawnAgent, the task has the parent's tool call, so it + // resets the automated-turn count. const child = await harness.spawn({ agentName: "watcher", idempotencyKey: "watch-1", input: "Watch the deploy.", + parentToolCallId: "spawn-watcher", }); await harness.drainQueuedWork(); diff --git a/packages/junior/tests/integration/event-wake-delay.test.ts b/packages/junior/tests/integration/event-wake-delay.test.ts index be455f84b5..880329f13c 100644 --- a/packages/junior/tests/integration/event-wake-delay.test.ts +++ b/packages/junior/tests/integration/event-wake-delay.test.ts @@ -15,6 +15,7 @@ import { type MailboxTurnWork, } from "@/chat/task-execution/mailbox-turn"; import { createConversationTurnWorker } from "@/chat/task-execution/conversation-turn"; +import { getAutomatedTurnLimitState } from "@/chat/services/automated-turn-limit"; import { getTurnRecord } from "@/chat/task-execution/checkpoint"; import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; import type { @@ -227,6 +228,11 @@ describe("event wake delay", () => { nowSpy.mockReturnValue(baseMs + 45_000); await expect(processNext()).resolves.toEqual({ status: "completed" }); expect(agentRuns).toHaveLength(2); + // The web Turn reset the automated-turn count, and the one event Turn + // counts once. A child agent uses the same core Turn save. + await expect( + getAutomatedTurnLimitState({ conversationId }), + ).resolves.toMatchObject({ consecutiveAutomatedTurns: 1 }); const savedEvents = await getConversationEventStore().loadHistory(conversationId); From 31d57a3a6938daafc35b8da44a74e41105a7a1df Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:00:20 +0000 Subject: [PATCH 10/21] test(task-execution): Cover the Turn lifecycle of every work owner Add one table to the durable queue tests. It runs each work owner through production routing with the real agent: a root web Turn, a root event Turn at a Slack Location, a root Slack mention, a dispatch, a child from a parent task, and a child from a watch event. Each owner gets new input, a yield and empty resume wake, a stop mid-Turn, an attempt that throws every time, and a resume that throws every time. The table found these bugs, which this commit fixes: - A stop that aborted a child Run saved the Run instead of ending the Turn as stopped. The stop check now runs in the shared worker lifecycle for root and child work. A Run that finished before the stop keeps its result. - A root event Turn that failed on its last attempt posted no error reply at its Slack Location. It now posts the standard failure reply. - A dispatch resume failed, because the wake and the resume read a Slack session Source that a dispatch Conversation does not have. They now use the saved Source of the dispatch. The table moves the durable queue helper onto the shared conversation work harness, so the file runs the agent from one place. The owner and signal rows are in tests/fixtures/turn-lifecycle.ts to keep the test file under the length limit. Dispatch stop and failure rows are not in the table yet. Dispatch work still runs its own lifecycle. The next commit moves it onto the shared one. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../chat/task-execution/conversation-turn.ts | 32 +- .../src/chat/task-execution/paused-turn.ts | 17 +- .../src/chat/task-execution/turn-wake.ts | 21 +- .../src/chat/task-execution/worker-turn.ts | 9 +- .../junior/tests/fixtures/model-stream.ts | 10 +- .../junior/tests/fixtures/turn-lifecycle.ts | 444 ++++++++++++++++++ .../tests/integration/durable-queue.test.ts | 165 +++++-- scripts/test-architecture-baseline.json | 3 +- 8 files changed, 634 insertions(+), 67 deletions(-) create mode 100644 packages/junior/tests/fixtures/turn-lifecycle.ts diff --git a/packages/junior/src/chat/task-execution/conversation-turn.ts b/packages/junior/src/chat/task-execution/conversation-turn.ts index 1a9559c3bb..ef5bf9a5a6 100644 --- a/packages/junior/src/chat/task-execution/conversation-turn.ts +++ b/packages/junior/src/chat/task-execution/conversation-turn.ts @@ -24,7 +24,12 @@ import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-life import type { ConversationTurnFailureCode } from "@/chat/conversations/history"; import { credentialContextForActor } from "@/chat/credentials/context"; import { getConversationEventStore, getConversationStore } from "@/chat/db"; -import { logException, setTags, withLogContext } from "@/chat/logging"; +import { + buildTurnFailureResponse, + logException, + setTags, + withLogContext, +} from "@/chat/logging"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; import { @@ -590,13 +595,6 @@ export function createConversationTurnWorker( }, }, saveResult: async (result) => { - if (stopSignal?.aborted) { - throw ( - stopSignal.reason ?? - new DOMException("Turn cancelled", "AbortError") - ); - } - modelFailureCaptureAttempted = result.diagnostics.outcome !== "success"; const finalized = finalizeFailedTurnReplyWithEvent({ @@ -756,6 +754,24 @@ export function createConversationTurnWorker( failureCode, turnId, }); + // Tell the people at the Location that the Turn failed. Web + // input has no provider Delivery. The dashboard shows the + // failed Turn. + if (deliverToProvider && failureEventId) { + try { + await deliverAssistantMessage( + buildTurnFailureResponse(failureEventId), + ); + } catch (deliveryError) { + captureConversationTurnFailure({ + conversationId: context.conversationId, + error: deliveryError, + failureCode: "delivery_failed", + runId: currentRunId, + turnId, + }); + } + } }, }, }); diff --git a/packages/junior/src/chat/task-execution/paused-turn.ts b/packages/junior/src/chat/task-execution/paused-turn.ts index f125e8d3de..e234799438 100644 --- a/packages/junior/src/chat/task-execution/paused-turn.ts +++ b/packages/junior/src/chat/task-execution/paused-turn.ts @@ -551,9 +551,20 @@ async function runPausedTurnInContext( excludeMessageId: userMessage.id, }); const sandboxRef = getPersistedSandboxState(currentState); - const routing = await resolveTurnSessionRouting({ - conversationId: payload.conversationId, - }); + // A dispatch Turn keeps the Source of its dispatch, as its first slice + // does. Its Conversation has no Slack session Source to read. + const routing = + dispatchId && activeTurn.source + ? { + ...(await resolveConversationRouting({ + conversationId: payload.conversationId, + })), + destination, + source: activeTurn.source, + } + : await resolveTurnSessionRouting({ + conversationId: payload.conversationId, + }); const source = routing.source; const routingDestination = routing.destination; diff --git a/packages/junior/src/chat/task-execution/turn-wake.ts b/packages/junior/src/chat/task-execution/turn-wake.ts index 65b771652c..89d34baf70 100644 --- a/packages/junior/src/chat/task-execution/turn-wake.ts +++ b/packages/junior/src/chat/task-execution/turn-wake.ts @@ -11,8 +11,8 @@ import { import type { Destination } from "@sentry/junior-plugin-api"; import type { ConversationStore } from "@/chat/conversations/store"; import { + resolveConversationRouting, resolveTurnSessionRouting, - type RequiredTurnSessionRouting, } from "@/chat/services/turn-session-routing"; import { failTurnRecord, @@ -71,12 +71,19 @@ export async function getPausedTurnRequest(args: { ) { return undefined; } - let routing: RequiredTurnSessionRouting; + let destination: Destination; try { - routing = await resolveTurnSessionRouting({ - conversationId: args.conversationId, - conversationStore: args.conversationStore, - }); + // A dispatch Turn keeps the Source of its dispatch. The wake needs only + // the Destination of its Conversation. + const routing = turn.dispatchId + ? await resolveConversationRouting(args) + : await resolveTurnSessionRouting(args); + if (!routing?.destination) { + throw new Error( + `Conversation ${args.conversationId} is missing durable routing metadata`, + ); + } + destination = routing.destination; } catch (error) { await failTurnRecord({ conversationId: args.conversationId, @@ -89,7 +96,7 @@ export async function getPausedTurnRequest(args: { return { conversationId: args.conversationId, - destination: routing.destination, + destination, turnId: args.turnId, expectedVersion: turn.version, traceContext: turn.traceContext, diff --git a/packages/junior/src/chat/task-execution/worker-turn.ts b/packages/junior/src/chat/task-execution/worker-turn.ts index ce6a2fb5a3..7b51952cd6 100644 --- a/packages/junior/src/chat/task-execution/worker-turn.ts +++ b/packages/junior/src/chat/task-execution/worker-turn.ts @@ -116,7 +116,14 @@ export async function runWorkerTurn(args: { shouldYield: context.shouldYield, }, }, - prepared.saveResult, + async (result) => { + // A stop that aborted the Run ends the Turn as stopped. A Run that + // finished before the stop keeps its result. + if (result.diagnostics.outcome !== "success") { + stopSignal?.throwIfAborted(); + } + return await prepared.saveResult(result); + }, ); if (outcome.status !== "completed" && stopSignal?.aborted) { return "stopped"; diff --git a/packages/junior/tests/fixtures/model-stream.ts b/packages/junior/tests/fixtures/model-stream.ts index 1a8b8e6ece..3555d5b99a 100644 --- a/packages/junior/tests/fixtures/model-stream.ts +++ b/packages/junior/tests/fixtures/model-stream.ts @@ -7,12 +7,15 @@ import { type FauxResponseStep, } from "@earendil-works/pi-ai/providers/faux"; -type FixedModelOutput = +/** One fixed model output for one model request. */ +export type FixedModelOutput = | { type: "text"; text: string; onRequest?: () => void; waitFor?: Promise | "abort"; + /** Stop waiting for the abort after this time, as a run that ignores it. */ + abortTimeoutMs?: number; } | { type: "toolCall"; @@ -55,6 +58,8 @@ function createAssistantMessage(output: FixedModelOutput): AssistantMessage { function createResponseStep(output: FixedModelOutput): FauxResponseStep { const message = createAssistantMessage(output); const { onRequest, waitFor } = output; + const abortTimeoutMs = + output.type === "text" ? output.abortTimeoutMs : undefined; if (!onRequest && !waitFor) { return message; } @@ -68,6 +73,9 @@ function createResponseStep(output: FixedModelOutput): FauxResponseStep { if (!signal.aborted) { await new Promise((resolve) => { signal.addEventListener("abort", () => resolve(), { once: true }); + if (abortTimeoutMs !== undefined) { + setTimeout(resolve, abortTimeoutMs); + } }); } } else if (waitFor) { diff --git a/packages/junior/tests/fixtures/turn-lifecycle.ts b/packages/junior/tests/fixtures/turn-lifecycle.ts new file mode 100644 index 0000000000..aa0fbb289e --- /dev/null +++ b/packages/junior/tests/fixtures/turn-lifecycle.ts @@ -0,0 +1,444 @@ +/** + * The work owners and lifecycle signals of the Turn lifecycle table in + * `integration/durable-queue.test.ts`. Each owner starts its own work through + * production entry points. Each signal sets the model output and says what the + * owner shows after the Turn ends. + */ +import { expect } from "vitest"; +import { createSlackSource } from "@sentry/junior-plugin-api"; +import { enqueueAgentDispatch } from "@/chat/agent-dispatch/work"; +import { + getDispatchConversationId, + getDispatchRecord, +} from "@/chat/agent-dispatch/store"; +import { + getAgentInvocation, + getLatestAgentInvocationForConversation, +} from "@/chat/agent-invocations/store"; +import { createAndEnqueueAgentInvocation } from "@/chat/agent-invocations/work"; +import { createAndEnqueueConversation } from "@/chat/conversations/web-input"; +import { getConversationEventStore, getConversationStore } from "@/chat/db"; +import { getEventCatalog } from "@/chat/events/runtime-catalog"; +import { createEventInboundMessage } from "@/chat/events/notification"; +import { runTimerWatchHeartbeat } from "@/chat/events/timers"; +import { + appendAndEnqueueInboundMessage, + CONVERSATION_WORK_MAX_RETRIES, +} from "@/chat/task-execution/store"; +import { createEventTools } from "@/chat/tools/events"; +import { createAgentDispatchTestRecord } from "./agent-dispatch"; +import { + invocationInput, + loadSubagentEvents, + parentConversationId, + prepareParentConversation, +} from "./agent-invocations"; +import { webTestActor } from "./conversation"; +import { + CONVERSATION_ID, + SLACK_BOT_USER_ID, + SLACK_DESTINATION, + type createConversationWorkHarness, + handleSlackWebhookAndFlush, + slackEnvelope, + slackWebhookRequest, +} from "./conversation-work"; +import { context, execute } from "./event-automations"; +import type { FixedModelOutput } from "./model-stream"; +import { slackApiOutbox } from "./slack-api-outbox"; + +type Harness = Awaited>; + +/** Texts of the Slack messages that the work posted. */ +export function slackReplies(): string[] { + return slackApiOutbox + .messages() + .map((call) => call.params.text) + .filter((text): text is string => typeof text === "string"); +} + +/** Where an owner sends what its Turn says. */ +type Delivery = "junior" | "slack" | "parent"; + +export interface Owner { + name: string; + delivery: Delivery; + /** Model output that ends a Turn of this owner with `text`. */ + reply?(text: string): FixedModelOutput; + /** Model output that the owner uses before its Turn under test. */ + setupOutputs?: FixedModelOutput[]; + /** Create the Conversation and queue new input. Return its id. */ + start(harness: Harness): Promise; + /** Run the Turns that must finish before the input under test. */ + setup?(args: { + conversationId: string; + harness: Harness; + process: (softYieldAfterMs?: number) => Promise; + }): Promise; + /** Read what the owner shows after the Turn ends. */ + read(conversationId: string): Promise>; + /** + * Wake results when every attempt throws, where the owner still runs its + * own retry lifecycle instead of the shared one. + */ + failedWakes?: string[]; + /** Wake results when every resume attempt throws. */ + failedResumeWakes?: string[]; + /** True when a failed Turn posts no error reply, by owner policy. */ + quietFailure?: boolean; + /** Signals that this owner does not handle through the shared lifecycle. */ + unsupported?: string[]; +} + +export const clock = { nowMs: 1_800_000_000_000 }; +export const closers: Array<() => Promise> = []; + +/** Read how the last Turn of a Conversation ended. */ +export async function lastTurnEnd(conversationId: string): Promise { + const last = (await getConversationEventStore().loadHistory(conversationId)) + .map((event) => event.data) + .filter( + (data) => data.type === "turn_completed" || data.type === "turn_failed", + ) + .at(-1); + if (!last) return "running"; + if (last.type === "turn_failed") return "failed"; + return last.outcome === "cancelled" ? "stopped" : "completed"; +} + +async function assistantMessages(conversationId: string): Promise { + return (await getConversationEventStore().loadHistory(conversationId)) + .map((event) => event.data) + .flatMap((data) => + data.type === "message" && data.role === "assistant" ? [data.text] : [], + ); +} + +/** Read the latest agent invocation of a child and its report to the parent. */ +async function readChild(conversationId: string) { + const latest = await getLatestAgentInvocationForConversation(conversationId); + const invocation = latest && (await getAgentInvocation(latest.invocationId)); + const ended = (await loadSubagentEvents(parentConversationId)).filter( + (event) => + event.type === "subagent_ended" && + event.subagentInvocationId === invocation?.invocationId, + ); + return { + invocation: invocation?.status, + parentMailbox: + invocation && "parentMailboxStatus" in invocation + ? invocation.parentMailboxStatus + : undefined, + reportsToParent: ended.length, + }; +} + +async function spawnChild(harness: Harness, idempotencyKey: string) { + const { conversationStore, fixture } = await prepareParentConversation(); + closers.push(async () => await fixture.close()); + const created = await createAndEnqueueAgentInvocation( + { ...invocationInput, idempotencyKey }, + { conversationStore, queue: harness.queue, state: harness.state }, + ); + return created.childConversationId; +} + +export const owners: Owner[] = [ + { + name: "root web", + delivery: "junior", + start: async (harness) => + ( + await createAndEnqueueConversation( + { + actor: webTestActor, + idempotencyKey: "lifecycle-web", + message: "Check the deploy.", + }, + { + conversationStore: getConversationStore(), + queue: harness.queue, + state: harness.state, + }, + ) + ).conversationId, + read: async (conversationId) => ({ + replies: await assistantMessages(conversationId), + }), + }, + { + name: "root event with a Location", + delivery: "slack", + start: async (harness) => { + const conversationId = "agent-dispatch:lifecycle-watch"; + await getConversationStore().recordActivity({ + conversationId, + destination: SLACK_DESTINATION, + nowMs: 1_000, + sessionSource: createSlackSource({ + channelId: SLACK_DESTINATION.channelId, + teamId: SLACK_DESTINATION.teamId, + visibility: "public", + }), + source: "event", + visibility: "public", + }); + await appendAndEnqueueInboundMessage({ + message: createEventInboundMessage({ + event: { + eventKey: "lifecycle-checks-failed", + eventType: "pull_request.checks.failed", + identifier: "getsentry/junior#1563", + namespace: "github", + occurredAtMs: 1_100, + trustedSummary: "The subscribed check failed.", + }, + receivedAtMs: 1_100, + subscription: { conversationId, id: "lifecycle-subscription" }, + text: "The subscribed check failed.", + }), + queue: harness.queue, + state: harness.state, + }); + return conversationId; + }, + read: async () => ({ replies: slackReplies() }), + }, + { + name: "root Slack", + delivery: "slack", + // Slack mailbox work requeues its message after a failed attempt and + // dead-letters it on the last attempt. + failedWakes: [ + ...Array.from( + { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, + () => "pending_requeued", + ), + "failed", + ], + // The Slack resume runner fails the Turn on the first error. + failedResumeWakes: ["completed"], + start: async (harness) => { + const response = await handleSlackWebhookAndFlush({ + request: slackWebhookRequest( + slackEnvelope({ text: `<@${SLACK_BOT_USER_ID}> check the deploy` }), + ), + services: { + getSlackAdapter: () => harness.adapter, + queue: harness.queue, + runtime: harness.runtime, + state: harness.state, + }, + }); + expect(response.status).toBe(200); + return CONVERSATION_ID; + }, + read: async () => ({ replies: slackReplies() }), + }, + { + name: "dispatch", + delivery: "slack", + // An Automation run posts only a declared message. Its failure shows on + // the Automation. + quietFailure: true, + // Dispatch work runs its own lifecycle. A stop fails its Turn, and a + // failed last attempt leaves its Turn open. + unsupported: ["stop mid-Turn", "every attempt throws"], + // A dispatch resumes through the Slack resume runner. + failedResumeWakes: ["completed"], + // A dispatch declares its result with a tool call. + reply: (message) => ({ + type: "toolCall", + name: "finishAutomationRun", + arguments: { result: "send_message", message }, + }), + start: async (harness) => { + const dispatch = await createAgentDispatchTestRecord("lifecycle"); + await enqueueAgentDispatch(dispatch, { + queue: harness.queue, + state: harness.state, + }); + return getDispatchConversationId(dispatch); + }, + read: async (conversationId) => ({ + dispatch: ( + await getDispatchRecord(conversationId.replace("agent-dispatch:", "")) + )?.status, + replies: slackReplies(), + }), + }, + { + name: "child from a parent task", + delivery: "parent", + start: async (harness) => await spawnChild(harness, "lifecycle-task"), + read: readChild, + }, + { + name: "child from a watch event", + delivery: "parent", + setupOutputs: [{ type: "text", text: "Watching." }], + start: async (harness) => await spawnChild(harness, "lifecycle-watch"), + setup: async ({ conversationId, harness, process }) => { + // Finish the parent task, then set a timer on the child. The timer + // event waits for its burst window, then starts a new agent + // invocation with an event Source. + await expect(process()).resolves.toMatchObject({ status: "completed" }); + await execute( + createEventTools({ ...context(), conversationId }, getEventCatalog()) + .watchTimer!, + { afterMs: 60_000, intent: "Check the deploy." }, + "lifecycle-timer", + ); + clock.nowMs += 120_000; + await runTimerWatchHeartbeat({ + nowMs: clock.nowMs, + queue: harness.queue, + }); + await expect(process()).resolves.toMatchObject({ + status: "pending_requeued", + }); + clock.nowMs += 120_000; + await expect(process(0)).resolves.toMatchObject({ + status: "pending_requeued", + }); + }, + read: readChild, + }, +]; + +interface Signal { + name: string; + outputs(args: { + reply: (text: string) => FixedModelOutput; + stop: () => void; + }): FixedModelOutput[]; + /** Make every agent run throw from this wake on. */ + failFromWake?: number; + /** Pause the first wake at its first safe boundary. */ + yieldFirst?: boolean; + /** What the owner shows after the Turn ends. */ + expected(owner: Owner): { + wakes: string[]; + turn: string; + shown: Record; + }; +} + +const REPLY = "Deploy checked."; +const ERROR_REPLY = expect.stringMatching(/^I ran into an internal error/); + +/** What a finished Turn shows on its owner. */ +function finished( + owner: Owner, + text: unknown, + end: "completed" | "failed" | "stopped", +) { + if (owner.delivery === "parent") { + // A child has no Delivery. It reports its end to the parent once. + return { + invocation: end, + parentMailbox: "appended", + reportsToParent: 1, + }; + } + return { + // A dispatch has no stopped status. A stop fails it. + ...(owner.name === "dispatch" + ? { dispatch: end === "completed" ? "completed" : "failed" } + : undefined), + replies: text ? [text] : [], + }; +} + +export const signals: Signal[] = [ + { + name: "new input completes", + outputs: ({ reply }) => [reply(REPLY)], + expected: (owner) => ({ + wakes: ["completed"], + turn: "completed", + shown: finished(owner, REPLY, "completed"), + }), + }, + { + name: "yield, then an empty wake resumes", + outputs: ({ reply }) => [ + { type: "toolCall", name: "systemTime", arguments: {} }, + reply(REPLY), + ], + yieldFirst: true, + expected: (owner) => ({ + wakes: ["yielded", "completed"], + turn: "completed", + shown: finished(owner, REPLY, "completed"), + }), + }, + { + name: "stop mid-Turn", + outputs: ({ stop }) => [ + { + type: "text", + text: "Never sent.", + onRequest: stop, + waitFor: "abort", + // An owner that ignores the stop finishes and fails this row. + abortTimeoutMs: 5_000, + }, + ], + expected: (owner) => ({ + // The stop request also wakes the Conversation. That wake finds no work. + wakes: ["completed", "no_work"], + turn: "stopped", + shown: finished(owner, undefined, "stopped"), + }), + }, + { + name: "every attempt throws", + outputs: () => [], + failFromWake: 0, + expected: (owner) => ({ + wakes: owner.failedWakes ?? [ + ...Array.from( + { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, + () => "failed", + ), + "completed", + ], + turn: "failed", + // Where a person reads the Turn, the last attempt posts an error reply. + shown: finished( + owner, + owner.delivery === "junior" || owner.quietFailure + ? undefined + : ERROR_REPLY, + "failed", + ), + }), + }, + { + name: "every resume attempt throws", + outputs: () => [{ type: "toolCall", name: "systemTime", arguments: {} }], + yieldFirst: true, + failFromWake: 1, + expected: (owner) => ({ + wakes: [ + "yielded", + ...(owner.failedResumeWakes ?? [ + ...Array.from( + { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, + () => "failed", + ), + "completed", + ]), + ], + turn: "failed", + shown: finished( + owner, + owner.delivery === "junior" || owner.quietFailure + ? undefined + : ERROR_REPLY, + "failed", + ), + }), + }, +]; diff --git a/packages/junior/tests/integration/durable-queue.test.ts b/packages/junior/tests/integration/durable-queue.test.ts index 852a0819fb..54533632e4 100644 --- a/packages/junior/tests/integration/durable-queue.test.ts +++ b/packages/junior/tests/integration/durable-queue.test.ts @@ -1,8 +1,6 @@ -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import type { StreamFn } from "@earendil-works/pi-agent-core"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { fauxAssistantMessage } from "@earendil-works/pi-ai/providers/faux"; import { createSlackSource } from "@sentry/junior-plugin-api"; -import { createConversationWork } from "@/chat/app/conversation-work"; import { FUNCTION_TIMEOUT_BUFFER_SECONDS, botConfig, @@ -20,13 +18,14 @@ import { import type { PiMessage } from "@/chat/pi/messages"; import { coerceThreadConversationState } from "@/chat/state/conversation"; import { hydrateConversationMessages } from "@/chat/conversations/messages"; -import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; +import { disconnectStateAdapter } from "@/chat/state/adapter"; import { getConversationStore } from "@/chat/db"; import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; import { LOST_WORKER_RESUME_MESSAGE } from "@/chat/task-execution/paused-turn"; import { appendAndEnqueueInboundMessage, CONVERSATION_WORK_LEASE_TTL_MS, + CONVERSATION_WORK_MAX_RETRIES, getConversationWorkState, requestConversationWork, startConversationWork, @@ -36,13 +35,11 @@ import { listTurnSummaries, saveTurnCheckpoint, } from "@/chat/task-execution/checkpoint"; -import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; import { CONVERSATION_ID, SLACK_DESTINATION, - createConversationWorkQueueTestAdapter, SLACK_BOT_USER_ID, - createSlackAdapterFixture, + createConversationWorkHarness, deferred, handleSlackWebhookAndFlush, slackEnvelope, @@ -54,10 +51,22 @@ import { queueSlackApiResponse, resetSlackApiMockState, } from "../msw/handlers/slack-api"; -import { createModelStream } from "../fixtures/model-stream"; +import { + createModelStream, + type FixedModelOutput, +} from "../fixtures/model-stream"; import { createModelAgentRunner } from "../fixtures/agent-runner"; import { createEventInboundMessage } from "@/chat/events/notification"; import { conversationTurnIdForMessage } from "@/chat/conversations/web-input"; +import { stopConversationTurn } from "@/chat/conversations/stop"; +import { + clock, + closers, + lastTurnEnd, + owners, + signals, + slackReplies, +} from "../fixtures/turn-lifecycle"; /** * Turn-lifecycle product outcomes for one conversation under the durable queue. @@ -88,39 +97,24 @@ function almostSpentStartedAtMs(remainingMs: number): number { * They may fake only model generation at the agent stream boundary and Slack * I/O at the adapter boundary. */ -async function slack(options: { modelStream?: StreamFn } = {}) { - const state = getStateAdapter(); - await state.connect(); - const wakes = createConversationWorkQueueTestAdapter(); - const adapter = createSlackAdapterFixture(); - const modelStream = - options.modelStream ?? - createModelStream([{ type: "text", text: "Deploy checked." }]); - const agentRunner = createModelAgentRunner(modelStream); - const work = createConversationWork({ - agentRunner, - conversationStore: getConversationStore(), - getSlackAdapter: () => adapter, - queue: wakes, +async function slack(options: { outputs?: FixedModelOutput[] } = {}) { + const agentRunner = createModelAgentRunner( + createModelStream( + options.outputs ?? [{ type: "text", text: "Deploy checked." }], + ), + ); + const work = await createConversationWorkHarness(agentRunner, { services: { agentRunner }, - state, }); + const wakes = work.queue; return { - state, + ...work, + agentRunner, wakes, - replies: () => - slackApiOutbox - .messages() - .map((call) => call.params.text) - .filter((text): text is string => typeof text === "string"), + replies: slackReplies, next: async (requestStartedAtMs?: number) => { // Use production worker shouldYield (lease + host request deadline). - const process = async () => - await processConversationQueueMessage(wakes.takeMessage(), { - queue: wakes, - run: work.run, - state, - }); + const process = async () => await work.process(wakes.takeMessage()); return requestStartedAtMs === undefined ? await process() : await runWithTurnRequestDeadline(process, requestStartedAtMs); @@ -141,10 +135,10 @@ async function slack(options: { modelStream?: StreamFn } = {}) { }), ), services: { - getSlackAdapter: () => adapter, + getSlackAdapter: () => work.adapter, queue: wakes, runtime: work.runtime, - state, + state: work.state, }, }), }; @@ -342,10 +336,10 @@ describe("durable queue contract", () => { it("runs one mention to one reply and leaves the conversation free", async () => { const q = await slack({ // Second reply is for the later user mention in expectNextTurn. - modelStream: createModelStream([ + outputs: [ { type: "text", text: "Deploy checked." }, { type: "text", text: "Deploy checked." }, - ]), + ], }); await expect(q.send()).resolves.toMatchObject({ status: 200 }); await expect(q.next()).resolves.toEqual({ status: "completed" }); @@ -362,7 +356,7 @@ describe("durable queue contract", () => { // Watch Conversations have their own id, not a Slack thread id. const watchConversationId = "agent-dispatch:watch-1"; const q = await slack({ - modelStream: createModelStream([ + outputs: [ { type: "text", text: "The subscribed check failed." }, { type: "text", text: "The subscribed check recovered." }, { type: "toolCall", name: "systemTime", arguments: {} }, @@ -370,7 +364,7 @@ describe("durable queue contract", () => { { type: "text", text: "Deploy checked." }, { type: "toolCall", name: "systemTime", arguments: {} }, { type: "text", text: "Never sent.", waitFor: "abort" }, - ]), + ], }); const store = getConversationStore(); await store.recordActivity({ @@ -556,13 +550,13 @@ describe("durable queue contract", () => { it("re-parks when a later attempt times out before any new work is saved", async () => { // JUNIOR-7G: a second timeout must preserve the saved tool work. const q = await slack({ - modelStream: createModelStream([ + outputs: [ { type: "toolCall", name: "systemTime", arguments: {} }, { type: "text", text: "Deploy checked.", waitFor: "abort" }, { type: "text", text: "Deploy checked.", waitFor: "abort" }, { type: "text", text: "Deploy checked." }, { type: "text", text: "Deploy checked." }, - ]), + ], }); await expect(q.send()).resolves.toMatchObject({ status: 200 }); @@ -629,10 +623,10 @@ describe("durable queue contract", () => { const startedAtMs = almostSpentStartedAtMs(remainingMs); const deadlineAtMs = startedAtMs + requestBudgetMs(); const q = await slack({ - modelStream: createModelStream([ + outputs: [ { type: "text", text: "Deploy checked." }, { type: "text", text: "Deploy checked." }, - ]), + ], }); await expect(q.send()).resolves.toMatchObject({ status: 200 }); @@ -816,3 +810,84 @@ describe("durable queue contract", () => { }); }); }); + +/** + * One table for the Turn lifecycle of every work owner. Each row runs real + * production routing: input reaches the queue, the conversation worker picks + * the owner, and the real agent runs with fixed model output. Root owners + * deliver to a person. A child owner reports to its parent agent instead. + */ +describe("Turn lifecycle for each work owner", () => { + beforeEach(() => { + resetSlackApiMockState(); + clock.nowMs = 1_800_000_000_000; + vi.spyOn(Date, "now").mockImplementation(() => clock.nowMs); + }); + afterEach(async () => { + for (const close of closers.splice(0)) await close(); + resetSlackApiMockState(); + await disconnectStateAdapter(); + vi.restoreAllMocks(); + }); + + describe.each(owners)("$name", (owner) => { + it.each( + signals.filter((signal) => !owner.unsupported?.includes(signal.name)), + )("$name", async (signal) => { + let conversationId = ""; + let stopping: Promise | undefined; + const harness = await slack({ + outputs: [ + ...(owner.setupOutputs ?? []), + ...signal.outputs({ + reply: owner.reply ?? ((text) => ({ type: "text", text })), + stop: () => { + stopping = stopConversationTurn({ + conversationId, + queue: harness.queue, + state: harness.state, + }); + }, + }), + ], + }); + conversationId = await owner.start(harness); + const process = async (softYieldAfterMs?: number) => + await harness.process( + harness.queue.takeMessageFor(conversationId), + softYieldAfterMs === undefined ? {} : { softYieldAfterMs }, + ); + await owner.setup?.({ conversationId, harness, process }); + // Run each wake of this Conversation until no work is left. + const wakes: string[] = []; + while ( + wakes.length < CONVERSATION_WORK_MAX_RETRIES + 2 && + harness.queue + .queuedMessages() + .some((message) => message.conversationId === conversationId) + ) { + if (signal.failFromWake === wakes.length) { + // A model error ends the Run with a failed outcome. Only an error + // that the runner throws, such as a lost store, retries the wake. + vi.spyOn(harness.agentRunner, "run").mockRejectedValue( + new Error("agent unavailable"), + ); + } + const result = await process( + signal.yieldFirst && wakes.length === 0 ? 0 : undefined, + ); + wakes.push(result.status); + // Each retry wake has a new idempotency key from the clock. + clock.nowMs += 1_000; + } + await stopping; + + const expected = signal.expected(owner); + expect({ + wakes, + turn: await lastTurnEnd(conversationId), + shown: await owner.read(conversationId), + }).toEqual(expected); + }); + }); +}); diff --git a/scripts/test-architecture-baseline.json b/scripts/test-architecture-baseline.json index ecec90d703..4cad06b625 100644 --- a/scripts/test-architecture-baseline.json +++ b/scripts/test-architecture-baseline.json @@ -33,7 +33,7 @@ "packages/junior/tests/integration/annotation-cards.test.ts": 7, "packages/junior/tests/integration/api/conversations/cancel-pending-messages.test.ts": 2, "packages/junior/tests/integration/conversation-turn-work.test.ts": 19, - "packages/junior/tests/integration/durable-queue.test.ts": 8, + "packages/junior/tests/integration/durable-queue.test.ts": 4, "packages/junior/tests/integration/event-wake-delay.test.ts": 4, "packages/junior/tests/integration/heartbeat-timers.test.ts": 2, "packages/junior/tests/integration/local-agent-runner.test.ts": 29, @@ -173,7 +173,6 @@ "packages/junior/tests/integration/agent-invocation-concurrency.test.ts": 1, "packages/junior/tests/integration/agent-invocation-work.test.ts": 5, "packages/junior/tests/integration/conversation-turn-work.test.ts": 9, - "packages/junior/tests/integration/durable-queue.test.ts": 2, "packages/junior/tests/integration/event-wake-delay.test.ts": 1, "packages/junior/tests/integration/slack/assistant-lifecycle-contract.test.ts": 2, "packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts": 1 From 9f6ea771030c5a1f9eca9a78b414c914bb70beab Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:14:16 +0000 Subject: [PATCH 11/21] ref(agent-dispatch): Run dispatch work on the shared worker lifecycle Dispatch work had its own copy of the ack, lost lease, stop, and last attempt logic. It now runs through runWorkerTurn, like root and child work. Dispatch keeps its own steps: it reads and projects the dispatch record, and its Slack runtime runs the Turn. runWorkerTurn accepts a prepared Turn that its owner runs. The owner says whether the Turn completed or paused, and acks when the Turn saved its input. This fixes two dispatch rows of the Turn lifecycle table: - A stop mid-Turn now reaches the dispatch Run and ends its Turn as stopped. Before, the Run ignored the stop and the Turn failed. - When every attempt throws, the last attempt now records the failed Turn. Before, only the dispatch record failed and the Turn stayed running. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../junior/src/chat/agent-dispatch/work.ts | 273 ++++++++++-------- .../junior/src/chat/slack/dispatch-turn.ts | 3 + .../src/chat/task-execution/worker-turn.ts | 81 ++++-- .../junior/tests/fixtures/turn-lifecycle.ts | 5 - .../tests/integration/durable-queue.test.ts | 4 +- 5 files changed, 216 insertions(+), 150 deletions(-) diff --git a/packages/junior/src/chat/agent-dispatch/work.ts b/packages/junior/src/chat/agent-dispatch/work.ts index bc169fcbff..e414c9e58d 100644 --- a/packages/junior/src/chat/agent-dispatch/work.ts +++ b/packages/junior/src/chat/agent-dispatch/work.ts @@ -9,13 +9,16 @@ import type { StateAdapter } from "chat"; import { z } from "zod"; import type { ConversationStore } from "@/chat/conversations/store"; import { credentialContextForActor } from "@/chat/credentials/context"; -import { getConversationStore, getSqlExecutor } from "@/chat/db"; +import { + getConversationEventStore, + getConversationStore, + getSqlExecutor, +} from "@/chat/db"; import { resolveActorIdentityId } from "@/chat/identities/actor"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { getConversationTurnBoundaryError, isCooperativeTurnYieldError, - isTurnInputCommitLostError, - TurnInputCommitLostError, } from "@/chat/runtime/turn"; import { getTurnRecord, @@ -33,6 +36,7 @@ import type { ConversationWorkerContext, ConversationWorkerResult, } from "@/chat/task-execution/worker"; +import { runWorkerTurn } from "@/chat/task-execution/worker-turn"; import { claimDispatchMailboxAppend, confirmDispatchMailboxAppend, @@ -115,6 +119,7 @@ export interface AgentDispatchConversationWorkerOptions { hooks: { ack: () => Promise; shouldYield?: () => boolean; + stopSignal?: AbortSignal; }, ) => Promise; } @@ -363,31 +368,41 @@ function missingAuthorizationReason( : `This run uses system credentials, which have no ${provider} access. Switch it to creator credentials and connect a ${provider} account.`; } -async function persistBlockedDispatchTurn( +/** + * End the dispatch Turn with a terminal outcome that no Run recorded: a + * blocking authorization error, or a failure on the last attempt. + */ +async function persistTerminalDispatchTurn( dispatch: DispatchRecord, - error: AuthorizationFlowDisabledError | PluginCredentialFailureError, + outcome: "blocked" | "failed", + errorMessage: string, ): Promise { const conversationId = getDispatchConversationId(dispatch); - const sessionId = getDispatchTurnId(dispatch.id); - const session = await getTurnRecord(conversationId, sessionId); + const turnId = getDispatchTurnId(dispatch.id); + const session = await getTurnRecord(conversationId, turnId); await recordTurnSummary({ actor: dispatch.actor, conversationId, destination: dispatch.destination, dispatchId: dispatch.id, - dispatchOutcome: "blocked", - turnId: sessionId, + dispatchOutcome: outcome, + turnId, sliceId: session?.sliceId ?? 1, source: dispatch.source, state: "failed", surface: "api", }); - await markDispatchBlocked( - dispatch.id, - error instanceof AuthorizationFlowDisabledError - ? missingAuthorizationReason(dispatch, error.provider) - : error.message, - ); + if (outcome === "blocked") { + await markDispatchBlocked(dispatch.id, errorMessage); + return; + } + await new ConversationTurnLifecycleService(getConversationEventStore()).fail({ + conversationId, + createdAtMs: Date.now(), + failureCode: "agent_run_failed", + turnId, + }); + await markDispatchFailed(dispatch.id, errorMessage); } /** Run one dispatch start or resume under the owning conversation lease. */ @@ -419,111 +434,131 @@ export function createAgentDispatchConversationWorker( ); } - let acknowledged = context.attempt.messages.length === 0; - const acknowledge = async (): Promise => { - if (acknowledged) { - return; - } - try { - await context.attempt.ack(); - } catch { - throw new TurnInputCommitLostError( - `Conversation work lease lost before dispatch inbox ack for ${context.conversationId}`, - ); + // Read the signal before any early return, so the worker clears a stop + // request that this wake handles. + const stopSignal = context.stopSignal?.(); + let result: DispatchTurnResult | undefined; + // Start the dispatch Turn, or resume it when a durable run exists. + const runDispatchTurn = async ( + durableResult: DurableDispatchTurnResult, + hooks: { + acknowledge(): Promise; + shouldYield(): boolean; + stopSignal: AbortSignal | undefined; + }, + ): Promise => { + if (durableResult.hasResumableRun) { + // A durable run proves the original input was already committed. + // The mailbox item is only a redelivery wake-up and must not + // restart it. + await hooks.acknowledge(); + await options.resumeTurn(dispatch, { shouldYield: hooks.shouldYield }); + return await readDispatchTurnResult(dispatch); } - acknowledged = true; + const runtimeResult = await options.runTurn(dispatch, { + ack: hooks.acknowledge, + shouldYield: hooks.shouldYield, + stopSignal: hooks.stopSignal, + }); + return runtimeResult.outcome + ? runtimeResult + : { ...runtimeResult, ...(await readDispatchTurnResult(dispatch)) }; }; - if (isTerminalDispatchStatus(dispatch.status)) { - await acknowledge(); - return { status: "completed" }; - } - const durableResult = await readDispatchTurnResult(dispatch); - if ( - durableResult.outcome === "blocked" || - durableResult.outcome === "completed" || - durableResult.outcome === "failed" - ) { - await projectDispatchTurnResult(dispatch.id, durableResult); - await acknowledge(); - return { status: "completed" }; - } - if (Date.now() - dispatch.createdAtMs > AGENT_DISPATCH_MAX_AGE_MS) { - await markDispatchFailed( - dispatch.id, - "Dispatch exceeded its maximum processing age", - ); - await acknowledge(); - return { status: "completed" }; - } - - const runningDispatch = await markDispatchRunning(dispatch.id); - if (!runningDispatch) { - throw new Error(`Dispatch record disappeared for ${dispatch.id}`); - } - if (isTerminalDispatchStatus(runningDispatch.status)) { - await acknowledge(); - return { status: "completed" }; - } - try { - const resumesDurableTurn = durableResult.hasResumableRun === true; - if (resumesDurableTurn && context.attempt.messages.length > 0) { - // A durable run proves the original input was already committed. The - // mailbox item is only a redelivery wake-up and must not restart it. - await acknowledge(); - } - let result: DispatchTurnResult; - if (resumesDurableTurn) { - await options.resumeTurn(dispatch, { - shouldYield: context.shouldYield, - }); - result = await readDispatchTurnResult(dispatch); - } else { - const runtimeResult = await options.runTurn(dispatch, { - ack: acknowledge, - shouldYield: context.shouldYield, - }); - result = runtimeResult.outcome - ? runtimeResult - : { - ...runtimeResult, - ...(await readDispatchTurnResult(dispatch)), - }; - } - if (!result.outcome) { - throw new Error( - `Dispatch turn ${dispatch.id} returned without a durable outcome`, - ); - } - await projectDispatchTurnResult(dispatch.id, result); - if (result.outcome && !acknowledged) { - await acknowledge(); - } - return result.outcome === "awaiting_resume" && context.shouldYield() - ? { status: "yielded" } - : { status: "completed" }; - } catch (error) { - if (isCooperativeTurnYieldError(error)) { - await markDispatchAwaitingResume(dispatch.id); - return { status: "yielded" }; - } - if (isTurnInputCommitLostError(error)) { - return { status: "lost_lease" }; - } - const blockingError = getDispatchBlockingError(error); - if (blockingError) { - await persistBlockedDispatchTurn(dispatch, blockingError); - await acknowledge(); - return { status: "completed" }; - } - if (!context.attempt.isFinalAttempt) { - throw error; - } - await markDispatchFailed( - dispatch.id, - error instanceof Error ? error.message : "Dispatch turn failed", - ); - await acknowledge(); - return { status: "completed" }; - } + return await runWorkerTurn({ + context, + inputCommitted: context.attempt.messages.length === 0, + stopSignal, + steps: { + prepare: async () => { + if (isTerminalDispatchStatus(dispatch.status)) { + return "finished"; + } + const durableResult = await readDispatchTurnResult(dispatch); + if ( + durableResult.outcome === "blocked" || + durableResult.outcome === "completed" || + durableResult.outcome === "failed" + ) { + await projectDispatchTurnResult(dispatch.id, durableResult); + return "finished"; + } + if (Date.now() - dispatch.createdAtMs > AGENT_DISPATCH_MAX_AGE_MS) { + await markDispatchFailed( + dispatch.id, + "Dispatch exceeded its maximum processing age", + ); + return "finished"; + } + const runningDispatch = await markDispatchRunning(dispatch.id); + if (!runningDispatch) { + throw new Error(`Dispatch record disappeared for ${dispatch.id}`); + } + if (isTerminalDispatchStatus(runningDispatch.status)) { + return "finished"; + } + if (stopSignal?.aborted) { + return "stopped"; + } + return { + execute: async (hooks) => { + try { + result = await runDispatchTurn(durableResult, hooks); + } catch (error) { + if (isCooperativeTurnYieldError(error)) { + return "suspended"; + } + throw error; + } + if (!result.outcome) { + throw new Error( + `Dispatch turn ${dispatch.id} returned without a durable outcome`, + ); + } + await hooks.acknowledge(); + return result.outcome === "awaiting_resume" && hooks.shouldYield() + ? "suspended" + : "completed"; + }, + }; + }, + stop: async () => { + const stopped = await readDispatchTurnResult(dispatch); + if (stopped.outcome && stopped.outcome !== "awaiting_resume") { + await projectDispatchTurnResult(dispatch.id, stopped); + return; + } + await markDispatchFailed(dispatch.id, "Dispatch was stopped"); + }, + suspend: async () => { + await markDispatchAwaitingResume(dispatch.id); + }, + complete: async () => { + if (result) { + await projectDispatchTurnResult(dispatch.id, result); + } + }, + block: async (error) => { + const blocking = getDispatchBlockingError(error); + if (!blocking) { + return false; + } + await persistTerminalDispatchTurn( + dispatch, + "blocked", + blocking instanceof AuthorizationFlowDisabledError + ? missingAuthorizationReason(dispatch, blocking.provider) + : blocking.message, + ); + return true; + }, + fail: async (error) => { + await persistTerminalDispatchTurn( + dispatch, + "failed", + error instanceof Error ? error.message : "Dispatch turn failed", + ); + }, + }, + }); }; } diff --git a/packages/junior/src/chat/slack/dispatch-turn.ts b/packages/junior/src/chat/slack/dispatch-turn.ts index e2b120b3a4..82ef5616c3 100644 --- a/packages/junior/src/chat/slack/dispatch-turn.ts +++ b/packages/junior/src/chat/slack/dispatch-turn.ts @@ -25,6 +25,7 @@ type ExecuteSlackTurn = ( onTurnOutcome?: (result: DispatchTurnResult) => void; shouldYield?: () => boolean; skipBackfill?: boolean; + stopSignal?: AbortSignal; }, ) => Promise; @@ -41,6 +42,7 @@ export function createSlackDispatchTurnRunner(options: { hooks: { ack: () => Promise; shouldYield?: () => boolean; + stopSignal?: AbortSignal; }, ): Promise { const state = getStateAdapter(); @@ -117,6 +119,7 @@ export function createSlackDispatchTurnRunner(options: { }, shouldYield: hooks.shouldYield, skipBackfill: true, + stopSignal: hooks.stopSignal, }); return { ...(errorMessage ? { errorMessage } : undefined), diff --git a/packages/junior/src/chat/task-execution/worker-turn.ts b/packages/junior/src/chat/task-execution/worker-turn.ts index 7b51952cd6..a9af5dc815 100644 --- a/packages/junior/src/chat/task-execution/worker-turn.ts +++ b/packages/junior/src/chat/task-execution/worker-turn.ts @@ -32,19 +32,34 @@ export interface PreparedWorkerTurn { saveResult(result: AgentRunResult): Promise; } +/** + * A prepared Turn that its owner runs, such as dispatch work through its + * provider runtime. `execute` says whether the Turn completed or paused at a + * boundary. It must call `acknowledge` when the Turn saved its input. + */ +export interface OwnerRunWorkerTurn { + execute(hooks: { + acknowledge(): Promise; + shouldYield(): boolean; + stopSignal: AbortSignal | undefined; + }): Promise<"completed" | "suspended">; +} + /** The steps that one work owner supplies to `runWorkerTurn`. */ export interface WorkerTurnSteps { /** * Start the Turn and read its Run. Return `stopped` when a stop came first. * Return `finished` when the work already ended and only the ack is left. */ - prepare(): Promise; + prepare(): Promise< + PreparedWorkerTurn | OwnerRunWorkerTurn | "finished" | "stopped" + >; /** Record a stopped Turn. The lifecycle acks after this step. */ stop(): Promise; /** Record a Run that paused at a boundary. */ suspend?(): Promise; /** Record a Run that waits for authorization, and say how the attempt ends. */ - awaitAuth( + awaitAuth?( outcome: Extract, ): Promise<"completed" | "paused">; /** Finish after `executeTurn` recorded the terminal Turn. An error retries. */ @@ -55,6 +70,13 @@ export interface WorkerTurnSteps { fail(error: unknown): Promise; } +function requireAgentRunner(agentRunner: AgentRunner | undefined) { + if (!agentRunner) { + throw new Error("A prepared Run needs an agent runner"); + } + return agentRunner; +} + function isInputCommitLost(error: unknown): boolean { const cause = getConversationTurnBoundaryError(error)?.cause ?? error; return isTurnInputCommitLostError(error) || isTurnInputCommitLostError(cause); @@ -66,7 +88,8 @@ function isInputCommitLost(error: unknown): boolean { * except on the last attempt or for an execution limit, where the Turn fails. */ export async function runWorkerTurn(args: { - agentRunner: AgentRunner; + /** Runs a prepared Run. An owner that runs its own Turn omits it. */ + agentRunner?: AgentRunner; context: ConversationWorkerContext; /** True when no attempt input needs an ack, such as on a resume. */ inputCommitted: boolean; @@ -105,26 +128,35 @@ export async function runWorkerTurn(args: { if (prepared === "finished") { return await end(); } - const outcome = await executeTurn( - args.agentRunner, - { - ...prepared.run, - ...(stopSignal ? { signal: stopSignal } : undefined), - durability: { - ...prepared.run.durability, - onInputCommitted: acknowledge, - shouldYield: context.shouldYield, - }, - }, - async (result) => { - // A stop that aborted the Run ends the Turn as stopped. A Run that - // finished before the stop keeps its result. - if (result.diagnostics.outcome !== "success") { - stopSignal?.throwIfAborted(); - } - return await prepared.saveResult(result); - }, - ); + const outcome = + "execute" in prepared + ? { + status: await prepared.execute({ + acknowledge, + shouldYield: context.shouldYield, + stopSignal, + }), + } + : await executeTurn( + requireAgentRunner(args.agentRunner), + { + ...prepared.run, + ...(stopSignal ? { signal: stopSignal } : undefined), + durability: { + ...prepared.run.durability, + onInputCommitted: acknowledge, + shouldYield: context.shouldYield, + }, + }, + async (result) => { + // A stop that aborted the Run ends the Turn as stopped. A Run + // that finished before the stop keeps its result. + if (result.diagnostics.outcome !== "success") { + stopSignal?.throwIfAborted(); + } + return await prepared.saveResult(result); + }, + ); if (outcome.status !== "completed" && stopSignal?.aborted) { return "stopped"; } @@ -133,6 +165,9 @@ export async function runWorkerTurn(args: { return { status: "yielded" }; } if (outcome.status === "awaiting_auth") { + if (!steps.awaitAuth) { + throw new Error("This work owner cannot wait for authorization"); + } return await end(await steps.awaitAuth(outcome)); } return "completed"; diff --git a/packages/junior/tests/fixtures/turn-lifecycle.ts b/packages/junior/tests/fixtures/turn-lifecycle.ts index aa0fbb289e..44ddc10624 100644 --- a/packages/junior/tests/fixtures/turn-lifecycle.ts +++ b/packages/junior/tests/fixtures/turn-lifecycle.ts @@ -86,8 +86,6 @@ export interface Owner { failedResumeWakes?: string[]; /** True when a failed Turn posts no error reply, by owner policy. */ quietFailure?: boolean; - /** Signals that this owner does not handle through the shared lifecycle. */ - unsupported?: string[]; } export const clock = { nowMs: 1_800_000_000_000 }; @@ -241,9 +239,6 @@ export const owners: Owner[] = [ // An Automation run posts only a declared message. Its failure shows on // the Automation. quietFailure: true, - // Dispatch work runs its own lifecycle. A stop fails its Turn, and a - // failed last attempt leaves its Turn open. - unsupported: ["stop mid-Turn", "every attempt throws"], // A dispatch resumes through the Slack resume runner. failedResumeWakes: ["completed"], // A dispatch declares its result with a tool call. diff --git a/packages/junior/tests/integration/durable-queue.test.ts b/packages/junior/tests/integration/durable-queue.test.ts index 54533632e4..b967544bf4 100644 --- a/packages/junior/tests/integration/durable-queue.test.ts +++ b/packages/junior/tests/integration/durable-queue.test.ts @@ -831,9 +831,7 @@ describe("Turn lifecycle for each work owner", () => { }); describe.each(owners)("$name", (owner) => { - it.each( - signals.filter((signal) => !owner.unsupported?.includes(signal.name)), - )("$name", async (signal) => { + it.each(signals)("$name", async (signal) => { let conversationId = ""; let stopping: Promise | undefined; const harness = await slack({ From 0ccaed9f962e4e8b224374bc9f6ba771608aaf9b Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:30:20 +0000 Subject: [PATCH 12/21] ref(task-execution): Let the Delivery owner post the turn limit notice core-turn.ts imported the Slack pause notice of the automated-turn limit. Core now takes the notice as a step from the owner of the Delivery. The Slack Turn and resume runners and the root mailbox worker pass the Slack notice. Child work has no Delivery, so it passes none, and core clears the notice claim as the Slack notice did for a Conversation without a Slack Location. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../junior/src/chat/app/conversation-work.ts | 2 + .../junior/src/chat/providers/slack/resume.ts | 2 + .../junior/src/chat/providers/slack/turn.ts | 7 ++- .../chat/slack/automated-turn-limit-notice.ts | 20 -------- .../chat/task-execution/conversation-turn.ts | 4 ++ .../src/chat/task-execution/core-turn.ts | 49 +++++++++++++++---- 6 files changed, 53 insertions(+), 31 deletions(-) diff --git a/packages/junior/src/chat/app/conversation-work.ts b/packages/junior/src/chat/app/conversation-work.ts index fa6986f3d6..191211dddf 100644 --- a/packages/junior/src/chat/app/conversation-work.ts +++ b/packages/junior/src/chat/app/conversation-work.ts @@ -32,6 +32,7 @@ import { import { createSlackRuntime } from "./factory"; import type { JuniorRuntimeServiceOverrides } from "./services"; import { createSlackSystemTurnDelivery } from "@/chat/providers/slack/system-turn"; +import { postAutomatedTurnLimitNoticeForConversation } from "@/chat/slack/automated-turn-limit-notice"; import { scheduleSessionCompletedPluginTasks, type ScheduleSessionCompletedPluginTasksOptions, @@ -155,6 +156,7 @@ export function createConversationWork( getSlackAdapter: options.getSlackAdapter, state: options.state, }), + postAutomatedTurnLimitNoticeForConversation, ); // A Slack Destination uses the Slack worker. All other root work must have // been selected above this function. diff --git a/packages/junior/src/chat/providers/slack/resume.ts b/packages/junior/src/chat/providers/slack/resume.ts index 6c177524aa..faf0ca99eb 100644 --- a/packages/junior/src/chat/providers/slack/resume.ts +++ b/packages/junior/src/chat/providers/slack/resume.ts @@ -50,6 +50,7 @@ import { saveTurnCheckpoint, } from "@/chat/task-execution/checkpoint"; import { recordAutomatedTurnLimit } from "@/chat/task-execution/core-turn"; +import { postAutomatedTurnLimitNoticeForConversation } from "@/chat/slack/automated-turn-limit-notice"; import { createSlackWebApiAssistantStatusSession, type AssistantStatusSession, @@ -677,6 +678,7 @@ async function resumeSlackTurnInContext( // A resumed Turn counts like a Turn that ran in one slice. await recordAutomatedTurnLimit({ conversationId: runArgs.conversationId, + postNotice: postAutomatedTurnLimitNoticeForConversation, source: run.source, }); if (reply.diagnostics.outcome === "success") { diff --git a/packages/junior/src/chat/providers/slack/turn.ts b/packages/junior/src/chat/providers/slack/turn.ts index 288d3b87e0..6e2c119a5b 100644 --- a/packages/junior/src/chat/providers/slack/turn.ts +++ b/packages/junior/src/chat/providers/slack/turn.ts @@ -120,6 +120,7 @@ import { } from "@/chat/services/turn-failure-response"; import { buildAuthPauseResponse } from "@/chat/services/auth-pause-response"; import { recordAutomatedTurnLimit } from "@/chat/task-execution/core-turn"; +import { postAutomatedTurnLimitNoticeForConversation } from "@/chat/slack/automated-turn-limit-notice"; import { AuthorizationFlowDisabledError } from "@/chat/services/auth-pause"; import { PluginCredentialFailureError } from "@/chat/services/plugin-auth-orchestration"; import { maybeApplyProviderDefaultConfigRequest } from "@/chat/services/provider-default-config"; @@ -1280,7 +1281,11 @@ export function createSlackTurn(deps: SlackTurnDeps) { conversationId, completedState, ); - await recordAutomatedTurnLimit({ conversationId, source }); + await recordAutomatedTurnLimit({ + conversationId, + postNotice: postAutomatedTurnLimitNoticeForConversation, + source, + }); } catch (saveError) { // The user already saw the reply. Record the save failure without // posting a second error reply. diff --git a/packages/junior/src/chat/slack/automated-turn-limit-notice.ts b/packages/junior/src/chat/slack/automated-turn-limit-notice.ts index 8f58880473..7e6110ba2c 100644 --- a/packages/junior/src/chat/slack/automated-turn-limit-notice.ts +++ b/packages/junior/src/chat/slack/automated-turn-limit-notice.ts @@ -10,7 +10,6 @@ import { logException, logWarn } from "@/chat/logging"; import { buildAutomatedTurnLimitResponse, clearAutomatedTurnLimitNoticeClaim, - type AutomatedTurnLimitUpdate, } from "@/chat/services/automated-turn-limit"; import type { Source } from "@sentry/junior-plugin-api"; import { resolveSlackReplyTarget, sendSlackReply } from "@/chat/slack/reply"; @@ -76,22 +75,3 @@ export async function postAutomatedTurnLimitNoticeForConversation(args: { return false; } } - -/** Post the pause notice after a finished Turn hits the limit. */ -export async function maybePostAutomatedTurnLimitNotice(args: { - conversationId: string; - maxTurns: number; - nowMs?: number; - source?: Source; - update: AutomatedTurnLimitUpdate | undefined; -}): Promise { - if (!args.update?.shouldPostNotice) { - return; - } - await postAutomatedTurnLimitNoticeForConversation({ - conversationId: args.conversationId, - maxTurns: args.maxTurns, - nowMs: args.nowMs, - source: args.source, - }); -} diff --git a/packages/junior/src/chat/task-execution/conversation-turn.ts b/packages/junior/src/chat/task-execution/conversation-turn.ts index ef5bf9a5a6..19eaa774dd 100644 --- a/packages/junior/src/chat/task-execution/conversation-turn.ts +++ b/packages/junior/src/chat/task-execution/conversation-turn.ts @@ -65,6 +65,7 @@ import { import { createTurnSandbox, saveFinishedTurn, + type PostAutomatedTurnLimitNotice, } from "@/chat/task-execution/core-turn"; import type { ConversationWorkerContext, @@ -206,10 +207,12 @@ function sharedAuthorIdentityId( * Create the shared mailbox worker for web and event Turns. Web input has no * provider Delivery, so its replies stay in Junior. `eventDelivery` is the * Delivery of event input, which sends replies to the Conversation Location. + * `postAutomatedTurnLimitNotice` posts the automated-turn pause notice there. */ export function createConversationTurnWorker( agentRunner: AgentRunner, eventDelivery?: DeliverMessage, + postAutomatedTurnLimitNotice?: PostAutomatedTurnLimitNotice, ) { return async ( context: ConversationWorkerContext, @@ -619,6 +622,7 @@ export function createConversationTurnWorker( conversation: completedState.conversation, conversationId: context.conversationId, destination, + postAutomatedTurnLimitNotice, result: reply, sandbox, source, diff --git a/packages/junior/src/chat/task-execution/core-turn.ts b/packages/junior/src/chat/task-execution/core-turn.ts index 77439b1398..53ae12136e 100644 --- a/packages/junior/src/chat/task-execution/core-turn.ts +++ b/packages/junior/src/chat/task-execution/core-turn.ts @@ -15,9 +15,12 @@ import { persistThreadStateById, } from "@/chat/runtime/thread-state"; import type { SandboxRef } from "@/chat/sandbox/ref"; -import { recordFinishedTurnForAutomatedLimit } from "@/chat/services/automated-turn-limit"; +import { logException } from "@/chat/logging"; +import { + clearAutomatedTurnLimitNoticeClaim, + recordFinishedTurnForAutomatedLimit, +} from "@/chat/services/automated-turn-limit"; import type { AgentRunResult } from "@/chat/services/turn-result"; -import { maybePostAutomatedTurnLimitNotice } from "@/chat/slack/automated-turn-limit-notice"; import type { ThreadConversationState } from "@/chat/state/conversation"; import { getTurnRecord, @@ -65,6 +68,16 @@ export function createTurnSandbox(args: { }; } +/** + * Post the pause notice of the automated-turn limit at the Location of a + * Conversation. The owner of the Delivery supplies it. + */ +export type PostAutomatedTurnLimitNotice = (args: { + conversationId: string; + maxTurns: number; + source: Source; +}) => Promise; + /** * Count one finished Turn toward the automated-turn limit, or reset the count * when the Turn answers its owner. A root's owner is a person. A child's @@ -74,6 +87,8 @@ export async function recordAutomatedTurnLimit(args: { conversationId: string; /** True when the input came from the owner of the Conversation. */ ownerInput?: boolean; + /** Posts the pause notice. Work without a Delivery omits it. */ + postNotice?: PostAutomatedTurnLimitNotice; source: Source; }): Promise { const maxTurns = botConfig.maxConsecutiveAutomatedTurns; @@ -83,14 +98,26 @@ export async function recordAutomatedTurnLimit(args: { // An owner input resets the count like a person's message. source: args.ownerInput ? undefined : args.source, }); - // A Conversation without a Slack Location skips the notice and clears its - // claim. - await maybePostAutomatedTurnLimitNotice({ - conversationId: args.conversationId, - maxTurns, - source: args.source, - update, - }); + if (!update?.shouldPostNotice) { + return; + } + if (args.postNotice) { + await args.postNotice({ + conversationId: args.conversationId, + maxTurns, + source: args.source, + }); + return; + } + // Work without a Delivery has nowhere to post the notice. Clear its claim + // so that a later Turn can post it. + try { + await clearAutomatedTurnLimitNoticeClaim({ + conversationId: args.conversationId, + }); + } catch (error) { + logException(error, "automated_turn_limit.notice_claim_clear_failed"); + } } /** @@ -104,6 +131,7 @@ export async function saveFinishedTurn(args: { conversationId: string; destination?: Destination; ownerInput?: boolean; + postAutomatedTurnLimitNotice?: PostAutomatedTurnLimitNotice; result: AgentRunResult; sandbox: TurnSandbox; source: Source; @@ -141,6 +169,7 @@ export async function saveFinishedTurn(args: { await recordAutomatedTurnLimit({ conversationId: args.conversationId, ownerInput: args.ownerInput, + postNotice: args.postAutomatedTurnLimitNotice, source: args.source, }); } From 6bf2038895c76fe3a5468754d783f52945ea0d60 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:43:47 +0000 Subject: [PATCH 13/21] fix(agent-invocations): Do not count grandchild input toward a child's turn limit A child Turn whose input is a result or message from its own child agent is work the child started. Restore the earlier rule: it neither counts toward nor resets the automated-turn limit. The child owner supplies this as a third TurnLimitInput state, so core has no child check. Root behavior is unchanged. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../junior/src/chat/agent-invocations/work.ts | 11 ++++-- .../src/chat/task-execution/core-turn.ts | 25 +++++++++---- .../agent-invocation-concurrency.test.ts | 35 ++++++++++++++++--- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/packages/junior/src/chat/agent-invocations/work.ts b/packages/junior/src/chat/agent-invocations/work.ts index d21e9fb303..c8636e2b49 100644 --- a/packages/junior/src/chat/agent-invocations/work.ts +++ b/packages/junior/src/chat/agent-invocations/work.ts @@ -457,14 +457,19 @@ async function saveAgentInvocationResult(args: { actor: args.invocation.actor, conversationId: args.invocation.childConversationId, destination: args.invocation.destination, - // A task or message from the parent resets the automated-turn limit, like - // a person's message on a root. - ownerInput: Boolean(args.invocation.parentToolCallId), result: args.result, sandbox: args.sandbox, source: args.invocation.source, surface: "internal", turnId: args.turnId, + // A task or message from the parent resets the automated-turn limit, like + // a person's message on a root. A result or message from a grandchild is + // work that the child started, so it does not count. Watch events count. + turnLimitInput: args.invocation.parentToolCallId + ? "owner" + : args.invocation.source.kind === "agent_invocation" + ? "own_work" + : undefined, }); const terminal = await completeAgentInvocation({ invocationId: args.invocation.invocationId, diff --git a/packages/junior/src/chat/task-execution/core-turn.ts b/packages/junior/src/chat/task-execution/core-turn.ts index 53ae12136e..3dd61aa09f 100644 --- a/packages/junior/src/chat/task-execution/core-turn.ts +++ b/packages/junior/src/chat/task-execution/core-turn.ts @@ -78,25 +78,38 @@ export type PostAutomatedTurnLimitNotice = (args: { source: Source; }) => Promise; +/** + * How the input of a finished Turn changes the automated-turn limit, when + * its Source alone does not say it. `owner`: the input came from the owner + * of the Conversation, so the count resets like a person's message does. + * `own_work`: the input is the result of work that the Conversation started, + * so the count does not change. + */ +export type TurnLimitInput = "owner" | "own_work"; + /** * Count one finished Turn toward the automated-turn limit, or reset the count * when the Turn answers its owner. A root's owner is a person. A child's - * owner is its parent agent. Post the pause notice when the limit is full. + * owner is its parent agent. Leave the count when the input is a result of + * the Conversation's own work. Post the pause notice when the limit is full. */ export async function recordAutomatedTurnLimit(args: { conversationId: string; - /** True when the input came from the owner of the Conversation. */ - ownerInput?: boolean; + /** Omit it to count the Turn by its Source. */ + input?: TurnLimitInput; /** Posts the pause notice. Work without a Delivery omits it. */ postNotice?: PostAutomatedTurnLimitNotice; source: Source; }): Promise { + if (args.input === "own_work") { + return; + } const maxTurns = botConfig.maxConsecutiveAutomatedTurns; const update = await recordFinishedTurnForAutomatedLimit({ conversationId: args.conversationId, maxTurns, // An owner input resets the count like a person's message. - source: args.ownerInput ? undefined : args.source, + source: args.input === "owner" ? undefined : args.source, }); if (!update?.shouldPostNotice) { return; @@ -130,13 +143,13 @@ export async function saveFinishedTurn(args: { conversation?: ThreadConversationState; conversationId: string; destination?: Destination; - ownerInput?: boolean; postAutomatedTurnLimitNotice?: PostAutomatedTurnLimitNotice; result: AgentRunResult; sandbox: TurnSandbox; source: Source; surface: AgentTurnSurface; turnId: string; + turnLimitInput?: TurnLimitInput; }): Promise { const { result } = args; await persistThreadStateById(args.conversationId, { @@ -168,7 +181,7 @@ export async function saveFinishedTurn(args: { } await recordAutomatedTurnLimit({ conversationId: args.conversationId, - ownerInput: args.ownerInput, + input: args.turnLimitInput, postNotice: args.postAutomatedTurnLimitNotice, source: args.source, }); diff --git a/packages/junior/tests/integration/agent-invocation-concurrency.test.ts b/packages/junior/tests/integration/agent-invocation-concurrency.test.ts index e82e9f14d8..9aa874882f 100644 --- a/packages/junior/tests/integration/agent-invocation-concurrency.test.ts +++ b/packages/junior/tests/integration/agent-invocation-concurrency.test.ts @@ -495,8 +495,25 @@ describe("agent invocation identity and concurrency", () => { it("wakes a child with its own timer by the same rules as a root Conversation", async () => { const clock = vi.spyOn(Date, "now").mockReturnValue(1_800_000_000_000); - const harness = await createHarness(taskModel); + // An action to run during the next model request of a child. + let duringNextRequest: (() => Promise) | undefined; + const harness = await createHarness((request) => { + const stream = taskModel(request); + return async (model, context, options) => { + const action = duringNextRequest; + duringNextRequest = undefined; + await action?.(); + return await stream(model, context, options); + }; + }); try { + // The task Turn starts a grandchild, as spawnAgent does. + duringNextRequest = async () => + await harness.spawn({ + idempotencyKey: "grandchild-1", + input: "Check one service.", + parentConversationId: child.childConversationId, + }); // As from spawnAgent, the task has the parent's tool call, so it // resets the automated-turn count. const child = await harness.spawn({ @@ -525,8 +542,14 @@ describe("agent invocation identity and concurrency", () => { // The timer runs on the child as the event Actor, with an event Source // and the grants of the person who set it. const requests = harness.requests(); - expect(requests).toHaveLength(2); - const timerRun = requests[1]!; + expect(requests.map((request) => request.conversationId)).toEqual([ + child.childConversationId, + expect.not.stringMatching(child.childConversationId), + // The grandchild result starts a Turn on the child. + child.childConversationId, + child.childConversationId, + ]); + const timerRun = requests[3]!; expect(timerRun).toMatchObject({ actor: EVENT_SYSTEM_ACTOR, conversationId: child.childConversationId, @@ -555,6 +578,8 @@ describe("agent invocation identity and concurrency", () => { status: "completed", }); // The timer Turn counts toward the automated-turn limit, as on a root. + // The grandchild result is work that the child started, so its Turn + // does not count. await expect( getAutomatedTurnLimitState({ conversationId: child.childConversationId, @@ -590,7 +615,7 @@ describe("agent invocation identity and concurrency", () => { clock.mockReturnValue(clock() + 600_000); await runTimerWatchHeartbeat({ nowMs: clock(), queue: harness.queue }); await harness.drainQueuedWork(); - expect(harness.requests()).toHaveLength(2); + expect(harness.requests()).toHaveLength(4); await expect(listAgents()).resolves.toMatchObject({ agents: [{ status: "stopped" }], }); @@ -621,6 +646,8 @@ describe("agent invocation identity and concurrency", () => { clock.mockReturnValue(clock() + 120_000); await harness.drainQueuedWork(); expect(harness.requests().map((request) => request.source.kind)).toEqual([ + "agent_invocation", + "agent_invocation", "agent_invocation", "event", "agent_invocation", From 5af8d3dd8b19a5f8954f52460825498e4f7f4282 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:17:03 +0000 Subject: [PATCH 14/21] ref(slack): Leave the resumed Turn limit count to the Slack slice The resumed Slack Turn started to count toward the automated-turn limit in this PR, with no test. Restore the main behavior. The change moves with the Slack runner conversion, where its test row belongs. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- packages/junior/src/chat/providers/slack/resume.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/junior/src/chat/providers/slack/resume.ts b/packages/junior/src/chat/providers/slack/resume.ts index faf0ca99eb..cffc7eb333 100644 --- a/packages/junior/src/chat/providers/slack/resume.ts +++ b/packages/junior/src/chat/providers/slack/resume.ts @@ -49,8 +49,6 @@ import { recordTurnSummary, saveTurnCheckpoint, } from "@/chat/task-execution/checkpoint"; -import { recordAutomatedTurnLimit } from "@/chat/task-execution/core-turn"; -import { postAutomatedTurnLimitNoticeForConversation } from "@/chat/slack/automated-turn-limit-notice"; import { createSlackWebApiAssistantStatusSession, type AssistantStatusSession, @@ -675,12 +673,6 @@ async function resumeSlackTurnInContext( }); } await runArgs.commitResult?.(reply); - // A resumed Turn counts like a Turn that ran in one slice. - await recordAutomatedTurnLimit({ - conversationId: runArgs.conversationId, - postNotice: postAutomatedTurnLimitNoticeForConversation, - source: run.source, - }); if (reply.diagnostics.outcome === "success") { shouldScheduleCompletedPluginTasks = true; return { From e0cc77abcd5c194d024460067d1929a40cea0358 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:17:06 +0000 Subject: [PATCH 15/21] fix(task-execution): Retry errors after the result save and close ended child Turns An error after the result save, such as a failed turn_completed write, went to the stop or fail step when a stop was pending. A child result that waited for an open grandchild then ended as skipped, and the parent never heard. runWorkerTurn now records the save, and any later error retries the attempt. The child report applies the stop-after-save rule, so the retry reports a replaced result and stops the grandchildren. Also: - A failed root or child Turn writes its idempotent start before its failure, so a setup error before the start leaves no lone turn_failed. - A failed or blocked child abandons its paused Turn record, as a stopped child did, so nothing can resume it. - The agent-invocations README names the latest invocation of any status as the owner of a late wake. The Turn lifecycle table gets a depth-2 child owner and rows for a stop before and after the Run, an error after the save with a pending stop, an execution limit, a setup error on the last attempt, and a child auth pause. Rows where the Slack runner differs are listed per owner as slackRunnerGaps. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../src/chat/agent-invocations/README.md | 3 +- .../junior/src/chat/agent-invocations/work.ts | 78 ++--- .../chat/task-execution/conversation-turn.ts | 22 +- .../src/chat/task-execution/worker-turn.ts | 13 +- .../junior/tests/fixtures/turn-lifecycle.ts | 287 +++++++++++++++--- .../tests/integration/durable-queue.test.ts | 70 ++++- 6 files changed, 375 insertions(+), 98 deletions(-) diff --git a/packages/junior/src/chat/agent-invocations/README.md b/packages/junior/src/chat/agent-invocations/README.md index efe57ccb21..feca22af51 100644 --- a/packages/junior/src/chat/agent-invocations/README.md +++ b/packages/junior/src/chat/agent-invocations/README.md @@ -404,4 +404,5 @@ open work. The child's own child agents stop the same way, so no result of theirs starts the child again. A stop request can outlive the run that it stopped. If it wakes the child -again, the latest `stopped` agent invocation handles the wake. +again, the latest agent invocation of the child handles the wake, whatever its +status. A finished agent invocation only repeats its idempotent finish. diff --git a/packages/junior/src/chat/agent-invocations/work.ts b/packages/junior/src/chat/agent-invocations/work.ts index c8636e2b49..b44d4f5bad 100644 --- a/packages/junior/src/chat/agent-invocations/work.ts +++ b/packages/junior/src/chat/agent-invocations/work.ts @@ -360,12 +360,15 @@ async function persistTerminalLifecycle( terminal: TerminalAgentInvocation, ): Promise { const turnId = getAgentInvocationTurnId(terminal.invocationId); - if (terminal.status === "stopped") { - // A child stopped between slices has a paused Turn record. Nothing may + if (terminal.status !== "completed") { + // A child that ends between slices has a paused Turn record. Nothing may // resume it. await abandonTurnRecord({ conversationId: terminal.childConversationId, - errorMessage: "Turn cancelled", + errorMessage: + terminal.status === "stopped" + ? "Turn cancelled" + : terminal.errorMessage, turnId, }); } @@ -540,8 +543,26 @@ export function createAgentInvocationWorker( throw new Error(`Agent invocation is missing for ${invocationId}`); } let invocation: AgentInvocation = loaded; + // Read the signal before any early return, so the worker clears a stop + // request that this wake handles. + const stopSignal = context.stopSignal?.(); // Report the result before ack so a failed parent append retries this work. - const report = async (terminal: TerminalAgentInvocation): Promise => { + const report = async (ended: TerminalAgentInvocation): Promise => { + let terminal = ended; + // A stop that comes after the save still stops the child agents of + // this child. Then they cannot report for a `replaced` result, so this + // result reports now. A retry after the save reports the same way. + if (stopSignal?.aborted) { + if (terminal.parentMailboxStatus === "replaced") { + const reported = await reportReplacedAgentInvocation( + terminal.invocationId, + ); + if (reported && isTerminalAgentInvocation(reported)) { + terminal = reported; + } + } + await stopChildAgentsOf(context.conversationId); + } await recordSubagentEnded(terminal); await options.enqueueResult?.(terminal); }; @@ -558,10 +579,18 @@ export function createAgentInvocationWorker( await report(terminal); }; - // Read the signal before any early return, so the worker clears a stop - // request that this wake handles. - const stopSignal = context.stopSignal?.(); const turnId = getAgentInvocationTurnId(invocation.invocationId); + const startTurn = async (): Promise => { + await new ConversationTurnLifecycleService( + getConversationEventStore(), + ).start({ + conversationId: invocation.childConversationId, + createdAtMs: invocation.createdAtMs, + inputMessageIds: [getAgentInvocationMessageId(invocation.invocationId)], + surface: "internal", + turnId, + }); + }; const endInvocation = async ( status: "blocked" | "failed" | "stopped", errorMessage: string, @@ -609,17 +638,7 @@ export function createAgentInvocationWorker( await finish(invocation); return "finished"; } - await new ConversationTurnLifecycleService( - getConversationEventStore(), - ).start({ - conversationId: invocation.childConversationId, - createdAtMs: invocation.createdAtMs, - inputMessageIds: [ - getAgentInvocationMessageId(invocation.invocationId), - ], - surface: "internal", - turnId, - }); + await startTurn(); const [persisted, projection, tree] = await Promise.all([ getPersistedThreadState(invocation.childConversationId), loadProjection({ @@ -682,12 +701,11 @@ export function createAgentInvocationWorker( }; }, // `stopChildAgent` already marked its work `stopped`. Any other stop is - // recorded here and reported to the parent. The stop also stops the + // recorded here and reported to the parent. The report also stops the // child agents of this child, so no result of theirs starts this // child again. stop: async () => { await endInvocation("stopped", "Agent invocation was stopped"); - await stopChildAgentsOf(context.conversationId); }, suspend: async () => { await markAgentInvocationPaused(invocation.invocationId); @@ -702,22 +720,7 @@ export function createAgentInvocationWorker( complete: async () => { // executeTurn already recorded the terminal Turn; only report the // result. - let terminal = await getAgentInvocation(invocation.invocationId); - // A stop that comes after the save still stops the child agents of - // this child. Then they cannot report for a `replaced` result, so - // this result reports now. - if (stopSignal?.aborted) { - if ( - terminal && - isTerminalAgentInvocation(terminal) && - terminal.parentMailboxStatus === "replaced" - ) { - terminal = await reportReplacedAgentInvocation( - terminal.invocationId, - ); - } - await stopChildAgentsOf(context.conversationId); - } + const terminal = await getAgentInvocation(invocation.invocationId); if (terminal && isTerminalAgentInvocation(terminal)) { await report(terminal); } @@ -731,6 +734,9 @@ export function createAgentInvocationWorker( return true; }, fail: async (error) => { + // A setup error can come before the Turn start. The start is + // idempotent, so each failed Turn also has its start. + await startTurn(); await endInvocation( "failed", error instanceof Error ? error.message : "Agent invocation failed", diff --git a/packages/junior/src/chat/task-execution/conversation-turn.ts b/packages/junior/src/chat/task-execution/conversation-turn.ts index 19eaa774dd..27cd0dd9ea 100644 --- a/packages/junior/src/chat/task-execution/conversation-turn.ts +++ b/packages/junior/src/chat/task-execution/conversation-turn.ts @@ -422,6 +422,15 @@ export function createConversationTurnWorker( assistantMessageDelivered = true; failureCode = "agent_run_failed"; }; + const startTurn = async (): Promise => { + await lifecycle.start({ + conversationId: context.conversationId, + createdAtMs: Date.now(), + inputMessageIds, + surface, + turnId, + }); + }; const prepare = async (): Promise< PreparedWorkerTurn | "finished" | "stopped" > => { @@ -462,13 +471,7 @@ export function createConversationTurnWorker( conversation, conversationId: context.conversationId, }); - await lifecycle.start({ - conversationId: context.conversationId, - createdAtMs: Date.now(), - inputMessageIds, - surface, - turnId, - }); + await startTurn(); if (stopSignal?.aborted) { return "stopped"; } @@ -751,6 +754,11 @@ export function createConversationTurnWorker( turnId, }); } + // A setup error can come before the Turn start. The start is + // idempotent, so each failed Turn also has its start. + if (!isResume) { + await startTurn(); + } await lifecycle.fail({ conversationId: context.conversationId, createdAtMs: Date.now(), diff --git a/packages/junior/src/chat/task-execution/worker-turn.ts b/packages/junior/src/chat/task-execution/worker-turn.ts index a9af5dc815..4c3544d765 100644 --- a/packages/junior/src/chat/task-execution/worker-turn.ts +++ b/packages/junior/src/chat/task-execution/worker-turn.ts @@ -86,6 +86,7 @@ function isInputCommitLost(error: unknown): boolean { * Run one Turn: prepare it, run the agent, and end the attempt. A lost lease * returns `lost_lease`. A stop ends the Turn as stopped. Other errors retry, * except on the last attempt or for an execution limit, where the Turn fails. + * An error after the result save always retries. */ export async function runWorkerTurn(args: { /** Runs a prepared Run. An owner that runs its own Turn omits it. */ @@ -118,6 +119,9 @@ export async function runWorkerTurn(args: { return { status }; }; + // After the result save, the Turn has its result. A later error must not + // run the stop or fail step, which would end the finished work again. + let saved = false; const advance = async (): Promise< ConversationWorkerResult | "completed" | "stopped" > => { @@ -154,7 +158,9 @@ export async function runWorkerTurn(args: { if (result.diagnostics.outcome !== "success") { stopSignal?.throwIfAborted(); } - return await prepared.saveResult(result); + const turnEnd = await prepared.saveResult(result); + saved = true; + return turnEnd; }, ); if (outcome.status !== "completed" && stopSignal?.aborted) { @@ -182,6 +188,11 @@ export async function runWorkerTurn(args: { if (isInputCommitLost(failure)) { return { status: "lost_lease" }; } + if (saved) { + // Retry instead of ending the Turn a second way. A child retry reports + // the saved result. + throw failure; + } if (stopSignal?.aborted) { ended = "stopped"; } else if (await steps.block?.(failure)) { diff --git a/packages/junior/tests/fixtures/turn-lifecycle.ts b/packages/junior/tests/fixtures/turn-lifecycle.ts index 44ddc10624..3314d60f52 100644 --- a/packages/junior/tests/fixtures/turn-lifecycle.ts +++ b/packages/junior/tests/fixtures/turn-lifecycle.ts @@ -12,7 +12,9 @@ import { getDispatchRecord, } from "@/chat/agent-dispatch/store"; import { + createAgentInvocation, getAgentInvocation, + getAgentInvocationTurnId, getLatestAgentInvocationForConversation, } from "@/chat/agent-invocations/store"; import { createAndEnqueueAgentInvocation } from "@/chat/agent-invocations/work"; @@ -25,12 +27,13 @@ import { appendAndEnqueueInboundMessage, CONVERSATION_WORK_MAX_RETRIES, } from "@/chat/task-execution/store"; +import { getTurnRecord } from "@/chat/task-execution/checkpoint"; import { createEventTools } from "@/chat/tools/events"; +import { TurnToolCallLimitExceededError } from "@/chat/services/turn-limit"; import { createAgentDispatchTestRecord } from "./agent-dispatch"; import { invocationInput, loadSubagentEvents, - parentConversationId, prepareParentConversation, } from "./agent-invocations"; import { webTestActor } from "./conversation"; @@ -45,6 +48,7 @@ import { } from "./conversation-work"; import { context, execute } from "./event-automations"; import type { FixedModelOutput } from "./model-stream"; +import type { AgentRunOutcome } from "@/chat/runtime/agent-run-outcome"; import { slackApiOutbox } from "./slack-api-outbox"; type Harness = Awaited>; @@ -86,22 +90,54 @@ export interface Owner { failedResumeWakes?: string[]; /** True when a failed Turn posts no error reply, by owner policy. */ quietFailure?: boolean; + /** + * Rows where the Slack runner ends the Turn another way. It does not use + * the shared lifecycle yet. Each value replaces part of the expected + * outcome of the row with that name. + */ + slackRunnerGaps?: Record>; +} + +/** What one row expects after the Turn ends. */ +export interface Expected { + wakes: string[]; + turn: string; + shown: Record; } export const clock = { nowMs: 1_800_000_000_000 }; export const closers: Array<() => Promise> = []; -/** Read how the last Turn of a Conversation ended. */ +/** + * Read how the last Turn of a Conversation ended. A Turn end without its + * start says so. + */ export async function lastTurnEnd(conversationId: string): Promise { - const last = (await getConversationEventStore().loadHistory(conversationId)) - .map((event) => event.data) + const events = ( + await getConversationEventStore().loadHistory(conversationId) + ).map((event) => event.data); + const last = events .filter( (data) => data.type === "turn_completed" || data.type === "turn_failed", ) .at(-1); - if (!last) return "running"; - if (last.type === "turn_failed") return "failed"; - return last.outcome === "cancelled" ? "stopped" : "completed"; + if (!last) { + return events.some((data) => data.type === "turn_started") + ? "running" + : "not started"; + } + const end = + last.type === "turn_failed" + ? "failed" + : last.outcome === "cancelled" + ? "stopped" + : last.outcome === "blocked" + ? "blocked" + : "completed"; + const started = events.some( + (data) => data.type === "turn_started" && data.turnId === last.turnId, + ); + return started ? end : `${end} without a start`; } async function assistantMessages(conversationId: string): Promise { @@ -112,33 +148,67 @@ async function assistantMessages(conversationId: string): Promise { ); } -/** Read the latest agent invocation of a child and its report to the parent. */ +/** + * Read the latest agent invocation of a child, its report to the parent, and + * whether its Turn record can still resume. + */ async function readChild(conversationId: string) { const latest = await getLatestAgentInvocationForConversation(conversationId); const invocation = latest && (await getAgentInvocation(latest.invocationId)); - const ended = (await loadSubagentEvents(parentConversationId)).filter( + if (!invocation) return {}; + const ended = ( + await loadSubagentEvents(invocation.parentConversationId) + ).filter( (event) => event.type === "subagent_ended" && - event.subagentInvocationId === invocation?.invocationId, + event.subagentInvocationId === invocation.invocationId, + ); + const turn = await getTurnRecord( + conversationId, + getAgentInvocationTurnId(invocation.invocationId), ); return { - invocation: invocation?.status, + invocation: invocation.status, parentMailbox: - invocation && "parentMailboxStatus" in invocation + "parentMailboxStatus" in invocation ? invocation.parentMailboxStatus : undefined, reportsToParent: ended.length, + turnRecordOpen: turn?.state === "running" || turn?.state === "paused", }; } -async function spawnChild(harness: Harness, idempotencyKey: string) { +/** Spawn a child of the root, or of another child at `depth` 2. */ +async function spawnChild( + harness: Harness, + idempotencyKey: string, + depth: 1 | 2 = 1, +) { const { conversationStore, fixture } = await prepareParentConversation(); closers.push(async () => await fixture.close()); + const services = { + conversationStore, + queue: harness.queue, + state: harness.state, + }; const created = await createAndEnqueueAgentInvocation( { ...invocationInput, idempotencyKey }, - { conversationStore, queue: harness.queue, state: harness.state }, + services, ); - return created.childConversationId; + if (depth === 1) { + return created.childConversationId; + } + // The child stays open, so its own child can start. + return ( + await createAndEnqueueAgentInvocation( + { + ...invocationInput, + idempotencyKey: `${idempotencyKey}-nested`, + parentConversationId: created.childConversationId, + }, + services, + ) + ).childConversationId; } export const owners: Owner[] = [ @@ -216,6 +286,25 @@ export const owners: Owner[] = [ ], // The Slack resume runner fails the Turn on the first error. failedResumeWakes: ["completed"], + slackRunnerGaps: { + // A stop after the Run drops the saved result. + "stop after the Run keeps the result": { turn: "stopped" }, + // An execution limit retries, then posts its own reply. + "an execution limit fails at once": { + wakes: [ + ...Array.from( + { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, + () => "pending_requeued", + ), + "failed", + ], + shown: { replies: [expect.stringMatching(/execution limit/)] }, + }, + // The failure has no Turn start when the start write failed. + "a setup error fails on the last attempt": { + turn: "failed without a start", + }, + }, start: async (harness) => { const response = await handleSlackWebhookAndFlush({ request: slackWebhookRequest( @@ -241,6 +330,19 @@ export const owners: Owner[] = [ quietFailure: true, // A dispatch resumes through the Slack resume runner. failedResumeWakes: ["completed"], + // A dispatch runs its Turn through the Slack runner. + slackRunnerGaps: { + // A stop before the Run ends the dispatch without a Turn. + "stop before the Run": { turn: "not started" }, + // A stop after the Run drops the saved result. + "stop after the Run keeps the result": { + turn: "stopped", + shown: { dispatch: "failed", replies: [] }, + }, + "a setup error fails on the last attempt": { + turn: "failed without a start", + }, + }, // A dispatch declares its result with a tool call. reply: (message) => ({ type: "toolCall", @@ -268,6 +370,12 @@ export const owners: Owner[] = [ start: async (harness) => await spawnChild(harness, "lifecycle-task"), read: readChild, }, + { + name: "child at depth 2", + delivery: "parent", + start: async (harness) => await spawnChild(harness, "lifecycle-nested", 2), + read: readChild, + }, { name: "child from a watch event", delivery: "parent", @@ -303,30 +411,65 @@ export const owners: Owner[] = [ interface Signal { name: string; + /** Owners of this row. Omit it for every owner. */ + appliesTo?(owner: Owner): boolean; outputs(args: { reply: (text: string) => FixedModelOutput; stop: () => void; }): FixedModelOutput[]; + /** Run after the owner setup, before the first wake. */ + before?(args: { conversationId: string; stop: () => void }): Promise; /** Make every agent run throw from this wake on. */ failFromWake?: number; + /** The error that each agent run throws. */ + failure?(): Error; + /** End every agent run with this outcome instead of the model output. */ + outcome?: AgentRunOutcome; + /** Request a stop after each Run returns, and wait until the Turn sees it. */ + stopAfterRun?: boolean; + /** + * Make a Turn lifecycle write fail: `complete` once, or `start` on every + * attempt. + */ + lifecycleFailure?: "complete" | "start"; /** Pause the first wake at its first safe boundary. */ yieldFirst?: boolean; /** What the owner shows after the Turn ends. */ - expected(owner: Owner): { - wakes: string[]; - turn: string; - shown: Record; - }; + expected(owner: Owner): Expected; } const REPLY = "Deploy checked."; const ERROR_REPLY = expect.stringMatching(/^I ran into an internal error/); +const isChild = (owner: Owner) => owner.delivery === "parent"; + +/** What a failed Turn shows on its owner. */ +function failed(owner: Owner) { + // Where a person reads the Turn, the last attempt posts an error reply. + return finished( + owner, + owner.delivery === "junior" || owner.quietFailure ? undefined : ERROR_REPLY, + "failed", + ); +} + +/** The wakes of work that throws on every attempt until the last one. */ +function failedAttempts(owner: Owner): string[] { + return ( + owner.failedWakes ?? [ + ...Array.from( + { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, + () => "failed", + ), + "completed", + ] + ); +} /** What a finished Turn shows on its owner. */ function finished( owner: Owner, text: unknown, - end: "completed" | "failed" | "stopped", + end: "blocked" | "completed" | "failed" | "stopped", ) { if (owner.delivery === "parent") { // A child has no Delivery. It reports its end to the parent once. @@ -334,6 +477,8 @@ function finished( invocation: end, parentMailbox: "appended", reportsToParent: 1, + // No Turn of an ended child can resume. + turnRecordOpen: false, }; } return { @@ -392,22 +537,9 @@ export const signals: Signal[] = [ outputs: () => [], failFromWake: 0, expected: (owner) => ({ - wakes: owner.failedWakes ?? [ - ...Array.from( - { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, - () => "failed", - ), - "completed", - ], + wakes: failedAttempts(owner), turn: "failed", - // Where a person reads the Turn, the last attempt posts an error reply. - shown: finished( - owner, - owner.delivery === "junior" || owner.quietFailure - ? undefined - : ERROR_REPLY, - "failed", - ), + shown: failed(owner), }), }, { @@ -427,13 +559,82 @@ export const signals: Signal[] = [ ]), ], turn: "failed", - shown: finished( - owner, - owner.delivery === "junior" || owner.quietFailure - ? undefined - : ERROR_REPLY, - "failed", - ), + shown: failed(owner), + }), + }, + { + name: "stop before the Run", + outputs: ({ reply }) => [reply("Never sent.")], + before: async ({ stop }) => stop(), + expected: (owner) => ({ + wakes: ["completed", "no_work"], + turn: "stopped", + shown: finished(owner, undefined, "stopped"), + }), + }, + { + name: "stop after the Run keeps the result", + outputs: ({ reply }) => [reply(REPLY)], + stopAfterRun: true, + expected: (owner) => ({ + wakes: ["completed", "no_work"], + turn: "completed", + shown: finished(owner, REPLY, "completed"), + }), + }, + { + // The child still has an open child agent, so its saved result waits + // for that work. The stop ends that work, so the result reports now. + name: "an error after the save while a stop is pending", + appliesTo: isChild, + outputs: ({ reply }) => [reply(REPLY)], + before: async ({ conversationId }) => { + await createAgentInvocation({ + ...invocationInput, + idempotencyKey: `${conversationId}-open-child`, + parentConversationId: conversationId, + }); + }, + stopAfterRun: true, + lifecycleFailure: "complete", + expected: (owner) => ({ + // The failed write retries the attempt. The retry reports the saved + // result. + wakes: ["failed", "completed", "no_work"], + turn: "completed", + shown: finished(owner, REPLY, "completed"), + }), + }, + { + name: "an execution limit fails at once", + outputs: () => [], + failFromWake: 0, + failure: () => new TurnToolCallLimitExceededError(1), + expected: (owner) => ({ + wakes: ["completed"], + turn: "failed", + shown: failed(owner), + }), + }, + { + name: "a setup error fails on the last attempt", + outputs: ({ reply }) => [reply("Never sent.")], + lifecycleFailure: "start", + expected: (owner) => ({ + wakes: failedAttempts(owner), + turn: "failed", + shown: failed(owner), + }), + }, + { + name: "an auth pause blocks the child", + appliesTo: isChild, + outputs: () => [], + outcome: { providerDisplayName: "GitHub", status: "awaiting_auth" }, + expected: (owner) => ({ + wakes: ["completed"], + turn: "blocked", + shown: finished(owner, undefined, "blocked"), }), }, ]; diff --git a/packages/junior/tests/integration/durable-queue.test.ts b/packages/junior/tests/integration/durable-queue.test.ts index b967544bf4..472650aa19 100644 --- a/packages/junior/tests/integration/durable-queue.test.ts +++ b/packages/junior/tests/integration/durable-queue.test.ts @@ -59,6 +59,7 @@ import { createModelAgentRunner } from "../fixtures/agent-runner"; import { createEventInboundMessage } from "@/chat/events/notification"; import { conversationTurnIdForMessage } from "@/chat/conversations/web-input"; import { stopConversationTurn } from "@/chat/conversations/stop"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { clock, closers, @@ -831,7 +832,8 @@ describe("Turn lifecycle for each work owner", () => { }); describe.each(owners)("$name", (owner) => { - it.each(signals)("$name", async (signal) => { + const rows = signals.filter((signal) => signal.appliesTo?.(owner) ?? true); + it.each(rows)("$name", async (signal) => { let conversationId = ""; let stopping: Promise | undefined; const harness = await slack({ @@ -839,16 +841,17 @@ describe("Turn lifecycle for each work owner", () => { ...(owner.setupOutputs ?? []), ...signal.outputs({ reply: owner.reply ?? ((text) => ({ type: "text", text })), - stop: () => { - stopping = stopConversationTurn({ - conversationId, - queue: harness.queue, - state: harness.state, - }); - }, + stop: () => stop(), }), ], }); + const stop = () => { + stopping = stopConversationTurn({ + conversationId, + queue: harness.queue, + state: harness.state, + }); + }; conversationId = await owner.start(harness); const process = async (softYieldAfterMs?: number) => await harness.process( @@ -856,6 +859,50 @@ describe("Turn lifecycle for each work owner", () => { softYieldAfterMs === undefined ? {} : { softYieldAfterMs }, ); await owner.setup?.({ conversationId, harness, process }); + await signal.before?.({ conversationId, stop }); + await stopping; + const runAgent = harness.agentRunner.run.bind(harness.agentRunner); + if (signal.outcome) { + vi.spyOn(harness.agentRunner, "run").mockResolvedValue(signal.outcome); + } + if (signal.stopAfterRun) { + vi.spyOn(harness.agentRunner, "run").mockImplementation(async (run) => { + const outcome = await runAgent(run); + stop(); + await stopping; + // The worker reads the stop marker on an interval. + await new Promise((resolve) => { + if (!run.signal || run.signal.aborted) return resolve(); + run.signal.addEventListener("abort", () => resolve(), { + once: true, + }); + setTimeout(resolve, 5_000); + }); + return outcome; + }); + } + if (signal.lifecycleFailure === "complete") { + vi.spyOn( + ConversationTurnLifecycleService.prototype, + "complete", + ).mockRejectedValueOnce(new Error("event store unavailable")); + } + if (signal.lifecycleFailure === "start") { + const start = ConversationTurnLifecycleService.prototype.start; + let failures = CONVERSATION_WORK_MAX_RETRIES; + vi.spyOn( + ConversationTurnLifecycleService.prototype, + "start", + ).mockImplementation(async function ( + this: ConversationTurnLifecycleService, + input, + ) { + if (failures-- > 0) { + throw new Error("event store unavailable"); + } + return await start.call(this, input); + }); + } // Run each wake of this Conversation until no work is left. const wakes: string[] = []; while ( @@ -868,7 +915,7 @@ describe("Turn lifecycle for each work owner", () => { // A model error ends the Run with a failed outcome. Only an error // that the runner throws, such as a lost store, retries the wake. vi.spyOn(harness.agentRunner, "run").mockRejectedValue( - new Error("agent unavailable"), + signal.failure?.() ?? new Error("agent unavailable"), ); } const result = await process( @@ -880,7 +927,10 @@ describe("Turn lifecycle for each work owner", () => { } await stopping; - const expected = signal.expected(owner); + const expected = { + ...signal.expected(owner), + ...owner.slackRunnerGaps?.[signal.name], + }; expect({ wakes, turn: await lastTurnEnd(conversationId), From 2cf76791276b0d3e95786a88158eb037d70d299b Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:47:00 +0000 Subject: [PATCH 16/21] ref(task-execution): Keep a setup failure without a Turn start and trim the lifecycle table A failed setup can end a Turn before its start write. Readers key turn_failed by turn id and do not need the start, so the extra start write is removed. The table rows now arrange their mocks in the row. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../junior/src/chat/agent-invocations/work.ts | 26 ++- .../chat/task-execution/conversation-turn.ts | 22 +-- .../junior/tests/fixtures/turn-lifecycle.ts | 172 +++++++++--------- .../tests/integration/durable-queue.test.ts | 52 +----- 4 files changed, 112 insertions(+), 160 deletions(-) diff --git a/packages/junior/src/chat/agent-invocations/work.ts b/packages/junior/src/chat/agent-invocations/work.ts index b44d4f5bad..081a5906d8 100644 --- a/packages/junior/src/chat/agent-invocations/work.ts +++ b/packages/junior/src/chat/agent-invocations/work.ts @@ -580,17 +580,6 @@ export function createAgentInvocationWorker( }; const turnId = getAgentInvocationTurnId(invocation.invocationId); - const startTurn = async (): Promise => { - await new ConversationTurnLifecycleService( - getConversationEventStore(), - ).start({ - conversationId: invocation.childConversationId, - createdAtMs: invocation.createdAtMs, - inputMessageIds: [getAgentInvocationMessageId(invocation.invocationId)], - surface: "internal", - turnId, - }); - }; const endInvocation = async ( status: "blocked" | "failed" | "stopped", errorMessage: string, @@ -638,7 +627,17 @@ export function createAgentInvocationWorker( await finish(invocation); return "finished"; } - await startTurn(); + await new ConversationTurnLifecycleService( + getConversationEventStore(), + ).start({ + conversationId: invocation.childConversationId, + createdAtMs: invocation.createdAtMs, + inputMessageIds: [ + getAgentInvocationMessageId(invocation.invocationId), + ], + surface: "internal", + turnId, + }); const [persisted, projection, tree] = await Promise.all([ getPersistedThreadState(invocation.childConversationId), loadProjection({ @@ -734,9 +733,6 @@ export function createAgentInvocationWorker( return true; }, fail: async (error) => { - // A setup error can come before the Turn start. The start is - // idempotent, so each failed Turn also has its start. - await startTurn(); await endInvocation( "failed", error instanceof Error ? error.message : "Agent invocation failed", diff --git a/packages/junior/src/chat/task-execution/conversation-turn.ts b/packages/junior/src/chat/task-execution/conversation-turn.ts index 27cd0dd9ea..19eaa774dd 100644 --- a/packages/junior/src/chat/task-execution/conversation-turn.ts +++ b/packages/junior/src/chat/task-execution/conversation-turn.ts @@ -422,15 +422,6 @@ export function createConversationTurnWorker( assistantMessageDelivered = true; failureCode = "agent_run_failed"; }; - const startTurn = async (): Promise => { - await lifecycle.start({ - conversationId: context.conversationId, - createdAtMs: Date.now(), - inputMessageIds, - surface, - turnId, - }); - }; const prepare = async (): Promise< PreparedWorkerTurn | "finished" | "stopped" > => { @@ -471,7 +462,13 @@ export function createConversationTurnWorker( conversation, conversationId: context.conversationId, }); - await startTurn(); + await lifecycle.start({ + conversationId: context.conversationId, + createdAtMs: Date.now(), + inputMessageIds, + surface, + turnId, + }); if (stopSignal?.aborted) { return "stopped"; } @@ -754,11 +751,6 @@ export function createConversationTurnWorker( turnId, }); } - // A setup error can come before the Turn start. The start is - // idempotent, so each failed Turn also has its start. - if (!isResume) { - await startTurn(); - } await lifecycle.fail({ conversationId: context.conversationId, createdAtMs: Date.now(), diff --git a/packages/junior/tests/fixtures/turn-lifecycle.ts b/packages/junior/tests/fixtures/turn-lifecycle.ts index 3314d60f52..01d5a769a9 100644 --- a/packages/junior/tests/fixtures/turn-lifecycle.ts +++ b/packages/junior/tests/fixtures/turn-lifecycle.ts @@ -4,7 +4,7 @@ * production entry points. Each signal sets the model output and says what the * owner shows after the Turn ends. */ -import { expect } from "vitest"; +import { expect, vi } from "vitest"; import { createSlackSource } from "@sentry/junior-plugin-api"; import { enqueueAgentDispatch } from "@/chat/agent-dispatch/work"; import { @@ -48,7 +48,8 @@ import { } from "./conversation-work"; import { context, execute } from "./event-automations"; import type { FixedModelOutput } from "./model-stream"; -import type { AgentRunOutcome } from "@/chat/runtime/agent-run-outcome"; +import type { AgentRunner } from "@/chat/runtime/agent-runner"; +import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { slackApiOutbox } from "./slack-api-outbox"; type Harness = Awaited>; @@ -90,11 +91,7 @@ export interface Owner { failedResumeWakes?: string[]; /** True when a failed Turn posts no error reply, by owner policy. */ quietFailure?: boolean; - /** - * Rows where the Slack runner ends the Turn another way. It does not use - * the shared lifecycle yet. Each value replaces part of the expected - * outcome of the row with that name. - */ + /** Expected parts of rows that the Slack runner still ends another way. */ slackRunnerGaps?: Record>; } @@ -108,36 +105,18 @@ export interface Expected { export const clock = { nowMs: 1_800_000_000_000 }; export const closers: Array<() => Promise> = []; -/** - * Read how the last Turn of a Conversation ended. A Turn end without its - * start says so. - */ +/** Read how the last Turn of a Conversation ended. */ export async function lastTurnEnd(conversationId: string): Promise { - const events = ( - await getConversationEventStore().loadHistory(conversationId) - ).map((event) => event.data); - const last = events + const last = (await getConversationEventStore().loadHistory(conversationId)) + .map((event) => event.data) .filter( (data) => data.type === "turn_completed" || data.type === "turn_failed", ) .at(-1); - if (!last) { - return events.some((data) => data.type === "turn_started") - ? "running" - : "not started"; - } - const end = - last.type === "turn_failed" - ? "failed" - : last.outcome === "cancelled" - ? "stopped" - : last.outcome === "blocked" - ? "blocked" - : "completed"; - const started = events.some( - (data) => data.type === "turn_started" && data.turnId === last.turnId, - ); - return started ? end : `${end} without a start`; + if (!last) return "not ended"; + if (last.type === "turn_failed") return "failed"; + if (last.outcome === "blocked") return "blocked"; + return last.outcome === "cancelled" ? "stopped" : "completed"; } async function assistantMessages(conversationId: string): Promise { @@ -148,10 +127,7 @@ async function assistantMessages(conversationId: string): Promise { ); } -/** - * Read the latest agent invocation of a child, its report to the parent, and - * whether its Turn record can still resume. - */ +/** Read the latest agent invocation of a child and its report to the parent. */ async function readChild(conversationId: string) { const latest = await getLatestAgentInvocationForConversation(conversationId); const invocation = latest && (await getAgentInvocation(latest.invocationId)); @@ -211,6 +187,16 @@ async function spawnChild( ).childConversationId; } +// Slack mailbox work requeues its message after a failed attempt and +// dead-letters it on the last attempt. +const slackFailedWakes = [ + ...Array.from( + { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, + () => "pending_requeued", + ), + "failed", +]; + export const owners: Owner[] = [ { name: "root web", @@ -275,15 +261,7 @@ export const owners: Owner[] = [ { name: "root Slack", delivery: "slack", - // Slack mailbox work requeues its message after a failed attempt and - // dead-letters it on the last attempt. - failedWakes: [ - ...Array.from( - { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, - () => "pending_requeued", - ), - "failed", - ], + failedWakes: slackFailedWakes, // The Slack resume runner fails the Turn on the first error. failedResumeWakes: ["completed"], slackRunnerGaps: { @@ -291,19 +269,9 @@ export const owners: Owner[] = [ "stop after the Run keeps the result": { turn: "stopped" }, // An execution limit retries, then posts its own reply. "an execution limit fails at once": { - wakes: [ - ...Array.from( - { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, - () => "pending_requeued", - ), - "failed", - ], + wakes: slackFailedWakes, shown: { replies: [expect.stringMatching(/execution limit/)] }, }, - // The failure has no Turn start when the start write failed. - "a setup error fails on the last attempt": { - turn: "failed without a start", - }, }, start: async (harness) => { const response = await handleSlackWebhookAndFlush({ @@ -333,15 +301,12 @@ export const owners: Owner[] = [ // A dispatch runs its Turn through the Slack runner. slackRunnerGaps: { // A stop before the Run ends the dispatch without a Turn. - "stop before the Run": { turn: "not started" }, + "stop before the Run": { turn: "not ended" }, // A stop after the Run drops the saved result. "stop after the Run keeps the result": { turn: "stopped", shown: { dispatch: "failed", replies: [] }, }, - "a setup error fails on the last attempt": { - turn: "failed without a start", - }, }, // A dispatch declares its result with a tool call. reply: (message) => ({ @@ -409,6 +374,30 @@ export const owners: Owner[] = [ }, ]; +interface BeforeArgs { + agentRunner: AgentRunner; + conversationId: string; + stop: () => void; + stopped: () => Promise; +} + +/** Request a stop after each Run, and wait until the Run sees it. */ +function stopAfterEachRun(args: BeforeArgs): void { + const run = args.agentRunner.run.bind(args.agentRunner); + vi.spyOn(args.agentRunner, "run").mockImplementation(async (request) => { + const outcome = await run(request); + args.stop(); + await args.stopped(); + // The worker reads the stop marker on an interval. + await new Promise((resolve) => { + if (!request.signal || request.signal.aborted) return resolve(); + request.signal.addEventListener("abort", () => resolve(), { once: true }); + setTimeout(resolve, 5_000); + }); + return outcome; + }); +} + interface Signal { name: string; /** Owners of this row. Omit it for every owner. */ @@ -417,21 +406,10 @@ interface Signal { reply: (text: string) => FixedModelOutput; stop: () => void; }): FixedModelOutput[]; - /** Run after the owner setup, before the first wake. */ - before?(args: { conversationId: string; stop: () => void }): Promise; + /** Arrange the row after the owner setup, before the first wake. */ + before?(args: BeforeArgs): Promise | void; /** Make every agent run throw from this wake on. */ failFromWake?: number; - /** The error that each agent run throws. */ - failure?(): Error; - /** End every agent run with this outcome instead of the model output. */ - outcome?: AgentRunOutcome; - /** Request a stop after each Run returns, and wait until the Turn sees it. */ - stopAfterRun?: boolean; - /** - * Make a Turn lifecycle write fail: `complete` once, or `start` on every - * attempt. - */ - lifecycleFailure?: "complete" | "start"; /** Pause the first wake at its first safe boundary. */ yieldFirst?: boolean; /** What the owner shows after the Turn ends. */ @@ -575,7 +553,7 @@ export const signals: Signal[] = [ { name: "stop after the Run keeps the result", outputs: ({ reply }) => [reply(REPLY)], - stopAfterRun: true, + before: stopAfterEachRun, expected: (owner) => ({ wakes: ["completed", "no_work"], turn: "completed", @@ -588,18 +566,20 @@ export const signals: Signal[] = [ name: "an error after the save while a stop is pending", appliesTo: isChild, outputs: ({ reply }) => [reply(REPLY)], - before: async ({ conversationId }) => { + before: async (args) => { await createAgentInvocation({ ...invocationInput, - idempotencyKey: `${conversationId}-open-child`, - parentConversationId: conversationId, + idempotencyKey: `${args.conversationId}-open-child`, + parentConversationId: args.conversationId, }); + stopAfterEachRun(args); + vi.spyOn( + ConversationTurnLifecycleService.prototype, + "complete", + ).mockRejectedValueOnce(new Error("event store unavailable")); }, - stopAfterRun: true, - lifecycleFailure: "complete", expected: (owner) => ({ - // The failed write retries the attempt. The retry reports the saved - // result. + // The failed write retries. The retry reports the saved result. wakes: ["failed", "completed", "no_work"], turn: "completed", shown: finished(owner, REPLY, "completed"), @@ -608,8 +588,11 @@ export const signals: Signal[] = [ { name: "an execution limit fails at once", outputs: () => [], - failFromWake: 0, - failure: () => new TurnToolCallLimitExceededError(1), + before: ({ agentRunner }) => { + vi.spyOn(agentRunner, "run").mockRejectedValue( + new TurnToolCallLimitExceededError(1), + ); + }, expected: (owner) => ({ wakes: ["completed"], turn: "failed", @@ -619,7 +602,21 @@ export const signals: Signal[] = [ { name: "a setup error fails on the last attempt", outputs: ({ reply }) => [reply("Never sent.")], - lifecycleFailure: "start", + // The start write fails on every attempt. + before: () => { + const start = ConversationTurnLifecycleService.prototype.start; + let failures = CONVERSATION_WORK_MAX_RETRIES; + vi.spyOn( + ConversationTurnLifecycleService.prototype, + "start", + ).mockImplementation(async function ( + this: ConversationTurnLifecycleService, + input, + ) { + if (failures-- > 0) throw new Error("event store unavailable"); + return await start.call(this, input); + }); + }, expected: (owner) => ({ wakes: failedAttempts(owner), turn: "failed", @@ -630,7 +627,12 @@ export const signals: Signal[] = [ name: "an auth pause blocks the child", appliesTo: isChild, outputs: () => [], - outcome: { providerDisplayName: "GitHub", status: "awaiting_auth" }, + before: ({ agentRunner }) => { + vi.spyOn(agentRunner, "run").mockResolvedValue({ + providerDisplayName: "GitHub", + status: "awaiting_auth", + }); + }, expected: (owner) => ({ wakes: ["completed"], turn: "blocked", diff --git a/packages/junior/tests/integration/durable-queue.test.ts b/packages/junior/tests/integration/durable-queue.test.ts index 472650aa19..368fde7e06 100644 --- a/packages/junior/tests/integration/durable-queue.test.ts +++ b/packages/junior/tests/integration/durable-queue.test.ts @@ -59,7 +59,6 @@ import { createModelAgentRunner } from "../fixtures/agent-runner"; import { createEventInboundMessage } from "@/chat/events/notification"; import { conversationTurnIdForMessage } from "@/chat/conversations/web-input"; import { stopConversationTurn } from "@/chat/conversations/stop"; -import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { clock, closers, @@ -859,50 +858,13 @@ describe("Turn lifecycle for each work owner", () => { softYieldAfterMs === undefined ? {} : { softYieldAfterMs }, ); await owner.setup?.({ conversationId, harness, process }); - await signal.before?.({ conversationId, stop }); + await signal.before?.({ + agentRunner: harness.agentRunner, + conversationId, + stop, + stopped: async () => await stopping, + }); await stopping; - const runAgent = harness.agentRunner.run.bind(harness.agentRunner); - if (signal.outcome) { - vi.spyOn(harness.agentRunner, "run").mockResolvedValue(signal.outcome); - } - if (signal.stopAfterRun) { - vi.spyOn(harness.agentRunner, "run").mockImplementation(async (run) => { - const outcome = await runAgent(run); - stop(); - await stopping; - // The worker reads the stop marker on an interval. - await new Promise((resolve) => { - if (!run.signal || run.signal.aborted) return resolve(); - run.signal.addEventListener("abort", () => resolve(), { - once: true, - }); - setTimeout(resolve, 5_000); - }); - return outcome; - }); - } - if (signal.lifecycleFailure === "complete") { - vi.spyOn( - ConversationTurnLifecycleService.prototype, - "complete", - ).mockRejectedValueOnce(new Error("event store unavailable")); - } - if (signal.lifecycleFailure === "start") { - const start = ConversationTurnLifecycleService.prototype.start; - let failures = CONVERSATION_WORK_MAX_RETRIES; - vi.spyOn( - ConversationTurnLifecycleService.prototype, - "start", - ).mockImplementation(async function ( - this: ConversationTurnLifecycleService, - input, - ) { - if (failures-- > 0) { - throw new Error("event store unavailable"); - } - return await start.call(this, input); - }); - } // Run each wake of this Conversation until no work is left. const wakes: string[] = []; while ( @@ -915,7 +877,7 @@ describe("Turn lifecycle for each work owner", () => { // A model error ends the Run with a failed outcome. Only an error // that the runner throws, such as a lost store, retries the wake. vi.spyOn(harness.agentRunner, "run").mockRejectedValue( - signal.failure?.() ?? new Error("agent unavailable"), + new Error("agent unavailable"), ); } const result = await process( From b77082984f38e9bcf2916021392760d80491ae5f Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:58:52 +0000 Subject: [PATCH 17/21] fix(task-execution): Do not post a second failure reply after a delivered reply A model failure reply reaches the Location before the save. When the save then fails on the last attempt, the fail step posted another error message. The fail step now skips the Location message when the Location already has a reply. A provider post also counts as delivered when its commit fails. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../chat/task-execution/conversation-turn.ts | 11 ++++- .../junior/tests/fixtures/turn-lifecycle.ts | 41 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/packages/junior/src/chat/task-execution/conversation-turn.ts b/packages/junior/src/chat/task-execution/conversation-turn.ts index 19eaa774dd..b1be6953e3 100644 --- a/packages/junior/src/chat/task-execution/conversation-turn.ts +++ b/packages/junior/src/chat/task-execution/conversation-turn.ts @@ -403,6 +403,8 @@ export function createConversationTurnWorker( location: conversationLocation, text: replyText, }); + // The Location has the reply, even when the commit below fails. + assistantMessageDelivered = true; } await commitAssistantMessage({ cards, @@ -760,8 +762,13 @@ export function createConversationTurnWorker( }); // Tell the people at the Location that the Turn failed. Web // input has no provider Delivery. The dashboard shows the - // failed Turn. - if (deliverToProvider && failureEventId) { + // failed Turn. A reply that the Location already has, such as + // a model failure reply, is not followed by a second message. + if ( + deliverToProvider && + failureEventId && + !assistantMessageDelivered + ) { try { await deliverAssistantMessage( buildTurnFailureResponse(failureEventId), diff --git a/packages/junior/tests/fixtures/turn-lifecycle.ts b/packages/junior/tests/fixtures/turn-lifecycle.ts index 01d5a769a9..f083723696 100644 --- a/packages/junior/tests/fixtures/turn-lifecycle.ts +++ b/packages/junior/tests/fixtures/turn-lifecycle.ts @@ -51,6 +51,7 @@ import type { FixedModelOutput } from "./model-stream"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { slackApiOutbox } from "./slack-api-outbox"; +import { getStateAdapter } from "@/chat/state/adapter"; type Harness = Awaited>; @@ -599,6 +600,46 @@ export const signals: Signal[] = [ shown: failed(owner), }), }, + { + // The model failure reply reaches the Location, then the save after it + // fails on the last attempt. The Turn fails with no second error reply. + name: "a save error after a delivered failure reply", + appliesTo: (owner) => owner.name === "root event with a Location", + outputs: () => [{ type: "error", errorMessage: "model unavailable" }], + before: ({ agentRunner, conversationId }) => { + // The earlier attempts throw before any reply. + let failures = CONVERSATION_WORK_MAX_RETRIES - 1; + const run = agentRunner.run.bind(agentRunner); + vi.spyOn(agentRunner, "run").mockImplementation(async (request) => { + if (failures-- > 0) throw new Error("agent unavailable"); + return await run(request); + }); + const state = getStateAdapter(); + const set = state.set.bind(state); + let saveFailed = false; + vi.spyOn(state, "set").mockImplementation(async (key, ...rest) => { + if ( + !saveFailed && + key === `thread-state:${conversationId}` && + slackReplies().length + ) { + saveFailed = true; + throw new Error("state unavailable"); + } + return await set(key, ...rest); + }); + }, + expected: (owner) => ({ + wakes: failedAttempts(owner), + turn: "failed", + // Only the model failure reply. + shown: { + replies: [ + expect.stringMatching(/^The model provider returned an error/), + ], + }, + }), + }, { name: "a setup error fails on the last attempt", outputs: ({ reply }) => [reply("Never sent.")], From b9cc6ef20cd9e2a257c2b8d685bdca6a0ccb4fbd Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:22:03 +0000 Subject: [PATCH 18/21] fix(agent-dispatch): Close the Turn record when a dispatch ends without a Run result A dispatch that failed on its last attempt or was blocked wrote a failed summary but kept a running or paused Turn record. A stop with no saved result only failed the dispatch. The fail step now fails the record, the block step abandons it, and the stop step abandons it and writes the cancelled Turn end, as root and child stops do. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../junior/src/chat/agent-dispatch/work.ts | 29 +++++++++++ .../junior/tests/fixtures/turn-lifecycle.ts | 51 +++++++++++++++---- .../tests/integration/durable-queue.test.ts | 4 ++ 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/packages/junior/src/chat/agent-dispatch/work.ts b/packages/junior/src/chat/agent-dispatch/work.ts index e414c9e58d..aff7f19656 100644 --- a/packages/junior/src/chat/agent-dispatch/work.ts +++ b/packages/junior/src/chat/agent-dispatch/work.ts @@ -21,6 +21,8 @@ import { isCooperativeTurnYieldError, } from "@/chat/runtime/turn"; import { + abandonTurnRecord, + failTurnRecord, getTurnRecord, listTurnSummaries, recordTurnSummary, @@ -392,10 +394,21 @@ async function persistTerminalDispatchTurn( state: "failed", surface: "api", }); + // A dispatch that ends between slices keeps an open Turn record. Nothing + // may resume it. if (outcome === "blocked") { + await abandonTurnRecord({ conversationId, errorMessage, turnId }); await markDispatchBlocked(dispatch.id, errorMessage); return; } + if (session) { + await failTurnRecord({ + conversationId, + errorMessage, + expectedVersion: session.version, + turnId, + }); + } await new ConversationTurnLifecycleService(getConversationEventStore()).fail({ conversationId, createdAtMs: Date.now(), @@ -527,6 +540,22 @@ export function createAgentDispatchConversationWorker( await projectDispatchTurnResult(dispatch.id, stopped); return; } + // End the Turn as root and child stops do, so nothing resumes it. + const conversationId = getDispatchConversationId(dispatch); + const turnId = getDispatchTurnId(dispatch.id); + await abandonTurnRecord({ + conversationId, + errorMessage: "Turn cancelled", + turnId, + }); + await new ConversationTurnLifecycleService( + getConversationEventStore(), + ).complete({ + conversationId, + createdAtMs: Date.now(), + outcome: "cancelled", + turnId, + }); await markDispatchFailed(dispatch.id, "Dispatch was stopped"); }, suspend: async () => { diff --git a/packages/junior/tests/fixtures/turn-lifecycle.ts b/packages/junior/tests/fixtures/turn-lifecycle.ts index f083723696..146af30659 100644 --- a/packages/junior/tests/fixtures/turn-lifecycle.ts +++ b/packages/junior/tests/fixtures/turn-lifecycle.ts @@ -10,6 +10,7 @@ import { enqueueAgentDispatch } from "@/chat/agent-dispatch/work"; import { getDispatchConversationId, getDispatchRecord, + getDispatchTurnId, } from "@/chat/agent-dispatch/store"; import { createAgentInvocation, @@ -266,6 +267,11 @@ export const owners: Owner[] = [ // The Slack resume runner fails the Turn on the first error. failedResumeWakes: ["completed"], slackRunnerGaps: { + // The resume runs the Turn after a stop request. + "stop after a yield": { + turn: "completed", + shown: { replies: ["Never sent."] }, + }, // A stop after the Run drops the saved result. "stop after the Run keeps the result": { turn: "stopped" }, // An execution limit retries, then posts its own reply. @@ -301,12 +307,10 @@ export const owners: Owner[] = [ failedResumeWakes: ["completed"], // A dispatch runs its Turn through the Slack runner. slackRunnerGaps: { - // A stop before the Run ends the dispatch without a Turn. - "stop before the Run": { turn: "not ended" }, // A stop after the Run drops the saved result. "stop after the Run keeps the result": { turn: "stopped", - shown: { dispatch: "failed", replies: [] }, + shown: { dispatch: "failed", replies: [], turnRecordOpen: false }, }, }, // A dispatch declares its result with a tool call. @@ -323,12 +327,18 @@ export const owners: Owner[] = [ }); return getDispatchConversationId(dispatch); }, - read: async (conversationId) => ({ - dispatch: ( - await getDispatchRecord(conversationId.replace("agent-dispatch:", "")) - )?.status, - replies: slackReplies(), - }), + read: async (conversationId) => { + const dispatchId = conversationId.replace("agent-dispatch:", ""); + const turn = await getTurnRecord( + conversationId, + getDispatchTurnId(dispatchId), + ); + return { + dispatch: (await getDispatchRecord(dispatchId))?.status, + replies: slackReplies(), + turnRecordOpen: turn?.state === "running" || turn?.state === "paused", + }; + }, }, { name: "child from a parent task", @@ -411,6 +421,8 @@ interface Signal { before?(args: BeforeArgs): Promise | void; /** Make every agent run throw from this wake on. */ failFromWake?: number; + /** Request a stop after this wake. */ + stopAfterWake?: number; /** Pause the first wake at its first safe boundary. */ yieldFirst?: boolean; /** What the owner shows after the Turn ends. */ @@ -462,8 +474,12 @@ function finished( } return { // A dispatch has no stopped status. A stop fails it. + // No Turn of an ended dispatch can resume. ...(owner.name === "dispatch" - ? { dispatch: end === "completed" ? "completed" : "failed" } + ? { + dispatch: end === "completed" ? "completed" : "failed", + turnRecordOpen: false, + } : undefined), replies: text ? [text] : [], }; @@ -541,6 +557,21 @@ export const signals: Signal[] = [ shown: failed(owner), }), }, + { + name: "stop after a yield", + outputs: ({ reply }) => [ + { type: "toolCall", name: "systemTime", arguments: {} }, + reply("Never sent."), + ], + yieldFirst: true, + stopAfterWake: 0, + expected: (owner) => ({ + // The resume wake ends the Turn. The stop wake finds no work. + wakes: ["yielded", "completed", "no_work"], + turn: "stopped", + shown: finished(owner, undefined, "stopped"), + }), + }, { name: "stop before the Run", outputs: ({ reply }) => [reply("Never sent.")], diff --git a/packages/junior/tests/integration/durable-queue.test.ts b/packages/junior/tests/integration/durable-queue.test.ts index 368fde7e06..d2745a29a3 100644 --- a/packages/junior/tests/integration/durable-queue.test.ts +++ b/packages/junior/tests/integration/durable-queue.test.ts @@ -884,6 +884,10 @@ describe("Turn lifecycle for each work owner", () => { signal.yieldFirst && wakes.length === 0 ? 0 : undefined, ); wakes.push(result.status); + if (signal.stopAfterWake === wakes.length - 1) { + stop(); + await stopping; + } // Each retry wake has a new idempotency key from the clock. clock.nowMs += 1_000; } From 4528c36cc8650853a7987a331cc12657415c14bc Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:34:32 +0000 Subject: [PATCH 19/21] test(task-execution): Fail every Turn start write in the setup error row The row let the start write pass again after the retries, so a start write in a fail step was not covered. Each start write now fails, and the row checks that the failure still reaches the owner. --- packages/junior/tests/fixtures/turn-lifecycle.ts | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/junior/tests/fixtures/turn-lifecycle.ts b/packages/junior/tests/fixtures/turn-lifecycle.ts index 146af30659..3808c92e48 100644 --- a/packages/junior/tests/fixtures/turn-lifecycle.ts +++ b/packages/junior/tests/fixtures/turn-lifecycle.ts @@ -674,20 +674,13 @@ export const signals: Signal[] = [ { name: "a setup error fails on the last attempt", outputs: ({ reply }) => [reply("Never sent.")], - // The start write fails on every attempt. + // Each start write fails, also the one in a fail step. The failure + // must still reach the owner. before: () => { - const start = ConversationTurnLifecycleService.prototype.start; - let failures = CONVERSATION_WORK_MAX_RETRIES; vi.spyOn( ConversationTurnLifecycleService.prototype, "start", - ).mockImplementation(async function ( - this: ConversationTurnLifecycleService, - input, - ) { - if (failures-- > 0) throw new Error("event store unavailable"); - return await start.call(this, input); - }); + ).mockRejectedValue(new Error("event store unavailable")); }, expected: (owner) => ({ wakes: failedAttempts(owner), From ea77f89526a381d185b606b3397f054aa6f62042 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:39:09 +0000 Subject: [PATCH 20/21] fix(agent-dispatch): End the dispatch before its Turn writes on the last attempt The fail and block steps wrote the Turn summary, the Turn record, and the Turn end before they marked the dispatch. When one of those writes failed, the dispatch stayed open with no retries left. The step now marks the dispatch first. --- .../junior/src/chat/agent-dispatch/work.ts | 9 ++++++-- .../junior/tests/fixtures/turn-lifecycle.ts | 23 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/junior/src/chat/agent-dispatch/work.ts b/packages/junior/src/chat/agent-dispatch/work.ts index aff7f19656..725bf05401 100644 --- a/packages/junior/src/chat/agent-dispatch/work.ts +++ b/packages/junior/src/chat/agent-dispatch/work.ts @@ -379,6 +379,13 @@ async function persistTerminalDispatchTurn( outcome: "blocked" | "failed", errorMessage: string, ): Promise { + // End the dispatch first. The retries are used up, so a failed Turn write + // below must not leave the dispatch open. + if (outcome === "blocked") { + await markDispatchBlocked(dispatch.id, errorMessage); + } else { + await markDispatchFailed(dispatch.id, errorMessage); + } const conversationId = getDispatchConversationId(dispatch); const turnId = getDispatchTurnId(dispatch.id); const session = await getTurnRecord(conversationId, turnId); @@ -398,7 +405,6 @@ async function persistTerminalDispatchTurn( // may resume it. if (outcome === "blocked") { await abandonTurnRecord({ conversationId, errorMessage, turnId }); - await markDispatchBlocked(dispatch.id, errorMessage); return; } if (session) { @@ -415,7 +421,6 @@ async function persistTerminalDispatchTurn( failureCode: "agent_run_failed", turnId, }); - await markDispatchFailed(dispatch.id, errorMessage); } /** Run one dispatch start or resume under the owning conversation lease. */ diff --git a/packages/junior/tests/fixtures/turn-lifecycle.ts b/packages/junior/tests/fixtures/turn-lifecycle.ts index 3808c92e48..c929e93138 100644 --- a/packages/junior/tests/fixtures/turn-lifecycle.ts +++ b/packages/junior/tests/fixtures/turn-lifecycle.ts @@ -688,6 +688,29 @@ export const signals: Signal[] = [ shown: failed(owner), }), }, + { + // The Turn end write fails on the last attempt. The dispatch must still + // end, so its Automation run does not stay open. + name: "a failed Turn end write still ends the dispatch", + appliesTo: (owner) => owner.name === "dispatch", + outputs: () => [], + failFromWake: 0, + before: () => { + vi.spyOn( + ConversationTurnLifecycleService.prototype, + "fail", + ).mockRejectedValue(new Error("event store unavailable")); + }, + expected: (owner) => ({ + // The last attempt throws after the dispatch ends. No work is left. + wakes: Array.from( + { length: CONVERSATION_WORK_MAX_RETRIES }, + () => "failed", + ), + turn: "not ended", + shown: failed(owner), + }), + }, { name: "an auth pause blocks the child", appliesTo: isChild, From a4a2ef08e6f78b0dc66b14a0f7e25a84d7f1a514 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:54:14 +0000 Subject: [PATCH 21/21] revert(task-execution): Drop the shared Turn layer and keep the child fix The shared Turn layer did not make the code simpler. It added worker-turn.ts and core-turn.ts with a seven-step callback interface, but the three Slack Turn copies still run their own lifecycle. The source grew by about 390 lines, and most review bugs on this PR came from that layer. This commit puts the branch back to the child fix (0b2d66b78): - Child work goes to the child worker, picked from the stored parent. - The last resume attempt counts as final, so a failing child tells its parent. - Child routing tests run through production work. It also keeps one small fix: a failed or blocked child closes its paused Turn record, as a stopped child did. The yielded-child test checks this. The removed work is kept on ref/shared-turn-lifecycle-attempt. Co-Authored-By: David Cramer <23610+dcramer@users.noreply.github.com> --- .../junior/src/chat/agent-dispatch/work.ts | 307 +++----- .../src/chat/agent-invocations/README.md | 3 +- .../junior/src/chat/agent-invocations/work.ts | 660 +++++++++------- .../junior/src/chat/app/conversation-work.ts | 2 - .../junior/src/chat/providers/slack/turn.ts | 15 +- .../chat/slack/automated-turn-limit-notice.ts | 20 + .../junior/src/chat/slack/dispatch-turn.ts | 3 - .../junior/src/chat/task-execution/README.md | 14 - .../chat/task-execution/conversation-turn.ts | 533 +++++++------ .../src/chat/task-execution/core-turn.ts | 243 ------ .../src/chat/task-execution/paused-turn.ts | 17 +- .../src/chat/task-execution/turn-wake.ts | 21 +- .../src/chat/task-execution/web-steering.ts | 48 +- .../src/chat/task-execution/worker-turn.ts | 227 ------ .../component/agent-invocation-worker.test.ts | 56 +- .../junior/tests/fixtures/model-stream.ts | 10 +- .../junior/tests/fixtures/turn-lifecycle.ts | 730 ------------------ .../agent-invocation-concurrency.test.ts | 38 +- .../integration/agent-invocation-work.test.ts | 7 + .../tests/integration/durable-queue.test.ts | 179 ++--- .../integration/event-wake-delay.test.ts | 6 - scripts/test-architecture-baseline.json | 3 +- 22 files changed, 961 insertions(+), 2181 deletions(-) delete mode 100644 packages/junior/src/chat/task-execution/core-turn.ts delete mode 100644 packages/junior/src/chat/task-execution/worker-turn.ts delete mode 100644 packages/junior/tests/fixtures/turn-lifecycle.ts diff --git a/packages/junior/src/chat/agent-dispatch/work.ts b/packages/junior/src/chat/agent-dispatch/work.ts index 725bf05401..bc169fcbff 100644 --- a/packages/junior/src/chat/agent-dispatch/work.ts +++ b/packages/junior/src/chat/agent-dispatch/work.ts @@ -9,20 +9,15 @@ import type { StateAdapter } from "chat"; import { z } from "zod"; import type { ConversationStore } from "@/chat/conversations/store"; import { credentialContextForActor } from "@/chat/credentials/context"; -import { - getConversationEventStore, - getConversationStore, - getSqlExecutor, -} from "@/chat/db"; +import { getConversationStore, getSqlExecutor } from "@/chat/db"; import { resolveActorIdentityId } from "@/chat/identities/actor"; -import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { getConversationTurnBoundaryError, isCooperativeTurnYieldError, + isTurnInputCommitLostError, + TurnInputCommitLostError, } from "@/chat/runtime/turn"; import { - abandonTurnRecord, - failTurnRecord, getTurnRecord, listTurnSummaries, recordTurnSummary, @@ -38,7 +33,6 @@ import type { ConversationWorkerContext, ConversationWorkerResult, } from "@/chat/task-execution/worker"; -import { runWorkerTurn } from "@/chat/task-execution/worker-turn"; import { claimDispatchMailboxAppend, confirmDispatchMailboxAppend, @@ -121,7 +115,6 @@ export interface AgentDispatchConversationWorkerOptions { hooks: { ack: () => Promise; shouldYield?: () => boolean; - stopSignal?: AbortSignal; }, ) => Promise; } @@ -370,57 +363,31 @@ function missingAuthorizationReason( : `This run uses system credentials, which have no ${provider} access. Switch it to creator credentials and connect a ${provider} account.`; } -/** - * End the dispatch Turn with a terminal outcome that no Run recorded: a - * blocking authorization error, or a failure on the last attempt. - */ -async function persistTerminalDispatchTurn( +async function persistBlockedDispatchTurn( dispatch: DispatchRecord, - outcome: "blocked" | "failed", - errorMessage: string, + error: AuthorizationFlowDisabledError | PluginCredentialFailureError, ): Promise { - // End the dispatch first. The retries are used up, so a failed Turn write - // below must not leave the dispatch open. - if (outcome === "blocked") { - await markDispatchBlocked(dispatch.id, errorMessage); - } else { - await markDispatchFailed(dispatch.id, errorMessage); - } const conversationId = getDispatchConversationId(dispatch); - const turnId = getDispatchTurnId(dispatch.id); - const session = await getTurnRecord(conversationId, turnId); + const sessionId = getDispatchTurnId(dispatch.id); + const session = await getTurnRecord(conversationId, sessionId); await recordTurnSummary({ actor: dispatch.actor, conversationId, destination: dispatch.destination, dispatchId: dispatch.id, - dispatchOutcome: outcome, - turnId, + dispatchOutcome: "blocked", + turnId: sessionId, sliceId: session?.sliceId ?? 1, source: dispatch.source, state: "failed", surface: "api", }); - // A dispatch that ends between slices keeps an open Turn record. Nothing - // may resume it. - if (outcome === "blocked") { - await abandonTurnRecord({ conversationId, errorMessage, turnId }); - return; - } - if (session) { - await failTurnRecord({ - conversationId, - errorMessage, - expectedVersion: session.version, - turnId, - }); - } - await new ConversationTurnLifecycleService(getConversationEventStore()).fail({ - conversationId, - createdAtMs: Date.now(), - failureCode: "agent_run_failed", - turnId, - }); + await markDispatchBlocked( + dispatch.id, + error instanceof AuthorizationFlowDisabledError + ? missingAuthorizationReason(dispatch, error.provider) + : error.message, + ); } /** Run one dispatch start or resume under the owning conversation lease. */ @@ -452,147 +419,111 @@ export function createAgentDispatchConversationWorker( ); } - // Read the signal before any early return, so the worker clears a stop - // request that this wake handles. - const stopSignal = context.stopSignal?.(); - let result: DispatchTurnResult | undefined; - // Start the dispatch Turn, or resume it when a durable run exists. - const runDispatchTurn = async ( - durableResult: DurableDispatchTurnResult, - hooks: { - acknowledge(): Promise; - shouldYield(): boolean; - stopSignal: AbortSignal | undefined; - }, - ): Promise => { - if (durableResult.hasResumableRun) { - // A durable run proves the original input was already committed. - // The mailbox item is only a redelivery wake-up and must not - // restart it. - await hooks.acknowledge(); - await options.resumeTurn(dispatch, { shouldYield: hooks.shouldYield }); - return await readDispatchTurnResult(dispatch); + let acknowledged = context.attempt.messages.length === 0; + const acknowledge = async (): Promise => { + if (acknowledged) { + return; } - const runtimeResult = await options.runTurn(dispatch, { - ack: hooks.acknowledge, - shouldYield: hooks.shouldYield, - stopSignal: hooks.stopSignal, - }); - return runtimeResult.outcome - ? runtimeResult - : { ...runtimeResult, ...(await readDispatchTurnResult(dispatch)) }; + try { + await context.attempt.ack(); + } catch { + throw new TurnInputCommitLostError( + `Conversation work lease lost before dispatch inbox ack for ${context.conversationId}`, + ); + } + acknowledged = true; }; - return await runWorkerTurn({ - context, - inputCommitted: context.attempt.messages.length === 0, - stopSignal, - steps: { - prepare: async () => { - if (isTerminalDispatchStatus(dispatch.status)) { - return "finished"; - } - const durableResult = await readDispatchTurnResult(dispatch); - if ( - durableResult.outcome === "blocked" || - durableResult.outcome === "completed" || - durableResult.outcome === "failed" - ) { - await projectDispatchTurnResult(dispatch.id, durableResult); - return "finished"; - } - if (Date.now() - dispatch.createdAtMs > AGENT_DISPATCH_MAX_AGE_MS) { - await markDispatchFailed( - dispatch.id, - "Dispatch exceeded its maximum processing age", - ); - return "finished"; - } - const runningDispatch = await markDispatchRunning(dispatch.id); - if (!runningDispatch) { - throw new Error(`Dispatch record disappeared for ${dispatch.id}`); - } - if (isTerminalDispatchStatus(runningDispatch.status)) { - return "finished"; - } - if (stopSignal?.aborted) { - return "stopped"; - } - return { - execute: async (hooks) => { - try { - result = await runDispatchTurn(durableResult, hooks); - } catch (error) { - if (isCooperativeTurnYieldError(error)) { - return "suspended"; - } - throw error; - } - if (!result.outcome) { - throw new Error( - `Dispatch turn ${dispatch.id} returned without a durable outcome`, - ); - } - await hooks.acknowledge(); - return result.outcome === "awaiting_resume" && hooks.shouldYield() - ? "suspended" - : "completed"; - }, - }; - }, - stop: async () => { - const stopped = await readDispatchTurnResult(dispatch); - if (stopped.outcome && stopped.outcome !== "awaiting_resume") { - await projectDispatchTurnResult(dispatch.id, stopped); - return; - } - // End the Turn as root and child stops do, so nothing resumes it. - const conversationId = getDispatchConversationId(dispatch); - const turnId = getDispatchTurnId(dispatch.id); - await abandonTurnRecord({ - conversationId, - errorMessage: "Turn cancelled", - turnId, - }); - await new ConversationTurnLifecycleService( - getConversationEventStore(), - ).complete({ - conversationId, - createdAtMs: Date.now(), - outcome: "cancelled", - turnId, - }); - await markDispatchFailed(dispatch.id, "Dispatch was stopped"); - }, - suspend: async () => { - await markDispatchAwaitingResume(dispatch.id); - }, - complete: async () => { - if (result) { - await projectDispatchTurnResult(dispatch.id, result); - } - }, - block: async (error) => { - const blocking = getDispatchBlockingError(error); - if (!blocking) { - return false; - } - await persistTerminalDispatchTurn( - dispatch, - "blocked", - blocking instanceof AuthorizationFlowDisabledError - ? missingAuthorizationReason(dispatch, blocking.provider) - : blocking.message, - ); - return true; - }, - fail: async (error) => { - await persistTerminalDispatchTurn( - dispatch, - "failed", - error instanceof Error ? error.message : "Dispatch turn failed", - ); - }, - }, - }); + if (isTerminalDispatchStatus(dispatch.status)) { + await acknowledge(); + return { status: "completed" }; + } + const durableResult = await readDispatchTurnResult(dispatch); + if ( + durableResult.outcome === "blocked" || + durableResult.outcome === "completed" || + durableResult.outcome === "failed" + ) { + await projectDispatchTurnResult(dispatch.id, durableResult); + await acknowledge(); + return { status: "completed" }; + } + if (Date.now() - dispatch.createdAtMs > AGENT_DISPATCH_MAX_AGE_MS) { + await markDispatchFailed( + dispatch.id, + "Dispatch exceeded its maximum processing age", + ); + await acknowledge(); + return { status: "completed" }; + } + + const runningDispatch = await markDispatchRunning(dispatch.id); + if (!runningDispatch) { + throw new Error(`Dispatch record disappeared for ${dispatch.id}`); + } + if (isTerminalDispatchStatus(runningDispatch.status)) { + await acknowledge(); + return { status: "completed" }; + } + try { + const resumesDurableTurn = durableResult.hasResumableRun === true; + if (resumesDurableTurn && context.attempt.messages.length > 0) { + // A durable run proves the original input was already committed. The + // mailbox item is only a redelivery wake-up and must not restart it. + await acknowledge(); + } + let result: DispatchTurnResult; + if (resumesDurableTurn) { + await options.resumeTurn(dispatch, { + shouldYield: context.shouldYield, + }); + result = await readDispatchTurnResult(dispatch); + } else { + const runtimeResult = await options.runTurn(dispatch, { + ack: acknowledge, + shouldYield: context.shouldYield, + }); + result = runtimeResult.outcome + ? runtimeResult + : { + ...runtimeResult, + ...(await readDispatchTurnResult(dispatch)), + }; + } + if (!result.outcome) { + throw new Error( + `Dispatch turn ${dispatch.id} returned without a durable outcome`, + ); + } + await projectDispatchTurnResult(dispatch.id, result); + if (result.outcome && !acknowledged) { + await acknowledge(); + } + return result.outcome === "awaiting_resume" && context.shouldYield() + ? { status: "yielded" } + : { status: "completed" }; + } catch (error) { + if (isCooperativeTurnYieldError(error)) { + await markDispatchAwaitingResume(dispatch.id); + return { status: "yielded" }; + } + if (isTurnInputCommitLostError(error)) { + return { status: "lost_lease" }; + } + const blockingError = getDispatchBlockingError(error); + if (blockingError) { + await persistBlockedDispatchTurn(dispatch, blockingError); + await acknowledge(); + return { status: "completed" }; + } + if (!context.attempt.isFinalAttempt) { + throw error; + } + await markDispatchFailed( + dispatch.id, + error instanceof Error ? error.message : "Dispatch turn failed", + ); + await acknowledge(); + return { status: "completed" }; + } }; } diff --git a/packages/junior/src/chat/agent-invocations/README.md b/packages/junior/src/chat/agent-invocations/README.md index feca22af51..efe57ccb21 100644 --- a/packages/junior/src/chat/agent-invocations/README.md +++ b/packages/junior/src/chat/agent-invocations/README.md @@ -404,5 +404,4 @@ open work. The child's own child agents stop the same way, so no result of theirs starts the child again. A stop request can outlive the run that it stopped. If it wakes the child -again, the latest agent invocation of the child handles the wake, whatever its -status. A finished agent invocation only repeats its idempotent finish. +again, the latest `stopped` agent invocation handles the wake. diff --git a/packages/junior/src/chat/agent-invocations/work.ts b/packages/junior/src/chat/agent-invocations/work.ts index 081a5906d8..ba29fb96e9 100644 --- a/packages/junior/src/chat/agent-invocations/work.ts +++ b/packages/junior/src/chat/agent-invocations/work.ts @@ -1,6 +1,10 @@ import { isDeepStrictEqual } from "node:util"; import type { StateAdapter } from "chat"; import { z } from "zod"; +import { + buildSteeringPiMessage, + withoutSavedSteeringMessages, +} from "@/chat/agent/prompt"; import type { AgentRun, AgentSteeringMessage } from "@/chat/agent/types"; import type { Location } from "@/chat/conversations/location"; import { @@ -8,32 +12,46 @@ import { sameActorIdentity, } from "@/chat/conversations/provenance"; import type { ConversationStore } from "@/chat/conversations/store"; -import { loadProjection } from "@/chat/conversations/projection"; +import { openConversationProjection } from "@/chat/conversations/projection"; import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; import { getConversationEventStore, getConversationStore } from "@/chat/db"; import { SYSTEM_IDENTITY_ID } from "@/chat/identities/actor"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; import { + AgentRunError, + executeTurn, finishTurn, type SavedTurnResult, } from "@/chat/runtime/turn-execution"; -import { getPersistedThreadState } from "@/chat/runtime/thread-state"; +import { + getPersistedSandboxState, + getPersistedThreadState, + persistThreadStateById, +} from "@/chat/runtime/thread-state"; import { abandonTurnRecord, failTurnRecord, getTurnRecord, + saveTurnCheckpoint, } from "@/chat/task-execution/checkpoint"; -import { getConversationTurnBoundaryError } from "@/chat/runtime/turn"; +import { + getConversationTurnBoundaryError, + isTurnInputCommitLostError, + TurnInputCommitLostError, +} from "@/chat/runtime/turn"; import { AuthorizationFlowDisabledError } from "@/chat/services/auth-pause"; +import { botConfig } from "@/chat/config"; +import { + recordFinishedTurnForAutomatedLimit, + resetAutomatedTurnLimit, +} from "@/chat/services/automated-turn-limit"; +import { maybePostAutomatedTurnLimitNotice } from "@/chat/slack/automated-turn-limit-notice"; import { PluginCredentialFailureError } from "@/chat/services/plugin-auth-orchestration"; +import { isTurnExecutionLimitExceededError } from "@/chat/services/turn-limit"; import { getAssistantReplyText } from "@/chat/services/assistant-reply"; import { getTerminalAssistantMessages } from "@/chat/pi/transcript"; -import { - createSteeringDrain, - createTurnSandbox, - saveFinishedTurn, - type TurnSandbox, -} from "@/chat/task-execution/core-turn"; +import type { PiMessage } from "@/chat/pi/messages"; +import type { SandboxRef } from "@/chat/sandbox/ref"; import { agentTypeInstructions, agentTypeStartsWithParentHistory, @@ -49,7 +67,6 @@ import type { ConversationWorkerContext, ConversationWorkerResult, } from "@/chat/task-execution/worker"; -import { runWorkerTurn } from "@/chat/task-execution/worker-turn"; import { completeAgentInvocation, createAgentInvocation, @@ -262,69 +279,91 @@ function createAgentMessageSteeringDrain( context: ConversationWorkerContext, running: AgentInvocation, ) { - return createSteeringDrain({ - context, - deferred: true, - turnId: getAgentInvocationTurnId(running.invocationId), - select: async (pending) => { - const readIds: string[] = []; - const joining: AgentInvocation[] = []; - for (const message of pending) { - const metadata = agentInvocationMailboxMetadataSchema.safeParse( - message.input.metadata, - ); - if (!metadata.success) { - continue; - } - const candidate = await getAgentInvocation( - metadata.data.agentInvocationId, - ); - // Only a parent task or message steers. A grandchild result or a - // watch event starts its own agent invocation. - if ( - !candidate?.parentToolCallId || - candidate.invocationId === running.invocationId - ) { - continue; - } - if (isTerminalAgentInvocation(candidate)) { - // An earlier drain marked this message but did not acknowledge it. - if (candidate.parentMailboxStatus === "steered") { - readIds.push(message.inboundMessageId); + const turnId = getAgentInvocationTurnId(running.invocationId); + return async ( + accept: (messages: AgentSteeringMessage[]) => Promise, + ): Promise => { + const accepted: AgentSteeringMessage[] = []; + await context.attempt.drain( + async (pending) => { + const readIds: string[] = []; + const joining: AgentInvocation[] = []; + for (const message of pending) { + const metadata = agentInvocationMailboxMetadataSchema.safeParse( + message.input.metadata, + ); + if (!metadata.success) { + continue; + } + const candidate = await getAgentInvocation( + metadata.data.agentInvocationId, + ); + // Only a parent task or message steers. A grandchild result or a + // watch event starts its own agent invocation. + if ( + !candidate?.parentToolCallId || + candidate.invocationId === running.invocationId + ) { + continue; + } + if (isTerminalAgentInvocation(candidate)) { + // An earlier drain marked this message but did not acknowledge it. + if (candidate.parentMailboxStatus === "steered") { + readIds.push(message.inboundMessageId); + } + continue; + } + if ( + !sameActorIdentity(candidate.actor, running.actor) || + !isDeepStrictEqual( + candidate.credentialContext, + running.credentialContext, + ) + ) { + break; } - continue; + joining.push(candidate); + readIds.push(message.inboundMessageId); } - if ( - !sameActorIdentity(candidate.actor, running.actor) || - !isDeepStrictEqual( - candidate.credentialContext, - running.credentialContext, - ) - ) { - break; + if (joining.length === 0) { + return readIds; } - joining.push(candidate); - readIds.push(message.inboundMessageId); - } - return { joining, readIds }; - }, - toSteeringMessage: (candidate): AgentSteeringMessage => ({ - provenance: agentInstructionProvenance, - text: agentInvocationInputText(candidate), - timestampMs: candidate.createdAtMs, - }), - // The Turn saves the messages before they are marked. - afterAccept: async (joining) => { - for (const candidate of joining) { - const terminal = await markAgentInvocationSteered( - candidate.invocationId, + // An earlier drain can save a message in the Turn and then fail + // before it marks the agent invocation. Do not give it to the Turn + // again. + const saved = + (await getTurnRecord(running.childConversationId, turnId)) + ?.piMessages ?? []; + const messages = withoutSavedSteeringMessages( + saved, + joining.map( + (candidate): AgentSteeringMessage => ({ + provenance: agentInstructionProvenance, + text: agentInvocationInputText(candidate), + timestampMs: candidate.createdAtMs, + }), + ), + buildSteeringPiMessage, ); - if (terminal && isTerminalAgentInvocation(terminal)) { - await recordSubagentEnded(terminal); + // The Turn saves the messages before they are marked. + if (messages.length > 0) { + await accept(messages); + accepted.push(...messages); } - } - }, - }); + for (const candidate of joining) { + const terminal = await markAgentInvocationSteered( + candidate.invocationId, + ); + if (terminal && isTerminalAgentInvocation(terminal)) { + await recordSubagentEnded(terminal); + } + } + return readIds; + }, + { deferred: true }, + ); + return accepted; + }; } /** @@ -445,35 +484,61 @@ function blockingInvocationError( : undefined; } -/** - * Save one completed agent result. The core Turn save comes first. The - * Delivery of a child then saves the result on its agent invocation. - */ +function isInvocationInputCommitLost(error: unknown): boolean { + const cause = getConversationTurnBoundaryError(error)?.cause ?? error; + return isTurnInputCommitLostError(cause); +} + +/** Save one completed agent result on its child agent invocation. */ async function saveAgentInvocationResult(args: { invocation: AgentInvocation; result: AgentRunResult; - sandbox: TurnSandbox; + sandboxRef?: SandboxRef; turnId: string; }) { const failed = args.result.diagnostics.outcome !== "success"; - await saveFinishedTurn({ - actor: args.invocation.actor, - conversationId: args.invocation.childConversationId, - destination: args.invocation.destination, - result: args.result, - sandbox: args.sandbox, - source: args.invocation.source, - surface: "internal", - turnId: args.turnId, - // A task or message from the parent resets the automated-turn limit, like - // a person's message on a root. A result or message from a grandchild is - // work that the child started, so it does not count. Watch events count. - turnLimitInput: args.invocation.parentToolCallId - ? "owner" - : args.invocation.source.kind === "agent_invocation" - ? "own_work" - : undefined, + await persistThreadStateById(args.invocation.childConversationId, { + sandboxRef: args.result.sandboxRef ?? args.sandboxRef, }); + if (args.result.piMessages?.length) { + await saveTurnCheckpoint({ + mode: "completed", + conversationId: args.invocation.childConversationId, + turnId: args.turnId, + durationMs: args.result.diagnostics.durationMs, + usage: args.result.diagnostics.usage, + destination: args.invocation.destination, + ...(failed + ? { + errorMessage: + args.result.diagnostics.errorMessage ?? "Agent invocation failed", + } + : undefined), + messages: args.result.piMessages, + actor: args.invocation.actor, + source: args.invocation.source, + surface: "internal", + }); + } + // As on a root, a watch event Turn counts toward the automated-turn limit, + // and a task or message from the parent resets it like a user message. + if (args.invocation.parentToolCallId) { + await resetAutomatedTurnLimit({ + conversationId: args.invocation.childConversationId, + }); + } else if (args.invocation.source.kind !== "agent_invocation") { + const maxTurns = botConfig.maxConsecutiveAutomatedTurns; + // A child has no Location, so this skips the notice and clears its claim. + await maybePostAutomatedTurnLimitNotice({ + conversationId: args.invocation.childConversationId, + maxTurns, + update: await recordFinishedTurnForAutomatedLimit({ + conversationId: args.invocation.childConversationId, + maxTurns, + source: args.invocation.source, + }), + }); + } const terminal = await completeAgentInvocation({ invocationId: args.invocation.invocationId, ...(failed @@ -538,31 +603,12 @@ export function createAgentInvocationWorker( context: ConversationWorkerContext, invocationId: string, ): Promise => { - const loaded = await getAgentInvocation(invocationId); - if (!loaded) { + let invocation = await getAgentInvocation(invocationId); + if (!invocation) { throw new Error(`Agent invocation is missing for ${invocationId}`); } - let invocation: AgentInvocation = loaded; - // Read the signal before any early return, so the worker clears a stop - // request that this wake handles. - const stopSignal = context.stopSignal?.(); // Report the result before ack so a failed parent append retries this work. - const report = async (ended: TerminalAgentInvocation): Promise => { - let terminal = ended; - // A stop that comes after the save still stops the child agents of - // this child. Then they cannot report for a `replaced` result, so this - // result reports now. A retry after the save reports the same way. - if (stopSignal?.aborted) { - if (terminal.parentMailboxStatus === "replaced") { - const reported = await reportReplacedAgentInvocation( - terminal.invocationId, - ); - if (reported && isTerminalAgentInvocation(reported)) { - terminal = reported; - } - } - await stopChildAgentsOf(context.conversationId); - } + const report = async (terminal: TerminalAgentInvocation): Promise => { await recordSubagentEnded(terminal); await options.enqueueResult?.(terminal); }; @@ -579,166 +625,252 @@ export function createAgentInvocationWorker( await report(terminal); }; - const turnId = getAgentInvocationTurnId(invocation.invocationId); - const endInvocation = async ( - status: "blocked" | "failed" | "stopped", - errorMessage: string, - ): Promise => { + let acknowledged = context.attempt.messages.length === 0; + const acknowledge = async (): Promise => { + if (acknowledged) { + return; + } + try { + await context.attempt.ack(); + } catch { + throw new TurnInputCommitLostError( + `Conversation work lease lost before invocation inbox ack for ${context.conversationId}`, + ); + } + acknowledged = true; + }; + + // Read the signal before any early return, so the worker clears a stop + // request that this wake handles. + const stopSignal = context.stopSignal?.(); + // `stopChildAgent` already marked its work `stopped`. Any other stop is + // recorded here and reported to the parent. + // The stop also stops the child agents of this child, so no result of + // theirs starts this child again. + const finishStopped = async (): Promise => { await finish( await completeAgentInvocation({ invocationId, - errorMessage, - status, + errorMessage: "Agent invocation was stopped", + status: "stopped", }), ); + await stopChildAgentsOf(context.conversationId); + await acknowledge(); + return { status: "completed" }; }; - return await runWorkerTurn({ - agentRunner, - context, - inputCommitted: context.attempt.messages.length === 0, - stopSignal, - steps: { - prepare: async () => { - if (invocation.childConversationId !== context.conversationId) { - throw new Error( - `Agent invocation ${invocationId} belongs to ${invocation.childConversationId}, not ${context.conversationId}`, - ); - } - if (context.destination) { - throw new Error( - `Agent invocation conversation ${context.conversationId} must not own a provider destination`, - ); - } - if (isTerminalAgentInvocation(invocation)) { - await finish(invocation); - return "finished"; - } - const projected = await projectTerminalTurn(invocation); - if (projected && isTerminalAgentInvocation(projected)) { - await finish(projected); - return "finished"; - } - invocation = - (await markAgentInvocationRunning(invocation.invocationId)) ?? - invocation; - // A stop can land between the read above and this write. - if (isTerminalAgentInvocation(invocation)) { - await finish(invocation); - return "finished"; - } - await new ConversationTurnLifecycleService( - getConversationEventStore(), - ).start({ - conversationId: invocation.childConversationId, - createdAtMs: invocation.createdAtMs, - inputMessageIds: [ - getAgentInvocationMessageId(invocation.invocationId), - ], - surface: "internal", - turnId, - }); - const [persisted, projection, tree] = await Promise.all([ - getPersistedThreadState(invocation.childConversationId), - loadProjection({ - conversationId: invocation.childConversationId, - }), - readChildDepthAndRootLocation(invocation.childConversationId), - ]); - const sandbox = createTurnSandbox({ - conversationId: invocation.childConversationId, - persisted, - }); - return { - run: { - conversationId: invocation.childConversationId, - parentConversationId: invocation.parentConversationId, - turnId, - runId: invocation.invocationId, - instruction: { - text: agentInvocationInputText(invocation), - }, - // Some types start with the parent's history. Later agent - // invocations read it from the child's own Conversation. - history: - agentTypeStartsWithParentHistory(invocation.agentType) && - projection.length === 0 - ? await loadForkHistory(invocation) - : projection, - actor: invocation.actor, - credentialContext: invocation.credentialContext, - destination: invocation.destination, - source: invocation.source, - ...(tree.location ? { location: tree.location } : undefined), - surface: "internal", - // TODO(dcramer): Issues #881 and #883 track a path for child runs - // to force interactive auth when a delegated tool requires - // credentials the parent can request. Today background children - // hard-fail instead of pausing for an OAuth link. - ...agentInvocationRunPolicy( - invocation, - tree.depth <= maxAgentDepth, - ), - state: { - sandboxRef: sandbox.ref, - }, - durability: { - drainSteeringMessages: createAgentMessageSteeringDrain( - context, - invocation, - ), - onSandboxRefChanged: sandbox.onSandboxRefChanged, - }, + const turnId = getAgentInvocationTurnId(invocation.invocationId); + const lifecycle = new ConversationTurnLifecycleService( + getConversationEventStore(), + ); + let sandboxRef: SandboxRef | undefined; + let history: PiMessage[]; + let location: Location | undefined; + let canSpawn = false; + try { + if (invocation.childConversationId !== context.conversationId) { + throw new Error( + `Agent invocation ${invocationId} belongs to ${invocation.childConversationId}, not ${context.conversationId}`, + ); + } + if (context.destination) { + throw new Error( + `Agent invocation conversation ${context.conversationId} must not own a provider destination`, + ); + } + if (isTerminalAgentInvocation(invocation)) { + await finish(invocation); + await acknowledge(); + return { status: "completed" }; + } + const projected = await projectTerminalTurn(invocation); + if (projected && isTerminalAgentInvocation(projected)) { + await finish(projected); + await acknowledge(); + return { status: "completed" }; + } + invocation = + (await markAgentInvocationRunning(invocation.invocationId)) ?? + invocation; + // A stop can land between the read above and this write. + if (isTerminalAgentInvocation(invocation)) { + await finish(invocation); + await acknowledge(); + return { status: "completed" }; + } + await lifecycle.start({ + conversationId: invocation.childConversationId, + createdAtMs: invocation.createdAtMs, + inputMessageIds: [getAgentInvocationMessageId(invocation.invocationId)], + surface: "internal", + turnId, + }); + const [persisted, projection, tree] = await Promise.all([ + getPersistedThreadState(invocation.childConversationId), + openConversationProjection({ + conversationId: invocation.childConversationId, + }), + readChildDepthAndRootLocation(invocation.childConversationId), + ]); + sandboxRef = getPersistedSandboxState(persisted); + // Some types start with the parent's history. Later agent invocations + // read it from the child's own Conversation. + history = + agentTypeStartsWithParentHistory(invocation.agentType) && + projection.messages.length === 0 + ? await loadForkHistory(invocation) + : projection.messages; + location = tree.location; + canSpawn = tree.depth <= maxAgentDepth; + } catch (error) { + if (isInvocationInputCommitLost(error)) { + return { status: "lost_lease" }; + } + if (!context.attempt.isFinalAttempt) { + throw error; + } + const terminal = await completeAgentInvocation({ + invocationId: invocation.invocationId, + errorMessage: + error instanceof Error ? error.message : "Agent invocation failed", + status: "failed", + }); + await finish(terminal); + await acknowledge(); + return { status: "completed" }; + } + + let outcome; + try { + outcome = await executeTurn( + agentRunner, + { + conversationId: invocation.childConversationId, + parentConversationId: invocation.parentConversationId, + turnId, + runId: invocation.invocationId, + instruction: { + text: agentInvocationInputText(invocation), + }, + history, + actor: invocation.actor, + credentialContext: invocation.credentialContext, + destination: invocation.destination, + source: invocation.source, + ...(location ? { location } : undefined), + surface: "internal", + ...(stopSignal ? { signal: stopSignal } : undefined), + // TODO(dcramer): Issues #881 and #883 track a path for child runs to + // force interactive auth when a delegated tool requires credentials + // the parent can request. Today background children hard-fail instead + // of pausing for an OAuth link. + ...agentInvocationRunPolicy(invocation, canSpawn), + state: { + sandboxRef, + }, + durability: { + drainSteeringMessages: createAgentMessageSteeringDrain( + context, + invocation, + ), + onInputCommitted: acknowledge, + shouldYield: context.shouldYield, + onSandboxRefChanged: async (nextSandboxRef) => { + sandboxRef = nextSandboxRef; + await persistThreadStateById(invocation.childConversationId, { + sandboxRef, + }); }, - saveResult: async (result) => - await saveAgentInvocationResult({ - invocation, - result, - sandbox, - turnId, - }), - }; - }, - // `stopChildAgent` already marked its work `stopped`. Any other stop is - // recorded here and reported to the parent. The report also stops the - // child agents of this child, so no result of theirs starts this - // child again. - stop: async () => { - await endInvocation("stopped", "Agent invocation was stopped"); - }, - suspend: async () => { - await markAgentInvocationPaused(invocation.invocationId); - }, - awaitAuth: async (outcome) => { - await endInvocation( - "blocked", - `Agent invocation requires ${outcome.providerDisplayName} authorization`, - ); - return "completed"; + }, }, - complete: async () => { - // executeTurn already recorded the terminal Turn; only report the - // result. - const terminal = await getAgentInvocation(invocation.invocationId); - if (terminal && isTerminalAgentInvocation(terminal)) { - await report(terminal); - } - }, - block: async (error) => { - const blocking = blockingInvocationError(error); - if (!blocking) { - return false; - } - await endInvocation("blocked", blocking.message); - return true; - }, - fail: async (error) => { - await endInvocation( - "failed", - error instanceof Error ? error.message : "Agent invocation failed", - ); - }, - }, - }); + async (result) => + await saveAgentInvocationResult({ + invocation, + result, + sandboxRef, + turnId, + }), + ); + } catch (error) { + // A save error has no AgentRunError wrapper. On the final attempt it + // must also fail the agent invocation, or the child stays open. + const runError = error instanceof AgentRunError ? error.cause : error; + if (isInvocationInputCommitLost(runError)) { + return { status: "lost_lease" }; + } + if (stopSignal?.aborted) { + return await finishStopped(); + } + const blocking = blockingInvocationError(runError); + if (blocking) { + const terminal = await completeAgentInvocation({ + invocationId: invocation.invocationId, + errorMessage: blocking.message, + status: "blocked", + }); + await finish(terminal); + await acknowledge(); + return { status: "completed" }; + } + // A retry cannot pass an execution limit, and its Turn record would + // hide the limit as a lost worker. + if ( + !context.attempt.isFinalAttempt && + !isTurnExecutionLimitExceededError(runError) + ) { + throw runError; + } + const terminal = await completeAgentInvocation({ + invocationId: invocation.invocationId, + errorMessage: + runError instanceof Error + ? runError.message + : "Agent invocation failed", + status: "failed", + }); + await finish(terminal); + await acknowledge(); + return { status: "completed" }; + } + + if (outcome.status !== "completed" && stopSignal?.aborted) { + return await finishStopped(); + } + if (outcome.status === "suspended") { + await markAgentInvocationPaused(invocation.invocationId); + return { status: "yielded" }; + } + if (outcome.status === "awaiting_auth") { + const terminal = await completeAgentInvocation({ + invocationId: invocation.invocationId, + errorMessage: `Agent invocation requires ${outcome.providerDisplayName} authorization`, + status: "blocked", + }); + await finish(terminal); + await acknowledge(); + return { status: "completed" }; + } + + // executeTurn already recorded the terminal Turn; only report the result. + let terminal = await getAgentInvocation(invocation.invocationId); + // A stop that comes after the save still stops the child agents of this + // child. Then they cannot report for a `replaced` result, so this result + // reports now. + if (stopSignal?.aborted) { + if ( + terminal && + isTerminalAgentInvocation(terminal) && + terminal.parentMailboxStatus === "replaced" + ) { + terminal = await reportReplacedAgentInvocation(terminal.invocationId); + } + await stopChildAgentsOf(context.conversationId); + } + if (terminal && isTerminalAgentInvocation(terminal)) { + await report(terminal); + } + await acknowledge(); + return { status: "completed" }; }; } diff --git a/packages/junior/src/chat/app/conversation-work.ts b/packages/junior/src/chat/app/conversation-work.ts index 191211dddf..fa6986f3d6 100644 --- a/packages/junior/src/chat/app/conversation-work.ts +++ b/packages/junior/src/chat/app/conversation-work.ts @@ -32,7 +32,6 @@ import { import { createSlackRuntime } from "./factory"; import type { JuniorRuntimeServiceOverrides } from "./services"; import { createSlackSystemTurnDelivery } from "@/chat/providers/slack/system-turn"; -import { postAutomatedTurnLimitNoticeForConversation } from "@/chat/slack/automated-turn-limit-notice"; import { scheduleSessionCompletedPluginTasks, type ScheduleSessionCompletedPluginTasksOptions, @@ -156,7 +155,6 @@ export function createConversationWork( getSlackAdapter: options.getSlackAdapter, state: options.state, }), - postAutomatedTurnLimitNoticeForConversation, ); // A Slack Destination uses the Slack worker. All other root work must have // been selected above this function. diff --git a/packages/junior/src/chat/providers/slack/turn.ts b/packages/junior/src/chat/providers/slack/turn.ts index 6e2c119a5b..e4c8c3227a 100644 --- a/packages/junior/src/chat/providers/slack/turn.ts +++ b/packages/junior/src/chat/providers/slack/turn.ts @@ -119,8 +119,8 @@ import { getAgentTurnDiagnosticsAttributes, } from "@/chat/services/turn-failure-response"; import { buildAuthPauseResponse } from "@/chat/services/auth-pause-response"; -import { recordAutomatedTurnLimit } from "@/chat/task-execution/core-turn"; -import { postAutomatedTurnLimitNoticeForConversation } from "@/chat/slack/automated-turn-limit-notice"; +import { recordFinishedTurnForAutomatedLimit } from "@/chat/services/automated-turn-limit"; +import { maybePostAutomatedTurnLimitNotice } from "@/chat/slack/automated-turn-limit-notice"; import { AuthorizationFlowDisabledError } from "@/chat/services/auth-pause"; import { PluginCredentialFailureError } from "@/chat/services/plugin-auth-orchestration"; import { maybeApplyProviderDefaultConfigRequest } from "@/chat/services/provider-default-config"; @@ -1281,10 +1281,17 @@ export function createSlackTurn(deps: SlackTurnDeps) { conversationId, completedState, ); - await recordAutomatedTurnLimit({ + const automatedTurnLimit = + await recordFinishedTurnForAutomatedLimit({ + conversationId, + maxTurns: botConfig.maxConsecutiveAutomatedTurns, + source, + }); + await maybePostAutomatedTurnLimitNotice({ conversationId, - postNotice: postAutomatedTurnLimitNoticeForConversation, + maxTurns: botConfig.maxConsecutiveAutomatedTurns, source, + update: automatedTurnLimit, }); } catch (saveError) { // The user already saw the reply. Record the save failure without diff --git a/packages/junior/src/chat/slack/automated-turn-limit-notice.ts b/packages/junior/src/chat/slack/automated-turn-limit-notice.ts index 7e6110ba2c..8f58880473 100644 --- a/packages/junior/src/chat/slack/automated-turn-limit-notice.ts +++ b/packages/junior/src/chat/slack/automated-turn-limit-notice.ts @@ -10,6 +10,7 @@ import { logException, logWarn } from "@/chat/logging"; import { buildAutomatedTurnLimitResponse, clearAutomatedTurnLimitNoticeClaim, + type AutomatedTurnLimitUpdate, } from "@/chat/services/automated-turn-limit"; import type { Source } from "@sentry/junior-plugin-api"; import { resolveSlackReplyTarget, sendSlackReply } from "@/chat/slack/reply"; @@ -75,3 +76,22 @@ export async function postAutomatedTurnLimitNoticeForConversation(args: { return false; } } + +/** Post the pause notice after a finished Turn hits the limit. */ +export async function maybePostAutomatedTurnLimitNotice(args: { + conversationId: string; + maxTurns: number; + nowMs?: number; + source?: Source; + update: AutomatedTurnLimitUpdate | undefined; +}): Promise { + if (!args.update?.shouldPostNotice) { + return; + } + await postAutomatedTurnLimitNoticeForConversation({ + conversationId: args.conversationId, + maxTurns: args.maxTurns, + nowMs: args.nowMs, + source: args.source, + }); +} diff --git a/packages/junior/src/chat/slack/dispatch-turn.ts b/packages/junior/src/chat/slack/dispatch-turn.ts index 82ef5616c3..e2b120b3a4 100644 --- a/packages/junior/src/chat/slack/dispatch-turn.ts +++ b/packages/junior/src/chat/slack/dispatch-turn.ts @@ -25,7 +25,6 @@ type ExecuteSlackTurn = ( onTurnOutcome?: (result: DispatchTurnResult) => void; shouldYield?: () => boolean; skipBackfill?: boolean; - stopSignal?: AbortSignal; }, ) => Promise; @@ -42,7 +41,6 @@ export function createSlackDispatchTurnRunner(options: { hooks: { ack: () => Promise; shouldYield?: () => boolean; - stopSignal?: AbortSignal; }, ): Promise { const state = getStateAdapter(); @@ -119,7 +117,6 @@ export function createSlackDispatchTurnRunner(options: { }, shouldYield: hooks.shouldYield, skipBackfill: true, - stopSignal: hooks.stopSignal, }); return { ...(errorMessage ? { errorMessage } : undefined), diff --git a/packages/junior/src/chat/task-execution/README.md b/packages/junior/src/chat/task-execution/README.md index 95144d46ef..a8b3a84248 100644 --- a/packages/junior/src/chat/task-execution/README.md +++ b/packages/junior/src/chat/task-execution/README.md @@ -11,20 +11,6 @@ accept the result. `loadTurnCheckpoint` and `saveTurnCheckpoint`. `turn-cursor.ts` stores the checkpoint and is private to this module. -`worker-turn.ts` owns the Turn lifecycle under the lease: the input ack, stop -checks, the Run, lost leases, and the choice to retry or fail on the last -attempt. Mailbox work and agent invocation work call `runWorkerTurn`. Each work -owner supplies its Run, its result save, and its finish steps. The lifecycle -does not check which owner called it. - -`core-turn.ts` owns the Turn steps that root and child work share with -different inputs: the sandbox ref save, the steering drain, the completed -Turn save, and the automated-turn limit. A child agent invocation is a core -Turn with no provider Delivery. Its own parts are only its run policy, the -history of a fork, and its Delivery, which saves the result on the agent -invocation and reports it to the parent. Slack Turns use only the -automated-turn limit step today. - `turn-wake.ts` wakes paused turns. `paused-turn.ts` runs them under the conversation lease. SQL conversation events store history. The Redis Turn cursor stores only the data that is needed to resume a Turn, including the diff --git a/packages/junior/src/chat/task-execution/conversation-turn.ts b/packages/junior/src/chat/task-execution/conversation-turn.ts index b1be6953e3..1a67c9b4d6 100644 --- a/packages/junior/src/chat/task-execution/conversation-turn.ts +++ b/packages/junior/src/chat/task-execution/conversation-turn.ts @@ -24,32 +24,35 @@ import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-life import type { ConversationTurnFailureCode } from "@/chat/conversations/history"; import { credentialContextForActor } from "@/chat/credentials/context"; import { getConversationEventStore, getConversationStore } from "@/chat/db"; -import { - buildTurnFailureResponse, - logException, - setTags, - withLogContext, -} from "@/chat/logging"; +import { logException, setTags, withLogContext } from "@/chat/logging"; import type { AgentRunner } from "@/chat/runtime/agent-runner"; +import { AgentRunError, executeTurn } from "@/chat/runtime/turn-execution"; import { buildDeliveredTurnStatePatch } from "@/chat/runtime/delivered-turn-state"; import { + getPersistedSandboxState, getPersistedThreadState, persistThreadStateById, } from "@/chat/runtime/thread-state"; import { + getConversationTurnBoundaryError, + isTurnInputCommitLostError, markTurnClosed, markTurnFailed, startActiveTurn, + TurnInputCommitLostError, } from "@/chat/runtime/turn"; import { completeAuthPauseTurn } from "@/chat/runtime/auth-pause-state"; import { getTurnUserMessage } from "@/chat/runtime/turn-user-message"; import { getAssistantReplyText } from "@/chat/services/assistant-reply"; +import { botConfig } from "@/chat/config"; import { buildConversationContext, markConversationMessage, normalizeConversationText, upsertConversationMessage, } from "@/chat/services/conversation-memory"; +import { recordFinishedTurnForAutomatedLimit } from "@/chat/services/automated-turn-limit"; +import { maybePostAutomatedTurnLimitNotice } from "@/chat/slack/automated-turn-limit-notice"; import { finalizeFailedTurnReplyWithEvent } from "@/chat/services/turn-failure-response"; import { clearPendingAuth } from "@/chat/services/pending-auth"; import { @@ -61,20 +64,12 @@ import { abandonTurnRecord, getTurnRecord, listTurnSummaries, + saveTurnCheckpoint, } from "@/chat/task-execution/checkpoint"; -import { - createTurnSandbox, - saveFinishedTurn, - type PostAutomatedTurnLimitNotice, -} from "@/chat/task-execution/core-turn"; import type { ConversationWorkerContext, ConversationWorkerResult, } from "@/chat/task-execution/worker"; -import { - runWorkerTurn, - type PreparedWorkerTurn, -} from "@/chat/task-execution/worker-turn"; import { processPluginTask, scheduleSessionCompletedPluginTasks, @@ -144,7 +139,13 @@ function resumedTimerWatchCredentialSubject( : undefined; } +function hasLostTurnInputCommit(error: unknown): boolean { + const cause = getConversationTurnBoundaryError(error)?.cause ?? error; + return isTurnInputCommitLostError(error) || isTurnInputCommitLostError(cause); +} + async function completeCancelledConversationTurn(args: { + acknowledge(): Promise; conversation: ThreadConversationState; conversationId: string; sandboxRef?: SandboxRef; @@ -188,6 +189,7 @@ async function completeCancelledConversationTurn(args: { outcome: "cancelled", turnId: args.turnId, }); + await args.acknowledge(); } /** @@ -204,15 +206,11 @@ function sharedAuthorIdentityId( } /** - * Create the shared mailbox worker for web and event Turns. Web input has no - * provider Delivery, so its replies stay in Junior. `eventDelivery` is the - * Delivery of event input, which sends replies to the Conversation Location. - * `postAutomatedTurnLimitNotice` posts the automated-turn pause notice there. + * Create the shared mailbox worker for web and event Turns. */ export function createConversationTurnWorker( agentRunner: AgentRunner, - eventDelivery?: DeliverMessage, - postAutomatedTurnLimitNotice?: PostAutomatedTurnLimitNotice, + deliverMessage?: DeliverMessage, ) { return async ( context: ConversationWorkerContext, @@ -242,8 +240,6 @@ export function createConversationTurnWorker( let event: EventMailboxMetadata["event"] | undefined; let batchEvents: ConversationMessageEvent[] = []; let authorIdentityId: string | undefined; - // The input owner selects Delivery. Source must not select it. - let eventInput = false; const storedConversation = await getConversationStore().get({ conversationId: context.conversationId, @@ -264,7 +260,6 @@ export function createConversationTurnWorker( const messages = resolved.batch.map((entry) => entry.message); const firstMetadata = first.message.input.metadata; if (isEventMailboxMetadata(firstMetadata)) { - eventInput = true; text = renderEventBatchText(messages); event = firstMetadata.event; batchEvents = eventBatchMessageEvents(messages); @@ -295,6 +290,7 @@ export function createConversationTurnWorker( const savedTurn = isResume ? await getTurnRecord(context.conversationId, turnId) : undefined; + let savedMessageIsEvent = false; if (isResume) { const userMessage = getTurnUserMessage(conversation, turnId); if (!userMessage) { @@ -302,7 +298,7 @@ export function createConversationTurnWorker( `Unable to locate the persisted user message for Turn "${turnId}"`, ); } - eventInput = isEventConversationMessage(userMessage); + savedMessageIsEvent = isEventConversationMessage(userMessage); // Resume has no new input. Restore Source and Actor from the Turn. // TODO(dcramer): Remove the saved Message fallback after no deployed Turn // cursor can omit Source or Actor. @@ -346,7 +342,14 @@ export function createConversationTurnWorker( }); const webActor = actor.platform === "web" ? actor : undefined; const conversationLocation = storedConversation?.location; - const deliverToProvider = Boolean(conversationLocation) && eventInput; + // TODO(dcramer): Remove the saved Message check after every deployed Turn + // cursor stores Event Source. + // TODO(dcramer): Remove this Source-based Delivery choice after the core + // Turn lifecycle stores assistant Messages and web and Event work + // each supplies optional provider Delivery. Source must not select Delivery. + const deliverToProvider = + Boolean(conversationLocation) && + (source.kind === "event" || savedMessageIsEvent); // TODO(dcramer): Stop deriving surface from Delivery after active Turn // lookup and reporting read Source for web and Event Turns. const surface = deliverToProvider ? ("slack" as const) : ("api" as const); @@ -365,12 +368,97 @@ export function createConversationTurnWorker( : undefined), }, async () => { - const sandbox = createTurnSandbox({ - conversation, - conversationId: context.conversationId, - persisted, - }); + let acknowledged = isResume || context.attempt.messages.length === 0; + const acknowledge = async (): Promise => { + if (acknowledged) { + return; + } + try { + await context.attempt.ack(); + } catch { + throw new TurnInputCommitLostError( + `Conversation work lease lost before mailbox Message acknowledgement for ${context.conversationId}`, + ); + } + acknowledged = true; + }; + + let sandboxRef: SandboxRef | undefined = + getPersistedSandboxState(persisted); + const initialSandboxRef = sandboxRef; const stopSignal = context.stopSignal?.(); + const completeCancelledTurn = + async (): Promise => { + try { + await completeCancelledConversationTurn({ + acknowledge, + conversation, + conversationId: context.conversationId, + sandboxRef, + turnId, + userMessageId, + }); + } catch (error) { + if (hasLostTurnInputCommit(error)) { + return { status: "lost_lease" }; + } + throw error; + } + return { status: "completed" }; + }; + + if (isResume) { + if (stopSignal?.aborted) { + return await completeCancelledTurn(); + } + } else { + upsertConversationMessage(conversation, { + id: userMessageId, + role: "user", + text: normalizeConversationText(text), + createdAtMs: startedAtMs, + author, + ...(authorIdentityId ? { authorIdentityId } : undefined), + meta: { + ...(attachments.length ? { attachments } : undefined), + explicitMention: true, + replied: false, + ...(source.kind === "web" + ? { source: "web" as const } + : undefined), + ...(event + ? { + eventType: event.eventType, + eventObjectType: event.objectType, + ...(event.trustedSummary + ? { trustedSummary: event.trustedSummary } + : undefined), + ...(batchEvents.length > 1 + ? { events: batchEvents } + : undefined), + } + : undefined), + }, + }); + await persistConversationMessages({ + conversation, + conversationId: context.conversationId, + }); + await lifecycle.start({ + conversationId: context.conversationId, + createdAtMs: Date.now(), + inputMessageIds, + surface, + turnId, + }); + if (stopSignal?.aborted) { + return await completeCancelledTurn(); + } + startActiveTurn({ + conversation, + nextTurnId: turnId, + }); + } let currentRunId: string | undefined; let assistantMessageDelivered = false; @@ -392,19 +480,17 @@ export function createConversationTurnWorker( const cards = await loadPendingMessageCards(context.conversationId); let deliveryResult: DeliveryResult | undefined; if (deliverToProvider) { - if (!conversationLocation || !eventDelivery) { + if (!conversationLocation || !deliverMessage) { throw new Error( `Conversation ${context.conversationId} cannot deliver to its Location`, ); } - deliveryResult = await eventDelivery({ + deliveryResult = await deliverMessage({ cards, conversationId: context.conversationId, location: conversationLocation, text: replyText, }); - // The Location has the reply, even when the commit below fails. - assistantMessageDelivered = true; } await commitAssistantMessage({ cards, @@ -424,64 +510,10 @@ export function createConversationTurnWorker( assistantMessageDelivered = true; failureCode = "agent_run_failed"; }; - const prepare = async (): Promise< - PreparedWorkerTurn | "finished" | "stopped" - > => { - if (isResume) { - if (stopSignal?.aborted) { - return "stopped"; - } - } else { - upsertConversationMessage(conversation, { - id: userMessageId, - role: "user", - text: normalizeConversationText(text), - createdAtMs: startedAtMs, - author, - ...(authorIdentityId ? { authorIdentityId } : undefined), - meta: { - ...(attachments.length ? { attachments } : undefined), - explicitMention: true, - replied: false, - ...(source.kind === "web" - ? { source: "web" as const } - : undefined), - ...(event - ? { - eventType: event.eventType, - eventObjectType: event.objectType, - ...(event.trustedSummary - ? { trustedSummary: event.trustedSummary } - : undefined), - ...(batchEvents.length > 1 - ? { events: batchEvents } - : undefined), - } - : undefined), - }, - }); - await persistConversationMessages({ - conversation, - conversationId: context.conversationId, - }); - await lifecycle.start({ - conversationId: context.conversationId, - createdAtMs: Date.now(), - inputMessageIds, - surface, - turnId, - }); - if (stopSignal?.aborted) { - return "stopped"; - } - startActiveTurn({ - conversation, - nextTurnId: turnId, - }); - } + try { if (stopSignal?.aborted) { - return "stopped"; + return await completeCancelledTurn(); } if (!isResume) { // Match Slack: new input supersedes an auth-paused Turn @@ -500,7 +532,7 @@ export function createConversationTurnWorker( ); for (const parked of authParked) { if (stopSignal?.aborted) { - return "stopped"; + return await completeCancelledTurn(); } await abandonTurnRecord({ conversationId: context.conversationId, @@ -529,7 +561,7 @@ export function createConversationTurnWorker( conversation, }); if (stopSignal?.aborted) { - return "stopped"; + return await completeCancelledTurn(); } const piMessages = await loadProjection({ conversationId: context.conversationId, @@ -538,8 +570,9 @@ export function createConversationTurnWorker( currentRunId = `conversation-run:${stableHex(turnId, String(startedAtMs))}`; setTags({ runId: currentRunId }); - return { - run: { + const outcome = await executeTurn( + agentRunner, + { conversationId: context.conversationId, turnId, runId: currentRunId, @@ -564,6 +597,7 @@ export function createConversationTurnWorker( ? { location: conversationLocation } : undefined), surface, + ...(stopSignal ? { signal: stopSignal } : undefined), ...(webActor ? { authorization: createWebAuthorization({ @@ -574,7 +608,7 @@ export function createConversationTurnWorker( : { disabledFeatures: ["interactive-auth"] as const }), state: { pendingAuth: conversation.processing.pendingAuth, - sandboxRef: sandbox.ref, + sandboxRef, }, delivery: deliverAssistantMessage, durability: { @@ -589,17 +623,32 @@ export function createConversationTurnWorker( }), } : undefined), - onSandboxRefChanged: sandbox.onSandboxRefChanged, + onInputCommitted: acknowledge, + shouldYield: context.shouldYield, + onSandboxRefChanged: async (nextSandboxRef) => { + sandboxRef = nextSandboxRef; + await persistThreadStateById(context.conversationId, { + conversation, + sandboxRef, + }); + }, recordPendingAuth: async (pendingAuth) => { conversation.processing.pendingAuth = pendingAuth; await persistThreadStateById(context.conversationId, { conversation, - sandboxRef: sandbox.ref, + sandboxRef, }); }, }, }, - saveResult: async (result) => { + async (result) => { + if (stopSignal?.aborted) { + throw ( + stopSignal.reason ?? + new DOMException("Turn cancelled", "AbortError") + ); + } + modelFailureCaptureAttempted = result.diagnostics.outcome !== "success"; const finalized = finalizeFailedTurnReplyWithEvent({ @@ -619,17 +668,44 @@ export function createConversationTurnWorker( sessionId: turnId, userMessageId, }); - await saveFinishedTurn({ - actor, + await persistThreadStateById(context.conversationId, { conversation: completedState.conversation, + sandboxRef: reply.sandboxRef ?? sandboxRef, + }); + if (reply.piMessages?.length) { + // Prefer the live checkpoint slice after yield/resume; first + // completion has no prior record and starts at slice 1. + const latest = await getTurnRecord( + context.conversationId, + turnId, + ); + await saveTurnCheckpoint({ + mode: "completed", + conversationId: context.conversationId, + turnId, + sliceId: latest?.sliceId ?? 1, + messages: reply.piMessages, + durationMs: reply.diagnostics.durationMs, + usage: reply.diagnostics.usage, + // TODO(dcramer): Remove checkpoint Destination after resume + // reads the Conversation Location. + destination, + source, + actor, + surface, + }); + } + const automatedTurnLimit = + await recordFinishedTurnForAutomatedLimit({ + conversationId: context.conversationId, + maxTurns: botConfig.maxConsecutiveAutomatedTurns, + source, + }); + await maybePostAutomatedTurnLimitNotice({ conversationId: context.conversationId, - destination, - postAutomatedTurnLimitNotice, - result: reply, - sandbox, + maxTurns: botConfig.maxConsecutiveAutomatedTurns, source, - surface, - turnId, + update: automatedTurnLimit, }); completedSuccessfully = reply.diagnostics.outcome === "success"; @@ -648,144 +724,123 @@ export function createConversationTurnWorker( outcome: "failed", }; }, - }; - }; + ); - return await runWorkerTurn({ - agentRunner, - context, - inputCommitted: isResume || context.attempt.messages.length === 0, - stopSignal, - steps: { - prepare, - stop: async () => { - await completeCancelledConversationTurn({ - conversation, - conversationId: context.conversationId, - sandboxRef: sandbox.ref, - turnId, - userMessageId, - }); - }, - awaitAuth: async () => { - // Close the live turn the same way Slack does after private-link - // delivery. The turn record stays paused for OAuth resume; only - // the conversation active pointer is cleared. - completeAuthPauseTurn({ - conversation, - sessionId: turnId, - }); - await persistThreadStateById(context.conversationId, { - conversation, - sandboxRef: sandbox.ref, - }); - return "paused"; - }, - complete: async () => { - if (completedSuccessfully) { - try { - await scheduleSessionCompletedPluginTasks( - { - conversationId: context.conversationId, - sessionId: turnId, - }, - { - send: async (message) => { - try { - await processPluginTask(message); - } catch (error) { - logException( - error, - "conversation.plugin.session_completion_task.failed", - { - conversationId: context.conversationId, - pluginName: message.plugin, - taskName: message.name, - turnId, - }, - ); - } - }, - }, - ); - } catch (error) { - logException( - error, - "conversation.plugin.session_completion_task.failed", - { - conversationId: context.conversationId, - turnId, - }, - ); - } - } - }, - fail: async (failure) => { - const failureEventId = - modelFailureCaptureAttempted && - failureCode === "agent_run_failed" - ? modelFailureEventId - : captureConversationTurnFailure({ - conversationId: context.conversationId, - error: failure, - failureCode, - runId: currentRunId, - turnId, - }); - try { - markTurnFailed({ - conversation, - nowMs: Date.now(), + if (outcome.status !== "completed" && stopSignal?.aborted) { + return await completeCancelledTurn(); + } + + if (outcome.status === "suspended") { + return { status: "yielded" }; + } + if (outcome.status === "awaiting_auth") { + // Close the live turn the same way Slack does after private-link + // delivery. The turn record stays paused for OAuth resume; only + // the conversation active pointer is cleared. + completeAuthPauseTurn({ + conversation, + sessionId: turnId, + }); + await persistThreadStateById(context.conversationId, { + conversation, + sandboxRef, + }); + await acknowledge(); + return { status: "paused" }; + } + if (completedSuccessfully) { + try { + await scheduleSessionCompletedPluginTasks( + { + conversationId: context.conversationId, sessionId: turnId, - userMessageId, - markConversationMessage, - }); - await persistThreadStateById(context.conversationId, { - conversation, - sandboxRef: sandbox.initial ?? null, - }); - } catch (persistenceError) { - captureConversationTurnFailure({ + }, + { + send: async (message) => { + try { + await processPluginTask(message); + } catch (error) { + logException( + error, + "conversation.plugin.session_completion_task.failed", + { + conversationId: context.conversationId, + pluginName: message.plugin, + taskName: message.name, + turnId, + }, + ); + } + }, + }, + ); + } catch (error) { + logException( + error, + "conversation.plugin.session_completion_task.failed", + { + conversationId: context.conversationId, + turnId, + }, + ); + } + } + + await acknowledge(); + return { status: "completed" }; + } catch (error) { + const failure = error instanceof AgentRunError ? error.cause : error; + if (hasLostTurnInputCommit(failure)) { + return { status: "lost_lease" }; + } + if (stopSignal?.aborted) { + return await completeCancelledTurn(); + } + if (!context.attempt.isFinalAttempt) { + throw failure; + } + + const failureEventId = + modelFailureCaptureAttempted && failureCode === "agent_run_failed" + ? modelFailureEventId + : captureConversationTurnFailure({ conversationId: context.conversationId, - error: persistenceError, - failureCode: "persistence_failed", + error: failure, + failureCode, runId: currentRunId, turnId, }); - } - await lifecycle.fail({ - conversationId: context.conversationId, - createdAtMs: Date.now(), - ...(failureEventId ? { eventId: failureEventId } : undefined), - failureCode, - turnId, - }); - // Tell the people at the Location that the Turn failed. Web - // input has no provider Delivery. The dashboard shows the - // failed Turn. A reply that the Location already has, such as - // a model failure reply, is not followed by a second message. - if ( - deliverToProvider && - failureEventId && - !assistantMessageDelivered - ) { - try { - await deliverAssistantMessage( - buildTurnFailureResponse(failureEventId), - ); - } catch (deliveryError) { - captureConversationTurnFailure({ - conversationId: context.conversationId, - error: deliveryError, - failureCode: "delivery_failed", - runId: currentRunId, - turnId, - }); - } - } - }, - }, - }); + try { + markTurnFailed({ + conversation, + nowMs: Date.now(), + sessionId: turnId, + userMessageId, + markConversationMessage, + }); + await persistThreadStateById(context.conversationId, { + conversation, + sandboxRef: initialSandboxRef ?? null, + }); + } catch (persistenceError) { + captureConversationTurnFailure({ + conversationId: context.conversationId, + error: persistenceError, + failureCode: "persistence_failed", + runId: currentRunId, + turnId, + }); + } + await lifecycle.fail({ + conversationId: context.conversationId, + createdAtMs: Date.now(), + ...(failureEventId ? { eventId: failureEventId } : undefined), + failureCode, + turnId, + }); + await acknowledge(); + return { status: "completed" }; + } }, ); }; diff --git a/packages/junior/src/chat/task-execution/core-turn.ts b/packages/junior/src/chat/task-execution/core-turn.ts deleted file mode 100644 index 3dd61aa09f..0000000000 --- a/packages/junior/src/chat/task-execution/core-turn.ts +++ /dev/null @@ -1,243 +0,0 @@ -/** - * Turn steps that every work owner shares. A root Turn and a child agent - * invocation give different inputs to the same steps. Only the run policy, - * the history of a fork, and Delivery stay with each owner. - */ -import type { Actor, Destination, Source } from "@sentry/junior-plugin-api"; -import { - buildSteeringPiMessage, - withoutSavedSteeringMessages, -} from "@/chat/agent/prompt"; -import type { AgentSteeringMessage } from "@/chat/agent/types"; -import { botConfig } from "@/chat/config"; -import { - getPersistedSandboxState, - persistThreadStateById, -} from "@/chat/runtime/thread-state"; -import type { SandboxRef } from "@/chat/sandbox/ref"; -import { logException } from "@/chat/logging"; -import { - clearAutomatedTurnLimitNoticeClaim, - recordFinishedTurnForAutomatedLimit, -} from "@/chat/services/automated-turn-limit"; -import type { AgentRunResult } from "@/chat/services/turn-result"; -import type { ThreadConversationState } from "@/chat/state/conversation"; -import { - getTurnRecord, - saveTurnCheckpoint, - type AgentTurnSurface, -} from "@/chat/task-execution/checkpoint"; -import type { InboundMessage } from "@/chat/task-execution/store"; -import type { ConversationWorkerContext } from "@/chat/task-execution/worker"; - -/** The sandbox of one Turn. Each change is saved on the Conversation. */ -export interface TurnSandbox { - /** The sandbox ref when the Turn started. */ - readonly initial: SandboxRef | undefined; - /** The latest sandbox ref of the Turn. */ - readonly ref: SandboxRef | undefined; - /** Save a new sandbox ref. Use it as `onSandboxRefChanged` of the Run. */ - onSandboxRefChanged(next: SandboxRef): Promise; -} - -/** - * Read the saved sandbox ref of a Conversation and save each change. When the - * owner keeps transcript state, `conversation` is saved with the ref. - */ -export function createTurnSandbox(args: { - conversation?: ThreadConversationState; - conversationId: string; - persisted: Record; -}): TurnSandbox { - const initial = getPersistedSandboxState(args.persisted); - let ref = initial; - return { - initial, - get ref() { - return ref; - }, - onSandboxRefChanged: async (next) => { - ref = next; - await persistThreadStateById(args.conversationId, { - ...(args.conversation - ? { conversation: args.conversation } - : undefined), - sandboxRef: ref, - }); - }, - }; -} - -/** - * Post the pause notice of the automated-turn limit at the Location of a - * Conversation. The owner of the Delivery supplies it. - */ -export type PostAutomatedTurnLimitNotice = (args: { - conversationId: string; - maxTurns: number; - source: Source; -}) => Promise; - -/** - * How the input of a finished Turn changes the automated-turn limit, when - * its Source alone does not say it. `owner`: the input came from the owner - * of the Conversation, so the count resets like a person's message does. - * `own_work`: the input is the result of work that the Conversation started, - * so the count does not change. - */ -export type TurnLimitInput = "owner" | "own_work"; - -/** - * Count one finished Turn toward the automated-turn limit, or reset the count - * when the Turn answers its owner. A root's owner is a person. A child's - * owner is its parent agent. Leave the count when the input is a result of - * the Conversation's own work. Post the pause notice when the limit is full. - */ -export async function recordAutomatedTurnLimit(args: { - conversationId: string; - /** Omit it to count the Turn by its Source. */ - input?: TurnLimitInput; - /** Posts the pause notice. Work without a Delivery omits it. */ - postNotice?: PostAutomatedTurnLimitNotice; - source: Source; -}): Promise { - if (args.input === "own_work") { - return; - } - const maxTurns = botConfig.maxConsecutiveAutomatedTurns; - const update = await recordFinishedTurnForAutomatedLimit({ - conversationId: args.conversationId, - maxTurns, - // An owner input resets the count like a person's message. - source: args.input === "owner" ? undefined : args.source, - }); - if (!update?.shouldPostNotice) { - return; - } - if (args.postNotice) { - await args.postNotice({ - conversationId: args.conversationId, - maxTurns, - source: args.source, - }); - return; - } - // Work without a Delivery has nowhere to post the notice. Clear its claim - // so that a later Turn can post it. - try { - await clearAutomatedTurnLimitNoticeClaim({ - conversationId: args.conversationId, - }); - } catch (error) { - logException(error, "automated_turn_limit.notice_claim_clear_failed"); - } -} - -/** - * Save a finished Run on its Conversation: the sandbox ref, the completed - * Turn checkpoint, and the automated-turn limit. `conversation` is the - * transcript state to save, when the owner keeps one. - */ -export async function saveFinishedTurn(args: { - actor: Actor; - conversation?: ThreadConversationState; - conversationId: string; - destination?: Destination; - postAutomatedTurnLimitNotice?: PostAutomatedTurnLimitNotice; - result: AgentRunResult; - sandbox: TurnSandbox; - source: Source; - surface: AgentTurnSurface; - turnId: string; - turnLimitInput?: TurnLimitInput; -}): Promise { - const { result } = args; - await persistThreadStateById(args.conversationId, { - ...(args.conversation ? { conversation: args.conversation } : undefined), - sandboxRef: result.sandboxRef ?? args.sandbox.ref, - }); - if (result.piMessages?.length) { - // Continue the slice of the latest checkpoint after a yield or resume. A - // Turn with no checkpoint yet starts at slice 1. - const latest = await getTurnRecord(args.conversationId, args.turnId); - await saveTurnCheckpoint({ - mode: "completed", - conversationId: args.conversationId, - turnId: args.turnId, - sliceId: latest?.sliceId ?? 1, - messages: result.piMessages, - durationMs: result.diagnostics.durationMs, - usage: result.diagnostics.usage, - ...(result.diagnostics.outcome !== "success" - ? { errorMessage: result.diagnostics.errorMessage ?? "Turn failed" } - : undefined), - // TODO(dcramer): Remove checkpoint Destination after resume reads the - // Conversation Location. - destination: args.destination, - source: args.source, - actor: args.actor, - surface: args.surface, - }); - } - await recordAutomatedTurnLimit({ - conversationId: args.conversationId, - input: args.turnLimitInput, - postNotice: args.postAutomatedTurnLimitNotice, - source: args.source, - }); -} - -/** - * Give pending mailbox input to the running Turn as steering messages. - * `select` picks the input that joins, and the mailbox ids to mark read. The - * first input that cannot join must stop the scan, so later input waits too. - * A message that a failed earlier drain already saved in the Turn does not - * join again. - */ -export function createSteeringDrain(args: { - context: ConversationWorkerContext; - /** Drain deferred input too, such as messages from a parent agent. */ - deferred?: boolean; - select(pending: InboundMessage[]): Promise<{ - joining: T[]; - readIds: string[]; - }>; - /** Save the joining input before the Turn reads it. */ - beforeAccept?(joining: T[]): Promise; - /** Mark the joining input after the Turn saved it. */ - afterAccept?(joining: T[]): Promise; - toSteeringMessage(item: T): AgentSteeringMessage; - turnId: string; -}) { - const { context } = args; - return async ( - accept: (messages: AgentSteeringMessage[]) => Promise, - ): Promise => { - const accepted: AgentSteeringMessage[] = []; - await context.attempt.drain( - async (pending) => { - const { joining, readIds } = await args.select(pending); - if (joining.length === 0) { - return readIds; - } - await args.beforeAccept?.(joining); - const saved = - (await getTurnRecord(context.conversationId, args.turnId)) - ?.piMessages ?? []; - const messages = withoutSavedSteeringMessages( - saved, - joining.map(args.toSteeringMessage), - buildSteeringPiMessage, - ); - if (messages.length > 0) { - await accept(messages); - accepted.push(...messages); - } - await args.afterAccept?.(joining); - return readIds; - }, - args.deferred ? { deferred: true } : undefined, - ); - return accepted; - }; -} diff --git a/packages/junior/src/chat/task-execution/paused-turn.ts b/packages/junior/src/chat/task-execution/paused-turn.ts index e234799438..f125e8d3de 100644 --- a/packages/junior/src/chat/task-execution/paused-turn.ts +++ b/packages/junior/src/chat/task-execution/paused-turn.ts @@ -551,20 +551,9 @@ async function runPausedTurnInContext( excludeMessageId: userMessage.id, }); const sandboxRef = getPersistedSandboxState(currentState); - // A dispatch Turn keeps the Source of its dispatch, as its first slice - // does. Its Conversation has no Slack session Source to read. - const routing = - dispatchId && activeTurn.source - ? { - ...(await resolveConversationRouting({ - conversationId: payload.conversationId, - })), - destination, - source: activeTurn.source, - } - : await resolveTurnSessionRouting({ - conversationId: payload.conversationId, - }); + const routing = await resolveTurnSessionRouting({ + conversationId: payload.conversationId, + }); const source = routing.source; const routingDestination = routing.destination; diff --git a/packages/junior/src/chat/task-execution/turn-wake.ts b/packages/junior/src/chat/task-execution/turn-wake.ts index 89d34baf70..65b771652c 100644 --- a/packages/junior/src/chat/task-execution/turn-wake.ts +++ b/packages/junior/src/chat/task-execution/turn-wake.ts @@ -11,8 +11,8 @@ import { import type { Destination } from "@sentry/junior-plugin-api"; import type { ConversationStore } from "@/chat/conversations/store"; import { - resolveConversationRouting, resolveTurnSessionRouting, + type RequiredTurnSessionRouting, } from "@/chat/services/turn-session-routing"; import { failTurnRecord, @@ -71,19 +71,12 @@ export async function getPausedTurnRequest(args: { ) { return undefined; } - let destination: Destination; + let routing: RequiredTurnSessionRouting; try { - // A dispatch Turn keeps the Source of its dispatch. The wake needs only - // the Destination of its Conversation. - const routing = turn.dispatchId - ? await resolveConversationRouting(args) - : await resolveTurnSessionRouting(args); - if (!routing?.destination) { - throw new Error( - `Conversation ${args.conversationId} is missing durable routing metadata`, - ); - } - destination = routing.destination; + routing = await resolveTurnSessionRouting({ + conversationId: args.conversationId, + conversationStore: args.conversationStore, + }); } catch (error) { await failTurnRecord({ conversationId: args.conversationId, @@ -96,7 +89,7 @@ export async function getPausedTurnRequest(args: { return { conversationId: args.conversationId, - destination, + destination: routing.destination, turnId: args.turnId, expectedVersion: turn.version, traceContext: turn.traceContext, diff --git a/packages/junior/src/chat/task-execution/web-steering.ts b/packages/junior/src/chat/task-execution/web-steering.ts index 15571e709b..8f46e5d577 100644 --- a/packages/junior/src/chat/task-execution/web-steering.ts +++ b/packages/junior/src/chat/task-execution/web-steering.ts @@ -1,5 +1,9 @@ import type { WebActor } from "@/chat/actor"; import type { AgentSteeringMessage } from "@/chat/agent/types"; +import { + buildSteeringPiMessage, + withoutSavedSteeringMessages, +} from "@/chat/agent/prompt"; import { readMessageAttachments } from "@/chat/attachments/input"; import { persistConversationMessages } from "@/chat/conversations/messages"; import { instructionProvenanceFor } from "@/chat/conversations/provenance"; @@ -12,7 +16,7 @@ import type { ConversationAuthor, ThreadConversationState, } from "@/chat/state/conversation"; -import { createSteeringDrain } from "@/chat/task-execution/core-turn"; +import { getTurnRecord } from "@/chat/task-execution/checkpoint"; import type { InboundMessage } from "@/chat/task-execution/store"; import type { ConversationWorkerContext } from "@/chat/task-execution/worker"; @@ -49,10 +53,11 @@ export function createWebSteeringDrain(args: { turnId: string; }) { const { actor, author, context, conversation, turnId } = args; - return createSteeringDrain({ - context, - turnId, - select: async (pending) => { + return async ( + accept: (messages: AgentSteeringMessage[]) => Promise, + ): Promise => { + const accepted: AgentSteeringMessage[] = []; + await context.attempt.drain(async (pending) => { const joining = pending.filter( (message) => message.source === "web" && @@ -60,14 +65,10 @@ export function createWebSteeringDrain(args: { legacyWebMailboxMetadataSchema.safeParse(message.input.metadata) .success, ); - return { - joining, - readIds: joining.map((message) => message.inboundMessageId), - }; - }, - // Save the transcript Message first. A failed write leaves the input - // pending instead of missing from the transcript. - beforeAccept: async (joining) => { + if (joining.length === 0) return []; + + // Save the transcript Message first. A failed write leaves the input + // pending instead of missing from the transcript. for (const message of joining) { const attachments = readMessageAttachments(message.input.attachments); upsertConversationMessage(conversation, { @@ -90,7 +91,22 @@ export function createWebSteeringDrain(args: { conversation, conversationId: context.conversationId, }); - }, - toSteeringMessage: (message) => steeringMessageFromWeb(message, actor), - }); + + // An earlier drain can save the messages in the Turn and fail before + // the mailbox ack. Do not give them to the Turn again. + const saved = + (await getTurnRecord(context.conversationId, turnId))?.piMessages ?? []; + const messages = withoutSavedSteeringMessages( + saved, + joining.map((message) => steeringMessageFromWeb(message, actor)), + buildSteeringPiMessage, + ); + if (messages.length > 0) { + await accept(messages); + accepted.push(...messages); + } + return joining.map((message) => message.inboundMessageId); + }); + return accepted; + }; } diff --git a/packages/junior/src/chat/task-execution/worker-turn.ts b/packages/junior/src/chat/task-execution/worker-turn.ts deleted file mode 100644 index 4c3544d765..0000000000 --- a/packages/junior/src/chat/task-execution/worker-turn.ts +++ /dev/null @@ -1,227 +0,0 @@ -/** - * Run one Turn under the conversation lease. Root mailbox work and child agent - * invocation work both use this lifecycle, so they ack, stop, retry, and fail - * the same way. Each work owner supplies its own steps. - */ -import type { AgentRun } from "@/chat/agent/types"; -import type { AgentRunOutcome } from "@/chat/runtime/agent-run-outcome"; -import type { AgentRunner } from "@/chat/runtime/agent-runner"; -import { - getConversationTurnBoundaryError, - isTurnInputCommitLostError, - TurnInputCommitLostError, -} from "@/chat/runtime/turn"; -import { - AgentRunError, - executeTurn, - type SavedTurnResult, -} from "@/chat/runtime/turn-execution"; -import { isTurnExecutionLimitExceededError } from "@/chat/services/turn-limit"; -import type { AgentRunResult } from "@/chat/services/turn-result"; -import type { - ConversationWorkerContext, - ConversationWorkerResult, -} from "@/chat/task-execution/worker"; - -/** - * The Run of a prepared Turn. The lifecycle adds the stop signal, the input - * acknowledgement, and the yield check. - */ -export interface PreparedWorkerTurn { - run: AgentRun; - saveResult(result: AgentRunResult): Promise; -} - -/** - * A prepared Turn that its owner runs, such as dispatch work through its - * provider runtime. `execute` says whether the Turn completed or paused at a - * boundary. It must call `acknowledge` when the Turn saved its input. - */ -export interface OwnerRunWorkerTurn { - execute(hooks: { - acknowledge(): Promise; - shouldYield(): boolean; - stopSignal: AbortSignal | undefined; - }): Promise<"completed" | "suspended">; -} - -/** The steps that one work owner supplies to `runWorkerTurn`. */ -export interface WorkerTurnSteps { - /** - * Start the Turn and read its Run. Return `stopped` when a stop came first. - * Return `finished` when the work already ended and only the ack is left. - */ - prepare(): Promise< - PreparedWorkerTurn | OwnerRunWorkerTurn | "finished" | "stopped" - >; - /** Record a stopped Turn. The lifecycle acks after this step. */ - stop(): Promise; - /** Record a Run that paused at a boundary. */ - suspend?(): Promise; - /** Record a Run that waits for authorization, and say how the attempt ends. */ - awaitAuth?( - outcome: Extract, - ): Promise<"completed" | "paused">; - /** Finish after `executeTurn` recorded the terminal Turn. An error retries. */ - complete(): Promise; - /** End the Turn on any attempt for an error that a retry cannot fix. Return true when handled. */ - block?(error: unknown): Promise; - /** Record the failure on the last attempt. */ - fail(error: unknown): Promise; -} - -function requireAgentRunner(agentRunner: AgentRunner | undefined) { - if (!agentRunner) { - throw new Error("A prepared Run needs an agent runner"); - } - return agentRunner; -} - -function isInputCommitLost(error: unknown): boolean { - const cause = getConversationTurnBoundaryError(error)?.cause ?? error; - return isTurnInputCommitLostError(error) || isTurnInputCommitLostError(cause); -} - -/** - * Run one Turn: prepare it, run the agent, and end the attempt. A lost lease - * returns `lost_lease`. A stop ends the Turn as stopped. Other errors retry, - * except on the last attempt or for an execution limit, where the Turn fails. - * An error after the result save always retries. - */ -export async function runWorkerTurn(args: { - /** Runs a prepared Run. An owner that runs its own Turn omits it. */ - agentRunner?: AgentRunner; - context: ConversationWorkerContext; - /** True when no attempt input needs an ack, such as on a resume. */ - inputCommitted: boolean; - steps: WorkerTurnSteps; - stopSignal: AbortSignal | undefined; -}): Promise { - const { context, steps, stopSignal } = args; - let acknowledged = args.inputCommitted; - const acknowledge = async (): Promise => { - if (acknowledged) { - return; - } - try { - await context.attempt.ack(); - } catch { - throw new TurnInputCommitLostError( - `Conversation work lease lost before mailbox acknowledgement for ${context.conversationId}`, - ); - } - acknowledged = true; - }; - const end = async ( - status: "completed" | "paused" = "completed", - ): Promise => { - await acknowledge(); - return { status }; - }; - - // After the result save, the Turn has its result. A later error must not - // run the stop or fail step, which would end the finished work again. - let saved = false; - const advance = async (): Promise< - ConversationWorkerResult | "completed" | "stopped" - > => { - const prepared = await steps.prepare(); - if (prepared === "stopped") { - return "stopped"; - } - if (prepared === "finished") { - return await end(); - } - const outcome = - "execute" in prepared - ? { - status: await prepared.execute({ - acknowledge, - shouldYield: context.shouldYield, - stopSignal, - }), - } - : await executeTurn( - requireAgentRunner(args.agentRunner), - { - ...prepared.run, - ...(stopSignal ? { signal: stopSignal } : undefined), - durability: { - ...prepared.run.durability, - onInputCommitted: acknowledge, - shouldYield: context.shouldYield, - }, - }, - async (result) => { - // A stop that aborted the Run ends the Turn as stopped. A Run - // that finished before the stop keeps its result. - if (result.diagnostics.outcome !== "success") { - stopSignal?.throwIfAborted(); - } - const turnEnd = await prepared.saveResult(result); - saved = true; - return turnEnd; - }, - ); - if (outcome.status !== "completed" && stopSignal?.aborted) { - return "stopped"; - } - if (outcome.status === "suspended") { - await steps.suspend?.(); - return { status: "yielded" }; - } - if (outcome.status === "awaiting_auth") { - if (!steps.awaitAuth) { - throw new Error("This work owner cannot wait for authorization"); - } - return await end(await steps.awaitAuth(outcome)); - } - return "completed"; - }; - - let ended: ConversationWorkerResult | "completed" | "stopped"; - try { - ended = await advance(); - } catch (error) { - // A save error has no AgentRunError wrapper. It fails the same way. - const failure = error instanceof AgentRunError ? error.cause : error; - if (isInputCommitLost(failure)) { - return { status: "lost_lease" }; - } - if (saved) { - // Retry instead of ending the Turn a second way. A child retry reports - // the saved result. - throw failure; - } - if (stopSignal?.aborted) { - ended = "stopped"; - } else if (await steps.block?.(failure)) { - ended = await end(); - } else if ( - // A retry cannot pass an execution limit, and its Turn record would - // hide the limit as a lost worker. - !context.attempt.isFinalAttempt && - !isTurnExecutionLimitExceededError(failure) - ) { - throw failure; - } else { - await steps.fail(failure); - ended = await end(); - } - } - if (typeof ended !== "string") { - return ended; - } - // An error after the Turn ends, or while a stop is recorded, retries the - // attempt. It must not run the stop or fail step. Those steps would end - // finished work again and skip its report to the parent. - try { - await (ended === "completed" ? steps.complete() : steps.stop()); - return await end(); - } catch (error) { - if (isInputCommitLost(error)) { - return { status: "lost_lease" }; - } - throw error; - } -} diff --git a/packages/junior/tests/component/agent-invocation-worker.test.ts b/packages/junior/tests/component/agent-invocation-worker.test.ts index 241974d8cc..0b491dafc7 100644 --- a/packages/junior/tests/component/agent-invocation-worker.test.ts +++ b/packages/junior/tests/component/agent-invocation-worker.test.ts @@ -177,33 +177,27 @@ describe("agent invocation worker", () => { parentConversationId: created.childConversationId, }); const stopRequest = new AbortController(); - const enqueueResult = vi - .fn(async () => {}) - .mockRejectedValueOnce(new Error("Parent mailbox unavailable")); - const worker = createAgentInvocationWorker( - { - run: async () => { - // The stop request arrives after the run finished. - stopRequest.abort(new Error("Conversation work stopped")); - return { - status: "completed" as const, - result: { - diagnostics: { - assistantMessageCount: 1, - modelId: "test-model", - outcome: "success" as const, - toolCalls: [], - toolErrorCount: 0, - toolResultCount: 0, - usedPrimaryText: true, - }, - text: "Partial result", + const worker = createAgentInvocationWorker({ + run: async () => { + // The stop request arrives after the run finished. + stopRequest.abort(new Error("Conversation work stopped")); + return { + status: "completed" as const, + result: { + diagnostics: { + assistantMessageCount: 1, + modelId: "test-model", + outcome: "success" as const, + toolCalls: [], + toolErrorCount: 0, + toolResultCount: 0, + usedPrimaryText: true, }, - }; - }, + text: "Partial result", + }, + }; }, - { enqueueResult }, - ); + }); const context = { attempt: { ack: vi.fn(async () => {}), @@ -218,21 +212,9 @@ describe("agent invocation worker", () => { stopSignal: () => stopRequest.signal, } satisfies ConversationWorkerContext; - // A failed report after the save retries the attempt. The pending stop - // must not end the saved result as stopped. - await expect(worker(context, created.invocationId)).rejects.toThrow( - "Parent mailbox unavailable", - ); - expect(context.attempt.ack).not.toHaveBeenCalled(); await expect(worker(context, created.invocationId)).resolves.toEqual({ status: "completed", }); - expect(enqueueResult).toHaveBeenLastCalledWith( - expect.objectContaining({ - parentMailboxStatus: "pending", - status: "completed", - }), - ); // The open helper would have reported for the child. The stop ends the // helper, so the child's own result reports now. diff --git a/packages/junior/tests/fixtures/model-stream.ts b/packages/junior/tests/fixtures/model-stream.ts index 3555d5b99a..1a8b8e6ece 100644 --- a/packages/junior/tests/fixtures/model-stream.ts +++ b/packages/junior/tests/fixtures/model-stream.ts @@ -7,15 +7,12 @@ import { type FauxResponseStep, } from "@earendil-works/pi-ai/providers/faux"; -/** One fixed model output for one model request. */ -export type FixedModelOutput = +type FixedModelOutput = | { type: "text"; text: string; onRequest?: () => void; waitFor?: Promise | "abort"; - /** Stop waiting for the abort after this time, as a run that ignores it. */ - abortTimeoutMs?: number; } | { type: "toolCall"; @@ -58,8 +55,6 @@ function createAssistantMessage(output: FixedModelOutput): AssistantMessage { function createResponseStep(output: FixedModelOutput): FauxResponseStep { const message = createAssistantMessage(output); const { onRequest, waitFor } = output; - const abortTimeoutMs = - output.type === "text" ? output.abortTimeoutMs : undefined; if (!onRequest && !waitFor) { return message; } @@ -73,9 +68,6 @@ function createResponseStep(output: FixedModelOutput): FauxResponseStep { if (!signal.aborted) { await new Promise((resolve) => { signal.addEventListener("abort", () => resolve(), { once: true }); - if (abortTimeoutMs !== undefined) { - setTimeout(resolve, abortTimeoutMs); - } }); } } else if (waitFor) { diff --git a/packages/junior/tests/fixtures/turn-lifecycle.ts b/packages/junior/tests/fixtures/turn-lifecycle.ts deleted file mode 100644 index c929e93138..0000000000 --- a/packages/junior/tests/fixtures/turn-lifecycle.ts +++ /dev/null @@ -1,730 +0,0 @@ -/** - * The work owners and lifecycle signals of the Turn lifecycle table in - * `integration/durable-queue.test.ts`. Each owner starts its own work through - * production entry points. Each signal sets the model output and says what the - * owner shows after the Turn ends. - */ -import { expect, vi } from "vitest"; -import { createSlackSource } from "@sentry/junior-plugin-api"; -import { enqueueAgentDispatch } from "@/chat/agent-dispatch/work"; -import { - getDispatchConversationId, - getDispatchRecord, - getDispatchTurnId, -} from "@/chat/agent-dispatch/store"; -import { - createAgentInvocation, - getAgentInvocation, - getAgentInvocationTurnId, - getLatestAgentInvocationForConversation, -} from "@/chat/agent-invocations/store"; -import { createAndEnqueueAgentInvocation } from "@/chat/agent-invocations/work"; -import { createAndEnqueueConversation } from "@/chat/conversations/web-input"; -import { getConversationEventStore, getConversationStore } from "@/chat/db"; -import { getEventCatalog } from "@/chat/events/runtime-catalog"; -import { createEventInboundMessage } from "@/chat/events/notification"; -import { runTimerWatchHeartbeat } from "@/chat/events/timers"; -import { - appendAndEnqueueInboundMessage, - CONVERSATION_WORK_MAX_RETRIES, -} from "@/chat/task-execution/store"; -import { getTurnRecord } from "@/chat/task-execution/checkpoint"; -import { createEventTools } from "@/chat/tools/events"; -import { TurnToolCallLimitExceededError } from "@/chat/services/turn-limit"; -import { createAgentDispatchTestRecord } from "./agent-dispatch"; -import { - invocationInput, - loadSubagentEvents, - prepareParentConversation, -} from "./agent-invocations"; -import { webTestActor } from "./conversation"; -import { - CONVERSATION_ID, - SLACK_BOT_USER_ID, - SLACK_DESTINATION, - type createConversationWorkHarness, - handleSlackWebhookAndFlush, - slackEnvelope, - slackWebhookRequest, -} from "./conversation-work"; -import { context, execute } from "./event-automations"; -import type { FixedModelOutput } from "./model-stream"; -import type { AgentRunner } from "@/chat/runtime/agent-runner"; -import { ConversationTurnLifecycleService } from "@/chat/conversations/turn-lifecycle"; -import { slackApiOutbox } from "./slack-api-outbox"; -import { getStateAdapter } from "@/chat/state/adapter"; - -type Harness = Awaited>; - -/** Texts of the Slack messages that the work posted. */ -export function slackReplies(): string[] { - return slackApiOutbox - .messages() - .map((call) => call.params.text) - .filter((text): text is string => typeof text === "string"); -} - -/** Where an owner sends what its Turn says. */ -type Delivery = "junior" | "slack" | "parent"; - -export interface Owner { - name: string; - delivery: Delivery; - /** Model output that ends a Turn of this owner with `text`. */ - reply?(text: string): FixedModelOutput; - /** Model output that the owner uses before its Turn under test. */ - setupOutputs?: FixedModelOutput[]; - /** Create the Conversation and queue new input. Return its id. */ - start(harness: Harness): Promise; - /** Run the Turns that must finish before the input under test. */ - setup?(args: { - conversationId: string; - harness: Harness; - process: (softYieldAfterMs?: number) => Promise; - }): Promise; - /** Read what the owner shows after the Turn ends. */ - read(conversationId: string): Promise>; - /** - * Wake results when every attempt throws, where the owner still runs its - * own retry lifecycle instead of the shared one. - */ - failedWakes?: string[]; - /** Wake results when every resume attempt throws. */ - failedResumeWakes?: string[]; - /** True when a failed Turn posts no error reply, by owner policy. */ - quietFailure?: boolean; - /** Expected parts of rows that the Slack runner still ends another way. */ - slackRunnerGaps?: Record>; -} - -/** What one row expects after the Turn ends. */ -export interface Expected { - wakes: string[]; - turn: string; - shown: Record; -} - -export const clock = { nowMs: 1_800_000_000_000 }; -export const closers: Array<() => Promise> = []; - -/** Read how the last Turn of a Conversation ended. */ -export async function lastTurnEnd(conversationId: string): Promise { - const last = (await getConversationEventStore().loadHistory(conversationId)) - .map((event) => event.data) - .filter( - (data) => data.type === "turn_completed" || data.type === "turn_failed", - ) - .at(-1); - if (!last) return "not ended"; - if (last.type === "turn_failed") return "failed"; - if (last.outcome === "blocked") return "blocked"; - return last.outcome === "cancelled" ? "stopped" : "completed"; -} - -async function assistantMessages(conversationId: string): Promise { - return (await getConversationEventStore().loadHistory(conversationId)) - .map((event) => event.data) - .flatMap((data) => - data.type === "message" && data.role === "assistant" ? [data.text] : [], - ); -} - -/** Read the latest agent invocation of a child and its report to the parent. */ -async function readChild(conversationId: string) { - const latest = await getLatestAgentInvocationForConversation(conversationId); - const invocation = latest && (await getAgentInvocation(latest.invocationId)); - if (!invocation) return {}; - const ended = ( - await loadSubagentEvents(invocation.parentConversationId) - ).filter( - (event) => - event.type === "subagent_ended" && - event.subagentInvocationId === invocation.invocationId, - ); - const turn = await getTurnRecord( - conversationId, - getAgentInvocationTurnId(invocation.invocationId), - ); - return { - invocation: invocation.status, - parentMailbox: - "parentMailboxStatus" in invocation - ? invocation.parentMailboxStatus - : undefined, - reportsToParent: ended.length, - turnRecordOpen: turn?.state === "running" || turn?.state === "paused", - }; -} - -/** Spawn a child of the root, or of another child at `depth` 2. */ -async function spawnChild( - harness: Harness, - idempotencyKey: string, - depth: 1 | 2 = 1, -) { - const { conversationStore, fixture } = await prepareParentConversation(); - closers.push(async () => await fixture.close()); - const services = { - conversationStore, - queue: harness.queue, - state: harness.state, - }; - const created = await createAndEnqueueAgentInvocation( - { ...invocationInput, idempotencyKey }, - services, - ); - if (depth === 1) { - return created.childConversationId; - } - // The child stays open, so its own child can start. - return ( - await createAndEnqueueAgentInvocation( - { - ...invocationInput, - idempotencyKey: `${idempotencyKey}-nested`, - parentConversationId: created.childConversationId, - }, - services, - ) - ).childConversationId; -} - -// Slack mailbox work requeues its message after a failed attempt and -// dead-letters it on the last attempt. -const slackFailedWakes = [ - ...Array.from( - { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, - () => "pending_requeued", - ), - "failed", -]; - -export const owners: Owner[] = [ - { - name: "root web", - delivery: "junior", - start: async (harness) => - ( - await createAndEnqueueConversation( - { - actor: webTestActor, - idempotencyKey: "lifecycle-web", - message: "Check the deploy.", - }, - { - conversationStore: getConversationStore(), - queue: harness.queue, - state: harness.state, - }, - ) - ).conversationId, - read: async (conversationId) => ({ - replies: await assistantMessages(conversationId), - }), - }, - { - name: "root event with a Location", - delivery: "slack", - start: async (harness) => { - const conversationId = "agent-dispatch:lifecycle-watch"; - await getConversationStore().recordActivity({ - conversationId, - destination: SLACK_DESTINATION, - nowMs: 1_000, - sessionSource: createSlackSource({ - channelId: SLACK_DESTINATION.channelId, - teamId: SLACK_DESTINATION.teamId, - visibility: "public", - }), - source: "event", - visibility: "public", - }); - await appendAndEnqueueInboundMessage({ - message: createEventInboundMessage({ - event: { - eventKey: "lifecycle-checks-failed", - eventType: "pull_request.checks.failed", - identifier: "getsentry/junior#1563", - namespace: "github", - occurredAtMs: 1_100, - trustedSummary: "The subscribed check failed.", - }, - receivedAtMs: 1_100, - subscription: { conversationId, id: "lifecycle-subscription" }, - text: "The subscribed check failed.", - }), - queue: harness.queue, - state: harness.state, - }); - return conversationId; - }, - read: async () => ({ replies: slackReplies() }), - }, - { - name: "root Slack", - delivery: "slack", - failedWakes: slackFailedWakes, - // The Slack resume runner fails the Turn on the first error. - failedResumeWakes: ["completed"], - slackRunnerGaps: { - // The resume runs the Turn after a stop request. - "stop after a yield": { - turn: "completed", - shown: { replies: ["Never sent."] }, - }, - // A stop after the Run drops the saved result. - "stop after the Run keeps the result": { turn: "stopped" }, - // An execution limit retries, then posts its own reply. - "an execution limit fails at once": { - wakes: slackFailedWakes, - shown: { replies: [expect.stringMatching(/execution limit/)] }, - }, - }, - start: async (harness) => { - const response = await handleSlackWebhookAndFlush({ - request: slackWebhookRequest( - slackEnvelope({ text: `<@${SLACK_BOT_USER_ID}> check the deploy` }), - ), - services: { - getSlackAdapter: () => harness.adapter, - queue: harness.queue, - runtime: harness.runtime, - state: harness.state, - }, - }); - expect(response.status).toBe(200); - return CONVERSATION_ID; - }, - read: async () => ({ replies: slackReplies() }), - }, - { - name: "dispatch", - delivery: "slack", - // An Automation run posts only a declared message. Its failure shows on - // the Automation. - quietFailure: true, - // A dispatch resumes through the Slack resume runner. - failedResumeWakes: ["completed"], - // A dispatch runs its Turn through the Slack runner. - slackRunnerGaps: { - // A stop after the Run drops the saved result. - "stop after the Run keeps the result": { - turn: "stopped", - shown: { dispatch: "failed", replies: [], turnRecordOpen: false }, - }, - }, - // A dispatch declares its result with a tool call. - reply: (message) => ({ - type: "toolCall", - name: "finishAutomationRun", - arguments: { result: "send_message", message }, - }), - start: async (harness) => { - const dispatch = await createAgentDispatchTestRecord("lifecycle"); - await enqueueAgentDispatch(dispatch, { - queue: harness.queue, - state: harness.state, - }); - return getDispatchConversationId(dispatch); - }, - read: async (conversationId) => { - const dispatchId = conversationId.replace("agent-dispatch:", ""); - const turn = await getTurnRecord( - conversationId, - getDispatchTurnId(dispatchId), - ); - return { - dispatch: (await getDispatchRecord(dispatchId))?.status, - replies: slackReplies(), - turnRecordOpen: turn?.state === "running" || turn?.state === "paused", - }; - }, - }, - { - name: "child from a parent task", - delivery: "parent", - start: async (harness) => await spawnChild(harness, "lifecycle-task"), - read: readChild, - }, - { - name: "child at depth 2", - delivery: "parent", - start: async (harness) => await spawnChild(harness, "lifecycle-nested", 2), - read: readChild, - }, - { - name: "child from a watch event", - delivery: "parent", - setupOutputs: [{ type: "text", text: "Watching." }], - start: async (harness) => await spawnChild(harness, "lifecycle-watch"), - setup: async ({ conversationId, harness, process }) => { - // Finish the parent task, then set a timer on the child. The timer - // event waits for its burst window, then starts a new agent - // invocation with an event Source. - await expect(process()).resolves.toMatchObject({ status: "completed" }); - await execute( - createEventTools({ ...context(), conversationId }, getEventCatalog()) - .watchTimer!, - { afterMs: 60_000, intent: "Check the deploy." }, - "lifecycle-timer", - ); - clock.nowMs += 120_000; - await runTimerWatchHeartbeat({ - nowMs: clock.nowMs, - queue: harness.queue, - }); - await expect(process()).resolves.toMatchObject({ - status: "pending_requeued", - }); - clock.nowMs += 120_000; - await expect(process(0)).resolves.toMatchObject({ - status: "pending_requeued", - }); - }, - read: readChild, - }, -]; - -interface BeforeArgs { - agentRunner: AgentRunner; - conversationId: string; - stop: () => void; - stopped: () => Promise; -} - -/** Request a stop after each Run, and wait until the Run sees it. */ -function stopAfterEachRun(args: BeforeArgs): void { - const run = args.agentRunner.run.bind(args.agentRunner); - vi.spyOn(args.agentRunner, "run").mockImplementation(async (request) => { - const outcome = await run(request); - args.stop(); - await args.stopped(); - // The worker reads the stop marker on an interval. - await new Promise((resolve) => { - if (!request.signal || request.signal.aborted) return resolve(); - request.signal.addEventListener("abort", () => resolve(), { once: true }); - setTimeout(resolve, 5_000); - }); - return outcome; - }); -} - -interface Signal { - name: string; - /** Owners of this row. Omit it for every owner. */ - appliesTo?(owner: Owner): boolean; - outputs(args: { - reply: (text: string) => FixedModelOutput; - stop: () => void; - }): FixedModelOutput[]; - /** Arrange the row after the owner setup, before the first wake. */ - before?(args: BeforeArgs): Promise | void; - /** Make every agent run throw from this wake on. */ - failFromWake?: number; - /** Request a stop after this wake. */ - stopAfterWake?: number; - /** Pause the first wake at its first safe boundary. */ - yieldFirst?: boolean; - /** What the owner shows after the Turn ends. */ - expected(owner: Owner): Expected; -} - -const REPLY = "Deploy checked."; -const ERROR_REPLY = expect.stringMatching(/^I ran into an internal error/); -const isChild = (owner: Owner) => owner.delivery === "parent"; - -/** What a failed Turn shows on its owner. */ -function failed(owner: Owner) { - // Where a person reads the Turn, the last attempt posts an error reply. - return finished( - owner, - owner.delivery === "junior" || owner.quietFailure ? undefined : ERROR_REPLY, - "failed", - ); -} - -/** The wakes of work that throws on every attempt until the last one. */ -function failedAttempts(owner: Owner): string[] { - return ( - owner.failedWakes ?? [ - ...Array.from( - { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, - () => "failed", - ), - "completed", - ] - ); -} - -/** What a finished Turn shows on its owner. */ -function finished( - owner: Owner, - text: unknown, - end: "blocked" | "completed" | "failed" | "stopped", -) { - if (owner.delivery === "parent") { - // A child has no Delivery. It reports its end to the parent once. - return { - invocation: end, - parentMailbox: "appended", - reportsToParent: 1, - // No Turn of an ended child can resume. - turnRecordOpen: false, - }; - } - return { - // A dispatch has no stopped status. A stop fails it. - // No Turn of an ended dispatch can resume. - ...(owner.name === "dispatch" - ? { - dispatch: end === "completed" ? "completed" : "failed", - turnRecordOpen: false, - } - : undefined), - replies: text ? [text] : [], - }; -} - -export const signals: Signal[] = [ - { - name: "new input completes", - outputs: ({ reply }) => [reply(REPLY)], - expected: (owner) => ({ - wakes: ["completed"], - turn: "completed", - shown: finished(owner, REPLY, "completed"), - }), - }, - { - name: "yield, then an empty wake resumes", - outputs: ({ reply }) => [ - { type: "toolCall", name: "systemTime", arguments: {} }, - reply(REPLY), - ], - yieldFirst: true, - expected: (owner) => ({ - wakes: ["yielded", "completed"], - turn: "completed", - shown: finished(owner, REPLY, "completed"), - }), - }, - { - name: "stop mid-Turn", - outputs: ({ stop }) => [ - { - type: "text", - text: "Never sent.", - onRequest: stop, - waitFor: "abort", - // An owner that ignores the stop finishes and fails this row. - abortTimeoutMs: 5_000, - }, - ], - expected: (owner) => ({ - // The stop request also wakes the Conversation. That wake finds no work. - wakes: ["completed", "no_work"], - turn: "stopped", - shown: finished(owner, undefined, "stopped"), - }), - }, - { - name: "every attempt throws", - outputs: () => [], - failFromWake: 0, - expected: (owner) => ({ - wakes: failedAttempts(owner), - turn: "failed", - shown: failed(owner), - }), - }, - { - name: "every resume attempt throws", - outputs: () => [{ type: "toolCall", name: "systemTime", arguments: {} }], - yieldFirst: true, - failFromWake: 1, - expected: (owner) => ({ - wakes: [ - "yielded", - ...(owner.failedResumeWakes ?? [ - ...Array.from( - { length: CONVERSATION_WORK_MAX_RETRIES - 1 }, - () => "failed", - ), - "completed", - ]), - ], - turn: "failed", - shown: failed(owner), - }), - }, - { - name: "stop after a yield", - outputs: ({ reply }) => [ - { type: "toolCall", name: "systemTime", arguments: {} }, - reply("Never sent."), - ], - yieldFirst: true, - stopAfterWake: 0, - expected: (owner) => ({ - // The resume wake ends the Turn. The stop wake finds no work. - wakes: ["yielded", "completed", "no_work"], - turn: "stopped", - shown: finished(owner, undefined, "stopped"), - }), - }, - { - name: "stop before the Run", - outputs: ({ reply }) => [reply("Never sent.")], - before: async ({ stop }) => stop(), - expected: (owner) => ({ - wakes: ["completed", "no_work"], - turn: "stopped", - shown: finished(owner, undefined, "stopped"), - }), - }, - { - name: "stop after the Run keeps the result", - outputs: ({ reply }) => [reply(REPLY)], - before: stopAfterEachRun, - expected: (owner) => ({ - wakes: ["completed", "no_work"], - turn: "completed", - shown: finished(owner, REPLY, "completed"), - }), - }, - { - // The child still has an open child agent, so its saved result waits - // for that work. The stop ends that work, so the result reports now. - name: "an error after the save while a stop is pending", - appliesTo: isChild, - outputs: ({ reply }) => [reply(REPLY)], - before: async (args) => { - await createAgentInvocation({ - ...invocationInput, - idempotencyKey: `${args.conversationId}-open-child`, - parentConversationId: args.conversationId, - }); - stopAfterEachRun(args); - vi.spyOn( - ConversationTurnLifecycleService.prototype, - "complete", - ).mockRejectedValueOnce(new Error("event store unavailable")); - }, - expected: (owner) => ({ - // The failed write retries. The retry reports the saved result. - wakes: ["failed", "completed", "no_work"], - turn: "completed", - shown: finished(owner, REPLY, "completed"), - }), - }, - { - name: "an execution limit fails at once", - outputs: () => [], - before: ({ agentRunner }) => { - vi.spyOn(agentRunner, "run").mockRejectedValue( - new TurnToolCallLimitExceededError(1), - ); - }, - expected: (owner) => ({ - wakes: ["completed"], - turn: "failed", - shown: failed(owner), - }), - }, - { - // The model failure reply reaches the Location, then the save after it - // fails on the last attempt. The Turn fails with no second error reply. - name: "a save error after a delivered failure reply", - appliesTo: (owner) => owner.name === "root event with a Location", - outputs: () => [{ type: "error", errorMessage: "model unavailable" }], - before: ({ agentRunner, conversationId }) => { - // The earlier attempts throw before any reply. - let failures = CONVERSATION_WORK_MAX_RETRIES - 1; - const run = agentRunner.run.bind(agentRunner); - vi.spyOn(agentRunner, "run").mockImplementation(async (request) => { - if (failures-- > 0) throw new Error("agent unavailable"); - return await run(request); - }); - const state = getStateAdapter(); - const set = state.set.bind(state); - let saveFailed = false; - vi.spyOn(state, "set").mockImplementation(async (key, ...rest) => { - if ( - !saveFailed && - key === `thread-state:${conversationId}` && - slackReplies().length - ) { - saveFailed = true; - throw new Error("state unavailable"); - } - return await set(key, ...rest); - }); - }, - expected: (owner) => ({ - wakes: failedAttempts(owner), - turn: "failed", - // Only the model failure reply. - shown: { - replies: [ - expect.stringMatching(/^The model provider returned an error/), - ], - }, - }), - }, - { - name: "a setup error fails on the last attempt", - outputs: ({ reply }) => [reply("Never sent.")], - // Each start write fails, also the one in a fail step. The failure - // must still reach the owner. - before: () => { - vi.spyOn( - ConversationTurnLifecycleService.prototype, - "start", - ).mockRejectedValue(new Error("event store unavailable")); - }, - expected: (owner) => ({ - wakes: failedAttempts(owner), - turn: "failed", - shown: failed(owner), - }), - }, - { - // The Turn end write fails on the last attempt. The dispatch must still - // end, so its Automation run does not stay open. - name: "a failed Turn end write still ends the dispatch", - appliesTo: (owner) => owner.name === "dispatch", - outputs: () => [], - failFromWake: 0, - before: () => { - vi.spyOn( - ConversationTurnLifecycleService.prototype, - "fail", - ).mockRejectedValue(new Error("event store unavailable")); - }, - expected: (owner) => ({ - // The last attempt throws after the dispatch ends. No work is left. - wakes: Array.from( - { length: CONVERSATION_WORK_MAX_RETRIES }, - () => "failed", - ), - turn: "not ended", - shown: failed(owner), - }), - }, - { - name: "an auth pause blocks the child", - appliesTo: isChild, - outputs: () => [], - before: ({ agentRunner }) => { - vi.spyOn(agentRunner, "run").mockResolvedValue({ - providerDisplayName: "GitHub", - status: "awaiting_auth", - }); - }, - expected: (owner) => ({ - wakes: ["completed"], - turn: "blocked", - shown: finished(owner, undefined, "blocked"), - }), - }, -]; diff --git a/packages/junior/tests/integration/agent-invocation-concurrency.test.ts b/packages/junior/tests/integration/agent-invocation-concurrency.test.ts index 9aa874882f..b5cf833c31 100644 --- a/packages/junior/tests/integration/agent-invocation-concurrency.test.ts +++ b/packages/junior/tests/integration/agent-invocation-concurrency.test.ts @@ -495,32 +495,12 @@ describe("agent invocation identity and concurrency", () => { it("wakes a child with its own timer by the same rules as a root Conversation", async () => { const clock = vi.spyOn(Date, "now").mockReturnValue(1_800_000_000_000); - // An action to run during the next model request of a child. - let duringNextRequest: (() => Promise) | undefined; - const harness = await createHarness((request) => { - const stream = taskModel(request); - return async (model, context, options) => { - const action = duringNextRequest; - duringNextRequest = undefined; - await action?.(); - return await stream(model, context, options); - }; - }); + const harness = await createHarness(taskModel); try { - // The task Turn starts a grandchild, as spawnAgent does. - duringNextRequest = async () => - await harness.spawn({ - idempotencyKey: "grandchild-1", - input: "Check one service.", - parentConversationId: child.childConversationId, - }); - // As from spawnAgent, the task has the parent's tool call, so it - // resets the automated-turn count. const child = await harness.spawn({ agentName: "watcher", idempotencyKey: "watch-1", input: "Watch the deploy.", - parentToolCallId: "spawn-watcher", }); await harness.drainQueuedWork(); @@ -542,14 +522,8 @@ describe("agent invocation identity and concurrency", () => { // The timer runs on the child as the event Actor, with an event Source // and the grants of the person who set it. const requests = harness.requests(); - expect(requests.map((request) => request.conversationId)).toEqual([ - child.childConversationId, - expect.not.stringMatching(child.childConversationId), - // The grandchild result starts a Turn on the child. - child.childConversationId, - child.childConversationId, - ]); - const timerRun = requests[3]!; + expect(requests).toHaveLength(2); + const timerRun = requests[1]!; expect(timerRun).toMatchObject({ actor: EVENT_SYSTEM_ACTOR, conversationId: child.childConversationId, @@ -578,8 +552,6 @@ describe("agent invocation identity and concurrency", () => { status: "completed", }); // The timer Turn counts toward the automated-turn limit, as on a root. - // The grandchild result is work that the child started, so its Turn - // does not count. await expect( getAutomatedTurnLimitState({ conversationId: child.childConversationId, @@ -615,7 +587,7 @@ describe("agent invocation identity and concurrency", () => { clock.mockReturnValue(clock() + 600_000); await runTimerWatchHeartbeat({ nowMs: clock(), queue: harness.queue }); await harness.drainQueuedWork(); - expect(harness.requests()).toHaveLength(4); + expect(harness.requests()).toHaveLength(2); await expect(listAgents()).resolves.toMatchObject({ agents: [{ status: "stopped" }], }); @@ -646,8 +618,6 @@ describe("agent invocation identity and concurrency", () => { clock.mockReturnValue(clock() + 120_000); await harness.drainQueuedWork(); expect(harness.requests().map((request) => request.source.kind)).toEqual([ - "agent_invocation", - "agent_invocation", "agent_invocation", "event", "agent_invocation", diff --git a/packages/junior/tests/integration/agent-invocation-work.test.ts b/packages/junior/tests/integration/agent-invocation-work.test.ts index 1bba4c592e..14e4a0d508 100644 --- a/packages/junior/tests/integration/agent-invocation-work.test.ts +++ b/packages/junior/tests/integration/agent-invocation-work.test.ts @@ -562,6 +562,13 @@ describe("agent invocation conversation work", () => { await expect(getAgentInvocation(invocationId)).resolves.toMatchObject( expected, ); + // An ended child leaves no paused Turn that a later wake could resume. + await expect( + getTurnRecord( + childConversationId, + getAgentInvocationTurnId(invocationId), + ), + ).resolves.not.toMatchObject({ state: "paused" }); const projection = await loadProjection({ conversationId: childConversationId, }); diff --git a/packages/junior/tests/integration/durable-queue.test.ts b/packages/junior/tests/integration/durable-queue.test.ts index d2745a29a3..852a0819fb 100644 --- a/packages/junior/tests/integration/durable-queue.test.ts +++ b/packages/junior/tests/integration/durable-queue.test.ts @@ -1,6 +1,8 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { StreamFn } from "@earendil-works/pi-agent-core"; import { fauxAssistantMessage } from "@earendil-works/pi-ai/providers/faux"; import { createSlackSource } from "@sentry/junior-plugin-api"; +import { createConversationWork } from "@/chat/app/conversation-work"; import { FUNCTION_TIMEOUT_BUFFER_SECONDS, botConfig, @@ -18,14 +20,13 @@ import { import type { PiMessage } from "@/chat/pi/messages"; import { coerceThreadConversationState } from "@/chat/state/conversation"; import { hydrateConversationMessages } from "@/chat/conversations/messages"; -import { disconnectStateAdapter } from "@/chat/state/adapter"; +import { disconnectStateAdapter, getStateAdapter } from "@/chat/state/adapter"; import { getConversationStore } from "@/chat/db"; import { recoverConversationWork } from "@/chat/task-execution/heartbeat"; import { LOST_WORKER_RESUME_MESSAGE } from "@/chat/task-execution/paused-turn"; import { appendAndEnqueueInboundMessage, CONVERSATION_WORK_LEASE_TTL_MS, - CONVERSATION_WORK_MAX_RETRIES, getConversationWorkState, requestConversationWork, startConversationWork, @@ -35,11 +36,13 @@ import { listTurnSummaries, saveTurnCheckpoint, } from "@/chat/task-execution/checkpoint"; +import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; import { CONVERSATION_ID, SLACK_DESTINATION, + createConversationWorkQueueTestAdapter, SLACK_BOT_USER_ID, - createConversationWorkHarness, + createSlackAdapterFixture, deferred, handleSlackWebhookAndFlush, slackEnvelope, @@ -51,22 +54,10 @@ import { queueSlackApiResponse, resetSlackApiMockState, } from "../msw/handlers/slack-api"; -import { - createModelStream, - type FixedModelOutput, -} from "../fixtures/model-stream"; +import { createModelStream } from "../fixtures/model-stream"; import { createModelAgentRunner } from "../fixtures/agent-runner"; import { createEventInboundMessage } from "@/chat/events/notification"; import { conversationTurnIdForMessage } from "@/chat/conversations/web-input"; -import { stopConversationTurn } from "@/chat/conversations/stop"; -import { - clock, - closers, - lastTurnEnd, - owners, - signals, - slackReplies, -} from "../fixtures/turn-lifecycle"; /** * Turn-lifecycle product outcomes for one conversation under the durable queue. @@ -97,24 +88,39 @@ function almostSpentStartedAtMs(remainingMs: number): number { * They may fake only model generation at the agent stream boundary and Slack * I/O at the adapter boundary. */ -async function slack(options: { outputs?: FixedModelOutput[] } = {}) { - const agentRunner = createModelAgentRunner( - createModelStream( - options.outputs ?? [{ type: "text", text: "Deploy checked." }], - ), - ); - const work = await createConversationWorkHarness(agentRunner, { +async function slack(options: { modelStream?: StreamFn } = {}) { + const state = getStateAdapter(); + await state.connect(); + const wakes = createConversationWorkQueueTestAdapter(); + const adapter = createSlackAdapterFixture(); + const modelStream = + options.modelStream ?? + createModelStream([{ type: "text", text: "Deploy checked." }]); + const agentRunner = createModelAgentRunner(modelStream); + const work = createConversationWork({ + agentRunner, + conversationStore: getConversationStore(), + getSlackAdapter: () => adapter, + queue: wakes, services: { agentRunner }, + state, }); - const wakes = work.queue; return { - ...work, - agentRunner, + state, wakes, - replies: slackReplies, + replies: () => + slackApiOutbox + .messages() + .map((call) => call.params.text) + .filter((text): text is string => typeof text === "string"), next: async (requestStartedAtMs?: number) => { // Use production worker shouldYield (lease + host request deadline). - const process = async () => await work.process(wakes.takeMessage()); + const process = async () => + await processConversationQueueMessage(wakes.takeMessage(), { + queue: wakes, + run: work.run, + state, + }); return requestStartedAtMs === undefined ? await process() : await runWithTurnRequestDeadline(process, requestStartedAtMs); @@ -135,10 +141,10 @@ async function slack(options: { outputs?: FixedModelOutput[] } = {}) { }), ), services: { - getSlackAdapter: () => work.adapter, + getSlackAdapter: () => adapter, queue: wakes, runtime: work.runtime, - state: work.state, + state, }, }), }; @@ -336,10 +342,10 @@ describe("durable queue contract", () => { it("runs one mention to one reply and leaves the conversation free", async () => { const q = await slack({ // Second reply is for the later user mention in expectNextTurn. - outputs: [ + modelStream: createModelStream([ { type: "text", text: "Deploy checked." }, { type: "text", text: "Deploy checked." }, - ], + ]), }); await expect(q.send()).resolves.toMatchObject({ status: 200 }); await expect(q.next()).resolves.toEqual({ status: "completed" }); @@ -356,7 +362,7 @@ describe("durable queue contract", () => { // Watch Conversations have their own id, not a Slack thread id. const watchConversationId = "agent-dispatch:watch-1"; const q = await slack({ - outputs: [ + modelStream: createModelStream([ { type: "text", text: "The subscribed check failed." }, { type: "text", text: "The subscribed check recovered." }, { type: "toolCall", name: "systemTime", arguments: {} }, @@ -364,7 +370,7 @@ describe("durable queue contract", () => { { type: "text", text: "Deploy checked." }, { type: "toolCall", name: "systemTime", arguments: {} }, { type: "text", text: "Never sent.", waitFor: "abort" }, - ], + ]), }); const store = getConversationStore(); await store.recordActivity({ @@ -550,13 +556,13 @@ describe("durable queue contract", () => { it("re-parks when a later attempt times out before any new work is saved", async () => { // JUNIOR-7G: a second timeout must preserve the saved tool work. const q = await slack({ - outputs: [ + modelStream: createModelStream([ { type: "toolCall", name: "systemTime", arguments: {} }, { type: "text", text: "Deploy checked.", waitFor: "abort" }, { type: "text", text: "Deploy checked.", waitFor: "abort" }, { type: "text", text: "Deploy checked." }, { type: "text", text: "Deploy checked." }, - ], + ]), }); await expect(q.send()).resolves.toMatchObject({ status: 200 }); @@ -623,10 +629,10 @@ describe("durable queue contract", () => { const startedAtMs = almostSpentStartedAtMs(remainingMs); const deadlineAtMs = startedAtMs + requestBudgetMs(); const q = await slack({ - outputs: [ + modelStream: createModelStream([ { type: "text", text: "Deploy checked." }, { type: "text", text: "Deploy checked." }, - ], + ]), }); await expect(q.send()).resolves.toMatchObject({ status: 200 }); @@ -810,98 +816,3 @@ describe("durable queue contract", () => { }); }); }); - -/** - * One table for the Turn lifecycle of every work owner. Each row runs real - * production routing: input reaches the queue, the conversation worker picks - * the owner, and the real agent runs with fixed model output. Root owners - * deliver to a person. A child owner reports to its parent agent instead. - */ -describe("Turn lifecycle for each work owner", () => { - beforeEach(() => { - resetSlackApiMockState(); - clock.nowMs = 1_800_000_000_000; - vi.spyOn(Date, "now").mockImplementation(() => clock.nowMs); - }); - afterEach(async () => { - for (const close of closers.splice(0)) await close(); - resetSlackApiMockState(); - await disconnectStateAdapter(); - vi.restoreAllMocks(); - }); - - describe.each(owners)("$name", (owner) => { - const rows = signals.filter((signal) => signal.appliesTo?.(owner) ?? true); - it.each(rows)("$name", async (signal) => { - let conversationId = ""; - let stopping: Promise | undefined; - const harness = await slack({ - outputs: [ - ...(owner.setupOutputs ?? []), - ...signal.outputs({ - reply: owner.reply ?? ((text) => ({ type: "text", text })), - stop: () => stop(), - }), - ], - }); - const stop = () => { - stopping = stopConversationTurn({ - conversationId, - queue: harness.queue, - state: harness.state, - }); - }; - conversationId = await owner.start(harness); - const process = async (softYieldAfterMs?: number) => - await harness.process( - harness.queue.takeMessageFor(conversationId), - softYieldAfterMs === undefined ? {} : { softYieldAfterMs }, - ); - await owner.setup?.({ conversationId, harness, process }); - await signal.before?.({ - agentRunner: harness.agentRunner, - conversationId, - stop, - stopped: async () => await stopping, - }); - await stopping; - // Run each wake of this Conversation until no work is left. - const wakes: string[] = []; - while ( - wakes.length < CONVERSATION_WORK_MAX_RETRIES + 2 && - harness.queue - .queuedMessages() - .some((message) => message.conversationId === conversationId) - ) { - if (signal.failFromWake === wakes.length) { - // A model error ends the Run with a failed outcome. Only an error - // that the runner throws, such as a lost store, retries the wake. - vi.spyOn(harness.agentRunner, "run").mockRejectedValue( - new Error("agent unavailable"), - ); - } - const result = await process( - signal.yieldFirst && wakes.length === 0 ? 0 : undefined, - ); - wakes.push(result.status); - if (signal.stopAfterWake === wakes.length - 1) { - stop(); - await stopping; - } - // Each retry wake has a new idempotency key from the clock. - clock.nowMs += 1_000; - } - await stopping; - - const expected = { - ...signal.expected(owner), - ...owner.slackRunnerGaps?.[signal.name], - }; - expect({ - wakes, - turn: await lastTurnEnd(conversationId), - shown: await owner.read(conversationId), - }).toEqual(expected); - }); - }); -}); diff --git a/packages/junior/tests/integration/event-wake-delay.test.ts b/packages/junior/tests/integration/event-wake-delay.test.ts index 880329f13c..be455f84b5 100644 --- a/packages/junior/tests/integration/event-wake-delay.test.ts +++ b/packages/junior/tests/integration/event-wake-delay.test.ts @@ -15,7 +15,6 @@ import { type MailboxTurnWork, } from "@/chat/task-execution/mailbox-turn"; import { createConversationTurnWorker } from "@/chat/task-execution/conversation-turn"; -import { getAutomatedTurnLimitState } from "@/chat/services/automated-turn-limit"; import { getTurnRecord } from "@/chat/task-execution/checkpoint"; import { processConversationQueueMessage } from "@/chat/task-execution/vercel-callback"; import type { @@ -228,11 +227,6 @@ describe("event wake delay", () => { nowSpy.mockReturnValue(baseMs + 45_000); await expect(processNext()).resolves.toEqual({ status: "completed" }); expect(agentRuns).toHaveLength(2); - // The web Turn reset the automated-turn count, and the one event Turn - // counts once. A child agent uses the same core Turn save. - await expect( - getAutomatedTurnLimitState({ conversationId }), - ).resolves.toMatchObject({ consecutiveAutomatedTurns: 1 }); const savedEvents = await getConversationEventStore().loadHistory(conversationId); diff --git a/scripts/test-architecture-baseline.json b/scripts/test-architecture-baseline.json index 4cad06b625..ecec90d703 100644 --- a/scripts/test-architecture-baseline.json +++ b/scripts/test-architecture-baseline.json @@ -33,7 +33,7 @@ "packages/junior/tests/integration/annotation-cards.test.ts": 7, "packages/junior/tests/integration/api/conversations/cancel-pending-messages.test.ts": 2, "packages/junior/tests/integration/conversation-turn-work.test.ts": 19, - "packages/junior/tests/integration/durable-queue.test.ts": 4, + "packages/junior/tests/integration/durable-queue.test.ts": 8, "packages/junior/tests/integration/event-wake-delay.test.ts": 4, "packages/junior/tests/integration/heartbeat-timers.test.ts": 2, "packages/junior/tests/integration/local-agent-runner.test.ts": 29, @@ -173,6 +173,7 @@ "packages/junior/tests/integration/agent-invocation-concurrency.test.ts": 1, "packages/junior/tests/integration/agent-invocation-work.test.ts": 5, "packages/junior/tests/integration/conversation-turn-work.test.ts": 9, + "packages/junior/tests/integration/durable-queue.test.ts": 2, "packages/junior/tests/integration/event-wake-delay.test.ts": 1, "packages/junior/tests/integration/slack/assistant-lifecycle-contract.test.ts": 2, "packages/junior/tests/integration/slack/conversation-turn-steering-behavior.test.ts": 1