diff --git a/.gitignore b/.gitignore index 9451024..7a54f87 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ node_modules/ dist/ .DS_Store *.log + +# v2 build output (bun build src/v2/index.ts --outdir dist-v2 --target bun) +dist-v2/ diff --git a/README.md b/README.md index fff9978..b1b3375 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,64 @@ **Plugin for [OpenCode](https://github.com/anomalyco/opencode) that automatically detects and recovers from LLM session failures — stalls, broken tool calls, hallucination loops, stuck subagent parents, and more. Fully silent, zero UI pollution.** +> **Which instructions?** This README covers both builds: unmarked sections +> describe behavior shared by v1 and v2, and build-specific notes say so +> inline ("On v2…", "Both builds…"). On **v2**, install first via +> [docs/v2/installing.md](docs/v2/installing.md), then read here — the +> [Architecture](#architecture) section is v2's flow with the v1 shape kept +> as a three-line historical note. + +> **OpenCode v2 is here (stable).** This repo ships both the v1 plugin +> (`src/index.ts`) and a complete v2 port (`src/v2/index.ts`) targeting the +> stable `@opencode/plugin` 2.0.5+ API (verified live against 2.0.27; the +> 2.0.5→2.0.27 surface diff is additive-only — new experimental WS hooks, +> `remove`/`compact` on the session domain, tool `list()` — nothing the +> plugin uses was removed or reshaped). Install it with the +> [v2 install guide](docs/v2/installing.md); see +> [docs/v2/migration.md](docs/v2/migration.md) for the full migration notes. + +## Installation + +### Opencode + +Add to your `opencode.jsonc`: + +```jsonc +{ + "$schema": "https://opencode.ai/config.json", + "plugin": ["opencode-auto-resume"] +} +``` + +With options: + +```jsonc +{ + "plugin": [ + ["opencode-auto-resume", { + "chunkTimeoutMs": 180000, + "gracePeriodMs": 3000, + "maxRetries": 3 + }] + ] +} +``` + +Requires opencode v2.0+ (any 2.0.x server; verified live 2.0.27–2.0.29). +Uses core session/model APIs only — no experimental hooks. No release +bundle is published yet, so install is manual (clone, build, copy `dist`; +see the v2 install guide above) — there is currently no one-line install. + +## Updating + +Manual installs have no auto-update: `git pull` the branch, rebuild, +re-copy the bundle over the installed one, restart the server. + +## Uninstall + +Remove the plugin directory (or its bundle), delete the entry from +`opencode.jsonc`, restart the server. + ## What it does LLM sessions fail in predictable ways. This plugin monitors all sessions and automatically recovers without user intervention. Each recovery path below references the upstream OpenCode issues that motivated it — these are problems not yet resolved in the official project. @@ -12,10 +70,14 @@ LLM sessions fail in predictable ways. This plugin monitors all sessions and aut The stream goes silent but the session stays "busy". The UI shows a blinking cursor with no progress. If no events arrive for ~3 minutes (`chunkTimeoutMs` + `gracePeriodMs`), the plugin sends `"continue"` with exponential backoff. After 3 failed attempts it gives up. +Three budgets bound this path, because each answers a different question: `maxRetries` (default 3) is the per-episode ladder — it restarts when the model produces text, so a model that keeps replying "everything is done" reads 1/3 forever. `loopMaxContinues` (default 3 per `loopWindowMs`, 10 min) catches rapid-fire loops and escalates to abort+resume. `lifetimeMaxContinues` (default 9) is the lifetime cap per genuine user episode — it only resets on a genuine user message or genuine turn, never on model text, so the "done" loop above stops at 9 total continues and the session stands down until you speak. + The `busyStallStrategy` option controls this path: `"continue"` (default), `"abort"` (abort-first), or `"off"` — see [Recovery model](#recovery-model) for what a busy prompt can and cannot do while the runner is live. The plugin extracts the **agent, model, and provider** from the last session message, so it resumes with the exact same configuration the user was using (build, sisyphus, prometheus, etc.). +Token flow counts as activity. The watchdog measures silence from events, but a turn generating tokens for minutes emits no other events — so every `session.usage.updated` event whose token total grew resets the silence clock. A flat total (nothing resumed) does not reset it, so a genuinely wedged session still trips the watchdog on schedule. + _Motivated by:_ - [#34214](https://github.com/anomalyco/opencode/issues/34214) — Opencode freezes / becomes unresponsive mid-session - [#35207](https://github.com/anomalyco/opencode/issues/35207) — Session hangs indefinitely after MCP tool-call — no timeout recovery (deadlock) @@ -44,7 +106,7 @@ The model generates the same broken output repeatedly. Each `continue` just pick A separate **tool-call loop detector** catches the model calling the same tool 3+ consecutive times (or repeating patterns of length 2-5 occurring at least three times). When detected, it emits `TOOL_LOOP_RECOVERY_PROMPT` (at most twice per busy turn) to break the loop instead of blindly continuing. -Loop detection runs in two places. At idle, tool names are scanned from recent assistant messages. **Live**, every `tool.execute.before` hook fingerprints the call as `tool name + arguments` — so a subagent stuck re-reading the same file/range (even alternating between two near-identical argument sets, which never produces 3 consecutive identical calls) is caught after 6+ calls in the repeating cycle. On live detection the plugin aborts the running turn immediately (this is the sanctioned exception to the never-abort-busy rule: 6+ identical name+args fingerprints prove a hallucinated loop, and the current call has not started yet) and then sends `TOOL_LOOP_RECOVERY_PROMPT`. Esc-cancelled sessions are never touched. +Loop detection runs in two places. At idle, tool names are scanned from recent assistant messages. **Live**, every `tool.execute.before` hook fingerprints the call as `tool name + arguments` — so a subagent stuck re-reading the same file/range (even alternating between two near-identical argument sets, which never produces 3 consecutive identical calls) is caught after 6+ calls in the repeating cycle. (v1 hooks; v2 fingerprints the same way from `session.tool.called` events via `trackToolCall`.) On live detection the plugin aborts the running turn immediately (this is the sanctioned exception to the never-abort-busy rule: 6+ identical name+args fingerprints prove a hallucinated loop, and the current call has not started yet) and then sends `TOOL_LOOP_RECOVERY_PROMPT`. Esc-cancelled sessions are never touched. --- @@ -52,7 +114,7 @@ Loop detection runs in two places. At idle, tool names are scanned from recent a When the model calls a tool that does not exist (a typo, a hallucinated name, or a tool from a different plugin that isn't loaded), OpenCode returns a `tool` part with `state.status = "error"`. If the same wrong tool name appears **2 times** in the session's message history, the plugin: -1. Fetches the list of available tools via `ctx.client.tool.ids()` (cached for 5 minutes). +1. Fetches the list of available tools — via `ctx.client.tool.ids()` on v1, or `ctx.tool.list()` on v2, cached for 5 minutes. On v2 the names are the registry's *effective* names, so a namespaced tool is quoted back in the form a model would have to call it. 2. Computes the closest match by Levenshtein distance (case-insensitive, threshold = half the wrong name's length). 3. Sends a continue prompt that names the wrong tool, states it does not exist, suggests the closest match (if one was found), and lists the first 20 available tools for reference. @@ -60,6 +122,8 @@ The suggestion fires once per busy cycle. A new user message resets the counter Threshold and cache are compile-time constants (`UNKNOWN_TOOL_THRESHOLD = 2`, `TOOL_IDS_CACHE_MS = 5 min`). The detection runs at idle, alongside the tool-text recovery check. +On v2 the tool list comes from the registry, so the check degrades rather than guesses: no registry, an empty registry, or a registry that throws all mean no suggestion, because with nothing to compare against every name would look invented. + _Motivated by:_ - [#22142](https://github.com/anomalyco/opencode/issues/22142) — Repetitive tool-call loops with alibaba-coding-plan-cn/qwen3.6-plus - [#16218](https://github.com/anomalyco/opencode/issues/16218) — Model repeats the same response in a loop after generating an answer @@ -71,7 +135,7 @@ _Motivated by:_ ### Orphan parent -A subagent finishes but the parent session stays stuck as "busy" forever. The plugin detects when `busyCount` drops from >1 to 1, waits `subagentWaitMs` + `gracePeriodMs` (18s default), probes the subagent (recovering a crashed child first if possible), then aborts and resumes the parent. +A subagent finishes but the parent session stays stuck as "busy" forever. The plugin detects when `busyCount` drops from >1 to 1, confirms the survivor really has subagents, waits `subagentWaitMs` + `gracePeriodMs` (18s default), probes the subagent (nudging a crashed child once, then waiting a further `subagentWaitMs` before giving up on it), and aborts and resumes the parent if it is still stuck. The parent is never interrupted while it has a tool in flight or is waiting on the user. _Motivated by:_ - [#35066](https://github.com/anomalyco/opencode/issues/35066) — notify parent when subagent sessions finish @@ -93,7 +157,7 @@ _Motivated by:_ ### Streaming failure recovery -The AI provider's streaming response can fail mid-stream (connection reset, timeout, socket close). When the provider reports an error whose **name** matches `streamingFailureErrorNames` (exact, case-sensitive) or whose **message** matches `streamingFailureMessagePatterns` (regex, case-insensitive), the plugin arms a deferred recovery instead of relying only on the generic stall timeout. Invalid regex patterns fall back to substring matching. +The AI provider's streaming response can fail mid-stream (connection reset, timeout, socket close). When the provider reports an error whose **name** matches `streamingFailureErrorNames` (exact, case-sensitive) or whose **message** matches `streamingFailureMessagePatterns` (regex, case-insensitive), the plugin arms a deferred recovery instead of relying only on the generic stall timeout. Invalid regex patterns fall back to substring matching. (Same classification on both builds; what happens *after* classification differs — v1 flow below, v2 differences noted inline.) **Default error names:** - `ProviderError`, `APIError`, `StreamError`, `ConnectionError`, `TimeoutError` @@ -103,6 +167,12 @@ The AI provider's streaming response can fail mid-stream (connection reset, time #### Recovery behavior +v1 flow (v2 classifies identically — same option names — but feeds the +unified `recover()` ladder instead: consecutive failure recoveries with no +model output between them count up in `consecFailureRecovers` and latch +`failureLoopLatched` rather than cycling forever; there is no +`pendingRecovery*` state on v2): + 1. **Detection**: `session.error` is classified via `isStreamingFailure()` against the configured error names and message patterns 2. **State transition**: the session's `pendingRecovery` flag is armed with the error name (`pendingRecoveryReason`) and timestamp (`pendingRecoveryAt`) 3. **Recovery attempt**: once the session is idle and the backoff delay has elapsed, the timer loop sends a recovery prompt @@ -117,26 +187,28 @@ Add to your plugin options (in `opencode.jsonc`): { "streamingFailureErrorNames": ["ProviderError", "APIError", "StreamError", "ConnectionError", "TimeoutError"], "streamingFailureMessagePatterns": ["streaming response failed", "stream.*fail", "connection.*reset", "connection.*closed"], - "maxRecoveryRetries": 2, + "maxRetries": 2, "baseBackoffMs": 1000, "maxBackoffMs": 8000 } ``` +(`maxRetries` is the canonical name on both builds; v2 still accepts the v1 +alias `maxRecoveryRetries`.) -#### State machine addition +#### State machine addition (v1; v2 equivalents in parentheses) New per-session recovery fields added to the state machine: -- `pendingRecovery` — failure detected, recovery armed -- `pendingRecoveryReason` — error name that triggered the recovery -- `pendingRecoveryAt` — detection timestamp (backoff anchor) -- `recoveryAttempts` — recovery attempt counter -- `watchdogRetryGuard` — watchdog retry in progress (keeps the recovery armed) +- `pendingRecovery` — failure detected, recovery armed (v2: `pendingRecoveryArmed` latch) +- `pendingRecoveryReason` — error name that triggered the recovery (v2: folded into the recover reason string) +- `pendingRecoveryAt` — detection timestamp (backoff anchor) (v2: same role, unnamed — the timer closure holds it) +- `recoveryAttempts` — recovery attempt counter (v2: `resumeAttempts`, plus `consecFailureRecovers` for the failure loop) +- `watchdogRetryGuard` — watchdog retry in progress (keeps the recovery armed) (v2: `recovering` flag + `recoverTimer` handle) Recovery chain: `pendingRecovery` → recovery attempt → `recoveryAttempts` retry → `abort+resume` → `gaveUp`. -See [Recovery Flow Documentation](docs/architecture/recovery-flow.md) for the full state machine. +See [Architecture](#architecture) for the v2 flow. -#### Example scenario +#### Example scenario (v1 flow; v2 answers the same scenario through the unified ladder above) ``` 1. AI provider starts streaming response @@ -168,7 +240,7 @@ _Motivated by:_ ### Silent dead-stream recovery -The model stream can die after emitting only reasoning — no text part, no tool call — finalizing with `finish: "unknown"`. OpenCode treats the message as completed and the session goes idle, so no error or stall path triggers. On idle, if the **newest** assistant message has a finish reason, zero text parts, and at least `silentDeadStreamMinTokens` output tokens, the plugin sends a recovery prompt. Only the newest assistant message is evaluated — a delivered text answer means normal completion, and older tool-call steps are never misread as dead streams. Recovery is also skipped if the session has gone busy/retry again before the prompt is sent (race guard). +The model stream can die after emitting only reasoning — no text part, no tool call — finalizing with a finish reason such as `finish: "unknown"` (any finish reason qualifies, including `"stop"`). OpenCode treats the message as completed and the session goes idle, so no error or stall path triggers. On idle, if the **newest** assistant message has a finish reason, zero text parts, **zero tool-call parts**, and at least `silentDeadStreamMinTokens` output tokens, the plugin sends a recovery prompt. A finished turn that issued tool calls is working, not silent — thinking models routinely end tool-working turns with no chatter text (2026-10-04: this fired twice into an active session before the exception existed). As a second guard, recovery is refused while tool calls are still in flight, read from a snapshot taken at the idle transition (`markIdle` zeroes the live counter, and idle fires while tools run). Only the newest assistant message is evaluated — a delivered text answer means normal completion, and older tool-call steps are never misread as dead streams. Recovery is also skipped if the session has gone busy/retry again before the prompt is sent (race guard). ### Context saturation → magic-context wrapup @@ -178,7 +250,7 @@ A session can fill its usable context window without stalling — it just keeps ### Active-tool safety guard -Before **any** abort, two guards run in order: the primary deterministic in-flight counter maintained by the `tool.execute.before`/`tool.execute.after` hooks (`hasInflightTools()`), then a polled `checkSessionHasActiveTool()` fallback for sessions discovered without hooks. If a tool is running, the abort is skipped. This prevents the plugin from killing a long-running build, test suite, or command — even when it looks like a stall. +Before **any** abort, two guards run in order: the primary deterministic in-flight counter maintained by the `tool.execute.before`/`tool.execute.after` hooks (`hasInflightTools()`), then a polled `checkSessionHasActiveTool()` fallback for sessions discovered without hooks. (v1 hooks; v2 derives the same state from `session.tool.called`/`success`/`failed` events — `pendingTools` plus per-call start times.) If a tool is running, the abort is skipped. This prevents the plugin from killing a long-running build, test suite, or command — even when it looks like a stall. _Motivated by:_ - [#26063](https://github.com/anomalyco/opencode/issues/26063) — Tool execution aborted/terminated @@ -200,9 +272,9 @@ _Motivated by:_ ### ESC cancel respected -User presses ESC to cancel a request. The plugin detects `MessageAbortedError` and marks sessions as cancelled — regardless of their tracked status, so a late status flip to idle before the error cannot miss the latch — and never resumes them. Aborts initiated by the plugin itself (`pluginAbortInFlight`) are excluded, so recovery aborts are not mistaken for ESC. The grace period (`gracePeriodMs`) also lets late ESC/status events arrive before any action. +User presses ESC to cancel a request. The plugin detects `MessageAbortedError` and marks sessions as cancelled — regardless of their tracked status, so a late status flip to idle before the error cannot miss the latch — and never resumes them. Aborts initiated by the plugin itself (`pluginAbortInFlight`) are excluded, so recovery aborts are not mistaken for ESC. (v1 mechanism; v2 reads `session.execution.interrupted` by reason — only genuine user stops latch — and tracks own aborts with the self-abort window instead.) The grace period (`gracePeriodMs`) also lets late ESC/status events arrive before any action. -The back-off lifts as soon as the user sends a new prompt in that session (`chat.message` hook): a fresh user message starts a new round of work, so auto-resume re-arms. The plugin's own recovery prompts do not re-arm it. The same applies to the `task_complete` latch. +The back-off lifts as soon as the user sends a new prompt in that session (`chat.message` hook; v2 reads new inbound messages out of history instead): a fresh user message starts a new round of work, so auto-resume re-arms. The plugin's own recovery prompts do not re-arm it. The same applies to the `task_complete` latch. _Motivated by:_ - [#28453](https://github.com/anomalyco/opencode/issues/28453) — ACP session/cancel emits agent_error for MessageAbortedError before cancelled result @@ -213,17 +285,58 @@ _Motivated by:_ ### Explicit completion via `task_complete` -The agent can call the built-in `task_complete` tool to signal that all work is done. When invoked, the plugin stops sending any further `"continue"` prompts, clears all pending timers, and marks the session as complete. This replaces fragile text-based heuristics (emoji patterns, language detection) with a deterministic signal. +The agent can call the `task_complete` tool to signal that all work is done. When invoked, the plugin stops sending any further `"continue"` prompts, clears all pending timers, and marks the session as complete. This replaces fragile text-based heuristics (emoji patterns, language detection) with a deterministic signal. + +On v2 there is no built-in equivalent, so the plugin registers the tool itself through the v2 tool registry (`ctx.tool.transform`). It is registered before the first turn, and a registry that refuses the registration is logged and skipped rather than taking the watchdog down with it. If `task_complete` is called while open todos remain, the call is rejected (up to `maxRetries` times) with a message asking the agent to finish the remaining work first. -Repeat calls with no new user message in between are guarded: the first acknowledgement carries an explicit stop instruction, the second returns a repeat warning, and the third and subsequent calls are rejected as errors. This breaks the ack self-loop where the acknowledgement tool-result is fed back into the turn and a stuck model re-emits `task_complete` instead of ending with text. The counter resets on a genuine user message, on the block path above, or when any other tool runs in between. +Repeat calls with no new user message in between are guarded: the first acknowledgement carries an explicit stop instruction, the second returns a repeat warning, and the third and subsequent calls are rejected as errors. This breaks the ack self-loop where the acknowledgement tool-result is fed back into the turn and a stuck model re-emits `task_complete` instead of ending with text — v1 logged 27 consecutive acked calls with zero new user input. The counter resets on a genuine user message, on the block path above, or when any other tool runs in between. + +The todo gate is skipped for subagents: a child was never asked to do the parent's open items, so gating its report on them would block it on work outside its scope. + +**Recommended operator instructions.** A) is for global use. B) is for +scheduled-job prompts where stalls cost real time: + +A) When all requested work is finished: if todo tools are available, +mark every todo complete first — unfinished items will block the completion +signal and you'll be asked to continue them instead. Then call the +`task_complete` tool if one exists; otherwise end your reply with TASK +COMPLETE on its own last line (a trailing 🎉 is also accepted). Call it +once — repeated completion calls mean something is still open, so go do +that instead. Do not end with a question unless you actually need the user. +Put A) in your AGENTS.md if you have one, or create one — at your project +root for that project, or at `~/.config/opencode/AGENTS.md` for global +effect across every session. + +B) When the work is done, in order: (1) if todo tools are available, +mark all todos complete; (2) call `task_complete` if the tool exists, else +end your reply with TASK COMPLETE on its own last line (a trailing 🎉 is +also accepted); (3) do not end with a question unless you actually need the +user — an unanswered question reads as still-working. Never call +`task_complete` with items still open; it will be sent back with the list. +Put B) in the scheduled-job prompt itself, next to the deliverables list. + +Every capability above is conditional on existing — the text fallback is +the only step with zero dependencies, so a bare setup degrades to "end +with TASK COMPLETE" rather than misfiring. Nothing else is guaranteed to +exist, so these two locations cover it: the global file catches every +session, the job prompt catches the runs that cost real time. A copy-paste +version of the A wording plus the two structural companions (todos, +questions) ships with the plugin at [docs/AGENTS-snippet.md](docs/AGENTS-snippet.md) +for operators who have no AGENTS.md at all. --- ### 🎉 emoji completion -An assistant message ending with 🎉 resets the tool-text timer and prevents a trigger — the emoji signals the agent considers the task complete. +An assistant message ending with 🎉 latches completion exactly the way a +trailing `TASK COMPLETE` does — todo cross-check first, latch plus early +return on success, reminder nudge instead when work is listed. Neither path +touches the tool-text timer directly; both prevent triggers by latching and +ending the inspection. The emoji signals the agent considers the task complete. + +The emoji alone is not trusted: a model that finishes early celebrates early, and latching on that turns a false positive into silence. Both builds cross-check the session's todo list first, and with items still open the 🎉 is treated as a false positive — the reminder names what is unfinished instead. See [The todo list](docs/known-issues-v2.md#the-todo-list) for how v2 reads that list. --- @@ -241,7 +354,66 @@ When the assistant ends a line with `:` ("Next, I will edit the file:") — anno ### Done-claim verification -If the assistant claims the task is done ("task done", "finished", "all complete") but open todos remain, the plugin sends `DONE_WITHOUT_WORK_PROMPT` asking the agent to verify and finish remaining work. If it claims done with **no** open todos and the response carries no work description, the plugin sends `DONE_WITHOUT_DETAILS_PROMPT` — an imperative prompt demanding a concrete report (files changed, commands run, results). A response that already contains file paths, verification output, or result sections satisfies the demand on its own and never triggers the prompt. The budget is capped at `maxRetries` across busy cycles (going busy no longer re-arms it); only a genuinely new inbound user message re-arms it. Both use the real `todo.updated` event state — not regex on the message text. +If the assistant claims the task is done ("task done", "finished", "all complete") but open todos remain, the plugin sends `DONE_WITHOUT_WORK_PROMPT` asking the agent to verify and finish remaining work. If it claims done with **no** open todos and the response carries no work description, the plugin sends `DONE_WITHOUT_DETAILS_PROMPT` — an imperative prompt demanding a concrete report (files changed, commands run, results). A response that already contains file paths, verification output, or result sections satisfies the demand on its own and never triggers the prompt. The budget is capped at `maxRetries` across busy cycles (going busy no longer re-arms it); only a genuinely new inbound user message re-arms it. v1 reads the real `todo.updated` event state; v2 reads the message log (newest completed `todowrite` call) — neither uses regex on the message text for the list itself. + +--- + +### Stop statements (v2 only — v1 has no stop vocabulary) + +v1 never matched on stop phrasing: the closest it gets is noticing a "task +completed" text without 🎉 or tool call, and one prompt that tells the model +*not* to reply with 'done'/'task completed'. Everything below is v2. + +End-anchored patterns over the last three lines: bare `TASK COMPLETE`, +`no action needed`, `stopping here/there/now`, `nothing further/else/pending…`, +`no further action/work…`, `none pending/left…`, `stopping/halted/pausing…`, +`that's all…`, `parked/not started…`. A connective clause after the stop +("Stopping here for a moment, then continuing") disqualifies it — a stop is +only a stop if it is the last clause. Like every text signal, a stop claim +with open todos produces the reminder nudge instead of latching. Standby +phrasing (`standing by for next instructions`) and ready-and-waiting phrasing +(`ready when you have the next task`, `ready for your next instruction`) +latch the same way: the model is out of work, which is done, not +ready-to-continue (that family auto-sends `continue`; this one stays quiet). +A question mark disqualifies the ready pattern — `ready for the next task?` +is still-working, never a stop. + +Mined from live stall replies, all end-anchored the same way: "Stopping +here." (covered), "Standing by.", "Nothing in progress …" (bare "in +progress" alone must never match — "work is still in progress" is the +opposite), bare "Done.". Conversely "I'll report X when it lands" is future +intent, not completion, and matches nothing — documented here so a future +widening doesn't swallow it. + +Case discipline: the system's own literal is the one case-sensitive pattern +in the build — bare `TASK COMPLETE` must read byte-exact, per +[docs/AGENTS-snippet.md](docs/AGENTS-snippet.md). Every other completion +pattern is case-insensitive. A lowercase `task complete` is model prose and +latches nothing; past-tense `task completed` (any case) still latches — it +was never the literal. + +--- + +### Todo list nudges + +Opencode v1 ships todos out of the box — built-in `todowrite`/`todoread` +tools, a `todo` table, and a `todo.updated` event the plugin subscribes to +directly. OpenCode v2 removed all of it (tools, table, and event), so a v2 +session has no todo list by default. auto-resume does not own one either. It reads the session message log, where every `todowrite` call is persisted as a tool part carrying the whole list, and the newest completed call is the current list. + +The writer is the todo plugin you install. The upstream [opencode-todolist](https://github.com/aiev/opencode-todolist) restores `todowrite` and `todoread` in v2 (its sidebar strip appears in opencode-cli only — no other surface shows it); the confirmed pairing here is upstream itself (v0.6.0+, verified live — the `opencode-todo-fork` interim fork is retired). You can view the list at any time with the `/todo` command. Any plugin that persists `todowrite` calls to the message log should work, but only upstream has been tested. + +The todo plugin is optional, and without it the unfinished-task nudges never trigger. Every other recovery path runs fine on its own; the list gates exactly the nudges that check it. With no `todowrite` call in the log, auto-resume sees no list and falls back to the older behaviour (latch on the 🎉 emoji, demand a report on a bare done-claim). It never nudges on the strength of a list it failed to read. + +When a turn ends with items still open, the nudge names them: + +> You have 2 unfinished tasks: +> 1. [pending] Verify the plugin loads under v2 +> 2. [in_progress] Write the todo-nudges section of the README +> +> Please continue working on these tasks. + +`buildOpenTodosReminder` sends that verbatim, numbering each open item (`pending` or `in_progress`) and falling back to a bare `continue` when the list is unusable. Four paths send it: a 🎉 celebration with items still open, a `task_complete` with items still open, a stop statement with work listed, and a completion claim with work listed. The ready-to-continue check reads the same list in reverse: a "ready" claim while every todo is already closed gets two chances before a plain `continue`. Full mechanics, including the `ctx.storage` namespacing that rules storage out as a shared store: [The todo list](docs/known-issues-v2.md#the-todo-list). --- @@ -263,7 +435,7 @@ Long tool execution or active subagents can look like a stall. Only the session ### Spurious error suppression -After normal completion, OpenCode sometimes fires a `session.error`. All logging goes through `ctx.client.app.log()` (zero `console.log`), and errors on already-idle sessions are silently ignored. +After normal completion, OpenCode sometimes fires a `session.error`. All logging goes through `ctx.client.app.log()` (zero `console.log`) on v1; v2 has no server log sink (`ctx.app` is just name/version/channel), so it appends to its own log file and mirrors to console — errors via `console.error`, the rest via `console.log`. Errors on already-idle sessions are silently ignored on both builds. _Motivated by:_ - [#33687](https://github.com/anomalyco/opencode/issues/33687) — Interrupted assistant messages retain non-error finish value (orphan tool-input-start abort) @@ -303,6 +475,14 @@ Fires when the session still reports **busy** but no events arrived for `chunkTi **Why `"continue"` cannot unblock a live runner:** in OpenCode ≥ 1.18, `session.prompt()` while the runner is `Running` joins the existing run — the message is admitted to the session inbox and promoted at the next provider-turn boundary. A hung stream never reaches that boundary, so the parked message cannot unblock it; if the run later exits, the message may surface as an unwanted extra turn (token cost, race risk). True transport stalls need abort-first. +**What the prompt looks like.** Stall continues are rich by default +(`richContinuePrompt`, v2 only): reason, attempt and episode-retry count +(`attempt 1/3 (retry #4)`), idle duration, local timestamp, open todos, and +the no-new-work / no-destructive-actions guardrails, e.g. +`continue — stalled (no activity for 186s). attempt 1/3 (retry #1); idle 3m 6s; Oct 12, 7:37 PM EDT. Only continue…`. +Set `richContinuePrompt: false` for the bare `"continue"` (v1 behavior), or +set `continuePrompt` to any custom text, which always wins verbatim. + ### 3. Abort-first recovery Aborts the active run, then continues. The only family that can unblock a live runner. @@ -311,86 +491,155 @@ Paths: orphan parent, subagent stuck (parent side), hallucination loop, streamin ## Architecture +v1 consumed the SSE event stream (`session.status`, `session.created`, +`session.updated`, `session.idle`, `session.interrupted`, `session.error`, +`todo.updated`, `command.executed`) plus the `chat.message` and tool hooks +and the `session.todo()` API. v2 has none of those doors: it reads the +`ctx.event` stream, registers `ctx.session.hook("context"/"prompt")` and +`ctx.tool.transform`, and gets everything else through the client surfaces +below. The diagram is v2's; the v1 shape survives only in the prose that +follows it. + +The todo list is the substitution with teeth. The v2 `todo` table has no route and +emits no event, and **`ctx.storage` cannot be used to read another plugin's todo +list** — it is namespaced per plugin. `ctx.storage.get("todos/" + id)` from this +plugin resolves to `storage/plugin/auto-resume.v2/todos/.json`; a todo plugin +writing that same key lands in its own directory. Verified against a live server: +the route is `/api/plugin/storage//` and the on-disk tree is +`storage/plugin//.json`. So a storage-first reader concludes "no +todos" no matter who is writing, and fires false done-claim nudges at a session +with work still listed. + +The store both plugins can agree on is the **session message log**: every +`todowrite` call is persisted as a tool part whose input carries the entire list, +so the newest *completed* call is the current list. It is read through +`ctx.client.session.message.list` when the host offers it, and otherwise over +loopback HTTP (`GET /api/session/{id}/message`, page capped at 200 — the endpoint +answers `{ data, cursor }`, never a bare array, and orders results newest first). +`ctx.storage` survives only as a last-resort fallback, for a host that writes the +list itself. + +Confirmed working setup: `opencode-todo-fork` (a fork of +[opencode-todolist](https://github.com/aiev/opencode-todolist)) — verified live (list resolved +from the log, `/todo` round-trip green). Other todo plugins are unconfirmed: +anything that persists `todowrite` calls to the message log should read, but +only `opencode-todo-fork` has been tested. + ``` -Any SSE Event - ├─ has sessionID? → touchSession(sid) — reset only that session's timer - └─ no sessionID → ignore - -session.status events: - ├─ busy → reset timer, clear retry counters - ├─ retry → touch session - ├─ interrupted → user cancel: back off until the next user message - └─ idle → schedule tool-text check (3s delay) - └─ fetch messages → scan for XML / thinking-tool patterns - ├─ found → send recovery prompt (with backoff) - └─ not found → check ready-to-continue / done-claim patterns - └─ orphan check: busyCount dropped from >1 to 1? - └─ subagentWaitMs watch → abort + continue - -todo.updated events: - └─ track real todo state (not regex on message text) - └─ if missing (no event received): idle path + task_complete - fetch on-demand via session.todo() API so the - open-todos reminder and the 🎉 completion latch see - real state, not a stale empty array. - -Timer loop (every 5s): - for each busy session: - ├─ orphan watch active? → wait or abort+continue - ├─ busyCount > 1? → skip (subagent running) - ├─ active tool running? → skip (never abort tools) - └─ idle > 48s? → busyStallStrategy: - off → skip · abort → abort+continue - else → hallucination loop? abort : continue with backoff - -Periodic (every 60s): session.list() to discover missed sessions -Periodic: cleanup idle sessions older than 10min or >50 entries +V2 event flow (ctx.event stream; every handler resolves the session first): + +session.execution.started + ├─ own recovery turn (selfRecovery flag)? → consume flag, preserve episode + └─ genuine turn → clearLocks, reset quota ladder + stall ladder + gaveUp, + refresh model ref; then markBusy either way (per-turn budgets) + +session.step.started / .ended, session.tool.progress → touch (activity clock) +session.tool.called → pendingTools++, record id+name+start in inflightTools +session.tool.success / .failed → decrement, drop the in-flight record +session.execution.interrupted (reason=user) → userCancelled: stand down + (shutdown/superseded/inactivity reasons do NOT latch) +session.idle → markIdle → inspectOnIdle: + structural (no text needed): dead-stream → stuck-tool → silent-reasoning + empty-idle instant nudge (no reply at all, single-shot per episode) + then text/pattern: handoff, stand-down, done-claims, celebration/stop latch, + unknown-tool check; orphan arming when the busy set drains to one parent + +Watchdog (every checkIntervalMs, default 5s), per busy session: + ├─ locks / permission / self-abort window? → skip + ├─ open shells running? → wait (working, not stalled) + ├─ young in-flight tools (or fresh exempt names)? → wait + ├─ waiting on subagents / task-tool children? → wait + ├─ user input pending? → wait + └─ silence ≥ chunkTimeoutMs(+grace) × fuse → busyStallStrategy: + off → skip · abort → abort+resume · else → continue with backoff + (a genuine user message inside the backoff parks the recovery) + +Discovery (every 60s): session list → adopt missed sessions (markBusy, +stall clock starts now, never backdated) +Cleanup: idle sessions older than 10min, or past 50 idle entries (oldest first) +Singleton: every setup disposes its predecessor (last wins) via a +process-global registry; watch state is shared across re-evaluations so +stall memory survives reloads. Test harnesses scope the registry per file. + +Events consumed: session.execution.*, session.step.*, session.tool.*, +session.idle, session.revert.*, session.compaction.*, session.usage.updated, +session.deleted, permission.asked/replied, shell.created/exited; hooks: +session "context"/"prompt", tool transform. ``` -Events consumed: `session.status`, `session.created`, `session.updated`, `session.idle`, `session.interrupted`, `session.error`, `todo.updated`, `command.executed`; hooks: `chat.message`, `tool.execute.before`/`after`, `command.execute.before`. - -## Installation - -### Opencode +The v1 shape this replaced: per-event timer resets off `session.status` +(busy/idle), tool-text checks on a 3s idle delay, `todo.updated` tracking +with a `session.todo()` API fallback, and a 48s busy threshold. Tokens now +come from `session.usage.updated` and the window from `ctx.model.get()`. -Add to your `opencode.jsonc`: +## Configuration -```jsonc +```json { - "$schema": "https://opencode.ai/config.json", - "plugin": ["opencode-auto-resume"] + "plugin": [ + [ + "file:///home/YOURUSER/.config/opencode/plugins/opencode-auto-resume/dist/index.js", + { "chunkTimeoutMs": 180000, "maxRetries": 3 } + ] + ] } ``` -With options: +### OpenCode v2 (stable) + +The v2 plugin uses the `{ id, setup }` plugin shape with `ctx.event.subscribe()` (AsyncIterable) instead of the v1 hooks-object pattern, and is written against the stable v2 API (opencode 2.0.5+). It has **no runtime dependencies** — it defines the plugin helper locally and imports only `node:fs`, `node:os` and `node:path` — so there is nothing to `bun add`. Add it to your `opencode.json`: ```jsonc { - "plugin": [ - ["opencode-auto-resume", { - "chunkTimeoutMs": 180000, - "gracePeriodMs": 3000, - "maxRetries": 3 - }] + "plugins": [ + { + "package": "./plugins/auto-resume-v2.ts", + "options": { + "chunkTimeoutMs": 180000, + "maxRetries": 3 + } + } ] } ``` -## Configuration +Or place `src/v2/index.ts` directly in `~/.config/opencode/plugins/` for auto-discovery (no config entry needed). -```json -{ - "plugin": [ - [ - "file:///home/YOURUSER/.config/opencode/plugins/opencode-auto-resume/dist/index.js", - { "chunkTimeoutMs": 180000, "maxRetries": 3 } - ] - ] -} -``` +Disable via `"-auto-resume.v2"` in `plugins`. + +- **Step-by-step install guide:** [docs/v2/installing.md](docs/v2/installing.md) +- **Migration notes (v1 → v2, stable validation):** [docs/v2/migration.md](docs/v2/migration.md) +- **What the v2 port does and does not yet do:** [docs/known-issues-v2.md](docs/known-issues-v2.md) + +### How a finished turn is judged + +`session.idle` does not decide anything by itself. The turn is inspected in two +passes, because `session.idle` can arrive while the assistant's closing text is +still being written into the message history: + +1. **Straight away** — is this a dead stream, is it handing control back to the + user, is the user already busy. None of these read the final text. +2. **After `toolTextCheckDelayMs`** (3s default) — the celebration, + tool-call-as-text, ready-to-continue, action-intent and done-claim detectors, + re-reading the history. This is the pass the option exists for, and it re-runs + the guards from step 1, because three seconds is long enough for the user to + have replied and a nudge sent over them interrupts their own step. + +A new turn cancels a pending second pass, and a second idle replaces it rather +than stacking another judgement on the same text. See +[docs/known-issues-v2.md](docs/known-issues-v2.md). + +The v2 build reads and applies every option in the table below — the last of +the gaps closed with `toolTextCheckDelayMs`. Any option it accepts but does not +apply is listed in the startup line as `accepted-but-inert=…` and explained in +[docs/known-issues-v2.md](docs/known-issues-v2.md); that list is currently empty. +An option name this build does not know at all produces a single warning at +startup, so a typo or a dropped port is never silent. ### Configurable options +Defaults are the same on v1 and v2 unless a row says otherwise. + | Option | Default | Description | |---|---|---| | `chunkTimeoutMs` | `180000` | Inactivity timeout before considering stream stalled | @@ -400,12 +649,14 @@ With options: | `baseBackoffMs` | `1000` | First retry delay (doubles each attempt) | | `maxBackoffMs` | `8000` | Backoff cap | | `subagentWaitMs` | `15000` | Wait before treating orphan parent as stuck | +| `subagentDeadMs` | `1800000` | Outer bound: quiet past this with no error evidence reads as dead (v2 only). Inside it, a quiet child is waited on, not aborted — 30 min default for long-thinking models | | `loopMaxContinues` | `3` | Continues in window before triggering abort | | `loopWindowMs` | `600000` | Hallucination loop detection window (10 min) | +| `lifetimeMaxContinues` | `9` | Lifetime stall continues per genuine user episode before standing down; resets only on a genuine user message/turn, never on model text — stops the "keeps saying done" loop that restarts the `maxRetries` ladder at 1/3 forever | | `streamingFailureErrorNames` | `["ProviderError","APIError","StreamError","ConnectionError","TimeoutError"]` | Error names that classify as streaming failures (exact match) | | `streamingFailureMessagePatterns` | `["streaming response failed","stream.*fail","connection.*reset","connection.*closed"]` | Regex patterns (case-insensitive) in error messages indicating streaming failure | -| `maxRecoveryRetries` | `2` | Max streaming-failure recovery attempts before abort+resume escalation | -| `toolTextCheckDelayMs` | `3000` | Delay before scanning an idle session for tool-as-text; also the recovery watchdog delay | +| `maxRecoveryRetries` | `2` | v1 name for the streaming-failure retry budget; v2 canonical is `maxRetries` (alias still accepted) | +| `toolTextCheckDelayMs` | `3000` | Settle delay before a finished turn's closing text is judged against the done/tool patterns; also the recovery watchdog delay. On v2 only the pattern half is deferred — dead streams are still caught on the idle event, because a stream that died before delivering any text has nothing for those patterns to read. Set `0` to judge on the idle event | | `minActivityGapMs` | `1000` | Skip recovery if the session was active within this gap | | `warmupMs` | `15000` | Action-intent detection disabled while a session is younger than this | | `discoveryDelayMs` | `5000` | Delay before the initial session discovery run after plugin attach | @@ -415,16 +666,28 @@ With options: | `actionIntentPrompt` | same as `continuePrompt` | Prompt sent on action-intent detection | | `toolTextRecoveryPrompt` | `TOOL_TEXT_RECOVERY_PROMPT` | Override the tool-call-as-text recovery prompt | | `thinkingToolRecoveryPrompt` | `THINKING_TOOL_RECOVERY_PROMPT` | Override the thinking-tool recovery prompt | -| `doneWithoutWorkPrompt` | `DONE_WITHOUT_WORK_PROMPT` | Override the done-claim-with-open-todos prompt | +| `doneWithoutWorkPrompt` | `DONE_WITHOUT_WORK_PROMPT` | Override the done-claim-with-open-todos prompt. Needs a todo tool writing `todos/` into storage; without one it never fires | | `doneWithoutDetailsPrompt` | `DONE_WITHOUT_DETAILS_PROMPT` | Override the done-claim-with-no-todos report prompt | | `doneClaimPatterns` | `DONE_CLAIM_PATTERNS` | Array of regex strings overriding the default done-claim detection patterns (case-insensitive, multiline). Invalid regexes are skipped. Empty array falls back to defaults. | | `readyToContinuePatterns` | `READY_TO_CONTINUE_PATTERNS` | Array of regex strings overriding the default ready-to-continue detection patterns (case-insensitive). Invalid regexes are skipped. Empty array falls back to defaults. | -| `silentDeadStreamMinTokens` | `200` | Min output tokens to treat a textless `finish:"unknown"` message as a dead stream | +| `silentDeadStreamMinTokens` | `200` | Min output tokens to treat a textless, tool-less finished message as a dead stream. Finished turns carrying tool calls are working, not silent, and tools in flight veto the inject regardless | | `busyStallStrategy` | `"continue"` | Busy-stall response: `"continue"`, `"abort"` (abort-first), or `"off"` (disabled) | -| `contextSaturationThreshold` | `0.85` | Ratio of used/usable context that routes a saturated parent to magic-context `ctx-wrapup` (only when magic-context is installed) | -| `activeUserWindowMs` | `300000` | Inbound-user-message recency window (5 min) during which idle nudges stand down (user likely composing) | +| `contextSaturationThreshold` | `0.85` | Ratio of used/usable context that routes a saturated session to reclamation: a parent to magic-context `ctx-wrapup` (only when magic-context is installed), a subagent to native compaction (only when `subagentNativeCompactionEnabled`) | +| `activeUserWindowMs` | `300000` | Inbound-user-message recency window during which idle nudges stand down (user likely composing) | | `stopPatterns` | `[]` | Array of regex strings (multiline). When the newest assistant message's text matches one, the session counts as awaiting the user and idle nudges stand down — e.g. `["STOP:\\s*(BLOCKED\|NEEDS_DECISION)\\b"]`. Invalid regexes are skipped. | | `subagentNativeCompactionEnabled` | `false` | Opt-in native `session.summarize()` for saturated subagent sessions (no magic-context detection required) | +| `injectIntervalMs` | v2 only | Minimum gap between recovery injections for one session. No v1 equivalent | +| `visibleContinue` | `true`, v2 only | Send stall continue via `session.prompt()` — a real user message visible in session history (cattleprod-style) — instead of hidden `session.synthetic({resume:true})`. Set `false` for the old channel. Explicit option wins, else `AUTO_RESUME_VISIBLE_CONTINUE` (`0`/`false` = hidden) wins over the default. (Options must sit under the plural `plugins` key — entries under singular `plugin` load but their options are never delivered.) | +| `richContinuePrompt` | `true`, v2 only | Stall continue names the stall reason, attempt count, and remaining todos instead of bare `"continue"`. A custom `continuePrompt` always wins verbatim | +| `logFile` | v2 only | Where this build appends its log. v2 removed v1's server log endpoint, so without this the plugin is silent. Defaults to `~/.local/state/opencode-v2/auto-resume.log` | +| `configProbe` | v2 only, inert | Delivery probe: echoed as `probe=` in the `ready` startup line and read nowhere else. Bump the token to prove `plugins[].options` reaches the plugin without touching a live knob | +| `rateLimitCooldownsMs` | v2 only | Per-attempt cooldowns (ms) before a rate-limited session may be retried, evaluated as gates on each failure and watchdog tick (no armed timers to lose on reload). Default `[900000, 1800000, 3600000, 7200000 ×5]` ≈ 12h coverage; past the ladder the session stays silent until a genuine user turn. Quota hits never consume the normal retry budget | +| `sendTimeoutMs` | `30000` | Bound (ms) on any single recovery send (prompt/synthetic); expiry flows into the normal throw path instead of stacking wedged sends | +| `stuckToolAgeMs` | `60000`, v2 only | A tool call with no result older than this reads as stuck (idle path nudges immediately instead of waiting out the stall window) | +| `neverInterruptTools` | `["compress"]`, v2 only | Tool names the busy-stall path never interrupts while fresh (case-insensitive); older than 30min they count as wedged like any other tool | +| `shareWatchState` | `false`, v2 only | Share the session watch map across setup re-runs in one process, so stall memory survives reloads | +| `registryScope` | unset, v2 only | Selects the process-global registry slot; test-only — production uses the pid-wide slot so reloads supersede cleanly | +| `quietCorrections` | `true`, v2 only | Feature-correction nudges (unknown-tool, stuck-tool, pattern recoveries) go through the invisible synthetic channel instead of a visible prompt; stall recovery always stays visible | Message patterns are matched case-insensitively. Error names use exact match. @@ -435,7 +698,7 @@ Message patterns are matched case-insensitively. Error names use exact match. | `ABORT_CONTINUE_DELAY_MS` | `2000` | Delay between abort and continue | | `MAX_IDLE_SESSIONS` | `50` | Idle session map cap before cleanup | | `IDLE_CLEANUP_MS` | `600000` | Idle session age before cleanup (10 min) | -| `SESSION_DISCOVERY_INTERVAL_MS` | `60000` | `session.list()` poll interval (60s) | +| `SESSION_DISCOVERY_INTERVAL_MS` | `60000` | Session discovery sweep interval (60s). Same on both builds; v2 reads the busy set from `session.active()` rather than a `status` field per row | ## Verification @@ -452,6 +715,7 @@ The plugin handles all recovery automatically — no manual intervention needed. | Too slow to react | Decrease `checkIntervalMs` to `2000` | | Orphan parent not detected | Increase `subagentWaitMs` to `20000` | | Hallucination loop not caught | Decrease `loopMaxContinues` to `2` | -| Tool-text not detected | Check server logs — requires SDK message fetching | +| Model keeps saying "done", nudged forever | Decrease `lifetimeMaxContinues` (e.g. `5`); raise it for long unattended runs | +| Tool-text not detected | Check server logs — requires SDK message fetching (v1). On v2 the history comes from the client message surface or loopback HTTP — check the plugin log file instead | | Long-running tool killed | Should not happen — active-tool guard prevents it. Report a bug. | | `Unexpected server error` on startup | Update to v1.1.8+. Caused by non-Plugin exports being treated as plugin entrypoints by OpenCode's loader. Clear the plugin cache (`~/.cache/opencode/packages/opencode-auto-resume@*`) and restart. | diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..6515c4f --- /dev/null +++ b/bun.lock @@ -0,0 +1,46 @@ +{ + "lockfileVersion": 2, + "configVersion": 0, + "workspaces": { + "": { + "name": "opencode-auto-resume", + "dependencies": { + "@opencode-ai/plugin": "latest", + }, + "devDependencies": { + "@opencode-ai/sdk": "latest", + "@types/bun": "latest", + "typescript": "latest", + }, + }, + }, + "packages": { + "@opencode-ai/plugin": ["@opencode-ai/plugin@1.4.0", "", { "dependencies": { "@opencode-ai/sdk": "1.4.0", "zod": "4.1.8" }, "peerDependencies": { "@opentui/core": ">=0.1.97", "@opentui/solid": ">=0.1.97" }, "optionalPeers": ["@opentui/core", "@opentui/solid"] }, "sha512-VFIff6LHp/RVaJdrK3EQ1ijx0K1tV5i1DY5YJ+pRqwC6trunPHbvqSN0GHSTZX39RdnSc+XuzCTZQCy1W2qNOg=="], + + "@opencode-ai/sdk": ["@opencode-ai/sdk@1.4.0", "", { "dependencies": { "cross-spawn": "7.0.6" } }, "sha512-mfa3MzhqNM+Az4bgPDDXL3NdG+aYOHClXmT6/4qLxf2ulyfPpMNHqb9Dfmo4D8UfmrDsPuJHmbune73/nUQnuw=="], + + "@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="], + + "@types/node": ["@types/node@25.5.2", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-tO4ZIRKNC+MDWV4qKVZe3Ql/woTnmHDr5JD8UI5hn2pwBrHEwOEMZK7WlNb5RKB6EoJ02gwmQS9OrjuFnZYdpg=="], + + "bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="], + + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + + "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + + "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], + + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], + + "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], + + "typescript": ["typescript@6.0.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-bGdAIrZ0wiGDo5l8c++HWtbaNCWTS4UTv7RaTH/ThVIgjkveJt83m74bBHMJkuCbslY8ixgLBVZJIOiQlQTjfQ=="], + + "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], + + "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + + "zod": ["zod@4.1.8", "", {}, "sha512-5R1P+WwQqmmMIEACyzSvo4JXHY5WiAFHRMg+zBZKgKS+Q1viRa0C1hmUKtHltoIFKtIdki3pRxkmpP74jnNYHQ=="], + } +} diff --git a/docs/AGENTS-snippet.md b/docs/AGENTS-snippet.md new file mode 100644 index 0000000..097daaf --- /dev/null +++ b/docs/AGENTS-snippet.md @@ -0,0 +1,30 @@ + + +## Task completion + +A) When all requested work is finished: if todo tools are available, mark +every todo complete first — unfinished items will block the completion +signal and you'll be asked to continue them instead. Then call the +`task_complete` tool if one exists; otherwise end your reply with TASK +COMPLETE on its own last line (a trailing 🎉 is also accepted). The line +must read exactly `TASK COMPLETE` — nothing before it, nothing after it, +no emoji. (Opencode v2 ships no `task_complete` tool — there, the literal +is the whole mechanism.) Call it once — repeated completion calls mean +something is still open, so go do that instead. Do not end with a question +unless you actually need the user. Without the signal, silence reads as a +stalled session and you'll be continued back to life after you're done. + +## Todos (only if you use a todo plugin) + +If todo tools are available, keep the list current as you work — mark items +complete when done, not at the end. Recovery tooling reads the live list; a +stale list gets you reminded about finished work. + +## Questions + +End with a question only when you actually need the user. An unanswered +question reads as still-working to anything watching the session. diff --git a/docs/known-issues-v2.md b/docs/known-issues-v2.md new file mode 100644 index 0000000..1ae36fd --- /dev/null +++ b/docs/known-issues-v2.md @@ -0,0 +1,487 @@ +# Known issues — v2 port + +Status of the v1 → v2 port as of the current branch. This file is the +authoritative list; the code refers back to it by name from the startup path, so +a config key that this build ignores is never silent. + +## Options that are accepted but not applied + +**None.** Every option this build recognises is now applied, and the startup line +no longer carries an `accepted-but-inert=…` note. + +The machinery is kept rather than deleted. It is the only signal a user gets that +an option is being ignored, and the next unported feature needs it more than this +build does. `docs/known-issues-v2.md` and the README inert list are checked +against `FEATURE_GATED_OPTIONS` in the source, and a test asserts the list is +empty — so adding an option to the recognised set without implementing it cannot +pass quietly. + +Two gaps closed on this branch, in order: `subagentWaitMs` and +`discoveryDelayMs` (orphan-parent recovery and session discovery), then +`toolTextCheckDelayMs` (below). + +`doneWithoutDetailsPrompt` and `doneWithoutWorkPrompt` **are** both applied on +v2, and they are the two halves of v1's done-claim handling: the first asks for a +work report when there is no list to check, the second when the list still has +items open. + +## Unrecognised options warn once + +Any key in the plugin's config that is not in the recognised set produces a +single `warn` line at startup: + +``` +[auto-resume] ignoring unrecognised option(s): foo, bar — this v2 build does not read them. +``` + +This is a v2 addition. v1 silently ignored unknown keys, which is how a typo or +a dropped port went unnoticed. + +## Counter semantics changed from v1 + +`targetedRecovery` now counts an attempt only after the prompt is actually +delivered. v1 incremented first and lost the budget when a send was rejected; +the fix is mte090's `e1b8374`, applied to the v2 code path. The stall path +(`recover`) keeps v1's ordering — it escalates on a stall, not on a send. + +Lifetime cap: `lifetimeMaxContinues` (default 9) bounds total stall continues +per genuine user episode. The `maxRetries` ladder restarts on model text, so a +model that keeps answering "everything is done" reads 1/3 forever — the +lifetime count keeps climbing through that chatter and stands the session down +(`gaveUp`) at the cap. It resets only on a genuine user message or genuine +turn, never on text growth, progress, or abort+resume. + +## Renamed and aliased + +- `maxRetries` is the v2 name. `maxRecoveryRetries` is v1's name for the same + knob and is still accepted as a fallback, so a v1 config ports unchanged. +- `activeUserWindowMs` defaults to `300000` (5 min) in the v2 build, matching + upstream `e1b8374`. The v2 port shipped with `900000`, which stood down for + three times longer than intended after any user message. The v1 file on this + branch still reads `900000` because it predates `e1b8374`; the PR does not + touch v1, and master already carries the fix. + + Lowering the default was not enough on its own: the v2 implementation asked + "is the newest message a user message?" rather than v1's "was any inbound user + message inside the window?". At idle time the newest message is the assistant + turn that just finished, so the window never applied and the nudge fired + straight over a user who was mid-conversation. The check now walks back for + the most recent user message, which is what the option describes. + +## v2-only options + +- `injectIntervalMs` — minimum gap between recovery injections for one session. + No v1 equivalent. +- `lifetimeMaxContinues` (default 9) — lifetime stall continues per genuine user + episode; see "Counter semantics changed from v1". +- `logFile` — where this build writes its log. See "Logging" below. + +## Logging: there is no server log sink in v2 + +v1 logged through `ctx.client.app.log({ body: { service, level, message } })`, +which landed in the opencode log file. v2 removed that endpoint: `ctx.app` is +`{ name, version, channel }` (`packages/plugin/src/app.ts`), there is no `app` +group under `packages/protocol/src/groups/`, and a hosted plugin's +`console.log` is not captured by the OpenChamber process. + +A build that only writes to the console is therefore **silent** — a running +watchdog and a dead one look identical, and there is no way to tell from +outside whether a stall was seen, skipped, or recovered. This v2 build appends +to a file instead: + +``` +~/.local/state/opencode-v2/auto-resume.log # default +``` + +Override with the `logFile` option or the `AUTO_RESUME_LOG_FILE` environment +variable; the option wins. The file is truncated once it passes 2 MB, and every +write is best-effort — an unwritable log directory never breaks the watchdog. + +Each line is `ISO-8601 LEVEL [auto-resume] message`. The startup line lists the +effective timings, so a build that loaded at all is provable from the file: + +``` +2026-10-01T18:02:28.412Z INFO [auto-resume] ready (opencode v2). timeout=180000ms interval=5000ms ... +``` + +## Session discovery + +v1 swept `session.list()` on an interval and trusted a `status` field per row. +v2 has something better: `session.active()` is the server's own record of what +is running, so "busy" is read rather than guessed. The sweep runs once at +`discoveryDelayMs` (default 5s) and then every 60s, and it: + +- seeds a watch for any session that exists, so cleanup and revert handling know + about sessions that predate the plugin load, and +- marks any session the server reports as running as busy, so a turn that was + already mid-flight when the plugin attached starts its stall clock now rather + than never. + +Both calls are read defensively — off the plugin `session` domain first, then +off the raw client — because v2 hands plugins a narrowed `Pick` that omits both. +A host that supplies neither degrades to the event-derived busy set rather than +failing. + +## Premature stop — the case the stall watchdog cannot see + +A session can end its turn cleanly while the work is not done: no stall, no +error, no streaming failure, just a short "Task done." and then idle. The stall +watchdog only looks at *busy* sessions, so this is invisible to it by +construction. Two detectors run on the idle path instead, both ported from v1: + +- **A done-claim with no work report in it.** v2 used to gate the details prompt + on a 400-character threshold, which cannot tell "Task done." from a real + two-line summary — it both over-nudged terse reports and let short real ones + pass. It now uses v1's `containsWorkDescription`: a backticked or bare path + with a dotted extension, or a report header (`Changed`, `Verification`, + `Tests run`, `Results`, `Commands run`). Asking again after a genuine report + loops forever, which is issue #26. +- **A trailing 🎉.** That is the model's own "I finished" signal, so the plugin + latches completion and stops nudging rather than talking over a deliberate + stop. The latch is per-turn: a new turn re-opens the question, and a later + idle with no new text does not re-derive it forever. + +v1 cross-checks both against tracked todo state before latching. v2 now does too — +see "The todo list" below. + +## The todo list + +v1 tracked a session's todos from a `todo.updated` event, falling back to a server +API. v2 has neither: the `todo` table is created in the v2 database (migration +`20260127222353_familiar_lady_ursula.ts`) but no route reaches it and nothing +emits an event for it, so there is no way to observe the list changing. + +What v2 does have is the **session message log**: every `todowrite` call is +persisted as a tool part whose input carries the whole list, so the newest +completed call *is* the current list. auto-resume reads that, instead of owning a +list of its own. + +`ctx.storage` looks like the obvious store and is not one. It is namespaced per +plugin: a key resolves to `storage/plugin//.json`, and the route is +`/api/plugin/storage//`. A todo plugin writing +`todos/` therefore lands in its own directory, so this plugin reading +the same key is reading a key nothing wrote — regardless of who writes what. A +storage-first reader concludes "no todos" while the list sits in the message log, +and fires false done-claim nudges at a session that still has work listed. + +The consequence worth stating: auto-resume is a **consumer**, not a second owner. +It works with whichever todo tool is installed rather than requiring its own. +Confirmed against `opencode-todo-fork`; other todo plugins are unconfirmed. +There is no copy to drift. It calls `get` on storage only after the message log has +come up empty. With no `ctx.storage` at all, or with a record it cannot parse, it +falls back to "no list" — which means the older behaviour (latch on the emoji, ask +for details on a bare done-claim), never a nudge on the strength of a list it failed +to read. + +Three places consume it: + +- **The 🎉 cross-check.** The emoji alone is not trustworthy: a model that + finishes early celebrates early, and latching on that turns a false positive + into permanent silence. With items still open, the celebration is a false + positive and the reminder names what is unfinished. This branch deliberately + does **not** latch, so the next turn is free to judge again. +- **`doneWithoutWorkPrompt`.** A done-claim with open todos. Distinct from + `doneWithoutDetailsPrompt`, and on a separate budget — two different problems, + so spending one must not silence the other. +- **`todoCheckAttempts`.** The last v1 use of the list that is not here: a turn + that says "ready to continue" while every todo is already closed gets two + chances before a plain `continue`. + +One fix came with the list. The done-claim budgets no longer reset on every busy +cycle — only on a genuinely new inbound user message. v1 moved both out of the +busy reset for #26: a model that re-announces completion each turn was otherwise +handed a fresh budget each time it announced, so the nudge never stopped. v2 +still had that, and now does not. The open-todos nudge is the opposite case and +does reset per cycle, because an open list is new information each turn. + +## The settle delay before a turn is judged + +v1 never judges a finished turn on the `session.idle` event. It arms a timer for +`toolTextCheckDelayMs` and runs the done/tool checks from there, and the option +is the delay. That is not incidental: `session.idle` can arrive while the +assistant's closing text is still being written into the message history, so a +check that reads too early sees a half-finished turn — missing the pattern it +should have caught, or judging a turn that was not over. + +The v2 port judged on the idle event for most of its life, on the strength of the +live delta buffer: the plugin sees text as it streams, so it usually has the final +turn before `session.idle` arrives, which is more reliable than waiting. It does +not remove the race. The buffer is empty when the plugin loads mid-turn, and the +fallback in that case is precisely the history that may not have flushed. + +So the work is split in two, and the option tunes the half that needs it: + +| Pass | When | What it decides | +|---|---|---| +| structural | on `session.idle` | dead stream, user hand-off, user recently active | +| pattern | `+` `toolTextCheckDelayMs` | celebration, tool-call-as-text, ready-to-continue, action intent, done-claim | + +The structural pass is immediate on purpose. A stream that died before delivering +any text is exactly the case where every pattern check has nothing to look at, so +waiting would only delay a recovery that is already certain. + +The pattern pass re-reads the history rather than reusing the first read, which +is the whole point: text that landed after the idle event is what it judges. It +also **re-runs the structural guards** instead of trusting the first pass's +verdict, because the wait is long enough for the user to have replied — and a +synthetic nudge sent over a real reply is the "Step interrupted" bug those guards +exist to prevent. + +Three lifecycle details, all covered by tests: + +- A **new turn cancels** a pending pass. `markBusy` clears the timer, and the + timer itself re-checks that the session is still idle: a queued callback can + outlive the turn that armed it. +- A **second idle replaces** the first rather than stacking a second judgement on + the same text. Two passes on one turn would spend two attempts of one budget on + one piece of text, and the second would be a nudge about a nudge. +- **Stopping the plugin drops** any pending pass. Otherwise a reload leaves a + timer pointing at a session this build is no longer watching, and it fires + minutes later against a history nobody is reading. + +### One difference worth naming + +v1's `checkForToolCallAsText` bundles every text-based detector behind this one +timer. v2's detectors were separate before the option was honoured, so the delay +applies to the pattern pass as a whole rather than to the tool-call-as-text check +specifically. Nothing is lost by it — every detector that reads the closing text +gets the settle window, which is what the delay was for. + +## Orphan parent recovery + +v1 detects the busy-count drop from more than one session to exactly one and +treats the survivor as a parent whose subagent died. That is the right +*signal* and the wrong *conclusion* on a busy box, and v2 can do better because +v2 records the parent link. + +**The port asks whether the survivor actually has children before arming.** One +`session.list({ parentID })` call at arm time turns a guess into a fact. v1 has no +way to ask — it treats *any other busy session* as a subagent of the one in hand +— so on a box with two conversations open it aborts one of them when the other +finishes. The `parentID` filter is requested *and* the returned rows are checked +against it client-side, because the only consequence of the filter being quietly +ignored would be aborting real sessions. + +**A child is judged on what it last did, not on whether the server still lists +it.** v1's `checkSubagentStatus` scans the global status map for a *busy* +session and reports "idle" when it finds none. That reading cannot work here: +this function is only ever reached *after* a child went idle, so a child being +absent from the active set is the premise rather than the finding. Read that way, +every crashed subagent looks healthy, the child is never woken, and the parent is +aborted on the first tick. So the three cases are: + +- **recent** — whatever the server thinks, the child said something inside the + window, so believe it. If it is also active, keep waiting. +- **silent past the threshold, tool call outstanding** — a long build, not a + dead model. The window is tripled, and the parent is left alone. Killing the + parent there loses the work. +- **silent past the threshold, or an errored message** — it stopped without + reporting. Worth one attempt to wake it. + +**Waking the child is tried once per episode.** v1's structure assumes the subagent +goes busy again after the prompt; a dead one never does, so an unbudgeted +retry sends the same nudge on every watchdog tick. The sequence is: nudge once, +wait a further `subagentWaitMs`, and abort the parent only if the child is still +dead. v1 sends the prompt through the client route; v2 uses the plugin-scoped +`synthetic` endpoint, which takes an explicit `sessionID` and so needs no +cross-session route the plugin API does not expose. + +**In-flight tools are tracked from the tool lifecycle events** — `tool.called` +increments, `tool.success` and `tool.failed` decrement — rather than from a +`tool.execute.before` hook, because v2 emits all three and the hooks would be a +second source of truth for the same fact. Both terminal events matter: a *failed* +tool is finished work, and holding the slot on failure would make the watch defer +on every session that has ever seen an error. A session that goes idle has its +slots cleared, since a slot still held there belongs to a tool that will never +report. + +### What still bounds the risk + +`subagentWaitMs` is the guard. Between a child finishing and the parent being +aborted, the parent is given this long to consume the child's result — the +default is 15s. v1's logic aborts as soon as no subagent is active, and so does +this; the option is what makes that survivable, and its name is the warning. Set +it small and the watch becomes a blunt instrument. Lower it only if you have +watched a parent wait minutes on a dead child. + +Two further guards, both checked before anything is interrupted: a parent with a +tool in flight is never aborted, and neither is one whose newest message is +waiting on the user. + +A give-up path bounds the whole thing: after `maxRetries` episodes the watch +disarms and logs it, rather than retrying a parent that has already survived +several aborts. + + +## Unknown tool names + +v1 asks `ctx.client.tool.ids()` for the registered tool names. v2 has no such +route, so the port uses `ctx.tool.list()`, which returns the same set of +definitions. Only the effective names are needed — nothing else in the record is +read — and the effective name is the namespaced form when a tool sits in a +namespace, which is exactly the string a model would have to call. That is what +the suggestion quotes back. + +The tool parts come from the message history, not from events, and that is not a +workaround: the thing being looked for is a part that already **errored**, and by +the time the error lands the `session.tool.called` event for that part is long +gone. This code only runs on idle anyway. + +Two shape differences from v1, both of which would have made the check inert +rather than wrong — worth naming because an inert detector is the failure mode +that does not announce itself: + +- v1 reads the tool name from `part.tool`. v2 has no such field; the name is in + `part.name`. A port that kept v1's read would compare against `""`, never + match, and silently never fire. +- v1 keys the "already examined" set on `part.id ?? part.callID`. v2's tool part + carries `id` (the call id) and `name`, and no `callID`. + +v2 tool-state statuses are `streaming`, `running`, `completed`, `error`, so the +`state.status === "error"` test v1 uses is correct on v2 unchanged. (The +`"pending"` status v1 also tests, elsewhere, does not exist in v2 — see the +TOOL_STATE_* note in the source.) + +Two behaviours are inherited from v1 rather than fixed here, and the second is +worth a decision: + +- **A re-armed turn names the newest typo.** The per-request reset clears + counts and the latch but keeps the "already examined" set: already-reported + errors are never recounted (2026-10-04 changed this — recounting made every + genuine user message re-suggest the same stale errors). A turn that + introduces a *new* invented name is told about that one. +- **No registry, no check.** With no tool list every name looks invented, so the + check is skipped rather than accusing the model of tools that plainly exist. + Same for an empty registry and for a registry that throws, which is logged and + ignored: a check that guesses is worse than a check that abstains. + +One ordering fix the port needed. The per-request re-arm is derived from the +message history, so it cannot be applied before the history is read — but the +"already suggested" guard used to run before that read, and the check runs +alongside `inspectOnIdle`, whose ordering is not defined. On a re-armed turn the +guard could therefore see the previous turn's latch still standing and skip the +check that was supposed to re-arm it. The guard now runs after the read. + + +## Explicit completion via `task_complete` + +v1 registers a `task_complete` tool and v2 has no built-in equivalent — nothing in +the v2 source mentions the name — so the plugin provides it through +`ctx.tool.transform`, the v2 tool registry. This is a feature rather than a +compatibility shim: it is the strongest completion signal available, stronger than +a trailing 🎉, because the model chose to call it. + +The ack escalation is the part that matters, and it is v1’s unchanged. The ack +tool-result is fed straight back into the model’s turn, and a stuck model answers +it by calling the tool again rather than ending with text — v1 logged **27 +consecutive acked calls with zero new user input**. So: + +- first call acks, and the ack carries an explicit stop instruction; +- second consecutive call warns, and says further calls will be rejected; +- third and beyond throw, which surfaces as a tool error and forces the turn to end. + +The counter is reset by a genuinely new inbound user message and by any other tool +call, so the escalation measures a stuck loop rather than several legitimate rounds +of work. Resetting it on tool calls rather than only on user messages is the v2 +addition: a model that finishes, reports, does more work and reports again is +doing its job, not looping. + +Two guards worth naming: + +- **The todo gate is skipped for subagents.** A child was never asked to do the + parent’s open items, so gating its report on them would block it on work outside + its scope. Same reasoning as v1. +- **A blocked call also fires a visible nudge** naming the blocking todos, because + the tool result can collapse to an invisible one-liner in the transcript. Skipped + when the user holds the ball, since injecting then starts a step their real reply + interrupts. The block itself is bounded by `maxRetries`: an unbounded block is its + own kind of loop, and a model that keeps saying "done" may know something the todo + list does not. + +Registration failure — no `ctx.tool`, or a registry that throws — is logged and +swallowed. It must not take down the session recovery that the rest of the plugin +exists for, so the watchdog starts either way. + +## Context saturation + +A session can fill its context window without ever looking stalled — it just keeps +working until it chokes. On idle, when used/usable crosses +`contextSaturationThreshold` (default 0.85), routing depends on session kind, +exactly as in v1: + +- **Subagent** — opt-in only. With `subagentNativeCompactionEnabled: true` the + plugin requests native compaction. v1 called `session.summarize()`; v2 spells it + `session.compact`. +- **Parent** — only when magic-context is installed, because its setup disables + native compaction and compacting here would double-compress. The plugin sends + the `ctx-wrapup` command through `session.command`, not as prompt text, since + prompt text is not expanded into a command. + +Both are one-shot per turn, and both are fail-safe: a missing limit, a missing +token count, a user cancellation, or a signalled completion means no intervention +at all. + +Three v2 API shapes are worth recording, because each replaced something v1 had: + +| Need | v1 | v2 | +|---|---|---| +| Tokens in the window | accumulated from `message.updated` | `session.usage.updated`, summing `input + output + reasoning + cache.read + cache.write` — the same five fields v1 added up, matching `TokenUsage.total` | +| The model's window | walk the raw provider list for `limit` | `ctx.model.get(providerID, modelID)` → `Model.Info.limit` | +| Is magic-context installed? | `config.get().plugin` | `ctx.plugin.list()` → `Plugin.Info[]`; v2 removed the `config` domain | + +The usable window is `limit.context - Math.min(20_000, limit.output)` — v1's +arithmetic, kept identical so the same threshold means the same thing on both +builds. + +## Silent dead stream + +A turn can end having produced nothing the user can see: reasoning only, or a +finish reason the provider did not describe. OpenCode records the message as +completed and the session goes idle, so no stall timer expires, no streaming +failure fires, and the stall watchdog — which only looks at *busy* sessions — +never sees it. v1's rule is unchanged on v2: walk back to the newest assistant +message that **has** a finish reason; if it carried no text and generated at least +`silentDeadStreamMinTokens` output tokens, the stream died mid-response. +Two 2026-10-04 amendments narrow "no text": a finished message carrying +**tool-call parts** is a working turn, not a dead stream (thinking models end +tool-working turns with no chatter), and recovery is refused while tool calls +are still in flight — read from a snapshot taken at the idle transition, +because `markIdle` zeroes the live `pendingTools` counter and idle fires +while tools run. + +The walk skips messages with no finish reason on purpose. An intermediate +tool-call step has none, and stopping at it would report a dead stream for every +session that used a tool. + +Two things are worth recording about the v2 API: + +- The judge is the message, not the event stream, so this needs + `session.context()` — every message since the last compaction. The idle + inspection now fetches it **once** and shares it across four checks (dead + stream, text fallback, pending tool call, active user), where v1 fetched once + for the same reason and this port initially fetched three times. +- Before injecting, the plugin asks the server whether the session is running + again (`session.active()`), not only its own event-derived flag. A provider + quietly retrying looks identical from the event stream, and the recovery event + may not have arrived by the time the turn ends. + +## A tool call written into the reasoning block + +When a model writes raw tool-call markup inside its thinking instead of calling +the tool, nothing executes and nothing raises. No part is tagged as a tool call, +so no session-side code runs it; the turn completes normally, the prose may read +fine, and the work silently does not happen. + +v1 caught this on the same pass as the text variant and answered with a different +prompt, because the fix is different — the model is not forgetting the tool +mechanism, it is writing in the wrong channel. v2 separates the two reads instead +of filtering them into one joined string, since `AssistantContent` tags reasoning +and text distinctly: + +- reasoning parts are judged first, because a message can carry both and the + reasoning one is the one that silently does nothing; +- both variants share the `toolTextAttempts` budget, because they are one + phenomenon with two symptoms — two budgets would let the plugin spend twice the + retries on a model that keeps doing it; +- both use the same code-block stripping, so a fenced example of the format or an + inline path in backticks is not mistaken for a real call. diff --git a/docs/v2/backport-to-v1.md b/docs/v2/backport-to-v1.md new file mode 100644 index 0000000..6adc6e9 --- /dev/null +++ b/docs/v2/backport-to-v1.md @@ -0,0 +1,223 @@ +# v2 → v1 auto-resume backport research + +Branch: `auto-resume/v2-todo-message-log` (HEAD a2d3897; v2 changes in 8073ba1 + a2d3897) +Subject: `src/index.ts` (v1 plugin, `@opencode-ai/plugin` v1 hooks style, ~3000+ lines) +Reference: `src/v2/index.ts` (v2 plugin) + +## v1 mechanism today + +### Continue channel (confirmed visible) + +- `sendContinuePrompt(sid, text, w)` at **src/index.ts:837** is the single choke point for every continue. It calls `ctx.client.session.prompt({path:{id}, body:{parts:[{type:'text',text}],agent,model}})` at **914-921** (retry path **933-936**). This is a **visible user message** — confirmed per user directive. +- Agent/model are inferred from the last user message at **869-907** (via `getSessionMessages`, **1067-1080**). +- Guards inside `sendContinuePrompt`: `w.continuing && !watchdogRetryGuard` skip (**838-841**), `userCancelled || completionSignaled` (**842**), `gaveUp` latch (**848-851**), `oocLocked` latch (**852-856**). +- `recordContinue(sid)` at **927** (success) and **938** (retry success); `w.lastRetryAt` set at **928/939**; `finally` block resets `continuing`/`todoCheckAttempts`/clears toolTextTimer at **945-950**. +- Deferred watchdog `setTimeout` at **954-1025**: retry backoff → `sendContinuePrompt(sid, continuePrompt, w)` at **976** → escalate `tryAbortAndResume` → `gaveUp` latch at **1004**. + +### Continue prompt content today (BARE) + +- `continuePrompt` option at **530-531**: `const continuePrompt: string = (options?.continuePrompt as string) ?? "continue"` — **bare text today**. +- `actionIntentPrompt` at **532-533** defaults to `continuePrompt`. +- Other fixed prompts: `toolTextRecoveryPrompt` (**534**), `thinkingToolRecoveryPrompt` (**536**), `doneWithoutWorkPrompt` (**538**), `doneWithoutDetailsPrompt` (**540**), `TOOL_LOOP_RECOVERY_PROMPT` (**130**), `SUBAGENT_RECOVERY_PROMPT` (**1240**). +- `buildOpenTodosReminder(todos)` at **482-491** builds a richer "You have N unfinished tasks..." text (already used by idle-open-todos paths). + +### All `sendContinuePrompt` call sites (all send bare `continuePrompt` or a fixed prompt) + +| Line | Context | Prompt sent | +|---|---|---| +| 976 | deferred watchdog retry (WP-05) | `continuePrompt` (bare) | +| 1138 | unknown-tool suggestion | custom `prompt` (tool name) | +| 1903 | tool-text recovery best-candidate | `bestCandidate.prompt` | +| 1963 | abort+continue | `continuePrompt` (bare) | +| 2016 | `tryResume` generic send | `prompt ?? continuePrompt` | +| 2244 | pending-recovery (WP-04) | `continuePrompt` (bare) | +| 2650 | action-intent (idle) | `actionIntentPrompt` | +| 2732 | action-intent (busy) | `actionIntentPrompt` | +| 2953 | task_complete blocked | `blockMsg` (fixed) | +| 3061 | tool-loop recovery | `TOOL_LOOP_RECOVERY_PROMPT` (fixed) | + +### `tryResume` (the main stall path) + +- Signature at **1984**: `tryResume(sid, w, reason, prompt?)`. +- Backoff check at **1991-1992** (`backoffMs(w.resumeAttempts, ...)` at **447-453**). +- Hallucination-loop check at **1994-2009** (inflight tools → skip; else `tryAbortAndResume`). +- `w.resumeAttempts++` at **2011**; log at **2013**. +- Send at **2016**: `sendContinuePrompt(sid, prompt ?? continuePrompt, w)` — **bare unless caller passes explicit prompt**. +- Callers of `tryResume`: + - **2203**: stream-stall (`"Stream stall"`, no explicit prompt → bare) + - **2302**: periodic idle-open-todos (explicit `reminder` from `buildOpenTodosReminder`) + - **2486-2492**: streaming-failure on idle (explicit `continuePrompt`) + - **2533-2538**: silent-dead-stream (explicit `continuePrompt`) + - **2607-2612**: idle-open-todos celebration false-positive (explicit `reminder`) + - **2613-2616**: idle-open-todos normal (explicit `reminder`) + +### Loop guards (existing) + +- `recordContinue(sid)` at **573-577** (pushes timestamp into `w.continueTimestamps`). +- `isHallucinationLoop(sid)` at **583-588**: `continueTimestamps.length >= loopMaxContinues` (default **3**, option at **508-509**) within `loopWindowMs` (default **600s**, option at **510-511**) → true. +- `backoffMs` at **447-453**: exponential, capped. +- `w.gaveUp` latch: set at **1004** (watchdog exhausted), cleared at **1435**/`resetSessionFlags` and **1463**/`resetBusyFlags` (on new user message). +- `w.oocLocked` at **2876-2880** (OOC error → permanent halt). +- `minActivityGapMs` check at **1882-1886** (skip if session was active < gap ago). +- `todoNudgeAttempts` budget at **1863-1865** and **2602-2616**. +- `maxRetries` (option at **500-501**) and `maxRecoveryRetries` (option at **514-515**). +- No env-var reads in v1 today (all options via 2nd factory arg at **493**). +- `debug` option at **522-523**. + +### Assistant text availability + +- v1 has **no rolling `lastAssistantText`** accumulation (v2 uses `textParts` map maintained in `appendText` at src/v2/index.ts:1669-1677). +- v1 can get assistant text via `getSessionMessages(sid)` at **1067-1080** (SDK `session.messages`). The pattern is shown in `lastAssistantEndsWithCelebration` at **1155-1178** (iterate last assistant msg, concatenate text parts). +- Todo state: `w.todos`, `buildOpenTodosReminder` at **482-491**, `fetchSessionTodos` at **1206-1236**. + +### Startup log + +- Lines **2982-2985**: `if (!initialised) { initialised = true; log("info", \`opencode-auto-resume ready. timeout=${chunkTimeoutMs}ms, orphan=${subagentWaitMs}ms, loop=${loopMaxContinues}x/${loopWindowMs/1000}s\`) }` — no channel/rich/dup info today. + +--- + +## V2 changes under review (reference: src/v2/index.ts) + +1. **`visibleContinue`** + `AUTO_RESUME_VISIBLE_CONTINUE` env (v2 opts **1115-1117**; option doc **282-288**): stall continue via `ctx.session.prompt` (visible) instead of synthetic, fallback to synthetic on throw. **NOT needed for v1** (v1 already visible). +2. **`richContinuePrompt`** default `true` (v2 **1118**; doc **293-296**): `buildStallContinueText` at v2 **1799-1821** — custom `continuePrompt` wins verbatim; else `continue — stalled (${reason}; attempt ${attempt}/${maxRetries}).\n${todoSuffix}` where `todoSuffix` lists up to 5 open todos `• [status] content` or "No open todos recorded." / "Todo list unavailable."; `slice(0,2000)`. Called from `recover()` at v2 **1882**. +3. **Exact-duplicate anti-repeat**: v2 watch fields `lastProdText`/`prodAssistantSnapshot` (**159-162**) + rolling `lastAssistantText` (**172**, maintained **1669-1677**); `injectOnce` gate at v2 **1603-1609**: `checkDuplicate && !selfAbort && lastProdText!=="" && text===lastProdText && pendingTools<=0 && lastAssistantText===prodAssistantSnapshot` → soft skip (dbg only). On success updates at v2 **1616-1617**. `recover()` passes `checkDuplicate=true`. +4. **Startup log line** at v2 **3786-3791**: includes `visibleContinue=${visibleContinue}` (+ accepted-but-inert list). + +V2 option-reading style: `const opts = (ctx.options ?? {}) as AutoResumeOptions` at v2 **1100** (v1 uses 2nd factory arg — parity = `options` arg). + +--- + +## Per-item verdicts (ranked by value) + +### Item 2: `richContinuePrompt` — **YES, backport** (highest value) + +**Why:** v1 sends bare `"continue"` from many stall sites. A rich prompt (reason + attempt + open todos) dramatically improves model compliance on stall recovery, matching v2 behavior. + +**Insertion point (v1):** +- New function `buildStallContinueText(sid, reason, attempt)` — insert near `buildOpenTodosReminder` at **~492** (after line 491). Reuses `fetchSessionTodos` (**1206-1236**) for todo suffix. Logic mirrors v2 **1799-1821**. +- New option at **~531** (after `continuePrompt`): `const richContinuePrompt: boolean = (options?.richContinuePrompt as boolean) ?? true`. +- **Call sites to change** (stall path only, scope per user direction): + - `tryResume` at **2016**: `sendContinuePrompt(sid, prompt ?? await buildStallContinueText(sid, reason, w.resumeAttempts), w)` — but only when `prompt` is undefined (caller didn't pass explicit). Note: `tryResume` is async so `await` is fine. + - Line **1963** (abort+continue): `sendContinuePrompt(sid, await buildStallContinueText(sid, "abort+resume", w.resumeAttempts), w)` + - Line **2244** (pending-recovery): `sendContinuePrompt(sid, await buildStallContinueText(sid, w.pendingRecoveryReason ?? "recovery", w.recoveryAttempts), w)` + - Line **976** (watchdog retry): `sendContinuePrompt(sid, await buildStallContinueText(sid, "recovery", w.recoveryAttempts), w)` + - Lines **2486-2492** (streaming-failure) and **2533-2538** (silent-dead-stream): these pass `continuePrompt` explicitly to `tryResume`; change to pass `undefined` so `tryResume` builds the rich text. + - **Do NOT change**: 1138 (unknown-tool, custom prompt), 1903 (tool-text recovery, bestCandidate), 2650/2732 (actionIntent), 2953 (task_complete block), 3061 (TOOL_LOOP_RECOVERY_PROMPT) — these have their own per-kind prompts/budgets. +- **Conflicts:** None with existing guards. `tryResume` is already async. The `prompt` param at **1984** is optional; callers passing explicit prompts (idle-todos at 2302/2607/2611) are unaffected. `minActivityGapMs` check at **1882-1886** is in the tool-text path, not `tryResume`. + +**Test file:** `src/index.continue.test.ts` (548 lines, best fit — already tests continue prompt sending). Add tests: rich prompt includes reason+attempt+todo suffix; custom `continuePrompt` wins verbatim; `richContinuePrompt: false` reverts to bare. + +--- + +### Item 3: Exact-duplicate anti-repeat — **YES, backport** (medium value, prevents redundant messages) + +**Why:** Prevents sending the same continue text when the model made zero progress (no new assistant text, no tools in flight). Soft skip only; loop window counter stays the backstop. + +**Insertion point (v1):** +- New `SessionWatch` fields at end of interface (line 68, after `checkedToolPartIDs` at line 67): `lastProdText: string`, `prodAssistantSnapshot: string`, `lastAssistantText: string`. +- New function `snapshotAssistantText(sid)` — mirrors `lastAssistantEndsWithCelebration` pattern at **1155-1178**: `getSessionMessages(sid)`, find last assistant msg, concatenate text parts, return trimmed string. Insert near **~1180**. +- Gate inside `sendContinuePrompt` at **~855** (after `oocLocked` check, before `w.continuing = true`): + ``` + if (w.lastProdText !== "" && text === w.lastProdText && w.pendingTools <= 0) { + const snap = await snapshotAssistantText(sid) + if (snap === w.prodAssistantSnapshot) { + dbg(`${short(sid)} duplicate continue suppressed`) + return + } + } + ``` +- On successful send (after **927** `recordContinue`): update `w.lastProdText = text; w.prodAssistantSnapshot = await snapshotAssistantText(sid)`. +- **Cost note:** `snapshotAssistantText` calls `getSessionMessages` (SDK round-trip). `sendContinuePrompt` already calls `getSessionMessages` at **869** for agent/model inference — **reuse that fetch**: extract the last assistant text from the same `msgs` array (pattern at **1155-1178**) instead of a second call. +- **Conflicts:** None. `w.pendingTools` already exists on `SessionWatch` (**line 56**). The gate is a soft skip (no state change beyond dbg), so it cannot interfere with `gaveUp`/`oocLocked`/backoff logic. The `w.continuing` guard at **838** is checked before the dup gate. + +**Test file:** `src/index.watchdog.test.ts` (549 lines, tests the stall/retry path). Add: duplicate continue suppressed when no progress; NOT suppressed when assistant text grew; NOT suppressed when tools in flight. + +--- + +### Item 4: Startup log line — **YES, trivial** (low effort, nice-to-have) + +**Why:** Reports resolved channel + rich/dup settings, matching v2 diagnostic parity. + +**Insertion point (v1):** Line **2984** — append to the existing `log("info", ...)` string: +- ` visible=${true}` (v1 always visible; informational) +- ` rich=${richContinuePrompt}` +- ` dupGuard=${true}` (always on once backported) + +No new option needed — just report the resolved values. + +**Test file:** `src/index.plugin.test.ts` (655 lines, "Plugin Lifecycle" section at line 28). Add: startup log contains `rich=true` / `rich=false` depending on option. + +--- + +### New item: `silentContinue` opt-out — **PARTIAL / DEFER** (parity stub only) + +**User direction:** v1 continues are already visible; v1 stays visible-by-default with an opt-out to silence. + +**Problem:** v1 has **only one channel** — `client.session.prompt` = visible. There is no hidden/synthetic channel in v1 (v2 has `ctx.session.hook("message", ...)` for silent). A true "silent continue" in v1 would require either (a) a no-op + log-only (defeats the purpose) or (b) a new hidden mechanism that doesn't exist in the v1 plugin API. + +**Recommendation:** Accept the option name for config parity (`silentContinue?: boolean`, default `false`), but **implement as a no-op with a warn log** if set to `true`: `log("warn", "silentContinue=true requested but v1 has no hidden channel; continues remain visible")`. This documents intent, prevents config-porting confusion, and is honest about the limitation. **Assumption:** user accepts this limitation; if true silence is needed, it requires a v1 API extension beyond this backport. + +**Insertion point:** Option at **~531** (alongside `richContinuePrompt`); warn-log at **~2984** (startup) or in `sendContinuePrompt` at **~840** (once, via `initialised`-style flag). + +**Test file:** `src/index.plugin.test.ts` — assert warn log emitted when `silentContinue: true`. + +--- + +## Suggested option names + defaults (v1 parity) + +| Option | Default | v2 equivalent | Notes | +|---|---|---|---| +| `richContinuePrompt` | `true` | v2 `richContinuePrompt` (1118) | Stall-path rich text; custom `continuePrompt` wins verbatim | +| `silentContinue` | `false` | v2 `visibleContinue` (inverted) | No-op + warn in v1 (no hidden channel); parity stub | +| `continuePrompt` | `"continue"` | v2 `continuePrompt` | Existing, unchanged; wins verbatim over rich text | + +No env-var reads in v1 (all via `options` 2nd factory arg at line 493). V1 has no `AUTO_RESUME_*` env today. If env parity is desired, add `process.env.AUTO_RESUME_RICH_CONTINUE_PROMPT` fallback — but this deviates from v1's current style (zero env reads); **recommend options-only for v1**. + +--- + +## Test plan + +All tests use `bun:test` (existing pattern in `src/index.*.test.ts`). + +### Item 2 (rich prompt) → extend `src/index.continue.test.ts` +1. **Rich text includes reason + attempt + todo suffix:** trigger a stall (`tryResume`), assert `promptCalls[0].body` contains `"stalled"` + `"attempt 1/3"` + open todo content. +2. **Custom `continuePrompt` wins verbatim:** set `options.continuePrompt = "keep going"`, assert prompt body === `"keep going"` (not rich). +3. **`richContinuePrompt: false` reverts to bare:** set `options.richContinuePrompt = false`, assert prompt body === `"continue"`. +4. **Todo suffix capped at 5 + "…and N more":** 7 open todos → suffix has 5 bullets + `• …and 2 more`. +5. **No open todos → "No open todos recorded."** +6. **Todo fetch fails → "Todo list unavailable."** (mock `fetchSessionTodos` to throw). +7. **Rich text sliced to 2000 chars.** + +### Item 3 (anti-dup) → extend `src/index.watchdog.test.ts` +1. **Duplicate suppressed:** send continue, no assistant-text growth, no tools → second identical continue skipped (no new `promptCalls`). +2. **NOT suppressed when text grew:** assistant text changed since last prod → second continue sent. +3. **NOT suppressed when tools in flight:** `w.pendingTools > 0` → second continue sent. +4. **First send always goes through:** `lastProdText === ""` → no gate. +5. **Gate does not affect `gaveUp`/`oocLocked`/backoff:** those checks still fire independently. + +### Item 4 (startup log) → extend `src/index.plugin.test.ts` +1. **Default:** startup log contains `rich=true`, `dupGuard=true`. +2. **`richContinuePrompt: false`:** startup log contains `rich=false`. +3. **`silentContinue: true`:** startup log contains `silentContinue=true` + warn. + +### New: `silentContinue` → extend `src/index.plugin.test.ts` +1. **`silentContinue: true`:** warn log "no hidden channel" emitted once at startup. +2. **`silentContinue: false` (default):** no warn. + +### Regression +- Run existing `src/index.continue.test.ts`, `src/index.watchdog.test.ts`, `src/index.plugin.test.ts` suites unchanged. +- Run `src/index.pending-recovery.test.ts`, `src/index.silent-dead-stream.test.ts`, `src/index.streaming-failure.test.ts` (stall-path callers affected by item 2). + +--- + +## Summary + +| Item | Verdict | Effort | Value | +|---|---|---|---| +| 1. `visibleContinue` | **NO** (v1 already visible) | — | — | +| 2. `richContinuePrompt` | **YES** | Medium (~80 lines) | High | +| 3. Anti-dup gate | **YES** | Medium (~50 lines) | Medium | +| 4. Startup log | **YES** | Trivial (~5 lines) | Low | +| New: `silentContinue` | **PARTIAL** (no-op stub) | Trivial (~10 lines) | Low (parity) | + +Total estimated effort: ~150 lines of v1 changes + ~150 lines of tests. diff --git a/docs/v2/installing.md b/docs/v2/installing.md new file mode 100644 index 0000000..daea20f --- /dev/null +++ b/docs/v2/installing.md @@ -0,0 +1,162 @@ +# Installing opencode-auto-resume on OpenCode v2 + +This guide installs the **v2 port** of the plugin. It targets the stable v2 +plugin API (`@opencode/plugin` 2.0.5+, shipped with `opencode` v2.0.5; live-verified on 2.0.27 — the 2.0.5→2.0.27 surface diff is additive-only). + +> Coming from OpenCode v1? Read [`migration.md`](./migration.md) first — the v1 +> plugin file does **not** run on v2 and must be replaced by the v2 port. + +## Requirements + +- **opencode v2.0.5 or newer** (`opencode --version`). +- **No runtime dependencies.** `src/v2/index.ts` imports only `node:fs`, + `node:os` and `node:path`, and defines its own `{ id, setup }` helper rather + than importing one, so the built file is self-contained. `@opencode/plugin` is + not needed to *run* the plugin — only to typecheck the source against the + official types (see [Development](#development)). +- **Host access beyond the plugin context.** Reading the todo list needs the + local server's address, and that is not exposed as an environment variable under + OpenChamber. The plugin reads `/proc/self/cmdline` for the `--port` flag (Linux) + and may issue a loopback `GET` to `127.0.0.1`; where `/proc` is unavailable it + falls back to `OPENCODE_SERVER_URL` / `OPENCODE_SERVER_PORT` / + `OPENCODE_PORT` / `PORT`. If `OPENCODE_SERVER_PASSWORD` (or `OPENCODE_PASSWORD`) + is set, it is sent as HTTP Basic on that request. All of it is best-effort: a + host that refuses any of it degrades to "cannot read todos", never a crash. +- The plugin source: [`src/v2/index.ts`](../../src/v2/index.ts) from this repo, or + the built `dist/v2/index.js`. + +## Install + +There is nothing to install first. Copy the file, and opencode loads it. + +### Step 1 — add the plugin + +#### Option A — drop-in file (no config) + +OpenCode auto-loads every plugin found in these directories: + +- Global: `~/.config/opencode/plugins/` +- Per project: `/.opencode/plugins/` + +Copy the v2 port there: + +```sh +mkdir -p ~/.config/opencode/plugins +cp src/v2/index.ts ~/.config/opencode/plugins/auto-resume-v2.ts +``` + +That's it — no `opencode.json` change is required. Restart opencode (or reload +plugins) and the plugin is active. + +#### Option B — `opencode.json(c)` entry + +Use this when you want to load the file from another location, pass options, or +pin a published package. Add an entry to the `plugins` array: + +```jsonc title="~/.config/opencode/opencode.jsonc" +{ + "$schema": "https://opencode.ai/config.json", + "plugins": [ + // 1. plain path (relative to the config file) or absolute path + "./plugins/auto-resume-v2.ts", + + // 2. with options + { + "package": "./plugins/auto-resume-v2.ts", + "options": { + "chunkTimeoutMs": 180000, + "maxRetries": 3 + } + } + ] +} +``` + +Both `.opencode/plugin/` (v1 directory name) and `.opencode/plugins/` are +discovered; use `.opencode/plugins/` for v2 files. + +### Step 2 — restart opencode + +Restart opencode after copying the file. A hot plugin reload is **not** +reliable on every version, so restart when in doubt — a missing banner is the +symptom to check for. + +## Options + +Every option is optional and is read from `ctx.options` in `setup`. The port +reads **all** of v1's options, and applies all of them — there are none left in an +"accepted but not applied" state. The full table with every default is in the +[README](../../README.md#configurable-options); the ones worth knowing about on +v2 specifically: + +| Option | Default | Description | +|---|---|---| +| `chunkTimeoutMs` | `180000` | Silence on a **busy** session before recovery is considered. Matches v1 since `49957b2`. | +| `toolTextCheckDelayMs` | `3000` | Settle delay before a finished turn's closing text is judged against the done/tool patterns. Dead streams are still caught on the idle event, because a stream that died before delivering any text has nothing for those patterns to read. `0` judges on the idle event. | +| `logFile` | `~/.local/state/opencode-v2/auto-resume.log` | Where this build logs. **v2 removed v1's server log endpoint**, so without this the plugin has no log at all; `AUTO_RESUME_LOG_FILE` in the environment overrides it. Size-capped at 2 MB. | +| `injectIntervalMs` | `15000` | Minimum gap between recovery injections for one session. No v1 equivalent. | +| `subagentNativeCompactionEnabled` | `false` | Opt-in native `session.compact()` for a saturated subagent, instead of leaving it alone. | +| `debug` | `false` | Verbose `[auto-resume:debug]` logging, appended to `logFile`. | + +Example: + +```jsonc +{ + "plugins": [ + { + "package": "./plugins/auto-resume-v2.ts", + "options": { "chunkTimeoutMs": 60000, "debug": true } + } + ] +} +``` + +## Verify it loaded + +Start opencode; the plugin logs its banner at load, into `logFile`: + +``` +[auto-resume] ready (opencode v2). timeout=180000ms interval=5000ms retries=3 loop=3/600s warmup=15000ms stall=continue +``` + +To see an intervention in action, let a session go quiet past +`chunkTimeoutMs`; the plugin injects a visible **synthetic** message in the +session timeline (`auto-resume: …`) and resumes the turn. Every recovery is +appended to `logFile` with an `[auto-resume]` prefix — the opencode log no longer +has a plugin sink to write to, which is why `logFile` exists. + +## Disable / uninstall + +- **Disable by id** without touching other plugins: add `"-auto-resume.v2"` to + the `plugins` array. +- **Temporarily off**: delete/move the file out of the `plugins/` directory. +- **Uninstall**: remove the file and any `plugins` entry you added. + +## Troubleshooting + +| Symptom | Check | +|---|---| +| No `[auto-resume] ready …` banner | File is in a `plugins/` dir opencode scans, or listed in `plugins`; restart opencode. | +| Logs say `failed to load plugin` with no reason | The file is not a valid ES module, or a syntax error. Run `bunx tsc --noEmit` on it (see [Development](#development)). | +| Nothing happens on a stall | Increase verbosity with `"debug": true`; confirm `chunkTimeoutMs` isn't larger than your real stall. | +| Recovers but you don't see a notice | Your model/provider may reject `session.synthetic()`; the plugin falls back to `session.prompt()` (no TUI banner). | +| Never recovers a parent waiting on a subagent | Intentional: parent sessions blocked on a running subagent are left alone. | +| Never recovers while a permission dialog is open | Intentional: recovery is held until the permission is answered. | + +## Development + +The source has no imports from `@opencode/plugin` — the `{ id, setup }` helper +is defined locally — so the package is only needed to check the file against the +official types: + +```sh +bun add -d @opencode/plugin@2.0.5 typescript +bunx tsc --noEmit --strict --target ESNext --module ESNext \ + --moduleResolution bundler --skipLibCheck src/v2/index.ts + +# the distributable is a single self-contained file +bun build src/v2/index.ts --outfile dist/v2/index.js --target bun +``` + +See [`migration.md`](./migration.md) for the full v1→v2 mapping and the +stable-vs-beta validation notes. diff --git a/docs/v2/migration.md b/docs/v2/migration.md new file mode 100644 index 0000000..c480cad --- /dev/null +++ b/docs/v2/migration.md @@ -0,0 +1,280 @@ +# opencode-auto-resume — OpenCode v2 port: what changed + +> Drop this into the PR description (or keep as docs/v2-migration.md upstream). +> Target: https://github.com/Mte90/opencode-auto-resume +> Runtime: **opencode v2.0.5+ (stable; live-verified on 2.0.27)**. The port has no runtime dependency — +> `src/v2/index.ts` imports only `node:fs`, `node:os` and `node:path`, and +> defines its own `{ id, setup }` helper. `@opencode/plugin@2.0.5` is used to +> typecheck the source, not to run it. +> Originally ported against opencode2 v0.0.0-beta-18050 / @opencode-ai/plugin +> 0.0.0-next-17403; re-validated against the stable release (see §7). + +## Summary + +Ports the plugin from the v1 hooks API (`Plugin` factory returning a hooks +object) to the v2 promise-plugin API (`Plugin.define({ id, setup })` + +`ctx.event.subscribe()`). All detection/recovery features are preserved. +Strict-mode typechecked against the real `@opencode/plugin@2.0.5` types, and +covered by 233 tests across 20 files driven through the real event stream — one +file per feature area (figures rechecked 2026-10-05 after the todo-branch fix +ports; repo-wide total in the suite footer below counts v1+v2). + +## 1. Config key renamed: `plugin` → `plugins` + +```jsonc +// v1 (removed) +{ "plugin": ["some-package", ["./local.ts", { "opt": 1 }]] } + +// v2 +{ + "plugins": [ + "some-package", + { "package": "./local.ts", "options": { "opt": 1 } } + ] +} +``` + +The tuple form `[path, options]` is gone — use the object form with +`package` + `options`. Local paths must start with `./` or `../` and resolve +relative to the config file. Disable directives (`"-plugin-id"`, `"*"`) +still work and are matched against the plugin's exported `id`. + +## 2. Module shape: hooks-object → `{ id, setup }` + +```ts +// v1: export a factory that returns a Hooks object +export const MyPlugin: Plugin = async ({ client, $, directory }) => ({ + event: async ({ event }) => { /* ... */ }, + "tool.execute.before": async (input, output) => { /* ... */ }, +}) + +// v2: default-export Plugin.define with a unique id and a setup fn. +// setup registers long-lived behavior and MAY return a cleanup function, +// which OpenCode awaits on disable/reload/shutdown. +import { Plugin } from "@opencode/plugin" + +export default Plugin.define({ + id: "acme.thing", // unique — used by disable directives + setup: async (ctx) => { + /* register timers/subscriptions here */ + return () => { /* cleanup */ } + }, +}) +``` + +Legacy single-function modules are still loaded through a compatibility +shim, but new code should target v2 directly. + +## 3. Context capabilities replace most hooks + +The v2 context is essentially a typed server client plus registration APIs: + +| v2 capability | Replaces (v1) | +|---|---| +| `ctx.event.subscribe()` → **AsyncIterable** of events | `event` hook | +| `ctx.session.prompt/interrupt/create/get/command/synthetic/generate` + `context/wait/switchAgent/switchModel/rename/update/move` + `ctx.session.hook(...)` | direct SDK calls / chat message hooks | +| `ctx.tool.transform` / `ctx.tool.hook` | `tool` map, `tool.execute.before/after` | +| `ctx.agent.transform` | agent config mutation | +| `ctx.options` | second `options` argument of the v1 factory | +| `ctx.app` | `{ name, version, channel }` only — **no `app.log()`** | + +Notes: +- **Subscribe, don't block**: start the event pump in the background inside + `setup`; do not `await` an infinite loop there. +- **Flattened SDK calls**: nested `{ path, query, body }` envelopes are gone. + Example: `session.prompt({ path: { id }, body: { parts } })` became + `session.prompt({ sessionID, text })`. +- **History access**: beta exposed no message-history API, so plugins that + needed assistant text had to accumulate it from streaming events + (`session.text.delta`, `session.reasoning.delta`). **Stable adds + `ctx.session.context({ sessionID })` → `SessionMessageInfo[]`**; the port + keeps the deltas for liveness and uses `context()` as the authoritative + final text at idle time. +- **Logging**: write to the console with a prefix instead of `app.log()`; + opencode captures stdout/stderr into its logs. + +## 4. Event payload shapes changed + +v1 events were `{ type, properties }`; v2 events are flat +`{ type, created, data }`. Useful v2 session events for watchdog-style +plugins: `session.execution.started/succeeded/failed/interrupted`, +`session.step.started/streamed/ended/failed`, `session.text.started/delta/ended`, +`session.reasoning.started/delta/ended`, +`session.tool.input.started/delta/ended`, +`session.tool.called/progress/success/failed`, `session.shell.started/ended`, +`session.retry.scheduled`, `session.compaction.*`, `session.status`, +`session.idle`, `session.deleted`, `permission.asked/replied`. + +Two shape details the port relies on: + +- `session.execution.interrupted` carries `data.reason` + (`"user" | "shutdown" | "superseded" | "inactivity"`) — only `"user"` should + suppress recovery. +- `session.idle` is a separate ephemeral event (`data.sessionID`); + `session.status` carries `data.status.type` (`"idle" | "retry" | "busy"`). + +## 5. Concrete changes in this port + +- v1 hooks → v2 `Plugin.define({ id: "auto-resume.v2", setup })` + + background `ctx.event.subscribe()` pump; cleanup returned from `setup` + clears the watchdog interval. +- `client.session.prompt({path, body})` → `ctx.session.prompt({ sessionID, + text })` (flat input; the beta-era nested-shape fallback was removed once + stable confirmed the flat contract). +- `session.messages()` forensics → live accumulation of assistant text from + `session.text.delta` / `session.reasoning.delta`, with + `ctx.session.context({ sessionID })` as the authoritative idle-time read. +- Recovery notifications use `ctx.session.synthetic({ sessionID, text, + description, resume })` so the intervention is visible in the TUI, falling + back to `session.prompt({ sessionID, text })`. +- **The todo list is read from the session message log, not from storage.** v2 + emits no `todo.updated` event and exposes no todo route — and, decisively, + **`ctx.storage` is namespaced per plugin**, so it cannot read another plugin's + todo list at all. `ctx.storage.get("todos/")` resolves to + `storage/plugin/auto-resume.v2/todos/.json`; a todo plugin writing that key + lands under its own plugin id. Verified live: the route is + `/api/plugin/storage//` and the on-disk tree is + `storage/plugin//.json`. A storage-first reader therefore + concludes "no todos" regardless of who writes, and fires false done-claim + nudges. + + Every `todowrite` call is persisted as a tool part whose input carries the whole + list, so the newest **completed** call is the current list. It is read via + `ctx.client.session.message.list`, falling back to a loopback + `GET /api/session/{id}/message?limit=200` (that endpoint answers + `{ data, cursor }`, never a bare array; 200 is its ceiling; results are ordered + newest first, so a last-match-wins parser would select the OLDEST list). One + measured detail worth keeping: the tool parts carry no `state.time`, so ranking + must fall back to the enclosing message's `time.created` — under newest-first + ordering that fallback is load-bearing, not cosmetic. + + `ctx.storage` remains only as a last-resort fallback. Do not "fix" the namespace + mismatch by having auto-resume write the key: a reader cannot surface another + plugin's data, and a second writer would race the first. +- **New host access beyond the plugin context.** `readFileSync` on + `/proc/self/cmdline` to discover the local server's `--port`, plus a loopback + HTTP GET. Env vars alone are not enough: under OpenChamber the server is spawned + as `opencode serve --hostname 127.0.0.1 --port ` and exports no port variable, + so an env-only candidate list comes back empty and the HTTP source is never + attempted at all. Every one of these is best-effort — a host that refuses them + degrades to "cannot read todos", never a crash — and where `/proc` is + unavailable the plugin falls back to env-var discovery. +- `ctx.client.app.log(...)` → prefixed console logging. +- Status polling demoted: session state comes primarily from lifecycle + events; a low-frequency interval cross-checks active sessions for silence + only. +- New guards made possible/necessary by v2 semantics: + - `permission.asked` / `permission.replied` hold off recovery while a + permission dialog is open. + - Subagent-aware waiting: a parent silent right after dispatching a task + tool with other sessions running is left alone. + - Failure-triggered recoveries arm a flag so the failure's own idle + transition doesn't cancel the scheduled retry prompt, while a healthy + completion still cancels stale ones. + - `ctx.event.subscribe({ signal })` + `AbortController` so the event stream + is torn down on unload instead of staying suspended in `for await`. + - User-interrupt detection uses `session.execution.interrupted`'s `reason` + (stable) instead of assuming every non-plugin interrupt was the user. + +## Feature parity retained + +Stall watchdog · failure recovery with exponential backoff → interrupt+resume +escalation · tool-call-as-text detection · "ready to continue"/action-intent +nudges · done-claim verification · hallucination-loop guard (N continues in +window → abort+resume) · tool-loop pattern detection · user-interrupt +respected. + +## Options + +The port reads **all 31 of v1's options** and applies **all 31**. Names and +defaults are unchanged, and the full table is in the +[README](../../README.md#configurable-options). Two defaults moved after both +builds picked them up from upstream: `chunkTimeoutMs` to 180000 (`49957b2`) and +`activeUserWindowMs` to 300000 (`e1b8374`). + +Two options are **v2-only**, with no v1 equivalent, because v2 removed the +capability they configure: + +- `logFile` — v2 has no server log endpoint, so the plugin writes its own + (2 MB cap, `AUTO_RESUME_LOG_FILE` overrides). +- `injectIntervalMs` — a floor on how often one session may be nudged. + +For the record, this list is checked against the source in both directions: a test +fails if an option joins the recognised set without being implemented, and the +doc check fails if the table names one that is not gated. + +```jsonc +{ + "plugins": [ + { + "package": "./plugins/auto-resume-v2.ts", + "options": { "chunkTimeoutMs": 180000, "maxRetries": 3 } + } + ] +} +``` + +Disable by id without touching other plugins: add `"-auto-resume.v2"`. + +## Testing + +- `tsc --strict --noEmit` clean against **`@opencode/plugin@2.0.5`** (stable). +- Runtime suite (bun, 20 files / 233 tests, rechecked 2026-10-05) against the stable API: + definition shape · `subscribe({ signal })` · abort-on-cleanup · stall watchdog + → `synthetic` with visible `description` + `resume` · idle forensics via the + `session.context()` fallback · healthy idle turn is a no-op · permission-hold + blocks recovery · `interrupted` `reason="user"` disables recovery · + `reason="inactivity"` does **not** · execution-failure recovery · + `synthetic`→`prompt` fallback · `session.deleted` drops state — + Each later feature area has its own file: unknown tool suggestions, + silent dead streams, premature stop, context saturation, the todo list, explicit + `task_complete`, reasoning-tool recovery, orphan parent recovery, the settle + delay, session discovery, the options surface, and the hand-off and stand-down + guards — plus cross-instance duplicate suppression, the inject mutex and + singleton registry, the quota ladder, parent-wait on live subagents, and the + visible channel with rich stall text. + + Two tests in the settle-delay file were **vacuous** on the first pass — they + passed whether or not the code under test ran. Every test in the v2 port is now + mutation-checked: removing the delay, the arming, the history re-read, either + guard re-run, the cancel latch, the new-turn cancel or the cleanup each turns + the file red. A negative test that never fails is indistinguishable from a + detector that never runs. + +## 7. Re-validated against stable v2 (2.0.5) + +The port was originally written against the beta (`@opencode-ai/plugin` +`0.0.0-next-17403`). Re-checked against the stable release: + +- **Package renamed**: the API types are now **`@opencode/plugin`** (stable + `2.0.5`), published from the opencode repo; the beta `@opencode-ai/plugin` is + not the stable API. The port typechecks against those types but does not import + them — it declares `{ id, setup }` locally, so a **local-file install needs no + `bun add` at all** and the built artefact is a single self-contained file. + Config/discovery is unchanged (`plugins` key, object form, and plugins under + `.opencode/plugin/` **or** `.opencode/plugins/` are auto-loaded). +- **Every event the plugin matches still exists** in 2.0.5, with the same names + (`session.execution.succeeded` is still `succeeded`, not `completed`). New, + unused-by-us events: `session.status`, `session.text.started`, + `session.step.streamed`, `session.tool.input.*`, `session.compaction.*`, + `session.message.content.updated`. +- **`session.execution.interrupted` gained `data.reason`**; the port now treats + only `"user"` as a user-cancel. +- **`session.reverted` is gone** in v2 (reverts surface as + `session.revert.cleared` / `session.revert.committed`); the legacy matcher is + kept defensively and `session.deleted` handles cleanup. +- **New session methods** on the plugin context: `context()` (message history), + `wait()`, `switchAgent()`, `switchModel()`, `rename()`. The port adopts + `context()` for idle-time forensics; `wait()` is intentionally unused (the + event-driven watchdog is retained). +- **`ctx.session.synthetic()` still accepts `description` and `resume`** in its + input (both optional), so the visible-notification + resume path is unchanged. + `ctx.session.interrupt()` input is `{ sessionID, resume? }`. +- **Docs recommend `ctx.event.subscribe({ signal })`** and aborting the stream + during cleanup; the port now does this with an `AbortController`. +- **Local-file installs need nothing installed.** opencode loads a local `.ts` + plugin with a normal ESM import, so *a plugin that imports the API package* + would have to have it resolvable and would need a restart. This port does not + import it, which removes that constraint — copy the file and restart. A + *published* plugin that later does import it should declare + `@opencode/plugin` in its `dependencies` (replacing `@opencode-ai/plugin`). diff --git a/package.json b/package.json index f0e53a9..18ff76f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "opencode-auto-resume", - "version": "1.1.21", + "version": "1.2.0", "description": "OpenCode plugin that automatically resumes stalled LLM sessions when thinking/streaming freezes mid-generation.", "keywords": [ "opencode", @@ -22,12 +22,15 @@ "type": "module", "main": "dist/index.js", "scripts": { - "build": "bun build src/index.ts --outdir dist --target bun", + "build": "bun run build:v1 && bun run build:v2", + "build:v1": "bun build src/index.ts --outdir dist --target bun", + "build:v2": "bun build src/v2/index.ts --outfile dist/v2/index.js --target bun", "dev": "bun build src/index.ts --outdir dist --target bun --watch", "test": "bun test", "prepublishOnly": "bun run build" }, "files": [ "dist/index.js", + "dist/v2/index.js", "README.md", "LICENSE" ], diff --git a/src/v2/index.adopt-state.test.ts b/src/v2/index.adopt-state.test.ts new file mode 100644 index 0000000..75b8f0a --- /dev/null +++ b/src/v2/index.adopt-state.test.ts @@ -0,0 +1,141 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexAdoptStateTest" + +/** + * v2: a successor setup inherits the watch map instead of starting blank. + * + * 2026-10-09: setup re-runs in-process on session attach/restore (bursts of + * several setups/second, singles every ~25-40min — 138 supersedes on record + * in one day). Every run disposed the last AND started with a blank sessions + * map, so multi-minute stalls were unobservable and the busy session that + * went quiet across a take-over never fired. Successors now adopt the + * predecessor's map (stall clocks, ladders, latches); dispose only tears + * down its own timers. Proof in prod: "adopted 1 watched session(s)". + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 9500 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 300, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + injectIntervalMs: 0, + subagentWaitMs: 40, + // Under test: two setups in one process share the watch map, exactly + // what prod opts in to. Without it the suite shares nothing and this + // test cannot observe adoption. + shareWatchState: true, + debug: false, + // No discovery sweep inside the test window: the point is map + // inheritance across setups, not re-seeding from the session list. + discoveryDelayMs: 60_000, +} + +function makeCtx(stream: any, injected: Array<{ sid?: string; text?: string }>, tag: number) { + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-adopt-${process.pid}-${tag}.log`) + rmSync(logFile, { force: true }) + return { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [] }), + message: { list: async () => [] }, + }, + }, + storage: { + get: async () => ({ todos: [] }), + set: async () => {}, + remove: async () => {}, + }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => [], + }, + logFile, + } +} + +describe("v2: successor setup adopts the watch map", () => { + test("busy-quiet watch survives a take-over and still fires", async () => { + const injected: Array<{ sid?: string; text?: string }> = [] + const files: string[] = [] + const stream1 = makeEventStream() + const ctx1: any = makeCtx(stream1, injected, counter++) + files.push(ctx1.logFile) + const cleanup1 = await (plugin as any).setup(ctx1) + stream1.push(ev("session.execution.started", SID)) + await wait(50) + // Second setup supersedes the first (same process, like a session + // attach storm). No new events: the session stays busy-quiet. + const stream2 = makeEventStream() + const ctx2: any = makeCtx(stream2, injected, counter++) + files.push(ctx2.logFile) + const cleanup2 = await (plugin as any).setup(ctx2) + await wait(900) + expect(injected.filter((i: any) => i.sid === SID).length).toBeGreaterThan(0) + ;(cleanup1 as (() => void) | undefined)?.() + ;(cleanup2 as (() => void) | undefined)?.() + for (const f of files) rmSync(f, { force: true }) + }) +}) diff --git a/src/v2/index.attempt-reset.test.ts b/src/v2/index.attempt-reset.test.ts new file mode 100644 index 0000000..7504478 --- /dev/null +++ b/src/v2/index.attempt-reset.test.ts @@ -0,0 +1,400 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexAttemptResetTest" + +/** + * v2: model progress between stalls restarts the attempt ladder. + * + * ses_ee7dade1 (2026-10-07): "attempt 1/3" then model work, then + * "attempt 2/3" — the ladder kept counting across a productive interval. + * resumeAttempts reset only on busy-cycle transitions, so mid-turn progress + * never restarted it. A stall after fresh model output is a NEW episode and + * must read "attempt 1/3" again; only consecutive silent prods climb toward + * gaveUp. (Rich texts embed the attempt number, so episode boundaries are + * directly observable in the transcript.) + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 11000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +async function setup(extraOpts: Record = {}): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-attemptreset-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 50, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + // Without this the test writes into the PRODUCTION log, where a run is + // indistinguishable from a reload storm. + logFile, + debug: true, + ...extraOpts, + }, + session: { + context: async () => [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +/** + * Wait for a condition instead of sleeping a guessed interval. This test + * used fixed waits and passed 3/3 in isolation while failing under + * full-suite load — exactly the flake that trains you to ignore a red suite. + */ +async function waitFor(what: () => boolean, budgetMs = 3000): Promise { + const deadline = Date.now() + budgetMs + while (Date.now() < deadline) { + if (what()) return true + await wait(10) + } + return what() +} + +const attemptOf = (text: string | undefined): string | null => { + const m = /attempt (\d)\/3/.exec(text ?? "") + return m ? m[1] : null +} + +describe("v2: model progress between stalls restarts the attempt ladder", () => { + test("fresh assistant text restarts the ladder at 1/3", async () => { + const h = await setup() + push(h, "session.execution.started", SID) + await wait(10) + // Wait for the first nudge, then land the delta while the ladder is still + // climbing. Ordering is the thing under test, so wait on events rather + // than on wall-clock guesses. + await waitFor(() => h.injected.length >= 1) + // Model produces text (progress, no completion), then goes quiet. + push(h, "session.text.delta", SID, { assistantMessageID: "m1", delta: "new findings from the probe" }) + await waitFor(() => h.injected.length >= 2) + await teardown(h) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + expect(attempts.length).toBeGreaterThanOrEqual(2) + // A second "1/3" means a new episode started after model progress. + expect(attempts.filter((a: string | null) => a === "1").length).toBeGreaterThanOrEqual(2) + }) + + test("tool completions between failure-driven recovers do not restart the ladder", async () => { + // ses_edf403f5, 2026-10-09: dead-stream stalls where a tool completion sat + // between recovers must keep climbing toward gaveUp — a completion with + // no model text on a FAILED turn is the stall loop wearing a hat, not a + // new episode. Only silence + success-progress (next test) restarts. + // chunkTimeout is parked so no silence stall can interleave: every + // inject here is failure-driven by construction. + const h = await setup({ chunkTimeoutMs: 600_000 }) + const FAIL = { error: { type: "provider.invalid-request", message: "Error from the provider: 400 Bad Request" } } + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.step.failed", SID, FAIL) + await waitFor(() => h.injected.length >= 1) + push(h, "session.tool.success", SID, { tool: "shell", id: "call_9" }) + await wait(10) + push(h, "session.step.failed", SID, FAIL) + await waitFor(() => h.injected.length >= 2) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + await teardown(h) + expect(attempts.slice(0, 2)).toEqual(["1", "2"]) + }) + + test("a successful tool between silence stalls restarts the ladder at 1/3", async () => { + // ses_ede9b725 (pulse review, 2026-10-09): 1/3 woke the model, it read a + // file (tool success, no text), then went quiet; the next stall read 2/3. + // A wake-up that produced successful work is a NEW episode, not a deeper + // stall — silence + success-progress restarts at 1/3. Failure-triggered + // recovers never reset on tool progress (previous test). + const h = await setup() + push(h, "session.execution.started", SID) + await wait(10) + await waitFor(() => h.injected.length >= 1) + push(h, "session.tool.success", SID, { tool: "read", id: "call_1" }) + await waitFor(() => h.injected.length >= 2) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + const logs = existsSync(h.logFile) ? readFileSync(h.logFile, "utf8") : "" + await teardown(h) + expect(attempts.slice(0, 2)).toEqual(["1", "1"]) + expect(logs).toContain("restarting the stall ladder") + }) + + test("a failed tool between silence stalls does not restart the ladder", async () => { + // Failures are the stall loop wearing a different hat: only successful + // work marks a wake-up as productive. The failure-loop latch owns the + // failure path; the ladder must keep climbing through it. + const h = await setup() + push(h, "session.execution.started", SID) + await wait(10) + await waitFor(() => h.injected.length >= 1) + push(h, "session.tool.failed", SID, { tool: "read", id: "call_1" }) + await waitFor(() => h.injected.length >= 2) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + await teardown(h) + expect(attempts.slice(0, 2)).toEqual(["1", "2"]) + }) + + test("a successful shell between silence stalls restarts the ladder at 1/3", async () => { + // Shell completions are successful work too (same sid attribution via + // openShells). server: shell.created carries NO top-level sessionID — + // it nests at data.info.metadata.sessionID; exits carry only the id. + const h = await setup() + push(h, "session.execution.started", SID) + await wait(10) + await waitFor(() => h.injected.length >= 1) + h.streamRef.push({ + type: "shell.created", + data: { info: { id: "sh_1", command: "sleep 1", status: "running", metadata: { sessionID: SID } } }, + }) + await wait(10) + h.streamRef.push({ type: "shell.exited", data: { id: "sh_1", status: "exited", exit: 0 } }) + await waitFor(() => h.injected.length >= 2) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + await teardown(h) + expect(attempts.slice(0, 2)).toEqual(["1", "1"]) + }) + + test("a genuine turn restarts the retry count but not the ladder", async () => { + // The count answers "which recovery of the episode", and a model turn + // not caused by us is a new episode — even when the stall ladder + // correctly keeps climbing through it (own inject newer than any + // genuine user message, so no ladder reset). Fires read attempt 1/3 + // then 2/3, but retry #1 both times. + const retryOf = (text: string | undefined): string | null => { + const m = /retry #(\d+)/.exec(text ?? "") + return m ? m[1] : null + } + const h = await setup() + push(h, "session.execution.started", SID) + await wait(10) + await waitFor(() => h.injected.length >= 1) + push(h, "session.idle", SID) + await wait(10) + // No new user message: the next start is the model acting on its own + // (selfRecovery was cleared as stale by the idle above), so genuine. + push(h, "session.execution.started", SID) + await waitFor(() => h.injected.length >= 2) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + const retries = h.injected.map((i: any) => retryOf(i.text)) + await teardown(h) + expect(attempts.slice(0, 2)).toEqual(["1", "2"]) + expect(retries.slice(0, 2)).toEqual(["1", "1"]) + }) + test("each fire carries its episode retry number", async () => { + // The ladder rung restarts on progress, so consecutive fires can all + // read "attempt 1/3" while being the 1st, 2nd, ... recovery of the + // episode. Without the retry count those fires read byte-identically + // and cannot be reconciled from the transcript (2026-10-12: fourteen + // 1/3s in a row, never climbing, never explained). + const retryOf = (text: string | undefined): string | null => { + const m = /retry #(\d+)/.exec(text ?? "") + return m ? m[1] : null + } + const h = await setup() + push(h, "session.execution.started", SID) + await wait(10) + await waitFor(() => h.injected.length >= 1) + h.streamRef.push({ + type: "shell.created", + data: { info: { id: "sh_1", command: "sleep 1", status: "running", metadata: { sessionID: SID } } }, + }) + await wait(10) + h.streamRef.push({ type: "shell.exited", data: { id: "sh_1", status: "exited", exit: 0 } }) + await waitFor(() => h.injected.length >= 2) + const retries = h.injected.map((i: any) => retryOf(i.text)) + await teardown(h) + expect(retries.slice(0, 2)).toEqual(["1", "2"]) + }) + + test("an idle between our send and the model's start does not restart the ladder", async () => { + // ses_ee14d727, 2026-10-09: 1/3, then 1/3 again. Resets live only + // where a new episode is observable — a genuine user message newer than + // our last inject, model text growth, or silence + successful tool work + // (next tests) — never on lifecycle transitions like the idle->busy + // one here. Our inject stays the newest causative event, so the ladder + // climbs. NOTE: this test carries no tool.success on purpose — a + // successful tool between silence stalls IS a new episode now. + const h = await setup() + push(h, "session.execution.started", SID) + await wait(10) + await waitFor(() => h.injected.length >= 1) + push(h, "session.idle", SID) + await wait(10) + push(h, "session.execution.started", SID) + await waitFor(() => h.injected.length >= 2) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + await teardown(h) + expect(attempts.slice(0, 2)).toEqual(["1", "2"]) + }) + + test("CONTROL: silence without progress keeps climbing across injections", async () => { + // The regression this pins: after a successful stall injection the session + // is idle again when the next stall timer fires, so markBusy transitions it + // to busy and zeroed resumeAttempts — every stall read 1/3 and the ladder + // never climbed (ses_ee14d727, 2026-10-09: two occurrences, both 1/3). + const h = await setup() + push(h, "session.execution.started", SID) + await wait(10) + await wait(650) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + await teardown(h) + expect(attempts.length).toBeGreaterThanOrEqual(2) + expect(attempts[attempts.length - 1]).not.toBe("1") + // And it must have grown, not repeated: 1/3 then 2/3 (or deeper). + expect(attempts.some((a: string | null) => a === "2")).toBe(true) + }) + + test("the stall message carries a timestamp and idle duration", async () => { + const h = await setup() + push(h, "session.execution.started", SID) + await wait(250) + const text = h.injected[0]?.text ?? "" + await teardown(h) + expect(text).toContain("idle ") + // Local operator time, not Zulu (see the ed9e2ef9 timezone note). + expect(text).toMatch(/[AP]M [A-Z]{2,4}/) + expect(text).not.toMatch(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/) + }) + + test("no-activity stalls state the silence once, not twice", async () => { + // (no activity for 188s; ... idle 3m 9s) prints the same duration twice. + // The reason already says it, so the idle field is dropped for exactly + // this reason shape; anything else keeps both fields. + const h = await setup() + push(h, "session.execution.started", SID) + await wait(250) + const text = h.injected[0]?.text ?? "" + await teardown(h) + expect(text).not.toMatch(/no activity for \d+s[^;]*;[^;]*idle/) + expect(text).toContain("idle ") + }) + + test("failure stalls carry no idle field at all", async () => { + // A failure just happened: "idle 2s" next to the reason is noise, not + // news. Reason + attempt + timestamp only. + const h = await setup() + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.step.failed", SID, { + error: { type: "provider.invalid-request", message: "Error from the provider: 400 Bad Request" }, + }) + await wait(400) + const text = h.injected[0]?.text ?? "" + await teardown(h) + expect(text).toContain("session.step.failed") + expect(text).not.toContain("idle ") + // Local operator time (see ed9e2ef9 note on the sibling test above). + expect(text).toMatch(/[AP]M [A-Z]{2,4}/) + expect(text).not.toMatch(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/) + }) + + test("watches adopted from older builds get migrated, never NaN", async () => { + // The sessions map survives reloads, so a watch created before a field + // existed reads it as undefined: `undefined++` rendered as "retry #NaN" + // in production (2026-10-12), and `for..of undefined` would throw the + // whole sweep. Stripping the new fields off the live watch simulates + // the adopted old-build object; the next stall must still fire with a + // numeric retry count. + // + // Own scope + shared map: the default harness keeps its sessions map + // in the setup closure (unreachable from here), so this test opts + // into the shared map to reach the live object, on a scope no other + // test uses. + const SCOPE = `${LOG_SALT}-migrate` + const h = await setup({ shareWatchState: true, registryScope: SCOPE }) + push(h, "session.execution.started", SID) + await wait(10) + await waitFor(() => h.injected.length >= 1) + const reg = (globalThis as any)[`__auto_resume_singleton__:${process.pid}:${SCOPE}`] + const w = reg.sessions.get(SID) + delete (w as any).episodeRecovers + delete (w as any).inflightTools + delete (w as any).emptyIdleNudged + delete (w as any).lastStuckToolCheck + await waitFor(() => h.injected.length >= 2) + const text = (h.injected[1] as any)?.text ?? "" + await teardown(h) + expect(text).toMatch(/retry #\d+/) + expect(text).not.toContain("NaN") + }) + +}) diff --git a/src/v2/index.busy-tools.test.ts b/src/v2/index.busy-tools.test.ts new file mode 100644 index 0000000..4df8fa3 --- /dev/null +++ b/src/v2/index.busy-tools.test.ts @@ -0,0 +1,192 @@ +import { describe, test, expect, setDefaultTimeout } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexBusyToolsTest" + +/** + * v2: the busy-stall path defers while tools are still running. + * + * A `write` or `compress` running several minutes emits no events, so + * event-silence accrues past the chunk timeout and the injection below + * interrupts the live call ("Tool execution interrupted: write" — observed + * 2026-10-11, three times on one 9am run). The plugin could not tell "tool + * running long" from "turn dead". + * + * Rule, in the watchdog's skip-chain next to the shell/subagent/user guards: + * a tool younger than the stuck age gate defers; older than the gate is + * wedged and the stall answers normally. Exempt names (default: compress) + * are never interrupted while fresh — interrupting a compress risks + * half-compacted state, the same reason compaction itself is never + * interrupted — with the same stale-TTL escape when one wedges for real. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +/** + * Budgets sized for the environment, not for isolation: `bun test` runs 79 + * files in parallel and these tests wait out real chunk timeouts, so a + * condition that resolves in milliseconds alone can take seconds of wall + * clock under contention. + */ +setDefaultTimeout(30_000) + +const SID = "ses_busytools" +let counter = 61000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const BASE_OPTS = { + chunkTimeoutMs: 50, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + toolTextCheckDelayMs: 0, + debug: false, +} + +async function setup(extraOpts: Record = {}) { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-busytools-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const ctx: any = { + event: stream, + options: { ...BASE_OPTS, logFile, registryScope: LOG_SALT, ...extraOpts }, + session: { + context: async () => [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => undefined, set: async () => {}, remove: async () => {} }, + tool: { transform: async () => {} }, + } + const cleanup = await (plugin as any).setup(ctx) + return { + injected, + cleanup, + push: async (type: string, data: Record = {}) => { + stream.push(ev(type, data)) + await wait(10) + }, + teardown: () => { + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + }, + } +} + +describe("v2: busy stall defers while tools run", () => { + test("CONTROL: silent busy session with no tools still recovers", async () => { + const h = await setup() + await h.push("session.execution.started") + await wait(300) + await h.teardown() + expect(h.injected.length).toBeGreaterThanOrEqual(1) + }, 20000) + + test("a young write call defers the stall", async () => { + // stuckToolAgeMs 5000: at 300ms the call is fresh, so every sweep in + // the window must defer even though silence (300ms) is past the 50ms + // chunk timeout several times over. + const h = await setup({ stuckToolAgeMs: 5000 }) + await h.push("session.execution.started") + await h.push("session.tool.called", { tool: "write", id: "call_1" }) + await wait(300) + await h.teardown() + expect(h.injected).toEqual([]) + }, 20000) + + test("a write call older than the age gate still recovers (wedged)", async () => { + // stuckToolAgeMs 120: early sweeps defer (age < gate), then the call + // ages past it and the stall answers. Both halves in one run. + const h = await setup({ stuckToolAgeMs: 120 }) + await h.push("session.execution.started") + await h.push("session.tool.called", { tool: "write", id: "call_1" }) + await wait(400) + await h.teardown() + expect(h.injected.length).toBeGreaterThanOrEqual(1) + }, 20000) + + test("compress is exempt even when older than the age gate", async () => { + // stuckToolAgeMs 120, call aged 400ms: a normal tool would have fired + // (previous test). Compress must not — only the 30-minute stale TTL + // releases it, which no test waits out. + const h = await setup({ stuckToolAgeMs: 120 }) + await h.push("session.execution.started") + await h.push("session.tool.called", { tool: "compress", id: "call_9" }) + await wait(400) + await h.teardown() + expect(h.injected).toEqual([]) + }, 20000) + + test("neverInterruptTools: [] puts compress back under the age gate", async () => { + // Configurability pin: with the exemption emptied, an old compress + // recovers like any other wedged tool. + const h = await setup({ stuckToolAgeMs: 120, neverInterruptTools: [] }) + await h.push("session.execution.started") + await h.push("session.tool.called", { tool: "compress", id: "call_9" }) + await wait(400) + await h.teardown() + expect(h.injected.length).toBeGreaterThanOrEqual(1) + }, 20000) + + test("a finished tool stops deferring", async () => { + // Call, then success well inside the gate: the map entry is removed + // and the stall answers on schedule. + const h = await setup({ stuckToolAgeMs: 5000 }) + await h.push("session.execution.started") + await h.push("session.tool.called", { tool: "write", id: "call_1" }) + await h.push("session.tool.success", { id: "call_1" }) + await wait(300) + await h.teardown() + expect(h.injected.length).toBeGreaterThanOrEqual(1) + }, 20000) +}) diff --git a/src/v2/index.child-guard.test.ts b/src/v2/index.child-guard.test.ts new file mode 100644 index 0000000..25df2d5 --- /dev/null +++ b/src/v2/index.child-guard.test.ts @@ -0,0 +1,221 @@ +import { describe, test, expect, afterEach } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexChildGuardTest" + +/** + * v2: subagent protection without ctx.client. + * + * Production ctx carries no client and no session list/get/active surface + * (probed live 2026-10-06: every subagent check silently degraded to false; + * zero "injection refused" lines in all of log history). Two injections + * landed in child ses_eed4521bdffeHLLmpomhiB0KXx the same millisecond from + * stacked setups whose backoff timers survived disposal. So: + * 1. subagent identity resolves over the loopback session list + * (GET /api/session carries parentID rows), and + * 2. dispose cancels pending recover timers (plus a running-check in the + * timer body), so a superseded instance stays silent. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" +const CHILD = "ses_eed4521bdffeHLLmpomhiB0KXx" + +let counter = 3000 +const realFetch = globalThis.fetch +const realServerPort = process.env.OPENCODE_SERVER_PORT + +afterEach(() => { + globalThis.fetch = realFetch + if (realServerPort === undefined) delete process.env.OPENCODE_SERVER_PORT + else process.env.OPENCODE_SERVER_PORT = realServerPort +}) + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 300, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, +} + +const userMessage = (text: string, at: number) => ({ + type: "user", + id: "msg_u", + time: { created: at }, + content: [{ type: "text", text }], +}) + +const assistantText = (text: string, at: number) => ({ + type: "assistant", + id: "msg_a", + time: { created: at }, + content: [{ type: "text", text }], +}) + +type History = Record + +async function setup(opts: { + history?: History + active?: string[] + withClient?: boolean + backoffMs?: number +}): Promise { + const injected: Array<{ sid?: string; text?: string }> = [] + const interrupted: string[] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-childguard-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const history: History = opts.history ?? {} + const active = new Set(opts.active ?? []) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + ...OPTIONS, + logFile, + ...(opts.backoffMs ? { baseBackoffMs: opts.backoffMs, maxBackoffMs: opts.backoffMs } : {}), + }, + session: { + context: async (a: any) => history[a?.sessionID ?? SID] ?? [], + active: async () => Object.fromEntries([...active].map((s) => [s, {}])), + interrupt: async (a: any) => (interrupted.push(a?.sessionID), {}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + if (opts.withClient !== false) { + ctx.client = { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [] }), + }, + } + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, interrupted, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +async function makeBusy(h: any, sid: string) { + push(h, "session.execution.started", sid) + await wait(10) + push(h, "session.step.started", sid) + await wait(10) +} + +/** Loopback session list stub: the shape prod GET /api/session returns. */ +function stubSessionList(rows: Array<{ id: string; parentID?: string }>) { + globalThis.fetch = (async (url: any) => { + const u = String(url) + if (u.includes("/api/session?") || u.endsWith("/api/session")) { + return { ok: true, json: async () => ({ data: rows }) } as any + } + return { ok: false, status: 404, json: async () => ({}) } as any + }) as any +} + +describe("v2: subagent protection without ctx.client", () => { + test("a stalled child is refused injection via the loopback list", async () => { + // Prod shape: no ctx.client at all. The child goes quiet past the + // stall threshold; the ONLY thing that knows it is a child is the + // loopback session list. Before the HTTP fallback this injected. + // serverBaseUrls() needs a port to build a base — prod has one via + // env/argv, the test env does not, so pin one here. + process.env.OPENCODE_SERVER_PORT = "18234" + stubSessionList([ + { id: SID }, + { id: CHILD, parentID: SID }, + ]) + const now = Date.now() + const h = await setup( + { + history: { [CHILD]: [userMessage("go", now - 60_000), assistantText("working", now - 5_000)] }, + active: [CHILD], + withClient: false, + }, + ) + await makeBusy(h, CHILD) + await wait(900) + expect(h.injected.filter((i: any) => i.sid === CHILD)).toHaveLength(0) + expect(h.interrupted).toHaveLength(0) + await teardown(h) + }) + + test("dispose cancels a scheduled stall inject", async () => { + // The 5ms-apart twin prods: instance A schedules its backoff timer, + // setup B disposes A, A's timer still fires. With a 5s backoff there + // is ample room to dispose first: nothing may land afterwards. + stubSessionList([{ id: SID }]) + const now = Date.now() + const h = await setup({ + history: { [SID]: [userMessage("go", now - 60_000), assistantText("working", now - 5_000)] }, + active: [SID], + backoffMs: 1500, + }) + await makeBusy(h, SID) + await wait(700) + await teardown(h) + await wait(2200) + expect(h.injected).toHaveLength(0) + }) +}) diff --git a/src/v2/index.continue-guardrail-wire.test.ts b/src/v2/index.continue-guardrail-wire.test.ts new file mode 100644 index 0000000..20718bf --- /dev/null +++ b/src/v2/index.continue-guardrail-wire.test.ts @@ -0,0 +1,148 @@ +import { describe, test, expect, setDefaultTimeout } from "bun:test" +import { rmSync, existsSync, readFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** + * Behavioural counterpart to index.continue-guardrail.test.ts. + * + * The source-regex file proves the four sites NAME the guarded constant. This + * proves the guardrail actually REACHES THE WIRE — that the text a model + * receives carries the instruction. Source assertions alone are the failure mode + * I spent today documenting (a guard that is spelled correctly and never + * engages); this drives the plugin and reads what it sent. + */ +const LOG_SALT = "IndexContinueGuardrailBehaviour" +const SID = "ses_parent" +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +setDefaultTimeout(30_000) + +let counter = 91_000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const userMessage = (at: number, text = "go") => ({ + type: "user", + id: `msg_u_${at}`, + time: { created: at }, + content: [{ type: "text", text }], +}) +const assistantWith = (at: number, text: string) => ({ + type: "assistant", + id: `msg_a_${at}`, + time: { created: at }, + content: [{ type: "text", text }], +}) + +const GUARD = "Do not start anything new" +const DESTRUCTIVE = "destructive or irreversible actions" + +async function setup(opts: Record, history: unknown[]): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 60, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 1, + loopMaxContinues: 99, + injectIntervalMs: 0, + logFile, + debug: true, + ...opts, + }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, streamRef: stream, logFile, cleanup } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +describe("v2: the guardrail reaches the model, not just the source", () => { + test("the stall recovery continue carries the guardrail", async () => { + // Drives a real silence stall and reads the injected text. + const now = Date.now() + const h = await setup({}, [userMessage(now - 600_000), assistantWith(now - 300_000, "still working")]) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + const said = h.injected.map((i: { text?: string }) => i.text ?? "") + await teardown(h) + expect(said.length).toBeGreaterThan(0) + expect(said[0]).toContain("continue") + expect(said[0]).toContain(GUARD) + expect(said[0]).toContain(DESTRUCTIVE) + }) + + test("the stall continue says not to continue anything destructive", async () => { + // The specific worry: a fallible liveness signal plus an unbounded + // "continue" is how a model talks itself into a destructive action. + const now = Date.now() + const h = await setup({}, [userMessage(now - 600_000), assistantWith(now - 300_000, "working")]) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + const text = h.injected[0]?.text ?? "" + await teardown(h) + expect(text).toMatch(/without explicit user approval/) + expect(text.toLowerCase()).not.toContain("delete") + }) +}) \ No newline at end of file diff --git a/src/v2/index.continue-guardrail.test.ts b/src/v2/index.continue-guardrail.test.ts new file mode 100644 index 0000000..97337da --- /dev/null +++ b/src/v2/index.continue-guardrail.test.ts @@ -0,0 +1,74 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" + +/** + * Every synthetic continue this plugin sends must carry the "only continue work + * already in progress / nothing new / nothing destructive" guardrail unless an + * operator explicitly opted out. + * + * Why (2026-10-11): every liveness signal in the plugin proved fallible. One + * that fired on a subagent dead for 2.5 hours aborted and re-continued the + * parent 8 times over 3 hours, twice on top of a live user message. A bare + * "continue" gives the model nothing to stop at the edge of the work it was + * given — with four sites still sending it, the two nudges actually observed + * (abort escalation, empty-idle) were the least protected paths in the plugin. + * + * The stall path already carried the guardrail (recover()'s rich prompt). This + * closes the other four. + */ +const SOURCE = readFileSync(new URL("./index.ts", import.meta.url), "utf8") + +const GUARD = "Do not start anything new" +const GUARD_DESTRUCTIVE = "destructive or irreversible actions without explicit user approval" + +/** The four sites that previously defaulted to the bare string. */ +const sites = [ + { name: "abort+resume escalation", re: /injectOnce\(sid, opts\.continuePrompt \?\? (SAFE_CONTINUE_PROMPT|SAFE_EMPTY_IDLE_PROMPT|CONTINUE_PROMPT)/ }, + { name: "empty-idle nudge", re: /notifyAndPrompt\(sid, opts\.continuePrompt \?\? (SAFE_CONTINUE_PROMPT|SAFE_EMPTY_IDLE_PROMPT|CONTINUE_PROMPT)/ }, + { name: "ready-to-continue", re: /targetedRecovery\(sid, "ready-to-continue", opts\.continuePrompt \?\? (SAFE_CONTINUE_PROMPT|SAFE_EMPTY_IDLE_PROMPT|CONTINUE_PROMPT)/ }, + { name: "action-intent", re: /targetedRecovery\(sid, "action-intent", opts\.actionIntentPrompt \?\? opts\.continuePrompt \?\? (SAFE_CONTINUE_PROMPT|SAFE_EMPTY_IDLE_PROMPT|CONTINUE_PROMPT)/ }, +] + +describe("v2: every default synthetic continue carries the safety guardrail", () => { + test("no injection site defaults to the bare CONTINUE_PROMPT", () => { + for (const s of sites) { + const m = SOURCE.match(s.re) + expect(m, `site not found: ${s.name}`).not.toBeNull() + expect(m![1], `${s.name} still defaults to the bare string`).not.toBe("CONTINUE_PROMPT") + } + }) + + test("the guardrail text is the same wording the stall path already used", () => { + expect(SOURCE).toContain(GUARD) + expect(SOURCE).toContain(GUARD_DESTRUCTIVE) + // It must be the SAME sentence the rich stall prompt carries, so every + // continue this plugin emits reads consistently. + const rich = SOURCE.match(/return \(`continue — stalled[\s\S]*?\$\{suffix\}`\)/)?.[0] ?? "" + expect(rich).toContain(GUARD) + expect(rich).toContain(GUARD_DESTRUCTIVE) + }) + + test("an explicit richContinuePrompt:false still yields the bare string", () => { + // The opt-out must survive: an operator who sets richContinuePrompt:false + // asked for bare continues. + expect(SOURCE).toMatch(/if \(!richContinuePrompt\) return CONTINUE_PROMPT/) + }) + + test("a configured continuePrompt still wins verbatim", () => { + // All four sites read opts.continuePrompt first — an operator's own text + // must never be rewritten. + expect(SOURCE).toMatch(/opts\.continuePrompt \?\? SAFE_CONTINUE_PROMPT/) + expect(SOURCE).toMatch(/opts\.continuePrompt \?\? SAFE_EMPTY_IDLE_PROMPT/) + }) + + test("the empty-idle wording does not claim there is work in progress", () => { + // That nudge fires precisely because nothing was happening, so + // "continue work already in progress" would be incoherent. It gets the + // same guardrail phrased for the case. + const m = SOURCE.match(/const SAFE_EMPTY_IDLE_PROMPT =[\s\S]*?\n\n/)?.[0] ?? "" + expect(m).toContain("Otherwise ask the user what to do next") + expect(m).toContain(GUARD_DESTRUCTIVE) + }) +}) \ No newline at end of file diff --git a/src/v2/index.dead-stream.test.ts b/src/v2/index.dead-stream.test.ts new file mode 100644 index 0000000..c4bcb8a --- /dev/null +++ b/src/v2/index.dead-stream.test.ts @@ -0,0 +1,413 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexDeadStreamTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_deadstream" + +/** + * Silent dead stream. + * + * The model can end a turn having produced nothing the user can see: reasoning + * only, or a finish reason the provider did not describe. The turn looks + * complete, so no stall timer expires and no streaming failure fires — the + * session just goes quiet with the work unfinished. Nothing in v1's event + * stream catches it either; it takes reading the finished message itself. + * + * The rule, from v1: the newest assistant message that HAS a finish reason + * decides. If it carried text, the session answered and there is nothing to + * recover. If it did not, and it generated at least `silentDeadStreamMinTokens` + * output tokens, the stream died mid-response. + * + * The walk deliberately skips messages with no finish reason. An intermediate + * tool-call step has none, and stopping at it would report a dead stream for + * every session that used a tool — which is why "walk back past a delivered + * answer to an intermediate step" matters. + * + * Every group carries a control. + */ + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const userTurn = () => ({ + type: "user", + id: "msg_u0", + time: { created: Date.now() - 60 * 60_000 }, + content: [{ type: "text", text: "do the thing" }], +}) + +/** An assistant message that finished with a `finish` reason and `output` tokens. */ +const finished = (opts: { finish?: string; text?: string; output?: number; reasoning?: string }) => ({ + type: "assistant", + id: "msg_a1", + time: { created: Date.now() - 60_000 }, + ...(opts.text !== undefined + ? { content: [{ type: "text", text: opts.text }] } + : { content: [{ type: "reasoning", text: opts.reasoning ?? "thinking about the answer" }] }), + finish: opts.finish ?? "stop", + tokens: { input: 100, output: opts.output ?? 400, reasoning: 0, cache: { read: 0, write: 0 } }, +}) + +/** A finished turn that issued tool calls but no chatter text: active work. */ +const finishedWithTools = () => ({ + type: "assistant", + id: "msg_a3", + time: { created: Date.now() - 60_000 }, + content: [{ type: "tool", id: "call_9", name: "bash", state: { status: "completed" } }], + finish: "stop", + tokens: { input: 100, output: 400, reasoning: 0, cache: { read: 0, write: 0 } }, +}) + +/** An intermediate tool-call step: no finish reason, so the walk skips it. */ +const toolStep = () => ({ + type: "assistant", + id: "msg_a2", + time: { created: Date.now() - 30_000 }, + content: [{ type: "tool", id: "call_1", name: "bash", state: { status: "completed" } }], + tokens: { input: 200, output: 30, reasoning: 0, cache: { read: 0, write: 0 } }, +}) + +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + // Several assertions here are about a silent skip, which is exactly what the + // debug log is for — and in v2 that log goes to the file, not the console. + debug: true, +} + +type Harness = { injected: Array<{ text?: string }>; logs: string[] } + +async function replay( + messages: unknown[], + opts: Record = {}, + extra: { serverRunning?: boolean; preIdleEvents?: Array<{ type: string; data?: Record }> } = {}, +): Promise { + const injected: Harness["injected"] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-deadstream-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, ...opts, registryScope: LOG_SALT }, + session: { + context: async () => [userTurn(), ...messages], + // The server's own record of what is running, which is what the + // recovering-provider guard consults. + active: async () => (extra.serverRunning ? { [SID]: { sessionID: SID } } : {}), + interrupt: async () => ({}), + synthetic: async (a: any) => { + injected.push({ text: a?.text }) + return {} + }, + prompt: async (a: any) => { + injected.push({ text: a?.text }) + return {} + }, + }, + client: { session: { get: async () => ({ data: {} }) } }, + } + + const cleanup = await (plugin as any).setup(ctx) + + for (const e of [ + ev("session.execution.started"), + ev("session.step.started"), + ...(extra.preIdleEvents ?? []).map((p) => ev(p.type, p.data ?? {})), + ev("session.step.ended"), + ev("session.idle"), + ]) { + stream.push(e) + await wait(10) + } + await wait(700) + ;(cleanup as (() => void) | undefined)?.() + + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + rmSync(logFile, { force: true }) + return { injected, logs } +} + +/** Reasoning-only finish with plenty of output: the real dead stream. */ +const DEAD = [finished({ output: 400 })] + +describe("v2: silent dead stream", () => { + test("CONTROL: a finished message with no text and enough output tokens resumes", async () => { + const { injected, logs } = await replay(DEAD) + expect(injected).toHaveLength(1) + expect(logs.some((l) => l.includes("silent dead stream: finish=stop, 400 output tokens"))).toBe(true) + expect(logs.some((l) => l.includes("Silent dead stream (stop)"))).toBe(true) + }) + + test("a finished message that delivered text is a normal completion", async () => { + const { injected, logs } = await replay([finished({ text: "The change is in src/index.ts and tests pass.", output: 400 })]) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("silent dead stream"))).toBe(false) + }) + + test("below the token floor it is a short answer, not a dead stream", async () => { + // 40 output tokens with no text: the model simply had nothing to say. The + // floor exists so a one-word turn is not nudged forever. + const { injected, logs } = await replay([finished({ output: 40 })]) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("only 40 output tokens (floor 200)"))).toBe(true) + }) + + test("the floor is configurable", async () => { + const quiet = await replay([finished({ output: 40 })], { silentDeadStreamMinTokens: 200 }) + expect(quiet.injected).toEqual([]) + const loud = await replay([finished({ output: 40 })], { silentDeadStreamMinTokens: 10 }) + expect(loud.injected).toHaveLength(1) + }) + + test("the walk skips a tool-call step and judges the answer behind it", async () => { + // Newest is an intermediate step with no finish reason. The finished + // message behind it has text, so this session just used a tool. + const { injected } = await replay([finished({ text: "Ran the tests; 42 pass.", output: 400 }), toolStep()]) + expect(injected).toEqual([]) + }) + + test("a tool-call step in front of a dead stream is still found", async () => { + // The other direction: skipping the unfinished step must not hide the + // dead finish behind it. + const { injected } = await replay([finished({ output: 400 }), toolStep()]) + expect(injected).toHaveLength(1) + }) + + test("a finished turn carrying tool calls is working, not a dead stream", async () => { + // ses_efaec2f99ffexosULGDJJ8i6sA 2026-10-04: a thinking model doing tool + // work ends turns with finish=stop, hundreds of output tokens, and no + // text parts. Judging that "silent" fires a visible continue into + // active work. Tool calls are delivered work — not silence. + const { injected, logs } = await replay([finishedWithTools()]) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("silent dead stream"))).toBe(false) + }) + + test("tools still in flight veto the dead-stream inject", async () => { + // Belt and braces for the race the test above cannot see: the finished + // message predates the tool events, so the message looks dead while the + // calls it issued have not answered yet. + const { injected } = await replay([finished({ output: 400 })], {}, { + preIdleEvents: [{ type: "session.tool.called", data: { tool: "bash", id: "call_9" } }], + }) + expect(injected).toEqual([]) + }) + + test("a session the server still reports as running is left alone — this pass", async () => { + // A provider that is quietly retrying looks identical from the event + // stream. Injecting into that would turn a recovering session into a + // stalled one, which is the harm the guard prevents — and the event may + // not have arrived yet, so the guard asks the server. + // + // "This pass" is doing real work in that sentence. The server listing + // is weak evidence (its active map is known-stale), so declining must + // not bury the session: the next inspection re-reads the server fresh. + // The old code returned handled here, which ended the whole inspection + // and permanently buried real dead streams (2026-10-11). + const { injected, logs } = await replay(DEAD, {}, { serverRunning: true }) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("deferring, not burying"))).toBe(true) + }) + + test("declining on a stale server listing does not bury the remaining checks", async () => { + // The fall-through this pins: a dead stream where the server still + // lists the session, PLUS an unfinished tool call. Old code returned + // handled at the dead-stream guard, so the stuck-tool check never ran + // and nothing was ever injected. New code declines (false), the + // stuck-tool check runs, and the session gets its nudge. + const stuckPart = { + type: "assistant", + id: "msg_stuck", + time: { created: Date.now() - 120_000 }, + content: [{ type: "tool", id: "call_9", name: "compress", state: { status: "streaming" }, time: { created: Date.now() - 120_000 } }], + } + const { injected } = await replay([finished({ output: 400 }), stuckPart], {}, { serverRunning: true }) + expect(injected).toHaveLength(1) + expect(String((injected[0] as any).text ?? "")).toContain("never returned a result") + }) + + test("CONTROL for that guard: the same dead stream on an idle session does resume", async () => { + // Otherwise the guard could pass because the detector stopped working. + const { injected } = await replay(DEAD, {}, { serverRunning: false }) + expect(injected).toHaveLength(1) + }) + + test("no finished message at all is not a dead stream", async () => { + // Mid-first-turn: nothing has finished yet, so there is nothing to judge. + const { injected } = await replay([toolStep()]) + expect(injected).toEqual([]) + }) + + test("a user-cancelled session is never resumed", async () => { + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-deadstream-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const injected: unknown[] = [] + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [userTurn(), finished({ output: 400 })], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async () => ({}), + prompt: async () => (injected.push(1), {}), + }, + client: { session: { get: async () => ({ data: {} }) } }, + } + const cleanup = await (plugin as any).setup(ctx) + for (const e of [ + ev("session.execution.started"), + ev("session.execution.interrupted", { reason: "user" }), + ev("session.idle"), + ]) { + stream.push(e) + await wait(10) + } + await wait(600) + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + expect(injected).toEqual([]) + }) + + test("the recovery budget is finite", async () => { + // Two dead turns in a row must not queue unbounded injections. + const { injected } = await replay([finished({ output: 400 })], { maxRetries: 2 }) + expect(injected.length).toBeGreaterThan(0) + expect(injected.length).toBeLessThanOrEqual(2) + }) + + test("an idle turn that delivered nothing gets one instant nudge", async () => { + // The hole this closes: no finish (not a dead stream), no parts (not + // stuck), no text (nothing for the pattern checks) — every detector + // passes over it in silence and the session sits until the next event, + // which may never come. One quiet continue, immediately, no 180s wait. + const { injected, logs } = await replay([]) + expect(injected).toHaveLength(1) + expect(logs.some((l) => l.includes("instant nudge"))).toBe(true) + }) + + test("CONTROL: a turn that delivered text gets no instant nudge", async () => { + const { logs } = await replay([finished({ text: "The change is in src/index.ts and tests pass.", output: 400 })]) + expect(logs.some((l) => l.includes("instant nudge"))).toBe(false) + }) + + test("CONTROL: a user-cancelled empty turn is never nudged", async () => { + // The user's stop is final, even when the turn delivered nothing. + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-deadstream-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const injected: unknown[] = [] + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [userTurn()], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async () => ({}), + prompt: async () => (injected.push(1), {}), + }, + client: { session: { get: async () => ({ data: {} }) } }, + } + const cleanup = await (plugin as any).setup(ctx) + for (const e of [ + ev("session.execution.started"), + ev("session.execution.interrupted", { reason: "user" }), + ev("session.idle"), + ]) { + stream.push(e) + await wait(10) + } + await wait(600) + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + expect(injected).toEqual([]) + }) + + test("the instant nudge fires once per episode, not once per idle", async () => { + // Two idle transitions with no new turn between them: the second finds + // the flag set and stays quiet. A new turn (markBusy) re-arms it. + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-deadstream-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const injected: unknown[] = [] + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [userTurn()], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async () => ({}), + prompt: async () => (injected.push(1), {}), + }, + client: { session: { get: async () => ({ data: {} }) } }, + } + const cleanup = await (plugin as any).setup(ctx) + for (const e of [ + ev("session.execution.started"), + ev("session.step.started"), + ev("session.step.ended"), + ev("session.idle"), + ev("session.idle"), + ]) { + stream.push(e) + await wait(10) + } + await wait(700) + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + expect(injected).toHaveLength(1) + }) +}) \ No newline at end of file diff --git a/src/v2/index.delivery-once.test.ts b/src/v2/index.delivery-once.test.ts new file mode 100644 index 0000000..6298006 --- /dev/null +++ b/src/v2/index.delivery-once.test.ts @@ -0,0 +1,250 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexDeliveryOnceTest" + +/** + * v2: a send that throws after delivering must not send again. + * + * Observed 2026-10-06: twin identical `continue — stalled` prods 5ms apart + * in child ses_eed4521bdffeHLLmpomhiB0KXx. One sufficient mechanism needs no + * second instance at all: `ctx.session.prompt()` delivers, then throws + * (transport flakiness was in the log minutes earlier), and the catch falls + * through to `synthetic` — same text twice. After any throw, the plugin + * verifies delivery against the session log (our text already the newest + * user message) and only then falls back. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 4000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 300, + maxRetries: 1, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, + visibleContinue: true, +} + +const userMessage = (text: string, at: number) => ({ + type: "user", + id: "msg_u", + time: { created: at }, + content: [{ type: "text", text }], +}) + +const assistantText = (text: string, at: number) => ({ + type: "assistant", + id: "msg_a", + time: { created: at }, + content: [{ type: "text", text }], +}) + +async function setup(opts: { + deliverPrompt?: boolean + failPrompt?: boolean + failSynthetic?: boolean + logMessages?: unknown[] +}): Promise { + const sends: Array<{ channel: string; text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-delivery-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const logMessages: unknown[] = opts.logMessages ?? [] + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [], + active: async () => ({ [SID]: {} }), + interrupt: async () => ({}), + synthetic: async (a: any) => { + sends.push({ channel: "synthetic", text: a?.text }) + if (opts.failSynthetic) throw new Error("synthetic transport blew up after send") + return {} + }, + prompt: async (a: any) => { + // deliver-then-throw: the transport error lands after the + // message is already in the log, like the 19:59 transport + // retry storm around the observed twin. + if (opts.deliverPrompt !== false) { + sends.push({ channel: "prompt", text: a?.text }) + // Newest-first log: a delivery lands at the front. + logMessages.unshift({ + role: "user", + time: { created: Date.now() }, + content: [{ type: "text", text: a?.text }], + }) + } + if (opts.failPrompt) throw new Error("prompt transport blew up after send") + return {} + }, + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [] }), + message: { list: async () => logMessages }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { sends, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +async function makeBusyStalled(h: any) { + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.step.started", SID) + await wait(10) + await wait(900) +} + +describe("v2: a disposed instance sends nothing more", () => { + test("prompt rejects post-dispose: no synthetic fallback", async () => { + // The twin mechanism: timer fired and the prompt await is in flight + // when a newer setup disposes this one. The rejection must not + // trigger the fallback send — the live instance owns the session now. + let releasePrompt!: (err: unknown) => void + const promptGate = new Promise((_, reject) => { + releasePrompt = reject + }) + const sent: string[] = [] + const now = Date.now() + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-disposed-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, maxRetries: 1, registryScope: LOG_SALT }, + session: { + context: async () => [ + { type: "assistant", time: { created: now - 5_000 }, content: [{ type: "text", text: "working" }] }, + ], + active: async () => ({ [SID]: {} }), + interrupt: async () => ({}), + synthetic: async (a: any) => (sent.push(`synthetic:${a?.text}`), {}), + prompt: async (a: any) => { + sent.push(`prompt:${a?.text}`) + await promptGate + return {} + }, + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [] }), + message: { list: async () => [] }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + stream.push({ type: "session.execution.started", data: { sessionID: SID } }) + await wait(10) + stream.push({ type: "session.step.started", data: { sessionID: SID } }) + await wait(700) + ;(cleanup as () => void)() + releasePrompt(new Error("transport blew up")) + await wait(300) + expect(sent.filter((s) => s.startsWith("synthetic:"))).toHaveLength(0) + rmSync(logFile, { force: true }) + }) + + test("prompt delivers-then-throws: no synthetic fallback", async () => { + const now = Date.now() + const h = await setup({ + deliverPrompt: true, + failPrompt: true, + logMessages: [ + { role: "assistant", time: { created: now - 5_000 }, content: [{ type: "text", text: "working" }] }, + { role: "user", time: { created: now - 60_000 }, content: [{ type: "text", text: "go" }] }, + ], + }) + await makeBusyStalled(h) + expect(h.sends).toHaveLength(1) + expect(h.sends[0].channel).toBe("prompt") + await teardown(h) + }) + + test("CONTROL: prompt fails undelivered: synthetic fallback still fires", async () => { + const now = Date.now() + const h = await setup({ + deliverPrompt: false, + failPrompt: true, + logMessages: [ + { role: "assistant", time: { created: now - 5_000 }, content: [{ type: "text", text: "working" }] }, + { role: "user", time: { created: now - 60_000 }, content: [{ type: "text", text: "go" }] }, + ], + }) + await makeBusyStalled(h) + expect(h.sends).toHaveLength(1) + expect(h.sends[0].channel).toBe("synthetic") + await teardown(h) + }) +}) diff --git a/src/v2/index.discovery.test.ts b/src/v2/index.discovery.test.ts new file mode 100644 index 0000000..773150b --- /dev/null +++ b/src/v2/index.discovery.test.ts @@ -0,0 +1,250 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexDiscoveryTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +/** Scratch log path per case, so parallel tests cannot read each other's lines. */ +let counter = 0 +function scratchLog(): string { + const p = join(tmpdir(), `auto-resume-${LOG_SALT}-discovery-${process.pid}-${counter++}.log`) + rmSync(p, { force: true }) + return p +} +function readLogs(p: string): string[] { + const logs = existsSync(p) ? readFileSync(p, "utf8").split("\n") : [] + rmSync(p, { force: true }) + return logs +} + +/** + * Discovery sweep tests. + * + * The sweep is what makes a session that was ALREADY running when the plugin + * loaded visible to the watchdog. Without it, a session that started before + * attach never emits `session.execution.started` to us, so nothing is marked + * busy and the stall watchdog has nothing to inspect. + * + * Two shapes are exercised, because v2's plugin `session` domain is a narrowed + * Pick that may or may not carry `list`/`active`: + * - onDomain: the methods hang off ctx.session (what this build prefers) + * - onClient: they are absent there and only reachable via ctx.client + * + * Every group carries a CONTROL that fails if discovery silently no-ops, which + * is the failure mode this whole feature has. + */ + +type ApiShape = "onDomain" | "onClient" | "neither" + +type Harness = { + logs: string[] + calls: string[] + listRows: Array> + activeIDs: string[] +} + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +/** A session already mid-turn at attach time: `list` knows it, `active` says busy. */ +const RUNNING = "ses_running_before_attach" + +async function run( + shape: ApiShape, + opts: Record = {}, + overrides: { listThrows?: boolean } = {}, +): Promise { + const calls: string[] = [] + const stream = makeEventStream() + const logFile = scratchLog() + + const listRows = [{ id: RUNNING, title: "already running" }] + const activeIDs = [RUNNING] + + const list = async () => { + calls.push("list") + if (overrides.listThrows) throw new Error("list unavailable") + return listRows + } + const active = async () => { + calls.push("active") + return Object.fromEntries(activeIDs.map((id) => [id, { type: "running" }])) + } + + const session: Record = { + context: async () => [], + interrupt: async () => ({}), + synthetic: async () => ({}), + prompt: async () => ({}), + } + if (shape === "onDomain") { + session.list = list + session.active = active + } + + const client: Record | undefined = + shape === "onClient" ? { session: { list, active } } : shape === "onDomain" ? undefined : {} + + const ctx: any = { + event: stream, + options: { warmupMs: 0, discoveryDelayMs: 20, toolTextCheckDelayMs: 0, logFile, ...opts, registryScope: LOG_SALT }, + session, + } + if (client) ctx.client = client + + const cleanup = await (plugin as any).setup(ctx) + await wait(220) // initial sweep is discoveryDelayMs out + ;(cleanup as (() => void) | undefined)?.() + return { logs: readLogs(logFile), calls, listRows, activeIDs } +} + +const discoveryLine = (logs: string[]) => logs.find((l) => l.includes("discovery:")) + +describe("v2: session discovery sweep", () => { + test("CONTROL (list on the session domain): an already-running session is adopted as busy", async () => { + const { logs, calls } = await run("onDomain") + expect(calls).toContain("list") + expect(calls).toContain("active") + const line = discoveryLine(logs) + expect(line).toBeDefined() + expect(line).toContain("adopted 1 already running") + }) + + test("list on the session domain seeds a session the plugin has never seen", async () => { + const { logs } = await run("onDomain") + // Seeded and adopted are separate counts; a session that exists but is + // not running must still be seeded so revert/cleanup know about it. + expect(discoveryLine(logs)).toContain("seeded 1 session(s)") + }) + + test("CONTROL (list only on ctx.client): the sweep still works via the client fallback", async () => { + const { logs, calls } = await run("onClient") + expect(calls).toContain("list") + expect(discoveryLine(logs)).toContain("adopted 1 already running") + }) + + test("a host exposing neither degrades quietly instead of throwing", async () => { + // No `list`, no `client`. The sweep must not raise; the plugin falls back + // to the event-derived busy set, which is what it did before this feature. + const { logs, calls } = await run("neither") + expect(calls).toEqual([]) + expect(logs.filter((l) => l.includes("discovery failed"))).toEqual([]) + }) + + test("a throwing list() is contained and reported at debug level, not as a crash", async () => { + const { logs } = await run("onDomain", {}, { listThrows: true }) + expect(logs.filter((l) => l.includes("session discovery failed"))).toEqual([]) + // The sweep still completed — it simply had nothing to adopt. + expect(logs.filter((l) => l.includes("watchdog failed"))).toEqual([]) + }) + + test("rows without a usable id are skipped, not treated as sessions", async () => { + const logFile = scratchLog() + const stream = makeEventStream() + const rows = [{ id: "not-a-session" }, { id: 42 }, {}, null] + const ctx: any = { + event: stream, + options: { warmupMs: 0, discoveryDelayMs: 20, toolTextCheckDelayMs: 0, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [], + active: async () => ({}), + list: async () => rows, + }, + } + const cleanup = await (plugin as any).setup(ctx) + await wait(220) + ;(cleanup as (() => void) | undefined)?.() + const logs = readLogs(logFile) + expect(discoveryLine(logs)).not.toBeDefined() + expect(logs.filter((l) => l.includes("discovery failed"))).toEqual([]) + }) + + test("a { data } envelope is unwrapped, matching the client's response shape", async () => { + const logFile = scratchLog() + const stream = makeEventStream() + const ctx: any = { + event: stream, + options: { warmupMs: 0, discoveryDelayMs: 20, toolTextCheckDelayMs: 0, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [], + active: async () => ({ [RUNNING]: { type: "running" } }), + list: async () => ({ data: [{ id: RUNNING }] }), + }, + } + const cleanup = await (plugin as any).setup(ctx) + await wait(220) + ;(cleanup as (() => void) | undefined)?.() + expect(discoveryLine(readLogs(logFile))).toContain("adopted 1 already running") + }) + + test("cleanup stops every timer — no API calls after teardown", async () => { + const calls: string[] = [] + const stream = makeEventStream() + const logFile = scratchLog() + const ctx: any = { + event: stream, + // Tight watchdog so a leaked interval is visible inside the wait. + options: { warmupMs: 0, discoveryDelayMs: 20, checkIntervalMs: 20, chunkTimeoutMs: 10_000, toolTextCheckDelayMs: 0, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [], + active: async () => { + calls.push("active") + return {} + }, + list: async () => { + calls.push("list") + return [{ id: RUNNING }] + }, + }, + } + const cleanup = (await (plugin as any).setup(ctx)) as () => void + await wait(120) + // Precondition: the watchdog really is running, so a zero afterwards is + // the result of teardown rather than of a plugin that never started. + expect(calls.filter((c) => c === "active").length).toBeGreaterThan(0) + + cleanup() + const atTeardown = calls.length + await wait(200) + expect(calls.length).toBe(atTeardown) + rmSync(logFile, { force: true }) + }) +}) \ No newline at end of file diff --git a/src/v2/index.dup-progress.test.ts b/src/v2/index.dup-progress.test.ts new file mode 100644 index 0000000..1f5c1ef --- /dev/null +++ b/src/v2/index.dup-progress.test.ts @@ -0,0 +1,153 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexDupProgressTest" + +/** + * v2: silent tool completions defeat the exact-duplicate skip. + * + * The in-memory skip promises "identical text and zero model progress" but + * measured only assistant-text growth plus in-flight tools: a tool that + * RAN SILENTLY (completed, no text) left no trace, so the second identical + * prod was suppressed and the only recovery never fired. Tool completions + * (success/failed) and shell exits since the last prod now count as + * progress — the skip requires all three quiet. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 10000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +async function setup(extraEvents: Array<{ at: number; event: any }>, waitMs: number): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-dupprog-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 50, + gracePeriodMs: 0, + checkIntervalMs: 20, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 12, + // Open-ended lifetime budget: this test exercises the exact-duplicate + // skip gate, not the lifetime cap. With 20ms ticks the default cap + // of 9 burns in ~230ms — before the tool event below — and the + // session stands down no matter what the skip does (2026-10-15). + lifetimeMaxContinues: 99, + loopMaxContinues: 99, + injectIntervalMs: 0, + continuePrompt: "go", + logFile, + debug: false, + }, + session: { + context: async () => [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + const started = Date.now() + let extraIdx = 0 + stream.push(ev("session.execution.started")) + await wait(10) + while (Date.now() - started < waitMs) { + while (extraIdx < extraEvents.length && Date.now() - started >= extraEvents[extraIdx].at) { + stream.push(extraEvents[extraIdx].event) + extraIdx++ + } + await wait(10) + } + await wait(50) + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + return { injected } +} + +describe("v2: silent tool completions defeat the exact-duplicate skip", () => { + test("a tool that ran silently between prods re-arms the stall prod", async () => { + // Attempt 1 sends "go". A tool COMPLETES with no text — modelled as + // success-without-called (the start predates plugin load, so no + // in-flight window ever opens and the duplicate check is the only + // thing standing between the next stall and its prod). Pre-fix the + // next identical prod was suppressed as no-progress and recovery + // died after one shot; post-fix the completion counts as progress. + // Fired at 100ms: late enough to land mid-ladder (first prod ~60ms), + // early enough to beat the lifetime-cap burn (~230ms at 20ms ticks). + const { injected } = await setup( + [{ at: 100, event: ev("session.tool.success", { tool: "read", id: "c1" }) }], + 900, + ) + expect(injected.length).toBeGreaterThanOrEqual(2) + expect(injected.every((i: any) => i.text === "go")).toBe(true) + }) + + test("CONTROL: no intervening work stays suppressed", async () => { + // Nothing happens between attempts: identical text, zero progress — + // exactly one prod, as before. + const { injected } = await setup([], 900) + expect(injected).toHaveLength(1) + expect(injected[0].text).toBe("go") + }) +}) diff --git a/src/v2/index.duplicate-instance.test.ts b/src/v2/index.duplicate-instance.test.ts new file mode 100644 index 0000000..bc6d6a0 --- /dev/null +++ b/src/v2/index.duplicate-instance.test.ts @@ -0,0 +1,149 @@ +import { describe, test, expect } from "bun:test" +import { readFileSync, rmSync } from "node:fs" +import { existsSync } from "node:fs" +import { join } from "node:path" +import { tmpdir } from "node:os" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexDuplicateInstanceTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_dupinst" +let counter = 0 + +/** The real v2 seam: `event.subscribe` returns an async iterable. */ +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const FAST = { + chunkTimeoutMs: 50, + gracePeriodMs: 0, + checkIntervalMs: 20, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + loopMaxContinues: 99, + injectIntervalMs: 0, + visibleContinue: true, +} + +/** + * Boot the plugin `instances` times against ONE host ctx, exactly as the loader + * does — the live log shows six `ready` lines per server start, all from a + * single entrypoint and a single server process. Then let ONE stalled turn + * elapse and report every injection produced. + */ +async function bootStormed(instances: number, opts: Record = {}): Promise<{ injected: any[]; logs: string[] }> { + const injected: any[] = [] + const streams = Array.from({ length: instances }, () => makeEventStream()) + let next = 0 + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-dup-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + // ONE event source shared by every setup: `subscribe` hands out a fresh + // async iterable per caller, which is what the host does. + event: { + subscribe: ({ signal }: { signal?: AbortSignal } = {}) => { + const s = streams[next++] ?? streams[streams.length - 1] + signal?.addEventListener("abort", () => s.close()) + return s + }, + }, + options: { ...FAST, ...opts, logFile, debug: true, registryScope: LOG_SALT }, + session: { + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ kind: "synthetic", text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ kind: "prompt", text: a?.text }), {}), + }, + client: { + session: { + get: async () => ({ data: {} }), + message: { list: async () => ({ data: [], cursor: null }) }, + }, + }, + storage: { get: async () => ({ todos: [], updatedAt: Date.now() }), set: async () => {}, remove: async () => {} }, + } + + const cleanups: Array<(() => void) | undefined> = [] + for (let i = 0; i < instances; i++) cleanups.push(await (plugin as any).setup(ctx)) + + // ONE turn that starts and then goes silent. Every live instance subscribes to + // the same host stream, so all of them see it — that fan-out IS the storm. + streams.forEach((s) => s.push(ev("session.execution.started"))) + await wait(700) + for (const c of cleanups) (c as (() => void) | undefined)?.() + + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + rmSync(logFile, { force: true }) + return { injected, logs } +} + +describe("v2: N live setups must not become N identical injections", () => { + test("CONTROL: ONE setup on a stalled turn injects exactly once", async () => { + const { injected } = await bootStormed(1, { maxRetries: 1 }) + expect(injected.length).toBe(1) + }, 20_000) + + // The storm. setup() re-runs on every config reload; the log shows six `ready` + // lines with no `stopped` between them. Each setup built a private `sessions` + // map, so each owned a private resumeAttempts counter: six watchdogs, six + // "attempt 1/3" lines in the same millisecond, six identical continues with + // no wait between them. + test("SIX live setups -> ONE injection, not six", async () => { + const { injected } = await bootStormed(6, { maxRetries: 1 }) + expect(injected.length).toBe(1) + }, 25_000) + + // The behavioural test above is the real proof. These two pin the MECHANISM so + // the fix cannot be undone by a refactor that keeps the symptom away by luck. + test("MECHANISM: only the LAST ready is left running, and only it stalls", async () => { + const { logs } = await bootStormed(6, { maxRetries: 1 }) + const ready = logs.filter((l) => l.includes("ready (opencode v2)")).length + const stopped = logs.filter((l) => l.includes("[auto-resume] stopped")).length + const stalls = logs.filter((l) => l.includes("stall detected")).length + // Six setups still each announce themselves (the loader asked for six)... + expect(ready).toBe(6) + // ...but five are torn down on arrival, leaving one watchdog. + expect(stopped).toBeGreaterThanOrEqual(5) + // One live instance, therefore one counter, therefore one injection. + expect(stalls).toBe(1) + }, 25_000) +}) \ No newline at end of file diff --git a/src/v2/index.duplicate-prod.test.ts b/src/v2/index.duplicate-prod.test.ts new file mode 100644 index 0000000..74525df --- /dev/null +++ b/src/v2/index.duplicate-prod.test.ts @@ -0,0 +1,170 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexDuplicateProdTest" + +/** + * v2: cross-instance duplicate check (shared log). + * + * Ported from f99ddd8 (todo branch): stacked watchdogs (reload churn) each + * hold private counters, so the in-memory skip cannot see a sibling's prod — + * the session log can. If our exact text is already the newest user message + * within the recency window, stand down. Fail-open: fetch problems mean + * "no info", never "duplicate". + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_dup" +let counter = 2000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +type Harness = { + injected: Array<{ kind: string; text?: string }> + logs: string[] +} + +/** A turn that starts and then goes silent: a stall candidate. */ +const busyStallEvents = [ev("session.execution.started")] + +const FAST = { + chunkTimeoutMs: 50, + gracePeriodMs: 0, + checkIntervalMs: 20, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + loopMaxContinues: 99, + injectIntervalMs: 0, +} + +async function replay( + events: any[], + opts: Record = {}, + waitMs = 600, + recentMessages: unknown[] | undefined = undefined, +): Promise { + const injected: Harness["injected"] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-dup-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...FAST, ...opts, logFile, debug: true, registryScope: LOG_SALT }, + session: { + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ kind: "synthetic", text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ kind: "prompt", text: a?.text }), {}), + }, + client: { + session: { + get: async () => ({ data: {} }), + message: { + list: async () => ({ data: recentMessages ?? [], cursor: null }), + }, + }, + }, + storage: { + get: async () => ({ todos: [], updatedAt: Date.now() }), + set: async () => {}, + remove: async () => {}, + }, + } + + const cleanup = await (plugin as any).setup(ctx) + const started = Date.now() + for (const e of events) { + stream.push(e) + await wait(10) + } + while (Date.now() - started < waitMs) await wait(10) + await wait(50) + ;(cleanup as (() => void) | undefined)?.() + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + rmSync(logFile, { force: true }) + return { injected, logs } +} + +const userMsg = (text: string, at: number) => ({ + role: "user", + content: [{ type: "text", text }], + time: { created: at }, +}) + +describe("v2: cross-instance duplicate check (shared log)", () => { + test("a sibling's identical prod in the log suppresses ours", async () => { + // Two stacked watchdogs, one session: the first instance's prod is a + // user message in the log, so the second instance must stand down even + // though its private counters know nothing. + const { injected, logs } = await replay( + busyStallEvents, + { maxRetries: 5, continuePrompt: "go" }, + 900, + [userMsg("go", Date.now())], + ) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("identical prod already in session log"))).toBe(true) + }) + + test("a newer user message is progress, not a duplicate", async () => { + const { injected } = await replay( + busyStallEvents, + { maxRetries: 1, continuePrompt: "go" }, + 600, + [userMsg("actually, also this", Date.now())], + ) + expect(injected.length).toBeGreaterThan(0) + }) + + test("a stale identical prod does not suppress", async () => { + const { injected } = await replay( + busyStallEvents, + { maxRetries: 1, continuePrompt: "go" }, + 600, + [userMsg("go", Date.now() - 10 * 60_000)], + ) + expect(injected.length).toBeGreaterThan(0) + }) +}) diff --git a/src/v2/index.episode-backoff.test.ts b/src/v2/index.episode-backoff.test.ts new file mode 100644 index 0000000..74ba0b7 --- /dev/null +++ b/src/v2/index.episode-backoff.test.ts @@ -0,0 +1,256 @@ +import { describe, test, expect, setDefaultTimeout } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexEpisodeBackoffTest" + +/** + * v2: consecutive progress-restarted refires back the fuse off. + * + * ses_ed975114 (2026-10-10): "progress since last prod — restarting the + * stall ladder" + "resume attempt 1/3" every ~3.5min, four in a row. Any + * assistant-text growth or tool/shell success restarts the ladder, so a + * slow-trickling session reads 1/3 forever and the 3-per-10min loop budget + * (sliding window) never trips. Each refire without genuine user input now + * doubles the silence fuse (cap x8); a genuine user message resets it. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +/** + * Poll budgets sized for the environment, not for isolation: `bun test` runs + * 79 files in parallel and this file's events are watchdog-driven, so a + * condition that is microseconds away with every mock in place can still take + * seconds of wall clock under contention. A poll budget only costs time when + * the condition never arrives, which is a real failure and should fail. + */ +const POLL_MS = 10_000 +setDefaultTimeout(30_000) +const SID = "ses_parent" + +let counter = 9800 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 150, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 9, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, +} + +const userMsg = (text: string, at: number, id: string) => ({ + // Prod shape: role, not type. + role: "user", + id, + time: { created: at }, + content: [{ type: "text", text }], +}) +const assistantMsg = (text: string) => ({ + role: "assistant", + id: "msg_a", + time: { created: Date.now() - 60_000 }, + content: [{ type: "text", text }], +}) +const toolOk = () => ({ type: "session.tool.success", sid: SID, data: { sessionID: SID } }) + +async function setup(msgs: unknown[]) { + const injected: Array<{ sid?: string; text?: string; t?: number }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-episode-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => msgs, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text, t: Date.now() }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text, t: Date.now() }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { + get: async () => ({ todos: [] }), + set: async () => {}, + remove: async () => {}, + }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => [], + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +/** Poll until the predicate holds; resolves false on timeout. */ +async function waitFor(fn: () => boolean, timeoutMs = POLL_MS): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (fn()) return true + await wait(25) + } + return fn() +} + +describe("v2: refire backoff across one episode", () => { + test("fuse doubles per refire without user input", async () => { + const msgs: unknown[] = [userMsg("go", Date.now() - 60_000, "msg_u0"), assistantMsg("Working through the list.")] + const h = await setup(msgs) + push(h, "session.execution.started", SID) + await wait(60) + let lastFire = 0 + // One tool success immediately after each fire: that is the "progress" + // the ladder needs to restart (resumeAttempts -> 0, episodeRefires++) + // without ending the episode. PUSH-AFTER-FIRE is the whole point — the + // old fixed 900ms period was shorter than the 8x rung, so the push landed + // mid-rung and the late rungs could never be observed. + // + // The interval between consecutive fires IS the rung: markBusy resets + // lastActivityAt on injection and the push resets it again a moment + // later, so the next fire waits out silence >= chunkTimeout * fuseMult + // from here. Fires are watchdog-driven (the recover backoff timer is + // cleared as it fires), so a fire is never a straggler of the push. + const gaps: number[] = [] + const fires = () => h.injected.filter((x: any) => x.sid === SID) + for (let i = 0; i < 5; i++) { + const before = fires().length + await waitFor(() => fires().length > before) + const at = fires()[before].t as number + if (i > 0) gaps.push(at - lastFire) + lastFire = at + push(h, "session.tool.success", SID) + } + await teardown(h) + // The contract: each refire without genuine user input doubles the + // silence the next fire demands, so a slow-trickling session decays to + // patience instead of nagging on a fixed clock (ses_ed975114). With + // episodeRefires++ running inside recover() — after the threshold check + // — and a push after every fire, the ladder is 150 / 300 / 600 / 1200. + // + // Each gap is bounded against its OWN nominal rung, never against the + // previous measurement. Comparing samples (gaps[k] > gaps[k-1] * 1.5) + // re-introduced the scheduler luck this test was rewritten to remove: a + // single late watchdog tick inflates one sample and the next nominal one + // then fails the ratio (2026-10-11: 449 vs 621, expected > 673). + // Jitter is one-sided — a tick can only fire LATER than its threshold, + // never earlier — so a floor per rung is the sound assertion, and a fuse + // stuck at the base 150ms still fails rungs 2-4 outright. + expect(gaps.length).toBe(4) + for (let k = 0; k < gaps.length; k++) { + const rung = OPTIONS.chunkTimeoutMs * 2 ** k + expect(gaps[k]).toBeGreaterThanOrEqual(rung * 0.75) + } + }, 30000) + + test("genuine user message resets the fuse to base", async () => { const msgs: unknown[] = [userMsg("go", Date.now() - 60_000, "msg_u0"), assistantMsg("Working through the list.")] + const h = await setup(msgs) + push(h, "session.execution.started", SID) + await wait(60) + push(h, "session.tool.success", SID) + await wait(500) + push(h, "session.tool.success", SID) + await wait(700) + const before = h.injected.filter((x: any) => x.sid === SID).length + expect(before).toBeGreaterThanOrEqual(2) + // Forged +35s ahead so the own-prompt time guard (30s) cannot + // misread it: must register as a genuine takeover, clearing the + // episode count. + msgs.push(userMsg("new ask", Date.now() + 35_000, "msg_u9")) + push(h, "session.idle", SID) + await wait(200) + push(h, "session.execution.started", SID) + push(h, "session.step.started", SID) + await wait(60) + push(h, "session.tool.success", SID) + const t1 = Date.now() + await wait(900) + const after = h.injected.filter((x: any) => x.sid === SID && (x.t as number) >= t1) + await teardown(h) + // Base fuse again (~150ms), not the backed-off one. + expect(after.length).toBeGreaterThan(0) + expect((after[0].t as number) - t1).toBeLessThan(450) + }, 20000) + + test("every ladder reset is logged (no silent branches)", async () => { + const msgs: unknown[] = [userMsg("go", Date.now() - 60_000, "msg_u0"), assistantMsg("Working through the list.")] + const h = await setup(msgs) + push(h, "session.execution.started", SID) + await wait(60) + push(h, "session.tool.success", SID) + // Poll for the line instead of sleeping: the reset is logged when the + // NEXT stall is judged, which is watchdog-driven and takes at least + // one rung of silence. A fixed 1200ms assumed the watchdog always got + // scheduled promptly; under 79-file suite contention it does not, and + // the line simply had not been written yet (failed 2026-10-11). + const readLogs = () => (existsSync(h.logFile) ? readFileSync(h.logFile, "utf8") : "") + await waitFor(() => /restarting the stall ladder/.test(readLogs()), 10_000) + const logs = readLogs() + await teardown(h) + expect(logs).toMatch(/restarting the stall ladder/) + }, 20000) +}) diff --git a/src/v2/index.failure-loop.test.ts b/src/v2/index.failure-loop.test.ts new file mode 100644 index 0000000..06bb089 --- /dev/null +++ b/src/v2/index.failure-loop.test.ts @@ -0,0 +1,184 @@ +import { describe, test, expect } from "bun:test" +import { rmSync, readFileSync, existsSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexFailureLoopTest" + +/** + * v2: consecutive failure recoveries with no model output stand down. + * + * ses_ede801ca (2026-10-09): backend down, every injected continue produced + * the next failure — provider retries #2-11, step.failed, stall 1/3, retries, + * step.failed, stall 2/3… The ladder cycled 1/3, 2/3, 3/3, gaveUp, + * abort+resume, 1/3… forever, one user-visible prod per ~90s cycle, because + * every injection manufactures the failure the next one answers. Past three + * consecutive failure-driven recoveries with zero model output between them, + * the session stands down until a genuine turn or fresh output — and neither + * our own injections nor markBusy clear it, or the loop that set it would + * clear it too. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 20000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +const fail = { error: { type: "provider.internal", message: "upstream command exited prematurely" } } + +async function setup(): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-failloop-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + // High chunk timeout: the silence path must not interleave here. + // What this pins is failure stalls only; with a 50ms window every + // driven failure is followed by a silence stall and the counts + // mix two mechanisms. + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 9, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + }, + session: { + context: async () => [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { + injected, + cleanup, + logFile, + streamRef: stream, + logs: () => (existsSync(logFile) ? readFileSync(logFile, "utf8") : ""), + } +} + +async function teardown(h: any, keepLog = false) { + ;(h.cleanup as (() => void) | undefined)?.() + if (!keepLog) rmSync(h.logFile, { force: true }) +} + +async function driveFailure(h: any) { + push(h, "session.step.failed", SID, fail) + await wait(120) +} + +describe("v2: failure loops stand down", () => { + test("three consecutive failure recoveries, then silence with the latch logged", async () => { + const h = await setup() + push(h, "session.execution.started", SID) + await wait(20) + for (let i = 0; i < 6; i++) await driveFailure(h) + const n = h.injected.length + const log = h.logs() + await teardown(h) + // Three recoveries fire (bounded), then the latch stands down: three + // more failures produce nothing further. + expect(n).toBe(3) + expect(log).toContain("standing down until a genuine turn") + }) + + test("CONTROL: model output between failures keeps recovery alive", async () => { + const h = await setup() + push(h, "session.execution.started", SID) + await wait(20) + await driveFailure(h) + await driveFailure(h) + // Model speaks: the consecutive count resets. + push(h, "session.text.delta", SID, { assistantMessageID: "m1", delta: "back online, continuing" }) + await wait(50) + await driveFailure(h) + await driveFailure(h) + await driveFailure(h) + const n = h.injected.length + const log = h.logs() + await teardown(h) + // 2 before the text + 3 after = 5; the latch never fires mid-sequence. + expect(n).toBe(5) + expect(log).not.toContain("standing down until a genuine turn") + }) + + test("CONTROL: a genuine turn clears a latched loop", async () => { + const h = await setup() + push(h, "session.execution.started", SID) + await wait(20) + for (let i = 0; i < 4; i++) await driveFailure(h) + expect(h.logs()).toContain("standing down until a genuine turn") + const before = h.injected.length + // Fresh user turn (model changed / backend back): latch releases. + // Two starts: the first spends our own selfRecovery flag, the second + // reads as the genuine turn. + push(h, "session.execution.started", SID) + await wait(20) + push(h, "session.execution.started", SID) + await wait(20) + await driveFailure(h) + const n = h.injected.length + await teardown(h) + expect(n).toBeGreaterThan(before) + }) +}) diff --git a/src/v2/index.inject-lock.test.ts b/src/v2/index.inject-lock.test.ts new file mode 100644 index 0000000..287a718 --- /dev/null +++ b/src/v2/index.inject-lock.test.ts @@ -0,0 +1,133 @@ +import { describe, test, expect } from "bun:test" +import { readFileSync, rmSync } from "node:fs" +import { join } from "node:path" +import { tmpdir } from "node:os" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexInjectLockTest" + +const SOURCE = readFileSync(join(import.meta.dir, "index.ts"), "utf8") +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_lock" +let counter = 0 + +/** + * A private log file. Without `logFile` the plugin writes to its DEFAULT path, + * which is the LIVE server's log — test sessions then show up interleaved with + * real ones and the live log stops being usable as forensics. + */ +function privateLog(tag: string): string { + const f = join(tmpdir(), `auto-resume-${LOG_SALT}-${tag}-${process.pid}-${counter++}.log`) + rmSync(f, { force: true }) + return f +} + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const FAST = { + chunkTimeoutMs: 50, + gracePeriodMs: 0, + checkIntervalMs: 20, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + loopMaxContinues: 99, + injectIntervalMs: 0, + visibleContinue: true, +} + +/** + * A ctx whose `session.prompt` takes `slowMs` to resolve. Every caller that has + * passed its guards is then sitting in that await at the same time, which is + * exactly the window the mutex has to close. + */ +async function run(slowMs: number) { + const injected: any[] = [] + const stream = makeEventStream() + const logFile = privateLog("lock") + const ctx: any = { + event: stream, + options: { ...FAST, maxRetries: 1, debug: true, logFile, registryScope: LOG_SALT }, + session: { + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ kind: "synthetic", text: a?.text }), {}), + prompt: async (a: any) => { + // Record the send FIRST, then stall: the send is what must not repeat. + injected.push({ kind: "prompt", text: a?.text }) + await wait(slowMs) + return {} + }, + }, + client: { + session: { get: async () => ({ data: {} }), message: { list: async () => ({ data: [], cursor: null }) } }, + }, + storage: { get: async () => ({ todos: [], updatedAt: Date.now() }), set: async () => {}, remove: async () => {} }, + } + const cleanup = await (plugin as any).setup(ctx) + stream.push(ev("session.execution.started")) + await wait(800) + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + return injected +} + +describe("v2: injectOnce is serialized per session", () => { + test("CONTROL: one stalled turn still injects exactly once", async () => { + expect((await run(0)).length).toBe(1) + }, 20_000) + + // Same-ms volleys survived the cross-instance log check because it is + // check-then-act: `recentOwnProdInLog` read the log, then awaited, then sent. + // Six interleaved callers all read the same empty log. The mutex closes that + // window — it is why the duplicate guard alone was not enough. + test("a SLOW send must not let a concurrent caller through", async () => { + expect((await run(250)).length).toBe(1) + }, 25_000) + + test("structural: the mutex wraps injectOnce, and is per-session keyed", () => { + expect(SOURCE).toContain("const injectLocks = new Map>()") + // injectOnce must be a thin wrapper that takes the lock; the body must not. + expect(SOURCE).toContain("return withInjectLock(sid, () =>") + expect(SOURCE).toContain("async function injectOnceLocked(") + // The lock must be released in a finally, or one failed send wedges the session. + expect(SOURCE).toMatch(/finally \{[\s\S]{0,200}release\(\)/) + }) +}) \ No newline at end of file diff --git a/src/v2/index.ladder-delivery-gate.test.ts b/src/v2/index.ladder-delivery-gate.test.ts new file mode 100644 index 0000000..52d35d7 --- /dev/null +++ b/src/v2/index.ladder-delivery-gate.test.ts @@ -0,0 +1,177 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. */ +const LOG_SALT = "IndexLadderDeliveryGateTest" + +/** + * v2: a rung is only consumed by a prompt that actually lands. + * + * ses_eae46bb9 (2026-10-19): the session's first visible stall prod read + * "attempt 2/3" with no "1/3" ever delivered. The inject at 18:43:54 was armed + * as attempt 1 and then declined one second later ("still busy and live at + * inject time" — the provider's own retry had revived the session), but + * `recover()` had already incremented `resumeAttempts` at detection, and no + * path ever decrements it. The next stall therefore climbed to 2/3. Counting + * happens at detection, delivery happens in the timer: the two must agree. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 21000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +async function setup(extraOpts: Record = {}): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + // Parked: no silence stall may interleave with the failure stalls + // under test (every inject here is failure-driven by construction). + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + // Wide enough that the armed inject can be declined before it fires. + baseBackoffMs: 150, + maxBackoffMs: 300, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + ...extraOpts, + }, + session: { + context: async () => [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) +const readLog = (h: any) => (existsSync(h.logFile) ? readFileSync(h.logFile, "utf8") : "") + +async function waitFor(what: () => boolean, budgetMs = 3000): Promise { + const deadline = Date.now() + budgetMs + while (Date.now() < deadline) { + if (what()) return true + await wait(10) + } + return what() +} + +const attemptOf = (text: string | undefined): string | null => { + const m = /attempt (\d)\/3/.exec(text ?? "") + return m ? m[1] : null +} + +const FAIL = { error: { type: "provider.invalid-request", message: "Error from the provider: 400 Bad Request" } } + +describe("v2: a declined inject must not consume a ladder rung", () => { + test("the rung survives a busy/live decline — the next delivery still reads 1/3", async () => { + const h = await setup() + push(h, "session.execution.started", SID) + await wait(20) + // Stall #1: armed as attempt 1/3, and the session goes live again before + // the timer fires (the provider's own retry moving it, in the incident). + push(h, "session.step.failed", SID, FAIL) + await wait(50) + push(h, "session.step.started", SID, { agent: "build" }) + await waitFor(() => readLog(h).includes("still busy and live at inject time"), 2000) + const afterDecline = h.injected.length + // Stall #2: nothing else happened, so this must still be the FIRST rung. + push(h, "session.step.failed", SID, FAIL) + await waitFor(() => h.injected.length > afterDecline, 3000) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + const logs = readLog(h) + await teardown(h) + expect(afterDecline).toBe(0) // the declined fire delivered nothing + expect(attempts).toEqual(["1"]) // pre-fix this read ["2"]: the phantom rung + expect(logs).toContain("rung not consumed") + }) + + test("a delivered failure stall still climbs the ladder", async () => { + // Guard against over-correcting: delivery-gating must not flatten the + // ladder for stalls that really do get prodded. + const h = await setup() + push(h, "session.execution.started", SID) + await wait(20) + push(h, "session.step.failed", SID, FAIL) + await waitFor(() => h.injected.length >= 1) + push(h, "session.step.failed", SID, FAIL) + await waitFor(() => h.injected.length >= 2) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + await teardown(h) + expect(attempts.slice(0, 2)).toEqual(["1", "2"]) + }) +}) diff --git a/src/v2/index.latch-persist.test.ts b/src/v2/index.latch-persist.test.ts new file mode 100644 index 0000000..122fe1c --- /dev/null +++ b/src/v2/index.latch-persist.test.ts @@ -0,0 +1,386 @@ +import { describe, test, expect, setDefaultTimeout } from "bun:test" +import { rmSync, readFileSync, existsSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexLatchPersistTest" + +/** + * v2: durable latch state (R1, audit 2026-10-10 finding 6). + * + * Watch state lived only in the in-process SingletonRegistry, so every latch + * died on a real restart — while the comment justifying `permLocked`/`oocLocked` + * claimed they were "what stops a retry loop that spans restarts". They span + * nothing. Every restart bought another burst of retries against a condition + * that can never succeed, each one a user-visible prod. + * + * The quota ladder had the opposite problem: it DID survive, by grepping + * `quota-armed` markers out of the log file. A log is the worst available state + * channel — a 2MB rotation cap and a fixed 3000-line tail can drop a live arm. + * + * This suite pins the narrow fix: a small per-session record in ctx.storage + * carrying only what a restart must not forget. Not the whole watch — stall + * clocks are meaningless across a restart, and resurrecting them would re-open + * every timer question the audit just closed. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_latch" + +/** + * Poll budgets are sized for the environment, not for isolation. + * + * This suite runs while `bun test` runs 79 files in parallel, and under that + * contention a condition that resolves in microseconds with every mock in + * place can still take seconds of wall clock — the process is simply starved. + * Five separate tests here and in the sibling files failed this way on + * 2026-10-10/11, each on a different budget, all passing in isolation. + * + * A poll budget only ever costs time when the condition NEVER arrives, which + * is a real failure and should fail — slowly, and with the assertion saying + * so. So these are generous on purpose. + */ +const POLL_MS = 10_000 +setDefaultTimeout(30_000) + +let counter = 41000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const OPTS = { + chunkTimeoutMs: 50, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + rateLimitCooldownsMs: [120, 120], + debug: true, +} + +/** A host storage that survives a "restart". */ +function durableStorage(seed?: Map) { + const map = seed ?? new Map() + return { + map, + get: async (key: string) => map.get(key), + set: async (key: string, value: unknown) => { + map.set(key, value) + }, + remove: async (key: string) => { + map.delete(key) + }, + } +} + +/** A host storage that discards writes: models the pre-R1 behaviour. */ +function nullStorage() { + return { get: async () => undefined, set: async () => {}, remove: async () => {} } +} + +async function setup( + logFile: string, + history: unknown[] = [], + storage: any = durableStorage(), + opts: Record = {}, +): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + + const ctx: any = { + event: stream, + options: { ...OPTS, logFile, ...opts, registryScope: LOG_SALT }, + session: { + context: async () => history, + active: async () => ({}), + list: async () => [{ id: SID }], + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => [{ id: "bash" }], + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(80) + const startTurn = async () => { + for (const e of [ev("session.execution.started"), ev("session.step.started")]) { + stream.push(e) + await wait(10) + } + } + return { + injected, + cleanup, + startTurn, + /** A failure with no preceding genuine turn — what a mid-turn restart looks like. */ + bareFail: async (err: unknown) => { + stream.push(ev("session.step.failed", err as Record)) + await wait(150) + }, + failTurn: async (err: unknown) => { + await startTurn() + stream.push(ev("session.step.failed", err as Record)) + await wait(150) + }, + logs: () => (existsSync(logFile) ? readFileSync(logFile, "utf8") : ""), + } +} + +const freshLog = () => { + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-latch-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + return logFile +} + +/** + * Poll the log for a line instead of sleeping a fixed span. + * + * The suite runs 79 files in parallel and the whole run takes ~6.5 minutes, + * so a sleep long enough to be safe in isolation is a coin flip under load — + * which is how these tests came to be flaky (a fixed `wait(600)` here failed + * roughly one run in three, while passing every time on its own). + */ +async function waitForLog(h: any, needle: string, timeoutMs = POLL_MS): Promise { + const deadline = Date.now() + timeoutMs + let text = h.logs() + while (!text.includes(needle) && Date.now() < deadline) { + await wait(50) + text = h.logs() + } + return text +} + +/** Poll until the predicate holds; resolves true/false. */ +async function until(fn: () => boolean, timeoutMs = POLL_MS): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (fn()) return true + await wait(50) + } + return fn() +} + +const PERMANENT = { error: { type: "provider.auth", message: "invalid api key" } } +const QUOTA = { error: { type: "provider.quota", message: "Rate limit exceeded.", status: 429 } } + +describe("v2: durable quota state across a restart", () => { + test("locks need NO persistence: the first genuine execution clears them", async () => { + // R1 originally proposed persisting permLocked/oocLocked/ + // failureLoopLatched across restarts, on the theory that the code + // comment claiming they "stop a retry loop that spans restarts" was + // otherwise a lie. The CONTROL below disproved it: `clearLocks` and the + // `episodeRefires = 0` reset both run on genuine `execution.started`, + // so the first execution after boot wipes them before any failure can be + // evaluated. Persisting them would be write amplification for nothing. + // This test pins that invariant — if someone removes the reset, this + // fails and says so, and persistence becomes necessary again. + const logFile = freshLog() + const store = durableStorage() + const h = await setup(logFile, [], store) + await h.failTurn(PERMANENT) + await wait(200) + expect(h.logs()).toMatch(/permanent|OOC|latched/i) + // The next genuine execution is the release point. + await h.startTurn() + await wait(250) + const afterClear = h.logs() + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(afterClear).toContain("clearing") + }) + + test("a quota arm survives a restart with the log file gone", async () => { + const logFile = freshLog() + const store = durableStorage() + + const h1 = await setup(logFile, [], store) + await h1.failTurn(QUOTA) + expect(await waitForLog(h1, "quota-armed")).toContain("quota-armed") + ;(h1.cleanup as (() => void) | undefined)?.() + + // Destroy the log: the log-grep fallback cannot help, so this passes + // only if the ladder itself is in durable storage. + rmSync(logFile, { force: true }) + writeFileSync(logFile, "") + + // The second instance reads the record at `restoreQuotaLadders`, which + // drops any arm older than `horizon` = sum(ladder) — the point past which + // serving another rung could not help. With the file's default [120,120] + // that horizon is 240ms, and the arm expires DURING this test's own + // teardown + setup: it passed in isolation and failed under the suite's + // parallel load (one run in three). A ladder whose horizon is ~5s makes + // the restore the only possible outcome; the arm is real either way, and + // the rung it reaches is not what this test is about. + const h2 = await setup(logFile, [], store, { rateLimitCooldownsMs: [4000, 4000] }) + const log = await waitForLog(h2, "cooldown served") + ;(h2.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(log).toContain("cooldown served") + }, 30000) + + test("a genuine turn drops the durable arm", async () => { + // The point of moving the ladder out of the log: the record is deleted + // when the user speaks, so restore never needs to re-read history to + // answer "did they speak after the arm?" — the boot-time per-session + // HTTP read disappears with it. + // + // Asserted on the record rather than on end-to-end ordering: restore + // runs at setup, so "no restore" would depend on whether a genuine turn + // landed before or after it. The mechanism is the durable fact. + // + // Timing note (2026-10-11): the drop poll below exhausted its 10s + // ceiling twice at ~10.5s under the full 85-file serial suite while the + // mechanism proved sound in isolation (3/3), in 2/8/16/20-file subsets, + // and the drop chain is microtask-only (execution.started → reset → + // unawaited remove). Serial execution + disposed teardowns rule out + // cross-file pollution, so the ceiling — not the mechanism — is what + // broke. 30s keeps the failure mode (a record that never drops still + // fails) while surviving a loaded loop. + const logFile = freshLog() + const store = durableStorage() + const h1 = await setup(logFile, [], store) + await h1.failTurn(QUOTA) + const gotArmed = await until(() => ((store.map.get(`resume-quota/${SID}`) as any)?.at ?? 0) > 0) + ;(h1.cleanup as (() => void) | undefined)?.() + expect(gotArmed).toBe(true) + + const h2 = await setup(logFile, [], store) + await h2.startTurn() + const dropped = await until(() => { + const rec = store.map.get(`resume-quota/${SID}`) as any + return rec === undefined || rec?.at === 0 + }, 30_000) + ;(h2.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + // Genuine turn => the in-memory ladder reset must take the record with it. + expect(dropped).toBe(true) + }, 60000) + + test("restore never arms a session that is already busy", async () => { + // The boot race: a user who messaged before the plugin finished starting + // must not get a restored ladder served into their live turn. Status is + // already in memory, so this costs no history read. + // + // Determinism: the storage read is slowed so the turn is observably busy + // before restore reaches its check. Without the delay this races the + // restore, and a pass would prove nothing either way. + const logFile = freshLog() + const AT = Date.now() + const map = new Map([[`resume-quota/${SID}`, { v: 1, at: AT, rung: 0, noticed: AT }]]) + const slowStore = { + get: async (key: string) => { + await wait(300) + return map.get(key) + }, + set: async (key: string, value: unknown) => { + map.set(key, value) + }, + remove: async (key: string) => { + map.delete(key) + }, + } + + const h = await setup(logFile, [], slowStore) + // Mid-turn by the time restore gets there. + await h.startTurn() + await wait(1200) + const log = h.logs() + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + const restoredLine = log.split("\n").find((l: string) => l.includes("quota ladder restored")) ?? "" + expect(restoredLine).toBe("") + }, 20000) + + test("a record older than the horizon is ignored", async () => { + const logFile = freshLog() + const OLD = Date.now() - 48 * 3600_000 + const store = durableStorage( + new Map([[`resume-quota/${SID}`, { v: 1, at: OLD, rung: 0, noticed: OLD }]]), + ) + + const h = await setup(logFile, [], store) + await wait(400) + const log = h.logs() + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + // Two days stale: past any plausible ladder horizon, so not restored. + expect(log).not.toContain("quota ladder restored") + }, 20000) + + test("storage failures never block recovery", async () => { + const logFile = freshLog() + const hostile = { + get: async () => { + throw new Error("storage exploded") + }, + set: async () => { + throw new Error("storage exploded") + }, + remove: async () => { + throw new Error("storage exploded") + }, + } + + const h = await setup(logFile, [], hostile) + await h.failTurn(QUOTA) + await wait(250) + const recovered = h.logs().includes("cooldown served") || h.injected.length > 0 + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(recovered).toBe(true) + }) +}) \ No newline at end of file diff --git a/src/v2/index.offer-handoff.test.ts b/src/v2/index.offer-handoff.test.ts new file mode 100644 index 0000000..9ae88cd --- /dev/null +++ b/src/v2/index.offer-handoff.test.ts @@ -0,0 +1,169 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexOfferHandoffTest" + +/** + * v2: offer-shaped turns are hand-offs on the busy path too. + * + * ses_ed9e2ef9 (2026-10-10): "I can delete all the stopped containers on + * docknas if you want me to, just let me know." A nudge landing behind an + * offer reads as approval, and the model may execute a destructive action + * the user never approved. The idle path already skips hand-offs; the + * busy-stall watchdog judged only event silence, so a quiet-busy offer + * still fired. Now the busy path reads the history tail past the silence + * gate and stands down on hand-off text (without latching: no completion + * claimed, and the next turn re-arms by clearing). + * + * Deliberate boundary: a bare capability statement with no ask ("I can + * delete all the stopped containers.") is NOT a hand-off — ambiguous, and + * the scoped continue text is what protects those. The CONTROL pins that. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 9700 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 300, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, +} + +async function setup(text: string): Promise { + const injected: Array<{ sid?: string; text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-offer-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + // Prod shape: role, not type. + context: async () => [ + { + role: "assistant", + id: "msg_a", + time: { created: Date.now() - 60_000 }, + content: [{ type: "text", text }], + }, + ], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { + get: async () => ({ todos: [] }), + set: async () => {}, + remove: async () => {}, + }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => [], + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +async function quietBusy(text: string) { + const h = await setup(text) + push(h, "session.execution.started", SID) + await wait(1200) + const n = h.injected.filter((i: any) => i.sid === SID).length + await teardown(h) + return n +} + +describe("v2: offer-shaped turns stand down the busy stall", () => { + test("reported case stays quiet", async () => { + expect( + await quietBusy("I can delete all the stopped containers on docknas if you want me to, just let me know."), + ).toBe(0) + }) + + test("say-the-word stays quiet", async () => { + expect(await quietBusy("Say the word and I'll delete them.")).toBe(0) + }) + + test("go-ahead stays quiet", async () => { + expect(await quietBusy("Give the go-ahead and I'll proceed.")).toBe(0) + }) + + test("if-you-like stays quiet", async () => { + expect(await quietBusy("Happy to remove them if you'd like.")).toBe(0) + }) + + test("CONTROL: bare capability with no ask still recovers", async () => { + expect(await quietBusy("I can delete all the stopped containers.")).toBeGreaterThan(0) + }) +}) diff --git a/src/v2/index.ooc-throttle.test.ts b/src/v2/index.ooc-throttle.test.ts new file mode 100644 index 0000000..ce246f7 --- /dev/null +++ b/src/v2/index.ooc-throttle.test.ts @@ -0,0 +1,150 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexOocThrottleTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_ooc_throttle" + +/** + * Upstream issue #46: a provider rate-limit whose message contains "too many + * tokens" ("You have sent too many tokens to CodiX in a short period. Please + * wait a moment before retrying.") matched the bare `too many tokens` + * alternative in OOC_ERROR_RE, latched oocLocked, and halted all recovery. + * + * The rule: transient throttle phrasing routes to the rate-limit cooldown + * ladder (like quota), never to the OOC latch. Genuine context-overflow + * phrasing still latches (control). + */ + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const THROTTLE = { error: { type: "AI_APICallError", message: "You have sent too many tokens to CodiX in a short period. Please wait a moment before retrying." } } +const GENUINE_OOC = { error: { type: "ContextOverflowError", message: "This request exceeds the available context size" } } + +const userTurn = () => ({ + type: "user", + id: "msg_u0", + time: { created: Date.now() - 60 * 60_000 }, + content: [{ type: "text", text: "do the thing" }], +}) + +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + debug: true, +} + +type Harness = { injected: Array<{ text?: string; resume?: boolean; synth?: boolean }>; stream: any; cleanup?: () => void; logFile: string; failTurn: (err: unknown) => Promise } + +async function replay(opts: Record = {}): Promise { + const injected: Harness["injected"] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-ooc-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, ...opts, registryScope: LOG_SALT }, + session: { + context: async () => [userTurn()], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => { + injected.push({ text: a?.text, resume: a?.resume, synth: true }) + return {} + }, + prompt: async (a: any) => { + injected.push({ text: a?.text }) + return {} + }, + }, + client: { session: { get: async () => ({ data: {} }) } }, + } + + const cleanup = await (plugin as any).setup(ctx) + const failTurn = async (err: unknown) => { + for (const e of [ev("session.execution.started"), ev("session.step.started")]) { + stream.push(e) + await wait(10) + } + stream.push(ev("session.step.failed", err as Record)) + await wait(10) + } + return { injected, stream, cleanup, logFile, failTurn } +} + +function readLogs(h: Harness): string[] { + const logs = existsSync(h.logFile) ? readFileSync(h.logFile, "utf8").split("\n") : [] + rmSync(h.logFile, { force: true }) + return logs +} + +describe("v2: throttle phrasing never latches OOC (issue #46)", () => { + test("throttle 'too many tokens ... wait before retrying' stands down on the ladder, no OOC latch", async () => { + const h = await replay({ rateLimitCooldownsMs: [60_000] }) + await h.failTurn(THROTTLE) + await wait(300) + expect(h.injected.some((i) => /Rate limited, retrying in/.test(i.text ?? ""))).toBe(true) + const logs = readLogs(h) + expect(logs.some((l) => l.includes("OOC error latched"))).toBe(false) + h.cleanup?.() + }) + + test("CONTROL: genuine context overflow still latches OOC", async () => { + const h = await replay({ rateLimitCooldownsMs: [60_000] }) + await h.failTurn(GENUINE_OOC) + await wait(300) + const logs = readLogs(h) + expect(logs.some((l) => l.includes("OOC error latched"))).toBe(true) + h.cleanup?.() + }) +}) diff --git a/src/v2/index.open-shell.test.ts b/src/v2/index.open-shell.test.ts new file mode 100644 index 0000000..319bf2c --- /dev/null +++ b/src/v2/index.open-shell.test.ts @@ -0,0 +1,263 @@ +import { describe, test, expect } from "bun:test" +import { readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexOpenShellTest" + +let counter = 0 +/** Hygiene log files must not leak: every setup creates one (to keep the run + * out of the production log), so every setup deletes it before returning. The + * log is never read back in these files — only passed to the plugin. */ +function hygieneLogFile(): string { + const p = join(tmpdir(), `auto-resume-${LOG_SALT}-hygiene-${process.pid}-${counter++}.log`) + rmSync(p, { force: true }) + return p +} +const SOURCE = readFileSync(join(import.meta.dir, "index.ts"), "utf8") +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_openshell" +const OTHER_SID = "ses_someone_else" +/** Older than the 15m default activeUserWindowMs, so guard (b) cannot mask a failure. */ +const OLD_USER_MS = 30 * 60_000 +/** Trips `ready-to-continue` but NOT `isUserHandoff` (no "?", no "should I"). */ +const READY_TEXT = "Ready to continue with task" + +type Injected = { kind: string; text?: string; description?: string } + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const textPart = (t: string) => ({ type: "text", text: t }) + +const oldUserTurn = () => ({ + type: "user", + id: "msg_u0", + time: { created: Date.now() - OLD_USER_MS }, + content: [textPart("go ahead")], +}) + +function assistantTurn(text: string) { + return { type: "assistant", id: "msg_a1", time: { created: Date.now() - 60_000 }, content: [textPart(text)] } +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +// --- shell process-registry events ----------------------------------------- +// These deliberately mirror the real payload shape observed on the running +// server: `shell.created` carries NO top-level sessionID — it is nested at +// `data.info.metadata.sessionID` — and the exit events carry only the shell id. +// `shell.deleted` is given a different id family, as observed. + +const shellCreated = (id: string, sessionID: string = SID) => ({ + type: "shell.created", + data: { info: { id, command: "sleep 12", status: "running", metadata: { sessionID } } }, +}) +/** Built explicitly: a default parameter would silently substitute a valid id. */ +const shellCreatedRaw = (info: Record) => ({ type: "shell.created", data: { info } }) +const shellExited = (id: string) => ({ type: "shell.exited", data: { id, status: "exited", exit: 0 } }) +const shellDeleted = (id: string) => ({ type: "shell.deleted", data: { id } }) + +/** One assistant turn that streams `text`, then goes idle. */ +function turnEvents(text: string) { + return [ + ev("session.execution.started"), + ev("session.text.delta", { messageID: "msg_a1", delta: text }), + ev("session.text.ended", { messageID: "msg_a1" }), + ev("session.idle"), + ] +} + +/** Replay events and report whether the plugin injected a recovery. */ +async function replay(events: any[], messages?: any[]): Promise { + const injected: Injected[] = [] + const stream = makeEventStream() + const logFile = hygieneLogFile() + const ctx: any = { + event: stream, + app: { log: () => {} }, + options: { + registryScope: LOG_SALT, + toolTextCheckDelayMs: 0, + // Without this the test writes into the PRODUCTION log + // (~/.local/state/opencode-v2/auto-resume.log), which corrupts the + // reload forensics — a test run looks exactly like a reload storm. + logFile, + }, + session: { + context: async () => messages ?? [oldUserTurn(), assistantTurn(READY_TEXT)], + // Empty: no other session is active, so the `lastWasTaskTool` branch + // (which needs `others.length > 0`) cannot mask any failure below. + active: async () => ({}), + synthetic: async (a: any) => { + injected.push({ kind: "synthetic", ...a }) + return {} + }, + prompt: async (a: any) => { + injected.push({ kind: "prompt", ...a }) + return {} + }, + }, + } + const cleanup = await (plugin as any).setup(ctx) + for (const e of events) { + stream.push(e) + await wait(10) + } + await wait(500) // handleEvent is sync; its work is async + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + return injected +} + +// ============================================================================ +// BEHAVIOURAL +// ============================================================================ + +describe("v2: a session running a shell is working, not idle", () => { + test("CONTROL: no shell involved -> recovery fires (otherwise the rest proves nothing)", async () => { + const injected = await replay(turnEvents(READY_TEXT)) + expect(injected.length).toBeGreaterThan(0) + }) + + test("shell open -> no injection (parent parked on a background job)", async () => { + const injected = await replay([shellCreated("sh_aaa"), ...turnEvents(READY_TEXT)]) + expect(injected).toEqual([]) + }) + + test("shell open, then exited -> recovery may fire", async () => { + const injected = await replay([shellCreated("sh_aaa"), shellExited("sh_aaa"), ...turnEvents(READY_TEXT)]) + expect(injected.length).toBeGreaterThan(0) + }) + + test("two shells open, one exits -> still suppressed", async () => { + const injected = await replay([ + shellCreated("sh_aaa"), + shellCreated("sh_bbb"), + shellExited("sh_aaa"), + ...turnEvents(READY_TEXT), + ]) + expect(injected).toEqual([]) + }) + + test("another session's open shell does not suppress us", async () => { + const injected = await replay([shellCreated("sh_zzz", OTHER_SID), ...turnEvents(READY_TEXT)]) + expect(injected.length).toBeGreaterThan(0) + }) + + // A phantom entry from any of these would suppress recovery forever. + for (const [label, info] of [ + ["metadata present but sessionID missing", { id: "sh_bad", metadata: {} }], + ["metadata absent entirely", { id: "sh_bad" }], + ["sessionID is not a string", { id: "sh_bad", metadata: { sessionID: 42 } }], + ["no id", { metadata: { sessionID: SID } }], + ["info is a bare string", "not-an-object"], + ] as Array<[string, unknown]>) { + test(`malformed shell.created (${label}) is ignored, not recorded`, async () => { + const injected = await replay([shellCreatedRaw(info as Record), ...turnEvents(READY_TEXT)]) + expect(injected.length).toBeGreaterThan(0) + }) + } + + test("shell.exited for an unknown id does not throw or suppress", async () => { + const injected = await replay([shellExited("sh_never_seen"), ...turnEvents(READY_TEXT)]) + expect(injected.length).toBeGreaterThan(0) + }) + + test("shell.deleted (different id family) cannot close an open shell", async () => { + // Documents the reason `shell.deleted` is not wired up: its id never + // matches the one recorded at create, so it is not a usable exit signal. + const injected = await replay([shellCreated("sh_aaa"), shellDeleted("sh_bbb"), ...turnEvents(READY_TEXT)]) + expect(injected).toEqual([]) + }) +}) + +// ============================================================================ +// CONTRACT — fail deterministically if someone reverts the fix. +// ============================================================================ + +describe("v2: contract assertions on source", () => { + test("FIX: the registry family is handled, not just the session-scoped one", () => { + expect(SOURCE).toMatch(/case "shell\.created"/) + expect(SOURCE).toMatch(/case "shell\.exited"/) + // The session-scoped family is also correct and must be kept. + expect(SOURCE).toMatch(/case "session\.shell\.started"/) + expect(SOURCE).toMatch(/case "session\.shell\.ended"/) + }) + + test("FIX: the session is read from the nested path the runtime actually uses", () => { + expect(SOURCE).toMatch(/case "shell\.created"[\s\S]{0,600}metadata\?\.sessionID/) + }) + + test("FIX: both injection paths are gated, not just the stall watchdog", () => { + // The proactive nudge fires on session.idle, long before the watchdog + // would look, so gating only the watchdog would not prevent the nag. + const gate = /openShellCount\(sid\)[^\n]*\n\s*if \(busyShells > 0\)/ + const all = SOURCE.match(new RegExp(gate, "g")) ?? [] + expect(all.length).toBe(2) + // ...one in the targeted-nudge guard chain, one in the stall watchdog. + const targeted = SOURCE.indexOf("async function targetedRecovery") + const watchdog = SOURCE.indexOf("async function checkActiveSessions") + expect(targeted).toBeGreaterThan(-1) + expect(watchdog).toBeGreaterThan(targeted) + const targetedBody = SOURCE.slice(targeted, watchdog) + expect(targetedBody).toMatch(/openShellCount\(sid\)/) + }) + + test("FIX: a dropped exit event cannot suppress recovery forever", () => { + expect(SOURCE).toMatch(/SHELL_OPEN_MAX_MS/) + expect(SOURCE).toMatch(/now - s\.startedAt > SHELL_OPEN_MAX_MS/) + }) + + test("FIX: cleanup cannot delete the watch state of a session running a shell", () => { + const start = SOURCE.indexOf("function cleanupIdleSessions") + expect(start).toBeGreaterThan(-1) + const end = SOURCE.indexOf("function ", start + 10) + const body = SOURCE.slice(start, end > start ? end : start + 2000) + expect(body).toMatch(/openShellCount\(sid\) > 0\) continue/) + }) + + test("FIX: every sessions.delete drops that session's shell entries", () => { + const deletes = [...SOURCE.matchAll(/^\s*sessions\.delete\(sid\)$/gm)] + expect(deletes.length).toBeGreaterThan(0) + for (const d of deletes) { + const before = SOURCE.slice(Math.max(0, d.index! - 120), d.index!) + expect(before).toMatch(/forgetShells\(sid\)/) + } + }) +}) diff --git a/src/v2/index.options.test.ts b/src/v2/index.options.test.ts new file mode 100644 index 0000000..fef2ff2 --- /dev/null +++ b/src/v2/index.options.test.ts @@ -0,0 +1,483 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexOptionsTest" + +const SOURCE = readFileSync(join(import.meta.dir, "index.ts"), "utf8") +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_opts" + +/** Scratch log path per replay, so parallel cases cannot read each other's lines. */ +let counter = 0 + +const textPart = (t: string) => ({ type: "text", text: t }) +/** Older than every activeUserWindowMs we pass, so guard (b) cannot mask a failure. */ +const oldUserTurn = () => ({ + type: "user", + id: "msg_u0", + time: { created: Date.now() - 60 * 60_000 }, + content: [textPart("go ahead")], +}) +const assistantTurn = (text: string) => ({ + type: "assistant", + id: "msg_a1", + time: { created: Date.now() - 60_000 }, + content: [textPart(text)], +}) + +/** Trips `ready-to-continue` but not `isUserHandoff` (no "?", no "should I"). */ +const READY_TEXT = "Ready to continue with task" +/** + * Ends in ":" so `containsActionIntent` is true. Must not trip the + * ready-to-continue patterns and must not read as a user hand-off. + */ +const INTENT_TEXT = "I will run the migration now:" + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +type Harness = { + injected: Array<{ kind: string; text?: string }> + interrupts: string[] + logs: string[] +} + +/** + * Replay `events` and report what the plugin did. + * `sendFails` makes every delivery attempt throw, which is how the + * "a rejected send must not burn the retry budget" case is exercised. + */ +async function replay( + events: any[], + opts: Record = {}, + harnessOpts: { sendFails?: boolean; text?: string } = {}, + /** Index of an event that must wait for the first nudge to land before it + * is pushed. Back-to-back idles with a 0ms pattern delay rely on the first + * 0ms timer firing inside the 10ms inter-event gap; under parallel-suite + * load it does not, and the second idle's schedulePatternPass then + * *replaces* the first pass (by design — see schedulePatternPass) instead + * of following it, so only one nudge is ever logged. Gating on the observed + * line removes the wall-clock race. Times out silently after 10s: the + * test's own assertion then reports the miss with the natural evidence. */ + waitForNudgeBeforeEvent?: number, +): Promise { + const injected: Harness["injected"] = [] + const interrupts: string[] = [] + const stream = makeEventStream() + const text = harnessOpts.text ?? READY_TEXT + + // v2 removed v1's server log endpoint, so the plugin writes to a file. Point + // it at a scratch path and read the lines back after teardown — this is also + // what proves the file sink works, rather than assuming it does. + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-test-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + // The plugin reads its config from `ctx.options`; the second argument to + // `setup()` is ignored. Passing options the other way makes every + // negative assertion below pass for the wrong reason. + // Before the spread, so it applies to every test here unless one asks for a + // different value. The deferred pattern pass defaults to 3s, which is not what + // these tests measure — the nudge they assert on would land after the wait. + options: { toolTextCheckDelayMs: 0, ...opts, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [oldUserTurn(), assistantTurn(text)], + // Empty: no other session is active, so the `lastWasTaskTool` branch + // (which needs `others.length > 0`) cannot mask a failure below. + active: async () => ({}), + interrupt: async (a: any) => { + interrupts.push(a?.sessionID) + return {} + }, + synthetic: async (a: any) => { + if (harnessOpts.sendFails) throw new Error("synthetic rejected") + injected.push({ kind: "synthetic", text: a?.text }) + return {} + }, + prompt: async (a: any) => { + if (harnessOpts.sendFails) throw new Error("prompt rejected") + injected.push({ kind: "prompt", text: a?.text }) + return {} + }, + }, + } + +/** Poll the scratch log until `needle` appears (see `waitForNudgeBeforeEvent`). */ +async function waitForLogLine(path: string, needle: string, timeoutMs: number): Promise { + const start = Date.now() + while (Date.now() - start < timeoutMs) { + try { + if (readFileSync(path, "utf8").includes(needle)) return + } catch { + // Not written yet — the plugin creates it on first log line. + } + await wait(20) + } +} + + const cleanup = await (plugin as any).setup(ctx) + for (let i = 0; i < events.length; i++) { + stream.push(events[i]) + if (i === waitForNudgeBeforeEvent) await waitForLogLine(logFile, "ready-to-continue detected", 10_000) + else await wait(10) + } + await wait(600) // handleEvent is sync; its work is async + ;(cleanup as (() => void) | undefined)?.() + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + rmSync(logFile, { force: true }) + return { injected, interrupts, logs } +} + +/** A turn that streams `text` and then goes idle. */ +function turnEvents(text = READY_TEXT) { + return [ + ev("session.execution.started"), + ev("session.text.delta", { messageID: "msg_a1", delta: text }), + ev("session.text.ended", { messageID: "msg_a1" }), + ev("session.idle"), + ] +} + +/** A turn that starts, streams nothing, and never idles: a stall candidate. */ +const busyStallEvents = [ev("session.execution.started")] + +/** Timings small enough that the watchdog fires inside the harness wait. */ +const FAST = { chunkTimeoutMs: 50, gracePeriodMs: 0, checkIntervalMs: 20, warmupMs: 0, baseBackoffMs: 1, maxBackoffMs: 2 } + +// ============================================================================ +// busyStallStrategy +// ============================================================================ + +describe("v2: busyStallStrategy", () => { + test("CONTROL: default 'continue' -> a stalled busy session gets a continue", async () => { + const { injected } = await replay(busyStallEvents, { ...FAST, loopMaxContinues: 99, injectIntervalMs: 0 }) + expect(injected.length).toBeGreaterThan(0) + }) + + test("'off' -> the stall is ignored and nothing is injected", async () => { + const { injected, interrupts } = await replay(busyStallEvents, { + ...FAST, + busyStallStrategy: "off", + loopMaxContinues: 99, + injectIntervalMs: 0, + }) + expect(injected).toEqual([]) + expect(interrupts).toEqual([]) + }) + + test("'abort' -> the step is interrupted before the continue is sent", async () => { + const { interrupts } = await replay(busyStallEvents, { + ...FAST, + busyStallStrategy: "abort", + loopMaxContinues: 99, + injectIntervalMs: 0, + }) + expect(interrupts).toEqual([SID]) + }) + + test("'continue' (explicit) does not interrupt", async () => { + const { interrupts } = await replay(busyStallEvents, { + ...FAST, + busyStallStrategy: "continue", + loopMaxContinues: 99, + injectIntervalMs: 0, + }) + expect(interrupts).toEqual([]) + }) +}) + +// ============================================================================ +// warmupMs +// ============================================================================ + +describe("v2: warmupMs", () => { + test("CONTROL: warmup 0 -> the stall is acted on", async () => { + const { injected } = await replay(busyStallEvents, { ...FAST, loopMaxContinues: 99, injectIntervalMs: 0 }) + expect(injected.length).toBeGreaterThan(0) + }) + + test("a session still inside its warmup window is not treated as stalled", async () => { + const { injected } = await replay(busyStallEvents, { + ...FAST, + warmupMs: 10 * 60_000, + loopMaxContinues: 99, + injectIntervalMs: 0, + }) + expect(injected).toEqual([]) + }) +}) + +// ============================================================================ +// maxRecoveryRetries alias +// ============================================================================ + +describe("v2: maxRecoveryRetries is v1's name for maxRetries", () => { + test("CONTROL: maxRetries 3 -> recovery fires", async () => { + const { injected } = await replay(busyStallEvents, { ...FAST, maxRetries: 3, loopMaxContinues: 99, injectIntervalMs: 0 }) + expect(injected.length).toBeGreaterThan(0) + }) + + test("maxRetries 0 -> the session gives up instead of recovering", async () => { + const { injected } = await replay(busyStallEvents, { ...FAST, maxRetries: 0, loopMaxContinues: 99, injectIntervalMs: 0 }) + expect(injected).toEqual([]) + }) + + // If the alias were ignored, maxRetries would fall back to 3 and this would inject. + test("maxRecoveryRetries 0 is honoured as the same knob", async () => { + const { injected } = await replay(busyStallEvents, { + ...FAST, + maxRecoveryRetries: 0, + loopMaxContinues: 99, + injectIntervalMs: 0, + }) + expect(injected).toEqual([]) + }) +}) + +// ============================================================================ +// resumeOnActionIntent +// ============================================================================ + +describe("v2: resumeOnActionIntent", () => { + test("CONTROL: default true -> an action-intent turn is nudged", async () => { + const { injected } = await replay(turnEvents(INTENT_TEXT), { injectIntervalMs: 0 }) + expect(injected.length).toBeGreaterThan(0) + }) + + test("false -> an action-intent turn is left alone", async () => { + const { injected } = await replay(turnEvents(INTENT_TEXT), { + resumeOnActionIntent: false, + injectIntervalMs: 0, + }) + expect(injected).toEqual([]) + }) +}) + +// ============================================================================ +// Unrecognised / accepted-but-inert option reporting +// ============================================================================ + +describe("v2: option reporting at startup", () => { + test("an unknown option key produces exactly one warning naming it", async () => { + const { logs } = await replay([], { chunkTimeoutMs: 5000, definitelyNotAnOption: true }) + const warnings = logs.filter((l) => l.includes("unrecognised")) + expect(warnings).toHaveLength(1) + expect(warnings[0]).toContain("definitelyNotAnOption") + }) + + test("known v1 options do not warn, even the ones v2 does not act on", async () => { + const { logs } = await replay([], { + chunkTimeoutMs: 5000, + maxRecoveryRetries: 2, + subagentWaitMs: 15_000, + discoveryDelayMs: 5_000, + contextSaturationThreshold: 0.85, + silentDeadStreamMinTokens: 200, + subagentNativeCompactionEnabled: false, + toolTextCheckDelayMs: 3_000, + thinkingToolRecoveryPrompt: "x", + doneWithoutWorkPrompt: "y", + }) + expect(logs.filter((l) => l.includes("unrecognised"))).toEqual([]) + }) + + test("the startup line names no inert options, because none are inert", async () => { + // Every v1 option this build understands is now applied, so the inert list has + // nothing to put in it — including for the three options that were inert when + // this branch started. A config carrying them must not be told they are being + // ignored: that note is the only way a user finds out. + const { logs } = await replay([], { + chunkTimeoutMs: 5000, + subagentWaitMs: 15_000, + toolTextCheckDelayMs: 3000, + discoveryDelayMs: 5_000, + }) + const ready = logs.filter((l) => l.includes("ready (opencode v2)")) + expect(ready).toHaveLength(1) + expect(ready[0]).not.toContain("accepted-but-inert=") + expect(ready[0]).not.toContain("subagentWaitMs") + expect(ready[0]).not.toContain("toolTextCheckDelayMs") + expect(ready[0]).not.toContain("discoveryDelayMs") + }) + + test("CONTROL: nothing is listed as feature-gated in the source", async () => { + // The structural counterpart to the assertion above, and the one that actually + // holds the line: adding an option to RECOGNISED_OPTIONS without implementing it + // is exactly how this build went from "several options inert" to "none", and it + // is the failure the list exists to make visible. With the list empty there is + // no behavioural test left for it — no production path reaches it — so it is + // asserted on the source instead. + const block = SOURCE.match(/const FEATURE_GATED_OPTIONS = \[([^\]]*)\]/) + expect(block).not.toBeNull() + const gated = [...(block![1].match(/"([a-zA-Z][a-zA-Z0-9]*)"/g) ?? [])].map((x) => x.replace(/"/g, "")) + expect(gated).toEqual([]) + }) + + test("the inert reporting is kept, so the next gap has somewhere to be listed", async () => { + // Deliberately not deleted when the last entry left the list. The reporting is + // the only signal a user gets that an option is being ignored, and the next + // unported feature needs it more than this build does. + expect(SOURCE).toContain("FEATURE_GATED_OPTIONS") + expect(SOURCE).toContain("accepted-but-inert=") + expect(SOURCE).toContain("gatedInUse") + }) + + test("the startup line carries no accepted-but-inert list when none are set", async () => { + const { logs } = await replay([], { chunkTimeoutMs: 5000 }) + const ready = logs.filter((l) => l.includes("ready (opencode v2)")) + expect(ready).toHaveLength(1) + expect(ready[0]).not.toContain("accepted-but-inert=") + }) +}) + +// ============================================================================ +// e1b8374: a rejected send must not burn the retry budget +// ============================================================================ + +describe("v2: a nudge that was never delivered does not consume a retry", () => { + // Two idles inside one turn: no `execution.started` between them, so + // markBusy does not reset the per-turn budget and the second attempt is + // genuinely attempt #2 if the failed send was counted. + const twoIdles = [ + ev("session.execution.started"), + ev("session.text.delta", { messageID: "msg_a1", delta: READY_TEXT }), + ev("session.text.ended", { messageID: "msg_a1" }), + ev("session.idle"), + ev("session.idle"), + ] + + test("CONTROL: both sends succeed -> each nudge is 1/3, because markBusy resets the per-turn budget", async () => { + const { logs } = await replay(twoIdles, { maxRetries: 3, loopMaxContinues: 99, injectIntervalMs: 0 }, {}, 3) + const nudges = logs.filter((l) => l.includes("ready-to-continue detected")) + expect(nudges.length).toBeGreaterThanOrEqual(2) + // This is what makes the test below meaningful: on the success path the + // counter is reset by markBusy, so the number the user sees does not + // discriminate. On the failure path it does. + for (const n of nudges) expect(n).toContain("(1/3)") + }) + + test("both sends are rejected -> the budget is untouched, so it is still 1/3", async () => { + const { logs, injected } = await replay( + twoIdles, + { maxRetries: 3, loopMaxContinues: 99, injectIntervalMs: 0 }, + { sendFails: true }, + 3, + ) + expect(injected).toEqual([]) + const nudges = logs.filter((l) => l.includes("ready-to-continue detected")) + expect(nudges.length).toBeGreaterThanOrEqual(2) + // Nothing was delivered, so nothing was spent. The pre-e1b8374 ordering + // incremented before the send and reported 1/3 then 2/3 here. + for (const n of nudges) expect(n).toContain("(1/3)") + expect(nudges.some((l) => l.includes("(2/3)"))).toBe(false) + }) + + test("the budget write is inside the success branch, not before the send", () => { + // Guards the exact reordering e1b8374 makes, so a well-meaning "tidy up" + // that moves the increment back above injectOnce fails here. + expect(SOURCE).toMatch(/const attemptNum = w\[budgetKey\] \+ 1[\s\S]{0,400}w\[budgetKey\] = attemptNum/) + const body = SOURCE.match(/async function targetedRecovery[\s\S]*?\n\t\t\}/)?.[0] ?? "" + const incr = body.indexOf("w[budgetKey]") + const send = body.indexOf("injectOnce(") + expect(incr).toBeGreaterThan(-1) + expect(send).toBeGreaterThan(-1) + expect(incr).toBeLessThan(send) + expect(body.slice(send)).toContain("w[budgetKey] = attemptNum") + }) +}) + +// ============================================================================ +// CONTRACT — fail deterministically if someone reverts a ported option. +// ============================================================================ + +describe("v2: contract assertions on source", () => { + test("every v1 option name is in the recognised set", () => { + const block = SOURCE.match(/const RECOGNISED_OPTIONS = new Set\(\[([\s\S]*?)\]\)/) + expect(block).not.toBeNull() + const recognised = new Set([...(block![1].match(/"([a-zA-Z][a-zA-Z0-9]*)"/g) ?? [])].map((s) => s.replace(/"/g, ""))) + // Sampled across every category: prompts, timing, pattern lists, the + // v1-only alias, and the two that were inert until this branch ported them. + for (const key of [ + "continuePrompt", + "actionIntentPrompt", + "toolTextRecoveryPrompt", + "doneWithoutDetailsPrompt", + "doneWithoutWorkPrompt", + "thinkingToolRecoveryPrompt", + "chunkTimeoutMs", + "warmupMs", + "minActivityGapMs", + "toolTextCheckDelayMs", + "subagentWaitMs", + "discoveryDelayMs", + "maxRecoveryRetries", + "busyStallStrategy", + "resumeOnActionIntent", + "doneClaimPatterns", + "readyToContinuePatterns", + "streamingFailureErrorNames", + "streamingFailureMessagePatterns", + "contextSaturationThreshold", + "subagentNativeCompactionEnabled", + "silentDeadStreamMinTokens", + "injectIntervalMs", + ]) { + expect(recognised.has(key)).toBe(true) + } + }) + + test("the declared options type and the recognised set cannot drift apart", () => { + const declared = new Set( + [...(SOURCE.match(/export interface AutoResumeOptions \{([\s\S]*?)\n\}/)?.[1] ?? "").matchAll( + /^\t([a-zA-Z][a-zA-Z0-9]*)\?:/gm, + )].map((m) => m[1]), + ) + const block = SOURCE.match(/const RECOGNISED_OPTIONS = new Set\(\[([\s\S]*?)\]\)/) + const recognised = new Set([...(block![1].match(/"([a-zA-Z][a-zA-Z0-9]*)"/g) ?? [])].map((s) => s.replace(/"/g, ""))) + expect([...declared].sort()).toEqual([...recognised].sort()) + }) + + test("the v2 active-user window matches v1 (5 min), not the 15 min the port shipped with", () => { + expect(SOURCE).toMatch(/DEFAULT_ACTIVE_USER_WINDOW_MS = 5 \* 60_000/) + }) +}) diff --git a/src/v2/index.orphan.test.ts b/src/v2/index.orphan.test.ts new file mode 100644 index 0000000..359b307 --- /dev/null +++ b/src/v2/index.orphan.test.ts @@ -0,0 +1,694 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexOrphanTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" +const CHILD = "ses_child" + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +/** Long timeouts: the orphan watch must fire well inside the silence check's own + * budget, and the point of several tests is that it does NOT wait for that. */ +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: true, +} + +const userMessage = (text: string, at: number, id = `msg_u_${at}`) => ({ + type: "user", + id, + time: { created: at }, + content: [{ type: "text", text }], +}) + +const assistantAt = (at: number, extra: unknown[] = [], err?: unknown) => ({ + type: "assistant", + id: "msg_a", + time: { created: at }, + ...(err !== undefined ? { error: err } : {}), + content: [{ type: "text", text: "working on it" }, ...extra], +}) + +type History = Record + +async function setup( + opts: { + history?: History + /** Session ids the server reports busy. */ + active?: string[] + /** Rows `session.list` returns. `parentID` is verified client-side. */ + listRows?: Array<{ id: string; parentID?: string }> + /** Drop `parentID` from list rows, simulating a host that ignores the filter. */ + listIgnoresFilter?: boolean + sessions?: Record + subagentWaitMs?: number + }, +): Promise { + const injected: Array<{ sid?: string; text?: string }> = [] + const interrupted: string[] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-orphan-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const history: History = opts.history ?? { [SID]: [userMessage("go", Date.now() - 60_000)] } + const active = new Set(opts.active ?? []) + const rows = opts.listRows ?? [{ id: SID }, { id: CHILD, parentID: SID }] + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, ...(opts.subagentWaitMs ? { subagentWaitMs: opts.subagentWaitMs } : {}), registryScope: LOG_SALT }, + session: { + context: async (a: any) => history[a?.sessionID ?? SID] ?? [], + active: async () => Object.fromEntries([...active].map((s) => [s, {}])), + interrupt: async (a: any) => (interrupted.push(a?.sessionID), {}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ + data: { id: path?.id, ...(opts.sessions?.[path?.id] ?? {}) }, + }), + list: async (a: any) => { + // A host that honours parentID filters server-side; one that does + // not returns everything and lets the caller verify. + if (opts.listIgnoresFilter || !a?.parentID) return { data: rows } + return { data: rows.filter((r) => r.parentID === a.parentID) } + }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + // Age past the warmup window so the watchdog considers the session at all. + await wait(60) + return { + injected, + interrupted, + history, + active, + ...({ cleanup, logFile, streamRef: stream } as any), + } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const logs = (h: any) => (existsSync(h.logFile) ? readFileSync(h.logFile, "utf8") : "") + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +/** Bring a session up busy: an execution and a step, no text. */ +async function makeBusy(h: any, sid: string) { + push(h, "session.execution.started", sid) + await wait(10) + push(h, "session.step.started", sid) + await wait(10) +} + +/** A child that goes busy and then quiet. */ +async function childGoesBusyThenIdle(h: any) { + await makeBusy(h, CHILD) + await wait(20) + // A finished session leaves the server active set. Leaving it in would have + // the watchdog re-mark the child busy, which is the server saying "still + // running" — and the whole premise of the watch is that it stopped. + h.active.delete(CHILD) + push(h, "session.idle", CHILD) + await wait(20) +} + +const A_LONG_WAY_AGO = Date.now() - 10 * 60_000 +// Past the 30m dead window: quiet this long with no error reads as crashed, +// while A_LONG_WAY_AGO (10m) now reads as waiting. Crashed-path tests use this. +const PAST_DEAD = Date.now() - 40 * 60_000 + +describe("v2: the orphan watch arms when a parent's subagents fall quiet", () => { + test("CONTROL: nothing is armed while only the parent is busy", async () => { + // The control for everything below: the watch is about subagents going + // quiet, not about a parent being busy. A parent alone must be left to the + // ordinary silence check. + const h = await setup({ active: [SID] }) + await makeBusy(h, SID) + await wait(400) + expect(h.interrupted).toHaveLength(0) + expect(logs(h)).not.toContain("orphan watch") + await teardown(h) + }) + + test("CONTROL: a session that outlived a busy session but has no subagents is left alone", async () => { + // Two unrelated conversations open at once and one of them finishes: from the + // busy-count transition alone that is indistinguishable from a parent whose + // child died. v1 cannot tell them apart and aborts the survivor. This one has + // no children at all, so the watch never arms. + const OTHER = "ses_stranger" + const h = await setup({ + active: [SID, OTHER], + history: { [SID]: [userMessage("go", A_LONG_WAY_AGO)] }, + listRows: [{ id: SID }], + }) + await makeBusy(h, SID) + await wait(20) + await makeBusy(h, OTHER) + await wait(20) + h.active.delete(OTHER) + push(h, "session.idle", OTHER) + await wait(500) + expect(logs(h)).not.toContain("orphan watch") + expect(h.interrupted).toHaveLength(0) + await teardown(h) + }) + + test("CONTROL: a subagent of a different parent does not arm this parent's watch", async () => { + // The child check is on the parentID link, not on "some child went quiet". A + // neighbour's fan-out finishing is not this session's business. + const OTHER_CHILD = "ses_other_child" + const h = await setup({ + active: [SID, OTHER_CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [OTHER_CHILD]: [assistantAt(A_LONG_WAY_AGO)], + }, + listRows: [{ id: SID }, { id: OTHER_CHILD, parentID: "ses_their_parent" }], + }) + await makeBusy(h, SID) + await wait(20) + await makeBusy(h, OTHER_CHILD) + await wait(20) + h.active.delete(OTHER_CHILD) + push(h, "session.idle", OTHER_CHILD) + await wait(500) + expect(logs(h)).not.toContain("orphan watch") + expect(h.interrupted).toHaveLength(0) + await teardown(h) + }) + + test("the watch is armed when the last child goes idle and a parent stays busy", async () => { + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(A_LONG_WAY_AGO)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + expect(logs(h)).toContain("orphan watch") + await teardown(h) + }) + + test("the watch does not act before subagentWaitMs has elapsed", async () => { + // subagentWaitMs is the whole knob. Firing early would kill parents that are + // simply slow. + const h = await setup({ + active: [SID, CHILD], + subagentWaitMs: 60_000, + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(A_LONG_WAY_AGO)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(400) + expect(h.interrupted).toHaveLength(0) + await teardown(h) + }) + + test("a busy watch is not re-armed by the clock, so it still fires", async () => { + // If a deferral pushed the start time out instead of leaving it alone, a + // watch could be held off forever by its own deferrals. + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(PAST_DEAD)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + expect(h.interrupted).toContain(SID) + await teardown(h) + }) +}) + +describe("v2: the orphan watch refuses to kill a working parent", () => { + test("a parent with a tool in flight is never aborted", async () => { + // The single most damaging thing this feature could do. A parent running a + // five-minute build looks exactly like a parent waiting on a dead child. + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(A_LONG_WAY_AGO)], + }, + }) + await makeBusy(h, SID) + await wait(20) + push(h, "session.tool.called", SID, { tool: "bash" }) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + expect(h.interrupted).toHaveLength(0) + expect(logs(h)).toContain("tool(s) in flight") + await teardown(h) + }) + + test("a finished tool releases the slot, and the watch then acts", async () => { + // The control for the test above: if the slot were never released the watch + // would defer forever and this feature would be inert. + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(PAST_DEAD)], + }, + }) + await makeBusy(h, SID) + await wait(20) + push(h, "session.tool.called", SID, { tool: "bash" }) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(200) + expect(h.interrupted).toHaveLength(0) + push(h, "session.tool.success", SID, { tool: "bash" }) + await wait(400) + expect(h.interrupted).toContain(SID) + await teardown(h) + }) + + test("a failed tool also releases the slot", async () => { + // A failed tool is finished work. Holding the slot on failure would make the + // watch defer on every session that has ever seen an error. + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(PAST_DEAD)], + }, + }) + await makeBusy(h, SID) + await wait(20) + push(h, "session.tool.called", SID, { tool: "bash" }) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(200) + push(h, "session.tool.failed", SID, { tool: "bash" }) + await wait(400) + expect(h.interrupted).toContain(SID) + await teardown(h) + }) + + test("a parent waiting on the user is never aborted", async () => { + // An open question is the user holding the ball, and injecting or aborting + // through that interrupts their answer. + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [ + userMessage("go", A_LONG_WAY_AGO), + { + type: "assistant", + id: "msg_q", + time: { created: A_LONG_WAY_AGO }, + content: [{ type: "tool", id: "c1", name: "question", state: { status: "running", input: {} } }], + }, + ], + [CHILD]: [assistantAt(A_LONG_WAY_AGO)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + expect(h.interrupted).toHaveLength(0) + await teardown(h) + }) + + test("a subagent that is still running is waited for, not treated as dead", async () => { + // The most damaging thing this feature could do: abort a parent in the middle + // of a healthy fan-out. One child has finished, which is what armed the watch; + // the other is mid-run, and the parent is waiting on it. + const LIVE = "ses_child_live" + const h = await setup({ + active: [SID, CHILD, LIVE], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(A_LONG_WAY_AGO)], + [LIVE]: [assistantAt(Date.now() - 1_000)], + }, + listRows: [{ id: SID }, { id: CHILD, parentID: SID }, { id: LIVE, parentID: SID }], + }) + await makeBusy(h, SID) + await wait(20) + await makeBusy(h, LIVE) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + expect(h.interrupted).toHaveLength(0) + await teardown(h) + }) + + test("a subagent that finished a moment ago is not treated as crashed", async () => { + // The control for the test above, and the reason the stuck fuse is measured + // from the child's last message rather than from when it went idle: a child + // that just finished is a parent making progress, not a parent stuck. + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(Date.now() - 1_000)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(300) + // One attempt to wake it, because a fresh-but-gone child is not "crashed" + // and must not be reported as one. + expect(logs(h)).not.toContain("reported an error") + await teardown(h) + }) + + test("a slow subagent with a tool outstanding gets a longer fuse", async () => { + // The control for the stuck threshold: 90s of silence is past the 60s + // threshold and would read as dead, but the outstanding tool part means it is + // a long build, so the fuse is tripled and the parent is left alone. + const LIVE = "ses_child_slow" + const h = await setup({ + active: [SID, CHILD, LIVE], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(A_LONG_WAY_AGO)], + [LIVE]: [ + assistantAt(Date.now() - 90_000, [ + { type: "tool", id: "c9", name: "bash", state: { status: "running", input: {} } }, + ]), + ], + }, + listRows: [{ id: SID }, { id: CHILD, parentID: SID }, { id: LIVE, parentID: SID }], + }) + await makeBusy(h, SID) + await wait(20) + await makeBusy(h, LIVE) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + expect(h.interrupted).toHaveLength(0) + expect(logs(h)).not.toContain("recovery prompt sent") + await teardown(h) + }) +}) + +describe("v2: a dead subagent is woken before the parent is killed", () => { + test("CONTROL: a stuck subagent is woken before the parent is killed", async () => { + // Waking the child is much cheaper than killing the parent: the child is one + // nudge from finishing and the parent's whole turn survives. + // + // subagentWaitMs is raised so the two stages are separately observable. The + // sequence matters and is not "recover, never abort" — it is nudge once, then + // wait a further subagentWaitMs, and only abort the parent if the child is + // still dead. Without the second wait a nudge and an abort would land together. + const h = await setup({ + active: [SID, CHILD], + subagentWaitMs: 400, + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(PAST_DEAD)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(700) + expect(h.interrupted).toHaveLength(0) + const nudged = h.injected.filter((i: any) => i.sid === CHILD && /stalled or timed out/.test(i.text ?? "")) + expect(nudged).toHaveLength(1) + // And only once, however many ticks pass. + await wait(400) + expect(h.injected.filter((i: any) => i.sid === CHILD)).toHaveLength(1) + await teardown(h) + }) + + test("a child that stays dead is escalated on the parent after a further wait", async () => { + // The control for the nudge: if the child never comes back, the parent is the + // only thing left to save, and it is waiting on a result that will not arrive. + const h = await setup({ + active: [SID, CHILD], + subagentWaitMs: 200, + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(PAST_DEAD)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(900) + expect(h.interrupted).toContain(SID) + await teardown(h) + }) + + test("a crashed subagent with no way to reach it falls back to aborting the parent", async () => { + // A crashed child cannot be woken. The parent is the only remaining thing to + // save, and it is waiting on a result that will never arrive. + const h = await setup({ + active: [SID, CHILD], + sessions: { [CHILD]: {} }, + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [{ type: "assistant", id: "msg_err", time: { created: A_LONG_WAY_AGO }, error: "stream disconnected", content: [] }], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + expect(h.interrupted).toContain(SID) + await teardown(h) + }) +}) + +describe("v2: the subagent list is verified, not trusted", () => { + test("CONTROL: a host that ignores the parentID filter does not cause a wrong abort", async () => { + // The filter is requested AND checked. If it were only requested, a host that + // ignored it would hand back every session on the box and the watch would + // read a stranger's children as this parent's. + const h = await setup({ + active: [SID, CHILD], + listIgnoresFilter: true, + listRows: [ + { id: SID }, + { id: CHILD, parentID: SID }, + { id: "ses_unrelated" }, + ], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(A_LONG_WAY_AGO)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + // ses_unrelated is not this session's child, so the busy child here is CHILD + // and the verdict is the same either way — the point is that nothing else in + // the list was treated as a child and no stranger was touched. + expect(h.interrupted).not.toContain("ses_unrelated") + await teardown(h) + }) + + test("a list that returns nothing means the watch is never armed", async () => { + // No rows could mean the API is unavailable rather than childless. Arming on + // that reading would abort a parent on the strength of a failed query, so an + // empty list abstains — and abstaining only costs a missed recovery. + const h = await setup({ + active: [SID, CHILD], + listRows: [], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(A_LONG_WAY_AGO)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + expect(h.interrupted).toHaveLength(0) + expect(logs(h)).toContain("has no subagents") + await teardown(h) + }) +}) + +// ses_ed7268728ffe6Wj6RWS7ME4DAN, 2026-10-11. Log: +// 18:29:56.341 …S7ME4DAN usage: … <- user "clean up any residue" lands +// 18:29:56.528 subagent …Ut64EfTB silent for 9227s +// subagent crashed and did not recover — aborting and resuming +// 18:29:56.528 WARN escalating: interrupt + fresh continue (2/2 per 10m) +// 18:29:58.539 resumed after abort +describe("v2: the orphan watch yields to the user and does not loop on one dead child", () => { + test("a user message after the watch armed suppresses the abort", async () => { + // `hasPendingUserInput` only sees a pending INTERACTIVE tool — a question + // awaiting an answer. An instruction is a plain user message and sailed + // through, so the abort landed on top of a live turn 187ms after the + // user spoke. The idle path always had this guard; the orphan path did not. + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(PAST_DEAD)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + // The user speaks AFTER the watch armed: an instruction, not a question. + // No execution.started here on purpose — that would markBusy and reset the + // watch (the code's own contract). The orphan path must notice the message + // on its own, from history, while the parent stays busy. + h.history[SID] = [ + userMessage("go", A_LONG_WAY_AGO), + userMessage("clean up any remaining residue", Date.now(), "msg_u_fresh"), + ] + await wait(500) + const said = logs(h) + await teardown(h) + expect(said).toContain("after the orphan watch armed — deferring orphan abort") + expect(said).not.toContain("subagent crashed and did not recover") + expect(h.interrupted).toHaveLength(0) + }) + + test("CONTROL: with no fresh user message the abort still happens", async () => { + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(PAST_DEAD)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + const said = logs(h) + await teardown(h) + expect(said).toContain("subagent crashed and did not recover") + expect(h.interrupted).toContain(SID) + }) + + test("one dead subagent does not re-escalate when the parent re-arms and works again", async () => { + // The real shape (ses_ed726872): the SAME dead child …Ut64EfTB produced 8 + // abort+resume escalations across 3 hours — not 8 in a burst. Within one + // armed episode `resumeAttempts` already caps repeats, so a burst-count + // cannot see the bug. What exposes it is RE-ARMING: the parent recovers, + // does real work, and the next 2-to-1 busy transition arms a fresh watch — + // which is where an earlier version of this fix wrongly cleared its own + // record, so the same dead child escalated again every cycle. + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(PAST_DEAD)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + expect(logs(h).split("subagent crashed and did not recover").length - 1).toBe(1) + + // The parent recovers and does real work, then the watch re-arms on the + // next child-idle transition. The child is still dead. + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + // Past the abort cooldown (~2s), which is what masked this in a short window. + await wait(2500) + + const said = logs(h) + const escalations = said.split("subagent crashed and did not recover").length - 1 + await teardown(h) + // Still one. Without the record surviving re-arm this climbs to 2. + expect(escalations).toBe(1) + expect(said).toContain("already escalated this episode") + }) + + test("CONTROL: a different stuck child still gets its own escalation", async () => { + // The per-child record must not over-block: a genuinely NEW stuck child + // (not the one already escalated) still earns its own abort. + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", A_LONG_WAY_AGO)], + [CHILD]: [assistantAt(PAST_DEAD)], + }, + }) + await makeBusy(h, SID) + await wait(20) + await childGoesBusyThenIdle(h) + await wait(500) + const first = logs(h).split("subagent crashed and did not recover").length - 1 + expect(first).toBe(1) + await teardown(h) + }) +}) diff --git a/src/v2/index.parent-wait.test.ts b/src/v2/index.parent-wait.test.ts new file mode 100644 index 0000000..d0ea3fa --- /dev/null +++ b/src/v2/index.parent-wait.test.ts @@ -0,0 +1,194 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexParentWaitTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" +const CHILD = "ses_child" + +let counter = 1000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 300, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, +} + +const userMessage = (text: string, at: number, id = `msg_u_${at}`) => ({ + type: "user", + id, + time: { created: at }, + content: [{ type: "text", text }], +}) + +const assistantAt = (at: number) => ({ + type: "assistant", + id: "msg_a", + time: { created: at }, + content: [{ type: "text", text: "working on it" }], +}) + +type History = Record + +async function setup(opts: { + history?: History + /** Session ids the server reports busy. */ + active?: string[] + /** Rows `session.list` returns. Defaults to parent + one linked child. */ + listRows?: Array<{ id: string; parentID?: string }> +}): Promise { + const injected: Array<{ sid?: string; text?: string }> = [] + const interrupted: string[] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-parentwait-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const history: History = opts.history ?? {} + const active = new Set(opts.active ?? []) + const rows = opts.listRows ?? [{ id: SID }, { id: CHILD, parentID: SID }] + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async (a: any) => history[a?.sessionID ?? SID] ?? [], + active: async () => Object.fromEntries([...active].map((s) => [s, {}])), + interrupt: async (a: any) => (interrupted.push(a?.sessionID), {}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async (a: any) => { + if (!a?.parentID) return { data: rows } + return { data: rows.filter((r) => (r as any).parentID === a.parentID) } + }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, interrupted, history, active, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +async function makeBusy(h: any, sid: string) { + push(h, "session.execution.started", sid) + await wait(10) + push(h, "session.step.started", sid) + await wait(10) +} + +const parentInjects = (h: any) => h.injected.filter((i: any) => i.sid === SID) + +describe("v2: busy-stall skips a parent waiting on a live subagent", () => { + test("no stall recovery while a live child works, even with no task-tool event", async () => { + // ses_ef73a206 shape: parent blocked on a subagent, silent past + // chunkTimeoutMs, never emitted session.tool.called for the dispatch + // (lastWasTaskTool false, pendingTools 0). The parentID link + recent + // child activity is the only evidence — and it must be enough. + const now = Date.now() + const h = await setup({ + active: [SID, CHILD], + history: { [SID]: [userMessage("go", now - 60_000)], [CHILD]: [assistantAt(now - 5_000)] }, + }) + await makeBusy(h, SID) + await wait(900) + expect(parentInjects(h)).toHaveLength(0) + expect(h.interrupted).toHaveLength(0) + await teardown(h) + }) + + test("no stall recovery while tools are held and children are unresolved", async () => { + // tool.called fired (pendingTools 1, lastWasTaskTool true) but the + // child is absent from the server active set — the old + // lastWasTaskTool+others guard misses, the parentID link must catch it. + const now = Date.now() + const h = await setup({ + active: [SID], + history: { [SID]: [userMessage("go", now - 60_000)], [CHILD]: [assistantAt(now - 5_000)] }, + }) + await makeBusy(h, SID) + push(h, "session.tool.called", SID, { tool: "subagent", input: { description: "research" } }) + await wait(900) + expect(parentInjects(h)).toHaveLength(0) + expect(h.interrupted).toHaveLength(0) + await teardown(h) + }) + + test("CONTROL: a truly stalled parent with no live children still recovers", async () => { + // No children at all on the parentID link — ordinary stall, must fire. + // (A merely-quiet child now reads as waiting, not dead, so the link + // itself must be absent for this control.) + const now = Date.now() + const h = await setup({ + active: [SID], + listRows: [{ id: SID }], + history: { [SID]: [userMessage("go", now - 60_000)] }, + }) + await makeBusy(h, SID) + await wait(900) + expect(parentInjects(h).length).toBeGreaterThan(0) + await teardown(h) + }) +}) diff --git a/src/v2/index.perm-failure.test.ts b/src/v2/index.perm-failure.test.ts new file mode 100644 index 0000000..160e682 --- /dev/null +++ b/src/v2/index.perm-failure.test.ts @@ -0,0 +1,163 @@ +import { describe, test, expect } from "bun:test" +import { rmSync, readFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexPermFailureTest" + +/** + * v2: permanent failures latch recovery out. + * + * ses_ee46693e (2026-10-08): `provider.invalid-request Model exo-free has been + * deprecated` matched no recovery category — not abort, not rate-limit, not + * OOC, not streaming — so it fell into generic stall recovery and injected + * `continue (attempt 1/3)` into a model that no longer exists. Twice, because + * 21 plugin reloads in 4 minutes gave each instance a fresh attempt counter. + * + * Attempt budgets cannot cover this: they are per-instance and reset on + * reload. Only the failure CLASS is stable across instances. So permanent + * failures latch like oocLocked — keyed on the message (deprecated / + * model-not-found shapes), never on `invalid-request` alone, which also + * carries transient 400s. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 13000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +async function setup(): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-permfail-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 50, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + }, + session: { + context: async () => [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const stepFailed = (h: any, message: string, type = "provider.invalid-request") => + push(h, "session.step.failed", SID, { error: { type, message } }) + +describe("v2: permanent failures latch recovery out", () => { + test("deprecated model error: no continue is injected", async () => { + const h = await setup() + push(h, "session.execution.started", SID) + await wait(20) + stepFailed(h, "Model exo-free has been deprecated.") + await wait(400) + const log = readFileSync(h.logFile, "utf8") + await teardown(h) + expect(h.injected.length).toBe(0) + expect(log).toContain("latched") + }) + + test("CONTROL: transient invalid-request still recovers", async () => { + const h = await setup() + push(h, "session.execution.started", SID) + await wait(20) + stepFailed(h, "Error from the provider: 400 Bad Request") + await wait(400) + await teardown(h) + expect(h.injected.length).toBeGreaterThanOrEqual(1) + }) + + test("a genuine new turn clears the latch", async () => { + const h = await setup() + push(h, "session.execution.started", SID) + await wait(20) + stepFailed(h, "Model exo-free has been deprecated.") + await wait(200) + expect(h.injected.length).toBe(0) + // User starts a fresh turn (model changed): latch must release. + push(h, "session.execution.started", SID) + await wait(20) + stepFailed(h, "Error from the provider: 500 upstream boom") + await wait(400) + await teardown(h) + expect(h.injected.length).toBeGreaterThanOrEqual(1) + }) +}) \ No newline at end of file diff --git a/src/v2/index.perm-gate-set.test.ts b/src/v2/index.perm-gate-set.test.ts new file mode 100644 index 0000000..95f2455 --- /dev/null +++ b/src/v2/index.perm-gate-set.test.ts @@ -0,0 +1,302 @@ +import { describe, test, expect } from "bun:test" +import { rmSync, readFileSync, existsSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexPermGateSetTest" + +/** + * v2: the whole permanent-failure gate set, table-driven. + * + * The classification is not invented here — it is opencode's own. Every gated + * value below is an `error.type` from the taxonomy the running binary maps + * (packages/llm/src/schema/errors.ts, `get retryable() === false`), or a status + * / message shape that source treats the same way. Per the audit in + * /tmp/opencode/error-taxonomy_findings.md: + * + * gate: provider.auth / provider.no-route / provider.unknown / + * provider.content-filter / provider.unsupported-operation, + * http 401 / 403, model-gone messages, history-rejection messages + * NOT gated: provider.rate-limit + provider.quota (15m→2h cooldown ladder, + * ~12h total, for 5h rolling Zen quotas), provider.internal, + * provider.transport, provider.timeout, provider.invalid-output, + * tool.execution, compaction.failed, and 404 (openai returns 404 for + * models that ARE available — provider/error.ts:26). + * + * Harness is prod-shape: no `ctx.client`, `session.active()` returns {}, and the + * loopback HTTP discovery paths are therefore unavailable — the gate must stand + * on the event payload alone, exactly as it does in production. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 14000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) +const push = (h: any, type: string, data: Record = {}) => h.streamRef.push(ev(type, data)) + +async function setup(): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-gateset-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + // Prod shape: `client` is undefined in production (probed live 2026-10-06) and + // must stay undefined here — the gate cannot lean on a discovery API. + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 50, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + }, + session: { + context: async () => [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const logsOf = (h: any) => (existsSync(h.logFile) ? readFileSync(h.logFile, "utf8") : "") + +/** Drive one failure event and report what the plugin did. */ +async function drive(errType: string, errMsg: string, status?: number) { + const h = await setup() + push(h, "session.execution.started") + await wait(20) + push(h, "session.step.failed", { error: { type: errType, message: errMsg, ...(status === undefined ? {} : { status }) } }) + await wait(300) + const injected = h.injected.length + const latched = logsOf(h).includes("permanent failure latched") + await teardown(h) + return { injected, latched } +} + +const GATED: Array<[string, string, number | undefined]> = [ + ["provider.auth", "Unauthorized: request was blocked by a gateway or proxy.", 401], + ["provider.auth", "Forbidden: your account lacks access to this model.", 403], + ["provider.auth", "no api key found for anthropic", undefined], + ["provider.no-route", "No LLM route for openai/gpt-5 using default", undefined], + ["provider.content-filter", "Your request was blocked by content policy", undefined], + // message-keyed (type too broad to gate on) + ["provider.invalid-request", "Model exo-free has been deprecated.", undefined], + // 404 does NOT rescue a gone model — only the "unknown catalog entry" shape. + ["provider.invalid-request", "Model exo-free has been deprecated.", 404], + ["provider.invalid-request", "Model not found: exo-free", undefined], + ["provider.invalid-request", "Referenced reasoning item 'rs_6abf:rs_0' was not found", undefined], + ["provider.invalid-request", "No tool output found for function call call_01a0", undefined], + ["provider.invalid-request", "The model gpt-9 has been retired", 400], +] + +const NOT_GATED: Array<[string, string, number | undefined]> = [ + ["provider.transport", "ConnectionRefused: Unable to connect", undefined], + ["provider.timeout", "request timed out after 30s", undefined], + ["provider.internal", "server_error: The model failed to generate a response.", 500], + ["provider.invalid-output", "OpenAI Chat stream ended without finish_reason", undefined], + ["provider.invalid-request", "Error from the provider: 400 Bad Request", 400], + // 404 + "model not found" keeps recovering (openai returns 404 for live models). + ["provider.invalid-request", "Error from the provider: 404 model not found", 404], + ["provider.quota", "Rate limit exceeded. Please try again later.", 429], + ["provider.rate-limit", "Rate limit exceeded. Please try again later.", 429], + ["tool.execution", "Command execution failed", undefined], + ["compaction.failed", "Compaction produced no summary", undefined], +] + +describe("v2: permanent-failure gate set", () => { + for (const [type, msg, status] of GATED) { + test(`GATED ${type} ${status ?? "-"}/${msg.slice(0, 44)}`, async () => { + const { injected, latched } = await drive(type, msg, status) + expect(latched).toBe(true) + expect(injected).toBe(0) + }) + } + + for (const [type, msg, status] of NOT_GATED) { + test(`RECOVERS ${type} ${status ?? "-"}/${msg.slice(0, 40)}`, async () => { + const { latched } = await drive(type, msg, status) + // Never latches. Whether it injects now or gates behind a cooldown + // ladder (rate-limit/quota: 15m first rung) is a separate concern. + expect(latched).toBe(false) + }) + } + + test("provider.unknown with transient abort gets ONE recovery attempt, then latches", async () => { + // provider.unknown is a catch-all for transient server hiccups (e.g. + // "Chat completion aborted. Please check the server console"). + // It must NOT hard-gate on the first occurrence. + const h = await setup() + push(h, "session.execution.started") + await wait(20) + push(h, "session.step.failed", { + error: { type: "provider.unknown", message: "Chat completion aborted. Please check the server console." }, + }) + await wait(300) + const afterFirst = h.injected.length + const latchedAfterFirst = logsOf(h).includes("permanent failure latched") + + // Second occurrence: the indeterminate budget is spent. + push(h, "session.step.failed", { + error: { type: "provider.unknown", message: "Chat completion aborted. Please check the server console." }, + }) + await wait(300) + const log = logsOf(h) + await teardown(h) + + expect(afterFirst).toBeGreaterThanOrEqual(1) + expect(latchedAfterFirst).toBe(false) + expect(h.injected.length).toBe(afterFirst) + expect(log).toContain("permanent failure latched") + expect(log).toContain("2 strikes") + }) + + test("an indeterminate type gets ONE recovery attempt, then latches", async () => { + // provider.unsupported-operation is the only gated-looking type with no + // upstream verdict: in the shipped binary, absent from the dev union, so + // no `retryable` getter was ever observable for it. Hard-gating on the + // first occurrence bets on unread semantics; never gating retries a wall + // forever. One attempt, then latch. + const h = await setup() + push(h, "session.execution.started") + await wait(20) + push(h, "session.step.failed", { + error: { type: "provider.unsupported-operation", message: "Model does not support tool use" }, + }) + await wait(300) + const afterFirst = h.injected.length + const latchedAfterFirst = logsOf(h).includes("permanent failure latched") + + // Second occurrence: the ladder is spent. + push(h, "session.step.failed", { + error: { type: "provider.unsupported-operation", message: "Model does not support tool use" }, + }) + await wait(300) + const log = logsOf(h) + await teardown(h) + + expect(afterFirst).toBeGreaterThanOrEqual(1) + expect(latchedAfterFirst).toBe(false) + expect(h.injected.length).toBe(afterFirst) + expect(log).toContain("permanent failure latched") + expect(log).toContain("2 strikes") + }) + + test("CONTROL: a genuine turn clears the strike count", async () => { + // Without the reset, the second occurrence below would latch and the + // session would lose recovery it should still have. + const h = await setup() + push(h, "session.execution.started") + await wait(20) + const unsupported = { error: { type: "provider.unsupported-operation", message: "nope" } } + push(h, "session.step.failed", unsupported) + await wait(150) + // Two starts, deliberately. Our recovery continue produces its own + // execution.started, which spends the selfRecovery flag; the user's + // arrival is the next one. Pushing only one lets our own turn masquerade + // as the user's, and the reset never runs. + push(h, "session.execution.started") + await wait(20) + push(h, "session.execution.started") + await wait(20) + push(h, "session.step.failed", unsupported) + await wait(300) + const log = logsOf(h) + await teardown(h) + expect(log).toContain("indeterminate failure") + expect(log).not.toContain("permanent failure latched") + expect(log).toContain("1/2") + }) + + test("quota stays on the cooldown ladder, not the permanent latch", async () => { + const h = await setup() + push(h, "session.execution.started") + await wait(20) + push(h, "session.step.failed", { error: { type: "provider.quota", message: "Rate limit exceeded.", status: 429 } }) + await wait(300) + const log = logsOf(h) + await teardown(h) + expect(log).not.toContain("permanent failure latched") + // The ladder is what should answer a 5h rolling quota. + expect(log).toMatch(/quota|rate limit|cooling|gate/i) + }) + + test("a genuine turn clears the latch and recovery resumes", async () => { + const h = await setup() + push(h, "session.execution.started") + await wait(20) + push(h, "session.step.failed", { error: { type: "provider.auth", message: "no api key", status: 401 } }) + await wait(150) + expect(h.injected.length).toBe(0) + expect(logsOf(h)).toContain("permanent failure latched") + // User re-auths and starts a fresh turn: latch must release. + push(h, "session.execution.started") + await wait(20) + push(h, "session.step.failed", { error: { type: "provider.transport", message: "ConnectionRefused" } }) + await wait(300) + await teardown(h) + expect(h.injected.length).toBeGreaterThanOrEqual(1) + }) +}) \ No newline at end of file diff --git a/src/v2/index.perm-lock-chokepoint.test.ts b/src/v2/index.perm-lock-chokepoint.test.ts new file mode 100644 index 0000000..ce9a618 --- /dev/null +++ b/src/v2/index.perm-lock-chokepoint.test.ts @@ -0,0 +1,232 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexPermLockChokepointTest" + +/** + * v2: the permanent/OOC locks belong at the injection choke point. + * + * The locks are enforced in `recover()` and `tryAbortAndResume()` — two of the + * five `injectOnce` call sites. The other three are separate features with + * their own triggers and their own budgets: the unknown-tool suggestion + * (`checkForUnknownToolCalls`), the task_complete blocked nudge, and + * `targetedRecovery` for intent/tool-loop patterns. + * + * That is a real hole, not a theoretical one. A permanently-failed session + * goes idle with its history intact, and the unknown-tool pass then reads that + * history and injects a suggestion — into a model that cannot answer at all. + * Same for the OOC lock, which predates this. Enforcing at `injectOnceLocked` + * closes all five paths and every future one, instead of five call sites. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_choke" + +let counter = 15000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: true, +} + +const OLD = Date.now() - 60 * 60_000 +const TOOLS = [{ id: "read" }, { id: "write" }, { id: "glob" }, { id: "bash" }, { id: "task_complete" }] + +const userMessage = (text: string, at: number) => ({ + type: "user", + id: `msg_u_${at}`, + time: { created: at }, + content: [{ type: "text", text }], +}) + +const toolPart = (id: string, name: string) => ({ + type: "tool", + id, + name, + time: { created: OLD }, + state: { status: "error", input: {}, error: "tool not found" }, +}) + +/** Carries a text part too, deliberately: a finished assistant message with no + * text part trips the silent-dead-stream detector first and would swallow this. */ +const assistantWith = (...content: unknown[]) => ({ + type: "assistant", + id: "msg_a_here", + time: { created: OLD + 1_000 }, + content: [{ type: "text", text: "Let me look at that." }, ...content], +}) + +/** Two invented calls to the same name: enough to earn a suggestion normally. */ +const TYPO_HISTORY = [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "globb")), + assistantWith(toolPart("call_2", "globb")), +] + +async function setup(history: unknown[]): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-choke-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + // Prod shape: no ctx.client, empty active set. + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => TOOLS, + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +async function goIdle(h: any) { + for (const e of [ + ev("session.execution.started"), + ev("session.step.started"), + ev("session.step.ended"), + ev("session.idle"), + ]) { + h.streamRef.push(e) + await wait(10) + } + await wait(250) +} + +const suggestions = (h: any) => h.injected.filter((i: any) => /does not exist/.test(i.text ?? "")) + +/** + * Reach idle WITHOUT pushing `session.execution.started`. That event is a genuine + * new turn and correctly clears both locks, so including it would test the wrong + * thing — the latch would already be gone before the idle passes ran. The step and + * idle events are enough: the unknown-tool pass is driven by `session.idle`. + */ +async function reachIdle(h: any) { + for (const e of [ev("session.step.started"), ev("session.step.ended"), ev("session.idle")]) { + h.streamRef.push(e) + await wait(10) + } + await wait(250) +} + +/** Latch a permanent failure, then let the idle passes run. */ +async function driveWithLatch(h: any) { + h.streamRef.push(ev("session.execution.started")) + await wait(20) + h.streamRef.push( + ev("session.step.failed", { error: { type: "provider.auth", message: "no api key found", status: 401 } }), + ) + await wait(150) + await reachIdle(h) +} + +describe("v2: locked sessions inject nothing from ANY path", () => { + test("CONTROL: without a latch, the unknown-tool suggestion is sent", async () => { + const h = await setup(TYPO_HISTORY) + await goIdle(h) + expect(suggestions(h)).toHaveLength(1) + await teardown(h) + }) + + test("a permanently failed session gets no suggestion", async () => { + const h = await setup(TYPO_HISTORY) + await driveWithLatch(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) + + test("an OOC-locked session gets no suggestion either", async () => { + const h = await setup(TYPO_HISTORY) + h.streamRef.push(ev("session.execution.started")) + await wait(20) + h.streamRef.push( + ev("session.step.failed", { + error: { type: "provider.invalid-request", message: "prompt is too long: 210000 tokens" }, + }), + ) + await wait(150) + await reachIdle(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) + + test("a genuine turn releases both locks and the suggestion returns", async () => { + const h = await setup(TYPO_HISTORY) + await driveWithLatch(h) + expect(suggestions(h)).toHaveLength(0) + // Fresh user turn (credentials fixed): locks clear, idle pass works again. + h.streamRef.push(ev("session.execution.started")) + await wait(20) + h.streamRef.push(ev("session.idle")) + await wait(300) + await teardown(h) + expect(suggestions(h)).toHaveLength(1) + }) +}) \ No newline at end of file diff --git a/src/v2/index.premature-stop.test.ts b/src/v2/index.premature-stop.test.ts new file mode 100644 index 0000000..1e8626b --- /dev/null +++ b/src/v2/index.premature-stop.test.ts @@ -0,0 +1,311 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexPrematureStopTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_premature" + +/** + * Premature-stop detection. + * + * This is the failure mode where the model ends its turn cleanly — no stall, no + * error, no streaming failure — while the work is not actually finished. The + * stall watchdog cannot see it: the session is idle, not wedged, so nothing is + * ever "too silent". What catches it is reading the last assistant text and + * asking whether it actually reports work. + * + * Two detectors: + * - endsWithCelebration: the model's own "finished" signal, which must latch + * rather than be nudged over. + * - containsWorkDescription: a done-claim with no work report in it, which gets + * exactly one request for details. + * + * Every group carries a control. The failure mode that matters here is a test + * that passes because the nudge never fired at all. + */ + +type Harness = { + injected: Array<{ kind: string; text?: string }> + logs: string[] +} + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +/** Old enough that activeUserWindowMs cannot mask a failure. */ +const userTurn = (ageMs = 60 * 60_000) => ({ + type: "user", + id: "msg_u0", + time: { created: Date.now() - ageMs }, + content: [{ type: "text", text: "do the thing" }], +}) +const assistantTurn = (text: string) => ({ + type: "assistant", + id: "msg_a1", + time: { created: Date.now() - 60_000 }, + content: [{ type: "text", text }], +}) + +/** One turn that streams `text` and then goes idle. */ +function turnEvents(text: string) { + return [ + ev("session.execution.started"), + ev("session.text.delta", { messageID: "msg_a1", delta: text }), + ev("session.text.ended", { messageID: "msg_a1" }), + ev("session.idle"), + ] +} + +/** Idle again with no new turn — what a latched completion must survive. */ +const idleAgain = [ev("session.idle")] + +const OPTIONS = { + // Long enough that the stall watchdog never fires: these tests are about the + // idle path only, and a stall injection would mask which path produced a nudge. + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + // Without this the second nudge in a two-turn test is debounced away as + // "too soon after the last injection", and the re-arm test fails for the + // wrong reason. + injectIntervalMs: 0, +} + +async function replay( + events: any[], + opts: Record = {}, + harnessOpts: { userAgeMs?: number } = {}, +): Promise { + const injected: Harness["injected"] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-premature-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, ...opts, registryScope: LOG_SALT }, + session: { + context: async () => [userTurn(harnessOpts.userAgeMs), assistantTurn(TEXT_FOR_HARNESS.value)], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => { + injected.push({ kind: "synthetic", text: a?.text }) + return {} + }, + prompt: async (a: any) => { + injected.push({ kind: "prompt", text: a?.text }) + return {} + }, + }, + } + + const cleanup = await (plugin as any).setup(ctx) + for (const e of events) { + stream.push(e) + await wait(10) + } + await wait(600) + ;(cleanup as (() => void) | undefined)?.() + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + rmSync(logFile, { force: true }) + return { injected, logs } +} + +/** + * The idle handler prefers the streamed delta buffer and only falls back to + * `session.context()` when it is empty. Tests that stream text exercise the + * buffer; this lets a case that must exercise the fallback set it explicitly. + */ +const TEXT_FOR_HARNESS = { value: "unrelated" } + +/** Terse done-claim with no work report — the premature stop. */ +const BARE_DONE = "Task done." + +/** Long, but no actual work report. Length alone must not save it. */ +const LONG_BUT_EMPTY = `Task done.\n\n${"I considered the situation carefully. ".repeat(30)}` + +/** Short, but a real report. Length alone must not condemn it. */ +const SHORT_BUT_REAL = "Done. Changed src/v2/index.ts" + +describe("v2: premature stop — a done-claim with no work in it", () => { + test("CONTROL: a bare 'Task done.' with nothing after it IS nudged", async () => { + const { injected } = await replay(turnEvents(BARE_DONE)) + expect(injected.length).toBeGreaterThan(0) + expect(injected[0].text).toContain("verify") + }) + + test("a long done-claim with no work report is still nudged", async () => { + // The bug this replaces: a 400-character gate let this through. + expect(LONG_BUT_EMPTY.length).toBeGreaterThan(400) + const { injected } = await replay(turnEvents(LONG_BUT_EMPTY)) + expect(injected.length).toBeGreaterThan(0) + }) + + test("a short done-claim that names a changed file is NOT nudged", async () => { + // The other half of the same bug: SHORT_BUT_REAL is under 400 characters. + expect(SHORT_BUT_REAL.length).toBeLessThan(400) + const { injected } = await replay(turnEvents(SHORT_BUT_REAL)) + expect(injected).toEqual([]) + }) + + test("a done-claim reporting tests run is NOT nudged", async () => { + const { injected } = await replay( + turnEvents("Done.\n\nTests run: 639 pass, 0 fail\nResults: all green"), + ) + expect(injected).toEqual([]) + }) + + test("a done-claim reporting verification is NOT nudged", async () => { + const { injected } = await replay(turnEvents("All done.\n\nVerification: tsc clean")) + expect(injected).toEqual([]) + }) + + test("the nudge is sent once per turn, not once per idle event", async () => { + const { injected } = await replay([...turnEvents(BARE_DONE), ...idleAgain, ...idleAgain]) + expect(injected.length).toBe(1) + }) + + test("a new turn re-arms the budget", async () => { + const { injected } = await replay([ + ...turnEvents(BARE_DONE), + ...idleAgain, + ...turnEvents(BARE_DONE), + ]) + expect(injected.length).toBe(2) + }) + + test("maxRetries caps how many details prompts one busy cycle can spend", async () => { + const { injected } = await replay(turnEvents(BARE_DONE), { maxRetries: 1 }) + expect(injected.length).toBe(1) + }) +}) + +describe("v2: celebration latches completion", () => { + test("CONTROL: a bare done-claim without 🎉 is still nudged", async () => { + const { injected } = await replay(turnEvents(BARE_DONE)) + expect(injected.length).toBeGreaterThan(0) + }) + + test("a trailing 🎉 latches completion instead of nudging", async () => { + const { injected, logs } = await replay(turnEvents("All set.\n\nChanged the parser. 🎉")) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("latching completion"))).toBe(true) + }) + + test("the latch survives a later idle with no new text", async () => { + // debug:true so the second (silent) visit is observable at all — before + // the log-sink fix these lines only reached a console nobody captures. + const { injected, logs } = await replay( + [...turnEvents("All set. 🎉"), ...idleAgain, ...idleAgain], + { debug: true }, + ) + expect(injected).toEqual([]) + expect(logs.filter((l) => l.includes("latching completion"))).toHaveLength(1) + expect(logs.some((l) => l.includes("completion already latched"))).toBe(true) + }) + + test("a new turn clears the latch, so a later done-claim is judged on its own", async () => { + const { injected, logs } = await replay([ + ...turnEvents("All set. 🎉"), + ...idleAgain, + // A fresh turn: budgets reset, latch resets, and this one is bare. + ...turnEvents(BARE_DONE), + ]) + expect(injected.length).toBeGreaterThan(0) + expect(logs.filter((l) => l.includes("latching completion"))).toHaveLength(1) + }) + + test("punctuation after the emoji still counts as celebration", async () => { + // v1 normalises trailing [.!?] before the check. + const { injected } = await replay(turnEvents("Wrapped it up. 🎉.")) + expect(injected).toEqual([]) + }) + + test("an emoji outside the last line does not latch", async () => { + // Boundary markers come in beginning/end pairs, so only the last line + // counts: an emoji anywhere earlier belongs to structure, not to the + // turn's close. The turn below ends mid-task with no other signal. + const { logs } = await replay( + turnEvents("🎉 Step one done.\n\nLine one.\nLine two.\nLine three.\nLine four.\nStill working on step two."), + ) + expect(logs.some((l) => l.includes("latching completion"))).toBe(false) + }) +}) + +describe("v2: premature-stop detection does not fight the other idle guards", () => { + test("a turn that hands off to the user is never nudged", async () => { + // A question is a legitimate stop, however terse. + const { injected } = await replay(turnEvents("Task done.\n\nWhich file should I edit?")) + expect(injected).toEqual([]) + }) + + test("a recent user message stands the nudge down", async () => { + // The user turn must be INSIDE the window for this to mean anything; the + // default harness turn is an hour old and would not stand down at all. + const { injected } = await replay(turnEvents(BARE_DONE), { activeUserWindowMs: 600_000 }, { userAgeMs: 30_000 }) + expect(injected).toEqual([]) + }) + + test("an old user message does NOT stand the nudge down", async () => { + // The control for the test above: same window, older turn, nudge fires. + const { injected } = await replay(turnEvents(BARE_DONE), { activeUserWindowMs: 600_000 }, { userAgeMs: 3_600_000 }) + expect(injected.length).toBeGreaterThan(0) + }) + + test("the fallback to session.context() judges the same way", async () => { + // No streamed text at all: the handler must read the authoritative history. + TEXT_FOR_HARNESS.value = BARE_DONE + try { + const { injected } = await replay([ev("session.execution.started"), ev("session.idle")]) + expect(injected.length).toBeGreaterThan(0) + } finally { + TEXT_FOR_HARNESS.value = "unrelated" + } + }) +}) \ No newline at end of file diff --git a/src/v2/index.prod-shape.test.ts b/src/v2/index.prod-shape.test.ts new file mode 100644 index 0000000..9d02693 --- /dev/null +++ b/src/v2/index.prod-shape.test.ts @@ -0,0 +1,381 @@ +import { describe, test, expect } from "bun:test" +import { rmSync, readFileSync, existsSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexProdShapeTest" + +/** + * v2: history readers must accept the PRODUCTION message shape. + * + * Audit 2026-10-10, verified against `opencode.db` (`message.data`): + * {"role":"user","time":{"created":1790213672332},"agent":"coder",...} + * + * The shape carries `role`, not `type`. Six readers checked only `.type` + * while their neighbours in the same file already used the tolerant + * `type ?? role` pattern, so the suite (938 green) validated a fiction: + * every fixture in the suite built `{type:"user"}` / `{type:"assistant"}`. + * + * The worst case fails OPEN. `restoreQuotaLadders` computed `lastUserAt = 0` + * for every session, so its guard `if (lastUserAt > mark.at) continue` never + * fired and a quota ladder re-armed for a session the user had already + * continued after a restart — a false nag injected into live work. + * + * These tests assert BOTH shapes work. Fixing by swapping to `role` alone + * would be the same bug wearing a different hat. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_prod" + +let counter = 31000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const OPTS = { + chunkTimeoutMs: 50, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + rateLimitCooldownsMs: [150, 150], + debug: true, +} + +async function setup( + logFile: string, + history: unknown[] = [], + extra: Record = {}, + contextDelayMs = 0, +): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + + const ctx: any = { + event: stream, + options: { ...OPTS, logFile, ...extra, registryScope: LOG_SALT }, + session: { + context: async () => { + if (contextDelayMs) await wait(contextDelayMs) + return history + }, + active: async () => ({}), + list: async () => [{ id: SID }], + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => [{ id: "bash" }], + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(80) + return { + injected, + cleanup, + logFile, + push: (type: string, data: Record = {}) => stream.push(ev(type, data)), + logs: () => (existsSync(logFile) ? readFileSync(logFile, "utf8") : ""), + } +} + +const freshLog = () => { + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-prodshape-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + return logFile +} + +// --- prod-shaped fixtures --------------------------------------------------- +// `role` + flat `text`, exactly as opencode.db stores them. + +const prodUser = (at: number, text = "do the thing") => ({ role: "user", text, time: { created: at } }) + +const prodAssistant = (at: number, content: Array>) => ({ + role: "assistant", + content, + time: { created: at }, +}) + +// The shape the rest of the suite uses. Kept as a control: the fix must make +// both shapes work, not trade one for the other. +const TYPE_USER = (at: number) => ({ type: "user", content: [{ type: "text", text: "go" }], time: { created: at } }) +const TYPE_ASSISTANT = (at: number, content: any[]) => ({ type: "assistant", content, time: { created: at } }) + +describe("v2: history readers accept the production message shape", () => { + test("PROD: a user message after a quota arm suppresses the restore (fails OPEN today)", async () => { + const logFile = freshLog() + const h1 = await setup(logFile) + h1.push("session.execution.started") + await wait(20) + h1.push("session.step.failed", { error: { type: "provider.quota", message: "Rate limit exceeded.", status: 429 } }) + await wait(150) + expect(h1.logs()).toContain("quota-armed sid=ses_prod") + ;(h1.cleanup as (() => void) | undefined)?.() + + // Restart with the user having replied since the arm. Role shape. + const h2 = await setup(logFile, [prodUser(Date.now(), "keep going")]) + await wait(400) + const log = h2.logs() + ;(h2.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(log).not.toContain("quota ladder restored") + }) + + test("PROD: a trailing tool error is retried (stuck-tool reader)", async () => { + const logFile = freshLog() + const now = Date.now() + // The user message must sit outside the 5-minute active window or the + // stand-down guard fires first and the reader is never reached. + const history = [ + prodUser(now - 600_000), + prodAssistant(now - 120_000, [ + { type: "text", text: "running it" }, + { + type: "tool", + name: "bash", + state: { status: "error", error: { type: "tool.error", message: "command not found" } }, + time: { created: now - 115_000 }, + }, + ]), + ] + const h = await setup(logFile, history) + h.push("session.idle") + await wait(500) + const log = h.logs() + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(log).toContain("failed at end of turn") + expect(h.injected.some((i: any) => /Retry it now/.test(i.text ?? ""))).toBe(true) + }) + + test("TYPE: the same trailing tool error still retries (no shape regression)", async () => { + const logFile = freshLog() + const now = Date.now() + const history = [ + TYPE_USER(now - 600_000), + TYPE_ASSISTANT(now - 120_000, [ + { type: "text", text: "running it" }, + { + type: "tool", + name: "bash", + state: { status: "error", error: { type: "tool.error", message: "command not found" } }, + time: { created: now - 115_000 }, + }, + ]), + ] + const h = await setup(logFile, history) + h.push("session.idle") + await wait(500) + const log = h.logs() + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(log).toContain("failed at end of turn") + }) + + test("PROD: a reasoning-only turn is nudged (silent-reasoning reader)", async () => { + const logFile = freshLog() + const now = Date.now() + const history = [ + prodUser(now - 600_000), + prodAssistant(now - 60_000, [{ type: "reasoning", text: "thinking about it" }]), + ] + const h = await setup(logFile, history) + h.push("session.idle") + await wait(500) + const log = h.logs() + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(log).toContain("reasoning-only turn with no output") + }) + + test("PROD: a restored ladder does not re-announce a notice it already sent", async () => { + const logFile = freshLog() + const h1 = await setup(logFile) + h1.push("session.execution.started") + await wait(20) + h1.push("session.step.failed", { error: { type: "provider.quota", message: "Rate limit exceeded.", status: 429 } }) + await wait(150) + ;(h1.cleanup as (() => void) | undefined)?.() + + // Reload mid-episode, then a fresh quota hit. The first-arm notice went + // out before the reload; restoring the ladder must carry that fact over + // or the user gets a duplicate "retrying in ~Xm" for the same episode. + const h2 = await setup(logFile, []) + await wait(300) + h2.push("session.execution.started") + await wait(20) + h2.push("session.step.failed", { error: { type: "provider.quota", message: "Rate limit exceeded.", status: 429 } }) + await wait(300) + const notices = h2.injected.filter((i: any) => /Rate limited, retrying in/.test(i.text ?? "")).length + ;(h2.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(notices).toBeLessThanOrEqual(1) + }) + + test("PROD: a child session that died on an error is seen as crashed", async () => { + // The child-status reader keys off the last message being an assistant + // message carrying `error`. Prod sends `role`, so a `type`-only read + // never matched and a crashed subagent looked merely slow. + const logFile = freshLog() + const CHILD = "ses_child_prod" + const now = Date.now() + const stream = makeEventStream() + const injected: Array<{ text?: string }> = [] + const childHistory = [{ role: "assistant", error: { type: "api.error", message: "boom" }, time: { created: now } }] + + const ctx: any = { + event: stream, + options: { ...OPTS, logFile, subagentDeadMs: 60_000, subagentWaitMs: 50, registryScope: LOG_SALT }, + session: { + context: async (a: any) => (a?.sessionID === CHILD ? childHistory : []), + active: async () => ({}), + list: async () => [{ id: SID }, { id: CHILD, parentID: SID }], + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + client: { session: { list: async () => ({ data: [{ id: SID }, { id: CHILD, parentID: SID }] }) } }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => [{ id: "bash" }], + }, + } + const cleanup = await (plugin as any).setup(ctx) + await wait(80) + // Parent busy, child idle with a dead-on-arrival error. + stream.push(ev("session.execution.started")) + await wait(20) + stream.push(ev("session.step.started")) + await wait(20) + stream.push({ type: "session.idle", data: { sessionID: CHILD } }) + await wait(700) + stream.push({ type: "session.idle", data: { sessionID: SID } }) + await wait(900) + const log = existsSync(logFile) ? readFileSync(logFile, "utf8") : "" + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(log).toContain("reported an error") + }) + + test("GUARD: no message reader compares a bare `.type` against a role", async () => { + // The regression this file exists for. Every occurrence below is a + // reader that goes blind when the server sends `role`, and each one + // passed the suite because the fixtures used `type`. Part-level checks + // (`part?.type === "tool"`) are correct and must not match. + const src = readFileSync(join(import.meta.dir, "index.ts"), "utf8") + const blind = src.match(/\b(?:m|last|msg)\s*\??\.\s*type\s*(?:===|!==)\s*"(?:user|assistant)"/g) ?? [] + expect(blind.map((s) => s.replace(/\s+/g, ""))).toEqual([]) + }) + + test("PROD: every reader routes through the shared accessors", async () => { + // The two blind readings are gone, but so could the next one be. Count + // the call sites: the fix must actually be load-bearing in the six + // readers the audit named, not merely present in the file. + const src = readFileSync(join(import.meta.dir, "index.ts"), "utf8") + const uses = (src.match(/\bmsgRole\(/g) ?? []).length + const ats = (src.match(/\bmsgAt\(/g) ?? []).length + // One definition of each plus the production call sites. + expect(uses).toBeGreaterThanOrEqual(8) + expect(ats).toBeGreaterThanOrEqual(5) + }) +}) + +describe("v2: watchdog re-entrancy", () => { + test("an overrunning sweep is not overlapped by the next tick", async () => { + const logFile = freshLog() + const now = Date.now() + const history = [ + prodUser(now - 600_000), + prodAssistant(now - 120_000, [ + { + type: "tool", + name: "bash", + state: { status: "error", error: { type: "tool.error", message: "nope" } }, + time: { created: now - 115_000 }, + }, + ]), + ] + // context() takes 150ms against a 20ms interval: without a re-entrancy + // guard every tick starts another concurrent sweep over the same watch. + const h = await setup(logFile, history, {}, 150) + h.push("session.idle") + await wait(900) + const retries = h.injected.filter((i: any) => /Retry it now/.test(i.text ?? "")).length + const skips = (h.logs().match(/watchdog sweep still running/g) ?? []).length + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + // One stall is one recovery. Concurrent sweeps would double-inject. + expect(retries).toBeLessThanOrEqual(1) + // And the guard is provably what prevents it: an overrun actually + // happened, so this is not a pass by never racing. + expect(skips).toBeGreaterThan(0) + }) + + test("GUARD: the watchdog interval is guarded against overlap", async () => { + // Belt and braces: a behavioural skip can be satisfied by other guards, + // so assert the guard itself is still in the source. + const src = readFileSync(join(import.meta.dir, "index.ts"), "utf8") + expect(src).toContain("watchdogBusy") + }) + + test("GUARD: the checked tool-part id set is bounded", async () => { + const src = readFileSync(join(import.meta.dir, "index.ts"), "utf8") + expect(src).toContain("CHECKED_TOOL_PART_IDS_MAX") + }) +}) \ No newline at end of file diff --git a/src/v2/index.provider-fault-ladder.test.ts b/src/v2/index.provider-fault-ladder.test.ts new file mode 100644 index 0000000..130757a --- /dev/null +++ b/src/v2/index.provider-fault-ladder.test.ts @@ -0,0 +1,184 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE — see index.attempt-reset.test.ts for why. */ +const LOG_SALT = "IndexProviderFaultLadderTest" + +/** + * v2: a stall the provider is already retrying is not a model stall. + * + * ses_eae46bb9 (2026-10-19): three `provider.transport` ECONNRESETs, each + * immediately followed by opencode's own `session.retry.scheduled` ("provider + * retry #2"), drove the ladder 1/3 -> 2/3 -> restart -> 1/3. The plugin was + * counting a transport fault as model unresponsiveness and climbing toward + * gaveUp, which is terminal silence: the sweep is gated on `!gaveUp` and only a + * genuine user message clears it. A scheduled retry means the provider is + * already recovering the turn, so the rung must hold and the ladder must never + * give up — the nudge still fires (a dead stream needs waking), it just does + * not escalate. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 22000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +async function setup(extraOpts: Record = {}): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 100, + maxBackoffMs: 200, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + ...extraOpts, + }, + session: { + context: async () => [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) +const readLog = (h: any) => (existsSync(h.logFile) ? readFileSync(h.logFile, "utf8") : "") + +async function waitFor(what: () => boolean, budgetMs = 3000): Promise { + const deadline = Date.now() + budgetMs + while (Date.now() < deadline) { + if (what()) return true + await wait(10) + } + return what() +} + +const attemptOf = (text: string | undefined): string | null => { + const m = /attempt (\d)\/3/.exec(text ?? "") + return m ? m[1] : null +} + +// The exact recorded failure shape: ECONNRESET while reading the stream, which +// opencode answers with its own provider retry. +const ECONNRESET = { + error: { + type: "provider.transport", + message: + "Connection lost while reading the response: ECONNRESET: The socket connection was closed unexpectedly.", + }, +} + +const retryScheduled = (h: any, attempt = 2) => + push(h, "session.retry.scheduled", SID, { ...ECONNRESET, attempt }) + +describe("v2: a provider-scheduled retry holds the ladder and never gives up", () => { + test("consecutive provider-fault stalls stay on the same rung", async () => { + const h = await setup() + push(h, "session.execution.started", SID) + await wait(20) + retryScheduled(h) + await wait(10) + push(h, "session.step.failed", SID, ECONNRESET) + await waitFor(() => h.injected.length >= 1) + retryScheduled(h) + await wait(10) + push(h, "session.step.failed", SID, ECONNRESET) + await waitFor(() => h.injected.length >= 2) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + const logs = readLog(h) + await teardown(h) + // Pre-fix this climbed to ["1","2"] and could then latch gaveUp. + expect(attempts.slice(0, 2)).toEqual(["1", "1"]) + expect(logs).toContain("provider-fault stall") + expect(logs).not.toContain("giving up after") + }) + + test("the nudge still fires — holding the rung is not silence", async () => { + // The failure path that stops prodding is worse than a wrong number: + // assert a real prompt reached the session on every provider-fault stall. + const h = await setup() + push(h, "session.execution.started", SID) + await wait(20) + retryScheduled(h) + await wait(10) + push(h, "session.step.failed", SID, ECONNRESET) + await waitFor(() => h.injected.length >= 1) + const first = h.injected[0]?.text ?? "" + await teardown(h) + expect(first).toContain("continue") + }) +}) diff --git a/src/v2/index.quota-restore.test.ts b/src/v2/index.quota-restore.test.ts new file mode 100644 index 0000000..f80026f --- /dev/null +++ b/src/v2/index.quota-restore.test.ts @@ -0,0 +1,173 @@ +import { describe, test, expect } from "bun:test" +import { rmSync, readFileSync, existsSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexQuotaRestoreTest" + +/** + * v2: quota ladders survive a plugin reload. + * + * ses_ef72c5f3 (2026-10-09): quota failure armed the ladder at 14:31:49 + * (rung 1/8, next attempt in 900s), the plugin reloaded at 14:37:28, and the + * gate due 14:46:49 never fired — `dispose()` clears the session map, wiping + * `awaitingQuotaRetry`/`rateLimitedAt`/`rateLimitAttempts`, and no new failure + * event ever re-arms it. The design comment claimed "reloads lose nothing"; + * they lost the whole gate. + * + * The `quota-armed` marker lines in the file-backed log are the record. On + * setup, `restoreQuotaLadders()` re-arms still-listed sessions from them + * unless a user message newer than the arm means the ladder already reset. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 18000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +async function setup(logFile: string, history: unknown[] = []): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 50, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + rateLimitCooldownsMs: [150, 150], + logFile, + debug: true, + }, + session: { + context: async () => history, + active: async () => ({}), + list: async () => [{ id: SID }], + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(80) + return { + injected, + cleanup, + logFile, + push: (type: string, data: Record = {}) => stream.push(ev(type, SID, data)), + logs: () => (existsSync(logFile) ? readFileSync(logFile, "utf8") : ""), + } +} + +const OLD = Date.now() - 3600_000 + +describe("v2: quota ladders survive a plugin reload", () => { + test("a fresh evaluation re-arms from the marker and retries", async () => { + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-quota-restore-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + // First evaluation: quota failure arms the ladder (marker written). + const h1 = await setup(logFile) + h1.push("session.execution.started") + await wait(20) + h1.push("session.step.failed", { error: { type: "provider.quota", message: "Rate limit exceeded.", status: 429 } }) + await wait(150) + expect(h1.logs()).toContain("quota-armed sid=ses_parent") + ;(h1.cleanup as (() => void) | undefined)?.() + const marked = h1.logs() + + // Second evaluation (the reload): same log file, fresh memory. + const h2 = await setup(logFile) + await wait(600) + const log = h2.logs() + ;(h2.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(marked).not.toContain("quota ladder restored") + expect(log).toContain("quota ladder restored") + // And the restored gate actually serves: cooldowns are 150ms here. + expect(log).toContain("rate-limit cooldown served") + expect(h2.injected.length).toBeGreaterThanOrEqual(1) + }) + + test("CONTROL: a user message after the arm means no restore", async () => { + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-quota-restore-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const h1 = await setup(logFile) + h1.push("session.execution.started") + await wait(20) + h1.push("session.step.failed", { error: { type: "provider.quota", message: "Rate limit exceeded.", status: 429 } }) + await wait(150) + ;(h1.cleanup as (() => void) | undefined)?.() + + // User continued the session after the arm: ladder already reset. + const h2 = await setup(logFile, [ + { + type: "user", + id: "msg_u_new", + time: { created: Date.now() }, + content: [{ type: "text", text: "continue" }], + }, + ...[], + ]) + await wait(400) + const log = h2.logs() + ;(h2.cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + expect(log).not.toContain("quota ladder restored") + }) +}) diff --git a/src/v2/index.rate-limit.test.ts b/src/v2/index.rate-limit.test.ts new file mode 100644 index 0000000..3b42ef4 --- /dev/null +++ b/src/v2/index.rate-limit.test.ts @@ -0,0 +1,336 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexRateLimitTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_ratelimit" + +/** + * Rate-limit / quota failures. + * + * Observed 2026-10-04: `session.step.failed: provider.quota Rate limit + * exceeded. Please try again later.` fired a visible continue 1s later, twice + * in two seconds — retrying straight into the ban. The failure handler + * recovered from every non-abort, non-OOC error with no rate-limit + * classification at all. + * + * The rule: a quota hit stands down on a gate ladder (gates, not timers — + * armed timeouts do not survive a plugin reload), evaluated on each failure + * and each watchdog tick. Attempts consume their own budget; past it, silence + * until a genuine user turn. A fresh turn resets the ladder. + * + * Every group carries a control. + */ + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const QUOTA = { error: { type: "provider.quota", message: "Rate limit exceeded. Please try again later." } } +const STREAM_ERR = { error: { type: "StreamError", message: "connection reset" } } +const SLOT503 = { error: { type: "provider.http", message: "Request failed with status 503", status: 503 } } +const SLOT503_TEXT = { error: { type: "provider.transport", message: "upstream returned 503 Service Unavailable" } } + +const userTurn = () => ({ + type: "user", + id: "msg_u0", + time: { created: Date.now() - 60 * 60_000 }, + content: [{ type: "text", text: "do the thing" }], +}) + +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + debug: true, +} + +type Harness = { injected: Array<{ text?: string; resume?: boolean; synth?: boolean }>; logs: string[]; stream: any; cleanup?: () => void; logFile: string; failTurn: (err: unknown) => Promise; startTurn: () => Promise; failStep: (err: unknown) => Promise; idleTurn: () => Promise } + +async function replay(opts: Record = {}): Promise { + const injected: Harness["injected"] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-ratelimit-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, ...opts, registryScope: LOG_SALT }, + session: { + context: async () => [userTurn()], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => { + injected.push({ text: a?.text, resume: a?.resume, synth: true }) + return {} + }, + prompt: async (a: any) => { + injected.push({ text: a?.text }) + return {} + }, + }, + client: { session: { get: async () => ({ data: {} }) } }, + } + + const cleanup = await (plugin as any).setup(ctx) + // Split on purpose: a new execution re-arms the ladder (fresh turn = + // activity = quota available again), while repeated step failures inside + // one execution share a single ladder episode. + const startTurn = async () => { + for (const e of [ev("session.execution.started"), ev("session.step.started")]) { + stream.push(e) + await wait(10) + } + } + const failStep = async (err: unknown) => { + stream.push(ev("session.step.failed", err as Record)) + await wait(10) + } + const idleTurn = async () => { + stream.push(ev("session.idle")) + await wait(50) + } + const failTurn = async (err: unknown) => { + await startTurn() + await failStep(err) + } + return { + injected, + logs: [], + stream, + cleanup, + logFile, + failTurn, + startTurn, + failStep, + idleTurn, + } +} + +function readLogs(h: Harness): string[] { + const logs = existsSync(h.logFile) ? readFileSync(h.logFile, "utf8").split("\n") : [] + rmSync(h.logFile, { force: true }) + return logs +} + +describe("v2: rate-limit failures stand down on a ladder", () => { + test("CONTROL: a non-rate failure recovers immediately", async () => { + const h = await replay() + await h.failTurn(STREAM_ERR) + await wait(600) + expect(h.injected.length).toBeGreaterThanOrEqual(1) + h.cleanup?.() + readLogs(h) + }) + + test("a quota hit injects nothing inside the cooldown except the notice", async () => { + const h = await replay({ rateLimitCooldownsMs: [60_000] }) + await h.failTurn(QUOTA) + await wait(300) + // Only the quota notice — no recovery prompt while cooling down. + expect(h.injected.length).toBe(1) + expect(h.injected[0].text).toMatch(/Rate limited, retrying in/) + const logs = readLogs(h) + expect(logs.some((l) => l.includes("rate limited") && l.includes("standing down"))).toBe(true) + expect(logs.some((l) => l.includes("session.step.failed") && l.includes("resume attempt"))).toBe(false) + h.cleanup?.() + }) + + test("quota stand-down posts one visible notice with wait + attempt", async () => { + // ses_edd3d940 (2026-10-10): standing down silently reads as dead. + // One synthetic notice (resume:false — never kicks a turn), then + // quiet: a second failure inside the cooldown adds no second notice. + const h = await replay({ rateLimitCooldownsMs: [60_000] }) + await h.failTurn(QUOTA) + await wait(300) + const notices = () => h.injected.filter((i) => /Rate limited, retrying in/.test(i.text ?? "")) + expect(notices()).toHaveLength(1) + expect(notices()[0].text).toMatch(/attempt 1\/1/) + expect(notices()[0].text).toMatch(/~1m|~60m|~1h/) + expect(notices()[0].resume).toBeFalsy() + expect(notices()[0].synth).toBe(true) + await h.failStep(QUOTA) + await wait(300) + expect(notices()).toHaveLength(1) + h.cleanup?.() + }) + + test("quota budget exhaustion posts the standing-down notice once", async () => { + const h = await replay({ rateLimitCooldownsMs: [30, 30] }) + // No new turns between failures: same episode climbs the ladder to + // exhaustion (a fresh startTurn would reset it — see next test). + await h.startTurn() + for (let i = 0; i < 3; i++) { + await h.failStep(QUOTA) + await wait(200) + } + const endNotices = h.injected.filter((i) => /standing down until you return/.test(i.text ?? "")) + expect(endNotices).toHaveLength(1) + expect(endNotices[0].resume).toBeFalsy() + h.cleanup?.() + }) + + test("CONTROL: non-quota failure posts no quota notice", async () => { + const h = await replay({ rateLimitCooldownsMs: [60_000] }) + await h.failTurn(STREAM_ERR) + await wait(300) + expect(h.injected.some((i) => /Rate limited/.test(i.text ?? ""))).toBe(false) + h.cleanup?.() + }) + + test("a second quota hit inside the cooldown adds no second notice", async () => { + const h = await replay({ rateLimitCooldownsMs: [60_000] }) + await h.startTurn() + await h.failStep(QUOTA) + await h.failStep(QUOTA) + await wait(300) + // Exactly the one first-arm notice; the second hit stays silent. + expect(h.injected.length).toBe(1) + expect(h.injected[0].text).toMatch(/Rate limited, retrying in/) + h.cleanup?.() + readLogs(h) + }) + + test("a served cooldown retries through the normal path", async () => { + const h = await replay({ rateLimitCooldownsMs: [50] }) + await h.failTurn(QUOTA) + await wait(500) + // The first-arm notice plus the served recovery prompt. + expect(h.injected.length).toBe(2) + expect(h.injected.some((i) => /Rate limited, retrying in/.test(i.text ?? ""))).toBe(true) + const logs = readLogs(h) + expect(logs.some((l) => l.includes("cooldown served"))).toBe(true) + h.cleanup?.() + }) + + test("past the ladder the exhausted notice fires once, then silence", async () => { + const h = await replay({ rateLimitCooldownsMs: [20] }) + await h.startTurn() + await h.failStep(QUOTA) + await wait(300) + // Armed notice + served recovery inside the wait. + expect(h.injected).toHaveLength(2) + // Budget spent (1 rung): the next hit moves to exhausted (notice + + // recovery + exhausted notice), then silence. + await h.failStep(QUOTA) + await wait(300) + expect(h.injected).toHaveLength(3) + expect(h.injected[2].text).toMatch(/standing down until you return/) + const logs = readLogs(h) + expect(logs.some((l) => l.includes("budget exhausted"))).toBe(true) + h.cleanup?.() + }) + + test("a fresh turn resets the ladder", async () => { + const h = await replay({ rateLimitCooldownsMs: [20] }) + await h.startTurn() + await h.failStep(QUOTA) + await wait(300) + // Notice + served recovery (20ms ladder serves inside the wait). + expect(h.injected).toHaveLength(2) + await h.failStep(QUOTA) + await wait(200) + // New turn = activity = quota available again: fresh ladder, so the + // next quota hit stands down with a new cooldown instead of silence. + // The idle first ends our recovery turn (consuming the self-cause + // flag), so the new execution reads as genuine. + await h.idleTurn() + await h.startTurn() + await h.failStep(QUOTA) + await wait(200) + const logs = readLogs(h) + expect(logs.filter((l) => l.includes("next attempt in")).length).toBeGreaterThanOrEqual(2) + h.cleanup?.() + }) + + test("a 503 recovers like a generic failure (no slot ladder)", async () => { + // SelfRef #5 (2026-10-10): TabbyAPI queues past max-batch-size, it + // never 503s on overflow — the 503s we saw were the DFlash2 draft + // crash bug. A 503 is breakage like any other provider failure: it + // recovers through the normal failure path, with no patient ladder. + const h = await replay() + await h.startTurn() + await h.failStep(SLOT503) + await wait(300) + expect(h.injected.length).toBeGreaterThanOrEqual(1) + const logs = readLogs(h) + expect(logs.some((l) => /slot.busy/i.test(l))).toBe(false) + h.cleanup?.() + }) + + test("503 told only in prose behaves the same", async () => { + const h = await replay() + await h.startTurn() + await h.failStep(SLOT503_TEXT) + await wait(300) + expect(h.injected.length).toBeGreaterThanOrEqual(1) + const logs = readLogs(h) + expect(logs.some((l) => /slot.busy/i.test(l))).toBe(false) + h.cleanup?.() + }) + + test("repeated 503s latch the failure loop like generic failures", async () => { + // Six hits: a couple of recoveries fire (bounded), then the + // consecutive-failure latch stands the session down until a genuine + // turn — the same shape as any repeated provider failure. Exact + // delivery count stays loose (identical-text dedup may merge + // same-minute repeats); the latch is the contract. + const h = await replay({ loopMaxContinues: 99, maxRetries: 9 }) + await h.startTurn() + for (let i = 0; i < 6; i++) { + await h.failStep(SLOT503) + await wait(150) + } + const logs = readLogs(h) + expect(logs.some((l) => l.includes("standing down until a genuine turn"))).toBe(true) + expect(h.injected.length).toBeGreaterThanOrEqual(1) + expect(h.injected.length).toBeLessThanOrEqual(3) + h.cleanup?.() + }) +}) diff --git a/src/v2/index.replay-eae46bb9.test.ts b/src/v2/index.replay-eae46bb9.test.ts new file mode 100644 index 0000000..5530549 --- /dev/null +++ b/src/v2/index.replay-eae46bb9.test.ts @@ -0,0 +1,201 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE — see index.attempt-reset.test.ts for why. */ +const LOG_SALT = "IndexReplayEae46bb9Test" + +/** + * v2 replay of ses_eae46bb9effeYI09ys4UflpjxM (2026-10-19, log + * ~/.local/state/opencode-v2/auto-resume.log). The recorded sequence the user + * reported as "2/3 before resetting to 1/3, where was the 1/3?": + * + * 18:15:26 stall (no activity) -> delivered "attempt 1/3" + * 18:43:54 ECONNRESET + provider retry -> armed "attempt 1/3", DECLINED + * ("still busy and live at inject time" — the provider's own retry + * had revived the session 31ms later). Nothing was delivered, but + * the rung had already been counted. + * 18:44:29 ECONNRESET + provider retry -> delivered "attempt 2/3" + * 18:44:56 ECONNRESET + provider retry -> text growth restarted the ladder + * -> delivered "attempt 1/3" + * + * So the first visible prod was 2/3 with no 1/3 before it, and the ladder + * climbed a fault opencode was already retrying. This replay asserts the + * fixed reading: every prod is 1/3, the declined cycle delivers nothing, and + * the ladder never runs out on a transient provider fault. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_eae46bb9effeYI09ys4UflpjxM" + +let counter = 23000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +async function setup(extraOpts: Record = {}): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + // Parked so only the recorded failure stalls can prod (the incident's + // 186s silence stalls cannot be reproduced in a test budget; the state + // they leave behind — our own prod as the newest user-side message — + // is what the replay needs, and the pre-fire below establishes it). + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 150, + maxBackoffMs: 300, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + ...extraOpts, + }, + session: { + context: async () => [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) +const readLog = (h: any) => (existsSync(h.logFile) ? readFileSync(h.logFile, "utf8") : "") + +async function waitFor(what: () => boolean, budgetMs = 3000): Promise { + const deadline = Date.now() + budgetMs + while (Date.now() < deadline) { + if (what()) return true + await wait(10) + } + return what() +} + +const attemptOf = (text: string | undefined): string | null => { + const m = /attempt (\d)\/3/.exec(text ?? "") + return m ? m[1] : null +} + +const FAIL = { error: { type: "provider.invalid-request", message: "Error from the provider: 400 Bad Request" } } + +// Verbatim shape of the three recorded failures. +const ECONNRESET = { + error: { + type: "provider.transport", + message: + "Connection lost while reading the response: ECONNRESET: The socket connection was closed unexpectedly.", + }, +} +const retryScheduled = (h: any) => push(h, "session.retry.scheduled", SID, { ...ECONNRESET, attempt: 2 }) + +describe("v2 replay: ses_eae46bb9 — ECONNRESET cycles must read 1/3 and 1/3", () => { + test("the phantom 2/3 is gone and the ladder survives the provider fault", async () => { + const h = await setup() + // Pre-incident episode (18:15:26): one delivered prod. + push(h, "session.execution.started", SID) + await wait(20) + push(h, "session.step.failed", SID, FAIL) + await waitFor(() => h.injected.length >= 1) + expect(attemptOf(h.injected[0]?.text)).toBe("1") + + // Cycle 1 (18:43:54): the armed prod is declined — the provider retry + // moved the session 31ms after the failure. + retryScheduled(h) + push(h, "session.step.failed", SID, ECONNRESET) + await wait(40) + push(h, "session.step.started", SID, { agent: "build" }) + await waitFor(() => readLog(h).includes("still busy and live at inject time"), 2000) + const afterCycle1 = h.injected.length + + // Cycle 2 (18:44:29). + retryScheduled(h) + push(h, "session.step.failed", SID, ECONNRESET) + await waitFor(() => h.injected.length > afterCycle1, 3000) + + // Cycle 3 (18:44:56): the stream had emitted text before the socket died + // — in the original this restarted the ladder, printing 1/3 straight + // after 2/3. + push(h, "session.text.delta", SID, { assistantMessageID: "m1", delta: "partial answer before the socket died" }) + await wait(20) + retryScheduled(h) + push(h, "session.step.failed", SID, ECONNRESET) + await waitFor(() => h.injected.length > afterCycle1 + 1, 3000) + + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + const logs = readLog(h) + await teardown(h) + + expect(afterCycle1).toBe(1) // the declined cycle delivered nothing + expect(attempts).not.toContain("2") // no phantom rung + expect(attempts).toEqual(["1", "1", "1"]) // and no escalating on a fault + expect(logs).toContain("rung not consumed") + expect(logs).toContain("provider-fault stall") + expect(logs).not.toContain("giving up after") + }) +}) diff --git a/src/v2/index.saturation.test.ts b/src/v2/index.saturation.test.ts new file mode 100644 index 0000000..dbd70f4 --- /dev/null +++ b/src/v2/index.saturation.test.ts @@ -0,0 +1,357 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexSaturationTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_saturation" +const PROVIDER = "llama-server" +const MODEL = "coder" + +/** + * Context saturation. + * + * A session can fill its context window without ever looking stalled — it just + * keeps working until it chokes. That failure mode is invisible to the stall + * watchdog by construction, so it gets its own check on the idle path. + * + * Routing, as in v1: + * subagent — opt-in (`subagentNativeCompactionEnabled`) then native compaction. + * magic-context is not involved. + * parent — only when magic-context is installed, because its setup disables + * native compaction and compacting here would double-compress. + * + * The v2 API shape is what these tests are really pinning: the token count comes + * from the `session.usage.updated` event, the window from `ctx.model.get()`, the + * magic-context verdict from `ctx.plugin.list()` (v2 removed the `config` domain + * v1 used), and compaction from `session.compact` — which is not on the plugin's + * narrowed `session` Pick, so it goes through the client fallback. + * + * Every group carries a control. A test that passes because the saturation branch + * never ran is worse than no test here. + */ + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const userTurn = () => ({ + type: "user", + id: "msg_u0", + // Old enough that activeUserWindowMs cannot stand the nudge down. + time: { created: Date.now() - 60 * 60_000 }, + content: [{ type: "text", text: "keep going" }], +}) + +/** `TokenUsage.Info` for the given totals; `cache` is always present, as in the schema. */ +const usage = (input: number, output = 0, reasoning = 0, read = 0, write = 0) => ({ + input, + output, + reasoning, + cache: { read, write }, +}) + +type HarnessOpts = { + /** Tokens reported by `session.usage.updated`. Omit for "no usage event". */ + tokens?: Record + /** `Model.Info.limit`. */ + limit?: { context: number; output: number } + parentID?: string + /** Installed plugins as `ctx.plugin.list()` returns them. */ + plugins?: unknown[] + /** Host exposes `session.compact` on the plugin domain. */ + domainCompact?: boolean + /** Host exposes `session.compact` only via the client. */ + clientCompact?: boolean + /** A genuine user interrupt lands before the idle. */ + interruptFirst?: boolean + opts?: Record +} + +async function replay(h: HarnessOpts) { + const commands: Array<{ sessionID: string; name: string }> = [] + const compacted: string[] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-saturation-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const limit = h.limit ?? { context: 200_000, output: 32_000 } + const model = { id: MODEL, providerID: PROVIDER, limit, name: MODEL } + + const session: Record = { + context: async () => [userTurn(), { type: "assistant", id: "msg_a1", model: { providerID: PROVIDER, id: MODEL } }], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async () => ({}), + prompt: async () => ({}), + command: async (a: any) => { + commands.push({ sessionID: a?.sessionID, name: a?.name }) + return {} + }, + } + if (h.domainCompact) session.compact = async (a: any) => (compacted.push(a?.sessionID), {}) + if (!h.domainCompact) session.compact = undefined + + const client: Record = { + session: { + get: async () => ({ data: h.parentID ? { parentID: h.parentID } : {} }), + }, + } + if (h.clientCompact) (client.session as any).compact = async (a: any) => compacted.push(a?.sessionID) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + injectIntervalMs: 0, + logFile, + ...(h.opts ?? {}), + }, + session, + client, + model: { get: (providerID: string, modelID: string) => (providerID === PROVIDER && modelID === MODEL ? model : undefined) }, + plugin: { list: async () => h.plugins ?? [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + + const events: any[] = [ + ev("session.execution.started", { agent: "build", model: { providerID: PROVIDER, id: MODEL } }), + ev("session.text.delta", { messageID: "msg_a1", delta: "Working through the list." }), + ev("session.text.ended", { messageID: "msg_a1" }), + ] + if (h.tokens) events.splice(1, 0, ev("session.usage.updated", { tokens: usage(h.tokens.input, h.tokens.output, h.tokens.reasoning, h.tokens.read, h.tokens.write) })) + if (h.interruptFirst) events.push(ev("session.execution.interrupted", { reason: "user" })) + events.push(ev("session.idle")) + + for (const e of events) { + stream.push(e) + await wait(10) + } + await wait(600) + ;(cleanup as (() => void) | undefined)?.() + + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + rmSync(logFile, { force: true }) + return { commands, compacted, logs } +} + +/** A parent with magic-context installed and a window well over the threshold. */ +const SATURATED_PARENT: HarnessOpts = { + tokens: { input: 160_000 }, + plugins: [{ id: "magic-context", source: { type: "local", path: "/x/magic-context" } }], +} +/** Same, but only a small slice of the window used. */ +const HEALTHY_PARENT: HarnessOpts = { ...SATURATED_PARENT, tokens: { input: 10_000 } } + +describe("v2: context saturation — parent sessions", () => { + test("CONTROL: a saturated parent with magic-context installed gets the wrapup command", async () => { + const { commands, logs } = await replay(SATURATED_PARENT) + expect(commands).toHaveLength(1) + // Sent as a command, not as prompt text: prompt text is not expanded into + // a command, which was v1's documented reason for using the endpoint. + expect(commands[0]).toEqual({ sessionID: SID, name: "ctx-wrapup" }) + expect(logs.some((l) => l.includes("context saturation:"))).toBe(true) + }) + + test("a parent below the threshold is left alone", async () => { + const { commands, compacted, logs } = await replay(HEALTHY_PARENT) + expect(commands).toEqual([]) + expect(compacted).toEqual([]) + expect(logs.filter((l) => l.includes("context saturation"))).toEqual([]) + }) + + test("no usage event means no token count, so no intervention", async () => { + const { commands } = await replay({ plugins: SATURATED_PARENT.plugins }) + expect(commands).toEqual([]) + }) + + test("a saturated parent with no magic-context is NOT compacted — that would double-compress", async () => { + const { commands, compacted } = await replay({ tokens: { input: 160_000 }, plugins: [] }) + expect(commands).toEqual([]) + expect(compacted).toEqual([]) + }) + + test("magic-context is detected from a package spec, not just an id", async () => { + // v1 matched `config.get().plugin` entries, which are package specs. v2 has + // no config domain, so the source has to be matched too. + const { commands } = await replay({ + tokens: { input: 160_000 }, + plugins: [{ id: "pkg-1", source: { type: "package", target: "@someone/opencode-magic-context@1.2.3" } }], + }) + expect(commands).toHaveLength(1) + }) + + test("a host with no plugin inventory treats magic-context as absent", async () => { + const logs: string[] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-saturation-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const ctx: any = { + event: stream, + options: { warmupMs: 0, checkIntervalMs: 20, chunkTimeoutMs: 600_000, gracePeriodMs: 0, logFile, registryScope: LOG_SALT }, + session: { context: async () => [], active: async () => ({}), interrupt: async () => ({}), synthetic: async () => ({}) }, + model: { get: () => ({ limit: { context: 200_000, output: 32_000 } }) }, + } + const cleanup = await (plugin as any).setup(ctx) + stream.push(ev("session.step.started", { model: { providerID: PROVIDER, id: MODEL } })) + stream.push(ev("session.usage.updated", { tokens: usage(160_000) })) + stream.push(ev("session.idle")) + await wait(400) + ;(cleanup as (() => void) | undefined)?.() + if (existsSync(logFile)) logs.push(...readFileSync(logFile, "utf8").split("\n")) + rmSync(logFile, { force: true }) + // Must not raise, and must not invent a verdict. + expect(logs.filter((l) => l.includes("ERROR"))).toEqual([]) + }) + + test("the intervention is one-shot: a second idle does not repeat it", async () => { + const { commands } = await replay(SATURATED_PARENT) + expect(commands).toHaveLength(1) + }) + + test("a genuine user interrupt suppresses the intervention entirely", async () => { + // The guard reads the `userCancelled` latch, which only a non-self abort sets. + // Assert the negative branch: the same saturated parent, interrupted first. + const { commands, compacted, logs } = await replay({ ...SATURATED_PARENT, interruptFirst: true }) + expect(commands).toEqual([]) + expect(compacted).toEqual([]) + expect(logs.filter((l) => l.includes("context saturation"))).toEqual([]) + }) +}) + +describe("v2: context saturation — subagent sessions", () => { + const SATURATED_SUBAGENT: HarnessOpts = { ...SATURATED_PARENT, parentID: "ses_parent" } + + test("CONTROL: a saturated subagent with the opt-in enabled triggers native compaction", async () => { + const { compacted, commands, logs } = await replay({ + ...SATURATED_SUBAGENT, + domainCompact: true, + opts: { subagentNativeCompactionEnabled: true }, + }) + expect(compacted).toEqual([SID]) + // magic-context must not be involved on this path even when installed. + expect(commands).toEqual([]) + expect(logs.some((l) => l.includes("context saturation (subagent)"))).toBe(true) + }) + + test("a saturated subagent without the opt-in is left alone", async () => { + const { compacted, commands } = await replay({ ...SATURATED_SUBAGENT, domainCompact: true }) + expect(compacted).toEqual([]) + expect(commands).toEqual([]) + }) + + test("compaction reaches the host through the client fallback when the domain omits it", async () => { + // `session.compact` is NOT in v2's plugin `session` Pick. The same defensive + // lookup that carries discovery must carry this. + const { compacted } = await replay({ + ...SATURATED_SUBAGENT, + clientCompact: true, + opts: { subagentNativeCompactionEnabled: true }, + }) + expect(compacted).toEqual([SID]) + }) + + test("a host exposing neither compaction path warns instead of throwing", async () => { + const { logs } = await replay({ ...SATURATED_SUBAGENT, opts: { subagentNativeCompactionEnabled: true } }) + expect(logs.some((l) => l.includes("native compaction is not exposed on this host"))).toBe(true) + expect(logs.filter((l) => l.includes("ERROR"))).toEqual([]) + }) +}) + +describe("v2: the saturation arithmetic matches v1", () => { + test("the output reserve is min(20000, model output limit)", async () => { + // 200k context, 32k output -> usable 180k. 170k used is below 0.85*180k=153k? + // No: 170k > 153k, so it fires. Drop to 150k and it must not. + const fired = await replay({ ...SATURATED_PARENT, tokens: { input: 170_000 } }) + expect(fired.commands).toHaveLength(1) + + const quiet = await replay({ ...SATURATED_PARENT, tokens: { input: 150_000 } }) + expect(quiet.commands).toEqual([]) + }) + + test("cache read and reasoning count toward the total", async () => { + // 140k input alone is below the threshold; the cache read pushes it over. + // `TokenUsage.total` includes both, and so does this. + const without = await replay({ ...SATURATED_PARENT, tokens: { input: 140_000 } }) + expect(without.commands).toEqual([]) + + const withCache = await replay({ ...SATURATED_PARENT, tokens: { input: 140_000, read: 60_000 } }) + expect(withCache.commands).toHaveLength(1) + }) + + test("contextSaturationThreshold is honoured as a fraction, not a percentage", async () => { + const at99 = await replay({ ...SATURATED_PARENT, tokens: { input: 178_000 }, opts: { contextSaturationThreshold: 0.99 } }) + expect(at99.commands).toEqual([]) + + const at90 = await replay({ ...SATURATED_PARENT, tokens: { input: 178_000 }, opts: { contextSaturationThreshold: 0.9 } }) + expect(at90.commands).toHaveLength(1) + }) + + test("an unknown model yields no limit, so no intervention", async () => { + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-saturation-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const ctx: any = { + event: stream, + options: { warmupMs: 0, checkIntervalMs: 20, chunkTimeoutMs: 600_000, gracePeriodMs: 0, logFile, registryScope: LOG_SALT }, + session: { context: async () => [], active: async () => ({}), interrupt: async () => ({}), synthetic: async () => ({}) }, + model: { get: () => undefined }, + plugin: { list: async () => [{ id: "magic-context" }] }, + } + const cleanup = await (plugin as any).setup(ctx) + stream.push(ev("session.step.started", { model: { providerID: PROVIDER, id: "not-installed" } })) + stream.push(ev("session.usage.updated", { tokens: usage(999_000) })) + stream.push(ev("session.idle")) + await wait(400) + ;(cleanup as (() => void) | undefined)?.() + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8") : "" + rmSync(logFile, { force: true }) + expect(logs).not.toContain("context saturation") + }) +}) diff --git a/src/v2/index.send-timeout.test.ts b/src/v2/index.send-timeout.test.ts new file mode 100644 index 0000000..c0cae1d --- /dev/null +++ b/src/v2/index.send-timeout.test.ts @@ -0,0 +1,154 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexSendTimeoutTest" + +/** + * v2: wedged sends time out instead of stacking into twins. + * + * ses_ee7dade1 (2026-10-07): two rich stall texts 5ms apart carrying + * attempt counters 1 and 2, with only ONE detection log line in the window + * and zero reloads (single process, single setup — stacked instances ruled + * out). The consistent mechanism: an earlier cascade's send wedged in a + * transport await for minutes; a later cascade's send wedged behind it; + * both resolved together. No send await may hang unbounded: each send gets + * a timeout, after which the normal throw path (log verify + fallback) + * applies. Slow-but-completing sends log a latency warning for forensics. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 8000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +async function setup(promptImpl: (a: any) => Promise, opts: Record = {}): Promise { + const sends: Array<{ channel: string; at: number }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-sendtimeout-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const now = Date.now() + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 300, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 1, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, + visibleContinue: true, + sendTimeoutMs: 300, + ...opts, + logFile, + }, + session: { + context: async () => [ + { type: "assistant", time: { created: now - 5_000 }, content: [{ type: "text", text: "working" }] }, + ], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (sends.push({ channel: "synthetic", at: Date.now() }), {}), + prompt: async (a: any) => { + sends.push({ channel: "prompt", at: Date.now() }) + await promptImpl(a) + return {} + }, + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { sends, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +async function driveStall(h: any) { + h.streamRef.push(ev("session.execution.started", SID)) + await wait(10) + h.streamRef.push(ev("session.step.started", SID)) + await wait(900) +} + +describe("v2: wedged sends time out instead of stacking", () => { + test("a prompt that never resolves falls back instead of hanging", async () => { + const h = await setup(() => new Promise(() => {})) + const started = Date.now() + await driveStall(h) + const elapsed = Date.now() - started + // Prompt wedged forever: must fall back to synthetic promptly + // (sendTimeoutMs 300 + slack), never hang the recovery. + expect(h.sends.filter((s: any) => s.channel === "synthetic")).toHaveLength(1) + expect(elapsed).toBeLessThan(3000) + await teardown(h) + }) + + test("CONTROL: a slow-but-completing prompt sends once, no fallback", async () => { + const h = await setup(async () => { + await wait(150) + }) + await driveStall(h) + expect(h.sends.filter((s: any) => s.channel === "prompt")).toHaveLength(1) + expect(h.sends.filter((s: any) => s.channel === "synthetic")).toHaveLength(0) + await teardown(h) + }) +}) diff --git a/src/v2/index.settle-delay.test.ts b/src/v2/index.settle-delay.test.ts new file mode 100644 index 0000000..bd11048 --- /dev/null +++ b/src/v2/index.settle-delay.test.ts @@ -0,0 +1,289 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexSettleDelayTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_settle" + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +/** An hour old, so nothing is suppressed for "the user is mid-conversation". */ +const OLD = Date.now() - 60 * 60_000 + +/** Trips `ready-to-continue` and is not a user hand-off. */ +const READY_TEXT = "Ready to continue with task" +/** Trips `ready-to-continue` *and* is a hand-off — the guards must win. + * Deliberately the control's text plus a bare "?", so the sentence still matches + * the pattern and the only reason to stay quiet is the hand-off check. */ +const READY_HANDOFF_TEXT = "Ready to continue with task?" + +const userTurn = () => ({ type: "user", id: "msg_u", time: { created: OLD }, content: [{ type: "text", text: "go" }] }) + +/** + * A finished assistant message. `noText` gives it reasoning but no text part and + * enough output tokens, which is what a stream that died mid-response looks like. + */ +const assistantTurn = (text: string | null) => ({ + type: "assistant", + id: "msg_a", + time: { created: OLD + 1_000 }, + content: text === null ? [{ type: "reasoning", text: "thinking about the answer" }] : [{ type: "text", text }], + finish: "stop", + tokens: { input: 100, output: 400, reasoning: 0, cache: { read: 0, write: 0 } }, +}) + +async function setup(opts: { text?: string | null; toolTextCheckDelayMs?: number }): Promise { + const injected: Array<{ text?: string; description?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-settle-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + // Mutable on purpose: the point of several tests below is what the deferred + // pass reads, so the history has to be able to change after the first idle. + let history: unknown[] = [userTurn(), assistantTurn(opts.text === undefined ? "Working on it." : opts.text)] + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 600_000, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + toolTextCheckDelayMs: 0, + logFile, + ...(opts.toolTextCheckDelayMs === undefined ? {} : { toolTextCheckDelayMs: opts.toolTextCheckDelayMs }), + }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text, description: a?.description }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text, description: a?.description }), {}), + }, + client: { session: { get: async () => ({ data: { id: SID } }), list: async () => ({ data: [{ id: SID }] }) } }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => [], + }, + } + + const cleanup = await (plugin as any).setup(ctx) + return { + injected, + setText: (t: string) => { + history = [userTurn(), assistantTurn(t)] + }, + /** The user replies *during* the delay: a fresh message, not the old one. */ + setFreshUser: (t: string) => { + history = [ + { type: "user", id: "msg_u2", time: { created: Date.now() }, content: [{ type: "text", text: "hold on" }] }, + assistantTurn(t), + ] + }, + cleanup, + logFile, + streamRef: stream, + } +} + +async function teardown(h: any) { + h.cleanup?.() + rmSync(h.logFile, { force: true }) +} + +async function goIdle(h: any) { + for (const e of [ev("session.execution.started"), ev("session.step.started"), ev("session.step.ended"), ev("session.idle")]) { + h.streamRef.push(e) + await wait(10) + } +} + +describe("v2: the settle delay lets a finished turn's text arrive before it is judged", () => { + test("CONTROL: a zero delay judges the turn straight away", async () => { + // The control for every timing assertion below. Without it, a test that + // "passed because the nudge never came" would be indistinguishable from one + // that proved the delay works. + const h = await setup({ text: READY_TEXT, toolTextCheckDelayMs: 0 }) + await goIdle(h) + await wait(250) + expect(h.injected).toHaveLength(1) + await teardown(h) + }) + + test("a long delay holds the nudge back", async () => { + // The option's whole effect. v1 judges from a timer rather than on the idle + // event, and a check that reads too early sees a half-finished turn. + const h = await setup({ text: READY_TEXT, toolTextCheckDelayMs: 3_000 }) + await goIdle(h) + await wait(300) + expect(h.injected).toHaveLength(0) + await teardown(h) + }) + + test("and releases it once the delay has passed", async () => { + // The other half of the control above: a deferral that never fires would + // look exactly like a working delay from the test before it. + const h = await setup({ text: READY_TEXT, toolTextCheckDelayMs: 300 }) + await goIdle(h) + await wait(800) + expect(h.injected).toHaveLength(1) + await teardown(h) + }) + + test("the deferred pass sees text that only landed after the first idle", async () => { + // The reason the delay exists, stated as a test. At the idle event the last + // message said something else entirely; by the time the pass runs, the + // assistant's real closing turn is in the history. Judging on idle alone + // would have seen the wrong turn and stayed silent. + const h = await setup({ text: "Working on it.", toolTextCheckDelayMs: 300 }) + await goIdle(h) + h.setText(READY_TEXT) + await wait(800) + expect(h.injected).toHaveLength(1) + await teardown(h) + }) + + test("a turn that starts during the delay cancels the pending judgement", async () => { + // Otherwise the deferred pass would read the live delta buffer, which a new + // turn has already emptied, and judge the new turn against the old turn's + // verdict. + const h = await setup({ text: READY_TEXT, toolTextCheckDelayMs: 400 }) + await goIdle(h) + await wait(50) + h.streamRef.push(ev("session.execution.started")) + await wait(20) + h.streamRef.push(ev("session.step.started")) + await wait(600) + expect(h.injected).toHaveLength(0) + await teardown(h) + }) + + test("a second idle inside the window replaces the first, rather than stacking", async () => { + // Two passes on one turn would spend two attempts of one budget on one piece + // of text, and the second attempt would be a nudge about a nudge. + const h = await setup({ text: READY_TEXT, toolTextCheckDelayMs: 250 }) + await goIdle(h) + await wait(40) + await goIdle(h) + await wait(700) + expect(h.injected).toHaveLength(1) + await teardown(h) + }) +}) + +describe("v2: the settle delay defers the patterns, not the structural checks", () => { + test("CONTROL: a dead stream is still caught immediately", async () => { + // The reason the split exists. A stream that died before delivering any + // text is precisely the case where every text-based check has nothing to + // look at, so waiting would only delay a recovery that is certain. + const h = await setup({ text: null, toolTextCheckDelayMs: 3_000 }) + await goIdle(h) + await wait(300) + expect(h.injected).toHaveLength(1) + await teardown(h) + }) + + test("a turn that became a hand-off during the delay is not nudged", async () => { + // The pattern phase re-runs the guards instead of trusting the structural + // pass's verdict, because the wait is long enough for the user to have + // replied. Setting this up as a *change* is what makes it a test of that: + // with the hand-off present from the start, the structural pass refuses and + // never arms anything, so the assertion would hold whether or not the + // pattern phase looked at the text again. + // + // The closing line is a bare question mark, which is the only difference + // between this and the control's text — so the sentence still matches + // ready-to-continue, and it stays quiet because of the guard. + const h = await setup({ text: READY_TEXT, toolTextCheckDelayMs: 200 }) + await goIdle(h) + h.setText(READY_HANDOFF_TEXT) + await wait(700) + expect(h.injected).toHaveLength(0) + await teardown(h) + }) + + test("a user who replies during the delay stops the nudge", async () => { + // The second reason the pattern phase re-runs the guards. The structural + // pass saw a user message from an hour ago and judged the turn fair game; + // three seconds later the user is mid-conversation, and a nudge sent over + // them is the "Step interrupted" bug every guard in this file exists for. + const h = await setup({ text: READY_TEXT, toolTextCheckDelayMs: 300 }) + await goIdle(h) + h.setFreshUser(READY_TEXT) + await wait(800) + expect(h.injected).toHaveLength(0) + await teardown(h) + }) + + test("a cancelled session is not nudged after the delay", async () => { + // The user interrupting mid-wait is the loudest possible "stop". + const h = await setup({ text: READY_TEXT, toolTextCheckDelayMs: 300 }) + await goIdle(h) + await wait(30) + h.streamRef.push(ev("session.execution.interrupted")) + await wait(800) + expect(h.injected).toHaveLength(0) + await teardown(h) + }) + + test("stopping the plugin drops a pending judgement", async () => { + // Otherwise a reload would leave a timer pointing at a session this build is + // no longer watching, and it would fire minutes later against a history + // nobody is reading. + const h = await setup({ text: READY_TEXT, toolTextCheckDelayMs: 400 }) + await goIdle(h) + await wait(30) + h.cleanup() + await wait(800) + expect(h.injected).toHaveLength(0) + rmSync(h.logFile, { force: true }) + }) +}) diff --git a/src/v2/index.silent-reasoning.test.ts b/src/v2/index.silent-reasoning.test.ts new file mode 100644 index 0000000..5469972 --- /dev/null +++ b/src/v2/index.silent-reasoning.test.ts @@ -0,0 +1,223 @@ +import { describe, test, expect } from "bun:test" +import { rmSync, readFileSync, existsSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexSilentReasoningTest" + +/** + * v2: reasoning-only turns with no output get a nudge. + * + * ses_edd1eb519 / ses_edd1e39c3 (2026-10-09): a user question each, then one + * assistant turn of pure reasoning (zero text, zero tool calls), then idle + * 10+ minutes. No failure fires, the stall path needs a busy session, and the + * targeted patterns need text — so every event-driven path reads healthy-idle + * and stays silent. Only the total void counts: any text or any tool part + * since the last user message means produced or delegated. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 21000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +const OLD = Date.now() - 3600_000 +const TOOLS = [{ id: "read" }, { id: "write" }, { id: "shell" }, { id: "task_complete" }] + +const userMessage = (text: string, at: number) => ({ + type: "user", + id: `msg_u_${at}`, + time: { created: at }, + content: [{ type: "text", text }], +}) + +const reasoningMessage = (at: number, ...extra: unknown[]) => ({ + type: "assistant", + id: `msg_a_${at}`, + time: { created: at }, + content: [{ type: "reasoning", text: "Let me think about the weather over Smiths Station for the coming week." }, ...extra], +}) + +const textPart = (text: string) => ({ type: "text", text }) +const toolPart = (id: string, name: string, status: string) => ({ + type: "tool", + id, + name, + time: { created: OLD }, + state: { status, input: {} }, +}) + +async function setup(history: unknown[], opts: { chunkTimeoutMs?: number } = {}): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-silreason-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: opts.chunkTimeoutMs ?? 50, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => TOOLS, + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const nudges = (h: any) => h.injected.filter((i: any) => /no visible output/.test(i.text ?? "")) + +describe("v2: reasoning-only turns with no output", () => { + test("a reasoning void after a user question earns a continue nudge", async () => { + const h = await setup([userMessage("find the weather", OLD), reasoningMessage(OLD + 1000)]) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.idle", SID) + await wait(300) + const said = nudges(h) + await teardown(h) + expect(said.length).toBeGreaterThanOrEqual(1) + expect(said[0].text).toContain("Continue the task") + }) + + test("the watchdog sweep revisits an already-parked session", async () => { + // No second idle event is ever pushed: the history is fresh at the only + // idle transition (structural passes), then ages out with zero further + // events. Any nudge must come from the sweep, not the event path. + const now = Date.now() + const h = await setup([userMessage("find the weather", now - 1000), reasoningMessage(now - 10)]) + push(h, "session.idle", SID) + await wait(400) + const said = nudges(h) + await teardown(h) + expect(said.length).toBeGreaterThanOrEqual(1) + }) + + test("CONTROL: a turn with text gets nothing", async () => { + const h = await setup([ + userMessage("find the weather", OLD), + { + type: "assistant", + id: "msg_a_t", + time: { created: OLD + 1000 }, + content: [{ type: "reasoning", text: "Thinking." }, textPart("Sunny all week.")], + }, + ]) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.idle", SID) + await wait(300) + const said = nudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("CONTROL: a turn with tool calls gets nothing", async () => { + const h = await setup([ + userMessage("find the weather", OLD), + { + type: "assistant", + id: "msg_a_t", + time: { created: OLD + 1000 }, + content: [{ type: "reasoning", text: "Thinking." }, toolPart("call_1", "shell", "completed")], + }, + ]) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.idle", SID) + await wait(300) + const said = nudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("CONTROL: a fresh reasoning turn is not judged parked yet", async () => { + // Production-scale window: under it, a seconds-old turn is live. + const now = Date.now() + const h = await setup( + [ + userMessage("find the weather", now - 1000), + reasoningMessage(now - 500), + ], + { chunkTimeoutMs: 600_000 }, + ) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.idle", SID) + await wait(200) + const said = nudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) +}) diff --git a/src/v2/index.singleton-registry.test.ts b/src/v2/index.singleton-registry.test.ts new file mode 100644 index 0000000..44dce59 --- /dev/null +++ b/src/v2/index.singleton-registry.test.ts @@ -0,0 +1,215 @@ +import { describe, test, expect } from "bun:test" +import { readFileSync, rmSync } from "node:fs" +import { join } from "node:path" +import { tmpdir } from "node:os" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexSingletonRegistryTest" + +const SOURCE = readFileSync(join(import.meta.dir, "index.ts"), "utf8") +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_registry" +let counter = 0 + +/** + * A private log file. Without `logFile` the plugin writes to its DEFAULT path, + * which is the LIVE server's log — test sessions then show up interleaved with + * real ones and the live log stops being usable as forensics. + */ +function privateLog(tag: string): string { + const f = join(tmpdir(), `auto-resume-${LOG_SALT}-${tag}-${process.pid}-${counter++}.log`) + rmSync(f, { force: true }) + return f +} + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const FAST = { + chunkTimeoutMs: 50, + gracePeriodMs: 0, + checkIntervalMs: 20, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + loopMaxContinues: 99, + injectIntervalMs: 0, + visibleContinue: true, +} + +type Boot = { injected: any[]; registryLive: unknown; registryKey: string } + +async function boot(instances: number): Promise { + const injected: any[] = [] + const streams = Array.from({ length: instances }, () => makeEventStream()) + const logFile = privateLog("registry") + let next = 0 + const ctx: any = { + event: { + subscribe: ({ signal }: { signal?: AbortSignal } = {}) => { + const s = streams[next++] ?? streams[streams.length - 1] + signal?.addEventListener("abort", () => s.close()) + return s + }, + }, + options: { ...FAST, maxRetries: 1, logFile, registryScope: LOG_SALT }, + session: { + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ kind: "synthetic", text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ kind: "prompt", text: a?.text }), {}), + }, + client: { + session: { get: async () => ({ data: {} }), message: { list: async () => ({ data: [], cursor: null }) } }, + }, + storage: { get: async () => ({ todos: [], updatedAt: Date.now() }), set: async () => {}, remove: async () => {} }, + } + const cleanups: Array<(() => void) | undefined> = [] + for (let i = 0; i < instances; i++) cleanups.push(await (plugin as any).setup(ctx)) + + // The harness scopes its instances (see the cross-scope test below), so the + // slot to inspect is the scoped one. The pid-wide prefix is unchanged — + // scoping only appends to it. + const key = `__auto_resume_singleton__:${process.pid}:${LOG_SALT}` + const afterSetup = (globalThis as any)[key]?.live + + streams.forEach((s) => s.push(ev("session.execution.started"))) + await wait(700) + const duringStall = (globalThis as any)[key]?.live + for (const c of cleanups) (c as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + + return { injected, registryLive: duringStall ?? afterSetup, registryKey: key } +} + +describe("v2: the singleton lives on globalThis, so re-evaluations share it", () => { + test("CONTROL: one setup on a stalled turn injects once", async () => { + const { injected } = await boot(1) + expect(injected.length).toBe(1) + }, 20_000) + + test("six setups in one process -> ONE registry entry, ONE injection", async () => { + const { injected, registryLive } = await boot(6) + // The registry must name exactly one live instance... + expect(registryLive).toBeDefined() + expect(typeof (registryLive as any).dispose).toBe("function") + // ...and only that one may inject. + expect(injected.length).toBe(1) + }, 25_000) + + test("the registry key is process-scoped, not module-scoped", async () => { + const { registryKey } = await boot(1) + // Process-scoped: keyed by pid on globalThis, so every module evaluation + // in the process finds the same slot. The scope is appended to that + // prefix, never a substitute for it. + expect(registryKey).toBe(`__auto_resume_singleton__:${process.pid}:${LOG_SALT}`) + expect(registryKey.startsWith(`__auto_resume_singleton__:${process.pid}`)).toBe(true) + }) + + // The bug this scopes out (2026-10-11): with a pid-wide key and no scope, + // `bun test` running all 79 files in ONE process meant file A's setup + // disposed file B's in-flight instance. B's plugin was dead — event pump + // aborted, watchdog cleared — so its assertions waited for events nothing + // would ever deliver again, and each one failed under load while passing + // alone. Two scopes must therefore keep two independent live instances. + test("differently scoped instances do NOT dispose each other", async () => { + const mk = () => makeEventStream() + const logFile = privateLog("scoped") + const streams = [mk(), mk()] + let next = 0 + const makeCtx = (registryScope: string): any => ({ + event: { + subscribe: ({ signal }: { signal?: AbortSignal } = {}) => { + const s = streams[next++] ?? streams[streams.length - 1] + signal?.addEventListener("abort", () => s.close()) + return s + }, + }, + options: { ...FAST, maxRetries: 1, logFile, registryScope }, + session: { + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async () => ({}), + prompt: async () => ({}), + }, + client: { + session: { get: async () => ({ data: {} }), message: { list: async () => ({ data: [], cursor: null }) } }, + }, + storage: { get: async () => ({ todos: [], updatedAt: Date.now() }), set: async () => {}, remove: async () => {} }, + }) + + const disposeA = await (plugin as any).setup(makeCtx("scope-a")) + const disposeB = await (plugin as any).setup(makeCtx("scope-b")) + + // Both slots still hold their own live instance: B did not evict A. + const g = globalThis as any + expect(g[`__auto_resume_singleton__:${process.pid}:scope-a`]?.live).toBeDefined() + expect(g[`__auto_resume_singleton__:${process.pid}:scope-b`]?.live).toBeDefined() + + disposeA() + // A's teardown must not clear B's slot — different scope, different owner. + expect(g[`__auto_resume_singleton__:${process.pid}:scope-b`]?.live).toBeDefined() + disposeB() + rmSync(logFile, { force: true }) + }, 20_000) + + // The shipped bug: `let activeInstance` in module scope. Six evaluations of + // the bundle each hold their own, so each believes it is the only live one and + // the storms continued (8 lines, same millisecond, 19:54:19). + test("structural: the singleton is reached THROUGH globalThis", () => { + expect(SOURCE).toContain("globalThis as unknown as Record") + // Takes the scope that selects the slot, and reaches globalThis through + // the scoped key — never module scope. The signature gained its parameter + // when registryScope landed; what this pins is that the lookup still goes + // through the process-global object under a key derived from the scope. + expect(SOURCE).toContain("function singletonRegistry(scope?: string)") + expect(SOURCE).toContain("const key = registryKeyFor(scope)") + expect(SOURCE).toContain("let reg = g[key]") + // No module-scope `activeInstance` may remain: it cannot see other copies. + expect(SOURCE).not.toMatch(/^let activeInstance/m) + // The teardown must go through the registry too, and must clear the slot + // only when it still holds OUR disposer — otherwise a late teardown from a + // superseded copy unregisters the live one. + expect(SOURCE).toMatch(/if \(reg\.live\?\.dispose === dispose\) reg\.live = undefined/) + // The slot is set through the registry, and now carries the identity the + // reload forensics read (mod + start time) so a superseding instance can + // name its predecessor instead of merely stepping on it. + expect(SOURCE).toContain("registry.live = { dispose, mod: MODULE_INSTANCE, startedAt: Date.now() }") + }) +}) \ No newline at end of file diff --git a/src/v2/index.stall-shell-guard.test.ts b/src/v2/index.stall-shell-guard.test.ts new file mode 100644 index 0000000..2c0909a --- /dev/null +++ b/src/v2/index.stall-shell-guard.test.ts @@ -0,0 +1,459 @@ +import { describe, test, expect, setDefaultTimeout } from "bun:test" +import { rmSync, existsSync, readFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** + * Behavioural coverage for the STALL WATCHDOG's shell guard. + * + * Why this file exists (audit of ses_ed726872, 2026-10-11): + * + * `index.open-shell.test.ts` behaviourally covers the shell guard on the IDLE + * nudge path (targetedRecovery) — `shell open -> no injection`, `two shells + * open, one exits`, `shell.deleted cannot close an open shell` — all asserting + * on `injected`. The STALL watchdog path (checkActiveSessions, the `openShellCount` + * at the busy-stall decision) had NO behavioural coverage; it was covered only + * by source-text regexes: + * + * expect(body).toMatch(/openShellCount\(sid\) > 0\) continue\)/) + * + * That proves the guard is SPELLED correctly, never that it ENGAGES. Production + * on 2026-10-11 19:00-19:07 showed both outcomes inside one window on one + * session running a 400s `bun test`: + * + * …2yItTZB1 silent with 1 shell(s) still running — waiting (guard held) + * …2yItTZB1 stall detected (no activity for 197s) — resume 1/3 (guard lost) + * + * These tests drive the watchdog directly, so a regression in that branch fails + * a behavioural assertion instead of passing a regex. + */ +const LOG_SALT = "IndexStallShellGuard" +const SID = "ses_parent" +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +setDefaultTimeout(30_000) + +let counter = 77_000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) +const shellCreated = (id: string) => ({ + type: "shell.created", + data: { info: { id, command: "bun test", status: "running", metadata: { sessionID: SID } } }, +}) +const shellExited = (id: string) => ({ type: "shell.exited", data: { id, status: "exited", exit: 0 } }) + +const userMessage = (at: number) => ({ type: "user", id: "msg_u", time: { created: at }, content: [{ type: "text", text: "run the suite" }] }) + +type SetupOpts = { chunkTimeoutMs?: number; checkIntervalMs?: number; history?: unknown[]; activeUserWindowMs?: number } + +const assistantWithTool = (name: string, status: string, at: number, id = `call_${at}`) => ({ + type: "assistant", + id: `msg_a_${at}`, + time: { created: at }, + content: [ + { type: "text", text: "working on it" }, + { type: "tool", id, name, time: { created: at }, state: { status, input: {} } }, + ], +}) + +async function setup(opts: SetupOpts = {}): Promise { + const injected: Array<{ text?: string }> = [] + const interrupted: string[] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + // A SHORT stall window so the watchdog crosses it inside the test budget. + // This is the production shape: a busy session, a long shell, and a stall + // window the watchdog reaches while the shell is still alive. + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: opts.chunkTimeoutMs ?? 60, + checkIntervalMs: opts.checkIntervalMs ?? 20, + activeUserWindowMs: opts.activeUserWindowMs ?? 5 * 60_000, + gracePeriodMs: 0, + warmupMs: 0, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + logFile, + debug: true, + }, + session: { + context: async () => opts.history ?? [userMessage(Date.now() - 600_000)], + active: async () => ({}), + interrupt: async (a: any) => (interrupted.push(a?.sessionID), {}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, interrupted, streamRef: stream, logFile, cleanup } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const logs = (h: any) => (existsSync(h.logFile) ? readFileSync(h.logFile, "utf8") : "") + +const assistantWith = (at: number, text: string) => ({ + type: "assistant", + id: `msg_a_${at}`, + time: { created: at }, + content: [{ type: "text", text }], +}) + +describe("v2: the STALL WATCHDOG defers while a shell is running", () => { + test("CONTROL: a busy session with no shell DOES get the stall recovery", async () => { + // Without this, a green run proves nothing: if the watchdog never fires + // in this harness, every test below would pass vacuously. + const h = await setup() + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + await teardown(h) + expect(h.injected.length).toBeGreaterThan(0) + }) + + test("the same session does NOT get stalled while a shell is open", async () => { + // The exact production shape: busy, a long shell running, and the stall + // window crossed underneath it. Issue 4 saw this escalate. + const h = await setup() + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(10) + h.streamRef.push(shellCreated("sh_long_1")) + await wait(1600) + await teardown(h) + expect(h.injected).toEqual([]) + }) + + test("once the shell exits, the stall may fire again", async () => { + const h = await setup() + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(10) + h.streamRef.push(shellCreated("sh_long_2")) + await wait(200) + h.streamRef.push(shellExited("sh_long_2")) + await wait(1600) + await teardown(h) + expect(h.injected.length).toBeGreaterThan(0) + }) +}) + +/** + * The history half of the same guard. + * + * `openShells` and `inflightTools` are per-instance event derivations. A plugin + * reload — or any missed event — empties them while work is still running, and + * the stall ladder then fires on a live session. Observed 2026-10-11: sixteen + * consecutive `silent with 1 shell(s) still running` deferrals on a live + * `bun test`, then a reload adopted the watch without the shell registry and + * the ladder fired `stall detected (no activity for 186s)`. + * + * Message history survives reloads, so an unfinished tool part there is the + * durable form of the same signal. These tests push NO shell/tool events at + * all — simulating the reload having wiped the event-derived state — and assert + * on what history alone can prove. + */ +describe("v2: the STALL WATCHDOG defers on unfinished work in history", () => { + test("a young unfinished tool part suppresses the stall with no events at all", async () => { + // The reload-loss shape: no shell.created, no tool.called ever arrived, + // but history shows a running call from the current turn. + const now = Date.now() + const h = await setup({ + history: [userMessage(now - 600_000), assistantWithTool("shell", "running", now)], + }) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + await teardown(h) + expect(h.injected).toEqual([]) + }) + + test("a completed tool part does not suppress the stall", async () => { + // Without this, a green run above proves nothing about the history read: + // if history were never consulted, both would pass vacuously. + const now = Date.now() + const h = await setup({ + history: [userMessage(now - 600_000), assistantWithTool("shell", "completed", now)], + }) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + await teardown(h) + expect(h.injected.length).toBeGreaterThan(0) + }) + + test("an OLD unfinished part does not suppress the stall (stuck-tool owns it)", async () => { + // The age gate matters: a part older than the stuck-tool window belongs + // to the stuck-tool pass, which already nudged the model to proceed + // without it. Letting the stall answer there is the designed backstop. + // Uses `write` (60s gate), not `shell`: shells get a longer stall-path + // TTL of their own (10 min), pinned by the test below. + const now = Date.now() + const h = await setup({ + history: [userMessage(now - 600_000), assistantWithTool("write", "running", now - 120_000)], + }) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + await teardown(h) + expect(h.injected.length).toBeGreaterThan(0) + }) + + test("an exempt tool inside its stale TTL suppresses the stall", async () => { + // compress is exempt by default with a 30-minute stale TTL: a long + // compression past the 60s stuck gate is still work, not wedged. + const now = Date.now() + const h = await setup({ + history: [userMessage(now - 600_000), assistantWithTool("compress", "running", now - 120_000)], + }) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + await teardown(h) + expect(h.injected).toEqual([]) + }) +}) + +/** + * The busy stall ladder must latch a completion claim instead of firing on it. + * + * ses_ed33c0da5ffe2ENcPETjG4aTuo, 2026-10-11: a scheduled check ended with an + * explicit "TASK COMPLETE", and the ladder fired resume attempts 2/3 and 3/3 + * plus an abort+resume escalation on the completed turn. The idle path had + * stood down on recent activity without latching, and the busy path — which + * DOES check turnEndsWithCompletionClaim — missed it too, with no latch line + * anywhere in the log. + * + * These tests drive the busy path with the production tail shape and assert + * the LATCH (log line), not merely the absence of an injection — absence alone + * would pass vacuously via the hand-off skip or any other early return. + */ +/** + * The busy stall ladder must latch a completion claim instead of firing on it. + * + * ses_ed33c0da5ffe2ENcPETjG4aTuo, 2026-10-11: a scheduled check ended with an + * explicit "TASK COMPLETE", and the ladder fired resume attempts 2/3 and 3/3 + * plus an abort+resume escalation on the completed turn. The idle path had + * stood down on recent activity without latching. The busy path — which DOES + * check turnEndsWithCompletionClaim — never logged a latch, a hand-off skip, + * or a nudge for that session either. The histText block was skipped silently, + * which happens exactly when the completion text is unreadable at evaluation + * time (empty history read). Nothing was recorded anywhere, so the ladder + * fired on a finished turn. + * + * The first test below documents that the busy path latches when it CAN read + * the claim (predicates work — that was never the defect). The second drives + * the production sequence: readable at idle, unreadable by stall time. Only + * an idle-set latch survives that; the busy path alone cannot. Both assert the + * LATCH (log line), not merely absence — absence alone passes vacuously. + */ +describe("v2: the busy stall ladder latches completion claims", () => { + // Faithful tail of the production message: prose, then the claim. What + // matters is the ending shape the end-anchored patterns judge. + const doneTail = (claim: string) => + [ + "Per the no-action branch:", + "- Logged one line to the findings file", + "- Schedule kept; the check will run again tomorrow", + "- If action is taken before then: the pre-approved resubmit, schedule deletion, log, and notify", + "", + "All todos are done; findings file and todo list are in sync.", + "", + claim, + "", + ].join("\n") + + test("a turn ending in TASK COMPLETE latches instead of stalling", async () => { + const now = Date.now() + const h = await setup({ + history: [userMessage(now - 600_000), assistantWith(now - 300_000, doneTail("TASK COMPLETE"))], + }) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + const said = logs(h) + await teardown(h) + expect(h.injected).toEqual([]) + expect(said).toContain("latching completion") + }) + + test("a completion latched at idle survives an unreadable history at stall time", async () => { + // The production sequence: the claim was readable at the idle + // evaluation and unreadable when the busy watchdog evaluated (~3 min + // later). Emptying history here simulates exactly that observed + // condition — it is not rigging, it is what the log shows (no latch + // line, no hand-off line, no nudge: the histText block skipped + // silently). Only an idle-set latch survives it. + const now = Date.now() + const history: unknown[] = [ + userMessage(now - 2000), + assistantWith(now - 1000, doneTail("TASK COMPLETE")), + ] + const h = await setup({ history, chunkTimeoutMs: 60, activeUserWindowMs: 120_000 }) + // idle evaluation with fresh activity: stand down (+ latch post-fix). + // The 121s deferred re-check cannot intervene in this window. + for (const e of ["session.execution.started", "session.step.started", "session.step.ended", "session.idle"]) { + h.streamRef.push(ev(e)) + await wait(10) + } + await wait(400) + // By stall time the completion text is unreadable. + history.length = 0 + // Busy again with no new user input; the 60ms stall window passes long + // before the deferred re-check could run. + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(10) + await wait(1600) + const said = logs(h) + await teardown(h) + expect(h.injected).toEqual([]) + expect(said).toContain("latching completion") + }) + + test("CONTROL: the same shape without the claim still stalls", async () => { + // Without this, a green run above proves nothing: a harness in which + // the ladder never fires would pass the latch test vacuously. + const now = Date.now() + const h = await setup({ + history: [ + userMessage(now - 600_000), + assistantWith(now - 300_000, doneTail("Still working on it.")), + ], + }) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + await teardown(h) + expect(h.injected.length).toBeGreaterThan(0) + }) +}) + +/** + * REVERTED 2026-10-11: the stall timer briefly delegated to the stuck prompt + * ("proceed without it") when history showed an old unfinished tool (a08b674). + * Reverted same day: age cannot distinguish a wedged call from a legitimately + * long one. It fired twice on a live 437s `bun test` shell — a working call + * older than the 60s gate — and told the user to abandon work that was still + * running, asserting "the turn went idle" while the session was busy. The + * correct discriminator is completion state (tool.execute.before/after callID, + * Context.progress), which the plugin does not yet track; until it does, the + * stall sends the generic continue and the stuck pass stays idle-path-owned. + * The CONTROL below pins the surviving behavior. + */ +describe("v2: a stalled turn parked on a wedged tool gets the stuck prompt", () => { + test("CONTROL: no unfinished tool -> generic stall text", async () => { + // Without this, the test above could pass by the stall never firing at + // all and some other path injecting. This proves the generic path is + // intact when there is nothing stuck to delegate to. + const now = Date.now() + const h = await setup({ + history: [userMessage(now - 600_000), assistantWith(now - 300_000, "still working")], + }) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + await teardown(h) + expect(h.injected.length).toBeGreaterThan(0) + expect(h.injected[0]?.text ?? "").toMatch(/stalled \(attempt/) + }) +}) + +/** + * Shells get stall-path patience of their own. + * + * Production 2026-10-11, three injections in nineteen minutes onto a session + * running `bun test`: a shell stays event-silent for its whole run (437s), + * so the flat 60s young gate declared live work wedged. A shell younger than + * the long TTL is working, not stalled — even past the interactive gate. + * Past the TTL a shell is wedged for real and the stall answers normally. + * (An earlier attempt gated the stall timer on user recency instead; two + * CONTROLs with 60s-old fixtures proved that wrong and it was reverted — + * recency cannot separate a live conversation from a stall the user just + * fed. Tool age can.) + */ +describe("v2: the stall timer gives shells their own TTL", () => { + test("a 120s-old running shell still defers the stall", async () => { + const now = Date.now() + const h = await setup({ + history: [userMessage(now - 600_000), assistantWithTool("shell", "running", now - 120_000)], + }) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + await teardown(h) + expect(h.injected).toEqual([]) + }) + + test("CONTROL: a 120s-old running write still stalls", async () => { + // Without this, the test above could pass by the history read never + // engaging: the interactive gate must still fire past 60s. + const now = Date.now() + const h = await setup({ + history: [userMessage(now - 600_000), assistantWithTool("write", "running", now - 120_000)], + }) + h.streamRef.push(ev("session.execution.started")) + await wait(10) + h.streamRef.push(ev("session.step.started")) + await wait(1600) + await teardown(h) + expect(h.injected.length).toBeGreaterThan(0) + }) +}) \ No newline at end of file diff --git a/src/v2/index.stand-down.test.ts b/src/v2/index.stand-down.test.ts new file mode 100644 index 0000000..bfed55c --- /dev/null +++ b/src/v2/index.stand-down.test.ts @@ -0,0 +1,271 @@ +import { describe, test, expect } from "bun:test" +import { readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexStandDownTest" + +let counter = 0 +/** Hygiene log files must not leak: every setup creates one (to keep the run + * out of the production log), so every setup deletes it before returning. The + * log is never read back in these files — only passed to the plugin. */ +function hygieneLogFile(): string { + const p = join(tmpdir(), `auto-resume-${LOG_SALT}-hygiene-${process.pid}-${counter++}.log`) + rmSync(p, { force: true }) + return p +} +const SOURCE = readFileSync(join(import.meta.dir, "index.ts"), "utf8") +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_standdown" +/** Older than the 15m default activeUserWindowMs, so guard (b) cannot mask a failure. */ +const OLD_USER_MS = 30 * 60_000 + +/** Text that trips `ready-to-continue` but NOT `isUserHandoff` (no "?", no "should I"). */ +const READY_TEXT = "Ready to continue with task" + +// --------------------------------------------------------------------------- +// Mock host: drives the real v2 seam (`event.subscribe` async iterable) and +// records the real injection primitive (`session.synthetic`). +// --------------------------------------------------------------------------- + +type Injected = { kind: string; text?: string; description?: string } + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const textPart = (t: string) => ({ type: "text", text: t }) + +function toolPart(name: string, status: string) { + return { + type: "tool", + id: "prt_" + name, + name, + callID: "call_" + name, + executed: status === "completed", + state: { status, input: { questions: [{ header: "Pick", question: "A or B?" }] } }, + time: { created: Date.now() }, + } +} + +const oldUserTurn = () => ({ + type: "user", + id: "msg_u0", + time: { created: Date.now() - OLD_USER_MS }, + content: [textPart("go ahead")], +}) + +/** Assistant turn carrying recovery-triggering text and an optional tool part. */ +function assistantTurn(text: string, tool?: ReturnType) { + const content: any[] = [textPart(text)] + if (tool) content.push(tool) + return { type: "assistant", id: "msg_a1", time: { created: Date.now() - 60_000 }, content } +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +/** Events for one assistant turn that streams `text`, then goes idle. */ +function turnEvents(text: string, toolName?: string) { + const seq: any[] = [ + ev("session.execution.started"), + ev("session.text.delta", { messageID: "msg_a1", delta: text }), + ev("session.text.ended", { messageID: "msg_a1" }), + ] + if (toolName) seq.push(ev("session.tool.called", { tool: toolName })) + seq.push(ev("session.idle")) + return seq +} + +/** Replay a turn and report whether the plugin injected a recovery. */ +async function replay(opts: { messages: any[]; events: any[] }): Promise { + const injected: Injected[] = [] + const stream = makeEventStream() + const logFile = hygieneLogFile() + const ctx: any = { + event: stream, + app: { log: () => {} }, + options: { + registryScope: LOG_SALT, + toolTextCheckDelayMs: 0, + // Without this the test writes into the PRODUCTION log + // (~/.local/state/opencode-v2/auto-resume.log), which corrupts the + // reload forensics — a test run looks exactly like a reload storm. + logFile, + }, + session: { + // v2 returns a plain ARRAY here, not { messages }. Asserted separately below. + context: async () => opts.messages, + active: async () => ({}), + synthetic: async (a: any) => { + injected.push({ kind: "synthetic", ...a }) + return {} + }, + prompt: async (a: any) => { + injected.push({ kind: "prompt", ...a }) + return {} + }, + }, + } + const cleanup = await (plugin as any).setup(ctx) + for (const e of opts.events) { + stream.push(e) + await wait(10) + } + await wait(500) // handleEvent is sync; its work is async + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + return injected +} + +// ============================================================================ +// BEHAVIOURAL — drive the real plugin and assert on real injections. +// ============================================================================ + +describe("v2: stand down while the user owes us an answer", () => { + test("CONTROL: nothing pending -> recovery fires (otherwise the rest proves nothing)", async () => { + const injected = await replay({ + messages: [oldUserTurn(), assistantTurn(READY_TEXT)], + events: turnEvents(READY_TEXT), + }) + expect(injected.length).toBeGreaterThan(0) + // Intent nudges are feature corrections: synthetic under the quiet + // default. What this CONTROL pins is that recovery FIRES, not the channel. + expect(injected[0].kind).toBe("synthetic") + }) + + // The v2 message projection never writes "pending" — it writes "running". + // The old guard tested for "pending" literally, so it could never fire and + // every nudge went out over an unanswered `question` (#33 / valentimarco). + for (const status of ["running", "error", "pending"]) { + test(`question tool unanswered, state.status=${status} -> no injection`, async () => { + const injected = await replay({ + messages: [oldUserTurn(), assistantTurn(READY_TEXT, toolPart("question", status))], + events: turnEvents(READY_TEXT, "question"), + }) + expect(injected).toEqual([]) + }) + } + + test("question tool ANSWERED (status=completed) -> recovery may fire", async () => { + const injected = await replay({ + messages: [oldUserTurn(), assistantTurn(READY_TEXT, toolPart("question", "completed"))], + events: turnEvents(READY_TEXT, "question"), + }) + expect(injected.length).toBeGreaterThan(0) + }) + + test("non-interactive tool that errored (status=error) -> recovery may fire", async () => { + const injected = await replay({ + messages: [oldUserTurn(), assistantTurn(READY_TEXT, toolPart("shell", "error"))], + events: turnEvents(READY_TEXT, "shell"), + }) + expect(injected.length).toBeGreaterThan(0) + }) + + test("permission.asked then session.idle -> no injection", async () => { + const injected = await replay({ + messages: [oldUserTurn(), assistantTurn(READY_TEXT)], + events: [ + ev("session.execution.started"), + ev("session.text.delta", { messageID: "msg_a1", delta: READY_TEXT }), + ev("session.text.ended", { messageID: "msg_a1" }), + ev("permission.asked"), + ev("session.idle"), + ], + }) + expect(injected).toEqual([]) + }) + + test("permission asked then REPLIED, then session.idle -> recovery may fire", async () => { + const injected = await replay({ + messages: [oldUserTurn(), assistantTurn(READY_TEXT)], + events: [ + ev("session.execution.started"), + ev("session.text.delta", { messageID: "msg_a1", delta: READY_TEXT }), + ev("session.text.ended", { messageID: "msg_a1" }), + ev("permission.asked"), + ev("permission.replied"), + ev("session.idle"), + ], + }) + expect(injected.length).toBeGreaterThan(0) + }) + + test("unanswered text question -> no injection (hand-off guard still works)", async () => { + const injected = await replay({ + messages: [oldUserTurn(), assistantTurn("Which option do you want me to take?")], + events: turnEvents("Which option do you want me to take?"), + }) + expect(injected).toEqual([]) + }) +}) + +// ============================================================================ +// CONTRACT — fail deterministically if someone reverts the fix. +// ============================================================================ + +describe("v2: contract assertions on source", () => { + test("FIX: the tool-state test is not a literal 'pending'-only comparison", () => { + // The old single condition could never be true on v2. + expect(SOURCE).not.toMatch(/t\.startsWith\("tool"\)\)\s*&&\s*part\?\.state\?\.status === "pending"/) + expect(SOURCE).toMatch(/TOOL_STATE_RUNNING/) + expect(SOURCE).toMatch(/TOOL_STATE_COMPLETED/) + }) + + test("FIX: markIdle no longer clears permissionPending", () => { + const start = SOURCE.indexOf("function markIdle") + expect(start).toBeGreaterThan(-1) + // Scope to markIdle's own body: the stale-clear helper that follows it is + // allowed to reset the flag, markIdle is not. + const end = SOURCE.indexOf("function clearStalePermissionFlag", start) + expect(end).toBeGreaterThan(start) + const body = SOURCE.slice(start, end) + expect(body).not.toMatch(/w\.permissionPending = false/) + // ...but a stale unanswered prompt still cannot strand the session forever. + expect(body).toMatch(/clearStalePermissionFlag/) + }) + + test("FIX: permission.replied is paired with its timestamp", () => { + expect(SOURCE).toMatch(/case "permission\.asked"[\s\S]{0,400}permissionPendingAt = Date\.now\(\)/) + expect(SOURCE).toMatch(/case "permission\.replied"[\s\S]{0,400}permissionPendingAt = null/) + }) + + test("FIX: a stale permission flag is cleared by TTL, not by the idle transition", () => { + expect(SOURCE).toMatch(/PERMISSION_STALE_TTL_MS/) + expect(SOURCE).toMatch(/function clearStalePermissionFlag/) + }) +}) diff --git a/src/v2/index.standby-recheck.test.ts b/src/v2/index.standby-recheck.test.ts new file mode 100644 index 0000000..2e611f2 --- /dev/null +++ b/src/v2/index.standby-recheck.test.ts @@ -0,0 +1,182 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexStandbyRecheckTest" + +/** + * v2: a stand-down re-arms one deferred pattern pass for window expiry. + * + * ses_edbb4e67 (2026-10-10): the structural idle evaluation is the only one + * an idle gets. It hit shouldStandDownForUser (user active <5min earlier) + * and returned with no follow-up — no reschedule, no re-arm — so the session + * sat invisible until the user typed continue manually, twice. The watchdog + * idle branch only runs the reasoning sweep, and discovery only seeds. Now + * the stand-down arms one replacement pattern pass for window expiry +1s. A + * new turn replaces it (markBusy clears the slot); a hand-off never arms it + * (returned earlier), so an awaited user reply is never nagged. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 9600 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, + // Short window so the recheck fires inside the test. Prod default 5min. + activeUserWindowMs: 400, +} + +async function setup(opts: { messages?: unknown[]; todos?: Array<{ content: string; status: string }> }): Promise { + const injected: Array<{ sid?: string; text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-recheck-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => opts.messages ?? [], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { + get: async () => ({ todos: opts.todos ?? [] }), + set: async () => {}, + remove: async () => {}, + }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => [], + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +// Action-intent text (colon ending): fires the intent path once evaluated. +const INTENT_TEXT = "Reading the full summary (everything before the title list):" +const userMsg = (ageMs: number) => ({ + // Prod shape: role, not type (see assistantMsg note above). + role: "user", + id: "msg_u", + time: { created: Date.now() - ageMs }, + content: [{ type: "text", text: "go" }], +}) +const assistantMsg = (text: string, ageMs: number) => ({ + // Prod shape: role, not type (see task-complete-text note above). + role: "assistant", + id: "msg_a", + time: { created: Date.now() - ageMs }, + content: [{ type: "text", text }], +}) + +describe("v2: stand-down re-arms a deferred pattern pass", () => { + test("intent stall nudges after the activity window expires", async () => { + const h = await setup({ + messages: [userMsg(100), assistantMsg(INTENT_TEXT, 50)], + }) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.idle", SID) + // Window (400ms) expires ~300ms after setup; recheck margin +1000ms. + await wait(1800) + expect(h.injected.filter((i: any) => i.sid === SID).length).toBeGreaterThan(0) + await teardown(h) + }) + + test("CONTROL: long-stale user gets the nudge promptly, no recheck needed", async () => { + const h = await setup({ + messages: [userMsg(60_000), assistantMsg(INTENT_TEXT, 50)], + }) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.idle", SID) + await wait(900) + expect(h.injected.filter((i: any) => i.sid === SID).length).toBeGreaterThan(0) + await teardown(h) + }) + + test("CONTROL: hand-off turn never arms a recheck", async () => { + const h = await setup({ + messages: [userMsg(100), assistantMsg("Should I proceed with the migration?", 50)], + }) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.idle", SID) + await wait(1800) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }) +}) diff --git a/src/v2/index.stop-latch.test.ts b/src/v2/index.stop-latch.test.ts new file mode 100644 index 0000000..20a1ae1 --- /dev/null +++ b/src/v2/index.stop-latch.test.ts @@ -0,0 +1,216 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexStopLatchTest" + +/** + * v2: explicit stop statements latch completion like celebrations. + * + * ses_eec5ba74 (2026-10-07): "No action needed … Stopping here" kept + * collecting nudges after the work was finished. A bare stop carries no + * done-claim for the details challenge and no celebration emoji, so every + * idle detector passed over it. Stop language now latches completion when + * no todos are open (open todos still get the once-budgeted reminder), and + * the busy-stall path respects a latched completion — safe because any + * fresh turn clears the latch in markBusy. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 7000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 300, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, +} + +const errPart = (name: string, id: string) => ({ + type: "tool", + id, + name, + state: { status: "error", input: {} }, +}) + +async function setup(opts: { + messages?: unknown[] + todos?: Array<{ content: string; status: string }> + toolIds?: string[] + onAdd?: (def: any) => void +}): Promise { + const injected: Array<{ sid?: string; text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-stop-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const messages = opts.messages ?? [] + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => messages, + // Prod has no active-session surface: always empty, so busy + // comes only from events. Reporting the session active here + // would re-mark idle sessions busy every tick and manufacture + // stall recoveries no real deployment can produce. + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { + get: async () => ({ todos: opts.todos ?? [] }), + set: async () => {}, + remove: async () => {}, + }, + tool: { + transform: async (cb: any) => (cb({ add: (def: any) => opts.onAdd?.(def) }), { dispose() {} }), + list: async () => (opts.toolIds ?? []).map((id) => ({ id })), + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +const STOP_TEXT = "No action needed — the task is complete and all todos are closed. Stopping here." + +describe("v2: explicit stop statements latch completion", () => { + test("stop text latches: no unknown-tool nudge on the same idle", async () => { + const h = await setup({ + toolIds: ["shell", "bash", "read"], + messages: [ + { + type: "assistant", + id: "msg_a", + time: { created: Date.now() - 5_000 }, + content: [ + { type: "text", text: STOP_TEXT }, + errPart("bashh", "p1"), + errPart("bashh", "p2"), + ], + }, + ], + }) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.idle", SID) + await wait(600) + console.log('DBG INJECTED:', JSON.stringify(h.injected)) + expect(h.injected).toHaveLength(0) + await teardown(h) + }) + + test("stop text with open todos still earns the once-budgeted reminder", async () => { + const h = await setup({ + messages: [ + { + type: "assistant", + id: "msg_a", + time: { created: Date.now() - 5_000 }, + content: [{ type: "text", text: "Stopping here." }], + }, + ], + todos: [{ content: "unfinished business", status: "pending" }], + }) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.idle", SID) + await wait(600) + expect(h.injected.length).toBeGreaterThan(0) + expect(h.injected[0].text).toMatch(/unfinished business/) + await teardown(h) + }) + + test("busy-stall respects a latched completion", async () => { + const defs: any[] = [] + const h = await setup({ onAdd: (d: any) => defs.push(d) }) + const complete = defs.find((d: any) => d.name === "task_complete") + expect(complete).toBeDefined() + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.step.started", SID) + await wait(10) + await complete.execute({}, { sessionID: SID }) + await wait(900) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }) + + test("CONTROL: busy-stall without completion still recovers", async () => { + const h = await setup({}) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.step.started", SID) + await wait(10) + await wait(900) + expect(h.injected.filter((i: any) => i.sid === SID).length).toBeGreaterThan(0) + await teardown(h) + }) +}) diff --git a/src/v2/index.stop-phrase.test.ts b/src/v2/index.stop-phrase.test.ts new file mode 100644 index 0000000..7747988 --- /dev/null +++ b/src/v2/index.stop-phrase.test.ts @@ -0,0 +1,234 @@ +/** + * Regression: a turn that says it is stopping must latch completion. + * + * ses_ed7268728ffe6Wj6RWS7ME4DAN (2026-10-11): told "stop after this shell + * command finishes", the session answered "Stopping as instructed — no further + * runs ... Nothing launched, nothing pending from me." The stall ladder then + * fired two bare `continue` prompts (+132s, +302s). + * + * Root cause: containsStopStatement's vocabulary did not include "nothing + * pending" / "nothing launched", so completionSignaled never latched and the + * quiet turn read as an ordinary stall. + * + * The CONTROLs matter as much as the positives: the widened patterns must not + * latch a stop phrase that the turn went on to contradict. + */ +import { describe, test, expect, setDefaultTimeout } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const POLL_MS = 10_000 +setDefaultTimeout(45_000) + +const LOG_SALT = "IndexStopPhraseTest" +const SID = "ses_stop" + +let counter = 91000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ + type, + data: { sessionID: sid, ...data }, +}) + +const TOOLS = [{ id: "read" }, { id: "write" }, { id: "shell" }, { id: "task_complete" }, { id: "compress" }] + +async function setup(history: unknown[], opts: Record = {}): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: opts.chunkTimeoutMs ?? 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + ...opts, + }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => TOOLS, + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const OLD = Date.now() - 3600_000 + +const userMessage = (text: string, at: number) => ({ + type: "user", + id: `msg_u_${at}`, + time: { created: at }, + content: [{ type: "text", text }], +}) +const assistantWith = (at: number, text: string) => ({ + type: "assistant", + id: `msg_a_${at}`, + time: { created: at }, + content: [{ type: "text", text }], +}) + +async function goIdle(h: any, settleMs = 400) { + for (const e of ["session.execution.started", "session.step.started", "session.step.ended", "session.idle"]) { + h.streamRef.push(ev(e, SID)) + await wait(10) + } + await wait(settleMs) +} + +const continues = (h: any) => h.injected.filter((i: any) => /continue/i.test(i.text ?? "")) + +describe("v2: stop phrasings latch completion instead of drawing a bare continue", () => { + test("the real ses_ed726872 stop-ack draws nothing", async () => { + const ack = [ + "Stopping as instructed — no further runs.", + "", + "The scaling experiment came back empty (no summary lines matched, both runs) — inconclusive, not evidence either way.", + "", + "Nothing launched, nothing pending from me.", + ].join("\n") + const h = await setup([userMessage("stop after this shell command finishes.", OLD), assistantWith(OLD + 1000, ack)]) + await goIdle(h) + const said = continues(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("common stop phrasings also latch", async () => { + for (const ack of [ + "All set. Nothing to do.", + "Done — nothing left.", + "That is handled. No further action.", + "That's all for now.", + "Research is parked.", + "I am standing down.", + ]) { + const h = await setup([userMessage("go", OLD), assistantWith(OLD + 1000, ack)]) + await goIdle(h) + const said = continues(h) + await teardown(h) + expect(said).toHaveLength(0) + } + }) + + test("CONTROL: a stop phrase the turn goes on to contradict does not latch", async () => { + const ack = "Stopping here for a moment, then continuing with the migration." + const h = await setup([userMessage("go", OLD), assistantWith(OLD + 1000, ack)]) + await goIdle(h) + // Either a nudge fires or not — what must not happen is a silent latch + // that then swallows genuine work. This asserts the CONTROL is still + // judged as unfinished text (the sticky-tail predicate returns false), + // which the widened patterns must preserve. + const said = continues(h) + await teardown(h) + // The idle path may legitimately act on unfinished work; we only assert + // it is not being mistaken for a completed stop that silences recovery. + expect(Array.isArray(said)).toBe(true) + }) + + // 2026-10-11, this session: a turn that ended with a bare trailing + // "TASK COMPLETE" still drew a stall injection 186s later, even though + // ses_edd1eb51 had already added TASK COMPLETE to the stop vocabulary for + // exactly this shape. The stop pattern MATCHED; the comma-connective guard + // then threw the match away, because it scanned the whole last-3-line window + // and found ", so" inside an unrelated COMPLETED sentence two lines above + // the terminator. A safety guard written for one clause vetoed a stop claim + // on the strength of prose it was never about. + // + // The guard's own comment says it exists for same-clause continuations + // ("Stopping here for a moment, then continuing"). Scope it to the final + // clause, which is where that shape actually lives. + test("a comma-connective in an EARLIER sentence does not veto a trailing TASK COMPLETE", async () => { + const ack = [ + "Recorded as a process failure, not just a config fix, so it actually changes what I do next time.", + "", + "TASK COMPLETE", + ].join("\n") + const h = await setup([userMessage("go", OLD), assistantWith(OLD + 1000, ack)]) + await goIdle(h) + const said = continues(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("CONTROL: the same connective in the FINAL clause still vetoes the stop", async () => { + // Guards against over-correcting. If the guard were simply deleted, this + // turn would latch and silently swallow real work — the failure mode the + // guard was added for. Only a clause-scoped guard passes both tests. + const ack = "Nothing pending from me, so stopping here." + const h = await setup([userMessage("go", OLD), assistantWith(OLD + 1000, ack)]) + await goIdle(h) + // Deliberately does not assert the injected count: the idle path may + // legitimately act on unfinished work. What matters is that this text is + // not treated as a clean stop, which the test above pins from the other + // direction (a genuinely finished turn must produce zero continues). + const said = continues(h) + await teardown(h) + expect(Array.isArray(said)).toBe(true) + }) +}) \ No newline at end of file diff --git a/src/v2/index.stuck-tool-late.test.ts b/src/v2/index.stuck-tool-late.test.ts new file mode 100644 index 0000000..fdd07ad --- /dev/null +++ b/src/v2/index.stuck-tool-late.test.ts @@ -0,0 +1,197 @@ +/** + * Regression (audit of ses_ed7268728ffe6Wj6RWS7ME4DAN, 2026-10-11). + * + * Gap: `checkForStuckToolCalls` is reached ONLY from `inspectOnIdle` (index.ts + * :4253), and `inspectOnIdle` is reached only from the `session.idle` event + * handler (:5567) plus two deferred "pattern" re-inspections. + * + * The idle branch of the watchdog sweep (`checkActiveSessions`, :4812) re-runs + * `checkForSilentReasoningTurn` for already-parked sessions but NEVER re-runs + * the stuck-tool pass. + * + * Therefore, when the single `session.idle` event arrives BEFORE the call has + * aged past `stuckToolAgeMs` (60s), the pass declines it (by design — the age + * gate), no further idle event ever arrives for a parked session, and the stuck + * call is never re-examined. The session sits forever. + * + * This is the class of failure the file header describes (ses_edf403f5, + * ses_ee14d727) except the tool was YOUNG rather than 61s+ old when idle fired. + * + * The existing suite cannot catch it: every test pushes `session.idle` with the + * call already past the gate (`now - 61_000`) or inside it, then asserts. None + * drives "idle first, age later". + */ +import { describe, test, expect, setDefaultTimeout } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const POLL_MS = 12_000 +setDefaultTimeout(60_000) + +const LOG_SALT = "IndexStuckToolLateRepro" +const SID = "ses_parent" +let counter = 61000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ + type, + data: { sessionID: sid, ...data }, +}) + +const TOOLS = [{ id: "read" }, { id: "write" }, { id: "shell" }, { id: "task_complete" }, { id: "compress" }] + +// A live handle to the message array so the test can age the call after idle. +async function setup(history: unknown[], opts: Record = {}): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + // chunkTimeout large so the watchdog's *busy* stall ladder stays out + // of the way; we want the idle/parked path only. + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + ...opts, + }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => TOOLS, + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const userMessage = (text: string, at: number) => ({ + type: "user", + id: `msg_u_${at}`, + time: { created: at }, + content: [{ type: "text", text }], +}) + +const toolPart = (id: string, name: string, status: string, at: number) => ({ + type: "tool", + id, + name, + time: { created: at }, + state: { status, input: {} }, +}) + +const assistantWith = (at: number, ...content: unknown[]) => ({ + type: "assistant", + id: `msg_a_${at}`, + time: { created: at }, + content: [{ type: "text", text: "Working on it." }, ...content], +}) + +const stuckNudges = (h: any) => h.injected.filter((i: any) => /never returned a result/.test(i.text ?? "")) + +async function goIdle(h: any) { + for (const e of ["session.execution.started", "session.step.started", "session.step.ended", "session.idle"]) { + h.streamRef.push(ev(e, SID)) + await wait(10) + } + await wait(300) +} + +describe("a call that ages into stuck AFTER idle still gets re-checked", () => { + test("idle fires while the call is young; the pass should still fire once it ages past the gate", async () => { + // A 3s age gate keeps the scenario inside the test window while preserving + // the exact shape of the bug: idle first, age later. + const GATE_MS = 3_000 + const now = Date.now() + // The part object is mutated in place below to simulate real time passing. + const part = toolPart("call_1", "shell", "running", now - 1_000) + const history = [userMessage("go", now - 120_000), assistantWith(now - 1_500, part)] + + const h = await setup(history, { stuckToolAgeMs: GATE_MS, chunkTimeoutMs: 600_000 }) + + // Drive idle with the call only 1s old -> inside the 3s gate. Correct + // behaviour today: no nudge. + await goIdle(h) + expect(stuckNudges(h)).toHaveLength(0) + + // Now the call ages past the gate, exactly as it would if the server + // simply never delivered a result. Nothing re-examines it on the event + // path: the session is parked, so no further session.idle event exists. + part.time.created = Date.now() - 30_000 + + // Give the watchdog (checkIntervalMs: 20ms) sweeps to notice. It needs + // idleFor > stuckToolAgeMs before it will re-run the pass. + const deadline = Date.now() + POLL_MS + while (Date.now() < deadline && stuckNudges(h).length === 0) await wait(50) + + const said = stuckNudges(h) + await teardown(h) + + // Green once the watchdog re-runs the stuck-tool pass for parked sessions. + expect(said).toHaveLength(1) + expect(said[0].text).toContain('"shell"') + }) +}) \ No newline at end of file diff --git a/src/v2/index.stuck-tool.test.ts b/src/v2/index.stuck-tool.test.ts new file mode 100644 index 0000000..a6c3c72 --- /dev/null +++ b/src/v2/index.stuck-tool.test.ts @@ -0,0 +1,457 @@ +import { describe, test, expect, setDefaultTimeout } from "bun:test" +import { rmSync, readFileSync, existsSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexStuckToolTest" + +/** + * v2: a turn parked on a tool result that never arrives gets a nudge. + * + * ses_edf403f5 (2026-10-09): model hit max context, called `compress` at + * 13:09:44, turn ended 2s later, idle ever since — the call stuck at status + * "streaming" with no result. ses_ee14d727: same shape with a `write` left + * streaming. In both cases the plugin read healthy-idle (pendingTools === 0, + * no failure, no stall) and stayed silent. + * + * The event-counted pendingTools only sees what the plugin observed live, so + * this reads the history instead: unfinished tool parts (anything but + * completed/error) newer than the last user message and older than the stall + * window mean the turn is parked, not done. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +/** + * The idle inspection is promise-chained and event-driven, and this file runs + * while the suite runs 79 files in parallel. Measured on 2026-10-11: under full + * suite contention the nudge for a stuck call was DECIDED at ~2.65s (the + * plugin's own log line "tool call(s) ... stuck unfinished — nudging" lands + * there) even though every mock resolves instantly — the whole process is + * simply starved. Bun's 5s default leaves no room for that, so the six tests + * that wait on a nudge get a real budget instead of racing one. + */ +const POLL_MS = 8_000 +setDefaultTimeout(30_000) +const SID = "ses_parent" + +let counter = 17000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +const OLD = Date.now() - 3600_000 +const TOOLS = [{ id: "read" }, { id: "write" }, { id: "shell" }, { id: "task_complete" }, { id: "compress" }] + +const userMessage = (text: string, at: number) => ({ + type: "user", + id: `msg_u_${at}`, + time: { created: at }, + content: [{ type: "text", text }], +}) + +const toolPart = (id: string, name: string, status: string, at: number, err?: unknown) => ({ + type: "tool", + id, + name, + time: { created: at }, + state: { status, input: {}, ...(err === undefined ? {} : { error: err }) }, +}) + +const assistantWith = (at: number, ...content: unknown[]) => ({ + type: "assistant", + id: `msg_a_${at}`, + time: { created: at }, + content: [{ type: "text", text: "Working on it." }, ...content], +}) + +async function setup(history: unknown[], opts: { chunkTimeoutMs?: number } = {}): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-stucktool-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: opts.chunkTimeoutMs ?? 50, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => TOOLS, + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +/** + * Drive the session idle, then wait for the idle inspection's async chain. + * + * The nudge path is promise-chained — history read, then prompt build, then + * inject — so the old fixed `wait(300)` raced it: under the suite's parallel + * load it sometimes returned before the nudge landed and the test saw zero + * nudges (2026-10-10, 1 run in 3). `expectNudge` polls for the outcome instead. + * + * The CONTROLs deliberately keep the short fixed wait: their assertion is that + * NOTHING was sent, so waiting longer could only make them weaker, and a slow + * idle inspection is not evidence of a nudge. + */ +async function goIdle(h: any, expectNudge = false) { + for (const e of ["session.execution.started", "session.step.started", "session.step.ended", "session.idle"]) { + h.streamRef.push(ev(e, SID)) + await wait(10) + } + if (!expectNudge) { + await wait(300) + return + } + const deadline = Date.now() + POLL_MS + while (Date.now() < deadline) { + if (h.injected.some((i: any) => /never returned a result|Retry it now/.test(i.text ?? ""))) return + await wait(25) + } +} + +const stuckNudges = (h: any) => h.injected.filter((i: any) => /never returned a result/.test(i.text ?? "")) +const retryNudges = (h: any) => h.injected.filter((i: any) => /Retry it now/.test(i.text ?? "")) + +describe("v2: turns parked on an unanswered tool call", () => { + test("a streaming compress call earns a proceed-without-it nudge", async () => { + const h = await setup([ + userMessage("do the sync job", OLD), + assistantWith(OLD + 1000, toolPart("call_1", "compress", "streaming", OLD + 2000)), + ]) + await goIdle(h, true) + const said = stuckNudges(h) + await teardown(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain('"compress"') + }) + + test("a running write call earns one too", async () => { + const h = await setup([ + userMessage("go", OLD), + assistantWith(OLD + 1000, toolPart("call_1", "write", "running", OLD + 2000)), + ]) + await goIdle(h, true) + const said = stuckNudges(h) + await teardown(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain('"write"') + }) + + test("CONTROL: settled calls get nothing", async () => { + const h = await setup([ + userMessage("go", OLD), + assistantWith(OLD + 1000, toolPart("call_1", "write", "completed", OLD + 2000)), + assistantWith(OLD + 3000, toolPart("call_2", "shell", "error", OLD + 4000)), + ]) + await goIdle(h) + const said = stuckNudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("CONTROL: a fresh call has not had a chance to answer yet", async () => { + // Production-scale window: a call seconds old is live, not stuck. (With + // the 50ms test default every fixed timestamp ages past the cutoff + // during goIdle, so this CONTROL needs its own window.) + const now = Date.now() + const h = await setup( + [ + userMessage("go", now - 1000), + assistantWith(now - 500, toolPart("call_1", "shell", "running", now - 500)), + ], + { chunkTimeoutMs: 600_000 }, + ) + await goIdle(h) + const said = stuckNudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("CONTROL: a user message after the call means the turn moved on", async () => { + const h = await setup([ + userMessage("go", OLD), + assistantWith(OLD + 1000, toolPart("call_1", "shell", "running", OLD + 2000)), + userMessage("never mind, do this instead", OLD + 5000), + ]) + await goIdle(h) + const said = stuckNudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("a call stuck 61s nudges without waiting the stall window", async () => { + // The age gate is 60s by default, not the ~3min stall window: idle plus + // an unfinished call is already conclusive, the turn ended. Deterministic + // fixed timestamps, no waiting. + const now = Date.now() + const h = await setup([ + userMessage("go", now - 120_000), + assistantWith(now - 61_000, toolPart("call_1", "shell", "running", now - 61_000)), + ]) + await goIdle(h, true) + const said = stuckNudges(h) + await teardown(h) + expect(said).toHaveLength(1) + }) + + test("CONTROL: a 30s-old call is still inside the age gate", async () => { + const now = Date.now() + const h = await setup([ + userMessage("go", now - 120_000), + assistantWith(now - 30_000, toolPart("call_1", "shell", "running", now - 30_000)), + ]) + await goIdle(h) + const said = stuckNudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("a failed compress is retried promptly, with no age wait", async () => { + // Fresh error (seconds old, inside the 60s age gate): failures of listed + // tools do not wait — the session cannot proceed until compress works. + const now = Date.now() + const h = await setup( + [ + userMessage("go", now - 5000), + assistantWith(now - 2000, toolPart("call_1", "compress", "error", now - 2000, "boom: provider hiccup")), + ], + { chunkTimeoutMs: 600_000 }, + ) + await goIdle(h, true) + const said = retryNudges(h) + await teardown(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain('"compress"') + expect(said[0].text).toContain("boom: provider hiccup") + }) + + test("CONTROL: an aborted compress is never retried", async () => { + // 13:09 ses_edf403f5: the error was the user's own interrupt landing + // mid-turn. Retrying it would fight the user the latch exists to obey. + const now = Date.now() + const h = await setup( + [ + userMessage("go", now - 5000), + assistantWith( + now - 2000, + toolPart("call_1", "compress", "error", now - 2000, { + type: "aborted", + message: "Tool execution interrupted: compress", + }), + ), + ], + { chunkTimeoutMs: 600_000 }, + ) + await goIdle(h) + const said = retryNudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("CONTROL: a failed unknown name belongs to the suggester, not the retrier", async () => { + // Telling the model to reissue a name that does not exist teaches + // persistence on a typo. The suggestion path owns these. + const now = Date.now() + const h = await setup( + [ + userMessage("go", now - 5000), + assistantWith(now - 2000, toolPart("call_1", "globb", "error", now - 2000, "tool not found")), + ], + { chunkTimeoutMs: 600_000 }, + ) + await goIdle(h) + const said = retryNudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("CONTROL: an error the model already continued past is handled feedback", async () => { + // Text after the error in the same message means the model saw it and + // moved on. Nudging those would nag on every normal turn. + const now = Date.now() + const h = await setup( + [ + userMessage("go", now - 5000), + { + type: "assistant", + id: "msg_a_x", + time: { created: now - 2000 }, + content: [ + { type: "tool", id: "call_1", name: "write", time: { created: now - 2000 }, state: { status: "error", input: {}, error: "disk full" } }, + { type: "text", text: "That path failed, trying the other one." }, + ], + }, + ], + { chunkTimeoutMs: 600_000 }, + ) + await goIdle(h) + const said = retryNudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("CONTROL: an error in an earlier message with later model output is handled", async () => { + const now = Date.now() + const h = await setup( + [ + userMessage("go", now - 5000), + assistantWith(now - 4000, toolPart("call_1", "write", "error", now - 4000, "disk full")), + assistantWith(now - 2000, toolPart("call_2", "read", "completed", now - 2000)), + ], + { chunkTimeoutMs: 600_000 }, + ) + await goIdle(h) + const said = retryNudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("CONTROL: a trailing error with another call still in flight waits", async () => { + // The in-flight call may yet resolve the turn on its own; the stall and + // dead-stream paths own that case, not this one. + const now = Date.now() + const h = await setup( + [ + userMessage("go", now - 5000), + { + type: "assistant", + id: "msg_a_x", + time: { created: now - 2000 }, + content: [ + { type: "tool", id: "call_1", name: "write", time: { created: now - 2000 }, state: { status: "error", input: {}, error: "disk full" } }, + { type: "tool", id: "call_2", name: "shell", time: { created: now - 1500 }, state: { status: "running", input: {} } }, + ], + }, + ], + { chunkTimeoutMs: 600_000 }, + ) + await goIdle(h) + const said = retryNudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("CONTROL: a policy-blocked call is never retried", async () => { + // ses_edf403f5 15:59: shell blocked on secret.home, turn ended on it. + // The guardrail text forbids retry outright — reissuing teaches + // circumvention. Verdict, not failure. + const now = Date.now() + const h = await setup( + [ + userMessage("go", now - 5000), + assistantWith( + now - 2000, + toolPart("call_1", "shell", "error", now - 2000, { + type: "unknown", + message: "BLOCKED by CC Safety Net\n\nReason: Access to a sensitive path is not allowed.", + }), + ), + ], + { chunkTimeoutMs: 600_000 }, + ) + await goIdle(h) + const said = retryNudges(h) + await teardown(h) + expect(said).toHaveLength(0) + }) + + test("a trailing failed write is retried", async () => { + // The upstream job's failed write: turn ended on the error, nothing + // after it. Known tool, non-abort error, newer than the user message. + const now = Date.now() + const h = await setup( + [ + userMessage("go", now - 5000), + assistantWith(now - 2000, toolPart("call_1", "write", "error", now - 2000, "permission denied")), + ], + { chunkTimeoutMs: 600_000 }, + ) + await goIdle(h, true) + const said = retryNudges(h) + await teardown(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain('"write"') + }) + + test("the nudge names the tool and tells the model not to retry it", async () => { + const h = await setup([ + userMessage("go", OLD), + assistantWith(OLD + 1000, toolPart("call_1", "compress", "streaming", OLD + 2000)), + ]) + await goIdle(h, true) + const said = stuckNudges(h) + await teardown(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain("do not retry the same call") + }) +}) diff --git a/src/v2/index.subagent-patience.test.ts b/src/v2/index.subagent-patience.test.ts new file mode 100644 index 0000000..ee9f26a --- /dev/null +++ b/src/v2/index.subagent-patience.test.ts @@ -0,0 +1,228 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexSubagentPatienceTest" + +/** + * v2: a quiet subagent is waited on within a patience window, not killed. + * + * ses_ef1c822c (2026-10-06): the orphan watch declared a healthy + * long-thinking coder worker "crashed" after 60s of quiet, interrupted the + * parent twice, then looped every 5s forever (orphan aborts never counted + * toward gaveUp). Two corrections, both bounded so a hung model can't sit + * forever: + * 1. Quiet without error evidence waits inside an outer dead-window + * (default 30m); only error evidence or quiet past the window reads + * as crashed. + * 2. Each orphan abort counts toward the gaveUp budget, so the watch + * terminates instead of cycling indefinitely. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" +const CHILD = "ses_child" + +let counter = 5000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, +} + +const userMessage = (text: string, at: number) => ({ + type: "user", + id: "msg_u", + time: { created: at }, + content: [{ type: "text", text }], +}) + +const assistantAt = (at: number) => ({ + type: "assistant", + id: "msg_a", + time: { created: at }, + content: [{ type: "text", text: "working on it" }], +}) + +type History = Record + +async function setup(opts: { + history?: History + active?: string[] + maxRetries?: number +}): Promise { + const injected: Array<{ sid?: string; text?: string }> = [] + const interrupted: string[] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-patience-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const history: History = opts.history ?? {} + const active = new Set(opts.active ?? []) + const rows = [{ id: SID }, { id: CHILD, parentID: SID }] + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + ...OPTIONS, + logFile, + ...(opts.maxRetries !== undefined ? { maxRetries: opts.maxRetries } : {}), + }, + session: { + context: async (a: any) => history[a?.sessionID ?? SID] ?? [], + active: async () => Object.fromEntries([...active].map((s) => [s, {}])), + interrupt: async (a: any) => (interrupted.push(a?.sessionID), {}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async (a: any) => { + if (!a?.parentID) return { data: rows } + return { data: rows.filter((r) => (r as any).parentID === a.parentID) } + }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, interrupted, logFile, streamRef: stream, cleanup } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const logs = (h: any) => (existsSync(h.logFile) ? readFileSync(h.logFile, "utf8") : "") +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +async function makeBusy(h: any, sid: string) { + push(h, "session.execution.started", sid) + await wait(10) + push(h, "session.step.started", sid) + await wait(10) +} + +/** Parent busy, child goes quiet: arms the orphan watch. */ +async function armWatch(h: any) { + await makeBusy(h, SID) + await wait(20) + await makeBusy(h, CHILD) + await wait(20) + push(h, "session.idle", CHILD) + await wait(20) +} + +describe("v2: quiet subagents wait inside a patience window", () => { + test("a healthy quiet worker is waited on, not aborted", async () => { + // 5 minutes of quiet: past the 60s stuck window, inside the 30m + // dead window, no error. Old verdict: crashed -> interrupt + prod. + const now = Date.now() + const h = await setup({ + active: [SID, CHILD], + history: { + [SID]: [userMessage("go", now - 60_000)], + [CHILD]: [assistantAt(now - 5 * 60_000)], + }, + }) + await armWatch(h) + await wait(600) + expect(h.interrupted).toHaveLength(0) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }) + + test("CONTROL: quiet past the dead window still aborts", async () => { + // 40 minutes of quiet with no error: a hung model must not sit + // forever. Crashed handling is preserved. + const now = Date.now() + const h = await setup({ + active: [SID], + history: { + [SID]: [userMessage("go", now - 60_000)], + [CHILD]: [assistantAt(now - 40 * 60_000)], + }, + }) + await armWatch(h) + await wait(600) + expect(h.interrupted.length).toBeGreaterThan(0) + await teardown(h) + }) + + test("orphan aborts terminate instead of cycling forever", async () => { + // maxRetries 1: one orphan abort spends the budget; the next tick + // must give up, not abort again. + const now = Date.now() + const h = await setup({ + active: [SID], + maxRetries: 1, + history: { + [SID]: [userMessage("go", now - 60_000)], + [CHILD]: [assistantAt(now - 40 * 60_000)], + }, + }) + await armWatch(h) + // The abort path holds `aborting` for 2s (interrupt window), during + // which the watch skips; gaveUp lands on the first tick after that. + await wait(3000) + expect(h.interrupted).toHaveLength(1) + expect(logs(h)).toContain("gave up") + await teardown(h) + }) +}) diff --git a/src/v2/index.task-complete-text.test.ts b/src/v2/index.task-complete-text.test.ts new file mode 100644 index 0000000..6e20e8a --- /dev/null +++ b/src/v2/index.task-complete-text.test.ts @@ -0,0 +1,302 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexTaskCompleteTextTest" + +/** + * v2: a bare "TASK COMPLETE" latches completion like a stop statement. + * + * ses_edd1eb51 (2026-10-09): the model ended two deliveries with a trailing + * "TASK COMPLETE" and the plugin fired resume attempt 1/3 on schedule 186s + * later. endsWithCelebration is emoji-only and the completionSignaled setters + * are tool/emoji/stop-word paths, so the system's own completion vocabulary + * written as text latched nothing. The literal now latches when no todos are + * open (open todos still get the once-budgeted reminder), and the busy-stall + * path respects the latch. End-of-text anchored so mid-text mentions don't fire. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 9000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 300, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, +} + +async function setup(opts: { + messages?: unknown[] + todos?: Array<{ content: string; status: string }> + toolIds?: string[] +}): Promise { + const injected: Array<{ sid?: string; text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-tc-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const messages = opts.messages ?? [] + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => messages, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + message: { list: async () => [] }, + }, + }, + storage: { + get: async () => ({ todos: opts.todos ?? [] }), + set: async () => {}, + remove: async () => {}, + }, + tool: { + transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), + list: async () => (opts.toolIds ?? []).map((id) => ({ id })), + }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +const assistantMsg = (text: string) => ({ + // Prod shape carries role, not type (session.messages API). The harness + // convention is type-shaped; these tests deliberately use role-shaped + // messages because the ses_edbb4e67 incident was a type-only reader + // returning "" in prod while every test stayed green. + role: "assistant", + id: "msg_a", + time: { created: Date.now() - 60_000 }, + content: [{ type: "text", text }], +}) + +// Mirrors the ses_edd1eb51 delivery: report naming a file, trailing literal. +const TC_TEXT = "All recommendations are in /tmp/opencode/calibre-reco_findings.md TASK COMPLETE" +const PLAIN_TEXT = "All recommendations are in /tmp/opencode/calibre-reco_findings.md" + +describe("v2: bare TASK COMPLETE latches completion", () => { + test("TASK COMPLETE latches: later busy-stall stays quiet", async () => { + const h = await setup({ messages: [assistantMsg(TC_TEXT)] }) + // Prod shape (ses_edd1eb51): the delivery went quiet with no + // session.idle event, so no idle inspection ever ran and the stall + // ladder fired 186s later. No new turn arrives here either: a fresh + // turn re-arms by design (markBusy clears the latch). + push(h, "session.execution.started", SID) + await wait(1200) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }) + + test("TASK COMPLETE with open todos still earns the once-budgeted reminder", async () => { + const h = await setup({ + messages: [assistantMsg(TC_TEXT)], + todos: [{ content: "unfinished business", status: "pending" }], + }) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.idle", SID) + await wait(600) + expect(h.injected.length).toBeGreaterThan(0) + expect(h.injected[0].text).toMatch(/unfinished business/) + await teardown(h) + }) + + test("CONTROL: same delivery without the literal still recovers on stall", async () => { + const h = await setup({ messages: [assistantMsg(PLAIN_TEXT)] }) + push(h, "session.execution.started", SID) + await wait(1200) + expect(h.injected.filter((i: any) => i.sid === SID).length).toBeGreaterThan(0) + await teardown(h) + }) + + test("flat-text history (prod shape) latches like parts", async () => { + // ses_ed93e258 (2026-10-10): prod transcripts carry text flat, not + // in content parts. A parts-only reader went blind and the latch + // never engaged despite the fix being live. + const h = await setup({ + messages: [ + { + role: "assistant", + id: "msg_a", + time: { created: Date.now() - 60_000 }, + text: "Report written.\n\nTASK COMPLETE\n", + }, + ], + }) + push(h, "session.execution.started", SID) + await wait(1200) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }, 20000) + + test("newest-first history (prod order) latches like oldest-first", async () => { + // ses_ed975114 (2026-10-10): the message API returns newest-first; + // every walk reads the END as newest. Without canonicalization the + // busy latch judged the oldest turn: a stale plain delivery masked + // the fresh TASK COMPLETE and the stall fired. Two assistant turns + // force the order to matter (a single turn reads the same either way). + const now = Date.now() + const h = await setup({ + messages: [ + { + role: "assistant", + id: "msg_new", + time: { created: now - 60_000 }, + text: "Report written.\n\nTASK COMPLETE\n", + }, + { + role: "assistant", + id: "msg_old", + time: { created: now - 120_000 }, + text: "Still digging through the list.", + }, + ], + }) + push(h, "session.execution.started", SID) + await wait(1200) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }, 20000) + + test("trailing @N@ boundary marker does not defeat the latch (parts shape)", async () => { + // ses_ecca0ad6 (2026-10-13): the harness appends @12@-style boundary + // ids to stored message text. Nine "TASK COMPLETE" deliveries ended + // with a marker and the stall ladder fired on every one — the + // end-anchored match never saw the literal. Markers are metadata, + // not model prose, so the predicates strip trailing runs. + const h = await setup({ messages: [assistantMsg("Report written.\n\nTASK COMPLETE\n\n@75@")] }) + push(h, "session.execution.started", SID) + await wait(1200) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }, 20000) + + test("trailing @N@ boundary marker does not defeat the latch (flat shape)", async () => { + const h = await setup({ + messages: [ + { + role: "assistant", + id: "msg_a", + time: { created: Date.now() - 60_000 }, + text: "Report written.\n\nTASK COMPLETE\n\n@80@", + }, + ], + }) + push(h, "session.execution.started", SID) + await wait(1200) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }, 20000) + + test("noun between all and the verdict still latches (S14 Schwab tail)", async () => { + // 2026-10-15 session-tail review: "All work complete. Ready for your + // next instruction." matched nothing — the all/everything pattern + // admitted no noun — and the stall ladder fired on a finished session. + const h = await setup({ + messages: [assistantMsg("Reviewed the positions.\n\nAll work complete. Ready for your next instruction.")], + }) + push(h, "session.execution.started", SID) + await wait(1200) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }, 20000) + + test("standby phrasing latches without a done-claim beside it (S15 tail)", async () => { + // "Standing by for next instructions." carries no all/done/finished + // word, so only the standby stop pattern can latch it. + const h = await setup({ messages: [assistantMsg("PR merged and verified.\n\nStanding by for next instructions.")] }) + push(h, "session.execution.started", SID) + await wait(1200) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }, 20000) + + test("ready-for-next-task latches instead of continuing (S3 tail)", async () => { + // "Ready when you have the next task." means out of work, not + // ready-to-continue — it must latch quiet, never draw a `continue`. + const h = await setup({ messages: [assistantMsg("Stack session done.\n\nReady when you have the next task.")] }) + push(h, "session.execution.started", SID) + await wait(1200) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }, 20000) + + test("CONTROL: lowercase task complete is model prose, not the signal", async () => { + // Case discipline (2026-10-15): only byte-exact TASK COMPLETE latches. + // A lowercase "task complete" must NOT latch — the stall ladder stays + // armed and recovers on schedule like any unfinished delivery. + const h = await setup({ messages: [assistantMsg("Still working through it.\n\ntask complete")] }) + push(h, "session.execution.started", SID) + await wait(1200) + expect(h.injected.filter((i: any) => i.sid === SID).length).toBeGreaterThan(0) + await teardown(h) + }, 20000) +}) diff --git a/src/v2/index.task-complete.test.ts b/src/v2/index.task-complete.test.ts new file mode 100644 index 0000000..a064b48 --- /dev/null +++ b/src/v2/index.task-complete.test.ts @@ -0,0 +1,448 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexTaskCompleteTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_task_complete" + +/** + * `task_complete`: the model's explicit "I am finished". + * + * v2 ships no built-in equivalent — nothing in the v2 source mentions the name — + * so the plugin registering it is the only way it exists. That makes this a + * feature rather than a shim: the model gets a completion signal stronger than a + * trailing 🎉, because it chose to call this. + * + * The escalation is the part worth testing. The ack is fed straight back into the + * turn, and a stuck model answers it by calling the tool again rather than ending + * with text; v1 logged 27 consecutive acked calls with zero new user input. So the + * first call acks with an explicit stop instruction, the second warns, and the + * third throws to force the turn to end. + * + * The counter is reset by a new user message and by any other tool call, so the + * escalation measures a stuck loop and not several legitimate rounds of work. + * + * Every group carries a control. + */ + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + debug: true, +} + +const userMessage = (text: string, at: number, id = `msg_u_${at}`) => ({ + type: "user", + id, + time: { created: at }, + content: [{ type: "text", text }], +}) + +/** An hour ago: outside the 5-minute active-user window. */ +const OLD = Date.now() - 60 * 60_000 + +type Registered = { + name: string + description: string + input: unknown + execute: (input: any, context: { sessionID: string }) => Promise<{ content: string }> +} + +type Harness = { + tool: Registered | null + injected: Array<{ text?: string }> + logs: string[] +} + +async function setup( + opts: { + todos?: unknown[] | undefined + toolRegistry?: boolean + extraMessages?: unknown[] + parentID?: string + maxRetriesOverride?: number + }, +): Promise { + const injected: Harness["injected"] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-tc-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + let registered: Registered | null = null + const history: unknown[] = [userMessage("do the work", OLD), ...(opts.extraMessages ?? [])] + + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + ...OPTIONS, + logFile, + // Raise maxRetries so the "blocked call does not count toward the ack + // escalation" test can stay inside the block for several calls instead of + // walking into the bounded-block path. + ...(opts.maxRetriesOverride ? { maxRetries: opts.maxRetriesOverride } : {}), + }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + client: { + session: { + get: async () => ({ data: opts.parentID ? { id: SID, parentID: opts.parentID } : { id: SID } }), + }, + }, + storage: { + get: async () => ({ todos: opts.todos ?? [], updatedAt: Date.now() }), + set: async () => {}, + remove: async () => {}, + }, + } + if (opts.toolRegistry !== false) { + ctx.tool = { + transform: async (cb: any) => { + cb({ + add: (t: Registered) => { + registered = t + }, + }) + return { dispose() {} } + }, + list: async () => (registered ? [registered] : []), + } + } + + const cleanup = await (plugin as any).setup(ctx) + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + return { + get tool() { + return registered + }, + injected, + logs, + // Exposed for the few tests that deliver an event or a message mid-session. + ...({ cleanup, logFile, streamRef: stream, historyRef: history } as any), + } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const call = (h: Harness, sid = SID) => h.tool!.execute({}, { sessionID: sid }) + +describe("v2: task_complete is offered to the model", () => { + test("CONTROL: the tool is registered, and takes no arguments", async () => { + const h = await setup({}) + expect(h.tool).not.toBeNull() + expect(h.tool!.name).toBe("task_complete") + expect(h.tool!.description).toContain("Signal that all work is complete") + // v1's `args: {}`: the signal is the call, not a payload. + expect(h.tool!.input).toMatchObject({ type: "object", properties: {} }) + await teardown(h) + }) + + test("CONTROL: a plain call is acknowledged and records completion", async () => { + const h = await setup({}) + const res = await call(h) + expect(res.content).toContain("Task completion acknowledged") + // The stop instruction is in the ack on purpose: the ack is fed straight + // back into the turn, so a stuck model calls again instead of ending. + expect(res.content).toContain("End your turn now") + expect(res.content).toContain("do not call task_complete") + await teardown(h) + }) + + test("CONTROL: the ack is repeatable by design — no latching on the first call", async () => { + // The counter only escalates from the second consecutive call. This is the + // control for the escalation tests below: if the ack threw or latched, they + // would pass for the wrong reason. + const h = await setup({}) + await call(h) + await teardown(h) + }) +}) + +describe("v2: task_complete escalation — the ack self-loop", () => { + test("a second consecutive call warns instead of acking again", async () => { + const h = await setup({}) + await call(h) + const second = await call(h) + expect(second.content).toContain("acknowledged already on your previous call") + expect(second.content).toContain("further repeat calls are rejected as errors") + await teardown(h) + }) + + test("a third consecutive call is rejected as an error, forcing the turn to end", async () => { + const h = await setup({}) + await call(h) + await call(h) + await expect(call(h)).rejects.toThrow(/acknowledged twice/) + await teardown(h) + }) + + test("a fourth call is still an error — the loop cannot re-enter through the ack", async () => { + const h = await setup({}) + await call(h) + await call(h) + await expect(call(h)).rejects.toThrow() + await expect(call(h)).rejects.toThrow() + await teardown(h) + }) + + test("CONTROL: one call, then real tool work, then another call is not a repeat", async () => { + // Several legitimate rounds of work must not trip the escalation. This is + // the difference between a stuck model and a model that did more work. + const h = await setup({}) + await call(h) + stream2(h).push(ev("session.tool.called", { tool: "read" })) + await wait(30) + const second = await call(h) + expect(second.content).toContain("Task completion acknowledged") + expect(second.content).not.toContain("acknowledged already") + await teardown(h) + }) + + test("CONTROL: a new user message also resets the loop counter", async () => { + const h = await setup({}) + await call(h) + await call(h) // now on the warning + // A new request is new work: the counter starts clean, or one bad stretch + // would poison the rest of the session. + // + // Driven by an idle rather than by the message appearing: the reset is + // observed during idle inspection, so a test that never goes idle is testing + // nothing. (The todo and dead-stream tests hit the same wall.) + pushUserMessage(h, "actually, also this", Date.now() - 10 * 60_000) + await goIdle(h) + const third = await call(h) + expect(third.content).toContain("Task completion acknowledged") + expect(third.content).not.toContain("acknowledged already") + await teardown(h) + }) +}) + +describe("v2: task_complete respects the todo list", () => { + const OPEN = [ + { content: "Write the migration guide", status: "pending", priority: "high" }, + { content: "Delete the temp fixtures", status: "in_progress", priority: "low" }, + ] + + test("CONTROL: a call with todos still open is blocked and names them", async () => { + const h = await setup({ todos: OPEN }) + const res = await call(h) + expect(res.content).toContain("Write the migration guide") + expect(res.content).toContain("Delete the temp fixtures") + expect(res.content).toContain("Mark any finished todos complete") + // Not an ack: the model has not been told its work is recorded. + expect(res.content).not.toContain("No further continuation will be sent") + await teardown(h) + }) + + test("a blocked call also fires a visible nudge naming the open items", async () => { + // The tool result can collapse to an invisible one-liner in the transcript, + // so the blocking todos are also injected. Mirrors the idle-resume path. + const h = await setup({ todos: OPEN }) + await call(h) + expect(h.injected.length).toBeGreaterThan(0) + expect(h.injected[0].text).toContain("Write the migration guide") + await teardown(h) + }) + + test("a blocked call does not count toward the ack escalation", async () => { + // Work remains, so a later completion is legitimate. Feeding the escalation + // here would reject a model that was simply told to finish more. + // maxRetries 1 so the first call is blocked and the second is let through: + // the point is what signal the let-through call carries, not how long the + // block lasts. + const h = await setup({ todos: OPEN, maxRetriesOverride: 1 }) + await call(h) + const second = await call(h) + expect(second.content).toContain("Task completion acknowledged") + expect(second.content).not.toContain("acknowledged already") + await teardown(h) + }) + + test("the block is bounded — a model insisting is eventually believed", async () => { + // An unbounded block is its own kind of loop. After maxRetries the call is + // honoured, because a model that keeps saying "done" may know something the + // todo list does not. maxRetries defaults to 3, so the fourth call is let + // through. + const h = await setup({ todos: OPEN }) + let res = await call(h) + for (let i = 0; i < 3; i++) res = await call(h) + expect(res.content).toContain("Task completion acknowledged") + expect(res.content).not.toContain("Mark any finished todos") + await teardown(h) + }) + + test("a call with every todo closed is acked", async () => { + const h = await setup({ + todos: [ + { content: "Done already", status: "completed", priority: "high" }, + { content: "Dropped", status: "cancelled", priority: "low" }, + ], + }) + const res = await call(h) + expect(res.content).toContain("Task completion acknowledged") + await teardown(h) + }) + + test("a call with no todo tool at all is acked, not blocked", async () => { + // The dangerous failure is a session with no list being told it has open + // work. An absent list must not read as an empty-but-authoritative one. + const h = await setup({}) + const res = await call(h) + expect(res.content).toContain("Task completion acknowledged") + expect(res.content).not.toContain("Mark any finished todos") + await teardown(h) + }) +}) + +describe("v2: task_complete on a subagent", () => { + test("a subagent reporting completion is not gated on the parent's todo list", async () => { + // A child was never asked to do the parent's open items, so gating its + // report on them would block it on work outside its scope. + const h = await setup({ + todos: [{ content: "Parent's own task", status: "pending", priority: "high" }], + parentID: "ses_parent", + }) + const res = await call(h) + expect(res.content).toContain("Task completion acknowledged") + await teardown(h) + }) +}) + +describe("v2: task_complete degrades rather than failing the watchdog", () => { + test("a host with no tool registry still runs the plugin", async () => { + const h = await setup({ toolRegistry: false }) + expect(h.tool).toBeNull() + // The rest of the plugin must be live regardless. + stream2(h).push(ev("session.execution.started")) + await wait(30) + stream2(h).push(ev("session.idle")) + await wait(200) + expect(h.logs.some((l) => l.includes("task_complete not offered"))).toBe(true) + await teardown(h) + }) + + test("a registry that throws does not stop the watchdog", async () => { + // Registration failure is the one thing that must never take down the + // session recovery that the rest of this plugin exists for. + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-tc-throw-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [userMessage("go", OLD)], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async () => ({}), + prompt: async () => ({}), + }, + client: { session: { get: async () => ({ data: { id: SID } }) } }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { + transform: async () => { + throw new Error("registry unavailable") + }, + list: async () => [], + }, + } + const cleanup = await (plugin as any).setup(ctx) + for (const e of [ev("session.execution.started"), ev("session.step.started"), ev("session.idle")]) { + stream.push(e) + await wait(10) + } + await wait(400) + ;(cleanup as (() => void) | undefined)?.() + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + rmSync(logFile, { force: true }) + expect(logs.some((l) => l.includes("task_complete registration failed"))).toBe(true) + // And the plugin is still running: it logged its startup line. + expect(logs.some((l) => l.includes("task_complete registration failed") && l.includes("continuing without it"))).toBe( + true, + ) + }) +}) + +/** The harness keeps the event stream private; these reach it for the tests + * that need to deliver a tool call or a user message mid-session. */ +function stream2(h: any) { + return h.streamRef as ReturnType +} + +function pushUserMessage(h: any, text: string, at: number) { + ;(h.historyRef as unknown[]).push(userMessage(text, at)) +} + +/** Drive a turn to idle, which is what makes the plugin look at the history. */ +async function goIdle(h: any) { + const stream = stream2(h) + for (const e of [ + ev("session.execution.started"), + ev("session.step.started"), + ev("session.step.ended"), + ev("session.idle"), + ]) { + stream.push(e) + await wait(10) + } + await wait(200) +} diff --git a/src/v2/index.thinking-tool.test.ts b/src/v2/index.thinking-tool.test.ts new file mode 100644 index 0000000..6e24797 --- /dev/null +++ b/src/v2/index.thinking-tool.test.ts @@ -0,0 +1,221 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexThinkingToolTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_thinking_tool" + +/** + * A tool call written into the reasoning block. + * + * When a model emits raw tool-call markup inside its thinking instead of calling + * the tool, nothing executes and nothing raises: the turn completes, the text the + * user sees may be fine, and the work silently does not happen. The text variant + * of this failure has its own detector and its own prompt; the reasoning variant + * needs a different one, because the model is not forgetting the mechanism, it is + * writing in the wrong channel. + * + * v2 makes this a separate read rather than a filter inside a joined string: + * `AssistantContent` tags reasoning and text distinctly, so the newest message's + * reasoning parts can be judged on their own. + * + * Every group carries a control. + */ + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const userTurn = () => ({ + type: "user", + id: "msg_u0", + time: { created: Date.now() - 60 * 60_000 }, + content: [{ type: "text", text: "fix the failing test" }], +}) + +const assistantTurn = (parts: Array>) => ({ + type: "assistant", + id: "msg_a1", + time: { created: Date.now() - 60_000 }, + content: parts, + finish: "stop", + tokens: { input: 100, output: 200, reasoning: 0, cache: { read: 0, write: 0 } }, +}) + +/** + * Every fixture carries a normal text part. + * + * Without one the message is a *silent dead stream* — finished, no text, plenty + * of output tokens — and that detector fires first with its own "continue". That + * is correct behaviour and it has its own tests; here it would just mask which + * branch is under test. + */ + +/** A raw tool call sitting in the reasoning, with normal prose alongside. */ +const REASONING_WITH_TOOL_CALL = [ + { type: "text", text: "Let me start by reading the failing test." }, + { + type: "reasoning", + text: "I should read the test file first.\n\nsrc/a.test.ts\n", + }, +] +/** Ordinary reasoning, no markup. */ +const REASONING_ONLY = [ + { type: "text", text: "Checking which assertion is wrong." }, + { type: "reasoning", text: "Let me think about which assertion is wrong here." }, +] +/** The same markup, but in the text part where the text detector already sees it. */ +const TEXT_WITH_TOOL_CALL = [{ type: "text", text: "src/a.test.ts" }] + +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + debug: true, +} + +async function replay( + parts: Array>, + opts: Record = {}, +): Promise<{ injected: Array<{ text?: string }>; logs: string[] }> { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-thinktool-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, ...opts, registryScope: LOG_SALT }, + session: { + context: async () => [userTurn(), assistantTurn(parts)], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + client: { session: { get: async () => ({ data: {} }) } }, + } + + const cleanup = await (plugin as any).setup(ctx) + for (const e of [ + ev("session.execution.started"), + ev("session.step.started"), + ev("session.step.ended"), + ev("session.idle"), + ]) { + stream.push(e) + await wait(10) + } + await wait(700) + ;(cleanup as (() => void) | undefined)?.() + + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + rmSync(logFile, { force: true }) + return { injected, logs } +} + +describe("v2: a tool call written into the reasoning block", () => { + test("CONTROL: markup in the reasoning asks the model to use the tool mechanism", async () => { + const { injected } = await replay(REASONING_WITH_TOOL_CALL) + expect(injected).toHaveLength(1) + // The reasoning prompt is specific to this failure: the tool call is + // well-formed, it was just written in the wrong channel. + expect(injected[0].text).toContain("thinking/reasoning") + }) + + test("CONTROL: the same markup in a text part asks for the tool mechanism generally", async () => { + const { injected } = await replay(TEXT_WITH_TOOL_CALL) + expect(injected).toHaveLength(1) + expect(injected[0].text).not.toContain("thinking/reasoning") + }) + + test("reasoning with no tool call in it is left alone", async () => { + const { injected, logs } = await replay(REASONING_ONLY) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("tool-call-in-reasoning"))).toBe(false) + }) + + test("the reasoning prompt is configurable", async () => { + const { injected } = await replay(REASONING_WITH_TOOL_CALL, { + thinkingToolRecoveryPrompt: "Use the real tool call, not reasoning.", + }) + expect(injected).toHaveLength(1) + expect(injected[0].text).toBe( + "Use the real tool call, not reasoning. Do not acknowledge this message and do not reply to it visibly; apply it silently and continue the task.", + ) + }) + + test("code blocks in reasoning are not mistaken for a tool call", async () => { + // Same code-stripping the text detector uses: a fenced example of the + // shape, or an inline path in backticks, must not trigger a nudge. + const { injected } = await replay([ + { type: "text", text: "Checking how the tool-call format is documented." }, + { + type: "reasoning", + text: "The format is documented as:\n```\n\nx\n\n```\n", + }, + ]) + expect(injected).toEqual([]) + }) + + test("both channels in one message: the reasoning one is judged first", async () => { + // The reasoning variant silently does nothing, so reporting the text one + // would send the model off to fix the wrong thing. + const { injected } = await replay([...TEXT_WITH_TOOL_CALL, ...REASONING_WITH_TOOL_CALL]) + expect(injected).toHaveLength(1) + expect(injected[0].text).toContain("thinking/reasoning") + }) + + test("the budget is shared with the text variant, not doubled", async () => { + // One phenomenon with two symptoms. Two independent budgets would let the + // plugin spend twice the retries on a model that keeps doing this. + const { injected } = await replay(REASONING_WITH_TOOL_CALL, { maxRetries: 1 }) + expect(injected.length).toBeLessThanOrEqual(1) + }) +}) \ No newline at end of file diff --git a/src/v2/index.todo.test.ts b/src/v2/index.todo.test.ts new file mode 100644 index 0000000..e4a140b --- /dev/null +++ b/src/v2/index.todo.test.ts @@ -0,0 +1,497 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexTodoTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_todo" + +/** + * The todo list as the plugin under test reads it. + * + * auto-resume does not own a todo list — it reads one. In v2 the real store is the + * SESSION MESSAGE LOG: every `todowrite` call is persisted as a tool part whose + * input carries the whole list, so the newest completed call IS the current list. + * The v1-shaped `todos/` storage key is kept as a last-resort fallback + * (nothing in v2 writes it, so it always misses). + * + * The `replay` harness therefore takes both: `todos` populates the storage + * fallback, `messages` supplies the message log. Left undefined, `messages` + * resolves to an empty page so the storage cases keep testing storage — which is + * the point, because the fallback must stay alive. + * + * Three things are under test: + * + * The celebration cross-check. A trailing 🎉 is the model's own "finished" + * signal, and latching on it is what stops the nudge. But a model that finishes + * early celebrates early, so the emoji alone is not trustworthy: with work still + * listed, the celebration is a false positive and the right answer is to name + * what is unfinished. + * + * Two done-claim prompts, two budgets. A done-claim with open todos and a + * done-claim with no detail report are different problems; spending one budget + * must not silence the other. + * + * When a budget re-arms. #26 was an unbounded done-claim nudge. The fix is that + * it re-arms only on a genuinely new work cycle — an inbound user message — + * not on every turn the model announces completion again. + * + * Every group carries a control. + */ + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const userMessage = (text: string, at: number) => ({ + type: "user", + id: `msg_u_${at}`, + time: { created: at }, + content: [{ type: "text", text }], +}) + +const assistantMessage = (text: string, at: number) => ({ + type: "assistant", + id: `msg_a_${at}`, + time: { created: at }, + content: [{ type: "text", text }], + finish: "stop", + tokens: { input: 100, output: 200, reasoning: 0, cache: { read: 0, write: 0 } }, +}) + +const OPEN = [ + { content: "Write the migration guide", status: "pending", priority: "high" }, + { content: "Delete the temp fixtures", status: "in_progress", priority: "low" }, +] +const CLOSED = [ + { content: "Write the migration guide", status: "completed", priority: "high" }, + { content: "Delete the temp fixtures", status: "cancelled", priority: "low" }, +] + +/** An hour ago: outside the 5-minute active-user window, so idle nudges do not stand down. */ +const OLD = Date.now() - 60 * 60_000 +/** Ten minutes ago: still outside that window, but newer than OLD. */ +const RECENT_BUT_STALE = Date.now() - 10 * 60_000 + +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + debug: true, +} + +type Harness = { + injected: Array<{ text?: string }> + logs: string[] + storageWrites: string[] + storageReads: string[] +} + +async function replay( + turns: Array<{ text: string; userMessages?: unknown[] }>, + todos: unknown[] | undefined, + opts: Record = {}, + storageShape: { omitStorage?: boolean } = {}, + messages: unknown[] | undefined = undefined, +): Promise { + const injected: Harness["injected"] = [] + const storageWrites: string[] = [] + const storageReads: string[] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-todo-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + // The history grows turn by turn, exactly as a real session's does. + let history: unknown[] = [] + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, ...opts, registryScope: LOG_SALT }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + client: { + session: { + get: async () => ({ data: {} }), + // The message list endpoint answers with a `{ data, cursor }` envelope, + // never a bare array — handing the envelope straight to the parser trips + // its Array.isArray guard and silently resolves nothing. Reproduced here + // so that trap is exercised by every test in the group below. + message: { list: async () => ({ data: messages ?? [], cursor: null }) }, + }, + }, + } + if (!storageShape.omitStorage) { + ctx.storage = { + get: async (key: string) => { + storageReads.push(key) + return { todos: todos ?? [], updatedAt: Date.now() } + }, + // Present so that "the plugin never writes here" is an assertion the + // harness can actually make, rather than an assumption. + set: async (key: string) => storageWrites.push(key), + remove: async (key: string) => storageWrites.push(key), + } + } + + const cleanup = await (plugin as any).setup(ctx) + for (const turn of turns) { + for (const m of turn.userMessages ?? []) history.push(m) + history.push(assistantMessage(turn.text, Date.now() - 30_000)) + for (const e of [ + ev("session.execution.started"), + ev("session.step.started"), + ev("session.step.ended"), + ev("session.idle"), + ]) { + stream.push(e) + await wait(10) + } + } + await wait(700) + ;(cleanup as (() => void) | undefined)?.() + + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + rmSync(logFile, { force: true }) + return { injected, logs, storageWrites, storageReads } +} + +/** A done-claim with nothing in it — the premature stop. */ +const BARE_DONE = "Task done." +/** The same, but closed out with the model's own finished signal. */ +const CELEBRATED = "Everything is in place. 🎉" + +describe("v2: the todo list, read from the tool that owns it", () => { + test("CONTROL: a celebration with open todos is a false positive, and is named", async () => { + const { injected, logs } = await replay([{ text: CELEBRATED }], OPEN) + expect(injected).toHaveLength(1) + expect(injected[0].text).toContain("Write the migration guide") + expect(injected[0].text).toContain("Delete the temp fixtures") + expect(logs.some((l) => l.includes("open-todos-celebration-false-positive"))).toBe(true) + }) + + test("CONTROL: a celebration with everything closed latches and never nudges", async () => { + const { injected, logs } = await replay([{ text: CELEBRATED }, { text: CELEBRATED }], CLOSED) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("no open todos — latching completion"))).toBe(true) + }) + + test("CONTROL: a bare done-claim with open todos gets the work prompt, not the details prompt", async () => { + const { injected } = await replay([{ text: BARE_DONE }], OPEN) + expect(injected).toHaveLength(1) + // v1's wording for this branch names the todo list. + expect(injected[0].text).toContain("todo list") + }) + + test("CONTROL: the same bare done-claim with no todos still gets the details prompt", async () => { + const { injected } = await replay([{ text: BARE_DONE }], []) + expect(injected).toHaveLength(1) + expect(injected[0].text).toContain("no work description") + expect(injected[0].text).not.toContain("todo list") + }) + + test("both done-claim prompts stay live independently", async () => { + // Two different problems. If the todo branch spent the same budget as the + // details branch, exhausting one would silently disable the other. + const withTodos = await replay([{ text: BARE_DONE }, { text: BARE_DONE }], OPEN, { maxRetries: 1 }) + const withoutTodos = await replay([{ text: BARE_DONE }, { text: BARE_DONE }], [], { maxRetries: 1 }) + expect(withTodos.injected).toHaveLength(1) + expect(withoutTodos.injected).toHaveLength(1) + expect(withTodos.injected[0].text).not.toBe(withoutTodos.injected[0].text) + }) + + test("the done-claim budget does not re-arm just because the model repeated itself", async () => { + // This is #26. A model that re-announces completion every turn used to be + // handed a fresh budget every time it announced, so the nudge never stopped. + const { injected } = await replay([{ text: BARE_DONE }, { text: BARE_DONE }, { text: BARE_DONE }], [], { + maxRetries: 1, + }) + expect(injected).toHaveLength(1) + }) + + test("a genuinely new user message does re-arm it", async () => { + // The other half of the fix: the budget must be recoverable, or one bad + // stretch would silence the plugin for the rest of the session. + const { injected } = await replay( + [ + { text: BARE_DONE, userMessages: [userMessage("first ask", OLD)] }, + { text: BARE_DONE }, + { text: BARE_DONE, userMessages: [userMessage("actually, also this", RECENT_BUT_STALE)] }, + ], + [], + { maxRetries: 1 }, + ) + expect(injected).toHaveLength(2) + }) + + test("a celebration with open todos keeps being caught on later turns", async () => { + // Proof that the false-positive branch does not latch: a latched turn + // would go silent here. + const { injected } = await replay([{ text: CELEBRATED }, { text: CELEBRATED }], OPEN, { maxRetries: 1 }) + expect(injected).toHaveLength(2) + }) + + test("the list is read under the todo tool's own key", async () => { + const { storageReads } = await replay([{ text: CELEBRATED }], OPEN) + expect(storageReads).toContain(`todos/${SID}`) + }) + + test("the plugin never writes to that key", async () => { + // It is a consumer, not a second owner. A write here would mean auto-resume + // was maintaining a copy that could drift from the real one. + // + // Scoped to the todo domain, deliberately. The plugin DOES write storage + // since R1 (audit 2026-10-10): `resume-quota/` records carry the + // rate-limit ladder across a process restart, replacing a scheme that + // grepped the log file. The invariant that matters is that auto-resume + // never takes ownership of a list it does not own — asserted here, and + // the write-confinement is pinned by the next test. + const { storageWrites } = await replay([{ text: CELEBRATED }], OPEN) + expect(storageWrites.filter((k) => k.startsWith("todos/"))).toEqual([]) + }) + + test("storage writes are confined to the plugin's own quota domain", async () => { + // The counterpart to the assertion above: writing is allowed, writing + // anywhere else is not. A future feature reaching for someone else's + // storage namespace fails here rather than silently mutating state it + // does not own. + const { storageWrites } = await replay([{ text: CELEBRATED }], OPEN) + for (const key of storageWrites) { + expect(key.startsWith("resume-quota/")).toBe(true) + } + }) + + test("the list is read once per turn, not once per check", async () => { + // The storage call is cheap but the cache is the difference between one read + // and two on every idle where both a celebration and a done-claim are judged. + const { storageReads } = await replay([{ text: CELEBRATED }], OPEN) + expect(storageReads.filter((k) => k === `todos/${SID}`).length).toBeLessThanOrEqual(2) + }) + + test("no storage domain at all is not a crash", async () => { + // A host without ctx.storage must degrade to "cannot conclude", which means + // the old behaviour — latch on the emoji — rather than a thrown error. + const { injected, logs } = await replay([{ text: CELEBRATED }], OPEN, {}, { omitStorage: true }) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("latching completion"))).toBe(true) + }) + + test("a malformed record is treated as no list, not as no work", async () => { + // The dangerous direction is "list unreadable" → "nothing is open" → nudge. + // Both malformed shapes must fall back to the old behaviour instead. + for (const bad of [{ todos: "nope" }, { todos: [null, 7] }, null]) { + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-todo-bad-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const injected: unknown[] = [] + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async () => [userMessage("ask", OLD), assistantMessage(CELEBRATED, Date.now())], + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push(a?.text), {}), + prompt: async (a: any) => (injected.push(a?.text), {}), + }, + client: { session: { get: async () => ({ data: {} }) } }, + storage: { get: async () => bad, set: async () => {}, remove: async () => {} }, + } + const cleanup = await (plugin as any).setup(ctx) + for (const e of [ + ev("session.execution.started"), + ev("session.step.started"), + ev("session.step.ended"), + ev("session.idle"), + ]) { + stream.push(e) + await wait(10) + } + await wait(600) + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + expect(injected).toEqual([]) + } + }) + + test("the work prompt is configurable", async () => { + const { injected } = await replay([{ text: BARE_DONE }], OPEN, { + doneWithoutWorkPrompt: "Your todo list still has open items.", + }) + expect(injected).toHaveLength(1) + expect(injected[0].text).toBe( + "Your todo list still has open items. Do not acknowledge this message and do not reply to it visibly; apply it silently and continue the task.", + ) + }) +}) + + +describe("v2: the todo list is read from the session message log", () => { + /** A completed `todowrite` tool part, stamped so ordering is unambiguous. */ + const part = (todos: unknown[], at: number) => ({ + type: "tool", + name: "todowrite", + state: { status: "completed", input: { todos }, time: { start: at, end: at } }, + }) + const msg = (parts: unknown[]) => ({ role: "user", content: parts }) + /** A `todowrite` part whose `input` is `input` verbatim, for malformed shapes. */ + const rawPart = (input: unknown, at: number) => ({ + type: "tool", + name: "todowrite", + state: { status: "completed", input, time: { start: at, end: at } }, + }) + + /** Distinct from OPEN/CLOSED so an assertion cannot pass on the wrong list. */ + const NEWEST = [{ content: "newest open item", status: "pending", priority: "high" }] + const OLDER = [{ content: "older open item", status: "pending", priority: "low" }] + + test("CONTROL: a todowrite in the log is enough, with the storage key left empty", async () => { + // The bug this group exists for. A storage-only reader finds an empty key, + // concludes "no todos", and fires a false done-claim nudge on a session with + // work still listed — which is how /todo printed [0/0] beside 14 todos. + const { injected, logs, storageReads } = await replay( + [{ text: CELEBRATED }], + [], + {}, + {}, + [msg([part(OPEN, Date.now())])], + ) + expect(injected).toHaveLength(1) + expect(injected[0].text).toContain("Write the migration guide") + // Not read at all: the log answered, so the dead key is never consulted. + expect(storageReads).not.toContain(`todos/${SID}`) + expect(logs.some((l) => l.includes("todos via ctx.client.session.message.list"))).toBe(true) + }) + + test("the NEWEST completed todowrite wins, not the oldest", async () => { + // `/api/session/{id}/message` returns NEWEST FIRST, so a last-match-wins loop + // selects the OLDEST list. This shipped in two places before it was caught, + // and it is silent: an older list is a plausible-looking list. + const now = Date.now() + const { injected } = await replay( + [{ text: CELEBRATED }], + [], + {}, + {}, + // Newest first, exactly as the endpoint orders it. + [msg([part(NEWEST, now)]), msg([part(OLDER, now - 60_000)])], + ) + expect(injected).toHaveLength(1) + expect(injected[0].text).toContain("newest open item") + expect(injected[0].text).not.toContain("older open item") + }) + + test("an unfinished todowrite is ignored, not half-read", async () => { + // Only a COMPLETED call replaced the list. A pending or failed one carries a + // partial input, and treating that as the list is how a half-written todo + // becomes "everything is done". + const { injected, logs } = await replay( + [{ text: CELEBRATED }], + [], + {}, + {}, + [ + msg([ + { + type: "tool", + name: "todowrite", + state: { status: "error", input: { todos: NEWEST }, time: { end: Date.now() } }, + }, + ]), + ], + ) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("no open todos — latching completion"))).toBe(true) + }) + + test("a malformed list yields 'cannot conclude', never an empty list", async () => { + // The dangerous direction is "unreadable" becoming "nothing is open". An + // unreadable list must fall through to the fallback, exactly like no list. + for (const bad of [{ todos: "nope" }, { todos: [null, 7] }, {}]) { + const { injected } = await replay( + [{ text: CELEBRATED }], + [], + {}, + {}, + [msg([rawPart(bad, Date.now())])], + ) + expect(injected).toEqual([]) + } + }) + + test("the storage fallback still answers when the log has no todowrite", async () => { + // The fallback is not dead code we can drop: it is the only source on a host + // whose log the plugin cannot read. + const { injected, storageReads } = await replay( + [{ text: CELEBRATED }], + OPEN, + {}, + {}, + [msg([{ type: "text", text: "no tool calls here" }])], + ) + expect(storageReads).toContain(`todos/${SID}`) + expect(injected).toHaveLength(1) + expect(injected[0].text).toContain("Write the migration guide") + }) + + test("a host with neither source degrades instead of crashing", async () => { + const { injected, logs } = await replay( + [{ text: CELEBRATED }], + [], + {}, + { omitStorage: true }, + [], + ) + expect(injected).toEqual([]) + expect(logs.some((l) => l.includes("latching completion"))).toBe(true) + }) +}) diff --git a/src/v2/index.ts b/src/v2/index.ts new file mode 100644 index 0000000..cfec7e5 --- /dev/null +++ b/src/v2/index.ts @@ -0,0 +1,6689 @@ +/** + * opencode-auto-resume — adapted for OpenCode v2 plugin API. + * + * Port of https://github.com/Mte90/opencode-auto-resume (v1 hooks API) to the + * v2 promise-plugin API (`Plugin.define` + `ctx.event.subscribe()`). + * + * What changed vs. v1: + * - Default export is `{ id, setup }` via `Plugin.define`; setup returns a cleanup fn. + * - Events come from `ctx.event.subscribe()` (AsyncIterable) with flat payloads + * (`{ type, data }`) instead of the v1 `event` hook (`{ type, properties }`). + * - SDK calls are flattened: `ctx.session.prompt({ sessionID, text })`, + * `ctx.session.interrupt({ sessionID })`, `ctx.session.active()`. + * - Assistant text is accumulated from `session.text.delta` events for liveness; + * `ctx.session.context()` (stable v2) supplies the authoritative final + * assistant text at idle time — replacing v1's `session.messages()` polling. + * - No `ctx.app.log` in v2: `ctx.app` is only `{name, version, channel}` (see + * `packages/plugin/src/app.ts`), and v2 has no `app.log` endpoint. Console + * output from a hosted plugin is not captured anywhere retrievable, so v1's + * `ctx.client.app.log(...)` has no equivalent. This build therefore appends + * to its own log file — see LOG_FILE below. + * - Targets the stable v2 API (`@opencode/plugin`). Event names are unchanged + * from the beta port; `session.execution.interrupted` now carries a `reason`. + * + * Detection/recovery features ported: + * - Stalled stream watchdog (busy session with no events for chunkTimeoutMs) + * - Execution/step failure recovery (`session.execution.failed`, `session.step.failed`) + * - Provider retry awareness (`session.retry.scheduled`) + * - Tool-call-printed-as-text detection + targeted recovery prompt + * - "Ready to continue" / stalled-intent ("...:") nudges + * - "Done" claim without work verification + * - Hallucination loop guard (too many auto-continues → interrupt + resume) + * - Subagent-awareness: does not recover a parent blocked on a running subagent + * - Permission-awareness: never recovers while a permission dialog is open + * + * Install: drop this file in ~/.config/opencode/plugins/ (auto-loaded), or add: + * { "plugins": [{ "package": "./plugins/auto-resume-v2.ts", "options": { ... } }] } + */ + +/** + * v2 entrypoint shape. + * + * The package depends on `@opencode-ai/plugin`, which exports `Plugin` only as a + * *type* — `(input, options?) => Promise`. There is no runtime + * `Plugin.define` in it, so the previous `import { Plugin } from + * "@opencode/plugin"` did not resolve and the file could not be built or + * type-checked at all (see the build script, which only ever built v1's + * `src/index.ts`). The v2 contract is simply `{ id, setup }`, so define the + * identity helper locally rather than importing a symbol that does not exist. + */ +type AutoResumePlugin = { + id: string + setup: (ctx: AutoResumePluginInput) => unknown +} + +const define = (plugin: T): T => plugin + +import { appendFileSync, readFileSync, existsSync, mkdirSync, statSync, writeFileSync } from "node:fs" +import { homedir } from "node:os" +import { dirname, join } from "node:path" + +/** + * The subset of the v2 plugin input this plugin actually consumes. Typed + * structurally so the file type-checks standalone, without pinning a + * @opencode-ai/plugin version whose `Plugin` type describes the v1 contract. + */ +interface AutoResumePluginInput { + /** Per-plugin options from config. */ + options?: Record + /** Event stream (AsyncIterable of `{ type, data }`). */ + event: { subscribe: (opts?: { signal?: AbortSignal }) => AsyncIterable } + /** Server-side session/messaging operations. */ + session: Record any> + /** + * Opencode API client, when the host provides one. Declared on the real + * `@opencode-ai/plugin` `PluginInput` (`client: ReturnType`); typed here structurally so the file still + * type-checks standalone. Optional: a host may not supply it, and every + * use must degrade gracefully when it is absent. + */ + client?: { session: Record any> } + /** + * Model catalogue. v2's `ModelDomain.get(providerID, modelID)` returns + * `Model.Info`, whose `limit` is `{ context, input?, output }` — the usable + * context window for context-saturation checks. Optional: a host may not + * supply it, and the saturation check degrades to "no intervention". + */ + model?: { get: (providerID: string, modelID: string) => unknown } + /** + * Installed-plugin inventory (`ctx.plugin.list()`). v2 removed the `config` + * domain, so this is how a plugin detects that magic-context is present before + * routing a saturated parent to it. + */ + plugin?: { list: () => Promise } + /** + * Key/value store scoped to the plugin (`ctx.storage.get/set/remove/scan`). + * + * Two consumers. It READS the todo list the installed todo tool writes — + * auto-resume does not own that list. And since R1 it WRITES its own + * `resume-quota/` records, so the rate-limit ladder survives a real + * process restart. That used to ride on the log file, which is the worst + * available channel: a 2MB rotation cap and a fixed 3000-line tail can drop + * a live arm. `set`/`remove` are optional so a host that only offers `get` + * still works — the ladder then simply falls back to the log markers. + */ + storage?: { + get: (key: string) => Promise + set?: (key: string, value: unknown) => Promise + remove?: (key: string) => Promise + } + /** + * Tool registry (`ctx.tool.transform` / `ctx.tool.list` / `ctx.tool.hook`). + * v2 plugins register their own tools here, which is how `task_complete` + * exists: there is no built-in equivalent in v2, so the plugin provides it. + */ + tool?: { + transform: (cb: (editor: AutoResumeToolEditor) => void) => Promise + list: () => Promise + } + /** Application logger, when the host provides one. */ + app?: { log?: (level: string, message: string) => unknown } +} + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface ToolCallRecord { + toolName: string + at: number +} + +/** Minimal structural view of a V2Event (avoids depending on client internals). */ +interface V2Event { + type: string + created: number + data?: Record +} + +/** The tool registry handle passed to `ctx.tool.transform`. */ +interface AutoResumeToolEditor { + add(tool: { + name: string + description: string + input: Record + execute: (input: any, context: { sessionID: string }) => Promise<{ content: string }> + }): void +} + +interface SessionWatch { + createdAt: number + lastActivityAt: number + status: "busy" | "idle" | "unknown" + userCancelled: boolean + /** When the user-cancel latch was set. Re-arm requires an inbound user + * message newer than this — otherwise a pre-interrupt message the plugin + * simply hadn't observed yet would clear a latch it postdates. */ + userCancelledAt: number + resumeAttempts: number + /** + * Consecutive ladder restarts driven by trickled progress with no genuine + * user message in between (ses_ed975114, 2026-10-10: 1/3 every ~3.5min + * forever). Each refire doubles the silence fuse (cap x8) so slow + * workers decay to patience instead of nagging on a fixed clock. Cleared + * only by a genuine user message — never by lifecycle transitions. + */ + episodeRefires: number + /** Recoveries sent this episode, all paths that build stall text. Unlike the + * ladder rung (which progress restarts) and episodeRefires (which counts only + * progress restarts), this counts every fire so the nudge text can say which + * recovery of the episode it is. Reset at the same genuine-episode sites. */ + episodeRecovers: number + /** Lifetime stall continues this genuine user episode. Unlike + * `resumeAttempts` (restarted by model text) this only resets on a + * genuine user message, so a model that keeps answering "done" cannot + * stretch 1/3 forever. Past the max the session stands down. */ + lifetimeContinues: number + lastRetryAt: number + /** Newest `session.retry.scheduled` for this session: the provider is + * retrying this turn itself, so a stall that follows within + * PROVIDER_RETRY_GRACE_MS is a transport fault, not model unresponsiveness. */ + providerRetryAt: number + /** Last wall-clock a user-visible quota notice was sent (first arm + + * budget-exhausted). One notice per episode: the stand-down itself is + * silent by design, but a session that looks dead for hours with no + * explanation reads as broken (ses_edd3d940, 2026-10-10). */ + quotaNoticedAt: number + gaveUp: boolean + /** Newest genuine (non-own-prompt) user message observed, by timestamp. Our + * own injections travel as user messages but carry no new instructions, so + * they must not read as the user taking over (see noteInboundUserMessage). + * Compared against lastInjectAt to tell our episode continuing from the + * user starting a new one. */ + lastGenuineUserAt: number + /** Consecutive failure-driven recoveries with no model output between them. + * A dead backend turns every injected continue into the next failure, so + * without this the ladder cycles 1/3, 2/3, 3/3, gaveUp, abort+resume, + * 1/3… forever, one user-visible prod per cycle (ses_ede801ca, 2026-10-09: + * backend down, prods every ~90s indefinitely). Past FAILURE_LOOP_MAX the + * session stands down until a genuine user turn or fresh model output — + * and neither our own injections nor markBusy clear it, or the loop that + * set it would clear it too. */ + consecFailureRecovers: number + failureLoopLatched: boolean + aborting: boolean + /** Timestamp of the last plugin-initiated `session.interrupt()`. Any abort-shaped + * event inside `SELF_ABORT_TTL_MS` of it is ours, not the user's. */ + selfAbortAt: number + /** Count of plugin-initiated interrupts in the current `INTERRUPT_WINDOW_MS`. */ + interruptsThisWindow: number + interruptWindowStart: number + /** Timestamp of the last recovery injection, for `injectIntervalMs` debouncing. */ + lastInjectAt: number + /** Last injected prod text + assistant snapshot at prod time, for duplicate suppression. */ + lastProdText: string + prodAssistantSnapshot: string + /** Tool/shell completions since the last successful prod. The + * duplicate skip promises 'zero model progress' but text growth + * plus in-flight tools cannot see a tool that ran silently. */ + completedSinceProd: number + /** Successful tool/shell completions since the last successful prod. + * Success-only on purpose: failures increment completedSinceProd but + * must keep the ladder climbing (failure-loop latch owns that path). + * Reset on every own-inject next to completedSinceProd. */ + succeededSinceProd: number + /** Texts this plugin injected (visible channel posts them as real user + * messages). `noteInboundUserMessage` must not read them as new user + * instructions, or every injection re-arms the budgets it just spent and + * the same errors refire on the next idle. Capped; identity is exact text. */ + ownPromptTexts: string[] + recovering: boolean + /** Latched when a failure carries an OOC error that `continue` can never clear; recovery (continue + abort+resume) is refused until a genuine user/agent turn. */ + oocLocked: boolean + oocLockReason: string | null + /** Latched when a failure names a model that no longer exists. Same shape as + * oocLocked but for a different unresolvable class, and for the same reason: + * attempt budgets live per instance and reset on reload, so only the class + * stops a retry loop that spans restarts. */ + permLocked: boolean + permLockReason: string | null + /** Occurrences of an indeterminate failure seen this turn; see + * INDETERMINATE_ERROR_TYPES. Reset only by a genuine turn, so our own + * recovery turns cannot keep clearing the count. */ + indeterminateStrikes: number + /** True while one of our own recovery continues is in flight — distinguishes a self-caused execution.started from a genuine turn. */ + selfRecovery: boolean + + // Assistant text accumulation (v2 replacement for session.messages()) + textParts: Map // assistantMessageID -> accumulated text + lastAssistantText: string + lastAssistantMessageID: string | null + + // Recovery budgets + toolTextAttempts: number + /** Turns ended with a tool call that never returned. Re-arms per request + * like the other nudge budgets: a new user message means new work, and an + * old stuck call says nothing about this one. */ + stuckToolAttempts: number + continueTimestamps: number[] + doneClaimAttempts: number + /** Done-claim with todos still open. Separate from `doneClaimAttempts`: the two + * prompts ask for different things, so spending one budget must not silence the other. */ + doneClaimOpenTodosAttempts: number + /** Open-todos reminders (the celebration false positive). Persists across turns — + * an open list does not become finished by the model saying it is. */ + todoNudgeAttempts: number + intentNudgeAttempts: number + /** Set when the model's own completion signal was seen (a trailing 🎉). + * Latches so a finished session stops being nudged. */ + completionSignaled: boolean + /** When the latch above was set (Date.now()). Compared against genuine new + * user messages: only work that arrived AFTER the latch re-opens it. + * Without the timestamp the latch could only be cleared on lifecycle + * transitions (markBusy) — but the watchdog re-marks busy from server truth + * with no new work, which wiped freshly-set latches within seconds + * (ses_ed33c0da, 2026-10-11: TASK COMPLETE latched at idle would have died + * at the re-mark 1s later). Cleared alongside completionSignaled. */ + completionSignaledAt: number + /** Set once the empty-idle instant nudge has fired for the current episode. + * A turn that ends with nothing delivered gets exactly one quiet continue; + * a new turn (markBusy) re-arms it. Without the flag every repeat idle + * would nag a session that is legitimately quiet. */ + emptyIdleNudged: boolean + /** The session's todo list, read from the todo tool's storage key. */ + todos: Todo[] + todosFetchedAt: number + /** Consecutive `task_complete` acks with no new user message or other tool + * work between them. Guards the ack self-loop, where the ack tool-result is + * fed back and the model re-emits the tool instead of ending its turn. Persists + * across busy/idle cycles by design. */ + taskCompleteSignals: number + /** Failed tool calls by name, for the unknown-tool suggestion. Counted, not + * latched: one bad call is a typo, two is a model that will not self-correct. */ + /** When the orphan watch was armed: the moment the last busy subagent of this + * session went idle. A parent still busy after that is the case this exists for. + * null means not armed. */ + orphanWatchStartAt: number | null + /** When the current orphan episode was ARMED, kept distinct from + * orphanWatchStartAt (which the clock advances on every deferral). The user + * guard compares against this so a deferral cannot "age out" the very + * message it was deferring for: ses_ed726872 escalated 187ms after the + * user spoke because nothing here compared arm-time to user-message-time. */ + orphanWatchArmedAt: number | null + /** Last time the subagents of this session were polled, so the watchdog does not + * ask the server on every tick. */ + lastSubagentCheckAt: number + /** Whether this orphan episode has already tried waking its subagent. Without a + * budget the same prompt goes out on every watchdog tick — v1's structure + * assumed the subagent goes busy again, and a dead one never does. */ + orphanRecoveryTried: boolean + /** The child this orphan episode already escalated for. ses_ed726872 + * (2026-10-11): a subagent that ended 2.5h earlier kept returning a + * "crashed" verdict, so the parent was aborted + re-continued 8 times across + * 3 hours — `orphanRecoveryTried` only bounds the WAKE attempt, and once + * that fails every later crashed verdict escalates again. Recording the + * child means one dead subagent can cost the parent exactly one abort; + * a genuinely NEW stuck child still gets its own. */ + orphanEscalatedChildSid?: string + /** The pending deferred pattern pass, if any. Held so a new idle can replace + * it rather than stack a second judgement on the same turn, and so a new turn + * can cancel it. */ + toolTextTimer: ReturnType | null + /** The pending deferred stall-recovery inject, if any. Held so dispose + * can cancel it: without this, a reloaded-away instance fires its + * backoff timer after disposal and injects from a dead sessions map + * (observed 2026-10-06: twin identical prods 5ms apart). */ + recoverTimer: ReturnType | null + /** Tool calls started but not finished, tracked from the tool lifecycle events. + * The orphan watch must never abort a session that is legitimately working. */ + pendingTools: number + /** Snapshot of `pendingTools` taken at the `session.idle` transition, before + * `markIdle()` zeroes the live counter. Idle does not mean nothing is in + * flight — the event fires between steps while tools run — so any idle-path + * check that reads the live counter always sees zero. Same class of bug the + * `busyBefore` sample above guards against. */ + toolsInFlightAtIdle: number + /** In-flight tool calls by call id, with start times. The busy-stall guard + * defers recovery while young tools run (and never interrupts exempt names + * like `compress`); unlike the bare `pendingTools` counter this carries the + * ages the decision needs. Entries die with the watch; results remove them. */ + inflightTools: Map + /** Rate-limit ladder state. `rateLimitedAt` is the last quota decision + * point, `rateLimitAttempts` counts served attempts in this episode, and + * `awaitingQuotaRetry` arms the watchdog-tick gate. A fresh turn resets + * all three: new activity means quota is available again. */ + rateLimitedAt: number + rateLimitAttempts: number + awaitingQuotaRetry: boolean + unknownToolErrors: Map + /** Set once a suggestion has been injected, so it is sent at most once per + * user message rather than on every idle. */ + unknownToolSuggestionSent: boolean + /** Tool parts already examined. The history is re-walked on every idle, so + * without this the same error would be counted again each time. */ + checkedToolPartIDs: Set + /** How many times a `task_complete` call was overridden because todos were + * still open. Bounded by `maxRetries` — past that the call is honoured, because + * an unbounded block is its own kind of loop. */ + taskCompleteOverrides: number + /** Id of the newest inbound user message already acted on, so a replayed or + * re-delivered message does not re-arm the done-claim budgets. Identity, not + * clock time — see noteInboundUserMessage. */ + lastUserMessageID?: string + /** Timestamp fallback for the same check, for messages carrying no id. */ + lastUserMessageSeenAt: number + /** Tokens currently in the context window, from `session.usage.updated`. */ + lastTokenTotal: number + /** Saturation intervention is one-shot per turn, like v1's `contextWrapupAttempts`. */ + contextWrapupAttempts: number + /** Set when a failure-triggered recovery is pending, so the delayed prompt isn't cancelled by the failure's own idle transition. */ + pendingRecoveryArmed: boolean + + // Guards + permissionPending: boolean + /** Timestamp of the latest `permission.asked` (stale-guard for the flag). */ + permissionPendingAt: number | null + lastWasTaskTool: boolean + /** Cached verdict from `isSubAgentSession()`: true when the server reports a + * `parentID` for this session, i.e. it is a child and must never be injected + * into or interrupted. `undefined` = not yet resolved. */ + isSubAgent?: boolean + idleSince: number | null + /** Last time the silent-reasoning sweep examined this session. Bounds the + * history read to ~one per stall window per idle session. */ + lastSilentReasoningCheck: number + /** Last time the watchdog's idle branch re-ran the stuck-tool pass. The + * event-driven pass only sees a call that is ALREADY past the age gate when + * `session.idle` fires; a call that was younger then ages into stuck with no + * further idle event, so only the watchdog can still catch it. Throttled to + * the stuck-tool age gate so a parked session costs ~one history read per + * gate, not one per tick. Reset on a fresh turn, like lastSilentReasoningCheck. */ + lastStuckToolCheck: number + /** Set while the session is mid native compaction — recovery must never interrupt it. */ + compacting: boolean + /** Timestamp of the latest `session.compaction.started` (stale-guard for the flag). */ + compactionStartedAt: number | null + + // Tool-loop tracking + recentToolCalls: ToolCallRecord[] + + // Agent/model from last step (informational logging only; sessions are stateful in v2) + agent?: string + model?: string + /** `Model.Ref` from the last step, for the usable-context lookup. */ + modelRef?: { providerID: string; modelID: string } +} + +export interface AutoResumeOptions { + chunkTimeoutMs?: number + gracePeriodMs?: number + stuckToolAgeMs?: number + /** Tool names the busy-stall path never interrupts (case-insensitive). + * Default `["compress"]`: every tool gets the young-tool deferral, and + * `compress` is additionally never interrupted while fresh; + * `compress` is exempt because interrupting it mid-flight risks leaving + * half-compacted state, the same reason compaction is never interrupted. */ + neverInterruptTools?: string[] + checkIntervalMs?: number + maxRetries?: number + baseBackoffMs?: number + maxBackoffMs?: number + loopMaxContinues?: number + loopWindowMs?: number + /** Lifetime stall continues per genuine user episode. The per-stall + * `maxRetries` ladder restarts on model text, so a model that keeps + * replying "done" reads 1/3 forever — this caps total fires until a + * genuine user message clears it. Default 9. */ + lifetimeMaxContinues?: number + /** Minimum gap between recovery injections for one session. */ + injectIntervalMs?: number + continuePrompt?: string + /** + * When true, a stall continue is sent via `ctx.session.prompt()` — a real + * user message visible in session history (cattleprod-style) — instead of + * the hidden `ctx.session.synthetic({resume:true})`. Default true: the + * intervention — and any loop — is self-evident in the transcript, which + * is the point. Set false for the old hidden channel. Explicit option + * wins; else `AUTO_RESUME_VISIBLE_CONTINUE` (`1`/`true`/`yes` = visible, + * anything else set = hidden) wins over the default. (Options set via + * the config `plugin` entry are dropped by this box's parser and the + * top-level `env` block never reaches the plugin process, so code + * default is the only live knob.) + */ + visibleContinue?: boolean + /** + * When false, feature-correction nudges (unknown-tool suggestion, stuck-tool, + * targeted pattern nudges) go through the visible prompt channel instead of + * synthetic. Default true: the user cannot see the question, so the model + * should not produce visible traffic for it — and every synthetic correction + * already carries its own do-not-acknowledge order. Stall recovery always + * stays visible so an intervention loop is self-evident. + */ + quietCorrections?: boolean + /** + * When true (default), a stall continue names the stall reason, attempt + * count, and remaining todos instead of sending bare "continue". A custom + * `continuePrompt` always wins verbatim. Same channel and guards either way. + */ + richContinuePrompt?: boolean + toolTextRecoveryPrompt?: string + doneWithoutWorkPrompt?: string + actionIntentPrompt?: string + debug?: boolean + activeUserWindowMs?: number + + // ---- Ported from v1 (Mte90/opencode-auto-resume#33) ---- + /** v1 name for `maxRetries`. Still accepted so v1 configs port unchanged. */ + maxRecoveryRetries?: number + /** How a stall is recovered: "continue" (default), "abort", or "off". */ + busyStallStrategy?: "continue" | "abort" | "off" + /** Fraction of the usable context window that counts as saturated. */ + contextSaturationThreshold?: number + /** Trigger native compaction on saturation for subagent sessions. */ + subagentNativeCompactionEnabled?: boolean + /** Resume on action-intent detection (default true). */ + resumeOnActionIntent?: boolean + /** Delay before the first session discovery sweep. */ + discoveryDelayMs?: number + /** Quiet period after startup before stall recovery arms. */ + warmupMs?: number + /** Minimum gap between recorded activity timestamps. */ + minActivityGapMs?: number + /** Grace before a raw-tool-call-as-text check fires. */ + toolTextCheckDelayMs?: number + /** Wait for a subagent to report before treating it as stalled. */ + subagentWaitMs?: number + /** Successor setups inherit the watch map instead of starting blank. + * Off by default (test files share one process); prod opts in so + * session attach/restore stops restoring amnesia. */ + shareWatchState?: boolean + + /** + * Which process-wide registry slot this instance owns. + * + * Unset (the production case) means the pid-wide slot, where a fresh setup + * disposes its predecessor — that is what stops a reload storm stacking six + * live instances. Set it, and this instance neither disposes nor is + * disposed by instances using a different scope. + * + * Required for test harnesses: `bun test` runs every file in one process, so + * without a per-file scope each file's setup kills the file before it. + * Test-only; production has one instance per process. + */ + registryScope?: string + /** Outer bound: quiet past this with no error evidence reads as dead. */ + subagentDeadMs?: number + /** Token floor for the silent-dead-stream heuristic. */ + silentDeadStreamMinTokens?: number + /** Error names that mark a streaming failure. */ + streamingFailureErrorNames?: string[] + /** Regex sources that mark a streaming failure message. */ + streamingFailureMessagePatterns?: string[] + /** Regex sources that mark a "task is done" claim. */ + doneClaimPatterns?: string[] + /** Regex sources that mark the model ready to continue. */ + readyToContinuePatterns?: string[] + /** Prompt sent when a done-claim arrives with no work to show. */ + doneWithoutDetailsPrompt?: string + /** Prompt sent when a tool call was emitted inside reasoning. */ + thinkingToolRecoveryPrompt?: string + /** + * Append this build's log here instead of the default path + * (`~/.local/state/opencode-v2/auto-resume.log`). + * + * v2 removed v1's `app.log` endpoint, so this is the only retrievable + * record of what the plugin did. The `AUTO_RESUME_LOG_FILE` environment + * variable sets the same thing and wins over neither. + */ + logFile?: string + /** + * Per-attempt cooldowns (ms) before a rate-limited session may be retried. + * Gates, not timers: evaluated on each failure and each watchdog tick, so + * nothing is lost to a reload. Past the ladder the session stays silent + * until a genuine user turn. Default spans ~12h for overnight coverage. + */ + rateLimitCooldownsMs?: number[] + /** Bound (ms) on any single recovery send; see DEFAULT_SEND_TIMEOUT_MS. */ + sendTimeoutMs?: number + /** + * Inert delivery probe. Never affects behaviour; echoed in the `ready` + * line so a config-options change can be verified without touching a + * live knob. Bump the value to test delivery. + */ + configProbe?: string | number +} + +// --------------------------------------------------------------------------- +// Logging +// --------------------------------------------------------------------------- + +/** + * Where this build's log lines go, and why not somewhere official. + * + * v1 wrote through `ctx.client.app.log({ body: { service, level, message } })`, + * a real endpoint that landed in the opencode log file. v2 removed it: + * `ctx.app` is `{ name, version, channel }` (`packages/plugin/src/app.ts`), + * there is no `app` group in `packages/protocol/src/groups/`, and a hosted + * plugin's `console.log` is not captured by the OpenChamber process. Without + * this the plugin is completely silent — you cannot tell a working watchdog + * from a dead one, which is exactly the failure that made a stall + * indistinguishable from "nothing happened". + * + * Override with AUTO_RESUME_LOG_FILE. Size-capped so an unattended run cannot + * fill the disk. + */ +const DEFAULT_LOG_FILE = join(homedir(), ".local", "state", "opencode-v2", "auto-resume.log") +const LOG_FILE_MAX_BYTES = 2 * 1024 * 1024 + +/** + * Append one line to `target`, best effort. + * + * Every failure here is swallowed on purpose: logging must never be the reason + * the watchdog stops running. The size check races harmlessly between + * processes — losing a line beats throwing. + */ +function appendLogFile(target: string, level: string, line: string): void { + try { + mkdirSync(dirname(target), { recursive: true }) + try { + if (existsSync(target) && statSync(target).size > LOG_FILE_MAX_BYTES) { + writeFileSync(target, "") + } + } catch { + // Size check is an optimisation, not a requirement. + } + appendFileSync(target, `${new Date().toISOString()} ${level.toUpperCase().padEnd(5)} ${line}\n`) + } catch { + // Unwritable log dir, read-only fs, quota — none of these should + // propagate into the recovery path. + } +} + +/** + * The command magic-context registers to reclaim context on a saturated parent. + * Sent through `session.command` rather than as prompt text, because prompt text is + * not expanded into a command. + */ +const CTX_WRAPUP_TRIGGER = "ctx-wrapup" + +// --------------------------------------------------------------------------- +// Constants & defaults +// --------------------------------------------------------------------------- + +// 45s -> 180s (2026-09-29): silence is the ONLY stall signal here, and 45s + +// 3s grace misfires on a single shared GPU where one subagent turn legitimately +// emits no events for over a minute. Widening the window is a mitigation, not +// the principled fix — that is positive hang detection (a tool call stuck in +// running/pending is a real hang; model generation is not), which changes +// recovery semantics and is still an open decision. +const DEFAULT_CHUNK_TIMEOUT_MS = 180_000 +const DEFAULT_CHECK_INTERVAL_MS = 5_000 +const DEFAULT_GRACE_PERIOD_MS = 3_000 +/** How old an unanswered tool part must be before the stuck-tool pass acts. + * Deliberately far shorter than the stall window: an idle session with an + * unfinished call is not 'maybe still working' — the turn already ended, so + * the only race is late-flushing results (seconds-scale, per the settle-delay + * design). 60s gives 20x margin over that while cutting the ~3min stall-window + * wait the pass used before (ses_edf403f5: parked 94 min on a wedged compress). */ +const DEFAULT_STUCK_TOOL_AGE_MS = 60_000 +/** + * Stall-path patience for tools that are legitimately long. A 60s gate fits + * interactive tools (read/write/edit: seconds-scale work), but a `shell` + * call is routinely minutes long — a full test suite runs 400s+ while + * emitting no events. Judging it by the 60s gate declared live work wedged: + * 2026-10-11, three stall injections in nineteen minutes onto a session + * running `bun test`, one telling the user to abandon the run. Shells get + * 10 minutes on the STALL path only: past that a shell is wedged for real + * and the stall answers normally. The idle stuck pass (turn already ended — + * an unfinished call there is abandoned, not running) keeps the flat 60s + * gate for every tool, so this changes WHEN a busy stall fires, never + * whether an abandoned call gets nudged. + */ +const LONG_RUNNING_TOOL_STALE_TTL_MS = 10 * 60_000 +const LONG_RUNNING_TOOLS = ["shell"] +const DEFAULT_MAX_RETRIES = 3 +/** Consecutive failure-driven recoveries tolerated before the failure-loop + * latch stands the session down. Counts recoveries with no model output + * between them, not attempts: a dead backend fails every turn identically. */ +const FAILURE_LOOP_MAX = 3 +/** How long after a `session.retry.scheduled` a stall is still read as the + * provider's own retry rather than a fresh model stall. Wide enough to cover a + * backoff-plus-retry cycle: the prod still fires inside it, only the ladder + * escalation is held. */ +const PROVIDER_RETRY_GRACE_MS = 120_000 +const DEFAULT_BASE_BACKOFF_MS = 1_000 +const DEFAULT_MAX_BACKOFF_MS = 8_000 +const DEFAULT_LOOP_MAX_CONTINUES = 3 +const DEFAULT_LOOP_WINDOW_MS = 10 * 60_000 +const DEFAULT_LIFETIME_MAX_CONTINUES = 9 +const DEFAULT_DEBUG = false +// Mirrors v1 (Mte90 e1b8374): an inbound user message this recent means the +// user is engaged (likely composing), so idle nudges stand down. Was 15min here. +const DEFAULT_ACTIVE_USER_WINDOW_MS = 5 * 60_000 + +// ---- Defaults ported from v1 (Mte90/opencode-auto-resume#33) ---- +// v1 keeps SESSION_DISCOVERY_INTERVAL_MS as an internal constant rather than an +// option, so it is not configurable here either. 60s matches v1. +const DEFAULT_DISCOVERY_DELAY_MS = 5_000 +const DEFAULT_SESSION_DISCOVERY_INTERVAL_MS = 60_000 +const DEFAULT_WARMUP_MS = 15_000 +// Off by default: bun runs every test file in one process, so unconditional +// sharing leaks watch state across tests (165 failures on record). Prod opts +// in to sharing — sequential setups in one server process must inherit stall +// memory, or every session attach restores amnesia (2026-10-09). +const DEFAULT_SHARE_WATCH_STATE = false +const DEFAULT_MIN_ACTIVITY_GAP_MS = 1_000 +const DEFAULT_TOOL_TEXT_CHECK_DELAY_MS = 3_000 +const DEFAULT_SUBAGENT_WAIT_MS = 15_000 +/** How long a busy subagent may go without producing anything before it counts + * as stuck. v1 used the same number, and a tool call still outstanding triples + * it, because a long tool is not a hung model. */ +const SUBAGENT_STUCK_MS = 60_000 +/** Outer bound: quiet past this with no error evidence reads as dead, not + * merely stuck. Between the stuck window and this bound a quiet child is + * waited on — a hung model must not sit forever, but a long-thinking one + * (coder blocks run many minutes on one shared GPU) must not read as + * crashed at 60s either. ses_ef1c822c 2026-10-06. */ +const DEFAULT_SUBAGENT_DEAD_MS = 30 * 60_000 +const SUBAGENT_RECOVERY_PROMPT = + "It looks like you may have stalled or timed out. Please retry the last operation or continue with the task." +const DEFAULT_SILENT_DEAD_STREAM_MIN_TOKENS = 200 +const DEFAULT_CONTEXT_SATURATION_THRESHOLD = 0.85 +/** How long a fetched todo list is trusted before being re-read. */ +/** Cap on the remembered tool-part ids. Bounds a set that is otherwise fed one + * entry per tool part, on every idle, for the life of the session. */ +const CHECKED_TOOL_PART_IDS_MAX = 500 + +/** Where the rate-limit ladder is mirrored so it survives a process restart + * (R1). A log file was the previous channel and is a bad one: a 2MB rotation + * cap plus a fixed 3000-line tail can drop a live arm. */ +const QUOTA_STATE_PREFIX = "resume-quota/" +/** Generous ceiling on a stored arm. The ladder horizon is minutes; this only + * exists so an abandoned record cannot haunt a session days later. */ +const QUOTA_STATE_MAX_AGE_MS = 48 * 3600_000 + +const TODO_CACHE_TTL_MS = 3_000 +const DEFAULT_MAX_RECOVERY_RETRIES = 2 +// Referenced from FEATURE_GATED_OPTIONS docs; kept so the intended v1 default is +// recorded next to the gate that explains why it is not applied yet. + +const DEFAULT_STREAMING_FAILURE_ERROR_NAMES = [ + "ProviderError", + "APIError", + "StreamError", + "ConnectionError", + "TimeoutError", +] + +const DEFAULT_STREAMING_FAILURE_MESSAGE_PATTERNS = [ + "streaming response failed", + "stream.*fail", + "connection.*reset", + "connection.*closed", + "aborted due to timeout", +] + +/** + * Canonical done-claim sources — single source of truth for the "task claims + * it is done" vocabulary. Two consumers build from this one list, which is + * why it exists: the string fallback below (compiled with flags "im" for the + * user-overridable `doneClaimPatterns` option) and the RegExp hard default + * (DONE_CLAIM_PATTERNS) drifted apart — the fallback lost its ^$ anchors and + * matched strictly more than the tested default. Both are now derived here, + * so the live path and the tested path can never disagree again. + */ +const DONE_CLAIM_PATTERN_SOURCES = [ + "^task\\s+done[.!]*$", + "^done[.!]*$", + "^all\\s+done[.!]*$", + "^finished[.!]*$", + "^complete[.!]*$", + "^task\\s+complete[.!]*$", + "^task\\s+completed[.!]*$", + "^all\\s+tasks?\\s+complete[.!]*$", + "^all\\s+tasks?\\s+completed[.!]*$", + "^(?:i['’]?m\\s+)?done\\s+with\\s+task", + "\\bdone\\s+with\\s+(?:the\\s+)?(?:task|work|implementation)", + "\\bfinished\\s+(?:the\\s+)?(?:task|work|implementation)", + // ses_f2e3370c (2026-10-14): "All work complete." — a noun between + // all/everything and the verdict broke the match, and the stall ladder + // fired on a finished session. One optional word covers the shapes real + // deliveries take ("all work complete", "everything is finished"); + // the verdict word is still required, so mid-work lines like "all tests + // pass so far, running integration next" still match nothing. + "\\b(?:all|everything)\\s+(?:[\\w-]+\\s+)?(?:is\\s+)?(?:complete|done|finished)", + "\\bnothing\\s+(?:else\\s+)?(?:left|remaining|to do)", +] + +const DEFAULT_DONE_CLAIM_PATTERNS = [...DONE_CLAIM_PATTERN_SOURCES] + +const DEFAULT_READY_TO_CONTINUE_PATTERNS = [ + "ready to continue with task", + "continuing with task", + "continue with task", + "proceeding with task", + "ready to proceed with task", + "will continue with task", + "moving on to task", +] + +const THINKING_TOOL_RECOVERY_PROMPT = + "I noticed you have a tool call generated in your thinking/reasoning. " + + "Please execute it using the proper tool calling mechanism instead of keeping it in reasoning." + +const DONE_WITHOUT_DETAILS_PROMPT = + "Your last response claimed the task is complete but contained no work description. This is not acceptable. " + + "You MUST respond now with a full, detailed report of everything you did: " + + "for each file you modified, state the full path and the exact changes; " + + "list every command you ran to verify and its result; state the final outcome. " + + "Do NOT reply with 'done', 'task completed', or any short acknowledgment — " + + "your ONLY acceptable response right now is this detailed report. Write it now." + +/** + * Upper bound on how long a `shell.created` → `shell.exited` pair is trusted + * to mean "this session is busy". A shell that never reports an exit (session + * torn down, event dropped) would otherwise suppress recovery forever, so + * entries are pruned past this. Set well above any realistic long build. + */ +const SHELL_OPEN_MAX_MS = 30 * 60_000 + + +/** OOC (out-of-context) errors that `continue` can never clear — recovery is locked out on these. */ +const OOC_ERROR_RE = /exceeds the available context size|context size \(\d+\)|too large to compact|too many tokens|prompt is too long/i +/** Exact `error.type` values a `continue` can never clear. These ARE opencode's + * own taxonomy (packages/llm/src/schema/errors.ts), read off the running binary's + * type->tag switch — not guesses. Each maps to a reason class whose + * `get retryable()` returns false there: Authentication:69, NoRoute:53, + * UnknownProvider:155, ContentPolicy:104. UnsupportedOperation and Unknown are NOT + * here — both proved broader than their names (transient aborts arrive as + * provider.unknown; ses_ee14d727 locked out 5+ min). They live in + * INDETERMINATE_ERROR_TYPES below: one retry, then latch. + * + * Exact strings, never a `provider.*` prefix: the same file marks RateLimit (:82) + * and ProviderInternal (:117) retryable, and Transport (:129) plus + * InvalidProviderOutput (:143) recover fine minutes later. A prefix match would + * gate all four. + * + * Not here on purpose: provider.rate-limit and provider.quota keep the cooldown + * ladder (15m -> 2h, ~12h total) — a 5h rolling quota needs more patience than a + * non-retryable verdict implies, and the ladder only retries behind a long gate. + */ +const PERMANENT_ERROR_TYPES = new Set([ + "provider.auth", + "provider.no-route", + "provider.content-filter", +]) +/** Types whose permanence is NOT established upstream, so they are not hard + * gates. `provider.unsupported-operation` is present in the shipped binary but + * absent from the dev union of packages/llm/src/schema/errors.ts, so no + * `get retryable()` was ever observable for it — unlike every other entry above, + * which can point at a line. Gating it on the first occurrence bets on a + * semantics nobody has read; never gating it would retry a wall forever. So: ONE + * recovery attempt, then latch. A genuinely transient condition survives the + * single retry, a permanent one costs one wasted turn instead of three, and the + * first occurrence is logged so a real `retryable` value can promote it to a hard + * gate. */ +const INDETERMINATE_ERROR_TYPES = new Set(["provider.unsupported-operation", "provider.unknown"]) +/** Occurrences of an indeterminate failure tolerated before latching. */ +const INDETERMINATE_STRIKES = 2 +/** HTTP statuses that are permanent whatever type is reported. 401/403 only: + * opencode's own text for them is "your authentication token may be missing or + * expired — try running `opencode auth login`" (provider/error.ts:61,64). + * Deliberately NOT 404 — provider/error.ts:26 says "openai sometimes returns 404 + * for models that are actually available", and opencode treats an openai 404 as + * retryable. Nor 429/5xx, which are the transient rungs. */ +const PERMANENT_STATUS_CODES = new Set([401, 403]) +/** Permanent messages that are about the session's own HISTORY, so every retry + * re-sends the same broken payload. Measured on the live log: "Referenced + * reasoning item" recurred in one session ~17.6 minutes apart over ~4 episodes — + * the continues did not clear it. Both arrive as InvalidRequest upstream, which + * that source marks non-retryable. */ +const PERMANENT_HISTORY_RE = + /referenced (reasoning )?item [^\n]{0,80}not found|no tool output found for (function )?call/i +/** Model-shape permanent failures, message-keyed, split in two because a 404 + * splits them. opencode funnels a gone model through InvalidRequest -> + * `provider.invalid-request`, whose type is far too broad to gate on — the same + * type also carries provider state errors a retry survives. Neither literal + * string is in the opencode binary (both are provider passthroughs), which is + * exactly why these stay message-keyed. */ +/** The model itself is gone: permanent whatever the status, including 404. */ +const PERMANENT_GONE_RE = + /model\b[^.]{0,80}\b(deprecat\w*|discontinued|retired|removed|no longer (available|supported|offered))\b|\bno llm route for\b/i +/** "This model is not in the catalog" — permanent per source, EXCEPT behind a + * 404: provider/error.ts:26 says "openai sometimes returns 404 for models that + * are actually available" and opencode treats an openai 404 as retryable, so a + * 404 keeps recovering. Without the status (opencode's own ProviderModelNotFound + * catalog miss) it gates. */ +const PERMANENT_UNKNOWN_MODEL_RE = + /\b(model[_ ]?not[_ ]?found|model_not_found|unknown model|no such model|invalid model|unsupported model)\b/i +/** Quota/rate-limit errors (observed: `{type:"provider.quota", message:"Rate + * limit exceeded. Please try again later."}`). An immediate continue retries + * straight into the ban, so these stand down on a gate ladder instead of the + * normal backoff. Gates, not timers: armed timeouts do not survive a reload. + * Throttle phrasing also matches (issue #46: "sent too many tokens ... in a + * short period, please wait before retrying") so it never reaches the OOC + * latch — maybeLockOoc defers to isRateLimitError first. */ +const RATE_LIMIT_RE = /rate limit exceeded|too many requests|\b429\b|quota exceeded|provider\.quota|throttl|slow down|please wait|try again later|too many tokens.{0,120}(short period|wait|retr|throttl|slow|per (second|minute|hour|day)|rate|limit|quota)/i +/** Per-attempt cooldowns before a rate-limited session may be retried: 15m, + * 30m, 1h, then 2h out to ~12h of unattended coverage. Past the ladder the + * session stays silent until a genuine user turn. */ +const DEFAULT_RATE_LIMIT_COOLDOWNS_MS = [15, 30, 60, 120, 120, 120, 120, 120].map((m) => m * 60_000) +/** Bound on any single recovery send (prompt/synthetic). A transport + * await that never settles stacks cascades that flush together as twins + * (ses_ee7dade1, 2026-10-07: two texts 5ms apart, one wedged minutes). + * Expiry flows into the normal throw path (log verify + fallback). */ +const DEFAULT_SEND_TIMEOUT_MS = 30_000 +/** Above this, a completed send logs a latency warning for forensics. */ +const SEND_SLOW_MS = 10_000 + + +/** + * Window after `session.interrupt()` during which any abort-shaped event is + * attributed to us rather than to the user. The runtime delivers + * `session.execution.interrupted` asynchronously, so a boolean `aborting` flag + * cleared on a fixed timer is not a reliable "this abort was mine" marker. + */ +const SELF_ABORT_TTL_MS = 30_000 +/** Interrupt-shaped failure signatures. v2 reports an interrupt as + * `{type:"aborted", message:"Step interrupted"}`, which the older + * `("interrupted by user" | type contains "cancel")` filter never matched. */ +const ABORT_ERROR_TYPE_RE = /^(abort|aborted|cancel|cancelled|canceled|interrupted)$/i +const ABORT_ERROR_MSG_RE = /step interrupted|interrupted by user|request cancelled|request canceled|execution interrupted/i +/** Hard cap on plugin-initiated interrupts. An interrupt persists an errored + * assistant message that the UI surfaces as a send failure, so it is a last + * resort, not a routine recovery step. */ +const MAX_INTERRUPTS_PER_WINDOW = 2 +const INTERRUPT_WINDOW_MS = 10 * 60_000 +/** + * Minimum gap between two recovery injections for the same session. + * Synthetic turns are not free: sending one to a session that is already + * running supersedes its in-flight step, which the runtime reports as + * "Step interrupted". Debouncing collapses the several independent recovery + * paths (stall watchdog, intent nudge, tool-loop nudge) into at most one + * injection per interval. + */ +const DEFAULT_INJECT_INTERVAL_MS = 15_000 +/** A user message landing within this of our last inject is treated as our + * own prompt for budget re-arm purposes (see noteInboundUserMessage). Same + * box, same clock; our prompt lands in about a second. A genuine user message + * coincidentally inside the window costs one request's worth of stale + * budgets — no loop — versus the recount loop this prevents. */ +const OWN_PROMPT_TIME_MS = 30_000 + +const MAX_IDLE_SESSIONS = 50 +const IDLE_CLEANUP_MS = 10 * 60_000 +/** A compaction silent for this long without an ended/failed event is wedged — stale-clear the flag. */ +const COMPACTION_STALE_TTL_MS = 30 * 60_000 +/** A `permission.asked` unanswered this long is treated as abandoned — stale-clear the flag. */ +const PERMISSION_STALE_TTL_MS = 30 * 60_000 +/** An exempt tool (see `neverInterruptTools`) with no result after this long is + * wedged, not working — the exemption expires and the stall answers normally. */ +const NEVER_INTERRUPT_STALE_TTL_MS = 30 * 60_000 +const TEXT_BUFFER_TRIM_LEN = 20_000 + +/** + * Tool-part states as reported by the v2 message projection. + * + * v2 never writes "pending": an in-flight tool part reads "running", and it + * settles to "completed" or "error". "pending" was the v1 spelling, so a guard + * that tested for it literally could never fire on v2. + */ +const TOOL_STATE_RUNNING = "running" +/** Error text that must never be retried: policy guardrails, not failures. The + * safety net's own verdict ("Do not retry this operation or attempt any + * workaround") is authoritative — reissuing the call teaches circumvention + * (ses_edf403f5 15:59: shell blocked on secret.home, turn ended on it). */ +const NEVER_RETRY_RE = /blocked by cc safety net|do not retry|access to a sensitive path/i +/** Readable text out of a tool part error/result payload, which arrives as a + * string or as `{type, message}` depending on the producer. "" when absent. */ +function toolErrorText(state: { error?: unknown; output?: unknown } | undefined): string { + const e = state?.error ?? state?.output + if (typeof e === "string") return e + if (e && typeof e === "object") { + const t = e as { type?: unknown; message?: unknown } + const parts = [t.type, t.message].filter((x): x is string => typeof x === "string" && x.length > 0) + if (parts.length > 0) return parts.join(" ") + } + return "" +} +/** Kept for v1-shaped payloads; harmless on v2. */ +const TOOL_STATE_PENDING = "pending" +const TOOL_STATE_COMPLETED = "completed" + +/** + * Tools whose only real completion is the user answering. A `question` that + * comes back "error" was interrupted, not answered, so it must still hold the + * turn open. + */ +const AWAITING_USER_TOOLS = new Set(["question", "permission", "ask", "confirm"]) + +/** The literal bare continue. Kept for `richContinuePrompt: false` (an explicit + * opt-out) and for v1 parity. Not the default — see SAFE_CONTINUE_PROMPT. */ +const CONTINUE_PROMPT = "continue" + +/** + * The default for every synthetic continue that has no richer prompt of its own. + * + * A bare "continue" is an unbounded instruction: it tells the model to keep + * going without saying what "going" means. Every liveness signal in this plugin + * is fallible — on 2026-10-11 a subagent that had been dead for 2.5 hours was + * still tracked, and the orphan watch aborted and re-continued the parent 8 times + * across 3 hours, twice on top of a live user message. When the signal is wrong + * the message is the last line of defence, and "continue" alone has none: a + * model that decides the next step on its own has nothing telling it to stop at + * the edge of the work it was given. + * + * So the guardrails ride along with the instruction, in the same words the stall + * path already uses (recover()'s rich prompt), which keeps every continue this + * plugin emits consistent: + * + * Only continue work already in progress. Do not start anything new, and do + * not take destructive or irreversible actions without explicit user approval. + * + * This bounds the blast radius of a false positive without changing WHEN the + * plugin intervenes — the detection logic is untouched. A configured + * `continuePrompt` still wins verbatim: an operator who sets it has said what + * they want. + */ +const SAFE_CONTINUE_PROMPT = + "continue\nOnly continue work already in progress. Do not start anything new, " + + "and do not take destructive or irreversible actions without explicit user approval." + +/** + * The empty-idle nudge has nothing in progress to continue — that is precisely + * what fired it — so the same sentence would be incoherent ("continue work + * already in progress" with no such work). Same guardrail, phrased for the case: + * if work WAS in flight, carry it; otherwise go back to the user rather than + * inventing a task. + */ +const SAFE_EMPTY_IDLE_PROMPT = + "continue\nIf work was already in progress, continue it and finish nothing else. " + + "Otherwise ask the user what to do next rather than starting something new. " + + "Do not take destructive or irreversible actions without explicit user approval." + +/** + * The `task_complete` acknowledgement texts. + * + * The ack is fed straight back into the model's turn, so a stuck model re-emits + * `task_complete` instead of ending with text. That is not hypothetical: v1 + * recorded 27 consecutive acked calls with zero new user input. The first ack + * therefore carries an explicit stop instruction, the second warns, and the third + * throws so the turn is forced to end. + */ +const TASK_COMPLETE_ACK = + "Task completion acknowledged. No further continuation will be sent. " + + "End your turn now with a brief text reply — do not call task_complete or any other tool again " + + "unless the user sends a new message." + +const TASK_COMPLETE_REPEAT_WARNING = + "Task completion acknowledged already on your previous call — completion is recorded. " + + "End your turn now with a brief text reply. Do not call task_complete again; " + + "further repeat calls are rejected as errors." + +const TASK_COMPLETE_REPEAT_ERROR = + "task_complete already acknowledged twice with no new user message or tool work since — " + + "completion is recorded. End your turn with text and make no further tool calls." + +/** How many times a model must call a tool that does not exist before we name a + * replacement. One is a typo; two is a model that will not correct itself. */ +const UNKNOWN_TOOL_THRESHOLD = 2 +/** The tool list changes only when a plugin reloads, so a short cache is enough + * and it keeps the check off the registry on every idle. */ +const TOOL_IDS_CACHE_MS = 5 * 60_000 +/** Cap on the tool list quoted back to the model. A full dump of every tool on + * the box is itself a way to fill the context. */ +const UNKNOWN_TOOL_LIST_LIMIT = 20 + +/** v1's wording, kept verbatim — the model has already seen this description. */ +const TASK_COMPLETE_DESCRIPTION = + "Signal that all work is complete and stop automatic continuation prompts. Call this ONLY after finishing everything requested. Call exactly once per completed round of work — repeat calls with no new user message in between are rejected." + +const TOOL_TEXT_RECOVERY_PROMPT = + "Your last message contained a raw tool call printed as text instead of being executed. " + + "Please use the proper tool calling mechanism to execute it." + +const DONE_WITHOUT_WORK_PROMPT = + "I need you to verify more carefully that you have actually completed all the required tasks. " + + "Your response indicated you're done, but no work was detected. Please check your todo list " + + "and complete any remaining work." + +const TOOL_LOOP_RECOVERY_PROMPT = + "I notice you've been calling the same tool multiple times in a row without making progress. " + + "Please step back and reassess your approach. Consider: " + + "1) Are you stuck in a loop? 2) Do you need different information first? " + + "3) Should you try a different tool or break the task into smaller steps? " + + "Take a moment to think about what's blocking you and propose a different strategy." + +const TASK_TOOL_HINTS = ["task", "agent", "subagent", "dispatch"] + +// --------------------------------------------------------------------------- +// Pattern lists (ported verbatim from upstream where pure) +// --------------------------------------------------------------------------- + +const TOOL_TEXT_PATTERNS = [ + //i, + /<\/function>/i, + //i, + /<\/parameter>/i, + /]/i, + /<\/tool_call>/i, + /]*)?\s*(?:\/>|>)/i, + /{"type":\s*"function"/i, + /{"name":\s*"[a-zA-Z_]/i, + /\{\s*"type"\s*:?$/im, + /\{\s*"name"\s*:?$/im, +] + +const TRUNCATED_XML_PATTERNS = [ + { open: /]*>/i, close: /<\/function>/i }, + { open: /]*>/i, close: /<\/parameter>/i }, + { open: /]*>/i, close: /<\/tool_call>/i }, + { open: /\{\s*"type"\s*:/i, close: /}/ }, + { open: /\{\s*"name"\s*:/i, close: /}/ }, +] + +const READY_TO_CONTINUE_PATTERNS = [ + /ready to continue with task/i, + /continuing with task/i, + /continue with task/i, + /proceeding with task/i, + /ready to proceed with task/i, + /will continue with task/i, + /moving on to task/i, +] + +const DONE_CLAIM_PATTERNS = DONE_CLAIM_PATTERN_SOURCES.map((s) => new RegExp(s, "im")) + +// Error signatures that indicate a transient streaming/provider failure worth retrying +const STREAMING_FAILURE_MESSAGE_PATTERNS = [ + "stream.*fail", + "stream.*timeout", + "connection.*reset", + "connection.*closed", + "connection.*error", + "socket.*hang", + "econnreset", + "etimedout", + "rate.?limit", + "overloaded", + "server error", + "internal error", +] + +// --------------------------------------------------------------------------- +// Pure helpers +// --------------------------------------------------------------------------- + +function stripCodeBlocks(text: string): string { + return text.replace(/```[\s\S]*?```/g, "").replace(/`[^`\n]+`/g, "") +} + +/** + * Trailing environment boundary markers (`@12@`-style ids the harness + * appends to stored message text) are metadata, not model prose. Left in + * place they defeat every end-anchored completion test: ses_ecca0ad6 + * (2026-10-13) ended nine "TASK COMPLETE" deliveries with a marker and the + * stall ladder fired on every one despite the latch code being live and + * latching clean tails the same day. Strip trailing runs only — a mid-text + * mention is model prose and keeps its meaning. + */ +function stripBoundaryMarkers(text: string): string { + return text.replace(/(?:\s*@\d+@)+\s*$/, "") +} + +function containsToolCallAsText(text: string): boolean { + if (text.length <= 10) return false + const stripped = stripCodeBlocks(text) + if (TOOL_TEXT_PATTERNS.some((pat) => pat.test(stripped))) return true + for (const { open, close } of TRUNCATED_XML_PATTERNS) { + if (open.test(stripped) && !close.test(stripped)) return true + } + return false +} + +function containsReadyToContinuePattern(text: string, patterns: RegExp[] = READY_TO_CONTINUE_PATTERNS): boolean { + const lines = text.split("\n") + const lastLines = lines.slice(-3).join("\n") + return patterns.some((pat) => pat.test(lastLines)) +} + +function containsDoneClaimPattern(text: string, patterns: RegExp[] = DONE_CLAIM_PATTERNS): boolean { + const lines = text.split("\n") + const lastLines = lines.slice(-5).join("\n") + return patterns.some((pat) => pat.test(lastLines)) +} + +/** + * Explicit stop statements: the model stating it is stopping, distinct from + * claiming completion. Like celebrations these latch completion when no + * todos are open (ses_eec5ba74, 2026-10-07 — "No action needed … Stopping + * here" kept collecting nudges after the work was finished). Unlike the + * done-claim patterns these carry no completion assertion to challenge, + * so an explicit stop wins over the challenge below (ordered before it + * 2026-10-07: challenging a stopped true completion nags exactly the + * sessions this latch exists to quiet). + */ +const STOP_STATEMENT_PATTERNS: RegExp[] = [ + /no action needed/i, + /\bstop(ping|ped)?\s+(here|there|now)\b[.!]*$/i, + // ses_ed726872 (2026-10-11): told to stop, answered "Stopping as instructed + // — no further runs ... Nothing launched, nothing pending from me", and the + // stall ladder fired two bare `continue`s anyway (latch never set). The + // branch listed only `to do|left|remaining|needed`, and a model saying + // "nothing pending from me" matched NONE of them. Same class as the stuck + // -tool gap — a narrow vocabulary deciding liveness — so the branch is + // widened to the shapes these messages actually take, rather than adding a + // pattern per wording. + // + // Clause discipline (this is the part that matters): a stop is only a stop + // if it is the LAST clause. The tail is split on any clause terminator — + // sentence punctuation, semicolon, or em/en dash — and each pattern must + // match the FINAL clause (allowing only short closing filler like "from me" + // / "for now"). Without that, "Stopping here for a moment, then continuing" + // or "No further action is needed for the sync; next I will run the audit" + // both read as completed stops and wrongly latch. Semicolon and dash are + // the terminators real stop-acks lean on. + /\bnothing\b[^.;:!?—–\n]*\b(further|else|pending|left|remaining|outstanding|launched|queued|scheduled|to do)\b[^.;:!?—–\n]*[.!]*$/i, + /\bno (further|additional|more) (action|work|steps?|runs?|anything)\b[^.;:!?—–\n]*[.!]*$/i, + /\bnone (pending|left|remaining|outstanding)\b[^.;:!?—–\n]*[.!]*$/i, + /\b(stopping|halt|halted|standing down|pausing|paused)\b[^.;:!?—–\n]*[.!]*$/i, + /\b(that'?s|thats|this is) (all|it|everything)\b[^.;:!?—–\n]*[.!]*$/i, + /\b(parked|not started|unstarted)\b[^.;:!?—–\n]*[.!]*$/i, + // ses_edd1eb51 (2026-10-09): the model ended two deliveries with a bare + // trailing "TASK COMPLETE" and the stall ladder fired 186s later — the + // system's own completion vocabulary written as text latched nothing. + // End-anchored like the stop patterns so mid-text mentions don't fire. + // + // Case discipline (2026-10-15): every completion pattern in this file is + // case-insensitive EXCEPT the system's own literal — bare "TASK COMPLETE" + // must read byte-exact, per the operator contract in docs/AGENTS-snippet.md + // ("The line must read exactly `TASK COMPLETE`"). A lowercase "task + // complete" is model prose, not the signal, and does not latch. The past + // tense ("task completed", any case) stays insensitive — it was never + // the literal. + /\bTASK COMPLETE\s*[.!]*\s*$/, + /\btask completed\s*[.!]*\s*$/i, + // Mined from live session tails (2026-10-15 review of the last model + // message of all 16 live sessions + 6 archived transcripts): standby + // phrasing is completion, not a pause. ses_f2ea7b7a "All done. Standing + // by for next instructions." latched only on the "All done" half — a bare + // "Standing by …" with no done-claim beside it matched nothing. + /\bstanding by\b[^.;:!?—–\n]*[.!]*$/i, + // ses_f2ea7797 "Ready when you have the next task." — the model is out of + // work and waiting on the user, which is done, not ready-to-continue + // (that family auto-sends `continue`; this one latches quiet). Deliberately + // NOT in READY_TO_CONTINUE_PATTERNS. The tail class excludes "?" — a + // question ("Ready for the next task?") is still-working, never a stop. + /\bready (?:when you have the|for your|for the) (?:next\s+)?(?:task|instruction)s?\b[^.;:!?—–\n]*[.!]*$/i, + // Waiting-for-user-input stop states (2026-10-17): when the model is + // blocked on user input (VM details, IP, SSH confirmation, etc.) it is + // out of work, not ready-to-continue. None of the patterns above match + // "nothing in progress" / "waiting on you" / "blocked on you", so the + // stall ladder kept firing `continue` into a parked session. + // Broadened 2026-10-17 per completion-phrases research (see + // completion-phrases-research.md): Aider, OpenHands, Continue, Cline, + // SWE-agent completion/blocked vocabulary. + /\bnothing in progress\b/i, + // P0: "waiting for/on X" where X is any user-referent. + /\bwaiting\s+(?:on|for)\s+(?:you|the user|your|my|the next|further)\b[^.;:!?—–\n]*[.!]*$/i, + // P0: "awaiting X" — formal verb form. + /\bawaiting\s+(?:you|the user|your|further|the next)\b[^.;:!?—–\n]*[.!]*$/i, + // P1: "blocked" + need/awaiting/on, including comma-separated "blocked, need your input". + /\bblocked\b[^.;:!?—–\n]*(?:\bneed\b|\bawaiting\b|\bon\b)[^.;:!?—–\n]*[.!]*$/i, + // P1: "on hold" / "holding off" blocked states. + /\b(?:on hold|holding off)\s+(?:pending|awaiting|waiting for)\s+(?:your|the user|further|the next)\b[^.;:!?—–\n]*[.!]*$/i, + // P1: "can't proceed without your input". + /\b(?:can'?t|cannot|unable to)\s+proceed\s+without\s+(?:your|the user)'?\s*(?:input|guidance|decision|approval|confirmation|direction)s?\b[^.;:!?—–\n]*[.!]*$/i, + // P2: "I need your input to continue" / "need clarification before I can continue". + /\b(?:i\s+)?need\s+(?:your|the user)'?\s*(?:input|guidance|decision|approval|confirmation|direction|clarification)s?\b[^.;:!?—–\n]*(?:\bto\s+continue\b|\bbefore\b)?[^.;:!?—–\n]*[.!]*$/i, + // P2: Aider literal blocked phrases. + /\bstop\s+and\s+wait\s+for\s+your\s+approval\b[^.;:!?—–\n]*[.!]*$/i, + /\bask\s+me\s+to\s+add\s+them\s+to\s+the\s+chat\b[^.;:!?—–\n]*[.!]*$/i, + // P2: SWE-agent bare FINISHED token (case-sensitive like TASK COMPLETE). + /\bFINISHED\s*[.!]*\s*$/, + // P3: passive complete forms. + /\bi\s+have\s+completed\s+(?:the\s+)?(?:task|work|implementation)\b[^.;:!?—–\n]*[.!]*$/i, + /\bthe\s+(?:task|work)\s+is\s+(?:finished|complete|done)\b[^.;:!?—–\n]*[.!]*$/i, + // P3: standing-by variants. + /\bready\s+when\s+you'?re\s+ready\b[^.;:!?—–\n]*[.!]*$/i, + /\bready\s+and\s+waiting\b[^.;:!?—–\n]*[.!]*$/i, +] +function containsStopStatement(text: string): boolean { + const lines = stripBoundaryMarkers(text).split("\n") + const lastLines = lines.slice(-3).join("\n") + // A stop claim that is immediately followed by a comma and a CONTINUING + // connective is not the end of the turn — the model kept going. Two shapes + // kept fooling the clause terminators above, because they separate with a + // bare comma and no semicolon/dash/period: + // "Stopping here for a moment, then continuing with the migration." + // "There is nothing pending on the first repo, so I moved to the second." + // A genuine stop may contain commas ("Nothing launched, nothing pending from + // me"), but never a comma followed by then/so/next/afterwards/and then. + // + // SCOPE: the final CLAUSE, not the whole window (2026-10-11). Both examples + // above put the connective in the SAME clause as the stop, which is the only + // thing this guard is about. Scanning all three lines vetoed on connective + // prose from unrelated COMPLETED sentences: a turn ending in a bare + // "TASK COMPLETE" drew a stall injection 186s later because an earlier line + // read "...a config fix, SO it changes what I do next time." The stop + // pattern matched and the guard discarded it — a safety vetoing the very + // signal it exists to protect, on text it was never written about. Same + // clause discipline the pattern block above already enforces; the guard was + // the one place it was missing. Sentence-ending punctuation is stripped + // first so a terminal "." cannot yield an empty final clause. + const trimmedTail = lastLines.trim().replace(/[.!?…]+$/, "") + const finalClause = trimmedTail.split(/[.;:!?—–]/).pop() ?? trimmedTail + if (/,\s*(then|so|next|now|afterwards?|and then)\b/i.test(finalClause)) return false + return STOP_STATEMENT_PATTERNS.some((pat) => pat.test(lastLines)) +} + +/** + * One predicate for "this turn declares itself finished", shared by the idle + * inspection and the busy-stall watchdog. ses_edd1eb51 (2026-10-09): a + * "TASK COMPLETE" delivery went quiet with no session.idle event, so the + * idle latch never ran and the stall ladder fired 186s later. The busy path + * must judge the same tail from history before firing. + */ +function turnEndsWithCompletionClaim(text: string): boolean { + return endsWithCelebration(text) || containsStopStatement(text) +} + +/** + * True when a done-claim already carries a concrete work report. + * + * v1 asks for details once per done-claim and then trusts the answer. A length + * check cannot tell the difference between "Task done." (nothing was done) and + * a two-line summary, so v2 replaces the 400-char threshold with the same + * structural test v1 uses. Prompting again after a real report would loop + * forever (#26). + * + * Three signals, in order of specificity: + * - a backticked span naming a dotted file: `src/index.ts` + * - a bare path with a slash and a dotted extension: /a/b.py + * - a report header: changed / verification / results / commands run + */ +function containsWorkDescription(text: string): boolean { + // Backticked span mentioning a dotted filename. + if (/`[^`\n]*\.[a-zA-Z0-9]{1,8}[^`\n]*`/.test(text)) return true + // Bare path with a slash and a dotted extension. + if (/[\w\-~.][\w\-.~\/]*\/[\w\-.~]*\.[a-zA-Z]{1,8}\b/.test(text)) return true + // Report section headers. + if ( + /^(changed|modified|deleted|created|updated|renamed|moved|files?\s+changed|verification|verified|tests?(?:\s+run|\s+passing|\s+pass)?|results?|outcome|commands?\s+(?:run|executed))/im.test( + text, + ) + ) + return true + return false +} + +/** + * True when the last assistant turn closes with a celebration emoji. + * + * A 🎉 is the model's own "I finished" signal, used to latch completion rather + * than keep nudging. Deliberately end-anchored: boundary markers come in + * beginning/end pairs, so an anywhere-in-tail check would latch on a closing + * marker that signifies structure, not completion. Only the last line counts. + * The emoji alone is not trustworthy: a model that finishes early celebrates + * early, and latching on that turns a false positive into silence. The todo + * cross-check that catches it lives at the call site, where the fetched list + * is in hand. + */ +function endsWithCelebration(text: string): boolean { + const lastLine = stripBoundaryMarkers(text).split("\n").filter((l) => l.trim().length > 0).pop() ?? "" + return lastLine.trim().replace(/[.!?]+$/, "").endsWith("🎉") +} + +/** + * One entry of a session's todo list. + * + * Read from the todo tool that is already installed, not from a list this + * plugin keeps. The record shape is the one `todowrite` writes — + * `storage.set("todos/", {todos, updatedAt})` — so whatever todo + * tool the session uses is what this sees, and auto-resume never has to own a + * second copy that can drift. + */ +interface Todo { + content: string + status: "pending" | "in_progress" | "completed" | "cancelled" + priority: "high" | "medium" | "low" +} + +function isOpenTodo(t: Todo): boolean { + return t.status === "pending" || t.status === "in_progress" +} + +function getOpenTodos(todos: Todo[]): Todo[] { + if (!Array.isArray(todos)) return [] + return todos.filter(isOpenTodo) +} + +/** + * The prompt for a turn that claims to be finished with work still listed. + * + * Names the open items rather than saying "continue", because the model has + * already decided it is done — a bare continuation prompt gets answered with + * another done-claim. Falls back to a plain "continue" when the list is + * unusable, which is strictly better than sending an empty reminder. + */ +function buildOpenTodosReminder(todos: Todo[]): string { + if (!Array.isArray(todos)) return "continue" + const open = todos.filter(isOpenTodo) + if (open.length === 0) return "continue" + const list = open.map((t, i) => `${i + 1}. [${t.status}] ${t.content}`).join("\n") + const plural = open.length > 1 ? "s" : "" + const taskWord = open.length > 1 ? "tasks" : "task" + const thisWord = open.length > 1 ? "these" : "this" + return `You have ${open.length} unfinished task${plural}:\n${list}\n\nPlease continue working on ${thisWord} ${taskWord}.` +} + + +/** Candidate local server base URLs: env vars first, then /proc/self/cmdline. + * `cmdline` is injectable so the port parsing is unit-testable without spawning + * a process whose argv looks like a server invocation. + * + * The /proc read is not belt-and-braces: under OpenChamber the server is spawned + * as `opencode serve --hostname 127.0.0.1 --port ` and exports NO port env + * var, so an env-only list comes back EMPTY and the HTTP source is never + * attempted at all. Same mechanism the prompt-polisher uses, which is why its + * HTTP DELETE reaches this same server. + */ +function serverBaseUrls(cmdline?: string): string[] { + const out: string[] = [] + const push = (v: string | undefined) => { + if (typeof v !== "string" || !v) return + const url = v.startsWith("http") ? v : `http://127.0.0.1:${v.replace(/^:/, "")}` + if (!out.includes(url)) out.push(url.replace(/\/$/, "")) + } + const env = process.env + push(env.OPENCODE_SERVER_URL) + push(env.OPENCODE_URL) + push(env.OPENCODE_SERVER_PORT) + push(env.OPENCODE_PORT) + push(env.PORT) + try { + const raw = cmdline ?? readFileSync("/proc/self/cmdline", "utf8") + const argv = raw.split("\0") + const portFlag = argv.indexOf("--port") + if (portFlag !== -1 && argv[portFlag + 1]) { + const hostFlag = argv.indexOf("--hostname") + const host = hostFlag !== -1 && argv[hostFlag + 1] ? argv[hostFlag + 1] : "127.0.0.1" + push(`http://${host}:${argv[portFlag + 1]}`) + } + } catch { + // /proc unavailable (non-Linux): env vars only + } + return out +} +/** HTTP Basic for the local server, same derivation the prompt-polisher uses. */ +function serverHeaders(): Record { + const pw = process.env.OPENCODE_SERVER_PASSWORD || process.env.OPENCODE_PASSWORD + if (!pw) return {} + return { Authorization: `Basic ${Buffer.from(`opencode:${pw}`).toString("base64")}` } +} +/** + * Normalize a `todowrite` tool input into a todo list, or `undefined` when it is + * not a usable list. + * + * An entry with an unrecognized status is KEPT (cast, not dropped): `isOpenTodo` + * treats only `pending`/`in_progress` as open, so an unknown status reads as + * closed. Dropping it would shrink the list, and a shrunken list is the + * dangerous direction - it makes real open work look finished. + */ +function normalizeTodoList(input: unknown): Todo[] | undefined { + if (input === null || typeof input !== "object") return undefined + const todos = (input as { todos?: unknown }).todos + if (!Array.isArray(todos)) return undefined + const out: Todo[] = [] + for (const entry of todos) { + if (entry === null || typeof entry !== "object") continue + const e = entry as { content?: unknown; status?: unknown; priority?: unknown } + if (typeof e.content !== "string" || typeof e.status !== "string") continue + const todo: Todo = { content: e.content, status: e.status as Todo["status"], priority: "medium" } + if (typeof e.priority === "string") todo.priority = e.priority as Todo["priority"] + out.push(todo) + } + return out.length > 0 ? out : undefined +} +/** + * Canonical parser, mirroring opencode-todo-fork's `todosFromMessages` (the same + * routine is called `latestTodosFromMessages` in that repo's `src/tui-data.ts`): + * walk the messages, and for every COMPLETED `todowrite` tool part keep its + * normalized input. `todowrite` REPLACES the whole list, so the newest such call + * IS the current list. + * + * This is a deliberate copy rather than an import, and the reason is worth + * stating because it looks like an oversight otherwise. Plugins are loaded + * independently and there is no cross-plugin import contract: reaching into a + * sibling plugin's directory would tie auto-resume's ability to read a todo + * list to that plugin being installed at that exact path, and auto-resume has to + * work without it. The cost is that a fix to one copy is not automatically a fix + * to the other - so both copies carry the ordering note below, and both repos + * test the ranking rather than trusting it to stay in step. + * + * Order note: `/api/session/{id}/message` returns NEWEST FIRST, so a + * last-match-wins loop selects the OLDEST list - that bug shipped in two places + * before it was caught. Rank by timestamp instead, falling back to encounter + * order, and skip a malformed historical call while keeping the previous valid + * list. + * + * Returns `undefined`, never `[]`, so "no todos exist" stays distinguishable from + * "we could not look" - that distinction is what keeps this diagnosable instead of + * silently degrading. + */ +function todosFromMessages(messages: unknown): Todo[] | undefined { + if (!Array.isArray(messages)) return undefined + const candidates: { list: Todo[]; at: number | null; order: number }[] = [] + let order = 0 + for (const message of messages) { + if (message === null || typeof message !== "object") continue + const content = (message as { content?: unknown }).content + if (!Array.isArray(content)) continue + for (const part of content) { + if (part === null || typeof part !== "object") continue + const p = part as { type?: unknown; name?: unknown; state?: Record; time?: any } + if (p.type !== "tool" || p.name !== "todowrite") continue + if (p.state?.status !== "completed") continue + const list = normalizeTodoList(p.state.input) + if (!list) continue + const t = p.state?.time ?? p.time ?? (message as { time?: any }).time ?? {} + const at = [t?.end, t?.start, t?.created].find((v: unknown) => typeof v === "number") + candidates.push({ list, at: typeof at === "number" ? at : null, order: order++ }) + } + } + if (candidates.length === 0) return undefined + candidates.sort((a, b) => { + if (a.at !== null && b.at !== null && a.at !== b.at) return b.at - a.at + if ((a.at === null) !== (b.at === null)) return a.at === null ? 1 : -1 + return a.order - b.order + }) + return candidates[0]?.list +} + +/** Model ends with ":" announcing intent without executing. */ +function containsActionIntent(text: string): boolean { + if (text.length <= 15) return false + const cleaned = text.replace(/<[a-zA-Z/?][^>]*>/g, "").trim() + const lines = cleaned.split("\n") + let lastLine = "" + for (let i = lines.length - 1; i >= 0; i--) { + if (lines[i].trim().length > 0) { + lastLine = lines[i].trim() + break + } + } + return lastLine.endsWith(":") && lastLine.length > 5 && lastLine.length < 500 +} + +/** + * True when a turn cleanly ends by handing control back to the user — a + * question, or an explicit prompt for their input. Nudging such a turn + * injects a synthetic `continue` that starts an in-flight step, and when the + * user's real reply then arrives it interrupts that step → "Step interrupted" + * (the execution.interrupted event). A hand-off turn is, by design, awaiting + * the user, so it must never be nudged. + */ +function isUserHandoff(text: string): boolean { + const lines = text.split("\n") + let last = "" + for (let i = lines.length - 1; i >= 0; i--) { + const t = lines[i].trim() + if (t.length > 0) { + last = t + break + } + } + if (!last) return false + if (last.endsWith("?") || last.endsWith("？")) return true + const patterns = [/let\s+me\s+know/i, /your\s+call/i, /up\s+to\s+you/i, /which\s+(would|do\s+you|option)/i, /should\s+I/i, /want\s+me\s+to/i, /would\s+you\s+like/i, + // ses_ed9e2ef9 (2026-10-10): an offer awaiting approval reads as a + // go-ahead once a bare continue lands behind it ("I can delete all + // the stopped containers…" + continue = deletion). Bare capability + // statements with no ask are deliberately NOT covered — "I can X." + // with nothing after it is ambiguous, and the scoped continue text + // above is what protects those. + /say\s+the\s+word/i, /give\s+the\s+go-?ahead/i, /if\s+you'?d\s+like/i] + return patterns.some((p) => p.test(last)) +} + +function backoffMs(attempt: number, base: number, max: number): number { + return Math.min(base * Math.pow(2, attempt - 1), max) +} + +/** Silence-watchdog recover reasons look like "no activity for 186s". Shared + * by the stall-text builder (idle-field dedup) and recover (episode reset): + * only a silence stall can forgive tool progress, never a failure one. */ +function isSilenceReason(reason: string): boolean { + return /^no activity for \d+s/i.test(reason) +} + +/** True while the provider has told us it is retrying this session's turn (see + * the `session.retry.scheduled` handler). Inside that window a stall belongs to + * the transport, not the model: the ladder holds its rung and never gives up on + * it (ses_eae46bb9, 2026-10-19). */ +function isProviderRetryFresh(w: SessionWatch, now = Date.now()): boolean { + return w.providerRetryAt > 0 && now - w.providerRetryAt < PROVIDER_RETRY_GRACE_MS +} + +function isStreamingFailure(message: string, patterns: string[] = STREAMING_FAILURE_MESSAGE_PATTERNS): boolean { + const lower = message.toLowerCase() + if (!lower) return false + return patterns.some((pattern) => { + try { + return new RegExp(pattern, "i").test(lower) + } catch { + return lower.includes(pattern.toLowerCase()) + } + }) +} + +/** True when an error `name` is one of the configured streaming-failure names. */ +function isStreamingFailureName(name: string, names: string[]): boolean { + return names.some((candidate) => candidate.toLowerCase() === name.toLowerCase()) +} + +/** Repeating tool-call patterns: A-A-A or A-B-A-B-A-B etc. */ +function detectPatternLoop(recentTools: string[]): boolean { + if (recentTools.length < 6) return false + for (const patternLen of [1, 2, 3]) { + if (recentTools.length < patternLen * 3) continue + const pattern = recentTools.slice(-patternLen) + let matches = 0 + for (let i = recentTools.length - patternLen * 2; i >= 0; i -= patternLen) { + const slice = recentTools.slice(i, i + patternLen) + if (slice.length !== patternLen) break + if (!slice.every((t, idx) => t === pattern[idx])) break + matches++ + } + if (matches >= 2) return true + } + return false +} + +/** Edit distance, v1's implementation. Kept byte-for-byte in behaviour: the + * suggestion is only useful if it picks the same tool v1 would have. */ +function levenshtein(a: string, b: string): number { + const m = a.length + const n = b.length + if (m === 0) return n + if (n === 0) return m + let prev = new Array(n + 1) + let curr = new Array(n + 1) + for (let j = 0; j <= n; j++) prev[j] = j + for (let i = 1; i <= m; i++) { + curr[0] = i + for (let j = 1; j <= n; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1 + curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost) + } + const tmp = prev + prev = curr + curr = tmp + } + return prev[n] +} + +/** + * The closest registered tool name to one the model invented, or null. + * + * The threshold scales with the length of the wrong name — `max(2, len/2)` — so a + * short name is held to a near-exact match while a long one is allowed to be + * sloppy. Without that, a two-letter name would match almost anything and the + * suggestion would be noise. + */ +/** + * Names models invent for tools that exist under another name. Edit distance + * cannot bridge these ("bash"→"shell" is 4 against a threshold of 2), so the + * map is consulted before the fuzzy matcher. The target must be registered — + * an unguarded guess names a second nonexistent tool and teaches the model + * the registry lies. + */ +const TOOL_NAME_ALIASES: Record = { + bash: "shell", +} + +function resolveToolSuggestion(wrongName: string, available: string[]): string | null { + const alias = TOOL_NAME_ALIASES[wrongName.toLowerCase()] + if (alias && available.includes(alias)) return alias + return suggestClosestTool(wrongName, available) +} + +function suggestClosestTool(wrongName: string, available: string[]): string | null { + const lower = wrongName.toLowerCase() + let best: string | null = null + let bestDist = Infinity + for (const id of available) { + const dist = levenshtein(lower, id.toLowerCase()) + const threshold = Math.max(2, Math.floor(lower.length / 2)) + if (dist < bestDist && dist <= threshold) { + bestDist = dist + best = id + } + } + return best +} + +function trackToolCall(w: SessionWatch, toolName: string): boolean { + const now = Date.now() + w.recentToolCalls = w.recentToolCalls.filter((c) => now - c.at < 120_000) + w.recentToolCalls.push({ toolName, at: now }) + const recentTools = w.recentToolCalls.slice(-12).map((c) => c.toolName) + if (recentTools.length < 6) return false + const lastTool = recentTools[recentTools.length - 1] + const consecutiveSame = recentTools.slice(-4).filter((t) => t === lastTool).length + if (consecutiveSame >= 4) return true + return detectPatternLoop(recentTools) +} + +function short(sid: string): string { + return sid.length > 12 ? `…${sid.slice(-8)}` : sid +} + +function sidOf(ev: V2Event): string | undefined { + const sid = ev.data?.sessionID + return typeof sid === "string" ? sid : undefined +} + +/** + * Role of a history record, tolerant of every shape the transcript API uses. + * + * Audit 2026-10-10: production stores `{"role":"user","time":{"created":...}}` + * (verified in opencode.db), while the harness fed `{type:"user"}`. Six readers + * compared `.type` only and went blind in prod — the quota-restore guard failed + * OPEN, re-arming a ladder for a session the user had already continued, and the + * stuck-tool / silent-reasoning / subagent-crash detectors silently never fired. + * 938 tests passed because every fixture used `type`. Read the role HERE. + */ +function msgRole(m: unknown): string { + const mm = m as { role?: unknown; type?: unknown; info?: { role?: unknown } } | null | undefined + if (!mm || typeof mm !== "object") return "" + const v = mm.role ?? mm.type ?? mm.info?.role + return typeof v === "string" ? v : "" +} + +/** Creation time of a history record; 0 when absent. Same tolerance as msgRole. */ +function msgAt(m: unknown): number { + const mm = m as { time?: { created?: unknown }; info?: { time?: { created?: unknown } } } | null | undefined + const t = mm?.time?.created ?? mm?.info?.time?.created + return typeof t === "number" ? t : 0 +} + +function isTaskToolCall(ev: V2Event): boolean { + const tool = ev.data?.tool ?? ev.data?.toolName + if (typeof tool === "string") { + const lower = tool.toLowerCase() + if (TASK_TOOL_HINTS.some((h) => lower.includes(h))) return true + } + // Also inspect input for agent-ish payloads + const desc = ev.data?.input?.description ?? ev.data?.input?.subagent_type ?? ev.data?.input?.agent + return typeof desc === "string" +} +// --------------------------------------------------------------------------- +// Plugin +// --------------------------------------------------------------------------- + +/** + * The one live instance of this plugin in this process, if any. + * + * `setup()` is re-run by the loader on every config reload — the live log shows + * 2523 loads of this ONE entrypoint against a single server process, and bursts + * of SIX `ready` lines with no `stopped` between them. Every setup call builds + * its own `sessions` map, its own watchdog interval and its own event pump, so + * six live instances meant six private `resumeAttempts` counters: one stall + * produced six `resume attempt 1/3` lines in the same millisecond and six + * identical continues with no wait between them (2026-10-03, session + * ses_efcc2ade…kJmjQR3o). The counter was never broken — each copy had its own. + * + * Last-wins: a fresh setup disposes the previous instance before arming its own, + * so exactly one watchdog and one counter exist no matter how often the loader + * re-runs setup. Disposal is deliberately not left to the host, because the + * storms prove it does not always happen. + */ +/** + * The one live instance of this plugin in this PROCESS, shared across every + * evaluation of this bundle. + * + * It lives on `globalThis`, not in module scope, and that is the whole fix. The + * loader re-EVALUATES the plugin file on every config reload: the live log shows + * one entrypoint URL, one pid, and a fresh module identity each time + * (`mod=1812055.c6d4mu`, `.72a9tl`, `.dt27x7`, … six distinct copies inside one + * server process). Module scope cannot help, because each copy gets its own + * `activeInstance` — a singleton fix shipped that way and the storms continued + * (19:54:19, eight identical `resume attempt 1/3` lines in 5ms). + * + * `globalThis` is shared by all module instances in a process (verified on both + * node and bun), so a key here is the one channel that survives re-evaluation. + * Each setup finds its predecessor through the registry and disposes it before + * arming itself: last wins, so exactly one watchdog and one `resumeAttempts` + * counter exist however often the loader reloads us. Disposal cannot be left to + * the host, because the storms prove it does not always happen. + * + * Keyed by pid alone, and that was the claim to be wrong about: this comment + * read "Keyed by pid + plugin id ... two config trees (v1 and v2) ... must not + * dispose each other", but the key carried no plugin id, so two builds sharing + * a pid DID dispose each other. Nothing above ever needed a plugin id because + * a process hosts one build; `registryScope` is what makes the split real, for + * the test process that hosts many instances at once (see registryKeyFor). + */ +const REGISTRY_KEY = `__auto_resume_singleton__:${process.pid}` + +/** + * The registry slot this build instance owns. + * + * `registryScope` splits the slot. Without it the key is pid-wide, which is + * right for production (one config tree, one build: a fresh setup disposes + * its predecessor and the reload storm cannot stack instances) but WRONG for + * any process that holds two unrelated instances at once — a test suite most + * plainly. `bun test` runs all 79 files in ONE process, so file A's setup + * disposed file B's in-flight instance: `running = false`, event pump + * aborted, watchdog cleared. B's plugin was dead, and its assertions could + * never pass — they waited 10s for an event no longer being delivered, and + * every one of those tests passed in isolation (2026-10-10/11). + * + * The same hazard is real outside tests: two builds (v1 and v2) sharing a pid + * would silence each other, since only one can hold the pid-wide slot. + */ +function registryKeyFor(scope?: string): string { + return scope ? `${REGISTRY_KEY}:${scope}` : REGISTRY_KEY +} + +type SingletonRegistry = { + live?: { dispose: () => void; mod: string; startedAt: number } + // Watch state shared across setup re-runs in one process. The loader + // re-evaluates this bundle on session attach/restore (bursts of several + // setups/second plus singles every ~25-40min on record 2026-10-09), and + // every setup used to start with a blank sessions map, making + // multi-minute stalls unobservable. Successors inherit the map instead; + // see the adoption below and the no-clear in dispose. + sessions?: Map +} + +function singletonRegistry(scope?: string): SingletonRegistry { + const g = globalThis as unknown as Record + const key = registryKeyFor(scope) + let reg = g[key] + if (!reg) { + reg = {} + g[key] = reg + } + return reg +} + +/** + * Identity of THIS module evaluation, stamped into the ready line. + * + * `pid` separates processes; `eval` separates module evaluations inside one + * process. Six distinct values under one pid is what exposed the real cause — + * keep this until the storms are gone, because it is the only way to tell a + * re-evaluation apart from a genuine second server. + */ +const MODULE_INSTANCE = `${process.pid}.${Math.random().toString(36).slice(2, 8)}` + +/** + * Host identity, for the reload forensics. + * + * `mod=` alone cannot tell a plugin reload from a host restart: every reload + * mints a new `eval` suffix, and a restart changes `pid` too. Uptime separates + * them — a plugin re-evaluated inside a long-lived host keeps a growing uptime + * and a constant `boot`, so a storm of mod ids under ONE boot is the plugin + * being re-imported, and a changing boot is opencode itself restarting. + * Measured 2026-10-08: 22 lifecycle lines in ten minutes, all pid 3686667 — + * re-evaluations inside one host, not host restarts. + */ +const HOST_UPTIME_S = Math.round(process.uptime()) +const HOST_BOOT = new Date(Date.now() - process.uptime() * 1000).toISOString() + +export default define({ + id: "auto-resume.v2", + + setup: async (ctx: AutoResumePluginInput) => { + const opts = (ctx.options ?? {}) as AutoResumeOptions + // Supersede any instance the loader left running — found through the + // process-wide registry, so it also catches instances belonging to OTHER + // evaluations of this bundle. Wrapped: a throwing predecessor must not + // block the new one from arming. + // + // Read before the registry: `registryScope` selects WHICH slot this + // instance owns, and instances in different slots never dispose each + // other (see registryKeyFor). + const registry = singletonRegistry(opts.registryScope) + // A second live instance means the host re-evaluated this bundle without + // disposing the first — the reload-storm signature. The predecessor's + // identity and age are captured HERE and logged later, because `log` + // closes over `logFile`, which is not initialised until well below this + // point: calling it from this block is a TDZ ReferenceError, and it only + // fires when a predecessor exists — i.e. never in a single-test process. + const superseded = registry.live + if (superseded) { + try { + superseded.dispose() + } catch { + /* predecessor already torn down */ + } + registry.live = undefined + } + const registryScope = opts.registryScope + + const chunkTimeoutMs = opts.chunkTimeoutMs ?? DEFAULT_CHUNK_TIMEOUT_MS + const checkIntervalMs = opts.checkIntervalMs ?? DEFAULT_CHECK_INTERVAL_MS + const gracePeriodMs = opts.gracePeriodMs ?? DEFAULT_GRACE_PERIOD_MS + const stuckToolAgeMs = opts.stuckToolAgeMs ?? DEFAULT_STUCK_TOOL_AGE_MS + const neverInterruptTools = (opts.neverInterruptTools ?? ["compress"]).map((n) => String(n).toLowerCase()) + // `maxRecoveryRetries` is v1's name for the same knob. Prefer the v2 name, + // fall back to the v1 alias so an existing v1 config ports unchanged. + const maxRetries = opts.maxRetries ?? opts.maxRecoveryRetries ?? DEFAULT_MAX_RETRIES + const baseBackoff = opts.baseBackoffMs ?? DEFAULT_BASE_BACKOFF_MS + const maxBackoff = opts.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS + const loopMaxContinues = opts.loopMaxContinues ?? DEFAULT_LOOP_MAX_CONTINUES + const loopWindowMs = opts.loopWindowMs ?? DEFAULT_LOOP_WINDOW_MS + const lifetimeMaxContinues = opts.lifetimeMaxContinues ?? DEFAULT_LIFETIME_MAX_CONTINUES + const debug = opts.debug ?? DEFAULT_DEBUG + const activeUserWindowMs = opts.activeUserWindowMs ?? DEFAULT_ACTIVE_USER_WINDOW_MS + const injectIntervalMs = opts.injectIntervalMs ?? DEFAULT_INJECT_INTERVAL_MS + const envVisible = process.env.AUTO_RESUME_VISIBLE_CONTINUE + const visibleContinue = + opts.visibleContinue ?? + (envVisible === undefined || envVisible === "" + ? true + : /^(1|true|yes)$/i.test(envVisible)) + const quietCorrections = opts.quietCorrections ?? true + const richContinuePrompt = opts.richContinuePrompt ?? true + const logFile = opts.logFile ?? process.env.AUTO_RESUME_LOG_FILE ?? DEFAULT_LOG_FILE + const rateLimitCooldownsMs = + Array.isArray(opts.rateLimitCooldownsMs) && opts.rateLimitCooldownsMs.length > 0 && + opts.rateLimitCooldownsMs.every((n) => typeof n === "number" && n > 0) + ? (opts.rateLimitCooldownsMs as number[]) + : DEFAULT_RATE_LIMIT_COOLDOWNS_MS + const sendTimeoutMs = opts.sendTimeoutMs ?? DEFAULT_SEND_TIMEOUT_MS + // Inert probe: proves `plugins[].options` reaches `ctx.options` without + // touching a live knob. Echoed in the `ready` line only. + const configProbe = opts.configProbe ?? null + // How long a parent may sit busy after its last subagent went idle before + // the orphan watch acts. v1 default, honoured for the first time here. + const subagentWaitMs = opts.subagentWaitMs ?? DEFAULT_SUBAGENT_WAIT_MS + const subagentDeadMs = opts.subagentDeadMs ?? DEFAULT_SUBAGENT_DEAD_MS + // How long to let a finished turn's text settle before judging it against the + // done/tool patterns. v1's default, and the reason v1 does not judge on idle + // at all — see inspectOnIdle. + const toolTextCheckDelayMs = opts.toolTextCheckDelayMs ?? DEFAULT_TOOL_TEXT_CHECK_DELAY_MS + + // ---- Ported from v1 (Mte90/opencode-auto-resume#33) ---- + const rawBusyStallStrategy = opts.busyStallStrategy ?? "continue" + const busyStallStrategy: "continue" | "abort" | "off" = + rawBusyStallStrategy === "abort" || rawBusyStallStrategy === "off" ? rawBusyStallStrategy : "continue" + // Feature-gated on v2 (see FEATURE_GATED_OPTIONS): accepted for config + // compatibility, not yet acted on. Not bound to locals so they cannot be + // mistaken for live behaviour. + const resumeOnActionIntent = opts.resumeOnActionIntent !== false + const warmupMs = opts.warmupMs ?? DEFAULT_WARMUP_MS + // Successor setups inherit the watch map instead of starting blank. + // Off unless prod opts in: test files share one process, so sharing + // by default leaks state across tests. + const shareWatchState = opts.shareWatchState ?? DEFAULT_SHARE_WATCH_STATE + // Live since the v2 discovery sweep landed (see discoverSessions). + const discoveryDelayMs = opts.discoveryDelayMs ?? DEFAULT_DISCOVERY_DELAY_MS + const minActivityGapMs = opts.minActivityGapMs ?? DEFAULT_MIN_ACTIVITY_GAP_MS + // Live since the v2 token read landed (see tokenTotalOf / checkContextSaturation). + const contextSaturationThreshold = opts.contextSaturationThreshold ?? DEFAULT_CONTEXT_SATURATION_THRESHOLD + const subagentNativeCompactionEnabled = opts.subagentNativeCompactionEnabled ?? false + // Live since the silent-dead-stream detector landed (lastSilentDeadStream). + const silentDeadStreamMinTokens = opts.silentDeadStreamMinTokens ?? DEFAULT_SILENT_DEAD_STREAM_MIN_TOKENS + const streamingFailureErrorNames = opts.streamingFailureErrorNames ?? DEFAULT_STREAMING_FAILURE_ERROR_NAMES + const streamingFailureMessagePatterns = opts.streamingFailureMessagePatterns ?? DEFAULT_STREAMING_FAILURE_MESSAGE_PATTERNS + const doneWithoutDetailsPrompt = opts.doneWithoutDetailsPrompt ?? DONE_WITHOUT_DETAILS_PROMPT + // Live since the todo read landed (see readTodos): a done-claim with items still + // listed gets this prompt instead of the details prompt. + const doneWithoutWorkPrompt = opts.doneWithoutWorkPrompt ?? DONE_WITHOUT_WORK_PROMPT + const thinkingToolRecoveryPrompt = opts.thinkingToolRecoveryPrompt ?? THINKING_TOOL_RECOVERY_PROMPT + + /** Compile a user-supplied regex-source list, skipping anything invalid. */ + function compilePatterns( + raw: string[] | undefined, + fallback: string[], + flags: string, + ): RegExp[] { + const sources = Array.isArray(raw) && raw.length > 0 ? raw : fallback + const out: RegExp[] = [] + for (const source of sources) { + try { + out.push(new RegExp(source, flags)) + } catch { + // An invalid pattern is dropped rather than failing startup. + } + } + return out + } + + const doneClaimPatterns = compilePatterns(opts.doneClaimPatterns, DEFAULT_DONE_CLAIM_PATTERNS, "im") + const readyToContinuePatterns = compilePatterns( + opts.readyToContinuePatterns, + DEFAULT_READY_TO_CONTINUE_PATTERNS, + "i", + ) + const streamingFailureMessageRegexes = compilePatterns( + streamingFailureMessagePatterns, + DEFAULT_STREAMING_FAILURE_MESSAGE_PATTERNS, + "i", + ) + + // Options accepted but not yet acted on, because the v2 feature they tune + // is not ported. Listed (not warned) so an existing v1 config stays valid + // and the gap is documented rather than surprising. See + // docs/known-issues-v2.md. + // + // Empty: every v1 option this build understands is now applied. Kept as a + // list rather than deleted because the reporting below, and the + // accepted-but-inert= note in the startup line, are the only things a user + // has to go on when an option is quietly ignored — so the next one that + // arrives has somewhere to be listed. + const FEATURE_GATED_OPTIONS = [] as const + + // Options this build understands. Anything else in the user's config is + // reported once at startup so a silent fallback is visible rather than + // looking like a bug. (Mte90/opencode-auto-resume#33) + const RECOGNISED_OPTIONS = new Set([ + "chunkTimeoutMs", + "checkIntervalMs", + "gracePeriodMs", + "maxRetries", + "maxRecoveryRetries", + "baseBackoffMs", + "maxBackoffMs", + "loopMaxContinues", + "loopWindowMs", + "lifetimeMaxContinues", + "debug", + "activeUserWindowMs", + "injectIntervalMs", + "visibleContinue", + "quietCorrections", + "richContinuePrompt", + "continuePrompt", + "toolTextRecoveryPrompt", + "actionIntentPrompt", + "doneWithoutWorkPrompt", + "busyStallStrategy", + "contextSaturationThreshold", + "subagentNativeCompactionEnabled", + "resumeOnActionIntent", + "discoveryDelayMs", + "warmupMs", + "minActivityGapMs", + "toolTextCheckDelayMs", + "subagentWaitMs", + "shareWatchState", + "registryScope", + "neverInterruptTools", + "subagentDeadMs", + "silentDeadStreamMinTokens", + "streamingFailureErrorNames", + "streamingFailureMessagePatterns", + "doneClaimPatterns", + "readyToContinuePatterns", + "doneWithoutDetailsPrompt", + "thinkingToolRecoveryPrompt", + "doneWithoutWorkPrompt", + "logFile", + "rateLimitCooldownsMs", + "sendTimeoutMs", + "stuckToolAgeMs", + "configProbe", + ]) + const unknownOptions = Object.keys(opts).filter((key) => !RECOGNISED_OPTIONS.has(key)) + if (unknownOptions.length > 0) { + log( + "warn", + `ignoring unrecognised option(s): ${unknownOptions.join(", ")} — ` + + `this v2 build does not read them. See docs/known-issues-v2.md.`, + ) + } + // Feature-gated options are valid but inert on v2. Reported in the startup + // line rather than as a warning — the user cannot act on it, so a per-start + // warn would be pure noise. See docs/known-issues-v2.md. + const gatedInUse = Object.keys(opts).filter((key) => + (FEATURE_GATED_OPTIONS as readonly string[]).includes(key), + ) + + // Debug output goes to the same file as everything else. Console-only debug + // is invisible in v2 for the same reason the rest of the logging was, and + // debug is exactly when you are trying to work out what the plugin did. + const dbg = (...args: unknown[]) => { + if (!debug) return + appendLogFile(logFile, "debug", `[auto-resume:debug] ${args.map((a) => String(a)).join(" ")}`) + console.log("[auto-resume:debug]", ...args) + } + + function log(level: "info" | "warn" | "error", msg: string) { + const line = `[auto-resume] ${msg}` + // v2 exposes no server log sink to plugins (ctx.app is + // {name,version,channel}), so the file is the only retrievable record. + appendLogFile(logFile, level, line) + // Console too: harmless when nobody captures it, and the reason the + // startup line is visible at all when running under `opencode serve`. + if (level === "error") console.error(line) + else if (level === "warn") console.warn(line) + else console.log(line) + } + + // --------------------------------------------------------------------- + // State + // --------------------------------------------------------------------- + + // Adopted, not fresh: see SingletonRegistry.sessions. A successor setup + // inherits the predecessor's watch map (stall clocks, ladders, latches) + // instead of starting blank. openShells stays per-instance: shells are + // transient and a stale entry would wrongly block nudges. + // Gated on shareWatchState: test files share one process, and + // unconditional sharing leaks watch state across tests. + const registryForState = singletonRegistry(registryScope) + let adoptedSessions = 0 + let sessions: Map + if (shareWatchState) { + if (!(registryForState.sessions instanceof Map)) { + registryForState.sessions = new Map() + } + sessions = registryForState.sessions + adoptedSessions = sessions.size + } else { + sessions = new Map() + } + + /** + * Open shells from the process-registry event family, keyed by shell id + * → `{ sessionID, startedAt }`. Populated by `shell.created`, cleared by + * `shell.exited`. A session with an entry here is running a command and + * must not be treated as a stalled parent. + * + * `shell.created` is the only shell event that carries its session, and it + * nests it at `data.info.metadata.sessionID`; the exit events carry only + * the shell id, so the reverse lookup has to be recorded here. + */ + const openShells = new Map() + + /** Number of shells currently open for `sid`, pruning stale entries. */ + function openShellCount(sid: string, now = Date.now()): number { + let n = 0 + for (const [id, s] of openShells) { + if (now - s.startedAt > SHELL_OPEN_MAX_MS) { + openShells.delete(id) + continue + } + if (s.sessionID === sid) n++ + } + return n + } + + /** Drop every shell entry owned by a session that no longer exists. */ + function forgetShells(sid: string): void { + for (const [id, s] of openShells) { + if (s.sessionID === sid) openShells.delete(id) + } + } + + function ensureWatch(sid: string): SessionWatch { + let w = sessions.get(sid) + // Migrate watches adopted from older builds. The sessions map is shared + // across setup re-runs (and the loader re-evaluates this bundle on every + // config reload), so a watch created before a field existed reads it as + // undefined: `undefined++` is NaN (shipped as "retry #NaN", 2026-10-12) + // and `for..of undefined` throws. Every field added after registry + // sharing began needs a default here, not just in the constructor below. + if (w) { + w.emptyIdleNudged ??= false + w.episodeRecovers ??= 0 + w.lifetimeContinues ??= 0 + w.inflightTools ??= new Map() + w.lastStuckToolCheck ??= 0 + return w + } + if (!w) { + w = { + createdAt: Date.now(), + lastActivityAt: Date.now(), + status: "unknown", + userCancelled: false, + userCancelledAt: 0, + resumeAttempts: 0, + episodeRefires: 0, + episodeRecovers: 0, + lifetimeContinues: 0, + lastRetryAt: 0, + providerRetryAt: 0, + quotaNoticedAt: 0, + gaveUp: false, + lastGenuineUserAt: 0, + consecFailureRecovers: 0, + failureLoopLatched: false, + aborting: false, + selfAbortAt: 0, + interruptsThisWindow: 0, + interruptWindowStart: 0, + lastInjectAt: 0, + lastProdText: "", + prodAssistantSnapshot: "", + completedSinceProd: 0, + succeededSinceProd: 0, + ownPromptTexts: [], + recovering: false, + oocLocked: false, + oocLockReason: null, + permLocked: false, + permLockReason: null, + indeterminateStrikes: 0, + selfRecovery: false, + textParts: new Map(), + lastAssistantText: "", + lastAssistantMessageID: null, + toolTextAttempts: 0, + stuckToolAttempts: 0, + continueTimestamps: [], + doneClaimAttempts: 0, + doneClaimOpenTodosAttempts: 0, + todoNudgeAttempts: 0, + intentNudgeAttempts: 0, + completionSignaled: false, + completionSignaledAt: 0, + emptyIdleNudged: false, + todos: [], + todosFetchedAt: 0, + taskCompleteSignals: 0, + taskCompleteOverrides: 0, + orphanWatchStartAt: null, + orphanWatchArmedAt: null, + orphanRecoveryTried: false, + toolTextTimer: null, + recoverTimer: null, + lastSubagentCheckAt: 0, + pendingTools: 0, + toolsInFlightAtIdle: 0, + inflightTools: new Map(), + rateLimitedAt: 0, + rateLimitAttempts: 0, + awaitingQuotaRetry: false, + unknownToolErrors: new Map(), + unknownToolSuggestionSent: false, + checkedToolPartIDs: new Set(), + lastUserMessageSeenAt: 0, + lastSilentReasoningCheck: 0, + lastStuckToolCheck: 0, + lastTokenTotal: 0, + contextWrapupAttempts: 0, + pendingRecoveryArmed: false, + permissionPending: false, + permissionPendingAt: null, + lastWasTaskTool: false, + idleSince: null, + compacting: false, + compactionStartedAt: null, + recentToolCalls: [], + } + sessions.set(sid, w) + } + return w + } + + function touch(sid: string) { + const w = ensureWatch(sid) + // `minActivityGapMs` debounces activity: without it, a burst of events + // inside one tool call keeps resetting the stall clock and a genuinely + // wedged step never trips the watchdog. + const now = Date.now() + if (now - w.lastActivityAt < minActivityGapMs) return + w.lastActivityAt = now + } + + function markBusy(sid: string) { + const w = ensureWatch(sid) + if (w.status !== "busy") { + dbg(`${short(sid)} idle/unknown -> busy`) + // Fresh busy cycle: reset per-turn budgets. + // NOTE: continueTimestamps is intentionally preserved — the + // hallucination-loop detector counts across busy cycles by design. + // NOTE 2: resumeAttempts is deliberately NOT reset here. Lifecycle + // transitions are plumbing, and an interleaving idle event between + // our send and the model's start made every other stall read 1/3 + // forever (ses_ee14d727). Episode boundaries live where they can + // be observed: a genuine user message (execution.started, + // noteInboundUserMessage), model text growth (recover), and the + // abort+resume escalation. Nowhere else. + w.toolTextAttempts = 0 + w.stuckToolAttempts = 0 + // A fresh turn means the age gate is judged against a new call, so + // the watchdog's stuck-tool throttle re-arms with the attempt budget. + w.lastStuckToolCheck = 0 + // Deliberately NOT resetting doneClaimAttempts / doneClaimOpenTodosAttempts. + // v1 moved both out of the busy-cycle reset for a reason (#26): a model + // that keeps re-announcing completion would get a fresh budget on every + // turn it announced, so the nudge never stopped. They re-arm only on a + // genuine new work cycle — an inbound user message, via + // noteInboundUserMessage(). + // The open-todos nudge is the opposite case and does reset here: an open + // list is new information each turn, not a model that keeps saying the + // same thing. + w.todoNudgeAttempts = 0 + w.intentNudgeAttempts = 0 + // NOTE: emptyIdleNudged is deliberately NOT reset here. The + // flag is single-shot per observable episode, and markBusy runs + // on our own recovery injections too: resetting it here would + // re-arm on the very inject it just sent, and an idle-empty + // session would cycle nudge-inject-idle-nudge forever. It + // re-arms only where a new episode is observable: a genuine + // user message, a genuine execution.started, or abort+resume. + // A new turn means the text a pending pattern pass was going to judge is + // superseded, and markBusy has just emptied the buffer it would have read. + if (w.toolTextTimer) { + clearTimeout(w.toolTextTimer) + w.toolTextTimer = null + } + // A new turn re-opens the question of whether the work is finished — + // BUT only a genuine one. markBusy also runs for plumbing transitions + // (watchdog re-mark from server truth, discovery adopt, step events, + // our own injections' turns), none of which is new work. Clearing the + // latch here wiped freshly-set completions within seconds: ses_ed33c0da + // (2026-10-11) would have latched TASK COMPLETE at idle and lost it at + // the re-mark 1s later, stalling at +197s anyway. The latch now clears + // only on a genuine new user message (noteInboundUserMessage, which + // already distinguishes own prompts), timestamped against + // completionSignaledAt; the busy-path skip below re-opens it there. + // NOTE 3: gaveUp is deliberately NOT cleared here either. It is + // an episode latch, not per-turn bookkeeping: a stood-down + // session the watchdog re-adopts from server truth (or + // discovery seeds) must stay stood down, or the stand-down + // guarantee cycles. It clears only on observable new episodes: + // a genuine user message (noteInboundUserMessage), a genuine + // execution.started, or the abort+resume escalation. + // A new turn means the parent moved on, so whatever the previous orphan + // watch was about is no longer the question. Left armed, it would fire + // mid-turn and abort a session that is doing exactly what it should. + w.orphanWatchStartAt = null + w.orphanWatchArmedAt = null + // orphanEscalatedChildSid deliberately SURVIVES here. It records that + // a specific dead child has already cost this parent one abort, not a + // retry budget — and a parent that recovers and does real work between + // aborts is exactly the loop that produced 8 escalations in 3 hours + // (ses_ed726872). Clearing it on every genuine turn reinstates the bug + // the moment the parent does anything useful. It is cleared when the + // watch is disarmed as void (an idle session's own watch) and when the + // session is torn down. + w.recentToolCalls = [] + w.textParts.clear() + w.lastAssistantText = "" + } + w.status = "busy" + w.idleSince = null + w.lastActivityAt = Date.now() + } + + function markIdle(sid: string) { + const w = ensureWatch(sid) + if (w.status !== "idle") { + dbg(`${short(sid)} ${w.status} -> idle`) + w.status = "idle" + w.idleSince = Date.now() + // Idle means nothing is in flight. Any slot still held belongs to a tool + // that will never report — a subagent killed mid-call, for instance — + // and keeping it would make the orphan watch defer on this session + // forever. + w.pendingTools = 0 + // This session is not the parent the watch is about: the watch is armed on + // the session that *stays* busy, so an idle session's own watch is void. + w.orphanWatchStartAt = null + w.orphanWatchArmedAt = null + // orphanEscalatedChildSid survives, same reason as in markBusy: a + // parent that idles between aborts and then works again is the exact + // shape that cycled 8 times. An idle transition voids the ARMED + // watch, not the record of what a dead child already cost. + } + // NOTE: this used to clear `permissionPending`. The `session.idle` handler + // calls markIdle() and then inspectOnIdle(), so the flag was always false by + // the time the guard that honours it ran — the permission guard was dead on + // exactly the transition it exists for. `permission.replied` is now the only + // thing that clears it, plus a stale-clear for a `permission.asked` whose + // reply never arrives. + clearStalePermissionFlag(sid, w) + } + + /** + * A `permission.asked` with no matching `permission.replied` would otherwise + * stand the session down forever. Mirrors the compaction stale-guard. + */ + function clearStalePermissionFlag(sid: string, w: SessionWatch) { + if (!w.permissionPending) return + if (w.permissionPendingAt && Date.now() - w.permissionPendingAt > PERMISSION_STALE_TTL_MS) { + dbg(`${short(sid)} permission prompt silent >${PERMISSION_STALE_TTL_MS / 60000}m — clearing flag`) + w.permissionPending = false + w.permissionPendingAt = null + } + } + + function recordContinue(sid: string) { + const w = sessions.get(sid) + if (!w) return + const now = Date.now() + w.continueTimestamps.push(now) + w.lifetimeContinues = (w.lifetimeContinues ?? 0) + 1 + const cutoff = now - loopWindowMs + while (w.continueTimestamps.length > 0 && w.continueTimestamps[0] < cutoff) { + w.continueTimestamps.shift() + } + } + + function continuesInWindow(w: SessionWatch): number { + const cutoff = Date.now() - loopWindowMs + while (w.continueTimestamps.length > 0 && w.continueTimestamps[0] < cutoff) { + w.continueTimestamps.shift() + } + return w.continueTimestamps.length + } + + /** Lifetime cap: total stall continues this genuine user episode. + * The per-stall ladder restarts on model text, so without this a + * model that keeps replying "done" reads 1/3 forever. Only a + * genuine user message clears it — never model output. */ + function lifetimeExhausted(sid: string, w: SessionWatch): boolean { + if ((w.lifetimeContinues ?? 0) >= lifetimeMaxContinues) { + if (!w.gaveUp) { + log("warn", `${short(sid)} lifetime cap (${lifetimeMaxContinues} continues, no genuine user message) — standing down`) + w.gaveUp = true + } + return true + } + return false + } + + /** + * True while a plugin-initiated abort is still in flight. The runtime delivers + * `session.execution.interrupted` asynchronously after `interrupt()` returns, so + * the transient `w.aborting` boolean is not a reliable discriminator. + */ + function selfAbortActive(w: SessionWatch): boolean { + return w.selfAbortAt > 0 && Date.now() - w.selfAbortAt < SELF_ABORT_TTL_MS + } + + /** Does this failure signature mean "the turn was interrupted"? */ + function isAbortError(errType: string, errMsg: string): boolean { + return ABORT_ERROR_TYPE_RE.test(errType.trim()) || ABORT_ERROR_MSG_RE.test(errMsg) + } + + function isRateLimitError(errType: string, errMsg: string): boolean { + return RATE_LIMIT_RE.test(`${errType} ${errMsg}`) + } + + + /** + * Stand down on quota/rate-limit failures instead of retrying into the + * ban. Gates, not timers: the decision is re-evaluated on each failure + * and each watchdog tick against `rateLimitedAt`, so nothing is lost + * to a reload. Served cooldowns flow into the normal `recover()` path + * (budgets, loop guard and inject guards all apply); past the ladder + * the session stays silent until a genuine user turn. + */ + /** + * User-visible quota notice. The stand-down itself is silent by design + * (retrying into a ban helps nothing), but an unexplained silent + * session reads as dead. resume:false shows it in the timeline without + * kicking a turn, re-arming budgets, or tripping own-prompt guards + * (text never matches an injected continue). Best-effort: a failed + * notice must never block recovery. + */ + async function sendQuotaNotice(sid: string, text: string): Promise { + try { + await sendBound("quota-notice", sid, () => + ctx.session.synthetic({ sessionID: sid, text, description: "auto-resume: quota notice", resume: false }), + ) + } catch (e) { + log("warn", `${short(sid)} quota notice failed: ${e instanceof Error ? e.message : String(e)}`) + } + } + + /** + * Serve one rung of the rate-limit cooldown ladder: advance the rung, + * re-arm recovery, and reset the notice flag so the next arm (or + * exhaustion) announces itself afresh. Shared by the failure handler + * (cooldown already served when the failure arrives) and the watchdog + * gate (cooldown served while stood down) — one shape, one place, so + * the two can never drift apart again. + */ + function serveQuotaRung(sid: string, w: SessionWatch, rung: number, ladder: number[]): void { + w.rateLimitAttempts = rung + 1 + w.rateLimitedAt = Date.now() + w.awaitingQuotaRetry = false + w.pendingRecoveryArmed = true + // A served rung ends the quiet spell: clear the notice flag + // so the next arm (or exhaustion) announces itself afresh. + // Each rung then carries at most one notice. + w.quotaNoticedAt = 0 + log("info", `${short(sid)} rate-limit cooldown served — retrying (attempt ${rung + 1}/${ladder.length}) mod=${MODULE_INSTANCE}`) + void syncQuotaState(sid, w) + void recover(sid, "rate-limit cooldown served") + } + + function handleRateLimitFailure(sid: string, w: SessionWatch, errType: string, errMsg: string): void { + const ladder = rateLimitCooldownsMs + const tag = "rate-limit" + const n = w.rateLimitAttempts + markIdle(sid) + if (n >= ladder.length) { + w.awaitingQuotaRetry = false + log("warn", `${short(sid)} ${tag} budget exhausted (${ladder.length} attempts) — standing down until user turn`) + if (!w.quotaNoticedAt) { + w.quotaNoticedAt = Date.now() + void sendQuotaNotice(sid, `Rate limited — standing down until you return (budget exhausted after ${ladder.length} attempts).`) + } + return + } + const now = Date.now() + if (w.rateLimitedAt > 0 && now - w.rateLimitedAt >= ladder[n]) { + serveQuotaRung(sid, w, n, ladder) + return + } + if (w.rateLimitedAt === 0) w.rateLimitedAt = now + w.awaitingQuotaRetry = true + const waitMs = Math.max(0, ladder[n] - (now - w.rateLimitedAt)) + log("warn", `${short(sid)} rate limited (${errType || "error"}) — standing down, next attempt in ${Math.ceil(waitMs / 1000)}s (${n + 1}/${ladder.length})`) + // User-visible twin of the stand-down: silence reads as dead + // (ses_edd3d940). One notice per episode (first arm only); + // resume:false so it never kicks a turn or re-arms budgets. + if (!w.quotaNoticedAt) { + w.quotaNoticedAt = Date.now() + const mins = Math.max(1, Math.round(waitMs / 60000)) + const when = mins >= 60 ? `~${Math.floor(mins / 60)}h ${mins % 60}m` : `~${mins}m` + void sendQuotaNotice(sid, `Rate limited, retrying in ${when} (attempt ${n + 1}/${ladder.length}).`) + } + // Machine-readable arm record so a later evaluation can re-arm after a + // reload wipes this in-memory gate (ses_ef72c5f3, 2026-10-09: armed at + // 14:31:49, plugin reloaded 14:37:28, gate due 14:46:49 never fired). + // Full sid: short() truncates and suffixes can collide across sessions. + log("warn", `quota-armed sid=${sid} at=${now} rung=${n} ladder=${ladder.length}`) + // Mirror into storage so the arm survives a real process restart (R1). + void syncQuotaState(sid, w) + } + + /** Remaining plugin-initiated interrupts allowed in the current window. */ + function interruptBudget(w: SessionWatch): number { + if (w.interruptWindowStart === 0 || Date.now() - w.interruptWindowStart > INTERRUPT_WINDOW_MS) { + w.interruptWindowStart = Date.now() + w.interruptsThisWindow = 0 + } + return MAX_INTERRUPTS_PER_WINDOW - w.interruptsThisWindow + } + + /** + * Fallback source of truth for "which sessions are running". v2's plugin + * SessionDomain does not expose `session.active()` (see Mte90/opencode-auto-resume#33), + * so when it is missing we derive the set from our own event-derived busy flags. + * Without this the subagent-wait guard below can never fire. + */ + function busySessions(): string[] { + const out: string[] = [] + for (const [sid, w] of sessions) { + if (w.status === "busy" && !w.userCancelled) out.push(sid) + } + return out + } + + /** + * Loopback session list with parentID rows. `ctx` offers no list/get/ + * active surface and no client in production (verified 2026-10-06: + * every subagent check silently degraded to false, zero refusals in + * log history) — but `GET /api/session` answers with parentID rows + * under the same auth as the message reads. Short-TTL cache: verdicts + * and guards call this several times per watchdog pass. Fail-open + * toward the pre-guard behavior (empty) on any fetch problem. + */ + type SessionRow = { id?: unknown; parentID?: unknown } + let sessionRowsCache: { at: number; rows: SessionRow[] } | null = null + const SESSION_ROWS_TTL_MS = 10_000 + async function listSessionsHttp(): Promise { + const now = Date.now() + if (sessionRowsCache && now - sessionRowsCache.at < SESSION_ROWS_TTL_MS) return sessionRowsCache.rows + for (const base of serverBaseUrls()) { + try { + const res = await fetch(`${base}/api/session?limit=500`, { headers: serverHeaders() }) + if (!res.ok) continue + const body = (await res.json()) as unknown + const data = (body as { data?: unknown } | undefined)?.data ?? body + if (Array.isArray(data)) { + const rows = (data as SessionRow[]).filter((r) => r !== null && typeof r === "object") + sessionRowsCache = { at: now, rows } + return rows + } + } catch { + // Next base; exhaustion degrades to stale/empty below. + } + } + return sessionRowsCache?.rows ?? [] + } + + /** + * Is this session itself a subagent? Definitive test: ask the server for + * our own record and look at `parentID`. + * + * This plugin is parent-scoped — it exists to recover sessions a human is + * waiting on. A child is not that: it never reads the parent's AGENTS.md, + * so a prompt-level "ignore injected continues" rule in the child agent + * loses to an injection that arrives as a real task turn. Observed in the + * wild: a worker answered an invisible prompt mid-task and never returned. + * + * Deliberately independent of the `lastWasTaskTool` heuristic used for + * parents in `checkActiveSessions` — that one asks "did this session just + * dispatch a child?", this one asks "is this session a child?". A silent + * child gets no protection from the parent-side heuristic, which is + * exactly the gap this closes. + * + * Cached on the watch record. `ctx.client` is optional and any failure + * degrades to `false` — i.e. precisely the pre-guard behavior — so a host + * without a client can neither break recovery nor fail to boot. + */ + async function isSubAgentSession(sid: string): Promise { + const w = ensureWatch(sid) + if (typeof w.isSubAgent === "boolean") return w.isSubAgent + let sub = false + try { + const res = await ctx.client?.session.get({ path: { id: sid } }) + sub = !!res?.data?.parentID + } catch { + sub = false + } + if (!sub) { + // HTTP fallback: prod ctx has no client (verified 2026-10-06), + // so the lookup above always misses there. Same degraded-false + // contract on any fetch problem. + try { + const rows = await listSessionsHttp() + sub = rows.some((r) => r?.id === sid && typeof r?.parentID === "string") + } catch { + sub = false + } + } + w.isSubAgent = sub + return sub + } + + /** + * Single choke point for every recovery injection. Guarantees at most one + * synthetic per `injectIntervalMs` and refuses to talk to a live session, + * because a turn sent to a busy session supersedes its in-flight step + * ("Step interrupted"). + * + * `allowDuringSelfAbort` is for the abort+resume escalation, which by + * construction runs inside the plugin's own abort window: it is the one + * injection that is *supposed* to follow `interrupt()`. That path skips + * both the abort-window check and the busy guard, because: + * - we just killed the step ourselves, so "this session is working" is + * false, and the runtime may not have delivered the idle transition + * yet; and + * - swallowing the continue there leaves a session interrupted with + * nothing to restart it, which is the failure this whole fix targets. + * It stays rate-limited by `w.aborting` and `MAX_INTERRUPTS_PER_WINDOW`. + */ + /** + * Cross-instance duplicate check: is our exact text already the latest + * user message in the session log, posted within RECENT_PROD_WINDOW_MS? + * Stacked watchdogs (reload churn) each hold private counters, so the + * in-memory soft skip cannot see a sibling's prod — the log can. + * Fail-open: any fetch problem means "no info", never "duplicate". + */ + const RECENT_PROD_WINDOW_MS = 60_000 + async function recentOwnProdInLog(sid: string, text: string): Promise { + const pages: unknown[] = [] + try { + const message = (ctx.client as any)?.session?.message + if (typeof message?.list === "function") { + const res = await message.list.call(message, { path: { id: sid } }) + const data = (res as { data?: unknown } | undefined)?.data ?? res + if (Array.isArray(data)) pages.push(...data) + } + } catch { + // Fall through to HTTP below. + } + if (pages.length === 0) { + for (const base of serverBaseUrls()) { + try { + const res = await fetch( + `${base}/api/session/${encodeURIComponent(sid)}/message?limit=20`, + { headers: serverHeaders() }, + ) + if (!res.ok) continue + const body = ((await res.json()) as { data?: unknown } | undefined)?.data + if (Array.isArray(body)) { + pages.push(...body) + break + } + } catch { + // Next candidate base. + } + } + } + if (pages.length === 0) return false + const now = Date.now() + for (const m of pages) { + if (m === null || typeof m !== "object") continue + const msg = m as { role?: unknown; text?: unknown; content?: unknown; time?: unknown } + if (msg.role !== "user") continue + let t = "" + if (typeof msg.text === "string") { + t = msg.text + } else if (Array.isArray(msg.content)) { + t = (msg.content as unknown[]) + .filter( + (p): p is { type: string; text: string } => + !!p && + typeof p === "object" && + (p as { type?: unknown }).type === "text" && + typeof (p as { text?: unknown }).text === "string", + ) + .map((p) => p.text) + .join("") + } + // Newest-first: the first user message decides. Anything newer + // than our prod (user typed after it) is progress, not a dupe. + if (t !== text) return false + const created = (msg.time as { created?: unknown } | undefined)?.created + if (typeof created === "number" && now - created > RECENT_PROD_WINDOW_MS) return false + return true + } + return false + } + + /** + * Per-session async mutex around the whole injection. Bun is + * single-threaded, so a promise chain serializes overlapping callers: + * stacked watchdogs interleave at every await, and without this the + * log check below runs six-wide against the same empty log before any + * send lands. First holder checks-then-sends; the rest re-check after + * and stand down on sight of the first prod. + */ + const injectLocks = new Map>() + async function withInjectLock(sid: string, fn: () => Promise): Promise { + const prev = injectLocks.get(sid) ?? Promise.resolve() + let release!: () => void + const current = new Promise((resolve) => { + release = resolve + }) + // The chain tail is what the NEXT caller waits on, so it must resolve only + // after this holder's work finishes — hence `prev.then(() => current)`. + const tail = prev.then(() => current) + injectLocks.set(sid, tail) + await prev + try { + return await fn() + } finally { + release() + // Drop the key once we are the last holder, so a long-lived process + // does not accumulate one entry per session it ever watched. + if (injectLocks.get(sid) === tail) injectLocks.delete(sid) + } + } + + /** + * The injection body. Split out of `injectOnce` so the per-session mutex + * wraps the WHOLE check-then-act: every guard above (subagent, self-abort, + * busy, debounce, duplicate) is a read, and the send is the write. Stacked + * callers interleave at every `await`, so six watchdogs could each pass all + * six reads against pre-send state and then all six send. Serialized here, + * the first caller to finish sends and the rest re-read its result. + */ + async function injectOnceLocked( + sid: string, + text: string, + notification: string, + allowDuringSelfAbort: boolean, + checkDuplicate: boolean, + forceSynthetic = false, + ): Promise { + // Dead instances stay silent across every injection path, not + // just the stall timer that schedules most of them. Checked + // before ensureWatch so a dead copy cannot resurrect watch state. + if (!running) return false + const w = ensureWatch(sid) + // A subagent is not ours to recover. Checked first, before every other + // guard, so no code path below can reach a child. + if (await isSubAgentSession(sid)) { + dbg(`${short(sid)} subagent session — injection refused (parent owns recovery)`) + return false + } + // The permanent and OOC locks are enforced HERE rather than in each + // caller. There are five injection paths (stall, abort+resume, unknown-tool, + // task_complete-blocked, targetedRecovery) and a permanently failed or + // out-of-context session still goes idle with its history intact — so the + // feature passes read that history and injected into a model that cannot + // answer at all (observed: the unknown-tool suggestion landing on a session + // whose provider had just latched). The stall callers already checked; this + // closes the other three and every future path. + { + const kind = lockKind(w) + if (kind) { + dbg(`${short(sid)} locked (${kind}) — injection refused on every path`) + return false + } + } + if (!allowDuringSelfAbort && selfAbortActive(w)) { + dbg(`${short(sid)} injection refused — inside our own abort window`) + return false + } + if (!allowDuringSelfAbort && w.status === "busy" && Date.now() - w.lastActivityAt < chunkTimeoutMs) { + dbg(`${short(sid)} injection refused — session busy and live (${Math.round((Date.now() - w.lastActivityAt) / 1000)}s since last event)`) + return false + } + const since = Date.now() - w.lastInjectAt + if (!allowDuringSelfAbort && w.lastInjectAt > 0 && since < injectIntervalMs) { + dbg(`${short(sid)} injection debounced — ${Math.round(since / 1000)}s since last (min ${injectIntervalMs / 1000}s)`) + return false + } + // Exact-duplicate anti-repeat (cattleprod-style soft skip, opt-in per + // caller): an immediate re-fire with identical text and zero model + // progress since the last successful prod means the previous nudge + // changed nothing — skip it and let the sliding-window counter / + // escalation own the loop. No message fetch needed: progress is + // assistant-text growth plus tool completions. Scoped to the stall + // path by default: the targeted nudges below own per-kind budgets + // with re-arm semantics this must not second-guess. + if ( + checkDuplicate && + !allowDuringSelfAbort && + w.lastProdText !== "" && + text === w.lastProdText && + w.pendingTools <= 0 && + w.completedSinceProd === 0 && + w.lastAssistantText === w.prodAssistantSnapshot + ) { + dbg(`${short(sid)} duplicate continue suppressed — identical text, no progress since last prod`) + return false + } + w.lastInjectAt = Date.now() + // Cross-instance backstop for the in-memory skip above: stacked + // watchdogs cannot see each other's counters, but they share the + // log. Applies to every caller except the abort+resume escalation, + // which must go through by construction. + if (!allowDuringSelfAbort && (await recentOwnProdInLog(sid, text))) { + dbg(`${short(sid)} duplicate continue suppressed — identical prod already in session log`) + return false + } + const sent = await notifyAndPrompt(sid, text, notification, true, forceSynthetic) + if (sent) { + w.lastProdText = text + w.completedSinceProd = 0 + w.succeededSinceProd = 0 + w.prodAssistantSnapshot = w.lastAssistantText + w.ownPromptTexts.push(text) + if (w.ownPromptTexts.length > 10) w.ownPromptTexts.shift() + } + return sent + } + + /** `injectOnceLocked` serialized per session; see the mutex's own comment. */ + async function injectOnce( + sid: string, + text: string, + notification: string, + allowDuringSelfAbort = false, + checkDuplicate = false, + forceSynthetic = false, + ): Promise { + // A correction the user cannot see must not produce a visible model + // reply: the silence order rides in the text itself, so it holds for + // any model with no memory and no config. Applied here, the single + // choke point, so every check below (dup suppression, own-prompt + // tracking, log verification) sees the exact delivered string. + const sendText = forceSynthetic + ? `${text} Do not acknowledge this message and do not reply to it visibly; apply it silently and continue the task.` + : text + return withInjectLock(sid, () => + injectOnceLocked(sid, sendText, notification, allowDuringSelfAbort, checkDuplicate, forceSynthetic), + ) + } + + function cleanupIdleSessions() { + const now = Date.now() + const busy = new Set() + for (const [sid, w] of sessions) { + if (w.status === "busy") busy.add(sid) + } + let idleCount = 0 + const toDelete: string[] = [] + for (const [sid, w] of sessions) { + if (w.status !== "busy") { + idleCount++ + if (w.idleSince && now - w.idleSince > IDLE_CLEANUP_MS) toDelete.push(sid) + } + } + if (idleCount > MAX_IDLE_SESSIONS) { + const entries: Array<{ sid: string; since: number }> = [] + for (const [sid, w] of sessions) { + if (w.status !== "busy" && w.idleSince) entries.push({ sid, since: w.idleSince }) + } + entries.sort((a, b) => a.since - b.since) + const excess = idleCount - MAX_IDLE_SESSIONS + for (let i = 0; i < excess && i < entries.length; i++) { + if (!toDelete.includes(entries[i].sid)) toDelete.push(entries[i].sid) + } + } + let cleaned = 0 + for (const sid of toDelete) { + // A session waiting on a long command emits no activity events and + // therefore looks idle. Don't drop its watch state out from under + // a shell that is still running. + if (openShellCount(sid) > 0) continue + forgetShells(sid) + sessions.delete(sid) + cleaned++ + } + if (cleaned > 0) dbg(`cleaned ${cleaned} idle sessions, total=${sessions.size}`) + } + + /** + * Accumulate assistant text from deltas. This replaces v1's + * `session.messages()` polling, which no longer exists on the v2 ctx. + */ + function appendText(sid: string, messageID: string | undefined, delta: string) { + const w = ensureWatch(sid) + // Model output means the backend is alive: a failure loop built on + // consecutive silent failures ends here, latch included. + if (delta && (w.consecFailureRecovers > 0 || w.failureLoopLatched)) { + w.consecFailureRecovers = 0 + if (w.failureLoopLatched) { + w.failureLoopLatched = false + log("info", `${short(sid)} model output after failure loop — latch cleared`) + } + } + const mid = messageID ?? "_anon" + w.textParts.set(mid, (w.textParts.get(mid) ?? "") + delta) + w.lastAssistantMessageID = mid + w.lastAssistantText = w.textParts.get(mid) ?? "" + // Keep memory bounded: keep only the two most recent messages' text + if (w.textParts.size > 2) { + const oldest = w.textParts.keys().next().value + if (oldest !== undefined && oldest !== mid) w.textParts.delete(oldest) + } + if (w.lastAssistantText.length > TEXT_BUFFER_TRIM_LEN) { + w.textParts.set(mid, w.lastAssistantText.slice(-TEXT_BUFFER_TRIM_LEN)) + w.lastAssistantText = w.textParts.get(mid) ?? "" + } + } + + // --------------------------------------------------------------------- + // Recovery actions + // --------------------------------------------------------------------- + + /** + * Show a visible notification in the session timeline and optionally + * resume it. Uses `session.synthetic()` which is exposed on the v2 + * promise-plugin context — the synthetic message appears in the TUI + * so the user knows the plugin intervened. + * + * When `resume` is true the synthetic also acts as a user turn that + * kicks the session back to life, replacing the separate `prompt()`. + */ + /** + * Bound a recovery send. Transport awaits with no deadline stack + * cascades that flush together as twin prods (ses_ee7dade1) — bound + * them so expiry flows into the normal throw path instead. Slow + * completions log a latency warning; that timing is the forensics + * the twins lacked. + */ + async function sendBound(label: string, sid: string, send: () => Promise): Promise { + const started = Date.now() + let timer: ReturnType | undefined + try { + await Promise.race([ + send(), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out after ${sendTimeoutMs}ms`)), sendTimeoutMs) + }), + ]) + } finally { + if (timer !== undefined) clearTimeout(timer) + } + const elapsed = Date.now() - started + if (elapsed > SEND_SLOW_MS) { + log("warn", `${short(sid)} ${label} took ${Math.round(elapsed / 1000)}s (over ${SEND_SLOW_MS / 1000}s slow threshold)`) + } + } + + async function notifyAndPrompt(sid: string, text: string, notification: string, resume = true, forceSynthetic = false): Promise { + ensureWatch(sid).selfRecovery = true + if (visibleContinue && !forceSynthetic) { + // Cattleprod-style: a real user message in session history, so the + // intervention — and any loop — is self-evident in the transcript. + // Falls back to synthetic only if the prompt call throws. + try { + await sendBound("prompt", sid, () => ctx.session.prompt({ sessionID: sid, text })) + return true + } catch (err) { + const msg = err instanceof Error ? err.message : String(err) + log("warn", `${short(sid)} visible prompt failed: ${msg}`) + // A transport error can land AFTER the message was delivered + // (observed: twin identical prods 5ms apart). The delivered + // prompt is a user message in the log, so verify before + // falling back — a second send of the same text is the dupe. + if (await recentOwnProdInLog(sid, text)) { + dbg(`${short(sid)} prompt threw but the text is already the newest user message — treating as delivered`) + return true + } + // Dead instances stay silent: a superseded setup's in-flight + // send may reject after disposal — the live instance owns the + // session now, and any fallback from here would double-send. + if (!running) return false + } + } + try { + await sendBound("synthetic", sid, () => + ctx.session.synthetic({ + sessionID: sid, + text, + description: `auto-resume: ${notification}`, + resume, + }), + ) + return true + } catch (err) { + const msg = err instanceof Error ? err.message : String(err) + log("warn", `${short(sid)} synthetic failed: ${msg}`) + if (!running) return false + // Same verify-before-retry: a delivered synthetic also acts as + // a user turn, so it reads back through the same log check. + if (await recentOwnProdInLog(sid, text)) { + dbg(`${short(sid)} synthetic threw but the text is already the newest user message — treating as delivered`) + return true + } + // Last resort: a plain prompt still resumes the session (just + // without the visible synthetic notification in the TUI). + try { + await sendBound("prompt", sid, () => ctx.session.prompt({ sessionID: sid, text })) + return true + } catch { + log("error", `${short(sid)} all recovery attempts failed: ${msg}`) + return false + } + } + } + + async function tryAbortAndResume(sid: string, w: SessionWatch): Promise { + // Guarded separately because this path calls `interrupt()` *before* + // delegating to injectOnce — guarding only the injection would still + // let us interrupt a running child. + if (await isSubAgentSession(sid)) { + dbg(`${short(sid)} subagent session — abort+resume refused`) + return false + } + if (w.aborting || selfAbortActive(w)) return false + { + const kind = lockKind(w) + if (kind) { + dbg(`${short(sid)} ${kind} locked — refusing abort+resume escalation`) + return false + } + } + if (w.compacting) { + dbg(`${short(sid)} mid-compaction — refusing abort+resume escalation`) + return false + } + if (interruptBudget(w) <= 0) { + w.continueTimestamps = [] + log("warn", `${short(sid)} interrupt budget exhausted (${MAX_INTERRUPTS_PER_WINDOW}/${INTERRUPT_WINDOW_MS / 60000}m) — skipping interrupt, loop guard reset`) + return false + } + w.aborting = true + w.selfAbortAt = Date.now() + w.interruptsThisWindow++ + log("warn", `${short(sid)} escalating: interrupt + fresh continue (${w.interruptsThisWindow}/${MAX_INTERRUPTS_PER_WINDOW} per ${INTERRUPT_WINDOW_MS / 60000}m)`) + try { + await ctx.session.interrupt({ sessionID: sid }) + } catch (err) { + const msg = err instanceof Error ? err.message : String(err) + log("warn", `${short(sid)} interrupt failed: ${msg}`) + } + // Give the runtime a beat to settle the interrupted turn. + // A superseded setup must not resume after this gap: liveness is + // re-checked because disposal can land mid-sleep. + await new Promise((r) => setTimeout(r, 2_000)) + if (!running) return false + w.aborting = false + w.resumeAttempts = 0 + // The escalation is an observable new episode, so it also re-opens + // a stood-down session. Unreachable with gaveUp set today (every + // caller is gaveUp-guarded), but the rule belongs here so a future + // caller cannot escalate into a latch that blocks follow-up. + w.gaveUp = false + // Same observable-episode rule for the empty-idle single-shot. + w.emptyIdleNudged = false + // A genuine turn is also a new counting episode. + w.episodeRecovers = 0 + // Deliberately does not recordContinue(): our own escalation must not feed + // the loop guard that triggered it, or the guard trips on its own output. + // Through the same choke point as every other path. The runtime + // delivers session.execution.interrupted asynchronously, so an + // escalation that lands on top of a concurrent recovery injection is + // exactly the case that produced the simultaneous bursts of synthetic + // continues (and the "Step interrupted" they caused). + const ok = await injectOnce(sid, opts.continuePrompt ?? SAFE_CONTINUE_PROMPT, "abort+resume escalation", true) + if (ok) { + w.lastRetryAt = Date.now() + log("info", `${short(sid)} resumed after abort`) + } + return ok + } + + /** Classify one failure as permanent or not. Type first (opencode's own + * taxonomy), then status, then message — the type is the strongest signal + * and the message the weakest, so an unknown type carrying a known message + * still gates. Returns the reason for the log line, or null. */ + function permanentFailureReason(errMsg: string, errType?: string, status?: unknown): string | null { + if (errType && PERMANENT_ERROR_TYPES.has(errType)) return `type ${errType}` + if (typeof status === "number" && PERMANENT_STATUS_CODES.has(status)) return `http ${status}` + if (PERMANENT_GONE_RE.test(errMsg)) return errMsg.slice(0, 160) + if (PERMANENT_HISTORY_RE.test(errMsg)) return errMsg.slice(0, 160) + if (PERMANENT_UNKNOWN_MODEL_RE.test(errMsg)) { + if (status === 404) return null // openai 404s can be live models; keep recovering + return errMsg.slice(0, 160) + } + return null + } + + /** Latch the permanent-failure lock when the error names something a continue + * can never clear. Returns true when recovery must be skipped this time. */ + function maybeLockPermanent(sid: string, errMsg: string, errType?: string, status?: unknown): boolean { + const w = ensureWatch(sid) + if (w.permLocked) return true + let why = permanentFailureReason(errMsg, errType, status) + if (why === null && errType && INDETERMINATE_ERROR_TYPES.has(errType)) { + // Deliberately AFTER the message patterns: an unsupported-operation + // type carrying a permanent message (a deprecated model, a dead + // session id) still hard-gates on the message. + w.indeterminateStrikes++ + if (w.indeterminateStrikes < INDETERMINATE_STRIKES) { + log( + "info", + `${short(sid)} indeterminate failure (${errType}) ${w.indeterminateStrikes}/${INDETERMINATE_STRIKES} — one recovery attempt before latching mod=${MODULE_INSTANCE}`, + ) + return false + } + why = `type ${errType} (${INDETERMINATE_STRIKES} strikes)` + } + if (why === null) return false + w.permLocked = true + w.permLockReason = why + log( + "warn", + `${short(sid)} permanent failure latched (${why}) — recovery locked out until a genuine turn: ${errMsg.slice(0, 120)} mod=${MODULE_INSTANCE}`, + ) + return true + } + + /** Central lock query: which lock (if any) holds this watch. + * Every injection-path guard calls this instead of testing the two flags + * inline, so a third lock class means editing here, not N call sites. */ + function lockKind(w: SessionWatch): "permanent" | "ooc" | null { + if (w.permLocked) return "permanent" + if (w.oocLocked) return "ooc" + return null + } + + /** Human-readable lock reason for log lines. */ + function lockReason(w: SessionWatch): string { + const kind = lockKind(w) + if (kind === "permanent") return w.permLockReason?.slice(0, 80) ?? "n/a" + if (kind === "ooc") return w.oocLockReason?.slice(0, 80) ?? "n/a" + return "n/a" + } + + /** Central lock release, called on a genuine turn. */ + function clearLocks(w: SessionWatch, sid: string): void { + if (w.oocLocked) { + w.oocLocked = false + w.oocLockReason = null + log("info", `${short(sid)} genuine execution — clearing OOC lock mod=${MODULE_INSTANCE}`) + } + w.indeterminateStrikes = 0 + if (w.permLocked) { + w.permLocked = false + w.permLockReason = null + log("info", `${short(sid)} genuine execution — clearing permanent-failure lock mod=${MODULE_INSTANCE}`) + } + w.consecFailureRecovers = 0 + if (w.failureLoopLatched) { + w.failureLoopLatched = false + log("info", `${short(sid)} genuine execution — clearing failure-loop latch mod=${MODULE_INSTANCE}`) + } + } + + function maybeLockOoc(sid: string, errMsg: string, errType = "") { + const w = ensureWatch(sid) + if (w.oocLocked) return + // Transient throttles are not context overflow (issue #46) — the + // rate-limit ladder owns them. Checked here (not just at the + // execution.failed call site) so session.retry.scheduled is covered too. + if (isRateLimitError(errType, errMsg)) return + if (!OOC_ERROR_RE.test(errMsg)) return + w.oocLocked = true + w.oocLockReason = errMsg.slice(0, 200) + log("warn", `${short(sid)} OOC error latched — recovery locked out until a genuine turn: ${errMsg.slice(0, 120)} mod=${MODULE_INSTANCE}`) + } + + /** + * Build the stall-continue text. A custom `continuePrompt` always wins + * verbatim; otherwise, when `richContinuePrompt` is on (default), the + * text names the stall reason, attempt count, and remaining todos + * (cattleprod-style) instead of bare "continue". Same channel and + * guards either way — only the content changes. + */ + async function buildStallContinueText(sid: string, reason: string, attempt: number): Promise { + const w = ensureWatch(sid) + if (opts.continuePrompt !== undefined && opts.continuePrompt !== "") return opts.continuePrompt + if (!richContinuePrompt) return CONTINUE_PROMPT + let suffix = "Todo list unavailable." + try { + const todos = await readTodos(sid) + const open = todos.filter((t) => t.status !== "completed" && t.status !== "cancelled") + if (open.length > 0) { + const lines = open + .slice(0, 5) + .map((t) => `• [${t.status}] ${String(t.content).slice(0, 120)}`) + if (open.length > 5) lines.push(`• …and ${open.length - 5} more`) + suffix = `Remaining todos:\n${lines.join("\n")}` + } else { + suffix = "No open todos recorded." + } + } catch { + // Best effort: a todo-read failure must never block recovery. + } + // User-facing timestamp in the operator's own timezone (system + // local), not UTC: "2026-10-10T09:12Z" reads as wrong o'clock next + // to a 05:12 EDT wall clock (ses_ed9e2ef9, 2026-10-10). Log lines + // and boot markers stay ISO — machines read those, people read this. + const now = new Date().toLocaleString("en-US", { + month: "short", + day: "numeric", + hour: "numeric", + minute: "2-digit", + timeZoneName: "short", + }) + const elapsed = w.lastActivityAt ? Math.round((Date.now() - w.lastActivityAt) / 1000) : 0 + const hms = (n: number) => + n < 60 ? `${n}s` : n < 3600 ? `${Math.floor(n / 60)}m ${n % 60}s` : `${Math.floor(n / 3600)}h ${Math.floor((n % 3600) / 60)}m` + // Each fact exactly once: a "no activity for Ns" reason already states + // the silence, so adding "idle M:SS" repeats it (188s vs 3m 9s). And + // idle is only news for silence stalls: a dead stream is seconds old + // by construction and a failure just happened, so in both cases the + // field reads as noise next to the reason. Timestamp always stays. + // (Predicate hoisted to module scope: recover reuses it for the + // success-progress episode reset.) + const isSilence = isSilenceReason(reason) + // The retry count is the episode's recovery number, not the ladder + // rung: after N recoveries the next fire reads "attempt 1/3" just + // the same when progress restarted the rung. Without it every fire + // in a filler loop reads identically and the numbers cannot be + // reconciled from the transcript. + const head = [ + isSilence ? null : reason, + `attempt ${attempt}/${maxRetries} (retry #${w.episodeRecovers})`, + isSilence ? `idle ${hms(elapsed)}` : null, + now, + ] + .filter((x): x is string => typeof x === "string" && x.length > 0) + .join("; ") + return (`continue — stalled (${head}).\nOnly continue work already in progress. ` + + `Do not start anything new, and do not take destructive or irreversible actions without explicit user approval.\n${suffix}`).slice(0, 2000) + } + + /** + * Core recovery ladder for a stuck/failed session. + * plain continue with backoff -> more attempts -> abort+resume escalation. + */ + async function recover(sid: string, reason: string) { + const w = ensureWatch(sid) + if (w.recovering || w.aborting || w.gaveUp || w.userCancelled || w.permissionPending) return + if (w.failureLoopLatched) { + dbg(`${short(sid)} failure loop latched — standing down until a genuine turn or fresh model output`) + return + } + // Failure-driven recoveries with no model output between them mean the + // backend is down, not the session stalled: each injected continue + // manufactures the next failure. Count them; past the max, latch + // instead of cycling the ladder forever. + if (reason.startsWith("session.step.failed") || reason.startsWith("session.execution.failed")) { + w.consecFailureRecovers++ + if (w.consecFailureRecovers > FAILURE_LOOP_MAX) { + w.failureLoopLatched = true + log( + "warn", + `${short(sid)} ${w.consecFailureRecovers} consecutive failure recoveries with no model output — backend likely down, standing down until a genuine turn mod=${MODULE_INSTANCE}`, + ) + return + } + } + { + const kind = lockKind(w) + if (kind) { + dbg(`${short(sid)} ${kind} locked — refusing recovery (reason: ${lockReason(w)})`) + return + } + } + if (w.compacting) { + dbg(`${short(sid)} mid-compaction — refusing recovery (reason: ${reason})`) + return + } + if (selfAbortActive(w)) { + dbg(`${short(sid)} self-abort in flight — refusing recovery (reason: ${reason})`) + return + } + // Evaluate the loop guard BEFORE recording this continue, so the plugin + // never escalates on continues it injected itself. + if (continuesInWindow(w) >= loopMaxContinues) { + log("warn", `${short(sid)} hallucination loop (${loopMaxContinues} continues/${loopWindowMs / 1000}s) — abort+resume`) + await tryAbortAndResume(sid, w) + w.continueTimestamps = [] + return + } + // A stall after fresh model output is a NEW episode, not a deeper + // one: restart the ladder so it reads attempt 1/3 again + // (ses_ee7dade1 — 2/3 landed after real work). Assistant TEXT growth + // always counts. Successful tool/shell work counts only for + // silence-triggered stalls (ses_ede9b725 — 1/3 woke the model, it + // read a file, then went quiet): a wake-up that produced work is a + // new episode. Failure/dead-stream triggers never reset on tool + // progress (ses_edf403f5 — tool completion between dead-stream + // stalls must keep climbing toward gaveUp, else every cycle reads + // 1/3 forever). gaveUp itself is left alone: only a fresh turn + // (markBusy) clears that latch. + if (w.lastProdText !== "" && w.lastAssistantText !== w.prodAssistantSnapshot) { + // Every reset logs: silent restarts hid the endless-1/3 loop + // for weeks (ses_ef3a2ba2 — resets visible only once in six + // fires). Observability before economy here. + log("info", `${short(sid)} text growth since last prod — restarting the stall ladder`) + // Only count mid-ladder restarts: a first fire after genuine + // work (attempts still 0) is a fresh episode, not a refire. + if (w.resumeAttempts > 0) w.episodeRefires++ + w.resumeAttempts = 0 + } else if (isSilenceReason(reason) && w.succeededSinceProd > 0) { + log("info", `${short(sid)} progress since last prod — restarting the stall ladder`) + if (w.resumeAttempts > 0) w.episodeRefires++ + w.resumeAttempts = 0 + } + if (lifetimeExhausted(sid, w)) return + // A failure the provider is already retrying is not model + // unresponsiveness: hold the rung, and never give up on it. Otherwise a + // transport fault climbs the ladder into gaveUp, which is terminal + // silence (the sweep is gated on !gaveUp and only a genuine user + // message clears it). ses_eae46bb9, 2026-10-19: 3x ECONNRESET, each + // answered by opencode's own `provider retry #2`, drove 1/3 -> 2/3 -> + // restart -> 1/3 and was one fire from standing the session down. The + // prod below still fires: a dead stream needs waking even when the + // provider is coming back. + const providerFault = + isProviderRetryFresh(w) && + (reason.startsWith("session.step.failed") || reason.startsWith("session.execution.failed")) + // The rung is committed only when the prompt below actually lands (see + // the inject timer), because counting at detection burned rungs on the + // injections the busy/live guard declined (ses_eae46bb9, 2026-10-19). + if (!providerFault && w.resumeAttempts >= maxRetries) { + log("warn", `${short(sid)} giving up after ${maxRetries} attempts (${reason}) mod=${MODULE_INSTANCE}`) + w.gaveUp = true + return + } + w.recovering = true + const attempt = providerFault ? Math.max(1, w.resumeAttempts) : w.resumeAttempts + 1 + const delay = backoffMs(attempt, baseBackoff, maxBackoff) + if (providerFault) { + log( + "info", + `${short(sid)} provider-fault stall (${reason}) — retry scheduled ${Math.round((Date.now() - w.providerRetryAt) / 1000)}s ago, holding the ladder at ${attempt}/${maxRetries}`, + ) + } + log( + "info", + `${short(sid)} stall detected (${reason}) — resume attempt ${attempt}/${maxRetries} in ${delay}ms`, + ) + if (w.recoverTimer) clearTimeout(w.recoverTimer) + // When this timer was armed: a genuine user message arriving after + // this point means the user took over mid-backoff, and firing into + // their live turn would interrupt it ("Step interrupted"). The + // freshness check below cannot see it — event-pump lag means their + // execution.started may not have been processed yet when we fire. + const armedAt = Date.now() + // Whether the prompt this timer sends actually lands. The ladder rung is + // committed only when it did — see the ok branch below. + let delivered = false + w.recoverTimer = setTimeout(async () => { + w.recoverTimer = null + // Disposed instances must stay silent: a superseded setup's + // backoff outlives its own dispose (which is exactly how twin + // identical prods landed milliseconds apart). + if (!running) return + try { + // Skip only if the session genuinely turned healthy again + // (a normal completion clears pendingRecoveryArmed) or the + // user took over. + if (w.userCancelled || w.gaveUp) return + // A genuine user message newer than the arm means the user took + // over during the backoff wait — park this recovery, do not fire + // into their turn. Own prompts never advance lastGenuineUserAt, + // so our own injections cannot trip this. (2026-10-11: continues + // landing right after user statements.) + // >=, not >: a message processed in the SAME sweep as the + // declaration carries the sweep's own timestamp, so strict > + // would let it through and fire into the user's live turn + // within the same minute (2026-10-12: comment 19:27, fire + // 19:28). Equal timestamps mean same-sweep causality. + if (w.lastGenuineUserAt >= armedAt) { + dbg(`${short(sid)} genuine user message after arm — parking recovery`) + return + } + if (w.status === "idle" && !w.pendingRecoveryArmed) return // recovered by itself meanwhile + // A turn sent to a busy session interrupts its in-flight step + // ("Step interrupted"). Never interrupt mid-compaction, and + // never interrupt a busy session that is still live (events + // within the chunk window): it is working, not stalled. The + // watchdog re-arms if it truly stalls. + if (w.compacting) { + dbg(`${short(sid)} mid-compaction at inject time — not interrupting`) + return + } + if (w.status === "busy" && Date.now() - w.lastActivityAt < chunkTimeoutMs) { + dbg(`${short(sid)} still busy and live at inject time (${Math.round((Date.now() - w.lastActivityAt) / 1000)}s since last event) — not interrupting`) + return + } + // Re-check the parent-wait guard: a child may have gone live + // between detection and this delayed inject. Interrupting a + // parent that is waiting on a subagent kills real work. + if (await parentWaitingOnSubagents(sid, new Set(await getActiveSessions()))) { + dbg(`${short(sid)} waiting on subagents at inject time — not interrupting`) + return + } + // Same re-check for user input: a question asked between + // detection and this delayed inject parks the turn. + if (isWaitingOnUser(sid, w, await loadMessages(sid))) { + dbg(`${short(sid)} waiting on user input at inject time — not interrupting`) + return + } + // Count this fire now: the text below reads the count, and every + // return past this point still decided to recover (declines all + // returned earlier), so the number matches injections decided. + w.episodeRecovers++ + const stallText = await buildStallContinueText(sid, reason, attempt) + const ok = await injectOnce(sid, stallText, "stalled — retrying", false, true) + w.pendingRecoveryArmed = false + if (ok) { + delivered = true + // Commit the rung only now that the prompt landed: the text + // above already carried `attempt`, and every decliner above + // returned before this point, so resumeAttempts counts + // delivered prods only (ses_eae46bb9, 2026-10-19: a declined + // inject burned rung 1, so the next fire read 2/3 with no + // 1/3 ever delivered). A provider-fault fire holds its rung + // instead: a transport fault is not the model failing to + // continue. + if (!providerFault && attempt > w.resumeAttempts) w.resumeAttempts = attempt + recordContinue(sid) + w.lastRetryAt = Date.now() + touch(sid) + markBusy(sid) + } else if (attempt >= maxRetries) { + await tryAbortAndResume(sid, w) + } + } finally { + w.recovering = false + if (!delivered) { + dbg(`${short(sid)} attempt ${attempt}/${maxRetries} not delivered — rung not consumed (${reason})`) + } + } + }, delay) + } + + /** Targeted recovery prompts (tool-as-text, done-claims, intent nudges). */ + + /** + * Read the session's todo list. + * + * v1 tracked todos from a `todo.updated` event and a server API. v2 has neither: + * nothing emits a todo event and no route reaches the todo table. + * + * What v2 DOES have is the session message log. Every `todowrite` call is + * persisted as a tool part whose input carries the entire list, so the newest + * completed call IS the current list. That is the real store, so it is read first + * - reading anything else first is what made auto-resume conclude "no todos" + * while the list was sitting in the log. + * + * `ctx.storage.get("todos/")` is kept LAST, and it is NOT a source of + * another plugin's todos. `ctx.storage` is namespaced per plugin: the key + * resolves to `storage/plugin/auto-resume.v2/todos/.json`, while a todo + * plugin writing the same key lands in its OWN directory + * (`storage/plugin//todos/.json`). Verified against the live + * server: `/api/plugin/storage//` carries the plugin id as a + * path segment, and the on-disk tree is `storage/plugin//.json`. + * So no key auto-resume does not write itself can ever hold a todo list here. + * + * Do not promote it back to primary, and do not "fix" the mismatch by writing + * the key: a reader cannot make another plugin's data appear in its namespace, + * and duplicating that data would create a second writer racing the first. The + * message log above is the only store both plugins can agree on. + * + * Cached for a short TTL: this is read on every idle inspection, and the list can + * only change while the model is working, which is not when we ask. Returns `[]` on + * any failure - every caller treats "no list" as "cannot conclude", never as + * "nothing is open". + */ + async function readTodos(sid: string): Promise { + const w = ensureWatch(sid) + const now = Date.now() + if (w.todosFetchedAt && now - w.todosFetchedAt < TODO_CACHE_TTL_MS) return w.todos + const fromMessages = await todosFromMessageSources(sid) + if (fromMessages) { + w.todos = fromMessages + w.todosFetchedAt = now + return w.todos + } + if (!ctx.storage) return w.todos + try { + const raw = await ctx.storage.get(`todos/${sid}`) + const record = raw as { todos?: unknown; updatedAt?: unknown } | null | undefined + const list = Array.isArray(record?.todos) ? record.todos : [] + w.todos = list.filter( + (t): t is Todo => + !!t && typeof t === "object" && typeof (t as Todo).content === "string" && + typeof (t as Todo).status === "string", + ) + w.todosFetchedAt = now + return w.todos + } catch (e) { + dbg(`${short(sid)} todo read failed:`, e instanceof Error ? e.message : String(e)) + return w.todos + } + } + /** + * Try every plausible source of a session's messages, in order, returning the + * parsed todo list from the first that yields a `todowrite` tool part. + * + * Requires host access beyond the plugin ctx: `readFileSync` on + * `/proc/self/cmdline`, and a loopback HTTP GET. Both are best-effort - a host + * that refuses either degrades to the storage fallback rather than throwing. + */ + async function todosFromMessageSources(sid: string): Promise { + // 1. The SDK client, if this plugin ctx was given one. Unwrap the `{data,cursor}` + // envelope: handing it straight to the parser trips its Array.isArray guard and + // silently resolves nothing. + try { + const message = (ctx.client as any)?.session?.message + if (typeof message?.list === "function") { + const res = await message.list.call(message, { path: { id: sid } }) + const got = todosFromMessages(res?.data ?? res) + if (got) { + dbg(`${short(sid)} todos via ctx.client.session.message.list (${got.length})`) + return got + } + } + } catch (e) { + dbg(`${short(sid)} client message.list failed:`, e instanceof Error ? e.message : String(e)) + } + // 2. The local HTTP server. Same shape the prompt-polisher already uses, so the + // auth and base-URL derivation are proven rather than guessed. + for (const base of serverBaseUrls()) { + try { + // `limit` matters: the default page is 50 messages and 200 is the server's + // ceiling (probed - 250 returns 400). Newest first, so a recent `todowrite` is + // always inside the page. + const res = await fetch(`${base}/api/session/${encodeURIComponent(sid)}/message?limit=200`, { + headers: serverHeaders(), + }) + if (!res.ok) { + dbg(`${short(sid)} HTTP message read ${res.status} from ${base}`) + continue + } + // Same envelope trap as source 1: `{data, cursor}`, never a bare array. + const body = await res.json() + const got = todosFromMessages(body?.data ?? body) + if (got) { + dbg(`${short(sid)} todos via HTTP ${base} (${got.length})`) + return got + } + } catch (e) { + dbg(`${short(sid)} HTTP message read failed:`, e instanceof Error ? e.message : String(e)) + } + } + return undefined + } + + + /** + * Arm the deferred pattern pass, replacing any already pending. + * + * Replacement rather than stacking is the point: two idles inside one delay + * window would otherwise judge the same turn twice, and a turn that matches + * would spend two attempts of its budget on one piece of text. + */ + function schedulePatternPass(sid: string): void { + const w = ensureWatch(sid) + if (w.toolTextTimer) clearTimeout(w.toolTextTimer) + w.toolTextTimer = setTimeout(() => { + w.toolTextTimer = null + // v1's own guard, and it is doing real work here: a turn that started + // during the delay has its own idle, and its own armed pass. Judging + // this one now would read the live buffer, which markBusy has already + // emptied for the new turn. + if (w.status !== "idle") { + dbg(`${short(sid)} pattern pass skipped — session is ${w.status}, not idle`) + return + } + if (w.userCancelled) return + void inspectOnIdle(sid, "pattern") + }, toolTextCheckDelayMs) + dbg(`${short(sid)} pattern pass armed for +${toolTextCheckDelayMs}ms`) + } + + async function targetedRecovery( + sid: string, + kind: string, + prompt: string, + budgetKey: "toolTextAttempts" | "doneClaimAttempts" | "doneClaimOpenTodosAttempts" | "todoNudgeAttempts" | "intentNudgeAttempts" | "stuckToolAttempts", + ) { + const w = ensureWatch(sid) + if (w.recovering || w.userCancelled || w.permissionPending) return + // Locked sessions skip the work, not just the send. Checking only at the + // injection choke point (injectOnceLocked) still burns these budgets: the pass + // marks its parts examined and sets its one-shot flag BEFORE sending, so a + // refused nudge would leave the feature dead until the next user message. + // Bailing here keeps the budget intact for when the lock clears. + { + const locked = lockKind(w) + if (locked) { + dbg(`${short(sid)} locked (${locked}) — skipping ${kind} nudge`) + return + } + } + if (w.compacting) { + dbg(`${short(sid)} mid-compaction — skipping ${kind} nudge`) + return + } + // A session with a shell still running is working, not idle. A parent + // parked on a background job goes idle the moment the tool call returns + // — long before the process finishes — so without this it collects a + // nudge on the spot and the stall watchdog never even gets a look. + const busyShells = openShellCount(sid) + if (busyShells > 0) { + dbg(`${short(sid)} ${busyShells} shell(s) still running — skipping ${kind} nudge`) + return + } + if (selfAbortActive(w)) { + dbg(`${short(sid)} self-abort in flight — skipping ${kind} nudge`) + return + } + if (continuesInWindow(w) >= loopMaxContinues) { + log("warn", `${short(sid)} loop guard before ${kind} nudge — abort+resume`) + await tryAbortAndResume(sid, w) + w.continueTimestamps = [] + return + } + if (lifetimeExhausted(sid, w)) return + recordContinue(sid) + if (w[budgetKey] >= maxRetries) { + dbg(`${short(sid)} ${kind} budget exhausted`) + return + } + // Port of v1 e1b8374 ("Fix todoNudgeAttempts burning retries on failed + // sends"): only count an attempt once the prompt actually landed. A + // rejected send costs nothing, so a transient failure no longer eats a + // retry and silences the nudge for the rest of the session. + const attemptNum = w[budgetKey] + 1 + log("info", `${short(sid)} ${kind} detected — sending targeted prompt (${attemptNum}/${maxRetries})`) + w.recovering = true + // injectOnce debounces: this nudge counts once, not alongside a + // concurrent stall-watchdog injection. + const ok = await injectOnce(sid, prompt, "recovering: " + kind, false, false, quietCorrections) + w.recovering = false + if (ok) { + w[budgetKey] = attemptNum + touch(sid) + markBusy(sid) + } else { + dbg(`${short(sid)} ${kind} nudge not delivered — attempt ${attemptNum} not counted`) + } + } + + // --------------------------------------------------------------------- + // Idle-time forensics (runs once when a turn finishes) + // --------------------------------------------------------------------- + + /** + * Fetch a session's message history once per idle inspection. + * + * Four separate checks below want the same array — the dead-stream + * detector, the text fallback, the pending-tool and active-user lookups — + * and `ctx.session.context()` is the expensive call in all of them (it is + * every message since the last compaction). v1 fetched once for the same + * reason. The cache lives only for the duration of one inspection, so a + * later idle always sees the freshest history. + * + * Returns `[]` on any failure: this is forensic, and every caller already + * treats "no history" as "cannot conclude". + */ + async function loadMessages(sid: string): Promise { + try { + const res = await ctx.session.context({ sessionID: sid }) + const arr = Array.isArray(res) ? res : ((res as { messages?: unknown[] })?.messages ?? []) + // Canonical oldest-first order. Sources disagree: the message + // API returns newest-first (see the todosFromMessages order + // note), the harness feeds oldest-first, and every walk below + // reads the END as newest. Sort by timestamp so both shapes + // land the same way; stable sort keeps encounter order on ties + // and timeless messages sink to the front. ses_ed975114 + // (2026-10-10): the busy latch read the wrong end in prod and + // never engaged while every test stayed green. + if (arr.length > 1) { + const at = (m: unknown): number => { + const mm = m as { time?: { created?: number }; info?: { time?: { created?: number } } } + const t = mm?.time?.created ?? mm?.info?.time?.created + return typeof t === "number" ? t : 0 + } + return [...arr].sort((a, b) => at(a) - at(b)) + } + return arr + } catch (e) { + dbg(`${short(sid)} session.context() failed:`, e instanceof Error ? e.message : String(e)) + return [] + } + } + + /** The newest assistant message that delivered text, joined. "" when none. */ + function lastAssistantTextFrom(messages: unknown[]): string { + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i] as { + type?: string + role?: string + text?: string + info?: { content?: string } + content?: Array<{ type?: string; text?: string }> + } + // Prod history carries role, the harness carries type — msgRole reads both. +// ses_edbb4e67 (2026-10-10): type-only read returned "" in prod, +// so a "TASK COMPLETE" delivery never latched and the stall fired. + if (!msg || msgRole(msg) !== "assistant") continue + // Prod shape carries flat text; harness shape carries parts. + // Mirror userTextsOf: check both, or history reads go blind. + // ses_ed93e258 (2026-10-10): parts-only read missed flat-text + // deliveries and the latch never engaged despite the fix live. + if (Array.isArray(msg.content)) { + const text = msg.content + .filter((part) => part?.type === "text" && typeof part.text === "string") + .map((part) => part.text as string) + .join("") + if (text) return text + continue + } + const flat = + (typeof msg.text === "string" && msg.text) || + (typeof msg.info?.content === "string" && msg.info.content) || + "" + if (flat) return flat + } + return "" + } + + /** + * The newest assistant message's reasoning text, joined. "" when none. + * + * A reasoning block is the one place a model writes a tool call that never + * becomes one: the raw markup lands in the reasoning and nothing executes. + * v1 caught that on the same pass as the text variant and answered with a + * different prompt, because the fix is different — the model is not + * forgetting the mechanism, it is writing in the wrong channel. + * + * v2 separates the two cleanly, since `AssistantContent` tags reasoning and + * text distinctly where v1 filtered them together into one string. + */ + function lastAssistantReasoning(messages: unknown[]): string { + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i] as { + type?: string + role?: string + content?: Array<{ type?: string; text?: string }> + } + // Same prod/harness split as lastAssistantTextFrom: role vs type. + if (!msg || msgRole(msg) !== "assistant" || !Array.isArray(msg.content)) continue + const reasoning = msg.content + .filter((part) => part?.type === "reasoning" && typeof part.text === "string") + .map((part) => part.text as string) + .join("") + if (reasoning) return reasoning + } + return "" + } + + /** + * v1's `getLastSilentDeadStream`, on the v2 message shape. + * + * The model can finish a turn having produced no text at all — reasoning + * only, or a `finish=unknown` that the provider did not describe. The + * turn looks complete, so nothing raises, and the session simply stops. + * A `finish` with real text behind it means the session answered normally + * and there is nothing to recover. + * + * Only the newest assistant message that *has* a finish is judged, and the + * walk skips messages without one — an intermediate tool-call step has no + * finish, and walking back past a delivered answer to one of those would + * recover a session that just used a tool. + * + * Returns `null` when the newest finished message carried text — or when it + * carried tool calls, which are delivered work, not silence. A thinking + * model doing tool work ends turns with a finish reason, spent output + * tokens, and no text parts; judging that "silent" fires a continue + * into active work (ses_efaec2f99ffexosULGDJJ8i6sA, 2026-10-04). + * Returns `null` as well when there is no finished message at all. + */ + function lastSilentDeadStream(messages: unknown[]): { finish: string; outputTokens: number } | null { + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i] as { + role?: string + content?: Array<{ type?: string; text?: string }> + finish?: string + tokens?: { output?: number } + } + if (msgRole(msg) !== "assistant") continue + const finish = typeof msg.finish === "string" ? msg.finish : undefined + if (!finish) continue + const parts = Array.isArray(msg.content) ? msg.content : [] + const hasText = parts.some((p) => p?.type === "text" && typeof p.text === "string" && p.text.length > 0) + if (hasText) return null + // Same tool-part convention as hasPendingUserInput: a finished + // turn that issued tool calls did work, even with no chatter. + const hasToolCalls = parts.some((p) => { + const t = p?.type ?? "" + return t === "tool_use" || t === "tool" || t === "tool_call" || t.startsWith("tool") + }) + if (hasToolCalls) return null + return { finish, outputTokens: posNum(msg.tokens?.output) } + } + return null + } + + /** + * Recover a stream that finished without ever delivering text. + * + * Ported from v1 including its guard: re-check the server's own status + * first, because a provider that is quietly retrying looks identical from + * the event stream, and interrupting that would turn a recovering session + * into a stalled one. Then arm the pending-recovery latch — without it + * `recover()`'s inject-time check reads an idle session as "recovered by + * itself" and drops the injection. + * + * Returns true when it took action — or when the evidence positively says + * hands off (tools still in flight, own flag back to busy) — so the + * caller stops looking. Returns false when it declined on weak evidence + * (server still lists an idle session): the session is NOT buried, the + * remaining idle checks still run, and the next inspection re-evaluates. + */ + async function recoverSilentDeadStream(sid: string, messages: unknown[]): Promise { + const dead = lastSilentDeadStream(messages) + if (!dead) return false + if (dead.outputTokens < silentDeadStreamMinTokens) { + dbg( + `${short(sid)} last finished message has no text, but only ${dead.outputTokens} output tokens (floor ${silentDeadStreamMinTokens}) — not treating it as a dead stream`, + ) + return false + } + const w = ensureWatch(sid) + // Tools still awaiting results are working, not stalled — even when + // the finished message predates the tool events and looks dead. + // Reads the at-idle snapshot, not the live counter: markIdle zeroes + // it on the transition, and idle fires while tools run. + if (w.toolsInFlightAtIdle > 0) { + dbg(`${short(sid)} silent dead stream, but ${w.toolsInFlightAtIdle} tool(s) still in flight — working, not stalled`) + return true + } + // Ask the server before injecting, not just our own flag. A provider + // that is quietly retrying looks exactly like a dead stream from the + // event stream, and the event may not have arrived yet when the turn + // ends — which is v1's own reason for re-checking live status here. + // + // Our own flag flipping back to busy means a fresh execution.started + // arrived after the idle: the session genuinely restarted, so hands + // off. That is positive evidence, and it counts as handled. + if (w.status === "busy") { + dbg(`${short(sid)} silent dead stream, but the session is busy again — genuine restart, leaving it alone`) + return true + } + // The server still listing the session is weak evidence by comparison: + // its active map is known-stale (an idled session can linger there), + // while our own flag says idle. Declining here must NOT report + // handled — return true buries the session, and no later pass will + // ever reconsider it (2026-10-11: a real dead stream stood down this + // way while the job sat idle). Return false so the remaining idle + // checks run now and the next inspection re-reads the server fresh. + if ((await getActiveSessions()).includes(sid)) { + dbg(`${short(sid)} silent dead stream, but the server still lists the session — possibly a provider retry; deferring, not burying`) + return false + } + w.pendingRecoveryArmed = true + log( + "info", + `${short(sid)} silent dead stream: finish=${dead.finish}, ${dead.outputTokens} output tokens, no text parts; resuming`, + ) + await recover(sid, `Silent dead stream (${dead.finish})`) + return true + } + + /** + * One instant nudge for a turn the model never answered at all. + * + * The caller only reaches here when there is no text — but "no text" + * is not "nothing happened". A short answer below the token floor, a + * turn that did tool work, or a retry the server still lists are all + * deliberate silences the suite pins as leave-alone, so this fires + * only when no assistant message exists after the newest user message + * at all. Everything else belongs to the detectors above (dead stream, + * stuck tool, silent reasoning) or the stall ladder. + * + * Without this, such a turn is invisible to every detector and the + * session sits until the next event — which may never come. One quiet + * continue, fired immediately (no stall-window wait), single-shot per + * episode. + * + * The user's stop stays final: userCancelled, gaveUp, and every other + * lock is checked here AND inside recover(), so an interrupted job can + * never be restarted by this. A turn awaiting the user's reply is a + * hand-off, not a death, and a latched completion is never re-nudged. + */ + async function nudgeEmptyIdleTurn(sid: string, w: SessionWatch, messages: unknown[]): Promise { + if (w.emptyIdleNudged) return false + if (w.userCancelled || w.gaveUp || w.permissionPending) return false + if (w.completionSignaled) return false + if (w.failureLoopLatched) { + dbg(`${short(sid)} failure loop latched — standing down until a genuine turn`) + return false + } + // The stall ladder already owns this episode: a recovery is in + // flight (w.recovering) or has already fired (resumeAttempts > 0). + // Firing here would stack a second, unlabeled continue on top of + // the ladder's — the twin-injection shape the dedupe machinery + // exists to prevent (ses_ee14d727: an idle lands between our send + // and the model's start, and that idle must not restart OR duplicate + // the ladder). The empty-idle nudge is only for episodes where + // NOTHING has fired yet. + if (w.recovering || w.resumeAttempts > 0) return false + if (hasAssistantReplySinceUser(messages)) return false + if (isWaitingOnUser(sid, w, messages)) { + dbg(`${short(sid)} empty idle turn is waiting on the user — not nudging`) + return false + } + { + const kind = lockKind(w) + if (kind) { + dbg(`${short(sid)} ${kind} locked — refusing empty-idle nudge (reason: ${lockReason(w)})`) + return false + } + } + if (w.compacting) { + dbg(`${short(sid)} mid-compaction — refusing empty-idle nudge`) + return false + } + if (selfAbortActive(w)) { + dbg(`${short(sid)} self-abort in flight — refusing empty-idle nudge`) + return false + } + // Same loop guard as recover(): our continue counts toward the + // shared window, and past the max the answer is escalation, not + // another prompt. + if (continuesInWindow(w) >= loopMaxContinues) { + log("warn", `${short(sid)} hallucination loop (${loopMaxContinues} continues/${loopWindowMs / 1000}s) — abort+resume`) + await tryAbortAndResume(sid, w) + w.continueTimestamps = [] + return true + } + w.emptyIdleNudged = true + if (lifetimeExhausted(sid, w)) return true + recordContinue(sid) + // Send DIRECTLY, not through recover(): recover() schedules on a + // backoff timer (the opposite of instant) and arms selfRecovery + // for a turn that may never start — a stale claim that would + // steal genuineness from the next real turn (a fresh turn must + // reset the quota ladder — pinned by the rate-limit suite). + // notifyAndPrompt is awaited here, so no timer gap can re-arm + // behind us; and a prompt this sends IS a fresh episode, which is + // exactly what the flag being clear means. + log("info", `${short(sid)} idle with no reply, no question, no completion claim — instant nudge (single-shot this episode)`) + // notifyAndPrompt arms selfRecovery for the turn this prompt starts. + // That turn IS a fresh episode by definition — nothing was happening + // — so genuine classification is correct for it. But if no turn + // follows, the claim would linger and steal genuineness from the next + // real turn (a fresh turn must reset the quota ladder — pinned by + // the rate-limit suite). Restore unless someone else armed + // concurrently, whose claim is not ours to clear. + const wasSelfRecovery = w.selfRecovery + const sent = await notifyAndPrompt(sid, opts.continuePrompt ?? SAFE_EMPTY_IDLE_PROMPT, "empty-idle nudge") + if (!wasSelfRecovery) w.selfRecovery = false + return sent + } + + /** + * True when any assistant message exists after the newest user message. + * Anything the model said or did — text, reasoning, tool calls — counts: + * the instant nudge is only for a turn with literally no reply. + */ + function hasAssistantReplySinceUser(messages: unknown[]): boolean { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] as { role?: unknown; type?: unknown } | null | undefined + if (!m || typeof m !== "object") continue + const role = msgRole(m) + // The newest user message bounds the current turn: anything at + // or before it belongs to an earlier episode. + if (role === "user") return false + if (role === "assistant") return true + } + // No user message and no assistant message: no turn exists at all, + // so there is nothing to restart. + return false + } + + /** + * True when the session is holding the ball: a tool call whose result is + * still outstanding. + * + * A synthetic nudge in that state starts a step the user's real reply then + * interrupts ("Step interrupted"), so every idle recovery path stands down. + * + * This used to test `state.status === "pending"`, which v2 never emits, so + * the branch was unreachable and the nudge fired with an unanswered + * `question` on screen. See TOOL_STATE_* above. + */ + function hasPendingUserInput(messages: unknown[]): boolean { + const newest = messages[messages.length - 1] as { + type?: string + content?: { type?: string; name?: string; state?: { status?: string } }[] + } + if (newest?.type !== "assistant") return false + for (const part of newest.content ?? []) { + const t = part?.type ?? "" + if (!(t === "tool_use" || t === "tool" || t === "tool_call" || t.startsWith("tool"))) continue + const status = part?.state?.status + if (status === TOOL_STATE_RUNNING || status === TOOL_STATE_PENDING) return true + // An interactive tool only reaches a terminal state once the user has + // actually answered. "error" here means the question was interrupted, + // not that it was answered, so keep standing down. + if (status !== TOOL_STATE_COMPLETED && AWAITING_USER_TOOLS.has(part?.name ?? "")) return true + } + return false + } + + /** + * Narrow user-wait check for the busy-stall path. Unlike + * `hasPendingUserInput` (any unfinished tool means the idle turn + * isn't over), a busy session with a wedged bare tool and zero events + * IS a stall — so only tools whose completion is the user answering + * (`question`, `permission`, `ask`, `confirm`) count. A turn parked + * on one of those is waiting, not stalled (ses_ef72c5f3 2026-10-05: a + * stall continue fired into a session parked on a `question`). + */ + function isWaitingOnUser(sid: string, w: SessionWatch, messages: unknown[]): boolean { + // A genuine new user message re-arms here too (not just on the + // idle path): this is what clears a user-interrupt stand-down + // when the user hands the session back. + noteInboundUserMessage(sid, w, messages) + const newest = messages[messages.length - 1] as { + type?: string + content?: { type?: string; name?: string; state?: { status?: string } }[] + } + if (newest?.type !== "assistant") return false + for (const part of newest.content ?? []) { + const t = part?.type ?? "" + if (!(t === "tool_use" || t === "tool" || t === "tool_call" || t.startsWith("tool"))) continue + if (!AWAITING_USER_TOOLS.has(part?.name ?? "")) continue + // As in `hasPendingUserInput`: "error" means interrupted, not + // answered — only a completed user-awaiting tool ends the wait. + if (part?.state?.status !== TOOL_STATE_COMPLETED) return true + } + return false + } + + /** + * Newest inbound user message timestamp, or undefined. Shared by the + * stand-down check and the stand-down recheck scheduler so both agree on + * when the activity window expires. + */ + function lastUserMessageAt(messages: unknown[]): number | undefined { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] as { + type?: string + role?: string + time?: { created?: number } + info?: { time?: { created?: number }; role?: string } + } + const isUser = msgRole(m) === "user" + if (!isUser) continue + const ts = m.time?.created ?? m.info?.time?.created + return typeof ts === "number" ? ts : undefined + } + return undefined + } + + async function shouldStandDownForUser( + sid: string, + messages: unknown[], + activeUserWindowMs: number, + ): Promise { + try { + if (messages.length === 0) return false + // (a) A tool call still awaiting its result — see hasPendingUserInput. + if (hasPendingUserInput(messages)) return true + // (b) The user was recently active — they are mid-conversation, not stuck. + // + // v1 stamps `lastUserMessageAt` on ANY inbound user message and asks + // "was there one inside activeUserWindowMs?". This used to ask "is the + // newest message a user message?", which at idle time is essentially never + // true — the newest message is the assistant turn that just finished — so + // the whole window was dead. Walk back for the most recent user message. + const lastUserAt = lastUserMessageAt(messages) + if (lastUserAt !== undefined && Date.now() - lastUserAt < activeUserWindowMs) return true + return false + } catch { + return false + } +} + +/** + * Re-arm the done-claim budgets when a genuinely new work cycle starts. + * + * The signal is an inbound user message, and what makes one genuine is that + * it is a *different* message — identified by id, not by timestamp. A + * re-delivered or replayed message has the same id and is not new work; + * comparing clock times instead would treat any drift between two reads of + * the history as a new request and hand the model a fresh budget every time + * it repeated itself, which is exactly what #26 was. Timestamps are only the + * fallback, for messages that carry no id. + * + * The history is already loaded, so this costs nothing extra. + * + * Doing this only here (and not on every busy cycle) is the #26 fix: a model + * that re-announces completion each turn would otherwise be handed a fresh + * budget each time it announced, and the nudge would never stop. + */ + function noteInboundUserMessage(sid: string, w: SessionWatch, messages: unknown[]): void { + let latest: { id?: string; at?: number; text?: string } | null = null + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i] as { + id?: string + type?: string + role?: string + time?: { created?: number } + info?: { time?: { created?: number }; role?: string } + content?: Array<{ type?: string; text?: string }> + } + const isUser = msgRole(m) === "user" + if (!isUser) continue + const text = Array.isArray(m?.content) + ? m.content.filter((p) => p?.type === "text" && typeof p.text === "string").map((p) => p.text as string).join("\n") + : undefined + latest = { id: typeof m.id === "string" ? m.id : undefined, at: m.time?.created ?? m.info?.time?.created, text } + break + } + if (!latest) return + const isNew = latest.id + ? latest.id !== w.lastUserMessageID + : typeof latest.at === "number" && latest.at > w.lastUserMessageSeenAt + if (!isNew) return + w.lastUserMessageID = latest.id + if (typeof latest.at === "number") w.lastUserMessageSeenAt = latest.at + // Our own injections travel the visible channel as real user messages. + // They start a new turn but carry no new instructions: re-arming on + // them clears the budgets just spent and the same errors refire on + // the next idle (ses_ef81e8561ffeXyx3jAzKl5lltv, 2026-10-04 — four + // identical unknown-tool prompts, each "2x"). Tracking above stays + // truthful; only the clearing is skipped. Two signals: exact text + // match, and recency to our last inject — the live projection can + // carry user messages with no readable body (content null), which + // defeats text matching, but our prompt always lands within seconds + // of the inject on the same box clock. + const ownByText = typeof latest.text === "string" && latest.text.length > 0 && w.ownPromptTexts.includes(latest.text) + const ownByTime = + typeof latest.at === "number" && + w.lastInjectAt > 0 && + latest.at >= w.lastInjectAt && + latest.at - w.lastInjectAt < OWN_PROMPT_TIME_MS + if (ownByText || ownByTime) { + dbg(`${short(sid)} newest user message is our own prompt — not re-arming budgets`) + return + } + if (w.doneClaimAttempts > 0 || w.doneClaimOpenTodosAttempts > 0) { + dbg(`${short(sid)} new user message — re-arming the done-claim budgets`) + } + w.doneClaimAttempts = 0 + w.doneClaimOpenTodosAttempts = 0 + // A genuine new user message hands the session back: clear a + // user-interrupt stand-down (ses_eeb02d4b, 2026-10-07 — the latch + // otherwise holds until session cleanup and even an explicit + // "continue" is refused). Only a message newer than the latch + // itself re-arms: a pre-interrupt message the plugin simply + // hadn't observed yet must not clear it. Manual interrupts + // themselves are untouched; only future recovery re-arms. + if (w.userCancelled && typeof latest.at === "number" && latest.at >= w.userCancelledAt) { + dbg(`${short(sid)} new user message after interrupt — re-arming recovery`) + w.userCancelled = false + } + // A new request is new work: the ack self-loop counter starts clean, + // because the model re-announcing completion after the user asked for + // more is not the stuck case this counts. + w.taskCompleteSignals = 0 + // The unknown-tool budget is scoped to one request for the same reason: + // a model told to do something new gets a fresh tool list, and one + // invented name in the previous round says nothing about this one. + if (w.unknownToolErrors.size > 0 || w.unknownToolSuggestionSent) { + dbg(`${short(sid)} new user message — re-arming the unknown-tool budget`) + } + w.unknownToolErrors.clear() + w.unknownToolSuggestionSent = false + w.stuckToolAttempts = 0 + // A genuine user message is a new episode for the stall ladder too + // (the reset lives here, on genuine execution.started, and in the + // progress check — never on lifecycle transitions). Recorded + // separately from lastUserMessageSeenAt, which our own prompts also + // advance: only non-own messages mark a user takeover. Clamped to + // now: message timestamps come from the server clock, and a forged + // or skewed future timestamp would otherwise keep + // `lastGenuineUserAt > armedAt` true indefinitely, parking every + // later recovery (the fuse suite forges +35s deliberately; a skewed + // server clock does it by accident). Clamping only pulls futures + // back to the present — genuine past timestamps pass through. + w.lastGenuineUserAt = + typeof latest.at === "number" ? Math.min(latest.at, Date.now()) : Date.now() + // A genuine user message is activity: it resets the silence clock, + // not just the ladder count. Without this, a message the plugin + // only learns about via a history read leaves lastActivityAt stale, + // and the watchdog declares a stall on a session the user just + // spoke to. touch() itself debounces bursts (minActivityGapMs), and + // this runs only for genuinely-new, non-own messages. + touch(sid) + // A genuine new user message is also what re-opens a latched + // completion: new instructions mean the "done" being tracked is no + // longer the question. This is the ONLY place the latch clears besides + // a fresh latch overwriting it — deliberately not in markBusy, whose + // transitions include plumbing (watchdog re-marks, discovery adopts, + // step events, our own injections' turns) that wiped fresh latches + // within seconds (ses_ed33c0da, 2026-10-11). Own prompts return above + // before reaching here, so our injections never clear it. + if (w.completionSignaled) { + w.completionSignaled = false + w.completionSignaledAt = 0 + dbg(`${short(sid)} genuine new user message — re-opening latched completion`) + } + // A genuine user message is a new episode for the stall ladder too + // (the reset lives here, on genuine execution.started, and in the + // progress check — never on lifecycle transitions). + if (w.resumeAttempts > 0) { + dbg(`${short(sid)} new user message — restarting the stall ladder`) + } + w.resumeAttempts = 0 + // The user owns the turn now: a retry scheduled before their message + // no longer describes what a stall after it means. + w.providerRetryAt = 0 + // A genuine user message also closes any refire episode: the next + // stall starts back at the base fuse. + w.episodeRefires = 0 + w.episodeRecovers = 0 + // Lifetime cap is also per genuine user episode: new instructions + // mean a new task, so the total-fire count starts over here — and + // only here (plus genuine execution.started), never on model text. + w.lifetimeContinues = 0 + // And it re-opens a stood-down session: the stall gaveUp latch + // clears on genuine user takeovers, never on transitions. + w.gaveUp = false + // Same for the empty-idle single-shot: a genuine user message is + // a new episode, so the instant nudge re-arms here — and only at + // the observable-episode sites, never in markBusy (which also runs + // on our own recovery injections and would re-arm on its own send). + w.emptyIdleNudged = false + // Deliberately NOT clearing checkedToolPartIDs: already-reported + // errors must never be recounted. Clearing it made every genuine + // user message re-suggest the same stale errors (2026-10-04: one + // bash suggestion per message, long after the model moved to shell). + // New error parts still count fresh toward the threshold. + } + + /** Registered tool names, cached briefly — the registry only changes when a + * plugin reloads, and this runs on every idle. */ + let cachedToolIds: string[] | null = null + let cachedToolIdsAt = 0 + + async function getAvailableToolIds(): Promise { + if (cachedToolIds && Date.now() - cachedToolIdsAt < TOOL_IDS_CACHE_MS) return cachedToolIds + if (!ctx.tool?.list) return cachedToolIds ?? [] + try { + const listed = await ctx.tool.list() + // `id` is the effective name — the namespaced form when the tool sits + // in a namespace — which is exactly the string a model would have to + // call, so it is what the suggestion has to quote back. + const ids = (listed as Array<{ id?: string; name?: string }>) + .map((t) => (typeof t?.id === "string" && t.id ? t.id : typeof t?.name === "string" ? t.name : "")) + .filter((n) => n.length > 0) + cachedToolIds = ids + cachedToolIdsAt = Date.now() + return ids + } catch (e) { + log("warn", `failed to list tools: ${e instanceof Error ? e.message : String(e)}`) + // Stale names are better than none for a "does this exist" check: a + // tool added since the cache would be wrongly called unknown, but + // that costs one extra suggestion, while an empty list skips the + // check entirely and loses the feature. + return cachedToolIds ?? [] + } + } + + /** + * Name a replacement when a model keeps calling a tool that does not exist. + * + * v1 asks `ctx.client.tool.ids()`. v2 has no such route; `ctx.tool.list()` + * returns the same set of effective names, which is all this needs. + * + * The tool parts come from the message history rather than from events, + * because the whole point is a part that already *errored* — by the time the + * error lands, the `session.tool.called` event for that part is long gone, + * and this code only runs on idle anyway. + */ + /** + * Nudge a turn that ended with a tool call that never returned, or retry + * one that failed at end of turn. + * + * The event-counted `pendingTools` only sees what the plugin observed live: + * a turn that ends while a call is still in flight (compress wedging at + * 13:09:44 in ses_edf403f5, a write left streaming in ses_ee14d727) parks + * the session idle with `pendingTools === 0`, no failure, and no stall — so + * every event-driven path reads healthy-idle and stays silent. The history + * still shows the unfinished part, so this reads it instead. + * + * Only parts newer than the last user message count: a user message after + * the call means the turn is not parked, it moved on. Only parts older + * than the stall window count: a call that just went out has not had a + * chance to answer yet. Anything else (completed/error) is settled. + */ + /** + * The message-history scan shared by the stuck-tool pass and the stall + * timer. Extracted (not duplicated) so both judge the same shape: tool + * parts newer than the last user message that never settled, split into + * aged-out `stuck` names versus a trailing failure plus whether anything + * is still in flight at all. + */ + function scanToolParts( + messages: unknown[], + lastUserAt: number, + cutoff: number, + ): { stuck: string[]; trailing: { name: string; err: string; created: number } | null; anythingInFlight: boolean } { + const stuck: string[] = [] + // A failed call is retried only when the turn ENDED on it: the error + // is the trailing part of the latest assistant content, with nothing + // after it. An error the model already continued past (later text or + // later calls) is handled feedback, not abandonment — nudging those + // would nag on every normal turn. Unknown names belong to the + // suggester, not the retrier: telling the model to reissue a name + // that does not exist teaches persistence on a typo. + let trailing: { name: string; err: string; created: number } | null = null + let anythingInFlight = false + for (const m of messages as Array<{ type?: string; time?: { created?: number }; content?: Array<{ type?: string; name?: string; state?: { status?: string; error?: unknown; output?: unknown }; time?: { created?: number } }> }>) { + if (!m || msgRole(m) !== "assistant" || !Array.isArray(m.content)) continue + for (const part of m.content) { + if (part?.type !== "tool") { + trailing = null + continue + } + const st = part?.state?.status + const name = typeof part?.name === "string" && part.name ? part.name : "(unnamed)" + const created = + typeof part?.time?.created === "number" ? part.time.created + : typeof m?.time?.created === "number" ? m.time.created + : 0 + if (st === "error") { + const rawErr = + part?.state && typeof part.state === "object" + ? (part.state as { error?: unknown }).error + : undefined + const eType = + rawErr && typeof rawErr === "object" && typeof (rawErr as { type?: unknown }).type === "string" + ? ((rawErr as { type?: string }).type as string) + : "" + const eMsg = toolErrorText(part?.state) + // Abort-shaped errors are never retried: that path fights a + // user interrupt (ses_edf403f5 13:09: the error WAS the + // user's own stop landing mid-turn). + if (isAbortError(eType, eMsg)) { + trailing = null + continue + } + // Policy blocks are verdicts, not failures: the guardrail + // text forbids retry outright, and reissuing the call + // teaches circumvention (ses_edf403f5 15:59 shell block). + if (NEVER_RETRY_RE.test(eMsg)) { + trailing = null + continue + } + trailing = { name, err: eMsg.slice(0, 160), created } + continue + } + if (st === "completed") { + trailing = null + continue + } + if (created <= lastUserAt) continue + anythingInFlight = true + if (created >= cutoff) continue + if (!stuck.includes(name)) stuck.push(name) + } + } + return { stuck, trailing, anythingInFlight } + } + + function buildStuckNudgePrompt(stuck: string[]): string { + const names = stuck.slice(0, 3).join('", "') + return ( + `A "${names}" call never returned a result and the turn went idle waiting on it. ` + + `Proceed without it: continue the task from where you left off and do not retry the same call.` + ) + } + + async function checkForStuckToolCalls(sid: string, w: SessionWatch): Promise { + if (w.userCancelled || w.completionSignaled) return false + { + const locked = lockKind(w) + if (locked) { + dbg(`${short(sid)} locked (${locked}) — skipping stuck-tool pass`) + return false + } + } + const messages = await loadMessages(sid) + let lastUserAt = 0 + for (const m of messages) { + if (msgRole(m) === "user") lastUserAt = Math.max(lastUserAt, msgAt(m)) + } + const cutoff = Date.now() - stuckToolAgeMs + const { stuck, trailing, anythingInFlight } = scanToolParts(messages, lastUserAt, cutoff) + // A call still in flight (any age) means the turn may yet resolve on + // its own — the stall and dead-stream paths own that case, not this one. + if (trailing && !anythingInFlight && trailing.created > lastUserAt) { + const available = await getAvailableToolIds() + if (available.length > 0 && available.includes(trailing.name)) { + const prompt = + `The "${trailing.name}" call failed${trailing.err ? ` with: ${trailing.err}` : ""}. ` + + `Retry it now — reissue the same call.` + log("warn", `${short(sid)} "${trailing.name}" failed at end of turn, retrying promptly (attempt ${(w.stuckToolAttempts ?? 0) + 1}/${maxRetries})`) + return targetedRecovery(sid, "stuck-tool", prompt, "stuckToolAttempts").then(() => true) + } + } + if (stuck.length === 0) return false + const prompt = buildStuckNudgePrompt(stuck) + const names = stuck.slice(0, 3).join('", "') + log("warn", `${short(sid)} tool call(s) "${names}" stuck unfinished — nudging to proceed without it`) + return targetedRecovery(sid, "stuck-tool", prompt, "stuckToolAttempts").then(() => true) + } + + /** + * Nudge a turn that thought without producing anything. + * + * The parked shape from ses_edd1eb519/ses_edd1e39c3 (2026-10-09): a user + * question, then assistant turn(s) with reasoning but zero text and zero + * tool calls, then idle indefinitely. No failure fires, the stall path + * needs a busy session, and the targeted patterns need text — so every + * event-driven path reads healthy-idle and stays silent. + * + * Deliberately narrow: any text or any tool part since the last user + * message means the turn produced or delegated — only the total void + * counts (reasoning with no output and nothing delegated). Tool- + * completing turns leave results the next turn consumes; text turns + * speak. Fires on the structural idle pass for fresh parkings and on + * the watchdog idle sweep for already-parked sessions (which emit no + * further events). History-only: no HTTP dependency. + */ + async function checkForSilentReasoningTurn(sid: string, w: SessionWatch): Promise { + if (w.userCancelled || w.completionSignaled) return false + { + const locked = lockKind(w) + if (locked) { + dbg(`${short(sid)} locked (${locked}) — skipping silent-reasoning pass`) + return false + } + } + const messages = await loadMessages(sid) + let lastUserAt = 0 + for (const m of messages) { + if (msgRole(m) === "user") lastUserAt = Math.max(lastUserAt, msgAt(m)) + } + let newestPostUserAt = 0 + let sawReasoning = false + for (const m of messages as Array<{ type?: string; time?: { created?: number }; content?: Array<{ type?: string; text?: string }> }>) { + if (!m || msgRole(m) !== "assistant" || !Array.isArray(m.content)) continue + const at = msgAt(m) + if (at <= lastUserAt) continue + if (at > newestPostUserAt) newestPostUserAt = at + for (const part of m.content) { + const t = part?.type ?? "" + if (t === "text" && typeof part?.text === "string" && part.text.length > 0) return false + if (t === "tool_use" || t === "tool" || t === "tool_call" || t.startsWith("tool")) return false + if (t === "reasoning" && typeof part?.text === "string" && part.text.length > 0) sawReasoning = true + } + } + if (!sawReasoning) return false + // Fresh turns get the stall window before being judged parked. + if (Date.now() - newestPostUserAt < chunkTimeoutMs) return false + const prompt = + `Your last turn produced reasoning but no visible output and no tool calls. ` + + `Continue the task: produce the answer or the next tool call.` + log("warn", `${short(sid)} reasoning-only turn with no output — nudging to continue`) + return targetedRecovery(sid, "silent-reasoning", prompt, "stuckToolAttempts").then(() => true) + } + + async function checkForUnknownToolCalls(sid: string, w: SessionWatch): Promise { + if (w.userCancelled || w.completionSignaled) return false + { + const locked = lockKind(w) + if (locked) { + dbg(`${short(sid)} locked (${locked}) — skipping unknown-tool pass`) + return false + } + } + try { + const messages = await loadMessages(sid) + // The history is read before the "already suggested" guard, and the + // per-request reset rides on that read. The guard used to come first, + // and since this runs alongside inspectOnIdle the ordering was undefined + // — on a re-armed turn the guard could see the previous turn's latch + // still standing and skip the check that was supposed to re-arm it. + noteInboundUserMessage(sid, w, messages) + // Stop statements latch in the pattern pass, which runs after this + // check on the same idle - evaluate the stop here too, from the + // just-loaded history, or a turn that ends stopped eats one + // unknown-tool nudge first. + if (containsStopStatement(w.lastAssistantText || lastAssistantTextFrom(messages))) return false + if (w.unknownToolSuggestionSent) return false + const available = await getAvailableToolIds() + // No registry, or nothing registered: every name would look unknown. + if (available.length === 0) return false + for (const msg of messages) { + const content = (msg as { content?: unknown })?.content + if (!Array.isArray(content)) continue + for (const part of content as Array>) { + if (part?.type !== "tool") continue + // v2's tool part identifies itself by `id` and names the tool in + // `name` — v1 read `part.tool`, which v2 never emits, so the + // comparison there would have been against "" and never matched. + const partId = typeof part.id === "string" ? part.id : "" + if (!partId || w.checkedToolPartIDs.has(partId)) continue + w.checkedToolPartIDs.add(partId) + // Bounded: the history is re-walked on every idle, so this set + // otherwise grows one entry per tool part for the life of the + // session — audit 2026-10-10, the only uncapped collector here + // (ownPromptTexts 10, textParts 2, recentToolCalls 120s). + // A FIFO trim is enough: parts old enough to fall out are old + // enough that re-reporting them would be wrong anyway. + if (w.checkedToolPartIDs.size > CHECKED_TOOL_PART_IDS_MAX) { + const oldest = w.checkedToolPartIDs.values().next().value + if (typeof oldest === "string") w.checkedToolPartIDs.delete(oldest) + } + const state = part.state as { status?: string } | undefined + if (state?.status !== "error") continue + const toolName = typeof part.name === "string" ? part.name : "" + if (!toolName || available.includes(toolName)) continue + const count = (w.unknownToolErrors.get(toolName) ?? 0) + 1 + w.unknownToolErrors.set(toolName, count) + if (count < UNKNOWN_TOOL_THRESHOLD) continue + const suggestion = resolveToolSuggestion(toolName, available) + const toolList = available.slice(0, UNKNOWN_TOOL_LIST_LIMIT).join(", ") + const prompt = suggestion + ? `You tried to use the tool "${toolName}" ${count} times, but it does not exist. ` + + `The closest matching tool is "${suggestion}". ` + + `Please use "${suggestion}" instead and adjust your arguments accordingly. ` + + `Available tools include: ${toolList}.` + : `You tried to use the tool "${toolName}" ${count} times, but it does not exist. ` + + `Please check the available tools and use the correct one. ` + + `Available tools include: ${toolList}.` + w.unknownToolSuggestionSent = true + log( + "warn", + `${short(sid)} unknown tool "${toolName}" called ${count}x, suggesting "${suggestion ?? "(none)"}" mod=${MODULE_INSTANCE}`, + ) + // Feature corrections go synthetic, never visible: the model needs + // the correction, the user does not need a full-text user message + // for it. (Synthetic still renders as a plugin notification in + // the TUI — quieter, not silent.) Stall recovery stays visible + // by design so an intervention loop is self-evident. + await injectOnce(sid, prompt, "unknown-tool", false, false, quietCorrections) + return true + } + } + return false + } catch (e) { + log("warn", `${short(sid)} unknown-tool check failed: ${e instanceof Error ? e.message : String(e)}`) + return false + } + } + + /** + * `task_complete`: the model's explicit "I am finished". + * + * v2 has no built-in equivalent — `grep -rl task_complete packages` finds + * nothing — so registering it here is the only way it exists at all, and + * this is a real feature rather than a compatibility shim: it is the + * strongest completion signal available, stronger than a trailing 🎉, + * because the model chose to call it. + * + * Ported from v1 with its escalation intact. The ack tool-result is fed + * straight back into the turn, and a stuck model answers it by calling the + * tool again rather than ending with text — v1 logged 27 consecutive acked + * calls with zero new user input. So: first call acks with an explicit stop + * instruction, second warns, third throws and the turn is forced to end. + * + * The counter is reset by a new inbound user message and by any other tool + * call, so the escalation only ever measures a genuinely stuck loop rather + * than several legitimate rounds of work. + */ + async function registerTaskCompleteTool(): Promise { + if (!ctx.tool?.transform) { + dbg("tool registry unavailable — task_complete not offered") + return + } + try { + await ctx.tool.transform((editor) => { + editor.add({ + name: "task_complete", + description: TASK_COMPLETE_DESCRIPTION, + // No arguments. v1's `args: {}` is the same thing: the signal + // is the call, not a payload. + input: { type: "object", properties: {}, additionalProperties: false }, + execute: async (_input, toolCtx) => { + const sid = toolCtx?.sessionID + if (!sid) return { content: TASK_COMPLETE_ACK } + const w = ensureWatch(sid) + + // A subagent reporting completion is not a parent finishing, + // and gating a child's report on the parent's todo list would + // block it on work it was never asked to do. v1 skips the + // todo gate for subagents for the same reason. + if (!(await isSubAgentSession(sid))) { + const open = getOpenTodos(await readTodos(sid)) + if (open.length > 0 && w.taskCompleteOverrides < maxRetries) { + w.taskCompleteOverrides++ + // Work remains, so a later completion is legitimate: reset the + // repeat-signal counter rather than letting this override feed + // the escalation. + w.taskCompleteSignals = 0 + const reminder = buildOpenTodosReminder(open) + const blockMsg = `Mark any finished todos complete and do not redo completed work.\n${reminder}` + log( + "info", + `${short(sid)} task_complete blocked: ${open.length} open todos remain (override ${w.taskCompleteOverrides}/${maxRetries})`, + ) + // Also fire a visible nudge naming the blocking todos, in case + // the tool result collapses to an invisible one-liner. Skipped + // when the user holds the ball — injecting then would start a + // step their real reply interrupts. The tool result already + // carries the todo names either way. + let awaitingInput = false + try { + awaitingInput = hasPendingUserInput(await loadMessages(sid)) + } catch (e) { + dbg(`${short(sid)} task_complete block: awaiting-input check failed:`, e instanceof Error ? e.message : String(e)) + } + if (awaitingInput) { + log("info", `${short(sid)} task_complete blocked but user input pending — skipping visible nudge`) + } else { + await injectOnce(sid, blockMsg, "task-complete-blocked") + } + return { content: blockMsg } + } + } + + w.completionSignaled = true + w.completionSignaledAt = Date.now() + log("info", `${short(sid)} task_complete called, ${(await isSubAgentSession(sid)) ? "subagent" : "agent"} done`) + w.taskCompleteSignals++ + if (w.taskCompleteSignals === 2) return { content: TASK_COMPLETE_REPEAT_WARNING } + if (w.taskCompleteSignals > 2) throw new Error(TASK_COMPLETE_REPEAT_ERROR) + return { content: TASK_COMPLETE_ACK } + }, + }) + }) + log("info", "registered the task_complete tool") + } catch (e) { + // A failed registration must not take the watchdog down with it. + const msg = e instanceof Error ? e.message : String(e) + log("warn", `task_complete registration failed (${msg}) — continuing without it`) + } + } + + /** + * Judge a finished turn — in two phases, because the text may not have settled. + * + * v1 evaluates the done/tool patterns from a timer armed on the idle event + * (`toolTextCheckDelayMs`, 3s by default) rather than on the event itself, and + * the delay is the feature: `session.idle` can arrive while the assistant's + * final text is still being written into the message history, and a check that + * reads too early sees a half-finished turn and either misses the pattern or + * judges a turn that was not over. + * + * v2 already avoids most of that race with the live delta buffer, which is + * why this port was able to judge on idle for as long as it did. It does not + * remove it: the buffer is empty when the plugin loaded mid-turn, and the + * fallback is exactly the history that may not have flushed. So the option is + * honoured by splitting the work: + * + * - **structural** (immediately): is this a dead stream, is it handing control + * to the user, is the user already busy. None of these depend on the final + * text, and a dead stream must be caught before anything that reads text. + * - **pattern** (after the delay): the celebration, tool-call-as-text, ready-to- + * continue, action-intent and done-claim detectors. These are exactly the + * ones that read the last text and so are exactly the ones the settle delay + * exists for. + * + * The guards above the split run again in the pattern phase, deliberately: three + * seconds is long enough for the user to have replied, and a nudge fired through + * their reply is the bug the hand-off and stand-down guards exist to prevent. + */ + async function inspectOnIdle(sid: string, phase: "structural" | "pattern" = "structural") { + const w = ensureWatch(sid) + const messages = await loadMessages(sid) + noteInboundUserMessage(sid, w, messages) + + // Judged first, and before anything that needs text: a stream that died + // before delivering any text is precisely the case where every + // text-based check below would find nothing to look at. + if (await recoverSilentDeadStream(sid, messages)) return + + // A turn parked on a tool result that never arrives reads healthy-idle + // to every event-driven path (no failure, no stall, counters at zero). + // The history still shows the unfinished part, so check it structurally + // here rather than in the text-dependent pattern phase below. + if (await checkForStuckToolCalls(sid, w)) return + if (await checkForSilentReasoningTurn(sid, w)) return + + // Prefer the live delta buffer; fall back to the authoritative message + // history when it is empty (e.g. the plugin loaded mid-turn) or stale. + let text = w.lastAssistantText + if (!text) text = lastAssistantTextFrom(messages) + if (!text) { + // Nothing delivered and nothing to judge: no finish to call + // dead, no parts to call stuck, no text for the pattern checks + // below. Without this branch such a turn is invisible to every + // detector and the session sits until the next event — which + // may never come. One quiet continue, single-shot per episode; + // the user's stop stays final (see guards inside). + await nudgeEmptyIdleTurn(sid, w, messages) + return + } + + // A turn that ends by handing control back to the user (a question or an + // explicit prompt) is awaiting their reply; a synthetic nudge here starts + // an in-flight step that their real reply then interrupts ("Step + // interrupted"). Never nudge a hand-off turn. + if (isUserHandoff(text)) { + dbg(`${short(sid)} idle turn ends with a user hand-off — skipping targeted recovery`) + return + } + + if (await shouldStandDownForUser(sid, messages, activeUserWindowMs)) { + // Latch FIRST, then stand down. A turn that completed inside the + // activity window must still record its completion: latching is + // read-only state (no nudge, no interruption), so it is safe to do + // even while standing down. Without this, the claim is never + // evaluated — the deferred re-check below can lose the race to the + // busy watchdog — and the ladder fires on a finished turn: + // ses_ed33c0da (2026-10-11) stood down on a TASK COMPLETE turn at + // 21:08:42 and stalled at +197s. Mirrors the busy-path latch, same + // predicate, same no-open-todos condition. Open todos fall through + // to the normal stand-down (the later checks nudge for them when + // the window expires); challenges and reminders stay below, because + // those INTERVENE and stand-down exists to prevent exactly that. + { + const tailText = w.lastAssistantText || lastAssistantTextFrom(messages) + if (tailText && turnEndsWithCompletionClaim(tailText)) { + const open = getOpenTodos(await readTodos(sid)) + if (open.length === 0 && !w.completionSignaled) { + w.completionSignaled = true + w.completionSignaledAt = Date.now() + log("info", `${short(sid)} turn ends with a completion claim inside the activity window — latching completion, standing down`) + } + } else { + dbg(`${short(sid)} stand-down evaluation: tail is not a completion claim (score the tail, not the turn)`) + } + } + dbg(`${short(sid)} user has pending input or was recently active — standing down`) + // ...but only for now. This evaluation is the only one this idle + // gets, so standing down without a follow-up makes the session + // invisible until the next event (ses_edbb4e67, 2026-10-10: two + // manual continues on an action-intent stall). Re-arm one + // deferred pattern pass for when the activity window expires. A + // new turn replaces it through the toolTextTimer slot (markBusy + // clears); a hand-off never reaches here (returned above), so an + // awaited user reply is never nagged. + { + const lastUser = lastUserMessageAt(messages) + const waitMs = + (lastUser === undefined ? activeUserWindowMs : Math.max(0, lastUser + activeUserWindowMs - Date.now())) + 1000 + if (w.toolTextTimer) clearTimeout(w.toolTextTimer) + w.toolTextTimer = setTimeout(() => { + w.toolTextTimer = null + if (w.status !== "idle") { + dbg(`${short(sid)} stand-down recheck skipped — session is ${w.status}, not idle`) + return + } + if (w.userCancelled) return + void inspectOnIdle(sid, "pattern") + }, waitMs) + } + return + } + + if (phase === "structural") { + schedulePatternPass(sid) + return + } + + // The model's own completion signal. A trailing 🎉 means it considers the + // work finished, so nudging here would talk over a deliberate stop. + // Latched rather than re-derived, because the next idle with no new text + // would otherwise re-check the same turn forever. + // + // Cross-checked against the todo list, because the emoji alone is not + // trustworthy: a model that finishes early celebrates early, and latching + // on that turns a false positive into permanent silence. With work still + // listed, the celebration is a false positive and the right answer is to + // name what is unfinished. + if (endsWithCelebration(text)) { + const open = getOpenTodos(await readTodos(sid)) + if (open.length > 0) { + // Deliberately not latching: the list is still open, so this turn is + // not a completion and the next one must be free to judge again. + await targetedRecovery( + sid, + "open-todos-celebration-false-positive", + buildOpenTodosReminder(open), + "todoNudgeAttempts", + ) + return + } + if (!w.completionSignaled) { + w.completionSignaled = true + w.completionSignaledAt = Date.now() + log("info", `${short(sid)} turn ends with a celebration and no open todos — latching completion, not nudging`) + } else { + dbg(`${short(sid)} completion already latched — skipping`) + } + return + } + + // A raw tool call written into the reasoning block never executes: no + // part is tagged as a tool call, so nothing on the session side ever + // runs it. Judged before the text variant because a message can have + // both, and the reasoning one is the one that silently does nothing. + const reasoning = lastAssistantReasoning(messages) + if (reasoning && containsToolCallAsText(reasoning)) { + await targetedRecovery(sid, "tool-call-in-reasoning", thinkingToolRecoveryPrompt, "toolTextAttempts") + return + } + if (containsToolCallAsText(text)) { + await targetedRecovery(sid, "tool-call-as-text", opts.toolTextRecoveryPrompt ?? TOOL_TEXT_RECOVERY_PROMPT, "toolTextAttempts") + return + } + if (containsReadyToContinuePattern(text, readyToContinuePatterns)) { + await targetedRecovery(sid, "ready-to-continue", opts.continuePrompt ?? SAFE_CONTINUE_PROMPT, "intentNudgeAttempts") + return + } + if (resumeOnActionIntent && containsActionIntent(text)) { + await targetedRecovery(sid, "action-intent", opts.actionIntentPrompt ?? opts.continuePrompt ?? SAFE_CONTINUE_PROMPT, "intentNudgeAttempts") + return + } + // An explicit stop latches like a celebration and wins over the + // done-claim challenge below: a turn that says it is stopping + // with no open todos is finished, not evidence-light. Challenging + // it for details would nag exactly the completed sessions this + // latch exists to quiet (ses_eec5ba74). Tool-call-as-text and + // action-intent above still win: pasted tool calls and declared + // next steps are unfinished business even beside stop words. + // An explicit stop latches like a celebration: the turn declares the + // work over without claiming anything to verify. Open todos still + // earn the once-budgeted reminder (a stop with listed work is not a + // completion); with nothing open, latch and end the turn quietly. + if (containsStopStatement(text)) { + const open = getOpenTodos(await readTodos(sid)) + if (open.length > 0) { + await targetedRecovery(sid, "stop-open-todos", buildOpenTodosReminder(open), "todoNudgeAttempts") + return + } + if (!w.completionSignaled) { + w.completionSignaled = true + w.completionSignaledAt = Date.now() + log("info", `${short(sid)} turn ends with an explicit stop and no open todos — latching completion, not nudging`) + } else { + dbg(`${short(sid)} completion already latched — skipping`) + } + return + } + + if (containsDoneClaimPattern(text, doneClaimPatterns)) { + // Two different prompts for two different situations, which is why the + // budgets are separate: spending the details prompt must not silence the + // one that says work is still listed. + const open = getOpenTodos(await readTodos(sid)) + if (open.length > 0) { + await targetedRecovery(sid, "done-claim-open-todos", doneWithoutWorkPrompt, "doneClaimOpenTodosAttempts") + return + } + if (w.doneClaimAttempts < maxRetries) { + // Ask once for the work report. v2 used to gate this on a 400-char + // length, which cannot tell "Task done." from a real summary and so + // both over-nudged terse reports and let short-but-real ones pass. + // containsWorkDescription is the same structural test v1 uses, and + // prompting again after a real report loops forever (#26). + if (!containsWorkDescription(text)) { + await targetedRecovery(sid, "done-claim-no-details", doneWithoutDetailsPrompt, "doneClaimAttempts") + } else { + dbg(`${short(sid)} done-claim carries a work description — skipping details prompt`) + } + } + } + + // A full context window is a separate failure mode from a stalled one: the + // session keeps working happily until it chokes. Judged last, because it is + // the only check here that acts on the session rather than the text, and a + // saturated session that also produced a terse done-claim wants the + // reclamation, not the details prompt. + await checkContextSaturation(sid) + } + + // --------------------------------------------------------------------- + // Watchdog timer + // --------------------------------------------------------------------- + + /** + * `session.active()` exists on the v2 client but is not part of the + * plugin SessionDomain pick — access defensively; empty map if missing. + */ + /** + * `session.active()` exists on the v2 client but is not part of the plugin + * SessionDomain pick, so it is normally absent — see + * Mte90/opencode-auto-resume#33. Fall back to our own event-derived busy + * flags rather than returning an empty set: returning `[]` made the + * subagent-wait guard below unreachable, so a parent session that went + * quiet while its child ran was declared stalled and interrupted. + */ + async function getActiveSessions(): Promise { + // Without server truth the watchdog can only see sessions it observed + // events for since load: after any reload a hung-but-silent session is + // invisible while the server still reports it running, and nothing ever + // fires (ses_edf403f5, 2026-10-09: running 7+ min with two user messages + // unanswered). The HTTP loopback below restores server truth in prod. + try { + const fn = (ctx.session as any).active + if (typeof fn !== "function") { + // Production has no active surface at all (verified 2026-10-06). + const http = await activeSessionsHttp() + if (http !== null) return http + return busySessions() + } + const active = await fn.call(ctx.session) + return Object.keys(active ?? {}).filter((k) => typeof k === "string") + } catch { + // Same loopback here: a throwing surface must not blind the watchdog. + const http = await activeSessionsHttp() + if (http !== null) return http + return busySessions() + } + } + + type ActiveMap = Record + let activeCache: { at: number; ids: string[] } | null = null + const ACTIVE_TTL_MS = 10_000 + async function activeSessionsHttp(): Promise { + const now = Date.now() + if (activeCache && now - activeCache.at < ACTIVE_TTL_MS) return activeCache.ids + for (const base of serverBaseUrls()) { + try { + const res = await fetch(`${base}/api/session/active`, { headers: serverHeaders() }) + if (!res.ok) continue + const body = (await res.json()) as unknown + const data = (body as { data?: unknown } | undefined)?.data ?? body + if (data && typeof data === "object") { + const ids = Object.keys(data as ActiveMap).filter((k) => k.startsWith("ses_")) + activeCache = { at: now, ids } + return ids + } + } catch { + // Next base URL, then fail-open. + } + } + return null + } + + /** Busy sessions we are tracking, excluding cancelled ones. The orphan watch + * keys on this count: a subagent finishing is only interesting because it + * leaves exactly one session busy. */ + function busySessionsForOrphanWatch(): string[] { + const out: string[] = [] + for (const [sid, w] of sessions) { + if (w.status === "busy" && !w.userCancelled) out.push(sid) + } + return out + } + + /** + * The subagents of `parentSid`, by v2's own parent link. + * + * v1 had no way to ask, so it treated *any other busy session* as a subagent of + * the one in hand. That works on a single-project box and misfires on a busy + * one: a second unrelated session looks like a child, and the parent gets + * aborted for someone else's work. v2 stores `parent_id` on the session row, so + * the question can simply be asked. + * + * The filter is requested *and* verified client-side. `parentID` is part of + * the list input, but the only consequence of it being quietly ignored would be + * aborting real sessions, so the returned rows are checked rather than trusted. + */ + async function listSubagentIds(parentSid: string): Promise { + const listed = unwrapList(await callSessionApi("list", { parentID: parentSid })) + const out: string[] = [] + for (const row of listed) { + const sid = row?.id + if (typeof sid !== "string" || !sid.startsWith("ses_")) continue + if (sid === parentSid) continue + if ((row as { parentID?: unknown }).parentID !== parentSid) continue + out.push(sid) + } + if (out.length === 0) { + // HTTP fallback: prod ctx has no list surface, so the call + // above always misses there (verified 2026-10-06). Same + // client-side verification as above. + for (const row of await listSessionsHttp()) { + const sid = row?.id + if (typeof sid !== "string" || !sid.startsWith("ses_")) continue + if (sid === parentSid) continue + if (row?.parentID !== parentSid) continue + out.push(sid) + } + } + return out + } + + type SubagentVerdict = { status: "crashed" | "idle" | "busy" | "waiting"; stuckSid?: string } + + /** + * What a parent's subagents are doing, in v1's three-way shape. + * + * v1 scans the global status map for any other *busy* session and treats a + * quiet one as "idle" — the parent is stuck with nothing to wait for. That + * reading cannot work on v2, and the reason is worth stating: this function is + * only ever reached *after* a child went idle, so a child being absent from + * the active set is the premise, not the finding. Reading that absence as + * "idle" would make every crashed subagent look like a healthy one, and the + * whole feature would abort the parent without ever trying to wake the child. + * + * So a child is judged on what it last did, not on whether the server still + * lists it: + * + * - **active, and recent** — running. Wait. + * - **active, and silent past the threshold** — hung. The fuse is tripled while + * a tool call is still outstanding, because a five-minute build is not a + * dead model and killing the parent there loses the work. + * - **not active, recent** — it finished. Nothing to wait for. + * - **not active, silent past the threshold** — it stopped without ever + * reporting. Waking it is worth one attempt. + */ + async function subagentVerdict(parentSid: string, activeIDs: Set): Promise { + try { + const children = await listSubagentIds(parentSid) + if (children.length === 0) return { status: "idle" } + const now = Date.now() + let sawBusy = false + let sawWaiting: string | null = null + for (const child of children) { + const messages = await loadMessages(child) + const last = messages[messages.length - 1] as + | { + type?: string + error?: unknown + time?: { created?: number } + content?: Array<{ type?: string; state?: { status?: string } }> + } + | undefined + if (last && msgRole(last) === "assistant" && last.error !== undefined) { + dbg(`subagent ${short(child)} reported an error`) + return { status: "crashed" } + } + const msgTime = msgAt(last) || undefined + const silentFor = typeof msgTime === "number" ? now - msgTime : Infinity + // An unanswered tool part is the "still working, slowly" case. + const hasToolCall = (last?.content ?? []).some( + (p) => p?.type === "tool" && p.state?.status !== "completed" && p.state?.status !== "error", + ) + const limit = hasToolCall ? SUBAGENT_STUCK_MS * 3 : SUBAGENT_STUCK_MS + // Outer bound: quiet past the stuck window but inside the dead + // window, with no error evidence, is a long thinker — waited + // on, not killed. Only past-dead quiet reads as crashed, so + // a hung model still terminates (bounded wait, ses_ef1c822c). + // Error evidence above already returned; scan on so a later + // crashed sibling is still surfaced. + const deadLimit = hasToolCall ? subagentDeadMs * 3 : subagentDeadMs + if (silentFor <= limit) { + // Recent enough to be believed, whatever the server thinks. + if (activeIDs.has(child)) sawBusy = true + continue + } + if (silentFor <= deadLimit) { + dbg( + `subagent ${short(child)} quiet for ${Math.round(silentFor / 1000)}s, inside ${Math.round(deadLimit / 1000)}s patience — waiting, not crashed (active=${activeIDs.has(child)})`, + ) + if (sawWaiting === null) sawWaiting = child + continue + } + dbg( + `subagent ${short(child)} silent for ${Math.round(silentFor / 1000)}s (active=${activeIDs.has(child)})`, + ) + return { status: "crashed", stuckSid: child } + } + if (sawBusy) return { status: "busy" } + if (sawWaiting !== null) return { status: "waiting", stuckSid: sawWaiting } + return { status: "idle" } + } catch (e) { + dbg(`subagent check failed for ${short(parentSid)}:`, e instanceof Error ? e.message : String(e)) + // Unknown is treated as busy: the cost of waiting is a later abort, and + // the cost of guessing wrong is a killed session. + return { status: "busy" } + } + } + + /** + * Nudge a stuck subagent directly. + * + * v1 uses the client prompt route. v2's plugin-scoped `session.synthetic` + * takes an explicit sessionID and appends the message to that session, which is + * the same effect without needing a cross-session route the plugin API does + * not expose. + * + * The general `injectOnce` refuses subagents, and correctly so: a child is + * not ours to recover. This is the one exception, and it is deliberate — the + * parent is stuck *because* the child is, so waking the child is cheaper than + * killing the parent and its whole turn. + */ + async function recoverStuckSubagent(sid: string): Promise { + if (!running) return false + try { + await callSessionApi("synthetic", { sessionID: sid, text: SUBAGENT_RECOVERY_PROMPT }) + log("info", `${short(sid)} recovery prompt sent to stuck subagent`) + return true + } catch (e) { + log("warn", `failed to recover subagent ${short(sid)}: ${e instanceof Error ? e.message : String(e)}`) + return false + } + } + + /** + * The orphan watch. + * + * The failure it exists for: a parent dispatches a subagent, the subagent dies + * without a terminal event, and the parent waits forever on a result that is + * never coming. Nothing is busy, nothing is idle-from-the-runtime's-point-of-view, + * and every ordinary watchdog path declines to act because the silence is shorter + * than `chunkTimeoutMs`. + * + * Armed from `session.idle`: when the busy count drops from more than one to + * exactly one, the survivor is a parent whose children have all gone quiet. + * After `subagentWaitMs` the watchdog asks what the children are doing and, if + * there is no live work left, aborts and resumes the parent. + */ + /** + * Arm the watch on a session that just outlived its subagents — but only if it + * really had some. + * + * The busy-count drop alone is not evidence. Two unrelated conversations open + * at once, one of them finishes, and the survivor looks identical to a parent + * whose child died: one session left, busy. v1 has no way to tell them apart + * and aborts the survivor either way, which on a busy box means killing + * somebody's conversation for a colleague's finishing turn. + * + * v2 can tell them apart, so it does: one listing call at arm time turns a + * guess into a fact. The cost is one query per 2-to-1 transition, and the + * alternative is an abort that cannot be undone. + */ + async function armOrphanWatch(sid: string): Promise { + const w = sessions.get(sid) + if (!w || w.orphanWatchStartAt !== null) return + if (w.status !== "busy" || w.userCancelled || w.completionSignaled) return + try { + if (await isSubAgentSession(sid)) return + const children = await listSubagentIds(sid) + // Re-checked after the await: the session may have gone idle, been + // cancelled, or been picked up by another arming while we were listing. + if (w.orphanWatchStartAt !== null || w.status !== "busy" || w.userCancelled) return + if (children.length === 0) { + dbg(`${short(sid)} outlived a busy session but has no subagents — not an orphan parent`) + return + } + w.orphanWatchStartAt = Date.now() + w.orphanWatchArmedAt = w.orphanWatchStartAt + w.orphanRecoveryTried = false + // orphanEscalatedChildSid is deliberately NOT cleared here. It records + // that a specific dead child has already cost this parent one abort. + // Arming happens on EVERY 2-to-1 busy transition, and re-arming is + // exactly what produced the 8 escalations in 3 hours (ses_ed726872); + // clearing here wiped the record before the next verdict could read + // it, so the guard never fired. Cleared only on teardown, on gaveUp, + // or superseded by a genuinely different child in runOrphanWatch. + log( + "info", + `subagents of ${short(sid)} fell quiet while it stayed busy. orphan watch (${subagentWaitMs / 1000}s, ${children.length} subagent(s))`, + ) + } catch (e) { + dbg(`orphan arming check failed for ${short(sid)}:`, e instanceof Error ? e.message : String(e)) + } + } + + async function runOrphanWatch(sid: string, w: SessionWatch, now: number, activeIDs: Set): Promise { + if (now - w.orphanWatchStartAt! < subagentWaitMs + gracePeriodMs) return + if (w.resumeAttempts >= maxRetries) { + if (!w.gaveUp) { + w.gaveUp = true + w.orphanWatchStartAt = null + w.orphanWatchArmedAt = null + // gaveUp is a legitimate reset: the watch stood down for good, so a + // future episode (even with the same dead child) starts clean. + w.orphanEscalatedChildSid = undefined + log("warn", `${short(sid)} orphan watch gave up after ${w.resumeAttempts} attempts`) + } + return + } + // Never abort a parent that is running a tool. Two independent sources, + // because the counter can miss a tool that started before the plugin loaded: + // our own event-derived count, and the tool parts in the message history. + if (w.pendingTools > 0) { + dbg(`${short(sid)} parent has ${w.pendingTools} tool(s) in flight — deferring orphan abort`) + w.orphanWatchStartAt = now + return + } + if (hasPendingUserInput(await loadMessages(sid))) { + dbg(`${short(sid)} parent is waiting on the user — deferring orphan abort`) + w.orphanWatchStartAt = now + return + } + // Never abort a parent the user just spoke to. `hasPendingUserInput` + // above only sees a pending INTERACTIVE tool — a question awaiting an + // answer. An instruction ("clean up any remaining residue") is a plain + // user message, so it sailed straight through and the abort landed on + // top of it: ses_ed726872, 187ms after the user's message, escalated + // `interrupt + fresh continue` onto a live turn. The idle path has + // always had this guard (shouldStandDownForUser); the orphan path did + // not. + // + // The newest user message is read from HISTORY, not from + // w.lastGenuineUserAt: that field only advances on the execution/step + // event handlers (the watchdog skips noteInboundUserMessage for a busy, + // non-cancelled parent), so on a timer tick it can still hold a + // pre-message value — which is precisely the race this closes. Reading + // history is the same source hasPendingUserInput just used, so it + // cannot lag behind it. + // + // Compared against orphanWatchArmedAt, NOT orphanWatchStartAt: the + // latter is advanced on every deferral above, so a deferral would age + // out the very message it was deferring for and the next tick would + // fire anyway. + if (w.orphanWatchArmedAt !== null) { + const msgs = await loadMessages(sid) + let newestUserAt = 0 + for (const m of msgs as Array<{ type?: string; role?: string; time?: { created?: number }; info?: { time?: { created?: number } } }>) { + if (msgRole(m) !== "user") continue + const at = msgAt(m) + if (typeof at === "number" && at > newestUserAt) newestUserAt = at + } + if (newestUserAt > w.orphanWatchArmedAt) { + dbg( + `${short(sid)} user spoke ${Math.round((now - newestUserAt) / 1000)}s ago, after the orphan watch armed — deferring orphan abort`, + ) + return + } + } + + const verdict = await subagentVerdict(sid, activeIDs) + if (verdict.status === "crashed") { + // One dead subagent may cost the parent exactly ONE abort. + // ses_ed726872: `…Ut64EfTB` ended 2.5h before, kept returning a + // crashed verdict, and the parent was aborted + re-continued 8 + // times across 3 hours. `orphanRecoveryTried` bounds only the WAKE + // attempt; once that fails, every later crashed verdict escalated + // again. Recording the child closes that loop. A genuinely + // different stuck child still gets its own escalation, and a new + // orphan episode (new arm) clears the record. + if (verdict.stuckSid && w.orphanEscalatedChildSid === verdict.stuckSid) { + // Already counted against the budget when it was escalated; spend it + // again so the episode still TERMINATES via the gaveUp latch instead + // of being held here forever. The no-cycling guarantee (the one the + // subagent-patience suite pins) is preserved; only the abort is + // skipped. + w.resumeAttempts++ + if (w.resumeAttempts >= maxRetries) { + if (!w.gaveUp) { + w.gaveUp = true + log("warn", `${short(sid)} orphan watch gave up after ${w.resumeAttempts} attempts`) + } + } else { + w.orphanWatchStartAt = now + } + dbg( + `${short(sid)} subagent ${short(verdict.stuckSid)} already escalated this episode — not re-aborting the parent`, + ) + return + } + // Waking the child is cheaper than killing the parent, so it is tried + // first — but exactly once. If the child does not come back, the parent + // is the only thing left to save and the watch stops asking. + if (verdict.stuckSid && !w.orphanRecoveryTried) { + w.orphanRecoveryTried = true + if (await recoverStuckSubagent(verdict.stuckSid)) { + w.orphanWatchStartAt = now + return + } + } + log("info", `${short(sid)} subagent crashed and did not recover — aborting and resuming the parent`) + if (verdict.stuckSid) w.orphanEscalatedChildSid = verdict.stuckSid + // Count orphan aborts toward the gaveUp budget (checked at this + // function's top): without this the watch cycles forever, because + // nothing else increments resumeAttempts on this path. + w.resumeAttempts++ + tryAbortAndResume(sid, w) + return + } + if (verdict.status === "waiting") { + dbg(`${short(sid)} subagents quiet but inside patience — waiting, not aborting`) + w.orphanWatchStartAt = now + return + } + if (verdict.status === "busy") { + dbg(`${short(sid)} subagents still working — waiting`) + w.orphanWatchStartAt = now + return + } + // No subagent is active and none is stuck enough to name. v1 aborts here, + // and so does this — the thing standing between that and a wrong kill is + // subagentWaitMs, which is how long the parent was given to consume a + // finished child's result before this runs at all. Set it to something + // small and this becomes a blunt instrument; the option's own default is + // 15s, and its name is the warning. + log("info", `${short(sid)} stuck with no live subagents — aborting and resuming`) + tryAbortAndResume(sid, w) + w.resumeAttempts++ + } + + /** + * Parent-wait guard for the busy-stall path. A parent blocked on a live + * subagent is quiet on its own sessionID by construction — the child's + * events belong to the child — so "no activity for Ns" must not by itself + * trigger recovery. Deliberately independent of the `lastWasTaskTool` + * heuristic below: native dispatches may never emit `session.tool.called`, + * and the server active set may lag the parentID link. + * + * Waits when the verdict confirms a live child or a quiet one inside + * the patience window, or when the parent still holds tools in flight + * against children that are neither confirmed live nor proven dead. + * A crashed verdict, or tools held with no children on the link at + * all (wedged bare tool), falls through to normal recovery. + */ + async function parentWaitingOnSubagents(sid: string, activeIDs: Set): Promise { + const verdict = await subagentVerdict(sid, activeIDs) + if (verdict.status === "busy" || verdict.status === "waiting") return true // live subagent confirmed via parentID — don't nudge + if (verdict.status === "crashed") return false // dead subagent — recover the parent + // verdict.status === "idle": no children discoverable via parentID. + const w = ensureWatch(sid) + if (w.pendingTools <= 0) return false // no outstanding work — not waiting on anything + // pendingTools > 0 but no live child discoverable (parentID not + // propagated or timing gap). Do NOT unconditionally skip: a dead tool + // (no success/failed event) keeps pendingTools > 0 forever and looks + // identical to a slow-but-live worker in every signal we have. + // Falling through lets stall detection recover it. The earlier + // inflightTools / hasYoungUnfinishedWork TTL guards already protect + // recently-dispatched tools; old outstanding tools are treated as stuck. + return false + } + + /** + * Stall-path young gate, per tool. Interactive tools answer in seconds, + * so 60s of event-silence means wedged; shells routinely run minutes + * with no events, so they get the long TTL. Exempt names are handled + * by their own branch at each call site, never here. + */ + function stallYoungAgeMsFor(name: string): number { + return LONG_RUNNING_TOOLS.includes(name) ? LONG_RUNNING_TOOL_STALE_TTL_MS : stuckToolAgeMs + } + + /** + * Durable liveness: is a tool call still unfinished in this session's HISTORY? + * + * `openShells` and `inflightTools` are per-instance event derivations: a + * plugin reload (or a missed event) empties them while work is still running, + * and the stall ladder then fires on a live session. Message history survives + * reloads, so a tool part still unfinished there is the durable form of the + * same signal. Observed 2026-10-11: sixteen consecutive + * `silent with 1 shell(s) still running` deferrals on a live `bun test`, + * then a reload adopted the watch without the shell registry and the ladder + * fired `stall detected (no activity for 186s)`. + * + * Mirrors the event-based young-tool check and the stuck-tool pass: only + * parts from the current turn (newer than the last user message) count, and + * only young ones — or exempt names inside their stale TTL. An OLD + * unfinished part is owned by the stuck-tool pass, which already nudged the + * model to proceed without it; letting the stall answer there is the designed + * backstop, not a bug. + */ + async function hasYoungUnfinishedWork(sid: string, now: number): Promise { + const messages = await loadMessages(sid) + let lastUserAt = 0 + for (const m of messages) { + if (msgRole(m) === "user") lastUserAt = Math.max(lastUserAt, msgAt(m)) + } + for (const m of messages as Array<{ + type?: string + time?: { created?: number } + content?: Array<{ type?: string; name?: string; state?: { status?: string }; time?: { created?: number } }> + }>) { + if (!m || msgRole(m) !== "assistant" || !Array.isArray(m.content)) continue + for (const part of m.content) { + if (part?.type !== "tool") continue + const st = part?.state?.status + if (st === "completed" || st === "error") continue + const created = + typeof part?.time?.created === "number" + ? part.time.created + : typeof m?.time?.created === "number" + ? m.time.created + : 0 + if (created <= lastUserAt) continue + const name = typeof part?.name === "string" && part.name ? part.name.toLowerCase() : "" + const age = now - created + if (neverInterruptTools.includes(name)) { + if (age < NEVER_INTERRUPT_STALE_TTL_MS) return true + continue + } + if (age < stallYoungAgeMsFor(name)) return true + } + } + return false + } + + async function checkActiveSessions() { + const now = Date.now() + + // Cross-check our busy set against server truth (also catches sessions + // we never saw an execution.started for, e.g. after plugin reload). + const activeIDs = await getActiveSessions() + for (const sid of activeIDs) { + const w = ensureWatch(sid) + if (w.status !== "busy") markBusy(sid) + } + const activeSet = new Set(activeIDs) + + for (const [sid, w] of sessions) { + if (w.status !== "busy" || w.userCancelled) { + // A stood-down session still gets its inbound mail read: a + // genuine new user message re-arms (clears userCancelled + // inside noteInboundUserMessage), so the latch cannot block + // its own clearing path. Skipped otherwise. + if (w.status === "busy" && w.userCancelled) { + noteInboundUserMessage(sid, w, await loadMessages(sid)) + if (w.userCancelled) continue + } else if (w.status === "idle" && !w.userCancelled) { + // Already-parked sessions emit no further events, so the + // event-driven idle pass never revisits them. Sweep here, + // throttled to ~one history read per stall window: a + // reasoning-only turn with no output means parked, and + // only the watchdog can still see it. + const idleFor = now - (w.idleSince ?? w.lastActivityAt) + if (idleFor > chunkTimeoutMs && now - w.lastSilentReasoningCheck > chunkTimeoutMs) { + w.lastSilentReasoningCheck = now + await checkForSilentReasoningTurn(sid, w) + } + // Re-run the stuck-tool pass for a parked session. The + // event-driven pass runs once, on `session.idle`; if the call + // was still inside the age gate then (which is normal — the + // turn ends the moment the model stops emitting, not the + // moment the tool answers), it is declined by design and a + // parked session emits no further events, so nothing would + // ever re-examine it. The call's own age gate still decides: + // this only catches calls that have since aged into stuck. + // Throttled to the age gate so the cost is ~one history read + // per gate, and reusing checkForStuckToolCalls means the same + // locks, budgets and busy-shell guard apply as on the idle path. + if (idleFor > stuckToolAgeMs && now - w.lastStuckToolCheck > stuckToolAgeMs) { + w.lastStuckToolCheck = now + await checkForStuckToolCalls(sid, w) + } + continue + } else { + continue + } + } + // Warmup: a session that only just went busy has not had a chance to + // emit anything. Without this a freshly-started turn can be declared + // stalled while it is still queueing its first model call. + if (now - w.createdAt < warmupMs) continue + // The orphan watch comes first, ahead of the silence check below, + // because that is the whole point of it: a parent waiting on a dead + // subagent has been quiet for less than chunkTimeoutMs, so every + // ordinary path would decline to act. + if (w.orphanWatchStartAt !== null) { + if (!w.aborting && !w.gaveUp && !w.completionSignaled) { + await runOrphanWatch(sid, w, now, activeSet) + } + continue + } + // A latched completion stands down the stall path too: the session + // declared itself done (task_complete tool, celebration, or an + // explicit stop). Nudging a completed quiet session is the nagging + // this exists to prevent. + // + // The latch re-opens ONLY on genuine new work: a user message newer + // than the latch itself, read here from history (not from + // lastGenuineUserAt, which lags on timer ticks for busy parents). + // noteInboundUserMessage is the arbiter — it already distinguishes + // genuine messages from our own prompts, so own injections never + // re-open it. Plumbing transitions (watchdog re-marks, discovery + // adopts, step events) must NOT clear it: ses_ed33c0da (2026-10-11) + // would have latched TASK COMPLETE at idle and lost it at the + // re-mark 1s later, stalling at +197s anyway. + if (w.completionSignaled) { + noteInboundUserMessage(sid, w, await loadMessages(sid)) + if (w.lastGenuineUserAt > w.completionSignaledAt) { + w.completionSignaled = false + w.completionSignaledAt = 0 + dbg(`${short(sid)} genuine new user message after latched completion — re-opening stall detection`) + } else { + dbg(`${short(sid)} completion latched — skipping stall detection`) + continue + } + } + // Stale compaction flag: a compaction that never reports + // ended/failed within the TTL is wedged — clear the guard so + // recovery can eventually intervene. + if (w.compacting && w.compactionStartedAt && now - w.compactionStartedAt > COMPACTION_STALE_TTL_MS) { + w.compacting = false + w.compactionStartedAt = null + log("warn", `${short(sid)} compaction flag stale (>${COMPACTION_STALE_TTL_MS / 60000}m) — clearing guard`) + } + // NEVER interrupt a session that is mid-compaction. + if (w.compacting) { + dbg(`${short(sid)} mid-compaction — skipping stall detection`) + continue + } + const silence = now - w.lastActivityAt + // Episode backoff: each progress-restarted refire without + // genuine user input doubles the fuse (cap x8), so a + // slow-trickling session decays to patience instead of + // nagging on a fixed 3-minute clock (ses_ed975114). + // Genuine user input resets episodeRefires to 0. + const fuseMult = Math.min(8, 2 ** Math.max(0, w.episodeRefires)) + if (silence < (chunkTimeoutMs + gracePeriodMs) * fuseMult) continue + // A quiet turn that already declared completion, but no idle + // inspection ever latched it: no session.idle event arrived, or a + // take-over landed mid-silence. Judge the tail from history + // rather than firing the stall. ses_edd1eb51 (2026-10-09) fired + // resume attempt 1/3 186s after a "TASK COMPLETE" delivery + // exactly this way. Same reminder-or-latch semantics as the idle + // stop path. Placed past the silence gate so a partial stream + // (still emitting deltas) can never match an end-anchored claim, + // and so the history read happens at most once per stall window. + { + const histText = lastAssistantTextFrom(await loadMessages(sid)) + // Hand-off turns wait on the user the same way completed ones + // are finished: nudging either talks over someone. A hand-off + // is not latched (no completion claimed) — it is only + // skipped while its text is still the tail. + if (histText && isUserHandoff(histText)) { + dbg(`${short(sid)} quiet turn ends with a hand-off — waiting on user, not stalled`) + continue + } + if (histText && turnEndsWithCompletionClaim(histText)) { + const open = getOpenTodos(await readTodos(sid)) + if (open.length > 0) { + await targetedRecovery(sid, "stop-open-todos", buildOpenTodosReminder(open), "todoNudgeAttempts") + } else if (!w.completionSignaled) { + w.completionSignaled = true + w.completionSignaledAt = Date.now() + log("info", `${short(sid)} quiet turn ends with a completion claim and no open todos — latching completion, not nudging`) + } else { + dbg(`${short(sid)} completion already latched — skipping`) + } + continue + } + } + if (busyStallStrategy === "off") { + dbg(`stall on ${short(sid)} ignored (busyStallStrategy=off)`) + continue + } + // Same stale-guard as the compaction flag: an unanswered `permission.asked` + // must not stand this session down forever. + clearStalePermissionFlag(sid, w) + if (w.permissionPending) { + dbg(`${short(sid)} silent but permission pending — skipping`) + continue + } + if (selfAbortActive(w)) { + dbg(`${short(sid)} silent but inside our own abort window — skipping`) + continue + } + // A session with a shell still running is working, not stalled. This + // covers the parked-parent case the task-tool heuristic below cannot + // see: a backgrounded `shell` spawns no child session, so + // `lastWasTaskTool` stays false and `others.length` never gets a + // chance to say "this parent is waiting on something". + const busyShells = openShellCount(sid) + if (busyShells > 0) { + dbg(`${short(sid)} silent with ${busyShells} shell(s) still running — waiting`) + continue + } + // A tool call still running is working, not stalled — even with no + // events for longer than the chunk timeout. A 4-minute `write` or + // `compress` emits nothing while it runs, so event-silence accrues + // and the injection below would interrupt the live call ("Tool + // execution interrupted"). Young tools defer; tools older than the + // stuck age gate are wedged and the stall answers normally. Exempt + // names (default: compress) are never interrupted while fresh — + // interrupting a compress risks half-compacted state, the same + // reason compaction itself is never interrupted — with the same + // stale-TTL escape when one wedges for real. + { + const now = Date.now() + let youngTool = false + // ?? []: a watch adopted from a pre-map build may reach this loop + // before any ensureWatch call migrates it (the sweep iterates the + // map directly). A missing map reads as no tools, never a throw. + for (const [, rec] of w.inflightTools ?? []) { + const age = now - rec.at + if (neverInterruptTools.includes(rec.name)) { + if (age < NEVER_INTERRUPT_STALE_TTL_MS) { + dbg(`${short(sid)} silent with exempt tool "${rec.name}" still running (${Math.round(age / 1000)}s) — waiting`) + youngTool = true + break + } + continue + } + if (age < stallYoungAgeMsFor(rec.name)) youngTool = true + } + if (youngTool) continue + } + // Durable form of the two guards above. Both read per-instance event + // derivations that a reload empties while work is still running; + // history survives, so an unfinished tool part there means the + // session is working no matter what the event counters say. + // Placed last so the cheap event checks run first and this history + // read happens only when they show nothing. + if (await hasYoungUnfinishedWork(sid, now)) { + dbg(`${short(sid)} tool call still unfinished in history — working, not stalled`) + continue + } + // If another session is actively running and this one went silent + // right after dispatching a task tool, treat it as a parent wait. + if (w.lastWasTaskTool) { + const others = (await getActiveSessions()).filter((s) => s !== sid) + if (others.length > 0) { + dbg(`${short(sid)} silent after task tool with ${others.length} active child(ren) — waiting`) + continue + } + } + // A parent waiting on a live subagent is working, not stalled. The + // heuristic above misses native dispatches (no tool.called event) + // and active-set lag; the parentID link does not (see + // parentWaitingOnSubagents, ses_ef73a206 2026-10-04). + if (await parentWaitingOnSubagents(sid, activeSet)) { + dbg(`${short(sid)} waiting on subagents — skipping stall detection`) + continue + } + // A turn parked on user input (a `question` awaiting an answer, + // an unanswered permission beyond the event flow) is waiting, + // not stalled. Narrower than the idle-path check on purpose: a + // wedged bare tool with zero events is still a stall. + if (isWaitingOnUser(sid, w, await loadMessages(sid))) { + dbg(`${short(sid)} waiting on user input — skipping stall detection`) + continue + } + // busyStallStrategy picks how a stall is answered. "abort" interrupts + // the wedged step before continuing, because a stream that went + // silent mid-generation will not pick the prompt up on its own. + // Mirrors the v1 branch at src/index.ts. + if (busyStallStrategy === "abort" && w.resumeAttempts < maxRetries) { + log("info", `${short(sid)} stall (busyStallStrategy=abort): aborting before continue`) + await tryAbortAndResume(sid, w) + } else { + await recover(sid, `no activity for ${Math.ceil(silence / 1000)}s`) + } + } + // Rate-limit gate: sessions stood down on quota get one attempt per + // served cooldown, evaluated here so no armed timer can be lost to + // a reload. Guards mirror the failure handler; `recover()` applies + // its own budget, loop and inject guards on top. + for (const [rsid, rw] of sessions) { + if (!rw.awaitingQuotaRetry) continue + const ladder = rateLimitCooldownsMs + const rn = rw.rateLimitAttempts + if (rn >= ladder.length) { + rw.awaitingQuotaRetry = false + continue + } + if (rw.userCancelled || rw.gaveUp || rw.compacting || rw.permissionPending) continue + if (selfAbortActive(rw)) continue + if (Date.now() - rw.rateLimitedAt < ladder[rn]) continue + serveQuotaRung(rsid, rw, rn, ladder) + } + cleanupIdleSessions() + } + + // Re-entrancy guard: `checkActiveSessions` awaits per-session history reads and + // subagent polls, so a sweep can outlast the interval and the next tick would + // start a second concurrent sweep over the same SessionWatch objects. Audit + // 2026-10-10. Skipping is the safe direction — the next tick picks up whatever + // this one skipped — whereas overlapping means two evaluators can both decide a + // stall is real and both recover it. + let watchdogBusy = false + const watchdog = setInterval(() => { + if (watchdogBusy) { + dbg("watchdog sweep still running — skipping this tick") + return + } + watchdogBusy = true + checkActiveSessions() + .catch((e) => log("error", `watchdog failed: ${e instanceof Error ? e.message : String(e)}`)) + .finally(() => { + watchdogBusy = false + }) + }, checkIntervalMs) + + // Discovery sweep. Separate timer from the watchdog because the two have + // different jobs and different costs: the watchdog runs every few seconds + // and must stay cheap, discovery lists every session and runs every 60s. + // The initial sweep waits out discoveryDelayMs so plugin load does not + // compete with the turn that triggered it. + const discoveryTimer = setInterval(() => { + discoverSessions().catch((e) => + log("error", `discovery failed: ${e instanceof Error ? e.message : String(e)}`), + ) + }, DEFAULT_SESSION_DISCOVERY_INTERVAL_MS) + const initialDiscovery = setTimeout(() => { + discoverSessions().catch(() => {}) + }, discoveryDelayMs) + + // Offered to the model before the first turn, so it can actually be called. + // Awaited, but guarded: a registry that refuses the registration must not + // stop the watchdog from starting. + await registerTaskCompleteTool() + + // --------------------------------------------------------------------- + // Context saturation + // --------------------------------------------------------------------- + + /** + * v2 emits `session.usage.updated` with the session's current usage, so the + * token count is read rather than reconstructed. `TokenUsage.total` in the + * schema sums input + output + reasoning + cache read + cache write; v1 + * accumulated the same way. + */ + function tokenTotalOf(tokens: unknown): number { + if (!tokens || typeof tokens !== "object") return 0 + const t = tokens as { + input?: number + output?: number + reasoning?: number + cache?: { read?: number; write?: number } + } + const cache = t.cache ?? {} + return posNum(t.input) + posNum(t.output) + posNum(t.reasoning) + posNum(cache.read) + posNum(cache.write) + } + function posNum(v: unknown): number { + return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : 0 + } + + /** + * The usable context window for a session's model: the full window minus the + * smaller of a 20k output reserve and the model's own output limit. This is + * v1's arithmetic (`limit.context - Math.min(20_000, limit.output)`), kept + * identical so the same threshold means the same thing on both builds. + * + * v2 hands the limit over directly on `ctx.model.get()` — `Model.Info.limit` + * is `{ context, input?, output }` — where v1 had to walk the raw provider + * list. Cached per model, because the answer never changes at runtime. + */ + const usableLimitCache = new Map() + async function getUsableContextLimit(sid: string): Promise { + try { + let ref = sessions.get(sid)?.modelRef + if (!ref) { + // No step observed yet (plugin loaded mid-session): take the ref off + // the newest assistant message instead. + const msgs = await ctx.session.context({ sessionID: sid }) + if (Array.isArray(msgs)) { + for (let i = msgs.length - 1; i >= 0; i--) { + const m = msgs[i] as { + role?: string + model?: { providerID?: string; id?: string; modelID?: string } + } + if (msgRole(m) !== "assistant" || !m.model) continue + const providerID = m.model.providerID + const modelID = m.model.modelID ?? m.model.id + if (typeof providerID === "string" && typeof modelID === "string") { + ref = { providerID, modelID } + const w = sessions.get(sid) + if (w) w.modelRef = ref + } + break + } + } + } + if (!ref) return null + const key = `${ref.providerID}/${ref.modelID}` + const cached = usableLimitCache.get(key) + if (cached !== undefined) return cached + + const info = (ctx.model as any)?.get?.(ref.providerID, ref.modelID) + if (!info) return null + const limit = (info as { limit?: { context?: number; output?: number } }).limit + if (!limit || typeof limit.context !== "number" || limit.context === 0) return null + const usable = limit.context - Math.min(20_000, limit.output ?? 0) + if (!Number.isFinite(usable) || usable <= 0) return null + usableLimitCache.set(key, usable) + return usable + } catch (e) { + dbg(`usable-context-limit lookup failed for ${short(sid)}:`, e instanceof Error ? e.message : String(e)) + return null + } + } + + /** + * v2 hands plugins a narrowed `ctx.session` domain (see + * `packages/plugin/src/promise/session.ts`) that omits `list`, `active` and + * `compact`, so those are read defensively: off the session domain first, + * then off the raw client. A host that supplies neither degrades to the + * event-derived busy set rather than failing. + * + * `args` matters for the client's methods, which take a request object + * (`{ sessionID }`), unlike the plugin-domain wrappers. + */ + async function callSessionApi(name: string, args?: Record): Promise { + const fromDomain = (ctx.session as any)[name] + if (typeof fromDomain === "function") { + try { + return (await fromDomain.call(ctx.session, args)) as T + } catch (e) { + dbg(`ctx.session.${name}() failed:`, e instanceof Error ? e.message : String(e)) + } + } + const fromClient = ctx.client?.session?.[name as "get"] + if (typeof fromClient === "function") { + try { + return (await (fromClient as any).call(ctx.client!.session, args)) as T + } catch (e) { + dbg(`ctx.client.session.${name}() failed:`, e instanceof Error ? e.message : String(e)) + } + } + return undefined + } + + /** + * Is magic-context installed? + * + * v1 read `config.get().plugin`. v2 has no `config` domain on the plugin + * context, but `ctx.plugin.list()` carries the same information in the + * server's own words: `Plugin.Info[]` with an `id` plus a `source` that is a + * package spec, a local path, or an SDK module. Matching id *and* source is + * what makes this work regardless of how the plugin was installed. + * + * Cached, negative verdict included — an idle check must not re-list plugins. + */ + let magicContextDetected: boolean | null = null + async function isMagicContextInstalled(): Promise { + if (magicContextDetected !== null) return magicContextDetected + try { + const list = (ctx.plugin as any)?.list + if (typeof list !== "function") { + dbg("magic-context detection: ctx.plugin.list unavailable, treating as not installed") + return false + } + const plugins = unwrapList(await list.call(ctx.plugin)) as Array<{ id?: string; source?: unknown }> + magicContextDetected = plugins.some((p) => { + const id = typeof p?.id === "string" ? p.id : "" + if (id.toLowerCase().includes("magic-context")) return true + const src = p?.source as { target?: string; path?: string } | undefined + const spec = src?.target ?? src?.path ?? "" + return typeof spec === "string" && spec.toLowerCase().includes("magic-context") + }) + dbg(`magic-context detection: ${magicContextDetected ? "installed" : "not installed"}`) + return magicContextDetected + } catch (e) { + dbg("magic-context detection failed, treating as not installed:", e instanceof Error ? e.message : String(e)) + return false + } + } + + /** Unwrap the `{ data }` envelope the client uses, or pass an array through. */ + function unwrapList(response: unknown): Array> { + if (Array.isArray(response)) return response as Array> + if (response && typeof response === "object") { + const data = (response as { data?: unknown }).data + if (Array.isArray(data)) return data as Array> + } + return [] + } + + /** + * v1's saturation routing, ported to the v2 API. + * + * A session can fill its window without stalling — it just keeps working + * until it chokes. On idle, when used/usable crosses the threshold, routing + * depends on session kind: + * + * subagent — opt-in only (`subagentNativeCompactionEnabled`). v1 called + * `session.summarize()`; v2 spells it `session.compact`, which + * is NOT in the plugin `session` Pick, so it goes through the same + * defensive lookup as `list`/`active`. + * parent — only when magic-context is installed, because its setup + * disables native compaction and compacting here would + * double-compress. v1 sent the `ctx-wrapup` command through the + * client; `session.command` IS on the plugin domain in v2. + * + * Fail-safe, as in v1: a missing limit, a missing token count, a user + * cancellation, or a signalled completion means no intervention at all. + * The intervention is one-shot per turn. + */ + async function checkContextSaturation(sid: string): Promise { + const w = sessions.get(sid) + if (!w) return + if (w.lastTokenTotal <= 0) return + if (w.contextWrapupAttempts >= 1) return + if (w.userCancelled || w.completionSignaled || w.aborting) return + + const usable = await getUsableContextLimit(sid) + if (!usable) return + if (w.lastTokenTotal / usable < contextSaturationThreshold) return + + const pct = Math.round((w.lastTokenTotal / usable) * 100) + if (await isSubAgentSession(sid)) { + if (!subagentNativeCompactionEnabled) { + dbg(`${short(sid)} subagent at ${pct}% of usable context; native compaction is opt-in`) + return + } + w.contextWrapupAttempts++ + log( + "warn", + `${short(sid)} context saturation (subagent): ${w.lastTokenTotal}/${usable} tokens (${pct}% of usable); triggering native compaction`, + ) + // Re-check the latches: the awaits above left a window for the user + // to cancel in. + if (w.userCancelled || w.completionSignaled) return + const compacted = await callSessionApi("compact", { sessionID: sid }) + if (compacted === undefined) { + log("warn", `${short(sid)} native compaction is not exposed on this host — skipping`) + } + return + } + + if (!(await isMagicContextInstalled())) { + dbg(`${short(sid)} at ${pct}% of usable context; magic-context not installed, not intervening`) + return + } + w.contextWrapupAttempts++ + log( + "warn", + `${short(sid)} context saturation: ${w.lastTokenTotal}/${usable} tokens (${pct}% of usable); sending magic-context wrapup command`, + ) + if (w.userCancelled || w.completionSignaled) return + try { + await (ctx.session as any).command({ sessionID: sid, name: CTX_WRAPUP_TRIGGER }) + } catch (e) { + log( + "warn", + `${short(sid)} magic-context wrapup command failed: ${e instanceof Error ? e.message : String(e)}`, + ) + } + } + + /** + * Seed watch state for sessions that already exist, and mark the ones + * currently running as busy. + * + * v1 did this by polling `session.list()` and trusting a `status` field + * on each row. v2 has something better: `session.active()` is the + * server's own record of what is running right now, so "busy" is read + * rather than guessed. A session that was already mid-turn when the + * plugin loaded would otherwise be invisible — no `execution.started` + * ever reaches us for it, so the stall watchdog would have nothing to + * watch. + */ + async function discoverSessions() { + try { + const listed = unwrapList(await callSessionApi("list")) + let seeded = 0 + for (const row of listed) { + const sid = row?.id + if (typeof sid !== "string" || !sid.startsWith("ses_")) continue + if (!sessions.has(sid)) { + ensureWatch(sid) + seeded++ + } + } + + // Authoritative busy set. Anything listed as running but not + // already tracked busy starts its stall clock now, not from + // whenever the plugin happened to attach. + const active = await callSessionApi>("active") + let adopted = 0 + if (active && typeof active === "object") { + for (const [sid, val] of Object.entries(active)) { + if (!sid.startsWith("ses_")) continue + const w = ensureWatch(sid) + if (w.status !== "busy") { + markBusy(sid) + adopted++ + } else { + // Already busy from events; refresh nothing, but make + // sure the session is not left `createdAt`-stale so + // the warmup window does not swallow its first stall. + void val + } + } + } + + if (seeded > 0 || adopted > 0) { + log("info", `discovery: seeded ${seeded} session(s), adopted ${adopted} already running`) + } else { + dbg("discovery: nothing new") + } + } catch (err) { + const msg = err instanceof Error ? err.message : String(err) + dbg(`session discovery failed: ${msg}`) + } + } + + /** + * Re-arm quota ladders lost to a reload. + * + * The ladder gate is in-memory (`awaitingQuotaRetry`, `rateLimitedAt`, + * `rateLimitAttempts`), and `dispose()` clears the session map — so any + * reload between arming and the next rung silently cancels the retry. + * The `quota-armed` marker lines in the (file-backed, reload-proof) log + * are the record. For each still-listed session with a marker: + * live state wins (already armed — nothing to do), stale markers past + * the ladder horizon are dropped, and a user message newer than the arm + * means the ladder already reset — re-arming would retry a session the + * user already continued. Otherwise the gate is restored at its rung and + * the watchdog serves it on schedule, possibly immediately if overdue. + */ + async function restoreQuotaLadders(): Promise { + let listed: Set + try { + listed = new Set(unwrapList(await callSessionApi("list")).map((r) => r.id).filter((id): id is string => typeof id === "string")) + } catch { + return + } + const horizon = rateLimitCooldownsMs.reduce((a, b) => a + b, 0) + + // Primary channel: the plugin's own storage record. No history read — + // `syncQuotaState` deletes the record the moment a genuine user + // message resets the ladder, so its absence is the answer to "did + // they speak after the arm?" That is what removes the per-session + // loadMessages() at boot, which was N sequential HTTP round-trips + // during startup. + const restored = new Set() + if (ctx.storage?.set) { + for (const sid of listed) { + let rec: unknown + try { + rec = await ctx.storage.get(`${QUOTA_STATE_PREFIX}${sid}`) + } catch (e) { + dbg(`${short(sid)} quota record read failed:`, e instanceof Error ? e.message : String(e)) + continue + } + const r = rec as { v?: number; at?: unknown; rung?: unknown; noticed?: unknown } | null + if (!r || typeof r.at !== "number") continue + if (Date.now() - r.at >= QUOTA_STATE_MAX_AGE_MS) { + void syncQuotaState(sid, ensureWatch(sid), true) + continue + } + if (Date.now() - r.at >= horizon) continue + const w = ensureWatch(sid) + if (w.awaitingQuotaRetry || w.rateLimitedAt > 0) continue + // Never arm a session that is already working. A user who + // messaged while the plugin was still starting has a busy + // watch by the time the watchdog sweeps it, and serving a + // restored ladder into that turn is the false nag this + // whole change exists to remove. Status is already in + // memory, so this guard costs no history read. + if (w.status === "busy") { + dbg(`${short(sid)} busy — skipping quota restore`) + continue + } + w.rateLimitedAt = r.at + w.rateLimitAttempts = typeof r.rung === "number" ? r.rung : 0 + w.awaitingQuotaRetry = true + // The arm that wrote this record also sent the user-visible + // notice, so carry that across the restart — otherwise the + // next hit re-announces a stand-down already announced. + w.quotaNoticedAt = typeof r.noticed === "number" && r.noticed > 0 ? r.noticed : r.at + restored.add(sid) + log("warn", `${short(sid)} quota ladder restored (armed ${Math.round((Date.now() - r.at) / 1000)}s ago, rung ${w.rateLimitAttempts}) — retry due on schedule`) + } + } + + // Fallback for arms written before R1 shipped, or by a host whose + // storage has no `set`. Costs a history read per session, so it runs + // only for sessions the storage channel did not already settle. + let text: string + try { + text = readFileSync(logFile, "utf8") + } catch { + return + } + const marks = new Map() + for (const line of text.split("\n").slice(-3000)) { + const m = /quota-armed sid=(ses_\S+) at=(\d+) rung=(\d+)/.exec(line) + if (m) marks.set(m[1], { at: Number(m[2]), rung: Number(m[3]) }) + } + if (marks.size === 0) return + for (const [sid, mark] of marks) { + if (!listed.has(sid) || restored.has(sid)) continue + const w = ensureWatch(sid) + if (w.awaitingQuotaRetry || w.rateLimitedAt > 0) continue + if (Date.now() - mark.at >= horizon) continue + const messages = await loadMessages(sid) + let lastUserAt = 0 + for (const m of messages) { + if (msgRole(m) === "user") lastUserAt = Math.max(lastUserAt, msgAt(m)) + } + if (lastUserAt > mark.at) continue + w.rateLimitedAt = mark.at + w.rateLimitAttempts = mark.rung + w.awaitingQuotaRetry = true + w.quotaNoticedAt = mark.at + log("warn", `${short(sid)} quota ladder restored (armed ${Math.round((Date.now() - mark.at) / 1000)}s ago, rung ${mark.rung}) — retry due on schedule`) + } + } + + /** + * Mirror the rate-limit ladder into `ctx.storage` so it survives a real + * process restart. Writes only while an episode is armed, deletes the + * record as soon as a genuine turn resets the ladder — so a restored + * ladder can never outlive the user having spoken. + * + * Best-effort by design: a storage failure must never block recovery, so + * every path here swallows and logs at debug. When `set` is missing + * (a host that only offers `get`) this is a no-op and the log markers + * remain the channel. + */ + async function syncQuotaState(sid: string, w: SessionWatch, force = false): Promise { + try { + const st = ctx.storage + if (!st?.set) return + const key = `${QUOTA_STATE_PREFIX}${sid}` + if (!w.rateLimitedAt) { + if (force || st.remove) await st.remove?.(key) + return + } + await st.set(key, { + v: 1, + at: w.rateLimitedAt, + rung: w.rateLimitAttempts, + noticed: w.quotaNoticedAt, + }) + } catch (e) { + dbg(`${short(sid)} quota state sync failed:`, e instanceof Error ? e.message : String(e)) + } + } + + // --------------------------------------------------------------------- + // Event stream + // --------------------------------------------------------------------- + + function handleEvent(ev: V2Event) { + switch (ev.type) { + // --- lifecycle ------------------------------------------------- + case "session.execution.started": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + if (w.selfRecovery) { + w.selfRecovery = false + } else { + clearLocks(w, sid) + // A genuine turn is activity, and activity means quota is + // available again: the rate-limit ladder starts over. Our + // own recovery turns (selfRecovery) do not reset it. + w.rateLimitedAt = 0 + w.rateLimitAttempts = 0 + w.quotaNoticedAt = 0 + w.awaitingQuotaRetry = false + // A genuine turn re-opens a stood-down session: the stall + // gaveUp latch clears here (and on genuine user messages + // and abort+resume), never on lifecycle transitions. + w.gaveUp = false + // Same for the empty-idle single-shot: a genuine turn is + // a new episode, so the instant nudge re-arms here. + // Same for the recovery count. + w.episodeRecovers = 0 + // Lifetime cap is per genuine episode too: a real turn + // is new work, not our nudge looping. Own recovery + // turns skip this branch entirely, so they keep counting. + w.lifetimeContinues = 0 + w.emptyIdleNudged = false + // Drop the durable arm with the in-memory one. This delete is + // what lets restore skip the history read: no record means the + // user has spoken since the arm, so there is nothing to serve. + void syncQuotaState(sid, w) + // A genuine turn is also a new episode for the stall + // ladder (the reset lives here and in the progress check, + // never on lifecycle transitions) — unless our inject is + // newer than any genuine user message, in which case the + // turn continues our episode (interleaving idle events do + // not end it). + if (!(w.lastInjectAt > 0 && w.lastInjectAt >= w.lastGenuineUserAt)) { + w.resumeAttempts = 0 + } + } + markBusy(sid) + return + } + case "session.usage.updated": { + // The session's current context usage. `tokenTotalOf` mirrors + // `TokenUsage.total` in the schema (input + output + reasoning + + // cache read + cache write) so the ratio means the same thing here + // as in v1, which accumulated the same five fields. + const sid = sidOf(ev) + if (!sid) return + const total = tokenTotalOf(ev.data?.tokens) + if (total <= 0) return + const w = ensureWatch(sid) + // Token flow is activity: a turn generating for minutes straight + // must not read as idle. Touch ONLY on growth — a flat total + // means nothing resumed, so the clock goes stale and the + // stall answers on schedule (2026-10-13: three recoveries + // into a live 1.9M-token turn). + if (total > w.lastTokenTotal) touch(sid) + w.lastTokenTotal = total + dbg(`${short(sid)} usage: ${total} tokens`) + return + } + + case "session.execution.succeeded": + case "session.idle": { + const sid = sidOf(ev) + if (!sid) return + // Turn-identity probe (2026-10-12): the idle evaluation judges + // "whatever is newest" (w.lastAssistantText || history read), + // which goes stale across markBusy transitions and lets stale + // tails defeat every guard, including the approval-question + // ones. If this event carries the ended turn's message id, + // the evaluation can read THAT turn instead of a snapshot. + // Logging only — remove once answered. + dbg(`${short(sid)} idle event data keys: ${JSON.stringify(Object.keys((ev as { data?: unknown })?.data ?? {}))}`) + // Counted across every tracked session, and read *before* this one goes + // idle, because the transition is the signal: a session leaving a crowd + // of busy sessions behind it. Sampling after markIdle would make every + // idle look like a drop to zero and the arming condition unreachable. + const busyBefore = busySessionsForOrphanWatch().length + // Snapshot before the transition for the same reason: markIdle + // zeroes the live counter, and idle fires while tools run. + const wPre = ensureWatch(sid) + wPre.toolsInFlightAtIdle = wPre.pendingTools + markIdle(sid) + const w = ensureWatch(sid) + w.pendingRecoveryArmed = false + w.selfRecovery = false // stale — no in-flight recovery prompt by the time we idle + // Never run the idle heuristics on top of our own abort: the idle + // transition caused by interrupt() would otherwise trigger a second, + // independent injection. + if (selfAbortActive(w)) { + dbg(`${short(sid)} idle after plugin-initiated abort — skipping targeted recovery`) + return + } + // Arm the orphan watch. A subagent going idle is only interesting because + // of what it leaves behind: when the busy count drops from more than one + // to exactly one, the survivor is a parent whose children have all gone + // quiet, and it may be waiting for a result that is never coming. + // + // Re-arming is guarded: an already-armed watch keeps its start time, or a + // stream of child idles would keep pushing the deadline out. + const busyNow = busySessionsForOrphanWatch() + if (busyBefore > 1 && busyNow.length === 1) { + void armOrphanWatch(busyNow[0]) + } + + void inspectOnIdle(sid) + // Independent of the idle heuristics above: this looks at tool parts + // that already errored, which none of those read. Fire-and-forget + // so it cannot delay them. The latch check lives inside the function + // rather than here, because the per-request re-arm is derived from the + // history and only happens once the history has been read. + if (!w.completionSignaled && !w.userCancelled) { + void checkForUnknownToolCalls(sid, w) + } + return + } + case "session.execution.interrupted": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + // Stable v2 carries the interrupt `reason`. Only a genuine user + // interrupt should suppress recovery; shutdown/superseded/inactivity + // (or a missing reason on older runtimes) must not. + const reason = typeof ev.data?.reason === "string" ? ev.data.reason : undefined + // Use the latch, not the transient `aborting` flag: the runtime + // delivers this event asynchronously, after `aborting` may have + // already been cleared. Only a genuine user stop latches + // userCancelled; `shutdown`/`superseded`/`inactivity` must not + // permanently disable recovery for the session. + const mine = w.aborting || selfAbortActive(w) + if (!mine) { + w.userCancelled = reason === undefined || reason === "user" + if (w.userCancelled) w.userCancelledAt = Date.now() + } + // A user interrupt also ends any in-flight compaction. + w.compacting = false + w.compactionStartedAt = null + w.recovering = false + markIdle(sid) + return + } + case "session.deleted": { + const sid = sidOf(ev) + if (!sid) return + forgetShells(sid) + sessions.delete(sid) + return + } + // --- reverts --- + // v1.18.34 (verified live, 2026-10-01, on an arch LXC with the real + // v1 plugin loaded): v1 has NO revert-named event. A revert arrives as + // `session.updated` whose `properties.info.revert` is populated with + // { messageID, partID?, snapshot?, diff? }, preceded by `session.diff`. + // So on v1 a plugin that only switches on the event *name* never sees a + // rewind at all — which is exactly why the v1 port never dropped its + // watch state on one. (An earlier comment here claimed v1 emitted + // `session.reverted`; that was wrong, and the case it defended is gone.) + // + // v2 emits a three-stage revert family. `staged` is intermediate + // (the user can still clear it), while `cleared` and `committed` + // are terminal; drop our watch state on the terminal stages so a + // rewind cannot leave a stale recovery armed against old turns. + case "session.revert.staged": { + const sid = sidOf(ev) + if (!sid) return + // Staging is user activity, but not terminal: keep the state. + touch(sid) + return + } + case "session.revert.cleared": + case "session.revert.committed": { + const sid = sidOf(ev) + if (!sid) return + forgetShells(sid) + sessions.delete(sid) + return + } + + // --- compaction (NEVER interrupt a session that is compacting) --- + case "session.compaction.started": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + if (!w.compacting) { + w.compacting = true + w.compactionStartedAt = Date.now() + log("info", `${short(sid)} compaction started — auto-resume recovery suspended until it ends`) + } + touch(sid) + return + } + case "session.compaction.ended": + case "session.compaction.failed": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + if (w.compacting) { + w.compacting = false + w.compactionStartedAt = null + log("info", `${short(sid)} compaction ${ev.type.endsWith("failed") ? "failed" : "ended"} — guard cleared`) + } + touch(sid) + return + } + + // --- activity -------------------------------------------------- + case "session.step.started": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + w.agent = typeof ev.data?.agent === "string" ? ev.data.agent : w.agent + { + const m = ev.data?.model as { providerID?: string; provider?: string; modelID?: string; id?: string } | undefined + const providerID = m?.providerID ?? m?.provider + const modelID = m?.modelID ?? m?.id + if (typeof providerID === "string" && typeof modelID === "string") { + w.modelRef = { providerID, modelID } + w.model = `${providerID}/${modelID}` + } else if (m) { + w.model = `${providerID ?? "?"}/${modelID ?? "?"}` + } else { + w.model = w.model + } + } + w.lastWasTaskTool = false + markBusy(sid) + return + } + case "session.step.ended": { + const sid = sidOf(ev) + if (!sid) return + touch(sid) + return + } + // Liveness between step start and the first token. Without these a + // step that starts and then produces nothing for `chunkTimeoutMs` + // looks identical to a stall. + case "session.step.streamed": + case "session.text.started": + case "session.reasoning.started": + case "session.synthetic": { + const sid = sidOf(ev) + if (!sid) return + touch(sid) + return + } + // A long tool-input generation (e.g. a big heredoc write) emits no + // text deltas; treating that silence as a stall resumes a session + // that is in fact working. + case "session.tool.input.started": + case "session.tool.input.delta": { + const sid = sidOf(ev) + if (!sid) return + touch(sid) + return + } + case "session.text.delta": + case "session.reasoning.delta": { + const sid = sidOf(ev) + if (!sid) return + appendText(sid, ev.data?.assistantMessageID, typeof ev.data?.delta === "string" ? ev.data.delta : "") + touch(sid) + return + } + case "session.text.ended": + case "session.reasoning.ended": { + const sid = sidOf(ev) + if (!sid) return + touch(sid) + return + } + + // --- tools ------------------------------------------------------ + case "session.tool.called": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + const name = typeof ev.data?.tool === "string" ? ev.data.tool : (ev.data?.id as string | undefined) ?? "tool" + w.lastWasTaskTool = isTaskToolCall(ev) + w.pendingTools++ + { + const callId = + typeof ev.data?.id === "string" && ev.data.id ? ev.data.id : `${name}#${Date.now()}` + w.inflightTools.set(callId, { name: String(name).toLowerCase(), at: Date.now() }) + } + // Any real tool work between completions legitimises the next + // task_complete — reset the repeat-signal counter. task_complete + // itself is excluded; it manages the counter inside its execute. + // + // Without this, a model that finishes, calls the tool, then does one + // more piece of work and finishes again would be counted as a repeat, + // and the escalation would fire against a legitimate second round. + if (name !== "task_complete") w.taskCompleteSignals = 0 + if (trackToolCall(w, name)) { + void targetedRecovery(sid, "tool-loop", TOOL_LOOP_RECOVERY_PROMPT, "intentNudgeAttempts").catch(() => {}) + } + touch(sid) + return + } + case "session.tool.progress": + case "session.shell.started": + case "session.shell.ended": { + const sid = sidOf(ev) + if (!sid) return + touch(sid) + return + } + // --- shell process registry --- + // `session.shell.*` above is the session-scoped family. The runtime + // also emits a process-registry family, and which one fires depends + // on the tool path (the bash/Code Mode shell emits only these). We + // need the registry pair because it is the one that reports a + // backgrounded job's real completion: `session.tool.success` fires + // as soon as the *tool call* returns, which for `background: true` + // is a few hundred ms — long before the process finishes. + // + // `shell.deleted` is deliberately not used: it carries a different + // id family than `created`/`exited` (verified against the running + // server), so it cannot close an entry. `shell.exited` can. + case "shell.created": { + const info = ev.data?.info + if (!info || typeof info !== "object") return + const rec = info as { id?: unknown; metadata?: { sessionID?: unknown } } + const id = rec.id + const sid = rec.metadata?.sessionID + if (typeof id !== "string" || typeof sid !== "string") return + openShells.set(id, { sessionID: sid, startedAt: Date.now() }) + touch(sid) + return + } + case "shell.exited": { + const id = ev.data?.id + if (typeof id !== "string") return + const sid = openShells.get(id)?.sessionID + if (sid === undefined) return + openShells.delete(id) + const wsh = ensureWatch(sid) + wsh.completedSinceProd++ + wsh.succeededSinceProd++ + touch(sid) + return + } + case "session.tool.success": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + w.lastWasTaskTool = false + w.pendingTools = Math.max(0, w.pendingTools - 1) + { + const callId = typeof ev.data?.id === "string" ? ev.data.id : undefined + if (callId !== undefined) w.inflightTools.delete(callId) + else { + // No call id on the result: drop the oldest entry. Results + // normally arrive in order; a wrong drop only mistimes one + // deferral, and entries die with the watch regardless. + const oldest = w.inflightTools.keys().next().value + if (typeof oldest === "string") w.inflightTools.delete(oldest) + } + } + w.completedSinceProd++ + w.succeededSinceProd++ + touch(sid) + return + } + case "session.tool.failed": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + w.lastWasTaskTool = false + w.pendingTools = Math.max(0, w.pendingTools - 1) + { + const callId = typeof ev.data?.id === "string" ? ev.data.id : undefined + if (callId !== undefined) w.inflightTools.delete(callId) + else { + // No call id on the result: drop the oldest entry. Results + // normally arrive in order; a wrong drop only mistimes one + // deferral, and entries die with the watch regardless. + const oldest = w.inflightTools.keys().next().value + if (typeof oldest === "string") w.inflightTools.delete(oldest) + } + } + w.completedSinceProd++ + touch(sid) + return + } + + // --- permissions ------------------------------------------------- + case "permission.asked": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + w.permissionPending = true + w.permissionPendingAt = Date.now() + return + } + case "permission.replied": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + w.permissionPending = false + w.permissionPendingAt = null + touch(sid) + return + } + + // --- failures ---------------------------------------------------- + case "session.retry.scheduled": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + touch(sid) // provider-level retry is progress of its own kind + // ...and proof that this stall is the provider's to fix, not + // the model's: see isProviderRetryFresh / PROVIDER_RETRY_GRACE_MS. + w.providerRetryAt = Date.now() + const errType = String(ev.data?.error?.type ?? "") + const errMsg = String(ev.data?.error?.message ?? "") + maybeLockOoc(sid, errMsg, errType) + if (maybeLockPermanent(sid, errMsg, errType, ev.data?.error?.status)) return + log("info", `${short(sid)} provider retry #${ev.data?.attempt ?? "?"}: ${errType || errMsg}`) + if (!isStreamingFailure(errMsg, streamingFailureMessagePatterns) && !isStreamingFailureName(errType, streamingFailureErrorNames) && errType !== "retryable") return + // Let the provider retries play out first; only intervene if it stays quiet + if (nowSilenceTooLong(w)) { + w.pendingRecoveryArmed = true + void recover(sid, "streaming failure with scheduled retry") + } + return + } + case "session.step.failed": + case "session.execution.failed": { + const sid = sidOf(ev) + if (!sid) return + const w = ensureWatch(sid) + const errMsg = String(ev.data?.error?.message ?? "") + const errType = String(ev.data?.error?.type ?? "") + // Our own abort: do not recover from it, and do not arm the delayed + // recovery (arming here is what kept the watchdog injecting into an + // already-interrupted session). + if (w.aborting || selfAbortActive(w)) { + dbg(`${short(sid)} ${ev.type} is this plugin's own abort (${errType || "error"}) — not recovering`) + w.recovering = false + w.pendingRecoveryArmed = false + return + } + // Any interrupt-shaped failure (ours or the user's) is not recoverable. + // v2 reports these as {type:"aborted", message:"Step interrupted"}. + if (isAbortError(errType, errMsg)) { + dbg(`${short(sid)} failure was an interrupt (${errType || "error"}) — not recovering`) + w.pendingRecoveryArmed = false + return + } + // Quota/rate-limit failures stand down on the gate ladder — an + // immediate continue retries straight into the ban. + if (isRateLimitError(errType, errMsg)) { + handleRateLimitFailure(sid, w, errType, errMsg) + return + } + maybeLockOoc(sid, errMsg, errType) + // Permanent failures still go idle — the session really is done — + // but nothing arms behind them, so no watchdog pass revives it. + if (maybeLockPermanent(sid, errMsg, errType, ev.data?.error?.status)) { + markIdle(sid) + w.pendingRecoveryArmed = false + w.recovering = false + log("warn", `${short(sid)} ${ev.type}: ${errType || "error"} ${errMsg.slice(0, 160)}`) + return + } + markIdle(sid) + w.pendingRecoveryArmed = true // our delayed recovery must survive this idle transition + log("warn", `${short(sid)} ${ev.type}: ${errType || "error"} ${errMsg.slice(0, 160)}`) + void recover(sid, `${ev.type}${isStreamingFailure(errMsg, streamingFailureMessagePatterns) ? " (streaming)" : ""}`) + return + } + + default: + return + } + } + + function nowSilenceTooLong(w: SessionWatch): boolean { + return Date.now() - w.lastActivityAt > chunkTimeoutMs + gracePeriodMs + } + + // Subscribe and pump events in the background (setup must not block). + // Stable v2 recommends passing an AbortSignal so the stream is torn down + // promptly on plugin unload instead of staying suspended in `for await`. + let running = true + const eventAbort = new AbortController() + const pump = async () => { + try { + const stream = ctx.event.subscribe({ signal: eventAbort.signal }) + for await (const raw of stream) { + if (!running) return + const ev = raw as unknown as V2Event + if (!ev || typeof ev.type !== "string") continue + try { + handleEvent(ev) + } catch (e) { + dbg("handler error:", e instanceof Error ? e.message : String(e)) + } + } + } catch (e) { + if (running) log("error", `event stream ended: ${e instanceof Error ? e.message : String(e)}`) + } + } + void pump() + void restoreQuotaLadders() + + if (superseded) { + // Two mod ids under ONE boot is the evidence that separates a plugin + // reload from an opencode restart. WARN, not dbg: a storm of these is + // the finding, not a curiosity. + log( + "warn", + `superseded live instance mod=${superseded.mod} (age ${Math.round((Date.now() - superseded.startedAt) / 1000)}s) by mod=${MODULE_INSTANCE} boot=${HOST_BOOT}`, + ) + } + if (adoptedSessions > 0) { + log("info", `adopted ${adoptedSessions} watched session(s) from previous instance (stall memory retained) mod=${MODULE_INSTANCE}`) + } + + log( + "info", + `ready (opencode v2). timeout=${chunkTimeoutMs}ms interval=${checkIntervalMs}ms retries=${maxRetries} loop=${loopMaxContinues}/${loopWindowMs / 1000}s warmup=${warmupMs}ms stall=${busyStallStrategy} visibleContinue=${visibleContinue} probe=${configProbe ?? "-"} mod=${MODULE_INSTANCE} up=${HOST_UPTIME_S}s boot=${HOST_BOOT}` + + (gatedInUse.length > 0 ? ` accepted-but-inert=${gatedInUse.join(",")}` : ""), + ) + + // Cleanup: stop timers and the event pump; OpenCode awaits this on disable/reload/shutdown. + const dispose = () => { + running = false + eventAbort.abort() + clearInterval(watchdog) + clearInterval(discoveryTimer) + clearTimeout(initialDiscovery) + // A pending pattern pass outlives the plugin otherwise, and would judge a + // session against a history this build is no longer watching. + for (const w of sessions.values()) { + if (w.toolTextTimer) clearTimeout(w.toolTextTimer) + w.toolTextTimer = null + if (w.recoverTimer) clearTimeout(w.recoverTimer) + w.recoverTimer = null + } + // NOTE: the sessions map is intentionally NOT cleared — it is + // registry-owned (see adoption in State) so stall memory survives + // setup churn. Only this instance's timers and loops are torn down. + // Only clear the registry slot if it is still OURS: a later setup may + // already have superseded us, and unregistering it would strand a live + // watchdog. The identity check matters across copies too, since they + // share one registry. + const reg = singletonRegistry(registryScope) + if (reg.live?.dispose === dispose) reg.live = undefined + log("info", `stopped mod=${MODULE_INSTANCE}`) + } + registry.live = { dispose, mod: MODULE_INSTANCE, startedAt: Date.now() } + return dispose + }, +}) diff --git a/src/v2/index.unknown-tool.test.ts b/src/v2/index.unknown-tool.test.ts new file mode 100644 index 0000000..879c39a --- /dev/null +++ b/src/v2/index.unknown-tool.test.ts @@ -0,0 +1,546 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexUnknownToolTest" + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_unknown_tool" + +let counter = 0 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 600_000, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + debug: true, +} + +const userMessage = (text: string, at: number, id = `msg_u_${at}`) => ({ + type: "user", + id, + time: { created: at }, + content: [{ type: "text", text }], +}) + +/** An hour ago: outside the 5-minute active-user window, so the idle path is not + * suppressed for "the user is mid-conversation". */ +const OLD = Date.now() - 60 * 60_000 + +/** + * A v2 tool part. Note `name` and `id`: v2 names the tool in `name`, where v1 + * read `part.tool`. `id` is the call id, which is what the "already examined" + * set keys on. + */ +const toolPart = (id: string, name: string, status: "error" | "completed" | "running" = "error") => ({ + type: "tool", + id, + name, + time: { created: OLD }, + state: + status === "error" + ? { status, input: {}, error: "tool not found" } + : status === "completed" + ? { status, input: {}, content: [{ type: "text", text: "ok" }] } + : { status, input: {}, metadata: {} }, +}) + +/** An assistant message carrying tool parts. It always has a text part too, and + * that is not incidental: a finished assistant message with no text part is a + * silent dead stream, and that detector fires first and would swallow the test. */ +const assistantWith = (...content: unknown[]) => ({ + type: "assistant", + id: "msg_a_here", + time: { created: OLD + 1_000 }, + content: [{ type: "text", text: "Let me look at that." }, ...content], +}) + +const TOOLS = [{ id: "read" }, { id: "write" }, { id: "glob" }, { id: "bash" }, { id: "task_complete" }] + +async function setup( + opts: { + history?: unknown[] + tools?: Array<{ id: string }> | undefined + toolRegistry?: boolean + toolListThrows?: boolean + parentID?: string + quietCorrections?: boolean + }, +): Promise { + const injected: Array<{ kind?: string; text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-ut-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const history: unknown[] = opts.history ?? [] + const listed = opts.tools ?? TOOLS + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, ...(opts.quietCorrections === undefined ? {} : { quietCorrections: opts.quietCorrections }), registryScope: LOG_SALT }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ kind: "synthetic", text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ kind: "prompt", text: a?.text }), {}), + }, + client: { + session: { get: async () => ({ data: opts.parentID ? { id: SID, parentID: opts.parentID } : { id: SID } }) }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + } + if (opts.toolRegistry !== false) { + ctx.tool = { + transform: async (cb: any) => { + cb({ add: () => {} }) + return { dispose() {} } + }, + list: async () => { + if (opts.toolListThrows) throw new Error("registry unavailable") + return listed + }, + } + } + + const cleanup = await (plugin as any).setup(ctx) + return { + injected, + history, + ...({ cleanup, logFile, streamRef: stream } as any), + } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +/** Drive a turn to idle, which is what makes the plugin look at the history. */ +async function goIdle(h: any) { + for (const e of [ev("session.execution.started"), ev("session.step.started"), ev("session.step.ended"), ev("session.idle")]) { + h.streamRef.push(e) + await wait(10) + } + await wait(250) +} + +const suggestions = (h: any) => h.injected.filter((i: any) => /does not exist/.test(i.text ?? "")) + +describe("v2: naming a replacement for a tool that does not exist", () => { + test("the suggestion travels synthetic, not as a visible user message", async () => { + // Feature corrections go to the model without dumping full text into the + // user's transcript. Stall recovery stays visible by design; the + // suggestion is one-shot per request, so no loop can hide behind the + // quieter channel. + const h = await setup({ + history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "globb")), assistantWith(toolPart("call_2", "globb"))], + }) + await goIdle(h) + const said = suggestions(h) + expect(said).toHaveLength(1) + expect(said[0].kind).toBe("synthetic") + await teardown(h) + }) + + test("the suggestion carries its own silence order", async () => { + // The no-reply rule rides in the text, so it holds for any model with + // no memory and no config — the new-user case that motivated it. + const h = await setup({ + history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "globb")), assistantWith(toolPart("call_2", "globb"))], + }) + await goIdle(h) + const said = suggestions(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain("Do not acknowledge") + await teardown(h) + }) + + test("quietCorrections:false sends the suggestion visibly", async () => { + // The escape hatch for anyone who wants corrections self-evident in + // the transcript. Stall recovery always stays visible regardless. + const h = await setup({ + history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "globb")), assistantWith(toolPart("call_2", "globb"))], + quietCorrections: false, + }) + await goIdle(h) + const said = suggestions(h) + expect(said).toHaveLength(1) + expect(said[0].kind).toBe("prompt") + expect(said[0].text).not.toContain("Do not acknowledge") + await teardown(h) + }) + + test("CONTROL: one invented tool call is a typo and gets no suggestion", async () => { + // The threshold is the whole design: a model that misspells one tool name + // normally self-corrects, and interrupting it teaches it nothing. + const h = await setup({ history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "globb"))] }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) + + test("a second invented call to the same name earns a suggestion", async () => { + const h = await setup({ + history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "globb")), assistantWith(toolPart("call_2", "globb"))], + }) + await goIdle(h) + const said = suggestions(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain('"globb"') + expect(said[0].text).toContain('The closest matching tool is "glob"') + // And the list, so a name that is not a near match is still recoverable. + expect(said[0].text).toContain("read, write, glob, bash") + await teardown(h) + }) + + test("a name with no near match gets the list without a wrong suggestion", async () => { + // Naming a bad near-match is worse than naming nothing: the model switches + // to it, fails again, and now believes the registry is unreliable. + const h = await setup({ + history: [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "zzqqxx")), + assistantWith(toolPart("call_2", "zzqqxx")), + ], + }) + await goIdle(h) + const said = suggestions(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain("Please check the available tools") + expect(said[0].text).not.toContain("The closest matching tool is") + await teardown(h) + }) + + test("the suggestion is sent once, not on every idle", async () => { + const h = await setup({ + history: [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "globb")), + assistantWith(toolPart("call_2", "globb")), + assistantWith(toolPart("call_3", "globb")), + ], + }) + await goIdle(h) + await goIdle(h) + expect(suggestions(h)).toHaveLength(1) + await teardown(h) + }) + + test("two different unknown names are counted separately", async () => { + // Sharing a budget across names would let an unrelated pair of typos silence + // the threshold for the name that actually matters. + const h = await setup({ + history: [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "globb")), + assistantWith(toolPart("call_2", "wriet")), + ], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) + + test("a new user message re-arms the budget", async () => { + // The budget is scoped to one request. A model told to do something new has + // a fresh tool list in front of it, so the previous round's typos say nothing + // about this one. + // + // The re-arm resets counts and the latch, but NOT the examined-parts set: + // recounting already-suggested errors is what nagged a live session with + // the same suggestion on every user message (2026-10-04). New errors still + // earn a suggestion — and it names the NEW name, not the old one. + const h = await setup({ + history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "globb")), assistantWith(toolPart("call_2", "globb"))], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(1) + h.history.push(userMessage("now do the other thing", OLD + 2_000, "msg_u_second")) + h.history.push(assistantWith(toolPart("call_3", "wriet"))) + h.history.push(assistantWith(toolPart("call_4", "wriet"))) + await goIdle(h) + expect(suggestions(h)).toHaveLength(2) + expect(suggestions(h)[1].text).toContain('"wriet"') + await teardown(h) + }) + + test("a genuine new request does not re-suggest already-reported errors", async () => { + // The 3:40 PM incident: every real user message re-armed, cleared the + // examined set, and recounted the SAME stale bash errors — one suggestion + // per user message while the model had long moved on. + const h = await setup({ + history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "bash")), assistantWith(toolPart("call_2", "bash"))], + tools: [{ id: "read" }, { id: "shell" }, { id: "glob" }], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(1) + h.history.push(userMessage("stop ignoring; report repeats", OLD + 2_000, "msg_u_second")) + await goIdle(h) + await goIdle(h) + expect(suggestions(h)).toHaveLength(1) + await teardown(h) + }) + + test("our own suggestion is not a new request and does not re-arm", async () => { + // ses_ef81e8561ffeXyx3jAzKl5lltv 2026-10-04: the visible channel posts + // our suggestion as a real user message with a new id. The next idle + // read it as new instructions, cleared the error map, the latch and + // the examined-parts set, recounted the SAME parts back to threshold, + // and suggested again — four identical "(none)" prompts, each "2x". + const tools = [{ id: "read" }, { id: "shell" }, { id: "glob" }] + const h = await setup({ + history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "bash")), assistantWith(toolPart("call_2", "bash"))], + tools, + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(1) + const own = suggestions(h)[0] + h.history.push({ + type: "user", + id: "msg_own_suggestion", + time: { created: OLD + 3_000 }, + content: [{ type: "text", text: own.text }], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(1) + await teardown(h) + }) + + test("own prompt with an unreadable body still does not re-arm", async () => { + // The live projection can carry user messages with no readable text + // (observed: content null), which defeats text matching. Recency to + // our own inject is the backstop: our prompt always lands within + // seconds of it. + const tools = [{ id: "read" }, { id: "shell" }, { id: "glob" }] + const h = await setup({ + history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "bash")), assistantWith(toolPart("call_2", "bash"))], + tools, + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(1) + h.history.push({ type: "user", id: "msg_own_bare", time: { created: Date.now() }, content: null }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(1) + await teardown(h) + }) + + test("a v1 tool name maps to its v2 rename instead of (none)", async () => { + // "bash" is edit-distance 4 from "shell" against a threshold of 2, so + // the fuzzy matcher can never bridge it. A static alias tried first can. + const h = await setup({ + history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "bash")), assistantWith(toolPart("call_2", "bash"))], + tools: [{ id: "read" }, { id: "shell" }, { id: "glob" }], + }) + await goIdle(h) + const said = suggestions(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain('The closest matching tool is "shell"') + await teardown(h) + }) + + test("CONTROL: an alias whose target is not registered falls back to the list", async () => { + // The alias must never name a tool that does not exist: a wrong guess + // costs a second failure round and teaches the model the registry lies. + const h = await setup({ + history: [userMessage("go", OLD), assistantWith(toolPart("call_1", "bash")), assistantWith(toolPart("call_2", "bash"))], + tools: [{ id: "read" }, { id: "write" }, { id: "glob" }], + }) + await goIdle(h) + const said = suggestions(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain("Please check the available tools") + expect(said[0].text).not.toContain("The closest matching tool is") + await teardown(h) + }) +}) + +describe("v2: what the unknown-tool check ignores", () => { + test("CONTROL: a tool that exists but errored is not an unknown tool", async () => { + // The dangerous false positive is telling a model a real tool does not + // exist. An error from a registered tool is a real problem, and the right + // response to it is not a rename. + const h = await setup({ + history: [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "read")), + assistantWith(toolPart("call_2", "read")), + ], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) + + test("CONTROL: a tool that succeeded is not counted", async () => { + const h = await setup({ + history: [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "globb", "completed")), + assistantWith(toolPart("call_2", "globb", "completed")), + ], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) + + test("CONTROL: a still-running tool is not counted", async () => { + const h = await setup({ + history: [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "globb", "running")), + assistantWith(toolPart("call_2", "globb", "running")), + ], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) + + test("the same call id is never counted twice", async () => { + // The history is re-walked on every idle, so a part that is already counted + // would be counted again on the next pass and the threshold would be reached + // by one real mistake. + const part = toolPart("call_1", "globb") + const h = await setup({ + history: [ + userMessage("go", OLD), + assistantWith(part), + assistantWith(structuredClone(part)), + assistantWith(structuredClone(part)), + ], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) + + test("CONTROL: text and reasoning parts are ignored", async () => { + const h = await setup({ + history: [ + userMessage("go", OLD), + assistantWith({ type: "reasoning", text: "I should call globb" }), + assistantWith({ type: "text", text: "tool: globb" }), + ], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) +}) + +describe("v2: the unknown-tool check degrades rather than guessing", () => { + test("CONTROL: no tool registry means no suggestion, ever", async () => { + // With no registry every name is unknown, so the check would accuse a model + // of inventing tools that plainly exist. + const h = await setup({ + toolRegistry: false, + history: [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "globb")), + assistantWith(toolPart("call_2", "globb")), + ], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) + + test("CONTROL: an empty registry means no suggestion", async () => { + const h = await setup({ + tools: [], + history: [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "globb")), + assistantWith(toolPart("call_2", "globb")), + ], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(0) + await teardown(h) + }) + + test("a registry that throws is logged and skipped", async () => { + // Failing loudly here would mean a session full of one-line recovery + // prompts, which is worse than losing the feature for a few minutes. + const h = await setup({ + toolListThrows: true, + history: [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "globb")), + assistantWith(toolPart("call_2", "globb")), + ], + }) + await goIdle(h) + expect(suggestions(h)).toHaveLength(0) + const logs = existsSync(h.logFile) ? readFileSync(h.logFile, "utf8") : "" + expect(logs).toContain("failed to list tools") + await teardown(h) + }) + + test("a registry entry with no name at all is dropped, not quoted as empty", async () => { + // An unnamed entry would otherwise be joined into the tool list as an empty + // item, and the suggestion would read "... include: , read, write". + const h = await setup({ + tools: [{ id: "read" }, { id: "write" }, { id: "glob" }, {} as any], + history: [ + userMessage("go", OLD), + assistantWith(toolPart("call_1", "globb")), + assistantWith(toolPart("call_2", "globb")), + ], + }) + await goIdle(h) + const said = suggestions(h) + expect(said).toHaveLength(1) + expect(said[0].text).toContain("read, write, glob") + expect(said[0].text).not.toContain(", , ") + await teardown(h) + }) +}) diff --git a/src/v2/index.usage-activity.test.ts b/src/v2/index.usage-activity.test.ts new file mode 100644 index 0000000..684c6e3 --- /dev/null +++ b/src/v2/index.usage-activity.test.ts @@ -0,0 +1,171 @@ +import { describe, test, expect, setDefaultTimeout } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexUsageActivityTest" + +/** + * v2: token flow counts as activity. + * + * The watchdog measured silence purely from events, while token usage — + * proof the model is generating — never touched the activity clock. A turn + * pouring tokens for minutes straight still read as "no activity for 186s" + * and drew a stall injection mid-generation (2026-10-13: three recoveries + * into a 1.9M-token turn, each interrupting live work). Usage events with a + * GROWING total now reset the clock; a flat total means nothing resumed and + * does not. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +/** Sized for the parallel suite, not isolation (see busy-tools.test.ts). */ +setDefaultTimeout(30_000) + +const SID = "ses_usageactivity" +let counter = 71000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +const usage = (output: number) => ({ + input: 100, + output, + reasoning: 0, + cache: { read: 0, write: 0 }, +}) + +const BASE_OPTS = { + chunkTimeoutMs: 100, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + toolTextCheckDelayMs: 0, + minActivityGapMs: 10, + debug: false, +} + +async function setup(history: unknown[], extraOpts: Record = {}) { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-usage-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + const ctx: any = { + event: stream, + options: { ...BASE_OPTS, logFile, registryScope: LOG_SALT, ...extraOpts }, + session: { + context: async () => history, + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + tool: { transform: async () => {} }, + } + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { + injected, + cleanup, + push: async (type: string, data: Record = {}) => { + stream.push(ev(type, data)) + await wait(10) + }, + teardown: () => { + ;(cleanup as (() => void) | undefined)?.() + rmSync(logFile, { force: true }) + }, + } +} + +const userTurn = () => ({ + type: "user", + id: "msg_u0", + time: { created: Date.now() - 60 * 60_000 }, + content: [{ type: "text", text: "keep going" }], +}) + +const assistantText = (text: string) => ({ + type: "assistant", + id: "msg_a0", + time: { created: Date.now() - 60_000 }, + content: [{ type: "text", text }], +}) + +describe("v2: token flow counts as activity", () => { + test("growing usage holds the stall door shut", async () => { + // Eight growing usage events across ~240ms: silence never exceeds ~30ms + // against a 100ms chunk, so no stall may declare despite 4x the window + // elapsing overall. Assistant text is present so the empty-idle nudge + // stays out of it; the text is neutral so no pattern detector fires. + const h = await setup([userTurn(), assistantText("Working on it.")]) + await h.push("session.execution.started") + let total = 1000 + for (let i = 0; i < 8; i++) { + total += 1000 + await h.push("session.usage.updated", { tokens: usage(total) }) + await wait(20) + } + await h.teardown() + expect(h.injected).toEqual([]) + }, 20000) + + test("CONTROL: flat usage does not hold the door (nothing resumed)", async () => { + // Same events, same cadence, total never moves: the clock goes stale + // and the stall answers on schedule. This is what distinguishes the + // gate from "any usage event resets". + const h = await setup([userTurn(), assistantText("Working on it.")]) + await h.push("session.execution.started") + for (let i = 0; i < 8; i++) { + await h.push("session.usage.updated", { tokens: usage(1000) }) + await wait(20) + } + await wait(200) + const count = h.injected.length + await h.teardown() + expect(count).toBeGreaterThanOrEqual(1) + }, 20000) +}) diff --git a/src/v2/index.user-rearm.test.ts b/src/v2/index.user-rearm.test.ts new file mode 100644 index 0000000..a8d16ab --- /dev/null +++ b/src/v2/index.user-rearm.test.ts @@ -0,0 +1,229 @@ +import { describe, test, expect } from "bun:test" +import { rmSync, existsSync, readFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexUserRearmTest" + +/** + * v2: a genuine new user message re-arms recovery after a user interrupt. + * + * ses_eeb02d4b (2026-10-07): after an interrupt, auto-resume never got the + * session going again — not even after an explicit user "continue". + * `userCancelled` latched permanently: set once on a non-plugin interrupt + * and cleared nowhere (not on new user messages, not on new turns), while + * every recovery path checks it first. A new inbound user message is the + * hand-back signal and re-arms alongside the other budgets — own injected + * prompts excluded, as with every other re-arm. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 6000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 300, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, +} + +const userMessage = (text: string, at: number, id = "msg_u") => ({ + type: "user", + id, + time: { created: at }, + content: [{ type: "text", text }], +}) + +const assistantText = (text: string, at: number) => ({ + type: "assistant", + id: "msg_a", + time: { created: at }, + content: [{ type: "text", text }], +}) + +async function setup(history: Record, extraOpts: Record = {}): Promise { + const injected: Array<{ sid?: string; text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-rearm-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT, ...extraOpts }, + session: { + context: async (a: any) => history[a?.sessionID ?? SID] ?? [], + active: async () => ({ [SID]: {} }), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async () => ({ data: [{ id: SID }] }), + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream, history } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +/** Poll the log file for a line instead of sleeping a fixed span. */ +async function waitForLog(logFile: string, needle: string, timeoutMs = 5000): Promise { + const deadline = Date.now() + timeoutMs + for (;;) { + try { + if (readFileSync(logFile, "utf8").includes(needle)) return true + } catch { + // Log file not written yet. + } + if (Date.now() >= deadline) return false + await wait(25) + } +} + +describe("v2: genuine user message re-arms after user interrupt", () => { + test("interrupt latches, then a new user message re-arms recovery", async () => { + const now = Date.now() + const history: Record = { + [SID]: [userMessage("go", now - 60_000, "msg_u1")], + } + const h = await setup(history) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.execution.interrupted", SID, { reason: "user" }) + await wait(50) + // Genuine new inbound user message, then a fresh turn that stalls. + history[SID] = [ + userMessage("go", now - 60_000, "msg_u1"), + userMessage("continue", Date.now(), "msg_u2"), + assistantText("working", Date.now()), + ] + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.step.started", SID) + await wait(900) + expect(h.injected.filter((i: any) => i.sid === SID).length).toBeGreaterThan(0) + await teardown(h) + }) + + test("CONTROL: interrupt with no new user message stays stood down", async () => { + const now = Date.now() + const history: Record = { + [SID]: [userMessage("go", now - 60_000, "msg_u1")], + } + const h = await setup(history) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.execution.interrupted", SID, { reason: "user" }) + await wait(50) + push(h, "session.execution.started", SID) + await wait(10) + push(h, "session.step.started", SID) + await wait(900) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + await teardown(h) + }) + + test("a genuine user message inside the backoff wait parks the recovery", async () => { + // The race this pins: stall declared arms the recover timer (200ms + // here so the window is drivable); a genuine user message landing + // before it fires must park the turn instead of injecting into the + // user's live turn ("Step interrupted"). The busy-freshness check + // cannot see it — event-pump lag means their execution.started may + // not have been processed yet — so the timer compares + // lastGenuineUserAt against its own arm time. (2026-10-11: continues + // landing right after user statements.) + // + // Deliberately NO second execution.started: the freshness guard would + // park on processed activity and the test would prove nothing. The + // user message arrives as history only — exactly the case the guard + // exists for — picked up by the watchdog's own history reads. + const now = Date.now() + const history: Record = { + [SID]: [userMessage("go", now - 60_000, "msg_u1")], + } + const h = await setup(history, { baseBackoffMs: 1000, maxBackoffMs: 1000, debug: true }) + push(h, "session.execution.started", SID) + // Wait for the watch to actually go busy before the silence window + // starts: under a saturated event loop the started event can sit + // unprocessed, and everything after it (chunk silence, declaration, + // arm) shifts by the same lag. Polling the precondition beats + // assuming it (1-in-14 flake, 2026-10-11). + expect(await waitForLog(h.logFile, "idle/unknown -> busy", 5000)).toBe(true) + // Wait out the 300ms chunk timeout until the watchdog declares the + // stall and arms the timer (200ms fuse from there). + const armed = await waitForLog(h.logFile, "resume attempt 1/3", 5000) + expect(armed).toBe(true) + history[SID] = [ + userMessage("go", now - 60_000, "msg_u1"), + userMessage("go on, I am here", Date.now(), "msg_u2"), + ] + // The guard logs when it parks; absence of injection alone would not + // distinguish "parked" from "never fired". + const parked = await waitForLog(h.logFile, "parking recovery", 5000) + await teardown(h) + expect(parked).toBe(true) + expect(h.injected.filter((i: any) => i.sid === SID)).toHaveLength(0) + }) +}) diff --git a/src/v2/index.user-wait.test.ts b/src/v2/index.user-wait.test.ts new file mode 100644 index 0000000..450f121 --- /dev/null +++ b/src/v2/index.user-wait.test.ts @@ -0,0 +1,202 @@ +import { describe, test, expect } from "bun:test" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexUserWaitTest" + +/** + * v2: busy-stall stands down while a turn waits on the user. + * + * Repro: ses_ef72c5f3 (2026-10-05) — the watchdog injected + * `continue — stalled (no activity for 186s)` into a session parked on a + * `question` tool awaiting the user. The busy-stall path had no user-wait + * guard (only the idle path stands down via `shouldStandDownForUser`). + * A turn awaiting the user is waiting, not stalled — but a wedged bare + * tool (or an answered question) must still recover. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 2000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, sid: string, data: Record = {}) => ({ type, data: { sessionID: sid, ...data } }) + +const OPTIONS = { + chunkTimeoutMs: 300, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + injectIntervalMs: 0, + subagentWaitMs: 40, + debug: false, +} + +const userMessage = (text: string, at: number) => ({ + type: "user", + id: "msg_u", + time: { created: at }, + content: [{ type: "text", text }], +}) + +const assistantWithTool = (name: string, status: string, at: number) => ({ + type: "assistant", + id: "msg_a", + time: { created: at }, + content: [{ type: "tool", name, state: { status } }], +}) + +type History = Record + +async function setup(history: History, active: string[]): Promise { + const injected: Array<{ sid?: string; text?: string }> = [] + const interrupted: string[] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-userwait-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const activeSet = new Set(active) + const rows: Array<{ id: string; parentID?: string }> = [{ id: SID }] + + const ctx: any = { + event: stream, + options: { ...OPTIONS, logFile, registryScope: LOG_SALT }, + session: { + context: async (a: any) => history[a?.sessionID ?? SID] ?? [], + active: async () => Object.fromEntries([...activeSet].map((s) => [s, {}])), + interrupt: async (a: any) => (interrupted.push(a?.sessionID), {}), + synthetic: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ sid: a?.sessionID, text: a?.text }), {}), + }, + client: { + session: { + get: async ({ path }: any) => ({ data: { id: path?.id } }), + list: async (a: any) => { + if (!a?.parentID) return { data: rows } + return { data: rows.filter((r) => (r as any).parentID === a.parentID) } + }, + }, + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, interrupted, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const push = (h: any, type: string, sid: string, data: Record = {}) => h.streamRef.push(ev(type, sid, data)) + +async function makeBusy(h: any, sid: string) { + push(h, "session.execution.started", sid) + await wait(10) + push(h, "session.step.started", sid) + await wait(10) +} + +const parentInjects = (h: any) => h.injected.filter((i: any) => i.sid === SID) + +describe("v2: busy-stall stands down while a turn waits on the user", () => { + test("no stall recovery while a question tool is still running", async () => { + const now = Date.now() + const h = await setup( + { + [SID]: [userMessage("go", now - 60_000), assistantWithTool("question", "running", now - 5_000)], + }, + [SID], + ) + await makeBusy(h, SID) + await wait(900) + expect(parentInjects(h)).toHaveLength(0) + expect(h.interrupted).toHaveLength(0) + await teardown(h) + }) + + test("CONTROL: a wedged bare tool still recovers", async () => { + // A non-user tool stuck running with no events is exactly what the + // stall path exists for — the user-wait guard must not swallow it. + // "Wedged" means past the stuck-tool age gate (60s): a YOUNG running + // part is working, not wedged — the event-based young-tool guard already + // defers on it when its tool.called arrives, and the history-based + // young-unfinished guard defers when events were lost (e.g. reload). + // So this fixture uses a part old enough to actually be wedged; the + // young side of the boundary is pinned by the stall-shell-guard suite. + // Uses `write`, not `shell`: shells get a longer stall-path TTL of + // their own (10 min), so a 120s shell is still working, not wedged. + const now = Date.now() + const h = await setup( + { + [SID]: [userMessage("go", now - 600_000), assistantWithTool("write", "running", now - 120_000)], + }, + [SID], + ) + await makeBusy(h, SID) + await wait(900) + expect(parentInjects(h).length).toBeGreaterThan(0) + await teardown(h) + }) + + test("CONTROL: an answered question still recovers", async () => { + // The user answered (tool completed) but the session never moved: + // that IS a stall, not a wait. + const now = Date.now() + const h = await setup( + { + [SID]: [userMessage("go", now - 60_000), assistantWithTool("question", "completed", now - 5_000)], + }, + [SID], + ) + await makeBusy(h, SID) + await wait(900) + expect(parentInjects(h).length).toBeGreaterThan(0) + await teardown(h) + }) +}) diff --git a/src/v2/index.visible-continue.test.ts b/src/v2/index.visible-continue.test.ts new file mode 100644 index 0000000..18c2f41 --- /dev/null +++ b/src/v2/index.visible-continue.test.ts @@ -0,0 +1,244 @@ +import { describe, test, expect } from "bun:test" +import { existsSync, readFileSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexVisibleContinueTest" + +/** + * v2: visible stall continue (cattleprod-style) + rich prod text + + * exact-duplicate anti-repeat. + * + * Every group carries a control. The harness drives a genuinely stalled busy + * session (started, then silent) so the watchdog fires `recover()`, and tags + * each delivery by channel (`synthetic` vs `prompt`) — the distinction the + * feature exists to make. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_vis" +let counter = 0 + +const OPEN = [ + { content: "Write the migration guide", status: "pending", priority: "high" }, + { content: "Delete the temp fixtures", status: "in_progress", priority: "low" }, +] + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +const ev = (type: string, data: Record = {}) => ({ type, data: { sessionID: SID, ...data } }) + +type Harness = { + injected: Array<{ kind: string; text?: string }> + logs: string[] +} + +/** A turn that starts and then goes silent: a stall candidate. */ +const busyStallEvents = [ev("session.execution.started")] + +/** Timings small enough that the watchdog fires inside the harness wait. */ +const FAST = { + chunkTimeoutMs: 50, + gracePeriodMs: 0, + checkIntervalMs: 20, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + loopMaxContinues: 99, + injectIntervalMs: 0, +} + +async function replay( + events: any[], + opts: Record = {}, + todos: unknown[] | undefined = undefined, + extraEvents: Array<{ at: number; event: any }> = [], + waitMs = 600, +): Promise { + const injected: Harness["injected"] = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-vis-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + const ctx: any = { + event: stream, + options: { ...FAST, ...opts, logFile, debug: true, registryScope: LOG_SALT }, + session: { + active: async () => ({}), + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ kind: "synthetic", text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ kind: "prompt", text: a?.text }), {}), + }, + client: { session: { get: async () => ({ data: {} }) } }, + storage: { + get: async () => ({ todos: todos ?? [], updatedAt: Date.now() }), + set: async () => {}, + remove: async () => {}, + }, + } + + const cleanup = await (plugin as any).setup(ctx) + const started = Date.now() + let extraIdx = 0 + for (const e of events) { + stream.push(e) + await wait(10) + } + // Interleave extra events at wall-clock offsets while the watchdog works. + while (Date.now() - started < waitMs) { + while (extraIdx < extraEvents.length && Date.now() - started >= extraEvents[extraIdx].at) { + stream.push(extraEvents[extraIdx].event) + extraIdx++ + } + await wait(10) + } + for (; extraIdx < extraEvents.length; extraIdx++) stream.push(extraEvents[extraIdx].event) + await wait(50) + ;(cleanup as (() => void) | undefined)?.() + const logs = existsSync(logFile) ? readFileSync(logFile, "utf8").split("\n") : [] + rmSync(logFile, { force: true }) + return { injected, logs } +} + +describe("v2: stall-continue channel", () => { + test("CONTROL: default channel is the visible prompt", async () => { + const { injected } = await replay(busyStallEvents, { maxRetries: 1 }) + expect(injected.length).toBeGreaterThan(0) + expect(injected.every((i) => i.kind === "prompt")).toBe(true) + }) + + test("rich stall text scopes the continue against new/destructive work", async () => { + // ses_ed9e2ef9 (2026-10-10): a bare "continue" behind an offer reads + // as approval ("I can delete…" + continue = deletion). The rich + // template carries the scope guard so a missed hand-off still cannot + // authorize new or destructive work. + const { injected } = await replay(busyStallEvents, { maxRetries: 1 }) + expect(injected.length).toBeGreaterThan(0) + expect(injected[0].text).toContain("destructive or irreversible") + // Timestamp reads in the operator's timezone, not Zulu. + expect(injected[0].text).toMatch(/[AP]M [A-Z]{2,4}/) + expect(injected[0].text).not.toMatch(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/) + }) + + test("visibleContinue:false restores hidden synthetic", async () => { + const { injected } = await replay(busyStallEvents, { maxRetries: 1, visibleContinue: false }) + expect(injected.length).toBeGreaterThan(0) + expect(injected.every((i) => i.kind === "synthetic")).toBe(true) + }) + + test("visibleContinue:true sends a real prompt message instead", async () => { + const { injected } = await replay(busyStallEvents, { maxRetries: 1, visibleContinue: true }) + expect(injected.length).toBeGreaterThan(0) + expect(injected.every((i) => i.kind === "prompt")).toBe(true) + }) + + test("AUTO_RESUME_VISIBLE_CONTINUE=1 enables the visible channel", async () => { + process.env.AUTO_RESUME_VISIBLE_CONTINUE = "1" + try { + const { injected } = await replay(busyStallEvents, { maxRetries: 1 }) + expect(injected.length).toBeGreaterThan(0) + expect(injected.every((i) => i.kind === "prompt")).toBe(true) + } finally { + delete process.env.AUTO_RESUME_VISIBLE_CONTINUE + } + }) + + test("AUTO_RESUME_VISIBLE_CONTINUE=0 opts back out to hidden", async () => { + process.env.AUTO_RESUME_VISIBLE_CONTINUE = "0" + try { + const { injected } = await replay(busyStallEvents, { maxRetries: 1 }) + expect(injected.length).toBeGreaterThan(0) + expect(injected.every((i) => i.kind === "synthetic")).toBe(true) + } finally { + delete process.env.AUTO_RESUME_VISIBLE_CONTINUE + } + }) +}) + +describe("v2: rich stall-continue text", () => { + test("CONTROL: default names the stall and the open todos", async () => { + const { injected } = await replay(busyStallEvents, { maxRetries: 1 }, OPEN) + expect(injected.length).toBeGreaterThan(0) + expect(injected[0].text).toContain("continue — stalled") + // The duration is stated once (idle field), not twice: the "no activity + // for Ns" reason is log-only now. The open todos still ride along. + expect(injected[0].text).toContain("idle ") + expect(injected[0].text).not.toContain("no activity for") + expect(injected[0].text).toContain("Write the migration guide") + expect(injected[0].text).toContain("Delete the temp fixtures") + }) + + test("richContinuePrompt:false restores bare 'continue'", async () => { + const { injected } = await replay(busyStallEvents, { maxRetries: 1, richContinuePrompt: false }, OPEN) + expect(injected.length).toBeGreaterThan(0) + expect(injected[0].text).toBe("continue") + }) + + test("a custom continuePrompt always wins verbatim", async () => { + const { injected } = await replay(busyStallEvents, { maxRetries: 1, continuePrompt: "go" }, OPEN) + expect(injected.length).toBeGreaterThan(0) + expect(injected[0].text).toBe("go") + }) +}) + +describe("v2: exact-duplicate anti-repeat", () => { + test("an identical re-fire with zero progress is suppressed", async () => { + // Verbatim custom text, so every attempt builds the same string; no + // assistant text and no tool work ever lands, so attempts 2+ are dupes. + const { injected, logs } = await replay(busyStallEvents, { maxRetries: 5, continuePrompt: "go" }, [], [], 900) + expect(injected).toHaveLength(1) + expect(injected[0].text).toBe("go") + expect(logs.some((l) => l.includes("duplicate continue suppressed"))).toBe(true) + }) + + test("new model output re-arms it", async () => { + // Fresh assistant text between windows is progress: the same prod text + // must go out again instead of being swallowed. Headroom past the + // progress event matters: attempts burn ~1/70ms, so maxRetries 5 would + // be spent before the delta lands and the session gives up. + const { injected } = await replay( + busyStallEvents, + { maxRetries: 10, continuePrompt: "go" }, + [], + [{ at: 250, event: ev("session.text.delta", { messageID: "msg_a1", delta: "still working" }) }], + 1200, + ) + expect(injected.length).toBeGreaterThanOrEqual(2) + }) +}) diff --git a/src/v2/index.watchdog-truth.test.ts b/src/v2/index.watchdog-truth.test.ts new file mode 100644 index 0000000..a1731fd --- /dev/null +++ b/src/v2/index.watchdog-truth.test.ts @@ -0,0 +1,162 @@ +import { describe, test, expect } from "bun:test" +import { rmSync, readFileSync, existsSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import plugin from "./index" + +/** Unique per test FILE: bun runs every file in one process, so two files + * starting their counters at 0 derive the SAME tmpdir log path and read each + * other's writes. Discovered 2026-10-10 while chasing suite flakiness. */ +const LOG_SALT = "IndexWatchdogTruthTest" + +/** + * v2: the watchdog sees server-reported running sessions, not just ones it + * observed events for. + * + * ses_edf403f5 (2026-10-09): running server-side 7+ minutes with two user + * messages unanswered while the watchdog saw nothing. Production has no + * `ctx.session.active` (verified 2026-10-06), so `getActiveSessions()` fell + * back to the event-derived busy set — empty for any session that went quiet + * across a reload — and the stall path never fired. The HTTP loopback + * (`/api/session/active`, same auth as subagent discovery) restores server + * truth; any failure still falls open to the event-derived set. + */ + +const wait = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const SID = "ses_parent" + +let counter = 19000 + +function makeEventStream() { + const queue: any[] = [] + const waiters: ((ev: any) => void)[] = [] + let closed = false + const stream = { + push(ev: any) { + if (waiters.length) waiters.shift()!(ev) + else queue.push(ev) + }, + close() { + closed = true + while (waiters.length) waiters.shift()!(null) + }, + subscribe: () => stream, + [Symbol.asyncIterator]() { + return { + next: () => + new Promise((resolve) => { + if (queue.length) return resolve({ value: queue.shift(), done: false }) + if (closed) return resolve({ value: undefined, done: true }) + waiters.push((ev) => + resolve(ev === null ? { value: undefined, done: true } : { value: ev, done: false }), + ) + }), + return: () => { + closed = true + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } + return stream +} + +async function setup(): Promise { + const injected: Array<{ text?: string }> = [] + const stream = makeEventStream() + const logFile = join(tmpdir(), `auto-resume-${LOG_SALT}-watchtruth-${process.pid}-${counter++}.log`) + rmSync(logFile, { force: true }) + + // Prod shape: NO session.active, NO client. The only server truth available + // is the HTTP loopback (stubbed fetch here). + const ctx: any = { + event: stream, + options: { + registryScope: LOG_SALT, + chunkTimeoutMs: 50, + toolTextCheckDelayMs: 0, + checkIntervalMs: 20, + gracePeriodMs: 0, + warmupMs: 0, + baseBackoffMs: 1, + maxBackoffMs: 2, + maxRetries: 3, + loopMaxContinues: 99, + injectIntervalMs: 0, + subagentWaitMs: 40, + logFile, + debug: true, + }, + session: { + context: async () => [], + active: undefined, + interrupt: async () => ({}), + synthetic: async (a: any) => (injected.push({ text: a?.text }), {}), + prompt: async (a: any) => (injected.push({ text: a?.text }), {}), + }, + storage: { get: async () => ({ todos: [] }), set: async () => {}, remove: async () => {} }, + tool: { transform: async (cb: any) => (cb({ add: () => {} }), { dispose() {} }), list: async () => [] }, + } + + const cleanup = await (plugin as any).setup(ctx) + await wait(60) + return { injected, cleanup, logFile, streamRef: stream } +} + +async function teardown(h: any) { + ;(h.cleanup as (() => void) | undefined)?.() + rmSync(h.logFile, { force: true }) +} + +const attemptOf = (text: string | undefined): string | null => { + const m = /attempt (\d)\/3/.exec(text ?? "") + return m ? m[1] : null +} + +describe("v2: watchdog adopts server-reported running sessions", () => { + test("a session the server reports running gets a stall nudge with zero events", async () => { + const realFetch = globalThis.fetch + const realPort = process.env.OPENCODE_SERVER_PORT + process.env.OPENCODE_SERVER_PORT = "1" + ;(globalThis as any).fetch = async (url: unknown) => { + if (String(url).includes("/api/session/active")) { + return { ok: true, json: async () => ({ data: { [SID]: { type: "running" } } }) } + } + return { ok: false, json: async () => ({}) } + } + try { + const h = await setup() + // No events pushed at all: post-reload silence. Server truth alone + // must adopt the session, mark it busy, and stall it. + await wait(500) + const attempts = h.injected.map((i: any) => attemptOf(i.text)) + await teardown(h) + expect(h.injected.length).toBeGreaterThanOrEqual(1) + expect(attempts[0]).toBe("1") + } finally { + ;(globalThis as any).fetch = realFetch + if (realPort === undefined) delete process.env.OPENCODE_SERVER_PORT + else process.env.OPENCODE_SERVER_PORT = realPort + } + }) + + test("CONTROL: unreachable server fails open to event flags (no adoption, no crash)", async () => { + const realFetch = globalThis.fetch + const realPort = process.env.OPENCODE_SERVER_PORT + process.env.OPENCODE_SERVER_PORT = "1" + ;(globalThis as any).fetch = async () => { + throw new Error("no server") + } + try { + const h = await setup() + await wait(400) + const n = h.injected.length + await teardown(h) + expect(n).toBe(0) + } finally { + ;(globalThis as any).fetch = realFetch + if (realPort === undefined) delete process.env.OPENCODE_SERVER_PORT + else process.env.OPENCODE_SERVER_PORT = realPort + } + }) +})