fix(cli): plain status, inspect and why print a successful poll's warnings, and the e2e covers a sealed engine relaunch

docs/reference/cli.md

@@ -388,6 +388,12 @@ Time on current step: 10 minutes on stateful-invariant-campaign
 Pace: 4 finished in the last 10m
 ```
 
+A poll that succeeds with diagnostics, such as a `WORKFLOW_DEADLINE_CANCEL_FAILED`
+or `WORKFLOW_STATE_SYNC_SKIPPED` warning, prints each one on stdout after these
+lines as `<severity>: <code>: <message>`, where a message can span several
+lines. The warnings do not change the exit status, so scripts should read them
+from the `diagnostics` array of `--json` output.
+
 `--watch` re-polls every `--interval` seconds (default 30) until the run
 reaches a terminal state (`succeeded`, `failed`, `timed-out`, or `canceled`),
 needs attention, is paused, or the poll fails. With `--json --watch`, every poll writes one

docs/reference/configuration.md

@@ -159,8 +159,9 @@ next synchronization requests cancellation again; a synchronization that fails
 or is skipped (for example `WORKFLOW_STATE_SYNC_SKIPPED` or
 `WORKFLOW_SYNC_IN_PROGRESS`) does not check the deadline at all. A run that
 finished first keeps its terminal outcome, with no timeout record. To bound an
-unattended run, run `ultrafuzz status <run-id>` periodically (for example from
-cron) and act on its warnings, or cancel it with `ultrafuzz cancel <run-id>`.
+unattended run, run `ultrafuzz status <run-id> --json` periodically (for example
+from cron) and act on the warnings in its `diagnostics` (plain `status` prints
+them on stdout), or cancel it with `ultrafuzz cancel <run-id>`.
 Workflow-side enforcement is tracked in
 [#1110](https://github.com/monad-developers/ultrafuzz/issues/1110).
 

packages/cli/src/command-shared.ts

@@ -131,8 +131,11 @@ export function commandFromRuntime<CommandName extends CliKnownCommand, Source>(
         : project === undefined
           ? (result.value as CliCommandDataMap[CommandName])
           : project(result.value),
+    // A successful result still prints its diagnostics, after the value; a failure leads with them.
     text: result.value
-      ? `${result.ok ? "" : diagnosticsText(result.diagnostics)}${text(result.value)}`
+      ? result.ok
+        ? `${text(result.value)}${diagnosticsText(result.diagnostics)}`
+        : `${diagnosticsText(result.diagnostics)}${text(result.value)}`
       : diagnosticsText(result.diagnostics),
     diagnostics: result.diagnostics
   };

packages/cli/src/commands/init.ts

@@ -1,7 +1,7 @@
 import { Command, Flags } from "@oclif/core";
 import { initProject } from "@ultrafuzz/runtime";
 
-import { commandFromRuntime, diagnosticsText, emitCommandResult, globalFlags, projectRoot } from "../command-shared.js";
+import { commandFromRuntime, emitCommandResult, globalFlags, projectRoot } from "../command-shared.js";
 
 export default class Init extends Command {
   static override summary = "Create project-owned Ultrafuzz surfaces";
@@ -22,9 +22,6 @@ export default class Init extends Command {
         ""
       ].join("\n")
     );
-    if (result.ok && result.diagnostics.length > 0) {
-      commandResult.text = `${commandResult.text ?? ""}${diagnosticsText(result.diagnostics)}`;
-    }
     emitCommandResult(this, "init", commandResult, flags.json === true);
   }
 }

packages/cli/src/commands/resume.ts

@@ -5,7 +5,6 @@ import {
   cliEntrypoint,
   cliIo,
   commandFromRuntime,
-  diagnosticsText,
   emitCommandResult,
   globalFlags,
   projectRoot
@@ -43,9 +42,6 @@ export default class Resume extends Command {
         ? `Submitted ${value.action}: ${value.workflow_run_id ?? value.run_id}\n`
         : `Run already active: ${value.workflow_run_id ?? value.run_id}; no new controller was started. If a pause is still draining, resume again once status reports paused; if its controller process just exited, resume again after 30 seconds.\n`
     );
-    if (result.ok && result.diagnostics.length > 0) {
-      commandResult.text = `${commandResult.text ?? ""}${diagnosticsText(result.diagnostics)}`;
-    }
     emitCommandResult(this, "resume", commandResult, flags.json);
   }
 }

packages/cli/test/e2e/campaign-resume.test.ts

@@ -358,7 +358,24 @@ function assertNoFinishedTaskRestarted(events: WorkflowEvent[]): void {
   assert.ok(finished.size > 0, "the workflow event stream has no finished tasks");
 }
 
-/** Launches the campaign and SIGKILLs its detached controller while `INTERRUPTED_NODE` runs. */
+/** Waits for the stub to hold `INTERRUPTED_NODE` in a process other than the `earlier` ones. */
+async function heldCall(campaign: Campaign, workflowRunId: string, label: string, earlier: number[] = []) {
+  return waitFor(label, 15 * MINUTE, () => {
+    const calls = agentCalls(campaign);
+    const call = calls.find(
+      (entry) => entry.node === INTERRUPTED_NODE && entry.event === "held" && !earlier.includes(entry.pid)
+    );
+    if (call === undefined && processesMentioning(workflowRunId).length === 0) {
+      assert.fail(`the workflow stopped while waiting for ${label}; agent calls: ${JSON.stringify(calls)}`);
+    }
+    return call;
+  });
+}
+
+/**
+ * Launches the campaign and SIGKILLs its detached engine while `INTERRUPTED_NODE` runs. Once the
+ * supervisor has relaunched the engine and the node runs again, SIGKILLs the whole controller.
+ */
 async function interruptMidRun(campaign: Campaign, mark: (phase: string) => void): Promise<void> {
   await ultrafuzz(campaign, ["init"], 2 * MINUTE);
   const configPath = path.join(campaign.project, "ultrafuzz.toml");
@@ -383,14 +400,19 @@ async function interruptMidRun(campaign: Campaign, mark: (phase: string) => void
   const [workflowRunId] = launched.workflow_ids;
   assert.ok(workflowRunId !== undefined, "run did not report its workflow run ID");
 
-  const held = await waitFor(`${INTERRUPTED_NODE} to start`, 15 * MINUTE, () => {
-    const calls = agentCalls(campaign);
-    const call = calls.find((entry) => entry.node === INTERRUPTED_NODE && entry.event === "held");
-    if (call === undefined && processesMentioning(workflowRunId).length === 0) {
-      assert.fail(`the workflow stopped before ${INTERRUPTED_NODE} started; agent calls: ${JSON.stringify(calls)}`);
-    }
-    return call;
+  const first = await heldCall(campaign, workflowRunId, `${INTERRUPTED_NODE} to start`);
+  // An engine killed on its own, as by the OOM killer, leaves its supervisor running. The supervisor
+  // relaunches it from the run's sealed execution snapshot with no `ultrafuzz` command.
+  const engines = processesMentioning(workflowRunId).filter((pid) => {
+    const argv = commandLine(pid) ?? "";
+    return argv.includes("\0up\0") && !argv.includes("\0supervise\0");
   });
+  assert.ok(engines.length > 0, "no detached engine process is running the workflow");
+  for (const pid of engines) kill(pid);
+  const held = await heldCall(campaign, workflowRunId, `the relaunched engine to rerun ${INTERRUPTED_NODE}`, [
+    first.pid
+  ]);
+  mark("engine relaunched");
   // A host crash takes down the detached engine and the supervisor that would otherwise restart it.
   const controller = processesMentioning(workflowRunId);
   assert.ok(controller.length > 0, "no detached controller process is running the workflow");
@@ -403,7 +425,7 @@ async function interruptMidRun(campaign: Campaign, mark: (phase: string) => void
 }
 
 test(
-  "a campaign whose controller is SIGKILLed mid-node resumes on the pinned engine without re-running finished nodes",
+  "a campaign whose engine and then whole controller are SIGKILLed mid-node resumes on the pinned engine without re-running finished nodes",
   { timeout: 45 * MINUTE, skip: process.platform === "linux" ? false : "finds the detached controller through /proc" },
   async (t) => {
     const campaign = prepareCampaign();
@@ -452,14 +474,19 @@ test(
         "RunFinished",
         `agent calls: ${JSON.stringify(calls)}`
       );
-      // Every agent ran once, except the node the kill interrupted, which ran again after resume.
+      // Every agent ran once, except the node the kills interrupted, which ran again in the relaunched
+      // engine and after resume.
       const starts = calls.filter((call) => call.event === "started");
       assert.deepEqual(
         AGENT_NODES.map((node) => [node, starts.filter((call) => call.node === node).length]),
-        AGENT_NODES.map((node) => [node, node === INTERRUPTED_NODE ? 2 : 1])
+        AGENT_NODES.map((node) => [node, node === INTERRUPTED_NODE ? 3 : 1])
       );
       assert.equal(events.truncated, false);
-      assert.equal(events.events.filter((event) => event.category === "RunStarted").length, 2);
+      assert.equal(events.events.filter((event) => event.category === "RunStarted").length, 3);
+      assert.ok(
+        events.events.some((event) => event.category === "RunAutoResumed"),
+        "no supervisor relaunch event"
+      );
       assertNoFinishedTaskRestarted(events.events);
 
       const health = await ultrafuzz<HealthValue>(campaign, ["status", runId]);

packages/cli/test/lifecycle-commands.test.ts

@@ -585,6 +585,19 @@ test("cancel distinguishes a submitted request from a confirmed cancellation", a
   assert.match(humanConfirmed.stdout, /^Cancellation confirmed: lifecycle-cli-run is canceled$/mu);
 });
 
+test("plain status prints the warnings of a successful poll", async (t) => {
+  // A cancel reply outside the runner's status contract fails the deadline cancel.
+  const { project, env, runRoot } = await launchedProject(t, { cancelStatus: "busy" });
+  const statePath = path.join(runRoot, "state.json");
+  const state = JSON.parse(fs.readFileSync(statePath, "utf8")) as Record<string, unknown>;
+  fs.writeFileSync(statePath, `${JSON.stringify({ ...state, workflow_deadline_at: "2000-01-01T00:00:00.000Z" })}\n`);
+
+  const status = await cli(project, ["status", RUN_ID], env);
+
+  assert.equal(status.code, 0, status.stderr);
+  assert.match(status.stdout, /^warning: WORKFLOW_DEADLINE_CANCEL_FAILED: /mu);
+});
+
 test("doctor reports install posture in human and JSON output", async (t) => {
   const { project, env } = await launchedProject(t);
   writeSmallTopology(project, "recon");