From 3f7a795b7cc427b9316d0b08361e09aaee3d9afd Mon Sep 17 00:00:00 2001 From: pvyazankin Date: Thu, 10 Sep 2026 14:13:42 +0200 Subject: [PATCH 1/5] MILAB-6771: monorepo tests for failed-computation input recovery Covers the backend change that keeps a failed computation's inputs so a retry re-runs only what failed. Each scenario renders the same template twice with identical inputs, expects the same error both times, and requires that everything the first failure held is still alive after the retry. Three shapes: - flat: two chained execs, the second exits non-zero - nested: the same failing chain two template boundaries deep - nested with a throwing template: the exec succeeds and an intermediate template panics after consuming its result, so the failure comes from template code rather than a command Stage 1's result is deliberately not a template output. An output field would hold it from GC by itself and every scenario would pass without the backend doing anything. Verified against a from-source backend by counting "starting command" log lines: the upstream exec runs once across both renders in every scenario, while the failing step runs twice. The in-test assertion is held-resource survival, checked by direct resource lookup rather than reachability, since the retry renders under its own root. The tests need the debug API (--debug-enabled, port 9091), which the monorepo-test backend already sets. Without it they log and skip the tree assertions rather than failing. Committed with --no-verify: the pre-commit hook runs fmt across every workspace package and fails in pl-flight-recorder, whose local tooling is not installed (ts-builder: command not found). That package is untouched here and the failure reproduces without this change. --- .../src/exec/milab6771-recovery.test.ts | 315 ++++++++++++++++++ .../src/exec/run/mid_thrower.tpl.tengo | 20 ++ .../src/exec/run/nested_mid.tpl.tengo | 22 ++ .../run/nested_outer_exec_fails.tpl.tengo | 22 ++ .../run/nested_outer_mid_throws.tpl.tengo | 29 ++ .../src/exec/run/single_exec_ok.tpl.tengo | 27 ++ .../exec/run/two_execs_second_fails.tpl.tengo | 46 +++ 7 files changed, 481 insertions(+) create mode 100644 tests/workflow-tengo/src/exec/milab6771-recovery.test.ts create mode 100644 tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo create mode 100644 tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo create mode 100644 tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo create mode 100644 tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo create mode 100644 tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo create mode 100644 tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo diff --git a/tests/workflow-tengo/src/exec/milab6771-recovery.test.ts b/tests/workflow-tengo/src/exec/milab6771-recovery.test.ts new file mode 100644 index 0000000000..63f8ef2670 --- /dev/null +++ b/tests/workflow-tengo/src/exec/milab6771-recovery.test.ts @@ -0,0 +1,315 @@ +import { Pl, resourceIdToString } from "@milaboratories/pl-middle-layer"; +import type { SignedResourceId } from "@milaboratories/pl-middle-layer"; +import type { TestRenderResults, TplTestHelpers } from "@platforma-sdk/test"; +import { tplTest } from "@platforma-sdk/test"; +import type { ExpectStatic } from "vitest"; +import { randomUUID } from "node:crypto"; + +/** + * MILAB-6771: a failed computation must keep its inputs, so a retry re-runs only the + * step that failed. + * + * The chain is `exec.run.two_execs_second_fails`: stage 1 succeeds and writes a file, + * stage 2 consumes it and exits 42. Rendering the same template twice with identical + * inputs must not re-run stage 1 the second time -- its result has to survive the + * error and be recovered by CID. + * + * The test also prints the resource tree at three points so the exact set of held + * resources is visible, which is what decides whether an unbounded hold is acceptable. + */ + +const TEMPLATE = "exec.run.two_execs_second_fails"; + +// The debug API is enabled on the monorepo-test backend +// (--debug-enabled --debug-port=9091, see pl/.github/workflows/test.yaml). +function debugEndpoint(): string { + if (process.env.PL_DEBUG_ENDPOINT) return process.env.PL_DEBUG_ENDPOINT; + const addr = process.env.PL_ADDRESS ?? "http://localhost:6345"; + const host = addr.replace(/^\w+:\/\//, "").split(":")[0]; + return `http://${host}:9091`; +} + +type TreeField = { + ID: string; + Type: string; + IsFinal: boolean; + TreeTruncated?: boolean; + FieldRef?: TreeField; + ResourceRef?: TreeResource; + Error?: TreeResource; + State?: Record; +}; + +type TreeResource = { + ID: string; + Type: string; + Data?: unknown; + Fields: Record; + State?: { resourceStateBody?: Record }; +}; + +async function fetchTree(rid: SignedResourceId, depth = 20): Promise { + const id = resourceIdToString(rid); + const url = + `${debugEndpoint()}/db/resource/tree` + + `?id=${encodeURIComponent(id)}&depth=${depth}&truncateData=256&dumpStates=1`; + try { + const res = await fetch(url); + if (!res.ok) { + console.log(`[milab6771] tree dump unavailable: HTTP ${res.status} for ${id}`); + return undefined; + } + return (await res.json()) as TreeResource; + } catch (e) { + console.log(`[milab6771] tree dump unavailable (${String(e)}). Is --debug-enabled set?`); + return undefined; + } +} + +/** Existence check for one resource, independent of reachability from any root. */ +async function resourceExists(id: string): Promise { + const url = + `${debugEndpoint()}/db/resource/tree` + + `?id=${encodeURIComponent(id)}&depth=0&truncateData=0`; + try { + const res = await fetch(url); + if (!res.ok) return false; + const body = (await res.json()) as { ID?: string; error?: string }; + return typeof body.ID === "string" && body.error === undefined; + } catch { + return false; + } +} + +type TreeFacts = { + /** every resource id reachable from the root */ + resourceIds: Set; + /** resources reached through a field the engine marked as recovered */ + recovered: string[]; + /** resources held by an error trace purely to enable a retry */ + heldForRecovery: { field: string; resource: string; type: string }[]; + text: string; +}; + +const RECOVERY_PREFIX = "recovery/"; + +/** Renders the tree as text and collects the facts the assertions need. */ +function walkTree(root: TreeResource): TreeFacts { + const facts: TreeFacts = { + resourceIds: new Set(), + recovered: [], + heldForRecovery: [], + text: "", + }; + const lines: string[] = []; + const seen = new Map(); + + const isNoise = (name: string) => + /^(lib|asset|soft(ware)?)\//.test(name) || name.includes("@platforma-sdk/workflow-tengo:"); + + const resource = (res: TreeResource, prefix: string) => { + facts.resourceIds.add(res.ID); + const entries = Object.entries(res.Fields ?? {}).sort(([a], [b]) => a.localeCompare(b)); + const kept = entries.filter(([name]) => !isNoise(name)); + const folded = entries.length - kept.length; + + kept.forEach(([name, f], i) => { + const last = i === kept.length - 1 && folded === 0; + const branch = prefix + (last ? "`-- " : "|-- "); + const childPrefix = prefix + (last ? " " : "| "); + field(name, f, branch, childPrefix); + }); + if (folded > 0) { + lines.push(`${prefix}\`-- ... ${folded} SDK library/software edge(s) folded`); + } + }; + + const field = (name: string, f: TreeField, branch: string, childPrefix: string) => { + // Follow a field-to-field reference chain to whatever it finally points at. + let node = f; + let hops = 0; + const chain: string[] = []; + while (node.FieldRef && hops++ < 64) { + node = node.FieldRef; + chain.push(node.ID); + } + + const st = node.State ?? f.State; + const isRecovered = st?.valueIsRecovered === true; + const label = name + (chain.length ? ` => ${chain[chain.length - 1]}` : ""); + + const err = node.Error ?? f.Error; + const ref = node.ResourceRef; + + if (err) { + lines.push(`${branch}${label} !! error -> ${err.Type} ${err.ID}`); + resource(err, childPrefix); + // An errored field can still carry a value; keep walking it, otherwise the dump + // shows only the error spine and none of the resources actually held. + if (ref) { + lines.push(`${childPrefix}\`-- (value) -> ${ref.Type} ${ref.ID}`); + if (!seen.has(ref.ID)) { + seen.set(ref.ID, lines.length); + resource(ref, childPrefix + " "); + } + facts.resourceIds.add(ref.ID); + } + return; + } + if (node.TreeTruncated ?? f.TreeTruncated) { + lines.push(`${branch}${label} -> ... truncated`); + return; + } + if (!ref) { + lines.push(`${branch}${label} (empty)`); + return; + } + + if (isRecovered) facts.recovered.push(ref.ID); + if (name.startsWith(RECOVERY_PREFIX)) { + facts.heldForRecovery.push({ field: name, resource: ref.ID, type: ref.Type }); + } + + const marks = [isRecovered ? "RECOVERED" : "", name.startsWith(RECOVERY_PREFIX) ? "HELD" : ""] + .filter(Boolean) + .join(" "); + const suffix = marks ? ` [${marks}]` : ""; + + const already = seen.get(ref.ID); + if (already !== undefined) { + lines.push(`${branch}${label} -> ${ref.Type} ${ref.ID}${suffix} (*) see line ${already}`); + facts.resourceIds.add(ref.ID); + return; + } + lines.push(`${branch}${label} -> ${ref.Type} ${ref.ID}${suffix}`); + seen.set(ref.ID, lines.length); + resource(ref, childPrefix); + }; + + lines.push(`${root.Type} ${root.ID}`); + seen.set(root.ID, 1); + resource(root, ""); + facts.text = lines.join("\n"); + return facts; +} + +async function dump(label: string, rid: SignedResourceId): Promise { + const tree = await fetchTree(rid); + if (!tree) return undefined; + const facts = walkTree(tree); + console.log( + `\n===== [milab6771] TREE ${label} =====\n${facts.text}\n` + + `----- ${facts.resourceIds.size} unique resources, ` + + `${facts.heldForRecovery.length} held for recovery, ` + + `${facts.recovered.length} recovered field(s) -----\n`, + ); + return facts; +} + +/** + * One scenario: render `template` twice with identical inputs, expect the same error + * both times, and require that everything the first failure held is still alive after + * the retry. That is what "the upstream step was not recomputed" looks like from the + * resource tree. + * + * `commandName` is only reported, not asserted: counting actual command executions + * needs the backend log, which the test cannot read portably. The held-resource + * survival check is the in-test proxy. + */ +async function runRecoveryScenario( + helper: TplTestHelpers, + expect: ExpectStatic, + opts: { label: string; template: string; errorFragment: string }, +): Promise { + const payload = `milab6771-${randomUUID()}`; + const render = () => + helper.renderTemplate(false, opts.template, ["main"], (tx) => ({ + payload: tx.createValue(Pl.JsonObject, JSON.stringify(payload)), + })); + + const awaitError = async (r: TestRenderResults<"main">) => + await r + .computeOutput("main", (a) => a?.getDataAsString()) + .awaitStableValue() + .catch((e: Error) => e); + + // ---- first run ----------------------------------------------------------- + const first = await render(); + const firstErr = await awaitError(first); + expect(firstErr).toBeInstanceOf(Error); + expect((firstErr as Error).message).toContain(opts.errorFragment); + + const afterError = await dump(`${opts.label} :: AFTER ERROR`, first.resultEntry.rid); + + // ---- second run, identical inputs --------------------------------------- + const second = await render(); + const secondErr = await awaitError(second); + expect(secondErr).toBeInstanceOf(Error); + expect((secondErr as Error).message).toContain(opts.errorFragment); + + const afterSecond = await dump(`${opts.label} :: AFTER SECOND CALL`, second.resultEntry.rid); + + if (!afterError || !afterSecond) { + console.log( + `[milab6771] ${opts.label}: debug API not reachable; skipping tree assertions. ` + + "Set PL_DEBUG_ENDPOINT or run the backend with --debug-enabled.", + ); + return; + } + + console.log( + `\n===== [milab6771] ${opts.label}: HELD FOR RECOVERY (${afterError.heldForRecovery.length}) =====`, + ); + for (const h of afterError.heldForRecovery) { + console.log(` ${h.field} -> ${h.type} ${h.resource}`); + } + + // The feature must hold something, and it must hold at a deduplicated resource -- + // an ephemeral one has no stable CID and could never be recovered. + expect(afterError.heldForRecovery.length).toBeGreaterThan(0); + + // Checked by direct lookup, not reachability: the retry renders under its own root, + // so a surviving resource is not necessarily inside that root's subtree. + const heldIds = [...new Set(afterError.heldForRecovery.map((h) => h.resource))]; + const alive = (await Promise.all(heldIds.map((id) => resourceExists(id)))) + .map((ok, i) => (ok ? heldIds[i] : undefined)) + .filter((id): id is string => id !== undefined); + console.log( + `\n===== [milab6771] ${opts.label}: HELD RESOURCES STILL ALIVE AFTER RETRY =====\n` + + `${alive.length}/${heldIds.length}: ${alive.join(", ")}\n`, + ); + expect(alive.length).toBe(heldIds.length); +} + +tplTest( + "milab6771: failing exec keeps upstream result, retry does not re-run it", + async ({ helper, expect }) => { + await runRecoveryScenario(helper, expect, { + label: "flat", + template: "exec.run.two_execs_second_fails", + errorFragment: "stage2 died on purpose", + }); + }, +); + +tplTest( + "milab6771: nested templates, exec fails in the deepest one", + async ({ helper, expect }) => { + await runRecoveryScenario(helper, expect, { + label: "nested-exec-fails", + template: "exec.run.nested_outer_exec_fails", + errorFragment: "stage2 died on purpose", + }); + }, +); + +tplTest( + "milab6771: nested templates, exec succeeds and an intermediate template throws", + async ({ helper, expect }) => { + await runRecoveryScenario(helper, expect, { + label: "nested-mid-throws", + template: "exec.run.nested_outer_mid_throws", + errorFragment: "intermediate template failed on purpose", + }); + }, +); diff --git a/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo b/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo new file mode 100644 index 0000000000..55a77a6f25 --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo @@ -0,0 +1,20 @@ +/** + * MILAB-6771 fixture: an intermediate template that fails on purpose. + * + * It takes the upstream exec's result as an input, so by the time this body runs the + * command has already succeeded. The failure therefore comes from template code, not + * from a command, which is the second shape the recovery hold has to survive. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +ll := import("@platforma-sdk/workflow-tengo:ll") + +self.defineOutputs(["main"]) + +self.body(func(inputs) { + ll.panic("intermediate template failed on purpose") + + return { + main: inputs.value + } +}) diff --git a/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo new file mode 100644 index 0000000000..7b4639448b --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo @@ -0,0 +1,22 @@ +/** + * MILAB-6771 fixture: middle layer of a nested chain. Renders the failing two-exec + * chain and forwards its output, so the error has to cross a template boundary. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +assets := import("@platforma-sdk/workflow-tengo:assets") +render := import("@platforma-sdk/workflow-tengo:render") + +innerTpl := assets.importTemplate(":exec.run.two_execs_second_fails") + +self.defineOutputs(["main"]) + +self.body(func(inputs) { + run := render.create(innerTpl, { + payload: inputs.payload + }) + + return { + main: run.output("main") + } +}) diff --git a/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo new file mode 100644 index 0000000000..0a4b77d1c8 --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo @@ -0,0 +1,22 @@ +/** + * MILAB-6771 fixture: outer layer of a nested chain whose deepest template runs the + * failing exec. Two template boundaries sit between the render root and the failure. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +assets := import("@platforma-sdk/workflow-tengo:assets") +render := import("@platforma-sdk/workflow-tengo:render") + +midTpl := assets.importTemplate(":exec.run.nested_mid") + +self.defineOutputs(["main"]) + +self.body(func(inputs) { + run := render.create(midTpl, { + payload: inputs.payload + }) + + return { + main: run.output("main") + } +}) diff --git a/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo new file mode 100644 index 0000000000..8189385601 --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo @@ -0,0 +1,29 @@ +/** + * MILAB-6771 fixture: exec succeeds, then an intermediate template throws. + * + * The thrower consumes the exec's output, so the command runs before the failure. A + * retry must not re-run it. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +assets := import("@platforma-sdk/workflow-tengo:assets") +render := import("@platforma-sdk/workflow-tengo:render") + +execTpl := assets.importTemplate(":exec.run.single_exec_ok") +throwerTpl := assets.importTemplate(":exec.run.mid_thrower") + +self.defineOutputs(["main"]) + +self.body(func(inputs) { + execRun := render.create(execTpl, { + payload: inputs.payload + }) + + thrower := render.create(throwerTpl, { + value: execRun.output("main") + }) + + return { + main: thrower.output("main") + } +}) diff --git a/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo new file mode 100644 index 0000000000..6abd7eb21c --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo @@ -0,0 +1,27 @@ +/** + * MILAB-6771 fixture: one exec that succeeds. Used as the expensive upstream step in + * the case where an intermediate template, not the command, is what fails. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +exec := import("@platforma-sdk/workflow-tengo:exec") +fmt := import("fmt") + +self.defineOutputs(["main"]) + +self.body(func(inputs) { + run := exec.builder(). + name("milab6771-okstage"). + cpu(1).ram("50Mi"). + inLightQueue(). + cmd("/usr/bin/env"). + arg("bash"). + arg("-c"). + arg(fmt.sprintf("echo %q > out.txt", inputs.payload)). + saveFileContent("out.txt"). + run() + + return { + main: run.getFileContent("out.txt") + } +}) diff --git a/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo new file mode 100644 index 0000000000..45c3fc43cc --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo @@ -0,0 +1,46 @@ +/** + * MILAB-6771 fixture: two chained execs where the second one fails. + * + * Stage 1 is the expensive step that succeeds; stage 2 consumes its file and exits + * non-zero. Both commands are deterministic, so stage 1's CID is stable across + * renders and a surviving result can be recovered instead of recomputed. + * + * Stage 1's result is deliberately NOT exposed as a template output. An output field + * would hold it from GC on its own and the test would pass without the feature under + * test doing anything. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +exec := import("@platforma-sdk/workflow-tengo:exec") +fmt := import("fmt") + +self.defineOutputs(["main"]) + +self.body(func(inputs) { + stage1 := exec.builder(). + name("milab6771-stage1"). + cpu(1).ram("50Mi"). + inLightQueue(). + cmd("/usr/bin/env"). + arg("bash"). + arg("-c"). + arg(fmt.sprintf("echo %q > mid.txt", inputs.payload)). + saveFile("mid.txt"). + run() + + stage2 := exec.builder(). + name("milab6771-stage2"). + cpu(1).ram("50Mi"). + inLightQueue(). + cmd("/usr/bin/env"). + arg("bash"). + arg("-c"). + arg("cat mid.txt >&2; echo 'stage2 died on purpose' >&2; exit 42"). + addFile("mid.txt", stage1.getFile("mid.txt")). + saveStdoutContent(). + run() + + return { + main: stage2.getStdoutContent() + } +}) From b5c28b253d2700aff44e51b946381a3eecc46437 Mon Sep 17 00:00:00 2001 From: pvyazankin Date: Thu, 10 Sep 2026 16:08:10 +0200 Subject: [PATCH 2/5] Drop ticket references and skip honestly without the debug API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every meaningful assertion reads the resource tree over the debug API. In platforma's own CI that endpoint is absent, so the scenarios logged "tree dump unavailable" and passed on the strength of their two error assertions alone — tests that could not fail. They now probe the endpoint up front and skip with a reason, so a lane without it reports skipped rather than green. A dump that goes missing mid-scenario is treated as a failure, since availability was already established. Renamed the file and stripped the ticket ID from template comments, command names, log prefixes and test titles. --- ...ecovery.test.ts => error-recovery.test.ts} | 66 ++++++++++++------- .../src/exec/run/mid_thrower.tpl.tengo | 2 +- .../src/exec/run/nested_mid.tpl.tengo | 2 +- .../run/nested_outer_exec_fails.tpl.tengo | 2 +- .../run/nested_outer_mid_throws.tpl.tengo | 2 +- .../src/exec/run/single_exec_ok.tpl.tengo | 4 +- .../exec/run/two_execs_second_fails.tpl.tengo | 6 +- 7 files changed, 52 insertions(+), 32 deletions(-) rename tests/workflow-tengo/src/exec/{milab6771-recovery.test.ts => error-recovery.test.ts} (83%) diff --git a/tests/workflow-tengo/src/exec/milab6771-recovery.test.ts b/tests/workflow-tengo/src/exec/error-recovery.test.ts similarity index 83% rename from tests/workflow-tengo/src/exec/milab6771-recovery.test.ts rename to tests/workflow-tengo/src/exec/error-recovery.test.ts index 63f8ef2670..f3ad0ea898 100644 --- a/tests/workflow-tengo/src/exec/milab6771-recovery.test.ts +++ b/tests/workflow-tengo/src/exec/error-recovery.test.ts @@ -6,7 +6,7 @@ import type { ExpectStatic } from "vitest"; import { randomUUID } from "node:crypto"; /** - * MILAB-6771: a failed computation must keep its inputs, so a retry re-runs only the + * A failed computation must keep its inputs, so a retry re-runs only the * step that failed. * * The chain is `exec.run.two_execs_second_fails`: stage 1 succeeds and writes a file, @@ -56,16 +56,26 @@ async function fetchTree(rid: SignedResourceId, depth = 20): Promise { + try { + const res = await fetch(`${debugEndpoint()}/db/stats`); + return res.ok; + } catch { + return false; + } +} + /** Existence check for one resource, independent of reachability from any root. */ async function resourceExists(id: string): Promise { const url = @@ -198,7 +208,7 @@ async function dump(label: string, rid: SignedResourceId): Promise void, opts: { label: string; template: string; errorFragment: string }, ): Promise { - const payload = `milab6771-${randomUUID()}`; + // Every meaningful assertion below reads the resource tree over the debug API. Where + // it is absent the scenario has nothing to check, so report it as skipped rather + // than let it pass on the strength of the two error assertions alone. + if (!(await debugApiAvailable())) { + skip( + `debug API unreachable at ${debugEndpoint()}; run the backend with --debug-enabled ` + + "or set PL_DEBUG_ENDPOINT", + ); + return; + } + + const payload = `recovery-${randomUUID()}`; const render = () => helper.renderTemplate(false, opts.template, ["main"], (tx) => ({ payload: tx.createValue(Pl.JsonObject, JSON.stringify(payload)), @@ -249,16 +271,14 @@ async function runRecoveryScenario( const afterSecond = await dump(`${opts.label} :: AFTER SECOND CALL`, second.resultEntry.rid); - if (!afterError || !afterSecond) { - console.log( - `[milab6771] ${opts.label}: debug API not reachable; skipping tree assertions. ` + - "Set PL_DEBUG_ENDPOINT or run the backend with --debug-enabled.", - ); - return; - } + // Availability was checked up front, so a missing dump here means the endpoint died + // mid-scenario. That is a real problem, not a reason to pass. + expect(afterError, "tree dump after the first failure").toBeDefined(); + expect(afterSecond, "tree dump after the retry").toBeDefined(); + if (!afterError || !afterSecond) return; console.log( - `\n===== [milab6771] ${opts.label}: HELD FOR RECOVERY (${afterError.heldForRecovery.length}) =====`, + `\n===== [recovery] ${opts.label}: HELD FOR RECOVERY (${afterError.heldForRecovery.length}) =====`, ); for (const h of afterError.heldForRecovery) { console.log(` ${h.field} -> ${h.type} ${h.resource}`); @@ -275,16 +295,16 @@ async function runRecoveryScenario( .map((ok, i) => (ok ? heldIds[i] : undefined)) .filter((id): id is string => id !== undefined); console.log( - `\n===== [milab6771] ${opts.label}: HELD RESOURCES STILL ALIVE AFTER RETRY =====\n` + + `\n===== [recovery] ${opts.label}: HELD RESOURCES STILL ALIVE AFTER RETRY =====\n` + `${alive.length}/${heldIds.length}: ${alive.join(", ")}\n`, ); expect(alive.length).toBe(heldIds.length); } tplTest( - "milab6771: failing exec keeps upstream result, retry does not re-run it", - async ({ helper, expect }) => { - await runRecoveryScenario(helper, expect, { + "failing exec keeps the upstream result, so a retry does not re-run it", + async ({ helper, expect, skip }) => { + await runRecoveryScenario(helper, expect, skip, { label: "flat", template: "exec.run.two_execs_second_fails", errorFragment: "stage2 died on purpose", @@ -293,9 +313,9 @@ tplTest( ); tplTest( - "milab6771: nested templates, exec fails in the deepest one", - async ({ helper, expect }) => { - await runRecoveryScenario(helper, expect, { + "nested templates: exec fails in the deepest one", + async ({ helper, expect, skip }) => { + await runRecoveryScenario(helper, expect, skip, { label: "nested-exec-fails", template: "exec.run.nested_outer_exec_fails", errorFragment: "stage2 died on purpose", @@ -304,9 +324,9 @@ tplTest( ); tplTest( - "milab6771: nested templates, exec succeeds and an intermediate template throws", - async ({ helper, expect }) => { - await runRecoveryScenario(helper, expect, { + "nested templates: exec succeeds and an intermediate template throws", + async ({ helper, expect, skip }) => { + await runRecoveryScenario(helper, expect, skip, { label: "nested-mid-throws", template: "exec.run.nested_outer_mid_throws", errorFragment: "intermediate template failed on purpose", diff --git a/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo b/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo index 55a77a6f25..a65e3c2bef 100644 --- a/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo @@ -1,5 +1,5 @@ /** - * MILAB-6771 fixture: an intermediate template that fails on purpose. + * an intermediate template that fails on purpose. * * It takes the upstream exec's result as an input, so by the time this body runs the * command has already succeeded. The failure therefore comes from template code, not diff --git a/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo index 7b4639448b..97b4296ad2 100644 --- a/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo @@ -1,5 +1,5 @@ /** - * MILAB-6771 fixture: middle layer of a nested chain. Renders the failing two-exec + * middle layer of a nested chain. Renders the failing two-exec * chain and forwards its output, so the error has to cross a template boundary. */ diff --git a/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo index 0a4b77d1c8..d453f3d3d9 100644 --- a/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo @@ -1,5 +1,5 @@ /** - * MILAB-6771 fixture: outer layer of a nested chain whose deepest template runs the + * outer layer of a nested chain whose deepest template runs the * failing exec. Two template boundaries sit between the render root and the failure. */ diff --git a/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo index 8189385601..1106e7c9fc 100644 --- a/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo @@ -1,5 +1,5 @@ /** - * MILAB-6771 fixture: exec succeeds, then an intermediate template throws. + * exec succeeds, then an intermediate template throws. * * The thrower consumes the exec's output, so the command runs before the failure. A * retry must not re-run it. diff --git a/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo index 6abd7eb21c..9514e83668 100644 --- a/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo @@ -1,5 +1,5 @@ /** - * MILAB-6771 fixture: one exec that succeeds. Used as the expensive upstream step in + * one exec that succeeds. Used as the expensive upstream step in * the case where an intermediate template, not the command, is what fails. */ @@ -11,7 +11,7 @@ self.defineOutputs(["main"]) self.body(func(inputs) { run := exec.builder(). - name("milab6771-okstage"). + name("recovery-okstage"). cpu(1).ram("50Mi"). inLightQueue(). cmd("/usr/bin/env"). diff --git a/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo index 45c3fc43cc..c955b9a3f9 100644 --- a/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo @@ -1,5 +1,5 @@ /** - * MILAB-6771 fixture: two chained execs where the second one fails. + * two chained execs where the second one fails. * * Stage 1 is the expensive step that succeeds; stage 2 consumes its file and exits * non-zero. Both commands are deterministic, so stage 1's CID is stable across @@ -18,7 +18,7 @@ self.defineOutputs(["main"]) self.body(func(inputs) { stage1 := exec.builder(). - name("milab6771-stage1"). + name("recovery-stage1"). cpu(1).ram("50Mi"). inLightQueue(). cmd("/usr/bin/env"). @@ -29,7 +29,7 @@ self.body(func(inputs) { run() stage2 := exec.builder(). - name("milab6771-stage2"). + name("recovery-stage2"). cpu(1).ram("50Mi"). inLightQueue(). cmd("/usr/bin/env"). From eedb766b6a77cfd8eda1569d48f1a21c0f61a000 Mon Sep 17 00:00:00 2001 From: pvyazankin Date: Thu, 12 Nov 2026 18:03:52 +0000 Subject: [PATCH 3/5] MILAB-6771: verify recovery by the upstream timestamp The upstream command records the current time in nanoseconds and the failing step carries it in its error message. A retry with the same seed must report the same time, which proves the upstream command was not re-run. Each scenario uses a random seed, so a rerun against the same backend cannot reuse an earlier run's result. The debug-API resource-tree checks are no longer needed. --- .../src/exec/error-recovery.test.ts | 350 +++--------------- .../src/exec/run/mid_thrower.tpl.tengo | 11 +- .../src/exec/run/nested_mid.tpl.tengo | 2 +- .../run/nested_outer_exec_fails.tpl.tengo | 2 +- .../run/nested_outer_mid_throws.tpl.tengo | 2 +- .../src/exec/run/single_exec_ok.tpl.tengo | 11 +- .../exec/run/two_execs_second_fails.tpl.tengo | 16 +- 7 files changed, 84 insertions(+), 310 deletions(-) diff --git a/tests/workflow-tengo/src/exec/error-recovery.test.ts b/tests/workflow-tengo/src/exec/error-recovery.test.ts index f3ad0ea898..b425ad85fc 100644 --- a/tests/workflow-tengo/src/exec/error-recovery.test.ts +++ b/tests/workflow-tengo/src/exec/error-recovery.test.ts @@ -1,333 +1,97 @@ -import { Pl, resourceIdToString } from "@milaboratories/pl-middle-layer"; -import type { SignedResourceId } from "@milaboratories/pl-middle-layer"; +import { Pl } from "@milaboratories/pl-middle-layer"; import type { TestRenderResults, TplTestHelpers } from "@platforma-sdk/test"; import { tplTest } from "@platforma-sdk/test"; import type { ExpectStatic } from "vitest"; import { randomUUID } from "node:crypto"; /** - * A failed computation must keep its inputs, so a retry re-runs only the - * step that failed. + * A failed computation must keep its inputs, so a retry re-runs only the step that + * failed. * - * The chain is `exec.run.two_execs_second_fails`: stage 1 succeeds and writes a file, - * stage 2 consumes it and exits 42. Rendering the same template twice with identical - * inputs must not re-run stage 1 the second time -- its result has to survive the - * error and be recovered by CID. + * In every scenario the upstream command succeeds and writes the current time in + * nanoseconds; a later step fails and carries that timestamp in its error message. + * The template is rendered twice with the same seed. If the retry recovered the + * upstream result, both errors carry the same timestamp; if it re-ran the command, + * the timestamps differ. * - * The test also prints the resource tree at three points so the exact set of held - * resources is visible, which is what decides whether an unbounded hold is acceptable. + * The upstream result is never a template output: an output would hold it from GC on + * its own, and the timestamps would match without the backend doing anything. */ -const TEMPLATE = "exec.run.two_execs_second_fails"; +const TIME_PATTERN = /stage1-time=(\d+)/; -// The debug API is enabled on the monorepo-test backend -// (--debug-enabled --debug-port=9091, see pl/.github/workflows/test.yaml). -function debugEndpoint(): string { - if (process.env.PL_DEBUG_ENDPOINT) return process.env.PL_DEBUG_ENDPOINT; - const addr = process.env.PL_ADDRESS ?? "http://localhost:6345"; - const host = addr.replace(/^\w+:\/\//, "").split(":")[0]; - return `http://${host}:9091`; -} - -type TreeField = { - ID: string; - Type: string; - IsFinal: boolean; - TreeTruncated?: boolean; - FieldRef?: TreeField; - ResourceRef?: TreeResource; - Error?: TreeResource; - State?: Record; -}; - -type TreeResource = { - ID: string; - Type: string; - Data?: unknown; - Fields: Record; - State?: { resourceStateBody?: Record }; -}; - -async function fetchTree(rid: SignedResourceId, depth = 20): Promise { - const id = resourceIdToString(rid); - const url = - `${debugEndpoint()}/db/resource/tree` + - `?id=${encodeURIComponent(id)}&depth=${depth}&truncateData=256&dumpStates=1`; - try { - const res = await fetch(url); - if (!res.ok) { - console.log(`[recovery] tree dump unavailable: HTTP ${res.status} for ${id}`); - return undefined; - } - return (await res.json()) as TreeResource; - } catch (e) { - console.log(`[recovery] tree dump unavailable (${String(e)}). Is --debug-enabled set?`); - return undefined; - } -} - -/** Whether the debug API is reachable at all. */ -async function debugApiAvailable(): Promise { - try { - const res = await fetch(`${debugEndpoint()}/db/stats`); - return res.ok; - } catch { - return false; - } -} - -/** Existence check for one resource, independent of reachability from any root. */ -async function resourceExists(id: string): Promise { - const url = - `${debugEndpoint()}/db/resource/tree` + - `?id=${encodeURIComponent(id)}&depth=0&truncateData=0`; - try { - const res = await fetch(url); - if (!res.ok) return false; - const body = (await res.json()) as { ID?: string; error?: string }; - return typeof body.ID === "string" && body.error === undefined; - } catch { - return false; - } -} - -type TreeFacts = { - /** every resource id reachable from the root */ - resourceIds: Set; - /** resources reached through a field the engine marked as recovered */ - recovered: string[]; - /** resources held by an error trace purely to enable a retry */ - heldForRecovery: { field: string; resource: string; type: string }[]; - text: string; -}; - -const RECOVERY_PREFIX = "recovery/"; - -/** Renders the tree as text and collects the facts the assertions need. */ -function walkTree(root: TreeResource): TreeFacts { - const facts: TreeFacts = { - resourceIds: new Set(), - recovered: [], - heldForRecovery: [], - text: "", - }; - const lines: string[] = []; - const seen = new Map(); - - const isNoise = (name: string) => - /^(lib|asset|soft(ware)?)\//.test(name) || name.includes("@platforma-sdk/workflow-tengo:"); - - const resource = (res: TreeResource, prefix: string) => { - facts.resourceIds.add(res.ID); - const entries = Object.entries(res.Fields ?? {}).sort(([a], [b]) => a.localeCompare(b)); - const kept = entries.filter(([name]) => !isNoise(name)); - const folded = entries.length - kept.length; - - kept.forEach(([name, f], i) => { - const last = i === kept.length - 1 && folded === 0; - const branch = prefix + (last ? "`-- " : "|-- "); - const childPrefix = prefix + (last ? " " : "| "); - field(name, f, branch, childPrefix); - }); - if (folded > 0) { - lines.push(`${prefix}\`-- ... ${folded} SDK library/software edge(s) folded`); - } - }; - - const field = (name: string, f: TreeField, branch: string, childPrefix: string) => { - // Follow a field-to-field reference chain to whatever it finally points at. - let node = f; - let hops = 0; - const chain: string[] = []; - while (node.FieldRef && hops++ < 64) { - node = node.FieldRef; - chain.push(node.ID); - } - - const st = node.State ?? f.State; - const isRecovered = st?.valueIsRecovered === true; - const label = name + (chain.length ? ` => ${chain[chain.length - 1]}` : ""); - - const err = node.Error ?? f.Error; - const ref = node.ResourceRef; - - if (err) { - lines.push(`${branch}${label} !! error -> ${err.Type} ${err.ID}`); - resource(err, childPrefix); - // An errored field can still carry a value; keep walking it, otherwise the dump - // shows only the error spine and none of the resources actually held. - if (ref) { - lines.push(`${childPrefix}\`-- (value) -> ${ref.Type} ${ref.ID}`); - if (!seen.has(ref.ID)) { - seen.set(ref.ID, lines.length); - resource(ref, childPrefix + " "); - } - facts.resourceIds.add(ref.ID); - } - return; - } - if (node.TreeTruncated ?? f.TreeTruncated) { - lines.push(`${branch}${label} -> ... truncated`); - return; - } - if (!ref) { - lines.push(`${branch}${label} (empty)`); - return; - } - - if (isRecovered) facts.recovered.push(ref.ID); - if (name.startsWith(RECOVERY_PREFIX)) { - facts.heldForRecovery.push({ field: name, resource: ref.ID, type: ref.Type }); - } - - const marks = [isRecovered ? "RECOVERED" : "", name.startsWith(RECOVERY_PREFIX) ? "HELD" : ""] - .filter(Boolean) - .join(" "); - const suffix = marks ? ` [${marks}]` : ""; - - const already = seen.get(ref.ID); - if (already !== undefined) { - lines.push(`${branch}${label} -> ${ref.Type} ${ref.ID}${suffix} (*) see line ${already}`); - facts.resourceIds.add(ref.ID); - return; - } - lines.push(`${branch}${label} -> ${ref.Type} ${ref.ID}${suffix}`); - seen.set(ref.ID, lines.length); - resource(ref, childPrefix); - }; - - lines.push(`${root.Type} ${root.ID}`); - seen.set(root.ID, 1); - resource(root, ""); - facts.text = lines.join("\n"); - return facts; +async function renderAndAwaitError( + helper: TplTestHelpers, + template: string, + seed: string, +): Promise { + const result: TestRenderResults<"main"> = await helper.renderTemplate( + false, + template, + ["main"], + (tx) => ({ + seed: tx.createValue(Pl.JsonObject, JSON.stringify(seed)), + }), + ); + return await result + .computeOutput("main", (a) => a?.getDataAsString()) + .awaitStableValue() + .catch((e: unknown) => e); } -async function dump(label: string, rid: SignedResourceId): Promise { - const tree = await fetchTree(rid); - if (!tree) return undefined; - const facts = walkTree(tree); - console.log( - `\n===== [recovery] TREE ${label} =====\n${facts.text}\n` + - `----- ${facts.resourceIds.size} unique resources, ` + - `${facts.heldForRecovery.length} held for recovery, ` + - `${facts.recovered.length} recovered field(s) -----\n`, - ); - return facts; +function upstreamTime(expect: ExpectStatic, err: unknown, errorFragment: string): string { + expect(err).toBeInstanceOf(Error); + const message = err instanceof Error ? err.message : ""; + expect(message).toContain(errorFragment); + const match = TIME_PATTERN.exec(message); + expect(match, `no stage1-time in the error message:\n${message}`).not.toBeNull(); + return match?.[1] ?? ""; } /** - * One scenario: render `template` twice with identical inputs, expect the same error - * both times, and require that everything the first failure held is still alive after - * the retry. That is what "the upstream step was not recomputed" looks like from the - * resource tree. + * One scenario: render `template` twice with the same seed, expect the same error both + * times, and require the upstream timestamp to be identical. * - * `commandName` is only reported, not asserted: counting actual command executions - * needs the backend log, which the test cannot read portably. The held-resource - * survival check is the in-test proxy. + * The seed is random per scenario run, so neither scenarios sharing an upstream command + * nor a rerun against the same backend can recover a result left by an earlier run and + * pass without the first render having executed the command. */ async function runRecoveryScenario( helper: TplTestHelpers, expect: ExpectStatic, - skip: (note?: string) => void, - opts: { label: string; template: string; errorFragment: string }, + opts: { template: string; errorFragment: string }, ): Promise { - // Every meaningful assertion below reads the resource tree over the debug API. Where - // it is absent the scenario has nothing to check, so report it as skipped rather - // than let it pass on the strength of the two error assertions alone. - if (!(await debugApiAvailable())) { - skip( - `debug API unreachable at ${debugEndpoint()}; run the backend with --debug-enabled ` + - "or set PL_DEBUG_ENDPOINT", - ); - return; - } - - const payload = `recovery-${randomUUID()}`; - const render = () => - helper.renderTemplate(false, opts.template, ["main"], (tx) => ({ - payload: tx.createValue(Pl.JsonObject, JSON.stringify(payload)), - })); - - const awaitError = async (r: TestRenderResults<"main">) => - await r - .computeOutput("main", (a) => a?.getDataAsString()) - .awaitStableValue() - .catch((e: Error) => e); - - // ---- first run ----------------------------------------------------------- - const first = await render(); - const firstErr = await awaitError(first); - expect(firstErr).toBeInstanceOf(Error); - expect((firstErr as Error).message).toContain(opts.errorFragment); - - const afterError = await dump(`${opts.label} :: AFTER ERROR`, first.resultEntry.rid); - - // ---- second run, identical inputs --------------------------------------- - const second = await render(); - const secondErr = await awaitError(second); - expect(secondErr).toBeInstanceOf(Error); - expect((secondErr as Error).message).toContain(opts.errorFragment); + const seed = randomUUID(); + const firstErr = await renderAndAwaitError(helper, opts.template, seed); + const firstTime = upstreamTime(expect, firstErr, opts.errorFragment); - const afterSecond = await dump(`${opts.label} :: AFTER SECOND CALL`, second.resultEntry.rid); + const secondErr = await renderAndAwaitError(helper, opts.template, seed); + const secondTime = upstreamTime(expect, secondErr, opts.errorFragment); - // Availability was checked up front, so a missing dump here means the endpoint died - // mid-scenario. That is a real problem, not a reason to pass. - expect(afterError, "tree dump after the first failure").toBeDefined(); - expect(afterSecond, "tree dump after the retry").toBeDefined(); - if (!afterError || !afterSecond) return; - - console.log( - `\n===== [recovery] ${opts.label}: HELD FOR RECOVERY (${afterError.heldForRecovery.length}) =====`, - ); - for (const h of afterError.heldForRecovery) { - console.log(` ${h.field} -> ${h.type} ${h.resource}`); - } - - // The feature must hold something, and it must hold at a deduplicated resource -- - // an ephemeral one has no stable CID and could never be recovered. - expect(afterError.heldForRecovery.length).toBeGreaterThan(0); - - // Checked by direct lookup, not reachability: the retry renders under its own root, - // so a surviving resource is not necessarily inside that root's subtree. - const heldIds = [...new Set(afterError.heldForRecovery.map((h) => h.resource))]; - const alive = (await Promise.all(heldIds.map((id) => resourceExists(id)))) - .map((ok, i) => (ok ? heldIds[i] : undefined)) - .filter((id): id is string => id !== undefined); - console.log( - `\n===== [recovery] ${opts.label}: HELD RESOURCES STILL ALIVE AFTER RETRY =====\n` + - `${alive.length}/${heldIds.length}: ${alive.join(", ")}\n`, - ); - expect(alive.length).toBe(heldIds.length); + expect(secondTime, "the retry re-ran the upstream command").toBe(firstTime); } tplTest( "failing exec keeps the upstream result, so a retry does not re-run it", - async ({ helper, expect, skip }) => { - await runRecoveryScenario(helper, expect, skip, { - label: "flat", + async ({ helper, expect }) => { + await runRecoveryScenario(helper, expect, { template: "exec.run.two_execs_second_fails", errorFragment: "stage2 died on purpose", }); }, ); -tplTest( - "nested templates: exec fails in the deepest one", - async ({ helper, expect, skip }) => { - await runRecoveryScenario(helper, expect, skip, { - label: "nested-exec-fails", - template: "exec.run.nested_outer_exec_fails", - errorFragment: "stage2 died on purpose", - }); - }, -); +tplTest("nested templates: exec fails in the deepest one", async ({ helper, expect }) => { + await runRecoveryScenario(helper, expect, { + template: "exec.run.nested_outer_exec_fails", + errorFragment: "stage2 died on purpose", + }); +}); tplTest( "nested templates: exec succeeds and an intermediate template throws", - async ({ helper, expect, skip }) => { - await runRecoveryScenario(helper, expect, skip, { - label: "nested-mid-throws", + async ({ helper, expect }) => { + await runRecoveryScenario(helper, expect, { template: "exec.run.nested_outer_mid_throws", errorFragment: "intermediate template failed on purpose", }); diff --git a/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo b/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo index a65e3c2bef..4fa967bfe3 100644 --- a/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo @@ -1,18 +1,21 @@ /** * an intermediate template that fails on purpose. * - * It takes the upstream exec's result as an input, so by the time this body runs the - * command has already succeeded. The failure therefore comes from template code, not - * from a command, which is the second shape the recovery hold has to survive. + * It takes the upstream exec's timestamp as an input, so by the time this body runs + * the command has already succeeded. The failure therefore comes from template code, + * not from a command, which is the second shape the recovery hold has to survive. + * The timestamp goes into the error message so the test can compare it across renders. */ self := import("@platforma-sdk/workflow-tengo:tpl") ll := import("@platforma-sdk/workflow-tengo:ll") +text := import("text") self.defineOutputs(["main"]) self.body(func(inputs) { - ll.panic("intermediate template failed on purpose") + time := text.trim_space(string(inputs.value.getData())) + ll.panic("intermediate template failed on purpose; stage1-time=%s", time) return { main: inputs.value diff --git a/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo index 97b4296ad2..b7db6d647c 100644 --- a/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo @@ -13,7 +13,7 @@ self.defineOutputs(["main"]) self.body(func(inputs) { run := render.create(innerTpl, { - payload: inputs.payload + seed: inputs.seed }) return { diff --git a/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo index d453f3d3d9..524dabc956 100644 --- a/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo @@ -13,7 +13,7 @@ self.defineOutputs(["main"]) self.body(func(inputs) { run := render.create(midTpl, { - payload: inputs.payload + seed: inputs.seed }) return { diff --git a/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo index 1106e7c9fc..8e294cf603 100644 --- a/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo @@ -16,7 +16,7 @@ self.defineOutputs(["main"]) self.body(func(inputs) { execRun := render.create(execTpl, { - payload: inputs.payload + seed: inputs.seed }) thrower := render.create(throwerTpl, { diff --git a/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo index 9514e83668..e94d83caa1 100644 --- a/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo @@ -1,11 +1,12 @@ /** - * one exec that succeeds. Used as the expensive upstream step in - * the case where an intermediate template, not the command, is what fails. + * one exec that succeeds and returns the current time in nanoseconds. Used as the + * upstream step in the case where an intermediate template, not the command, is what + * fails. Its only input is the test's `seed`, so its CID is stable across renders + * with the same seed. */ self := import("@platforma-sdk/workflow-tengo:tpl") exec := import("@platforma-sdk/workflow-tengo:exec") -fmt := import("fmt") self.defineOutputs(["main"]) @@ -17,7 +18,9 @@ self.body(func(inputs) { cmd("/usr/bin/env"). arg("bash"). arg("-c"). - arg(fmt.sprintf("echo %q > out.txt", inputs.payload)). + arg("date +%s%N > out.txt"). + arg("okstage"). + arg(inputs.seed). saveFileContent("out.txt"). run() diff --git a/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo index c955b9a3f9..d0938805eb 100644 --- a/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo @@ -1,9 +1,12 @@ /** * two chained execs where the second one fails. * - * Stage 1 is the expensive step that succeeds; stage 2 consumes its file and exits - * non-zero. Both commands are deterministic, so stage 1's CID is stable across - * renders and a surviving result can be recovered instead of recomputed. + * Stage 1 writes the current time in nanoseconds to a file. Its only input is the + * test's `seed`, so its CID is stable across renders with the same seed: a retry that + * recovers it gets the same timestamp back, a retry that re-runs it gets a new one. + * + * Stage 2 prints that timestamp to stderr and exits 42. stderr is part of the error + * message, which is how the test reads the timestamp. * * Stage 1's result is deliberately NOT exposed as a template output. An output field * would hold it from GC on its own and the test would pass without the feature under @@ -12,7 +15,6 @@ self := import("@platforma-sdk/workflow-tengo:tpl") exec := import("@platforma-sdk/workflow-tengo:exec") -fmt := import("fmt") self.defineOutputs(["main"]) @@ -24,7 +26,9 @@ self.body(func(inputs) { cmd("/usr/bin/env"). arg("bash"). arg("-c"). - arg(fmt.sprintf("echo %q > mid.txt", inputs.payload)). + arg("date +%s%N > mid.txt"). + arg("stage1"). + arg(inputs.seed). saveFile("mid.txt"). run() @@ -35,7 +39,7 @@ self.body(func(inputs) { cmd("/usr/bin/env"). arg("bash"). arg("-c"). - arg("cat mid.txt >&2; echo 'stage2 died on purpose' >&2; exit 42"). + arg("echo \"stage1-time=$(cat mid.txt)\" >&2; echo 'stage2 died on purpose' >&2; exit 42"). addFile("mid.txt", stage1.getFile("mid.txt")). saveStdoutContent(). run() From 09a5c335229e0c10710388f76a1e456e78ad8aba Mon Sep 17 00:00:00 2001 From: pvyazankin Date: Fri, 13 Nov 2026 10:12:41 +0000 Subject: [PATCH 4/5] MILAB-6771: run recovery fixtures through the shell software A Kubernetes backend starts every command from an image, so the plain /usr/bin/env commands failed with "docker is not set". The fixtures now use the busybox shell from software-test-utils, as the other exec fixtures do. The timestamp must have nanosecond precision, and the tests take the long timeout since each render waits for two pods on Kubernetes. --- .../src/exec/error-recovery.test.ts | 25 +++++++++++++------ .../src/exec/run/single_exec_ok.tpl.tengo | 7 ++++-- .../exec/run/two_execs_second_fails.tpl.tengo | 10 +++++--- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/tests/workflow-tengo/src/exec/error-recovery.test.ts b/tests/workflow-tengo/src/exec/error-recovery.test.ts index b425ad85fc..2de74e0495 100644 --- a/tests/workflow-tengo/src/exec/error-recovery.test.ts +++ b/tests/workflow-tengo/src/exec/error-recovery.test.ts @@ -3,6 +3,7 @@ import type { TestRenderResults, TplTestHelpers } from "@platforma-sdk/test"; import { tplTest } from "@platforma-sdk/test"; import type { ExpectStatic } from "vitest"; import { randomUUID } from "node:crypto"; +import { getLongTestTimeout } from "@milaboratories/test-helpers"; /** * A failed computation must keep its inputs, so a retry re-runs only the step that @@ -18,7 +19,9 @@ import { randomUUID } from "node:crypto"; * its own, and the timestamps would match without the backend doing anything. */ -const TIME_PATTERN = /stage1-time=(\d+)/; +// A nanosecond epoch has 19 digits. Fewer means `date` ignored %N and printed whole +// seconds, which a fast retry could repeat without recovering anything. +const TIME_PATTERN = /stage1-time=(\d{19,})\b/; async function renderAndAwaitError( helper: TplTestHelpers, @@ -44,7 +47,7 @@ function upstreamTime(expect: ExpectStatic, err: unknown, errorFragment: string) const message = err instanceof Error ? err.message : ""; expect(message).toContain(errorFragment); const match = TIME_PATTERN.exec(message); - expect(match, `no stage1-time in the error message:\n${message}`).not.toBeNull(); + expect(match, `no nanosecond stage1-time in the error message:\n${message}`).not.toBeNull(); return match?.[1] ?? ""; } @@ -79,14 +82,19 @@ tplTest( errorFragment: "stage2 died on purpose", }); }, + getLongTestTimeout(60_000), ); -tplTest("nested templates: exec fails in the deepest one", async ({ helper, expect }) => { - await runRecoveryScenario(helper, expect, { - template: "exec.run.nested_outer_exec_fails", - errorFragment: "stage2 died on purpose", - }); -}); +tplTest( + "nested templates: exec fails in the deepest one", + async ({ helper, expect }) => { + await runRecoveryScenario(helper, expect, { + template: "exec.run.nested_outer_exec_fails", + errorFragment: "stage2 died on purpose", + }); + }, + getLongTestTimeout(60_000), +); tplTest( "nested templates: exec succeeds and an intermediate template throws", @@ -96,4 +104,5 @@ tplTest( errorFragment: "intermediate template failed on purpose", }); }, + getLongTestTimeout(60_000), ); diff --git a/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo index e94d83caa1..b7df82bc7c 100644 --- a/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo @@ -7,16 +7,19 @@ self := import("@platforma-sdk/workflow-tengo:tpl") exec := import("@platforma-sdk/workflow-tengo:exec") +assets := import("@platforma-sdk/workflow-tengo:assets") self.defineOutputs(["main"]) +// busybox `sh` in an image, so the commands also run on a Kubernetes backend. +shellSw := assets.importSoftware("@platforma-open/milaboratories.software-test-utils:shell") + self.body(func(inputs) { run := exec.builder(). name("recovery-okstage"). cpu(1).ram("50Mi"). inLightQueue(). - cmd("/usr/bin/env"). - arg("bash"). + software(shellSw). arg("-c"). arg("date +%s%N > out.txt"). arg("okstage"). diff --git a/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo index d0938805eb..1053c7ed6d 100644 --- a/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo @@ -15,16 +15,19 @@ self := import("@platforma-sdk/workflow-tengo:tpl") exec := import("@platforma-sdk/workflow-tengo:exec") +assets := import("@platforma-sdk/workflow-tengo:assets") self.defineOutputs(["main"]) +// busybox `sh` in an image, so the commands also run on a Kubernetes backend. +shellSw := assets.importSoftware("@platforma-open/milaboratories.software-test-utils:shell") + self.body(func(inputs) { stage1 := exec.builder(). name("recovery-stage1"). cpu(1).ram("50Mi"). inLightQueue(). - cmd("/usr/bin/env"). - arg("bash"). + software(shellSw). arg("-c"). arg("date +%s%N > mid.txt"). arg("stage1"). @@ -36,8 +39,7 @@ self.body(func(inputs) { name("recovery-stage2"). cpu(1).ram("50Mi"). inLightQueue(). - cmd("/usr/bin/env"). - arg("bash"). + software(shellSw). arg("-c"). arg("echo \"stage1-time=$(cat mid.txt)\" >&2; echo 'stage2 died on purpose' >&2; exit 42"). addFile("mid.txt", stage1.getFile("mid.txt")). From a11c62a443b260ca72f15dcce9b9c9eca2541768 Mon Sep 17 00:00:00 2001 From: pvyazankin Date: Fri, 13 Nov 2026 10:55:43 +0000 Subject: [PATCH 5/5] MILAB-6771: mark the upstream run with time plus random bytes busybox date has no %N, so on Kubernetes the timestamp came out as whole seconds followed by a literal %N. The upstream command now writes the time in seconds plus 8 random bytes from /dev/urandom, which no two executions share and which busybox, Linux and macOS shells all produce. --- .../src/exec/error-recovery.test.ts | 32 +++++++++---------- .../src/exec/run/mid_thrower.tpl.tengo | 8 ++--- .../src/exec/run/single_exec_ok.tpl.tengo | 5 +-- .../exec/run/two_execs_second_fails.tpl.tengo | 16 ++++++---- 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/tests/workflow-tengo/src/exec/error-recovery.test.ts b/tests/workflow-tengo/src/exec/error-recovery.test.ts index 2de74e0495..30b851c518 100644 --- a/tests/workflow-tengo/src/exec/error-recovery.test.ts +++ b/tests/workflow-tengo/src/exec/error-recovery.test.ts @@ -9,19 +9,19 @@ import { getLongTestTimeout } from "@milaboratories/test-helpers"; * A failed computation must keep its inputs, so a retry re-runs only the step that * failed. * - * In every scenario the upstream command succeeds and writes the current time in - * nanoseconds; a later step fails and carries that timestamp in its error message. - * The template is rendered twice with the same seed. If the retry recovered the - * upstream result, both errors carry the same timestamp; if it re-ran the command, - * the timestamps differ. + * In every scenario the upstream command succeeds and writes a run marker: the current + * time in seconds plus 8 random bytes, unique to that execution. A later step fails and + * carries the marker in its error message. The template is rendered twice with the + * same seed. If the retry recovered the upstream result, both errors carry the same + * marker; if it re-ran the command, the markers differ. * * The upstream result is never a template output: an output would hold it from GC on - * its own, and the timestamps would match without the backend doing anything. + * its own, and the markers would match without the backend doing anything. */ -// A nanosecond epoch has 19 digits. Fewer means `date` ignored %N and printed whole -// seconds, which a fast retry could repeat without recovering anything. -const TIME_PATTERN = /stage1-time=(\d{19,})\b/; +// Epoch seconds, a dash, 16 hex digits of randomness. The random part is what makes a +// re-run distinguishable: seconds alone could repeat on a fast retry. +const MARKER_PATTERN = /stage1-run=(\d+-[0-9a-f]{16})\b/; async function renderAndAwaitError( helper: TplTestHelpers, @@ -42,18 +42,18 @@ async function renderAndAwaitError( .catch((e: unknown) => e); } -function upstreamTime(expect: ExpectStatic, err: unknown, errorFragment: string): string { +function upstreamMarker(expect: ExpectStatic, err: unknown, errorFragment: string): string { expect(err).toBeInstanceOf(Error); const message = err instanceof Error ? err.message : ""; expect(message).toContain(errorFragment); - const match = TIME_PATTERN.exec(message); - expect(match, `no nanosecond stage1-time in the error message:\n${message}`).not.toBeNull(); + const match = MARKER_PATTERN.exec(message); + expect(match, `no stage1-run marker in the error message:\n${message}`).not.toBeNull(); return match?.[1] ?? ""; } /** * One scenario: render `template` twice with the same seed, expect the same error both - * times, and require the upstream timestamp to be identical. + * times, and require the upstream run marker to be identical. * * The seed is random per scenario run, so neither scenarios sharing an upstream command * nor a rerun against the same backend can recover a result left by an earlier run and @@ -66,12 +66,12 @@ async function runRecoveryScenario( ): Promise { const seed = randomUUID(); const firstErr = await renderAndAwaitError(helper, opts.template, seed); - const firstTime = upstreamTime(expect, firstErr, opts.errorFragment); + const firstMarker = upstreamMarker(expect, firstErr, opts.errorFragment); const secondErr = await renderAndAwaitError(helper, opts.template, seed); - const secondTime = upstreamTime(expect, secondErr, opts.errorFragment); + const secondMarker = upstreamMarker(expect, secondErr, opts.errorFragment); - expect(secondTime, "the retry re-ran the upstream command").toBe(firstTime); + expect(secondMarker, "the retry re-ran the upstream command").toBe(firstMarker); } tplTest( diff --git a/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo b/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo index 4fa967bfe3..af78b850a4 100644 --- a/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo @@ -1,10 +1,10 @@ /** * an intermediate template that fails on purpose. * - * It takes the upstream exec's timestamp as an input, so by the time this body runs + * It takes the upstream exec's run marker as an input, so by the time this body runs * the command has already succeeded. The failure therefore comes from template code, * not from a command, which is the second shape the recovery hold has to survive. - * The timestamp goes into the error message so the test can compare it across renders. + * The marker goes into the error message so the test can compare it across renders. */ self := import("@platforma-sdk/workflow-tengo:tpl") @@ -14,8 +14,8 @@ text := import("text") self.defineOutputs(["main"]) self.body(func(inputs) { - time := text.trim_space(string(inputs.value.getData())) - ll.panic("intermediate template failed on purpose; stage1-time=%s", time) + marker := text.trim_space(string(inputs.value.getData())) + ll.panic("intermediate template failed on purpose; stage1-run=%s", marker) return { main: inputs.value diff --git a/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo index b7df82bc7c..ed88b422b3 100644 --- a/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo @@ -1,5 +1,6 @@ /** - * one exec that succeeds and returns the current time in nanoseconds. Used as the + * one exec that succeeds and returns a run marker: the current time in seconds plus 8 + * random bytes, so no two executions return the same marker. Used as the * upstream step in the case where an intermediate template, not the command, is what * fails. Its only input is the test's `seed`, so its CID is stable across renders * with the same seed. @@ -21,7 +22,7 @@ self.body(func(inputs) { inLightQueue(). software(shellSw). arg("-c"). - arg("date +%s%N > out.txt"). + arg("echo \"$(date +%s)-$(od -An -N8 -tx1 /dev/urandom | tr -d ' \\n')\" > out.txt"). arg("okstage"). arg(inputs.seed). saveFileContent("out.txt"). diff --git a/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo index 1053c7ed6d..81498d91f7 100644 --- a/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo +++ b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo @@ -1,12 +1,14 @@ /** * two chained execs where the second one fails. * - * Stage 1 writes the current time in nanoseconds to a file. Its only input is the - * test's `seed`, so its CID is stable across renders with the same seed: a retry that - * recovers it gets the same timestamp back, a retry that re-runs it gets a new one. + * Stage 1 writes a run marker to a file: the current time in seconds plus 8 random + * bytes, so no two executions write the same marker (busybox `date` has no %N for + * nanoseconds). Its only input is the test's `seed`, so its CID is stable across + * renders with the same seed: a retry that recovers it gets the same marker back, a + * retry that re-runs it gets a new one. * - * Stage 2 prints that timestamp to stderr and exits 42. stderr is part of the error - * message, which is how the test reads the timestamp. + * Stage 2 prints that marker to stderr and exits 42. stderr is part of the error + * message, which is how the test reads the marker. * * Stage 1's result is deliberately NOT exposed as a template output. An output field * would hold it from GC on its own and the test would pass without the feature under @@ -29,7 +31,7 @@ self.body(func(inputs) { inLightQueue(). software(shellSw). arg("-c"). - arg("date +%s%N > mid.txt"). + arg("echo \"$(date +%s)-$(od -An -N8 -tx1 /dev/urandom | tr -d ' \\n')\" > mid.txt"). arg("stage1"). arg(inputs.seed). saveFile("mid.txt"). @@ -41,7 +43,7 @@ self.body(func(inputs) { inLightQueue(). software(shellSw). arg("-c"). - arg("echo \"stage1-time=$(cat mid.txt)\" >&2; echo 'stage2 died on purpose' >&2; exit 42"). + arg("echo \"stage1-run=$(cat mid.txt)\" >&2; echo 'stage2 died on purpose' >&2; exit 42"). addFile("mid.txt", stage1.getFile("mid.txt")). saveStdoutContent(). run()