diff --git a/tests/workflow-tengo/src/exec/error-recovery.test.ts b/tests/workflow-tengo/src/exec/error-recovery.test.ts new file mode 100644 index 0000000000..30b851c518 --- /dev/null +++ b/tests/workflow-tengo/src/exec/error-recovery.test.ts @@ -0,0 +1,108 @@ +import { Pl } from "@milaboratories/pl-middle-layer"; +import type { TestRenderResults, TplTestHelpers } from "@platforma-sdk/test"; +import { tplTest } from "@platforma-sdk/test"; +import type { ExpectStatic } from "vitest"; +import { randomUUID } from "node:crypto"; +import { getLongTestTimeout } from "@milaboratories/test-helpers"; + +/** + * A failed computation must keep its inputs, so a retry re-runs only the step that + * failed. + * + * In every scenario the upstream command succeeds and writes a run marker: the current + * time in seconds plus 8 random bytes, unique to that execution. A later step fails and + * carries the marker in its error message. The template is rendered twice with the + * same seed. If the retry recovered the upstream result, both errors carry the same + * marker; if it re-ran the command, the markers differ. + * + * The upstream result is never a template output: an output would hold it from GC on + * its own, and the markers would match without the backend doing anything. + */ + +// Epoch seconds, a dash, 16 hex digits of randomness. The random part is what makes a +// re-run distinguishable: seconds alone could repeat on a fast retry. +const MARKER_PATTERN = /stage1-run=(\d+-[0-9a-f]{16})\b/; + +async function renderAndAwaitError( + helper: TplTestHelpers, + template: string, + seed: string, +): Promise { + const result: TestRenderResults<"main"> = await helper.renderTemplate( + false, + template, + ["main"], + (tx) => ({ + seed: tx.createValue(Pl.JsonObject, JSON.stringify(seed)), + }), + ); + return await result + .computeOutput("main", (a) => a?.getDataAsString()) + .awaitStableValue() + .catch((e: unknown) => e); +} + +function upstreamMarker(expect: ExpectStatic, err: unknown, errorFragment: string): string { + expect(err).toBeInstanceOf(Error); + const message = err instanceof Error ? err.message : ""; + expect(message).toContain(errorFragment); + const match = MARKER_PATTERN.exec(message); + expect(match, `no stage1-run marker in the error message:\n${message}`).not.toBeNull(); + return match?.[1] ?? ""; +} + +/** + * One scenario: render `template` twice with the same seed, expect the same error both + * times, and require the upstream run marker to be identical. + * + * The seed is random per scenario run, so neither scenarios sharing an upstream command + * nor a rerun against the same backend can recover a result left by an earlier run and + * pass without the first render having executed the command. + */ +async function runRecoveryScenario( + helper: TplTestHelpers, + expect: ExpectStatic, + opts: { template: string; errorFragment: string }, +): Promise { + const seed = randomUUID(); + const firstErr = await renderAndAwaitError(helper, opts.template, seed); + const firstMarker = upstreamMarker(expect, firstErr, opts.errorFragment); + + const secondErr = await renderAndAwaitError(helper, opts.template, seed); + const secondMarker = upstreamMarker(expect, secondErr, opts.errorFragment); + + expect(secondMarker, "the retry re-ran the upstream command").toBe(firstMarker); +} + +tplTest( + "failing exec keeps the upstream result, so a retry does not re-run it", + async ({ helper, expect }) => { + await runRecoveryScenario(helper, expect, { + template: "exec.run.two_execs_second_fails", + errorFragment: "stage2 died on purpose", + }); + }, + getLongTestTimeout(60_000), +); + +tplTest( + "nested templates: exec fails in the deepest one", + async ({ helper, expect }) => { + await runRecoveryScenario(helper, expect, { + template: "exec.run.nested_outer_exec_fails", + errorFragment: "stage2 died on purpose", + }); + }, + getLongTestTimeout(60_000), +); + +tplTest( + "nested templates: exec succeeds and an intermediate template throws", + async ({ helper, expect }) => { + await runRecoveryScenario(helper, expect, { + template: "exec.run.nested_outer_mid_throws", + errorFragment: "intermediate template failed on purpose", + }); + }, + getLongTestTimeout(60_000), +); diff --git a/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo b/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo new file mode 100644 index 0000000000..af78b850a4 --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo @@ -0,0 +1,23 @@ +/** + * an intermediate template that fails on purpose. + * + * It takes the upstream exec's run marker as an input, so by the time this body runs + * the command has already succeeded. The failure therefore comes from template code, + * not from a command, which is the second shape the recovery hold has to survive. + * The marker goes into the error message so the test can compare it across renders. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +ll := import("@platforma-sdk/workflow-tengo:ll") +text := import("text") + +self.defineOutputs(["main"]) + +self.body(func(inputs) { + marker := text.trim_space(string(inputs.value.getData())) + ll.panic("intermediate template failed on purpose; stage1-run=%s", marker) + + return { + main: inputs.value + } +}) diff --git a/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo new file mode 100644 index 0000000000..b7db6d647c --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo @@ -0,0 +1,22 @@ +/** + * middle layer of a nested chain. Renders the failing two-exec + * chain and forwards its output, so the error has to cross a template boundary. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +assets := import("@platforma-sdk/workflow-tengo:assets") +render := import("@platforma-sdk/workflow-tengo:render") + +innerTpl := assets.importTemplate(":exec.run.two_execs_second_fails") + +self.defineOutputs(["main"]) + +self.body(func(inputs) { + run := render.create(innerTpl, { + seed: inputs.seed + }) + + return { + main: run.output("main") + } +}) diff --git a/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo new file mode 100644 index 0000000000..524dabc956 --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/nested_outer_exec_fails.tpl.tengo @@ -0,0 +1,22 @@ +/** + * outer layer of a nested chain whose deepest template runs the + * failing exec. Two template boundaries sit between the render root and the failure. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +assets := import("@platforma-sdk/workflow-tengo:assets") +render := import("@platforma-sdk/workflow-tengo:render") + +midTpl := assets.importTemplate(":exec.run.nested_mid") + +self.defineOutputs(["main"]) + +self.body(func(inputs) { + run := render.create(midTpl, { + seed: inputs.seed + }) + + return { + main: run.output("main") + } +}) diff --git a/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo b/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo new file mode 100644 index 0000000000..8e294cf603 --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/nested_outer_mid_throws.tpl.tengo @@ -0,0 +1,29 @@ +/** + * exec succeeds, then an intermediate template throws. + * + * The thrower consumes the exec's output, so the command runs before the failure. A + * retry must not re-run it. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +assets := import("@platforma-sdk/workflow-tengo:assets") +render := import("@platforma-sdk/workflow-tengo:render") + +execTpl := assets.importTemplate(":exec.run.single_exec_ok") +throwerTpl := assets.importTemplate(":exec.run.mid_thrower") + +self.defineOutputs(["main"]) + +self.body(func(inputs) { + execRun := render.create(execTpl, { + seed: inputs.seed + }) + + thrower := render.create(throwerTpl, { + value: execRun.output("main") + }) + + return { + main: thrower.output("main") + } +}) diff --git a/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo new file mode 100644 index 0000000000..ed88b422b3 --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo @@ -0,0 +1,34 @@ +/** + * one exec that succeeds and returns a run marker: the current time in seconds plus 8 + * random bytes, so no two executions return the same marker. Used as the + * upstream step in the case where an intermediate template, not the command, is what + * fails. Its only input is the test's `seed`, so its CID is stable across renders + * with the same seed. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +exec := import("@platforma-sdk/workflow-tengo:exec") +assets := import("@platforma-sdk/workflow-tengo:assets") + +self.defineOutputs(["main"]) + +// busybox `sh` in an image, so the commands also run on a Kubernetes backend. +shellSw := assets.importSoftware("@platforma-open/milaboratories.software-test-utils:shell") + +self.body(func(inputs) { + run := exec.builder(). + name("recovery-okstage"). + cpu(1).ram("50Mi"). + inLightQueue(). + software(shellSw). + arg("-c"). + arg("echo \"$(date +%s)-$(od -An -N8 -tx1 /dev/urandom | tr -d ' \\n')\" > out.txt"). + arg("okstage"). + arg(inputs.seed). + saveFileContent("out.txt"). + run() + + return { + main: run.getFileContent("out.txt") + } +}) diff --git a/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo new file mode 100644 index 0000000000..81498d91f7 --- /dev/null +++ b/tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo @@ -0,0 +1,54 @@ +/** + * two chained execs where the second one fails. + * + * Stage 1 writes a run marker to a file: the current time in seconds plus 8 random + * bytes, so no two executions write the same marker (busybox `date` has no %N for + * nanoseconds). Its only input is the test's `seed`, so its CID is stable across + * renders with the same seed: a retry that recovers it gets the same marker back, a + * retry that re-runs it gets a new one. + * + * Stage 2 prints that marker to stderr and exits 42. stderr is part of the error + * message, which is how the test reads the marker. + * + * Stage 1's result is deliberately NOT exposed as a template output. An output field + * would hold it from GC on its own and the test would pass without the feature under + * test doing anything. + */ + +self := import("@platforma-sdk/workflow-tengo:tpl") +exec := import("@platforma-sdk/workflow-tengo:exec") +assets := import("@platforma-sdk/workflow-tengo:assets") + +self.defineOutputs(["main"]) + +// busybox `sh` in an image, so the commands also run on a Kubernetes backend. +shellSw := assets.importSoftware("@platforma-open/milaboratories.software-test-utils:shell") + +self.body(func(inputs) { + stage1 := exec.builder(). + name("recovery-stage1"). + cpu(1).ram("50Mi"). + inLightQueue(). + software(shellSw). + arg("-c"). + arg("echo \"$(date +%s)-$(od -An -N8 -tx1 /dev/urandom | tr -d ' \\n')\" > mid.txt"). + arg("stage1"). + arg(inputs.seed). + saveFile("mid.txt"). + run() + + stage2 := exec.builder(). + name("recovery-stage2"). + cpu(1).ram("50Mi"). + inLightQueue(). + software(shellSw). + arg("-c"). + arg("echo \"stage1-run=$(cat mid.txt)\" >&2; echo 'stage2 died on purpose' >&2; exit 42"). + addFile("mid.txt", stage1.getFile("mid.txt")). + saveStdoutContent(). + run() + + return { + main: stage2.getStdoutContent() + } +})