Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 108 additions & 0 deletions tests/workflow-tengo/src/exec/error-recovery.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import { Pl } from "@milaboratories/pl-middle-layer";
import type { TestRenderResults, TplTestHelpers } from "@platforma-sdk/test";
import { tplTest } from "@platforma-sdk/test";
import type { ExpectStatic } from "vitest";
import { randomUUID } from "node:crypto";
import { getLongTestTimeout } from "@milaboratories/test-helpers";

/**
* A failed computation must keep its inputs, so a retry re-runs only the step that
* failed.
*
* In every scenario the upstream command succeeds and writes a run marker: the current
* time in seconds plus 8 random bytes, unique to that execution. A later step fails and
* carries the marker in its error message. The template is rendered twice with the
* same seed. If the retry recovered the upstream result, both errors carry the same
* marker; if it re-ran the command, the markers differ.
*
* The upstream result is never a template output: an output would hold it from GC on
* its own, and the markers would match without the backend doing anything.
*/

// Epoch seconds, a dash, 16 hex digits of randomness. The random part is what makes a
// re-run distinguishable: seconds alone could repeat on a fast retry.
const MARKER_PATTERN = /stage1-run=(\d+-[0-9a-f]{16})\b/;

async function renderAndAwaitError(
helper: TplTestHelpers,
template: string,
seed: string,
): Promise<unknown> {
const result: TestRenderResults<"main"> = await helper.renderTemplate(
false,
template,
["main"],
(tx) => ({
seed: tx.createValue(Pl.JsonObject, JSON.stringify(seed)),
}),
);
return await result
.computeOutput("main", (a) => a?.getDataAsString())
.awaitStableValue()
.catch((e: unknown) => e);
}

function upstreamMarker(expect: ExpectStatic, err: unknown, errorFragment: string): string {
expect(err).toBeInstanceOf(Error);
const message = err instanceof Error ? err.message : "";
expect(message).toContain(errorFragment);
const match = MARKER_PATTERN.exec(message);
expect(match, `no stage1-run marker in the error message:\n${message}`).not.toBeNull();
return match?.[1] ?? "";
}

/**
* One scenario: render `template` twice with the same seed, expect the same error both
* times, and require the upstream run marker to be identical.
*
* The seed is random per scenario run, so neither scenarios sharing an upstream command
* nor a rerun against the same backend can recover a result left by an earlier run and
* pass without the first render having executed the command.
*/
async function runRecoveryScenario(
helper: TplTestHelpers,
expect: ExpectStatic,
opts: { template: string; errorFragment: string },
): Promise<void> {
const seed = randomUUID();
const firstErr = await renderAndAwaitError(helper, opts.template, seed);
const firstMarker = upstreamMarker(expect, firstErr, opts.errorFragment);

const secondErr = await renderAndAwaitError(helper, opts.template, seed);
const secondMarker = upstreamMarker(expect, secondErr, opts.errorFragment);

expect(secondMarker, "the retry re-ran the upstream command").toBe(firstMarker);
}

tplTest(
"failing exec keeps the upstream result, so a retry does not re-run it",
async ({ helper, expect }) => {
await runRecoveryScenario(helper, expect, {
template: "exec.run.two_execs_second_fails",
errorFragment: "stage2 died on purpose",
});
},
getLongTestTimeout(60_000),
);

tplTest(
"nested templates: exec fails in the deepest one",
async ({ helper, expect }) => {
await runRecoveryScenario(helper, expect, {
template: "exec.run.nested_outer_exec_fails",
errorFragment: "stage2 died on purpose",
});
},
getLongTestTimeout(60_000),
);

tplTest(
"nested templates: exec succeeds and an intermediate template throws",
async ({ helper, expect }) => {
await runRecoveryScenario(helper, expect, {
template: "exec.run.nested_outer_mid_throws",
errorFragment: "intermediate template failed on purpose",
});
},
getLongTestTimeout(60_000),
);
23 changes: 23 additions & 0 deletions tests/workflow-tengo/src/exec/run/mid_thrower.tpl.tengo
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/**
* an intermediate template that fails on purpose.
*
* It takes the upstream exec's run marker as an input, so by the time this body runs
* the command has already succeeded. The failure therefore comes from template code,
* not from a command, which is the second shape the recovery hold has to survive.
* The marker goes into the error message so the test can compare it across renders.
*/

self := import("@platforma-sdk/workflow-tengo:tpl")
ll := import("@platforma-sdk/workflow-tengo:ll")
text := import("text")

self.defineOutputs(["main"])

self.body(func(inputs) {
marker := text.trim_space(string(inputs.value.getData()))
ll.panic("intermediate template failed on purpose; stage1-run=%s", marker)

return {
main: inputs.value
}
})
22 changes: 22 additions & 0 deletions tests/workflow-tengo/src/exec/run/nested_mid.tpl.tengo
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/**
* middle layer of a nested chain. Renders the failing two-exec
* chain and forwards its output, so the error has to cross a template boundary.
*/

self := import("@platforma-sdk/workflow-tengo:tpl")
assets := import("@platforma-sdk/workflow-tengo:assets")
render := import("@platforma-sdk/workflow-tengo:render")

innerTpl := assets.importTemplate(":exec.run.two_execs_second_fails")

self.defineOutputs(["main"])

self.body(func(inputs) {
run := render.create(innerTpl, {
seed: inputs.seed
})

return {
main: run.output("main")
}
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/**
* outer layer of a nested chain whose deepest template runs the
* failing exec. Two template boundaries sit between the render root and the failure.
*/

self := import("@platforma-sdk/workflow-tengo:tpl")
assets := import("@platforma-sdk/workflow-tengo:assets")
render := import("@platforma-sdk/workflow-tengo:render")

midTpl := assets.importTemplate(":exec.run.nested_mid")

self.defineOutputs(["main"])

self.body(func(inputs) {
run := render.create(midTpl, {
seed: inputs.seed
})

return {
main: run.output("main")
}
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/**
* exec succeeds, then an intermediate template throws.
*
* The thrower consumes the exec's output, so the command runs before the failure. A
* retry must not re-run it.
*/

self := import("@platforma-sdk/workflow-tengo:tpl")
assets := import("@platforma-sdk/workflow-tengo:assets")
render := import("@platforma-sdk/workflow-tengo:render")

execTpl := assets.importTemplate(":exec.run.single_exec_ok")
throwerTpl := assets.importTemplate(":exec.run.mid_thrower")

self.defineOutputs(["main"])

self.body(func(inputs) {
execRun := render.create(execTpl, {
seed: inputs.seed
})

thrower := render.create(throwerTpl, {
value: execRun.output("main")
})

return {
main: thrower.output("main")
}
})
34 changes: 34 additions & 0 deletions tests/workflow-tengo/src/exec/run/single_exec_ok.tpl.tengo
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/**
* one exec that succeeds and returns a run marker: the current time in seconds plus 8
* random bytes, so no two executions return the same marker. Used as the
* upstream step in the case where an intermediate template, not the command, is what
* fails. Its only input is the test's `seed`, so its CID is stable across renders
* with the same seed.
*/

self := import("@platforma-sdk/workflow-tengo:tpl")
exec := import("@platforma-sdk/workflow-tengo:exec")
assets := import("@platforma-sdk/workflow-tengo:assets")

self.defineOutputs(["main"])

// busybox `sh` in an image, so the commands also run on a Kubernetes backend.
shellSw := assets.importSoftware("@platforma-open/milaboratories.software-test-utils:shell")

self.body(func(inputs) {
run := exec.builder().
name("recovery-okstage").
cpu(1).ram("50Mi").
inLightQueue().
software(shellSw).
arg("-c").
arg("echo \"$(date +%s)-$(od -An -N8 -tx1 /dev/urandom | tr -d ' \\n')\" > out.txt").
arg("okstage").
arg(inputs.seed).
saveFileContent("out.txt").
run()

return {
main: run.getFileContent("out.txt")
}
})
54 changes: 54 additions & 0 deletions tests/workflow-tengo/src/exec/run/two_execs_second_fails.tpl.tengo
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/**
* two chained execs where the second one fails.
*
* Stage 1 writes a run marker to a file: the current time in seconds plus 8 random
* bytes, so no two executions write the same marker (busybox `date` has no %N for
* nanoseconds). Its only input is the test's `seed`, so its CID is stable across
* renders with the same seed: a retry that recovers it gets the same marker back, a
* retry that re-runs it gets a new one.
*
* Stage 2 prints that marker to stderr and exits 42. stderr is part of the error
* message, which is how the test reads the marker.
*
* Stage 1's result is deliberately NOT exposed as a template output. An output field
* would hold it from GC on its own and the test would pass without the feature under
* test doing anything.
*/

self := import("@platforma-sdk/workflow-tengo:tpl")
exec := import("@platforma-sdk/workflow-tengo:exec")
assets := import("@platforma-sdk/workflow-tengo:assets")

self.defineOutputs(["main"])

// busybox `sh` in an image, so the commands also run on a Kubernetes backend.
shellSw := assets.importSoftware("@platforma-open/milaboratories.software-test-utils:shell")

self.body(func(inputs) {
stage1 := exec.builder().
name("recovery-stage1").
cpu(1).ram("50Mi").
inLightQueue().
software(shellSw).
arg("-c").
arg("echo \"$(date +%s)-$(od -An -N8 -tx1 /dev/urandom | tr -d ' \\n')\" > mid.txt").
arg("stage1").
arg(inputs.seed).
saveFile("mid.txt").
run()

stage2 := exec.builder().
name("recovery-stage2").
cpu(1).ram("50Mi").
inLightQueue().
software(shellSw).
arg("-c").
arg("echo \"stage1-run=$(cat mid.txt)\" >&2; echo 'stage2 died on purpose' >&2; exit 42").
addFile("mid.txt", stage1.getFile("mid.txt")).
saveStdoutContent().
run()

return {
main: stage2.getStdoutContent()
}
})
Loading