From d3052ea4d7395909127c45b0510ae2c16dbf6c99 Mon Sep 17 00:00:00 2001 From: Vitalii Popov Date: Mon, 16 Nov 2026 17:23:47 +0100 Subject: [PATCH] fix(pl-client): an EphStdMap is never final in the tree A ready EphStdMap was final, so the tree stopped re-reading it. A map is ready once its entries resolve to resources, not once those resources settle. An entry that fails later puts its error on the map after ready, and the tree never saw it. The v2 effect hit this: on the k8s backend the import of a missing library file failed after the effect map was ready, and "a failed import fails the effect" read no error for minutes. The map is now never final. The delta benchmark gets an arm of ephemeral maps that compares the old row with the new one. Co-Authored-By: Claude Opus 5.5 --- .changeset/eph-std-map-never-final.md | 5 + lib/node/pl-client/src/core/final.ts | 32 ++-- lib/node/pl-client/src/core/finality.test.ts | 2 +- .../middle_layer/finality_stop_rules.test.ts | 4 - lib/node/pl-tree/src/delta_benchmark.test.ts | 161 +++++++++++++----- .../pl-tree/src/synchronized_tree.test.ts | 76 +++++++++ 6 files changed, 224 insertions(+), 56 deletions(-) create mode 100644 .changeset/eph-std-map-never-final.md diff --git a/.changeset/eph-std-map-never-final.md b/.changeset/eph-std-map-never-final.md new file mode 100644 index 0000000000..bd21971e10 --- /dev/null +++ b/.changeset/eph-std-map-never-final.md @@ -0,0 +1,5 @@ +--- +"@milaboratories/pl-client": patch +--- + +A tree no longer holds a ready `EphStdMap` final. An error that reaches the map after it is ready, such as the failed import of a v2 effect, now gets into the tree. diff --git a/lib/node/pl-client/src/core/final.ts b/lib/node/pl-client/src/core/final.ts index f3490ff1b3..41d79038e6 100644 --- a/lib/node/pl-client/src/core/final.ts +++ b/lib/node/pl-client/src/core/final.ts @@ -6,11 +6,15 @@ export { ResourceTypeName, ResourceTypePrefix }; export type { FinalResourceDataPredicate } from "./finality"; export { readyOrDuplicateOrError } from "./finality"; -// The finality tables, one layer per consumer. Every layer accepts two backend writes after +// The finality tables, one layer per consumer. Every layer accepts three backend writes after // final: a data-loss error cascade, lifting the error of a stored blob found lost or corrupt to -// the resources built from it, and the first duplicate's `hasOriginalListeners` flag on its -// original, backend-internal and never on the wire. An entry's `why` names any other write -// it accepts. A layer adds only the types whose later writes its consumer does not observe. +// the resources built from it; a late error, lifted by the core from an entry that fails after +// the resource above it is ready, because ready asks only that every input points to a resource +// with a final status, and work below that resource can still fail; and the first duplicate's +// `hasOriginalListeners` flag on its original, backend-internal and never on the wire. A reader +// of a final resource misses the first two. An entry's `why` names any other write it accepts, +// and a type that must show a late error gets a `never` entry. A layer adds only the types whose +// later writes its consumer does not observe. // Where the backend's filter stops earlier than the predicate. An early stop is safe: the // backend sends a body-less stop marker, and the streaming loader fetches every stopped resource @@ -36,6 +40,9 @@ const value = (name: string): FinalityEntry => ({ why: "a value, written once at creation", }); +const noFillAfterReady = + "no controller fills it after ready; a late error from an entry below it is not seen"; + const settledAtReady = (name: string, why: string): FinalityEntry => ({ match: { name }, rule: "readyOrDuplicateOrError", @@ -50,8 +57,8 @@ const never = (match: FinalityEntry["match"], why: string): FinalityEntry => ({ }); /** The base layer: nothing observable (state, fields, KV) changes after the rule holds, whatever - * the consumer prunes. Exceptions: the data-loss error cascade, the `hasOriginalListeners` flag, - * and the writes its entries name. */ + * the consumer prunes. Exceptions: the data-loss error cascade, the late error, the + * `hasOriginalListeners` flag, and the writes its entries name. */ export const StrictFinality: FinalityTable = FinalityTable.of("strict", [ value(ResourceTypeName.JsonObject), value(ResourceTypeName.JsonGzObject), @@ -92,7 +99,6 @@ export const StrictFinality: FinalityTable = FinalityTable.of("strict", [ ...[ ResourceTypeName.StdMap, ResourceTypeName.StdMapSlash, - ResourceTypeName.EphStdMap, ResourceTypeName.PFrame, ResourceTypeName.ParquetChunk, ResourceTypeName.BContext, @@ -102,9 +108,7 @@ export const StrictFinality: FinalityTable = FinalityTable.of("strict", [ ResourceTypeName.BResolveSingleNoResult, ResourceTypeName.BQueryResult, ResourceTypeName.TengoLib, - ].map((name) => - settledAtReady(name, "its inputs are final at ready, and no controller fills it afterwards"), - ), + ].map((name) => settledAtReady(name, noFillAfterReady)), ...[ ResourceTypeName.TengoTemplate, ResourceTypeName.SoftwareInfo, @@ -122,7 +126,7 @@ export const StrictFinality: FinalityTable = FinalityTable.of("strict", [ { match: { prefix: ResourceTypePrefix.PColumnData }, rule: "readyOrDuplicateOrError", - why: "its inputs are final at ready, and no controller fills it afterwards", + why: noFillAfterReady, stopRule: earlyOnFieldErrors, }, { @@ -155,6 +159,12 @@ export const StrictFinality: FinalityTable = FinalityTable.of("strict", [ "as resolution finishes", stopRule: earlyOnFieldErrorsWithOutputs, }, + never( + { name: ResourceTypeName.EphStdMap }, + "it is ready once its entries resolve to resources, not once those resources settle; an " + + "entry that fails later, such as a v2 effect's failed import, puts its error on the map " + + "after ready", + ), never({ name: ResourceTypeName.UserProject }, "blocks and fields come and go over its life"), never({ name: ResourceTypeName.Projects }, "projects are added and removed"), never({ name: ResourceTypeName.ClientRoot }, "its fields are added and removed"), diff --git a/lib/node/pl-client/src/core/finality.test.ts b/lib/node/pl-client/src/core/finality.test.ts index 557635e288..15f09e4cda 100644 --- a/lib/node/pl-client/src/core/finality.test.ts +++ b/lib/node/pl-client/src/core/finality.test.ts @@ -131,7 +131,6 @@ describe("the built-in tables", () => { "Dummy", "StdMap", "std/map", - "EphStdMap", "PFrame", "ParquetChunk", "BContext", @@ -146,6 +145,7 @@ describe("the built-in tables", () => { "SoftwareInfo", ]); const neverFinal = new Set([ + "EphStdMap", "UserProject", "Projects", "ClientRoot", diff --git a/lib/node/pl-middle-layer/src/middle_layer/finality_stop_rules.test.ts b/lib/node/pl-middle-layer/src/middle_layer/finality_stop_rules.test.ts index 2444b0e299..09894f5bc5 100644 --- a/lib/node/pl-middle-layer/src/middle_layer/finality_stop_rules.test.ts +++ b/lib/node/pl-middle-layer/src/middle_layer/finality_stop_rules.test.ts @@ -32,10 +32,6 @@ function expectedStopRules(): Filter { treeFilter.resourceTypeEq(ResourceTypeName.StdMapSlash), treeFilter.readyOrDuplicateOrError(), ), - treeFilter.and( - treeFilter.resourceTypeEq(ResourceTypeName.EphStdMap), - treeFilter.readyOrDuplicateOrError(), - ), treeFilter.and( treeFilter.resourceTypeEq(ResourceTypeName.PFrame), treeFilter.readyOrDuplicateOrError(), diff --git a/lib/node/pl-tree/src/delta_benchmark.test.ts b/lib/node/pl-tree/src/delta_benchmark.test.ts index 75c0eb817f..0d6b3a5815 100644 --- a/lib/node/pl-tree/src/delta_benchmark.test.ts +++ b/lib/node/pl-tree/src/delta_benchmark.test.ts @@ -1,7 +1,13 @@ import { test } from "vitest"; -import { field, hasCapability, TestHelpers } from "@milaboratories/pl-client"; -import type { PlClient, PlTransaction, SignedResourceId } from "@milaboratories/pl-client"; -import { DefaultFinalResourceDataPredicate } from "@milaboratories/pl-client"; +import { field, hasCapability, Pl, TestHelpers, TreeFinality } from "@milaboratories/pl-client"; +import type { + FieldType, + FinalResourceDataPredicate, + PlClient, + PlTransaction, + ResourceRef, + SignedResourceId, +} from "@milaboratories/pl-client"; import { TestStructuralResourceType1 } from "./test_utils"; import { PlTreeState } from "./state"; import { constructTreeLoadingRequest, initialTreeLoadingStat, loadTreeState } from "./sync"; @@ -18,6 +24,10 @@ import type { ExtendedResourceData } from "./state"; * * PL_TREE_BENCH=1 pnpm exec vitest run src/delta_benchmark.test.ts * + * A second run seeds ephemeral maps of values in place of the structs, and crosses the arms with + * two finality tables: the tree's own, and the tree's own plus the row that held a ready + * EphStdMap final. The difference is the polling cost of a map that is never final. + * * Delta arms report as skipped on a backend without `treeChangedSince:v2`, rather than * silently measuring the fallback. */ @@ -44,59 +54,117 @@ const ARMS: Arm[] = [ { label: "backend-delta prune=off", mode: "backend-delta", pruning: false }, ]; +/** The finality table an arm runs under. */ +type Finality = { name: string; predicate: FinalResourceDataPredicate }; + +const TREE_FINALITY: Finality = { name: "tree", predicate: TreeFinality.predicate() }; + +const MAP_FINAL_AT_READY: Finality = { + name: "map-final-at-ready", + predicate: TreeFinality.extend("map final at ready", [ + { + match: { name: Pl.EphStdMap.name }, + rule: "readyOrDuplicateOrError", + why: "the row that held a ready EphStdMap final, for the cost comparison", + }, + ]).predicate(), +}; + /** Stands in for the real project pruning, without importing the middle layer. */ const benchPruning = (r: ExtendedResourceData): FieldData[] => r.fields.filter((f) => !f.name.startsWith("pruneMe")); +type SeededTree = { root: SignedResourceId; leaves: SignedResourceId[] }; + +/** What a child of the root is, what it holds, and which resources the polls write KV on. */ +type Shape = { + name: string; + child: (tx: PlTransaction) => ResourceRef; + childField: FieldType; + /** `key` is unique per leaf: equal values deduplicate into one resource. */ + leaf: (tx: PlTransaction, key: string) => ResourceRef; + /** Runs after the child gets its fields. */ + close: (tx: PlTransaction, child: ResourceRef) => void; + touchedName: string; + touched: (tree: SeededTree) => SignedResourceId[]; +}; + +const STRUCT_SHAPE: Shape = { + name: "structs", + child: (tx) => tx.createStruct(TestStructuralResourceType1, PAYLOAD), + childField: "Dynamic", + leaf: (tx) => tx.createStruct(TestStructuralResourceType1, PAYLOAD), + close: () => {}, + touchedName: "a leaf (quiet-parent shape)", + touched: (tree) => tree.leaves, +}; + +const MAP_SHAPE: Shape = { + name: "EphStdMaps of json values", + child: (tx) => tx.createEphemeral(Pl.EphStdMap), + childField: "Input", + leaf: (tx, key) => + tx.createValue(Pl.JsonObject, JSON.stringify({ key, payload: PAYLOAD.toString() })), + // A map is ready, and so final under the old row, only with its inputs locked. + close: (tx, child) => tx.lockInputs(child), + // A json value is final at creation, so a KV write on it is never read again. + touchedName: "the root", + touched: (tree) => [tree.root], +}; + async function seedTree( pl: PlClient, -): Promise<{ root: SignedResourceId; leaves: SignedResourceId[] }> { + shape: Shape, +): Promise<{ root: SignedResourceId; touched: SignedResourceId[] }> { return await pl.withWriteTx( "BenchSeed", async (tx) => { const root = tx.createStruct(TestStructuralResourceType1, PAYLOAD); - const rootField = field(tx.clientRoot, "benchRoot"); + const rootField = field(tx.clientRoot, `benchRoot-${shape.name}`); tx.createField(rootField, "Dynamic"); tx.setField(rootField, root); const leaves: Promise[] = []; for (let c = 0; c < CHILDREN; c++) { - const child = tx.createStruct(TestStructuralResourceType1, PAYLOAD); + const child = shape.child(tx); const cf = field(root, `child${c}`); tx.createField(cf, "Dynamic"); tx.setField(cf, child); // A field the pruning function removes, so prune=on and prune=off differ. - const pruned = tx.createStruct(TestStructuralResourceType1, PAYLOAD); + const pruned = shape.leaf(tx, `pruned${c}`); const pf = field(child, "pruneMe"); - tx.createField(pf, "Dynamic"); + tx.createField(pf, shape.childField); tx.setField(pf, pruned); for (let g = 0; g < GRANDCHILDREN; g++) { - const grand = tx.createStruct(TestStructuralResourceType1, PAYLOAD); + const grand = shape.leaf(tx, `leaf${c}.${g}`); const gf = field(child, `g${g}`); - tx.createField(gf, "Dynamic"); + tx.createField(gf, shape.childField); tx.setField(gf, grand); leaves.push(grand.globalId); } + shape.close(tx, child); } await tx.commit(); - return { root: await root.globalId, leaves: await Promise.all(leaves) }; + const tree = { root: await root.globalId, leaves: await Promise.all(leaves) }; + return { root: tree.root, touched: shape.touched(tree) }; }, { sync: true }, ); } -/** One mutation between polls: a KV write on a leaf. The parent is not rewritten, so this is - * the quiet-parent shape, which is the case frontier seeding exists to reach. */ -async function touchLeaf(pl: PlClient, leaf: SignedResourceId, arm: string, round: number) { +/** One mutation between polls: a KV write on a resource the shape names. On a leaf the parent is + * not rewritten, so this is the quiet-parent shape, which is the case frontier seeding exists to + * reach. */ +async function touch(pl: PlClient, target: SignedResourceId, arm: string, round: number) { await pl.withWriteTx( "BenchTouch", async (tx: PlTransaction) => { // Keyed by arm: the tree is shared and never reset, and state.ts compares KV values, // so a later arm rewriting the same key with the same bytes observes no change at all. - tx.setKValue(leaf, `bench-${arm}-${round}`, Buffer.from(`r${round}`)); + tx.setKValue(target, `bench-${arm}-${round}`, Buffer.from(`r${round}`)); await tx.commit(); }, { sync: true }, @@ -122,13 +190,15 @@ type Row = { async function runArm( pl: PlClient, arm: Arm, - seed: { root: SignedResourceId; leaves: SignedResourceId[] }, + seed: { root: SignedResourceId; touched: SignedResourceId[] }, + finality: Finality, + label: string, ): Promise { const caps = pl.serverInfo.capabilities ?? []; if (arm.mode === "backend-delta" && !hasCapability(caps, "treeChangedSince:v2")) return undefined; // Scalar, not an array: the constructor takes SignedResourceId | Set. - const state = new PlTreeState(seed.root, DefaultFinalResourceDataPredicate); + const state = new PlTreeState(seed.root, finality.predicate); const stat: TreeLoadingStat = initialTreeLoadingStat(); let token: Uint8Array | undefined; let changedAtColdLoad = 0; @@ -137,9 +207,8 @@ async function runArm( // purpose: it is where the arms are meant to look alike. for (let cycle = 0; cycle <= POLL_CYCLES; cycle++) { if (cycle > 0) { - const leaf = seed.leaves[cycle % seed.leaves.length]; - if (leaf !== undefined) - await touchLeaf(pl, leaf, arm.label.trim().replace(/\s+/g, "-"), cycle); + const target = seed.touched[cycle % seed.touched.length]; + if (target !== undefined) await touch(pl, target, label.trim().replace(/\s+/g, "-"), cycle); } const request = constructTreeLoadingRequest(state, { @@ -162,7 +231,7 @@ async function runArm( } return { - arm: arm.label, + arm: label, roundTrips: stat.roundTrips, resources: stat.retrievedResources, bytes: stat.retrievedResourceDataBytes + stat.retrievedKeyValueBytes, @@ -176,19 +245,20 @@ async function runArm( }; } -function report(rows: Row[], skipped: string[], failed: string[] = []) { +function report(shape: Shape, rows: Row[], skipped: string[], failed: string[]) { const pad = (s: string | number, n: number) => String(s).padStart(n); + const armWidth = Math.max("arm".length, ...rows.map((r) => r.arm.length)) + 1; const lines = [ "", "=== tree loading cost ===", - `tree: ${CHILDREN} children x ${GRANDCHILDREN} grandchildren, ${POLL_CYCLES} polls after load,`, - ` one KV write on a leaf between polls (quiet-parent shape)`, + `tree: ${CHILDREN} ${shape.name} x ${GRANDCHILDREN} leaves, ${POLL_CYCLES} polls after load,`, + ` one KV write on ${shape.touchedName} between polls`, "", - `${"arm".padEnd(28)} ${pad("trips", 6)} ${pad("res", 6)} ${pad("bytes", 8)} ${pad("seeds", 6)} ${pad("resolv", 7)} ${pad("unchgd", 7)} ${pad("wasted", 8)} ${pad("chgd", 5)} ${pad("pruned", 7)} ${pad("ms", 7)}`, + `${"arm".padEnd(armWidth)}${pad("trips", 6)} ${pad("res", 6)} ${pad("bytes", 8)} ${pad("seeds", 6)} ${pad("resolv", 7)} ${pad("unchgd", 7)} ${pad("wasted", 8)} ${pad("chgd", 5)} ${pad("pruned", 7)} ${pad("ms", 7)}`, ]; for (const r of rows) { lines.push( - `${r.arm.padEnd(28)} ${pad(r.roundTrips, 6)} ${pad(r.resources, 6)} ${pad(r.bytes, 8)} ${pad(r.seeds, 6)} ${pad(r.resolutions, 7)} ${pad(r.unchanged, 7)} ${pad(r.wastedBytes, 8)} ${pad(r.changedSteady, 5)} ${pad(r.prunedFields, 7)} ${pad(r.ms, 7)}`, + `${r.arm.padEnd(armWidth)}${pad(r.roundTrips, 6)} ${pad(r.resources, 6)} ${pad(r.bytes, 8)} ${pad(r.seeds, 6)} ${pad(r.resolutions, 7)} ${pad(r.unchanged, 7)} ${pad(r.wastedBytes, 8)} ${pad(r.changedSteady, 5)} ${pad(r.prunedFields, 7)} ${pad(r.ms, 7)}`, ); } @@ -234,27 +304,38 @@ function report(rows: Row[], skipped: string[], failed: string[] = []) { console.log(lines.join("\n")); } -test("benchmark: tree loading cost by algorithm", async () => { - if (process.env.PL_TREE_BENCH !== "1") return; - +async function runBenchmark(shape: Shape, finalities: Finality[]) { await TestHelpers.withTempRoot(async (pl) => { - const seed = await seedTree(pl); + const seed = await seedTree(pl, shape); const rows: Row[] = []; const skipped: string[] = []; const failed: string[] = []; - for (const arm of ARMS) { - // One arm failing must not lose the other five: this runs against a real backend, and - // an algorithm that errors is itself a result worth reporting. - try { - const row = await runArm(pl, arm, seed); - if (row === undefined) skipped.push(arm.label.trim()); - else rows.push(row); - } catch (e: unknown) { - failed.push(`${arm.label.trim()}: ${e instanceof Error ? e.message : String(e)}`); + for (const finality of finalities) { + for (const arm of ARMS) { + // One arm failing must not lose the others: this runs against a real backend, and + // an algorithm that errors is itself a result worth reporting. + const label = `${arm.label} ${finality.name}`; + try { + const row = await runArm(pl, arm, seed, finality, label); + if (row === undefined) skipped.push(label.trim()); + else rows.push(row); + } catch (e: unknown) { + failed.push(`${label.trim()}: ${e instanceof Error ? e.message : String(e)}`); + } } } - report(rows, skipped, failed); + report(shape, rows, skipped, failed); }); +} + +test("benchmark: tree loading cost by algorithm", async () => { + if (process.env.PL_TREE_BENCH !== "1") return; + await runBenchmark(STRUCT_SHAPE, [TREE_FINALITY]); +}, 600_000); + +test("benchmark: polling cost of an ephemeral map that is never final", async () => { + if (process.env.PL_TREE_BENCH !== "1") return; + await runBenchmark(MAP_SHAPE, [MAP_FINAL_AT_READY, TREE_FINALITY]); }, 600_000); diff --git a/lib/node/pl-tree/src/synchronized_tree.test.ts b/lib/node/pl-tree/src/synchronized_tree.test.ts index 6fb952a07c..145368c0da 100644 --- a/lib/node/pl-tree/src/synchronized_tree.test.ts +++ b/lib/node/pl-tree/src/synchronized_tree.test.ts @@ -3,6 +3,7 @@ import { DefaultFinalResourceDataPredicate, field, GrantType, + Pl, PlClient, TestHelpers, UnauthenticatedPlClient, @@ -847,3 +848,78 @@ test("a tree of shared roots survives a re-login: it rediscovers the roots under } } }, 60_000); + +test("an error that reaches an ephemeral map after it is ready gets into the tree", async () => { + await TestHelpers.withTempRoot(async (pl) => { + const ids = await pl.withWriteTx( + "LateMapError", + async (tx) => { + const importer = tx.createStruct(TestStructuralResourceType1); + tx.createField(field(importer, "blob"), "Output"); + tx.lockInputs(importer); + + const entry = tx.createStruct(TestStructuralResourceType1); + tx.createField(field(entry, "file"), "Input", field(importer, "blob")); + tx.lockInputs(entry); + + const imports = tx.createEphemeral(Pl.EphStdMap); + tx.createField(field(imports, "source"), "Input", entry); + tx.lockInputs(imports); + + const effect = tx.createEphemeral(Pl.EphStdMap); + tx.createField(field(effect, "imports"), "Input", imports); + tx.lockInputs(effect); + + const holder = tx.createEphemeral(Pl.EphStdMap); + tx.createField(field(holder, "effect"), "Input", effect); + tx.lockInputs(holder); + + const rootField = field(tx.clientRoot, "lateMapError"); + tx.createField(rootField, "Dynamic"); + tx.setField(rootField, holder); + await tx.commit(); + return { importer: await importer.globalId, holder: await holder.globalId }; + }, + { sync: true }, + ); + + const tree = await SynchronizedTreeState.init(pl, ids.holder, { + stopPollingDelay: 60_000, + pollingInterval: 10, + }); + try { + const readImports = () => + Computable.make((c) => { + const effect = c.accessor(tree.entry()).node().traverse("effect"); + return effect?.getIsReadyOrError() ? effect.traverse("imports")?.id : undefined; + }); + expect(await readImports().getValue()).toBeDefined(); + + await pl.withWriteTx( + "LateMapErrorSet", + async (tx) => { + tx.setFieldError(field(ids.importer, "blob"), tx.createError("late map error")); + await tx.commit(); + }, + { sync: true }, + ); + + await expect + .poll( + async () => { + await tree.refreshState(); + try { + await readImports().getValue(); + return "no error"; + } catch (e) { + return String(e); + } + }, + { timeout: 20_000, interval: 100 }, + ) + .toMatch(/\/imports"[\s\S]*late map error/); + } finally { + await tree.terminate(); + } + }); +}, 60_000);