diff --git a/.changeset/eph-std-map-never-final.md b/.changeset/eph-std-map-never-final.md new file mode 100644 index 0000000000..bd21971e10 --- /dev/null +++ b/.changeset/eph-std-map-never-final.md @@ -0,0 +1,5 @@ +--- +"@milaboratories/pl-client": patch +--- + +A tree no longer holds a ready `EphStdMap` final. An error that reaches the map after it is ready, such as the failed import of a v2 effect, now gets into the tree. diff --git a/lib/node/pl-client/src/core/final.ts b/lib/node/pl-client/src/core/final.ts index f3490ff1b3..41d79038e6 100644 --- a/lib/node/pl-client/src/core/final.ts +++ b/lib/node/pl-client/src/core/final.ts @@ -6,11 +6,15 @@ export { ResourceTypeName, ResourceTypePrefix }; export type { FinalResourceDataPredicate } from "./finality"; export { readyOrDuplicateOrError } from "./finality"; -// The finality tables, one layer per consumer. Every layer accepts two backend writes after +// The finality tables, one layer per consumer. Every layer accepts three backend writes after // final: a data-loss error cascade, lifting the error of a stored blob found lost or corrupt to -// the resources built from it, and the first duplicate's `hasOriginalListeners` flag on its -// original, backend-internal and never on the wire. An entry's `why` names any other write -// it accepts. A layer adds only the types whose later writes its consumer does not observe. +// the resources built from it; a late error, lifted by the core from an entry that fails after +// the resource above it is ready, because ready asks only that every input points to a resource +// with a final status, and work below that resource can still fail; and the first duplicate's +// `hasOriginalListeners` flag on its original, backend-internal and never on the wire. A reader +// of a final resource misses the first two. An entry's `why` names any other write it accepts, +// and a type that must show a late error gets a `never` entry. A layer adds only the types whose +// later writes its consumer does not observe. // Where the backend's filter stops earlier than the predicate. An early stop is safe: the // backend sends a body-less stop marker, and the streaming loader fetches every stopped resource @@ -36,6 +40,9 @@ const value = (name: string): FinalityEntry => ({ why: "a value, written once at creation", }); +const noFillAfterReady = + "no controller fills it after ready; a late error from an entry below it is not seen"; + const settledAtReady = (name: string, why: string): FinalityEntry => ({ match: { name }, rule: "readyOrDuplicateOrError", @@ -50,8 +57,8 @@ const never = (match: FinalityEntry["match"], why: string): FinalityEntry => ({ }); /** The base layer: nothing observable (state, fields, KV) changes after the rule holds, whatever - * the consumer prunes. Exceptions: the data-loss error cascade, the `hasOriginalListeners` flag, - * and the writes its entries name. */ + * the consumer prunes. Exceptions: the data-loss error cascade, the late error, the + * `hasOriginalListeners` flag, and the writes its entries name. */ export const StrictFinality: FinalityTable = FinalityTable.of("strict", [ value(ResourceTypeName.JsonObject), value(ResourceTypeName.JsonGzObject), @@ -92,7 +99,6 @@ export const StrictFinality: FinalityTable = FinalityTable.of("strict", [ ...[ ResourceTypeName.StdMap, ResourceTypeName.StdMapSlash, - ResourceTypeName.EphStdMap, ResourceTypeName.PFrame, ResourceTypeName.ParquetChunk, ResourceTypeName.BContext, @@ -102,9 +108,7 @@ export const StrictFinality: FinalityTable = FinalityTable.of("strict", [ ResourceTypeName.BResolveSingleNoResult, ResourceTypeName.BQueryResult, ResourceTypeName.TengoLib, - ].map((name) => - settledAtReady(name, "its inputs are final at ready, and no controller fills it afterwards"), - ), + ].map((name) => settledAtReady(name, noFillAfterReady)), ...[ ResourceTypeName.TengoTemplate, ResourceTypeName.SoftwareInfo, @@ -122,7 +126,7 @@ export const StrictFinality: FinalityTable = FinalityTable.of("strict", [ { match: { prefix: ResourceTypePrefix.PColumnData }, rule: "readyOrDuplicateOrError", - why: "its inputs are final at ready, and no controller fills it afterwards", + why: noFillAfterReady, stopRule: earlyOnFieldErrors, }, { @@ -155,6 +159,12 @@ export const StrictFinality: FinalityTable = FinalityTable.of("strict", [ "as resolution finishes", stopRule: earlyOnFieldErrorsWithOutputs, }, + never( + { name: ResourceTypeName.EphStdMap }, + "it is ready once its entries resolve to resources, not once those resources settle; an " + + "entry that fails later, such as a v2 effect's failed import, puts its error on the map " + + "after ready", + ), never({ name: ResourceTypeName.UserProject }, "blocks and fields come and go over its life"), never({ name: ResourceTypeName.Projects }, "projects are added and removed"), never({ name: ResourceTypeName.ClientRoot }, "its fields are added and removed"), diff --git a/lib/node/pl-client/src/core/finality.test.ts b/lib/node/pl-client/src/core/finality.test.ts index 557635e288..15f09e4cda 100644 --- a/lib/node/pl-client/src/core/finality.test.ts +++ b/lib/node/pl-client/src/core/finality.test.ts @@ -131,7 +131,6 @@ describe("the built-in tables", () => { "Dummy", "StdMap", "std/map", - "EphStdMap", "PFrame", "ParquetChunk", "BContext", @@ -146,6 +145,7 @@ describe("the built-in tables", () => { "SoftwareInfo", ]); const neverFinal = new Set([ + "EphStdMap", "UserProject", "Projects", "ClientRoot", diff --git a/lib/node/pl-middle-layer/src/middle_layer/finality_stop_rules.test.ts b/lib/node/pl-middle-layer/src/middle_layer/finality_stop_rules.test.ts index 2444b0e299..09894f5bc5 100644 --- a/lib/node/pl-middle-layer/src/middle_layer/finality_stop_rules.test.ts +++ b/lib/node/pl-middle-layer/src/middle_layer/finality_stop_rules.test.ts @@ -32,10 +32,6 @@ function expectedStopRules(): Filter { treeFilter.resourceTypeEq(ResourceTypeName.StdMapSlash), treeFilter.readyOrDuplicateOrError(), ), - treeFilter.and( - treeFilter.resourceTypeEq(ResourceTypeName.EphStdMap), - treeFilter.readyOrDuplicateOrError(), - ), treeFilter.and( treeFilter.resourceTypeEq(ResourceTypeName.PFrame), treeFilter.readyOrDuplicateOrError(), diff --git a/lib/node/pl-tree/src/delta_benchmark.test.ts b/lib/node/pl-tree/src/delta_benchmark.test.ts index 75c0eb817f..0d6b3a5815 100644 --- a/lib/node/pl-tree/src/delta_benchmark.test.ts +++ b/lib/node/pl-tree/src/delta_benchmark.test.ts @@ -1,7 +1,13 @@ import { test } from "vitest"; -import { field, hasCapability, TestHelpers } from "@milaboratories/pl-client"; -import type { PlClient, PlTransaction, SignedResourceId } from "@milaboratories/pl-client"; -import { DefaultFinalResourceDataPredicate } from "@milaboratories/pl-client"; +import { field, hasCapability, Pl, TestHelpers, TreeFinality } from "@milaboratories/pl-client"; +import type { + FieldType, + FinalResourceDataPredicate, + PlClient, + PlTransaction, + ResourceRef, + SignedResourceId, +} from "@milaboratories/pl-client"; import { TestStructuralResourceType1 } from "./test_utils"; import { PlTreeState } from "./state"; import { constructTreeLoadingRequest, initialTreeLoadingStat, loadTreeState } from "./sync"; @@ -18,6 +24,10 @@ import type { ExtendedResourceData } from "./state"; * * PL_TREE_BENCH=1 pnpm exec vitest run src/delta_benchmark.test.ts * + * A second run seeds ephemeral maps of values in place of the structs, and crosses the arms with + * two finality tables: the tree's own, and the tree's own plus the row that held a ready + * EphStdMap final. The difference is the polling cost of a map that is never final. + * * Delta arms report as skipped on a backend without `treeChangedSince:v2`, rather than * silently measuring the fallback. */ @@ -44,59 +54,117 @@ const ARMS: Arm[] = [ { label: "backend-delta prune=off", mode: "backend-delta", pruning: false }, ]; +/** The finality table an arm runs under. */ +type Finality = { name: string; predicate: FinalResourceDataPredicate }; + +const TREE_FINALITY: Finality = { name: "tree", predicate: TreeFinality.predicate() }; + +const MAP_FINAL_AT_READY: Finality = { + name: "map-final-at-ready", + predicate: TreeFinality.extend("map final at ready", [ + { + match: { name: Pl.EphStdMap.name }, + rule: "readyOrDuplicateOrError", + why: "the row that held a ready EphStdMap final, for the cost comparison", + }, + ]).predicate(), +}; + /** Stands in for the real project pruning, without importing the middle layer. */ const benchPruning = (r: ExtendedResourceData): FieldData[] => r.fields.filter((f) => !f.name.startsWith("pruneMe")); +type SeededTree = { root: SignedResourceId; leaves: SignedResourceId[] }; + +/** What a child of the root is, what it holds, and which resources the polls write KV on. */ +type Shape = { + name: string; + child: (tx: PlTransaction) => ResourceRef; + childField: FieldType; + /** `key` is unique per leaf: equal values deduplicate into one resource. */ + leaf: (tx: PlTransaction, key: string) => ResourceRef; + /** Runs after the child gets its fields. */ + close: (tx: PlTransaction, child: ResourceRef) => void; + touchedName: string; + touched: (tree: SeededTree) => SignedResourceId[]; +}; + +const STRUCT_SHAPE: Shape = { + name: "structs", + child: (tx) => tx.createStruct(TestStructuralResourceType1, PAYLOAD), + childField: "Dynamic", + leaf: (tx) => tx.createStruct(TestStructuralResourceType1, PAYLOAD), + close: () => {}, + touchedName: "a leaf (quiet-parent shape)", + touched: (tree) => tree.leaves, +}; + +const MAP_SHAPE: Shape = { + name: "EphStdMaps of json values", + child: (tx) => tx.createEphemeral(Pl.EphStdMap), + childField: "Input", + leaf: (tx, key) => + tx.createValue(Pl.JsonObject, JSON.stringify({ key, payload: PAYLOAD.toString() })), + // A map is ready, and so final under the old row, only with its inputs locked. + close: (tx, child) => tx.lockInputs(child), + // A json value is final at creation, so a KV write on it is never read again. + touchedName: "the root", + touched: (tree) => [tree.root], +}; + async function seedTree( pl: PlClient, -): Promise<{ root: SignedResourceId; leaves: SignedResourceId[] }> { + shape: Shape, +): Promise<{ root: SignedResourceId; touched: SignedResourceId[] }> { return await pl.withWriteTx( "BenchSeed", async (tx) => { const root = tx.createStruct(TestStructuralResourceType1, PAYLOAD); - const rootField = field(tx.clientRoot, "benchRoot"); + const rootField = field(tx.clientRoot, `benchRoot-${shape.name}`); tx.createField(rootField, "Dynamic"); tx.setField(rootField, root); const leaves: Promise[] = []; for (let c = 0; c < CHILDREN; c++) { - const child = tx.createStruct(TestStructuralResourceType1, PAYLOAD); + const child = shape.child(tx); const cf = field(root, `child${c}`); tx.createField(cf, "Dynamic"); tx.setField(cf, child); // A field the pruning function removes, so prune=on and prune=off differ. - const pruned = tx.createStruct(TestStructuralResourceType1, PAYLOAD); + const pruned = shape.leaf(tx, `pruned${c}`); const pf = field(child, "pruneMe"); - tx.createField(pf, "Dynamic"); + tx.createField(pf, shape.childField); tx.setField(pf, pruned); for (let g = 0; g < GRANDCHILDREN; g++) { - const grand = tx.createStruct(TestStructuralResourceType1, PAYLOAD); + const grand = shape.leaf(tx, `leaf${c}.${g}`); const gf = field(child, `g${g}`); - tx.createField(gf, "Dynamic"); + tx.createField(gf, shape.childField); tx.setField(gf, grand); leaves.push(grand.globalId); } + shape.close(tx, child); } await tx.commit(); - return { root: await root.globalId, leaves: await Promise.all(leaves) }; + const tree = { root: await root.globalId, leaves: await Promise.all(leaves) }; + return { root: tree.root, touched: shape.touched(tree) }; }, { sync: true }, ); } -/** One mutation between polls: a KV write on a leaf. The parent is not rewritten, so this is - * the quiet-parent shape, which is the case frontier seeding exists to reach. */ -async function touchLeaf(pl: PlClient, leaf: SignedResourceId, arm: string, round: number) { +/** One mutation between polls: a KV write on a resource the shape names. On a leaf the parent is + * not rewritten, so this is the quiet-parent shape, which is the case frontier seeding exists to + * reach. */ +async function touch(pl: PlClient, target: SignedResourceId, arm: string, round: number) { await pl.withWriteTx( "BenchTouch", async (tx: PlTransaction) => { // Keyed by arm: the tree is shared and never reset, and state.ts compares KV values, // so a later arm rewriting the same key with the same bytes observes no change at all. - tx.setKValue(leaf, `bench-${arm}-${round}`, Buffer.from(`r${round}`)); + tx.setKValue(target, `bench-${arm}-${round}`, Buffer.from(`r${round}`)); await tx.commit(); }, { sync: true }, @@ -122,13 +190,15 @@ type Row = { async function runArm( pl: PlClient, arm: Arm, - seed: { root: SignedResourceId; leaves: SignedResourceId[] }, + seed: { root: SignedResourceId; touched: SignedResourceId[] }, + finality: Finality, + label: string, ): Promise { const caps = pl.serverInfo.capabilities ?? []; if (arm.mode === "backend-delta" && !hasCapability(caps, "treeChangedSince:v2")) return undefined; // Scalar, not an array: the constructor takes SignedResourceId | Set. - const state = new PlTreeState(seed.root, DefaultFinalResourceDataPredicate); + const state = new PlTreeState(seed.root, finality.predicate); const stat: TreeLoadingStat = initialTreeLoadingStat(); let token: Uint8Array | undefined; let changedAtColdLoad = 0; @@ -137,9 +207,8 @@ async function runArm( // purpose: it is where the arms are meant to look alike. for (let cycle = 0; cycle <= POLL_CYCLES; cycle++) { if (cycle > 0) { - const leaf = seed.leaves[cycle % seed.leaves.length]; - if (leaf !== undefined) - await touchLeaf(pl, leaf, arm.label.trim().replace(/\s+/g, "-"), cycle); + const target = seed.touched[cycle % seed.touched.length]; + if (target !== undefined) await touch(pl, target, label.trim().replace(/\s+/g, "-"), cycle); } const request = constructTreeLoadingRequest(state, { @@ -162,7 +231,7 @@ async function runArm( } return { - arm: arm.label, + arm: label, roundTrips: stat.roundTrips, resources: stat.retrievedResources, bytes: stat.retrievedResourceDataBytes + stat.retrievedKeyValueBytes, @@ -176,19 +245,20 @@ async function runArm( }; } -function report(rows: Row[], skipped: string[], failed: string[] = []) { +function report(shape: Shape, rows: Row[], skipped: string[], failed: string[]) { const pad = (s: string | number, n: number) => String(s).padStart(n); + const armWidth = Math.max("arm".length, ...rows.map((r) => r.arm.length)) + 1; const lines = [ "", "=== tree loading cost ===", - `tree: ${CHILDREN} children x ${GRANDCHILDREN} grandchildren, ${POLL_CYCLES} polls after load,`, - ` one KV write on a leaf between polls (quiet-parent shape)`, + `tree: ${CHILDREN} ${shape.name} x ${GRANDCHILDREN} leaves, ${POLL_CYCLES} polls after load,`, + ` one KV write on ${shape.touchedName} between polls`, "", - `${"arm".padEnd(28)} ${pad("trips", 6)} ${pad("res", 6)} ${pad("bytes", 8)} ${pad("seeds", 6)} ${pad("resolv", 7)} ${pad("unchgd", 7)} ${pad("wasted", 8)} ${pad("chgd", 5)} ${pad("pruned", 7)} ${pad("ms", 7)}`, + `${"arm".padEnd(armWidth)}${pad("trips", 6)} ${pad("res", 6)} ${pad("bytes", 8)} ${pad("seeds", 6)} ${pad("resolv", 7)} ${pad("unchgd", 7)} ${pad("wasted", 8)} ${pad("chgd", 5)} ${pad("pruned", 7)} ${pad("ms", 7)}`, ]; for (const r of rows) { lines.push( - `${r.arm.padEnd(28)} ${pad(r.roundTrips, 6)} ${pad(r.resources, 6)} ${pad(r.bytes, 8)} ${pad(r.seeds, 6)} ${pad(r.resolutions, 7)} ${pad(r.unchanged, 7)} ${pad(r.wastedBytes, 8)} ${pad(r.changedSteady, 5)} ${pad(r.prunedFields, 7)} ${pad(r.ms, 7)}`, + `${r.arm.padEnd(armWidth)}${pad(r.roundTrips, 6)} ${pad(r.resources, 6)} ${pad(r.bytes, 8)} ${pad(r.seeds, 6)} ${pad(r.resolutions, 7)} ${pad(r.unchanged, 7)} ${pad(r.wastedBytes, 8)} ${pad(r.changedSteady, 5)} ${pad(r.prunedFields, 7)} ${pad(r.ms, 7)}`, ); } @@ -234,27 +304,38 @@ function report(rows: Row[], skipped: string[], failed: string[] = []) { console.log(lines.join("\n")); } -test("benchmark: tree loading cost by algorithm", async () => { - if (process.env.PL_TREE_BENCH !== "1") return; - +async function runBenchmark(shape: Shape, finalities: Finality[]) { await TestHelpers.withTempRoot(async (pl) => { - const seed = await seedTree(pl); + const seed = await seedTree(pl, shape); const rows: Row[] = []; const skipped: string[] = []; const failed: string[] = []; - for (const arm of ARMS) { - // One arm failing must not lose the other five: this runs against a real backend, and - // an algorithm that errors is itself a result worth reporting. - try { - const row = await runArm(pl, arm, seed); - if (row === undefined) skipped.push(arm.label.trim()); - else rows.push(row); - } catch (e: unknown) { - failed.push(`${arm.label.trim()}: ${e instanceof Error ? e.message : String(e)}`); + for (const finality of finalities) { + for (const arm of ARMS) { + // One arm failing must not lose the others: this runs against a real backend, and + // an algorithm that errors is itself a result worth reporting. + const label = `${arm.label} ${finality.name}`; + try { + const row = await runArm(pl, arm, seed, finality, label); + if (row === undefined) skipped.push(label.trim()); + else rows.push(row); + } catch (e: unknown) { + failed.push(`${label.trim()}: ${e instanceof Error ? e.message : String(e)}`); + } } } - report(rows, skipped, failed); + report(shape, rows, skipped, failed); }); +} + +test("benchmark: tree loading cost by algorithm", async () => { + if (process.env.PL_TREE_BENCH !== "1") return; + await runBenchmark(STRUCT_SHAPE, [TREE_FINALITY]); +}, 600_000); + +test("benchmark: polling cost of an ephemeral map that is never final", async () => { + if (process.env.PL_TREE_BENCH !== "1") return; + await runBenchmark(MAP_SHAPE, [MAP_FINAL_AT_READY, TREE_FINALITY]); }, 600_000); diff --git a/lib/node/pl-tree/src/synchronized_tree.test.ts b/lib/node/pl-tree/src/synchronized_tree.test.ts index 6fb952a07c..145368c0da 100644 --- a/lib/node/pl-tree/src/synchronized_tree.test.ts +++ b/lib/node/pl-tree/src/synchronized_tree.test.ts @@ -3,6 +3,7 @@ import { DefaultFinalResourceDataPredicate, field, GrantType, + Pl, PlClient, TestHelpers, UnauthenticatedPlClient, @@ -847,3 +848,78 @@ test("a tree of shared roots survives a re-login: it rediscovers the roots under } } }, 60_000); + +test("an error that reaches an ephemeral map after it is ready gets into the tree", async () => { + await TestHelpers.withTempRoot(async (pl) => { + const ids = await pl.withWriteTx( + "LateMapError", + async (tx) => { + const importer = tx.createStruct(TestStructuralResourceType1); + tx.createField(field(importer, "blob"), "Output"); + tx.lockInputs(importer); + + const entry = tx.createStruct(TestStructuralResourceType1); + tx.createField(field(entry, "file"), "Input", field(importer, "blob")); + tx.lockInputs(entry); + + const imports = tx.createEphemeral(Pl.EphStdMap); + tx.createField(field(imports, "source"), "Input", entry); + tx.lockInputs(imports); + + const effect = tx.createEphemeral(Pl.EphStdMap); + tx.createField(field(effect, "imports"), "Input", imports); + tx.lockInputs(effect); + + const holder = tx.createEphemeral(Pl.EphStdMap); + tx.createField(field(holder, "effect"), "Input", effect); + tx.lockInputs(holder); + + const rootField = field(tx.clientRoot, "lateMapError"); + tx.createField(rootField, "Dynamic"); + tx.setField(rootField, holder); + await tx.commit(); + return { importer: await importer.globalId, holder: await holder.globalId }; + }, + { sync: true }, + ); + + const tree = await SynchronizedTreeState.init(pl, ids.holder, { + stopPollingDelay: 60_000, + pollingInterval: 10, + }); + try { + const readImports = () => + Computable.make((c) => { + const effect = c.accessor(tree.entry()).node().traverse("effect"); + return effect?.getIsReadyOrError() ? effect.traverse("imports")?.id : undefined; + }); + expect(await readImports().getValue()).toBeDefined(); + + await pl.withWriteTx( + "LateMapErrorSet", + async (tx) => { + tx.setFieldError(field(ids.importer, "blob"), tx.createError("late map error")); + await tx.commit(); + }, + { sync: true }, + ); + + await expect + .poll( + async () => { + await tree.refreshState(); + try { + await readImports().getValue(); + return "no error"; + } catch (e) { + return String(e); + } + }, + { timeout: 20_000, interval: 100 }, + ) + .toMatch(/\/imports"[\s\S]*late map error/); + } finally { + await tree.terminate(); + } + }); +}, 60_000);