diff --git a/.gitignore b/.gitignore index ae4e0c1..7c0bfb7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ target/ __pycache__/ *.pyc .DS_Store +python/build/ +python/*.egg-info/ +python/dist/ diff --git a/README.md b/README.md index ea259c8..871852b 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,15 @@ contracts. Registration alone never starts execution. See [world lifecycle and protocol](docs/worlds.md) and [inspection metadata](docs/inspection.md). Existing `ProgramInstance` and shared-host callers remain supported. +Saved [library artifacts](docs/library-artifacts.md) import exact definition pins +and scenarios without execution. Attached clients share the small +[Python SDK](python/README.md); their connection lifetime does not own worlds. +The owner supervises [managed workers and durable admissions](docs/managed-workers.md), +while application packages own prompts, tools and provider behavior. Explicit +[JSON recovery](docs/managed-checkpoints.md) and optional +[local Iceberg publication](docs/managed-iceberg.md) restore into fresh generations; +neither automatically retries external calls. + ## MCP ```sh diff --git a/docs/checkpoints.md b/docs/checkpoints.md index 9c425c9..bc6e537 100644 --- a/docs/checkpoints.md +++ b/docs/checkpoints.md @@ -59,3 +59,10 @@ restored source before exposing product operations. For a composition, the caller can include its manifest and returned resolution in that metadata. Checkpoints retain the installed source and inputs, but do not implicitly retain registry records or reconstruct a composition resolution. + +Managed worlds expose this same JSON format through explicit receipt-based +Checkpoint/Restore operations. They bind registry/dependency pins, public admission, +origin execution and actual lowering automatically, and rebuild that admission +state before activating restoration. See [managed checkpoints](managed-checkpoints.md). +Backend restore retains its caller's hosted process control, capture path and +observer options while constructing the fresh candidate. diff --git a/docs/evidence/managed-iceberg-runtime.json b/docs/evidence/managed-iceberg-runtime.json new file mode 100644 index 0000000..4ec4adf --- /dev/null +++ b/docs/evidence/managed-iceberg-runtime.json @@ -0,0 +1,1240 @@ +{ + "schema_version": 1, + "recorded_at_utc": "2026-09-29T17:23:52.626414+00:00", + "scope": "Managed local SQLite/local FileIO Iceberg checkpoint transport; isolated native composition fixture, not assembled HHMM/Observer or remote R2 acceptance", + "status": "passed", + "candidate": { + "source_sha256": { + "src/world_storage.rs": "02c073e77af6c331af0fe6b53f38a7986ff0df3c703f9bba06213257161d6e94", + "src/world_persistence.rs": "9283e01a4184e31d0dec228edbbe0d06671cf013c447d85626f9f2b69fb31802", + "src/iceberg_checkpoint.rs": "2568ea8623b5943ea6715e2a6e2107c766e073256182bdd2ec306bd1ac128ea3", + "src/worlds.rs": "efbdd17101e2183e8dfec58a15bac9e36dc3d5c82efb62be6d2cb2a44b6d8365", + "src/bin/ddlog-worlds.rs": "a7b3a9f61001bd1ee59212deb997ee506dbf84f7109d37704e056b78fae29049", + "tests/managed_iceberg.rs": "05e4a6a343038435bbf81111434fcd22e9356f71eae00165094b6893c304ac0f" + }, + "host_binary_sha256": "10c832981c1a5a3fb150dccc04a9c57daa653bec34af086a31f4808b5772954a", + "candidate_path": "/private/tmp/hhmm-x0-20260929/ddlog-worlds-managed-iceberg", + "test_binary_sha256": "05b66ce691e8888ffd74d3e3072ffa1a912999199b1b53ed2e34c5a10112c236" + }, + "source_sha256": { + "src/world_storage.rs": "02c073e77af6c331af0fe6b53f38a7986ff0df3c703f9bba06213257161d6e94", + "src/world_persistence.rs": "9283e01a4184e31d0dec228edbbe0d06671cf013c447d85626f9f2b69fb31802", + "src/iceberg_checkpoint.rs": "2568ea8623b5943ea6715e2a6e2107c766e073256182bdd2ec306bd1ac128ea3", + "src/worlds.rs": "efbdd17101e2183e8dfec58a15bac9e36dc3d5c82efb62be6d2cb2a44b6d8365", + "src/bin/ddlog-worlds.rs": "a7b3a9f61001bd1ee59212deb997ee506dbf84f7109d37704e056b78fae29049", + "tests/managed_iceberg.rs": "05e4a6a343038435bbf81111434fcd22e9356f71eae00165094b6893c304ac0f", + "src/checkpoint.rs": "4f05f81cf2ee596ef13042cd518826d4e162f66e33983103d2027cf53f064a3c", + "src/instance.rs": "a9c2dd6ad061608e23bc62ca984ea42c6174e75585e8dba7a18f497768f2f835", + "src/world_admission.rs": "86490f4f5f9e7348789f9cae96abcebc1b85cc0e4bf1ee924011e287b7c69925", + "src/world_workers.rs": "c056602e755099dcd4153ad28e6f24e663650e4307280fe00f859ef6fae026cd", + "src/lib.rs": "2b37bf482a72528403a95a1fc7b948f84cba18b8b2107e38b74737327a249b69", + "docs/managed-iceberg.md": "6d29be784922756a5987b108d7c66339ed86b45668f9df9c97ff7d62b91907f1", + "docs/managed-checkpoints.md": "86ec951531e140347b4fcb783b00be97267af14a1deea22bc43be958367c81c2", + "docs/iceberg-checkpoints.md": "d21f0daa8d4d45854e2af12afbc2bdedce4d99de7c8375fbde8f44a7761e7207", + "docs/worlds.md": "6d3ba3ddf847453e5229b81474731e9aafc23de4960d100f44fb1c5620ca6f88", + "docs/managed-iceberg-next.md": "9ac064857378dbd206963582b2ecd2427a9f1d77db1f7a95444dc764f1c2574b" + }, + "validation": { + "host_rust": "1.95.0", + "native_ddlog": "1.2.3", + "native_rust": "1.65.0", + "host_target": "/private/tmp/hhmm-x0-20260929/runtime-target", + "native_target": "/private/tmp/hhmm-x0-20260929/runtime-native-target", + "fmt": "passed (owned Rust files, skip_children=true)", + "clippy": "passed --offline --lib --bin ddlog-worlds -- -D warnings with and without --features iceberg; feature test managed_iceberg also checked", + "diff_check": "passed (owned tracked files); new owned files checked for trailing whitespace", + "tests": { + "managed_iceberg_simulated_transport_real_catalog": { + "passed": 4, + "seconds": 7.01 + }, + "managed_iceberg_native_composition": { + "passed": 1, + "seconds": 185.84 + }, + "memory_runtime_existing_iceberg_engine": { + "passed": 1, + "seconds": 0.63 + }, + "worlds_json_persistence": { + "passed": 8, + "seconds": 8.97 + }, + "worlds_admission": { + "passed": 4, + "seconds": 4.46 + }, + "workers_socket_attached_worker_admission_replay_restore_and_owner_shutdown": { + "passed": 1, + "seconds": 2.11 + }, + "worlds_test_worlds_run_scenarios_asynchronously_and_report_results": { + "passed": 1, + "seconds": 1.07 + } + }, + "capture_test_correction": "A prior run passed restore and update/retraction but asserted inspection before the 100 ms capture tailer polled. The fixture now waits at most five seconds for available capture with timely events, matching the existing native JSON test contract. No production change was needed; final native run passed.", + "native_driver": "Supported build-ddlog.sh/build-native-artifact.py, owned by the separate build lane; no edits by this slice", + "provider_calls": 0, + "active_stores_accessed": false, + "existing_owners_modified": false + }, + "proof": { + "ambiguous_record_reconciled": true, + "blocked_catalog_query_ms": 8.581750000000001, + "blocked_catalog_status_ms": 31.942333000000005, + "capture_generation": 2, + "capture_state": "available", + "catalog_snapshot_id": "3670372088214088639", + "native": true, + "provider_calls": 0, + "public_composition": true, + "published": { + "availability": "catalog_unverified", + "error": null, + "receipt": { + "checkpoint_sha256": "764a265aaee4b5b75ea1a6a7caa80ecff56478af791b448716c66fa7fc3070f9", + "format": "iceberg", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "a925df9813475fef0d33760ad7d10819ebbbf77e6242c50c1192c98dceb062d2", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "26065-1790702406512034000-0" + }, + "program": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "Input_source" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "Output_echo" + } + ], + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "published_at_unix_ms": 1790702515183, + "receipt_id": "26065-1790702514478248000-6", + "schema_version": 1, + "storage": { + "envelope_sha256": "b5737b4a612861c41891dbe117333cb29f3ee7241d18a57017106edf3830d641", + "object_sha256": "d94f8054da7c9aceef7cf37b9a558a62efa25d1c0a5f8a2d2e872af567cea94d", + "object_uri": "file:///private/var/folders/qq/pxfnjsk517x5rc0m044m3fnm0000gn/T/managed-iceberg-26065-0/local store/warehouse/objects/26065-1790702514478248000-6.parquet", + "profile": "local", + "profile_sha256": "d8b20caf7a1de92fd19928eee0690efca12458f671b51668f00f01e1ae5226d7", + "snapshot_id": "3670372088214088639", + "table": [ + "runtime", + "checkpoints" + ], + "table_uuid": "01a0ee2f-f167-73c1-bc65-8d64b9a01c46" + } + }, + "schema_version": 1, + "status": "published" + }, + "restore": { + "build": null, + "definition": { + "label": "Iceberg recovery", + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + } + }, + "error": null, + "generation": 2, + "history": [ + { + "at_unix_ms": 1790702406513, + "error": null, + "generation": 0, + "restore_requested": null, + "restored_from": null, + "state": "created" + }, + { + "at_unix_ms": 1790702406561, + "error": null, + "generation": 1, + "restore_requested": null, + "restored_from": null, + "state": "starting" + }, + { + "at_unix_ms": 1790702439045, + "error": null, + "generation": 1, + "restore_requested": null, + "restored_from": null, + "state": "running" + }, + { + "at_unix_ms": 1790702514968, + "error": null, + "generation": 1, + "restore_requested": null, + "restored_from": null, + "state": "stopped" + }, + { + "at_unix_ms": 1790702515278, + "error": null, + "generation": 2, + "restore_requested": { + "checkpoint_sha256": "764a265aaee4b5b75ea1a6a7caa80ecff56478af791b448716c66fa7fc3070f9", + "format": "iceberg", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "a925df9813475fef0d33760ad7d10819ebbbf77e6242c50c1192c98dceb062d2", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "26065-1790702406512034000-0" + }, + "program": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "Input_source" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "Output_echo" + } + ], + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "published_at_unix_ms": 1790702515183, + "receipt_id": "26065-1790702514478248000-6", + "schema_version": 1, + "storage": { + "envelope_sha256": "b5737b4a612861c41891dbe117333cb29f3ee7241d18a57017106edf3830d641", + "object_sha256": "d94f8054da7c9aceef7cf37b9a558a62efa25d1c0a5f8a2d2e872af567cea94d", + "object_uri": "file:///private/var/folders/qq/pxfnjsk517x5rc0m044m3fnm0000gn/T/managed-iceberg-26065-0/local store/warehouse/objects/26065-1790702514478248000-6.parquet", + "profile": "local", + "profile_sha256": "d8b20caf7a1de92fd19928eee0690efca12458f671b51668f00f01e1ae5226d7", + "snapshot_id": "3670372088214088639", + "table": [ + "runtime", + "checkpoints" + ], + "table_uuid": "01a0ee2f-f167-73c1-bc65-8d64b9a01c46" + } + }, + "restored_from": null, + "state": "starting" + }, + { + "at_unix_ms": 1790702591694, + "error": null, + "generation": 2, + "restore_requested": { + "checkpoint_sha256": "764a265aaee4b5b75ea1a6a7caa80ecff56478af791b448716c66fa7fc3070f9", + "format": "iceberg", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "a925df9813475fef0d33760ad7d10819ebbbf77e6242c50c1192c98dceb062d2", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "26065-1790702406512034000-0" + }, + "program": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "Input_source" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "Output_echo" + } + ], + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "published_at_unix_ms": 1790702515183, + "receipt_id": "26065-1790702514478248000-6", + "schema_version": 1, + "storage": { + "envelope_sha256": "b5737b4a612861c41891dbe117333cb29f3ee7241d18a57017106edf3830d641", + "object_sha256": "d94f8054da7c9aceef7cf37b9a558a62efa25d1c0a5f8a2d2e872af567cea94d", + "object_uri": "file:///private/var/folders/qq/pxfnjsk517x5rc0m044m3fnm0000gn/T/managed-iceberg-26065-0/local store/warehouse/objects/26065-1790702514478248000-6.parquet", + "profile": "local", + "profile_sha256": "d8b20caf7a1de92fd19928eee0690efca12458f671b51668f00f01e1ae5226d7", + "snapshot_id": "3670372088214088639", + "table": [ + "runtime", + "checkpoints" + ], + "table_uuid": "01a0ee2f-f167-73c1-bc65-8d64b9a01c46" + } + }, + "restored_from": { + "checkpoint_sha256": "764a265aaee4b5b75ea1a6a7caa80ecff56478af791b448716c66fa7fc3070f9", + "format": "iceberg", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "a925df9813475fef0d33760ad7d10819ebbbf77e6242c50c1192c98dceb062d2", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "26065-1790702406512034000-0" + }, + "program": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "Input_source" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "Output_echo" + } + ], + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "published_at_unix_ms": 1790702515183, + "receipt_id": "26065-1790702514478248000-6", + "schema_version": 1, + "storage": { + "envelope_sha256": "b5737b4a612861c41891dbe117333cb29f3ee7241d18a57017106edf3830d641", + "object_sha256": "d94f8054da7c9aceef7cf37b9a558a62efa25d1c0a5f8a2d2e872af567cea94d", + "object_uri": "file:///private/var/folders/qq/pxfnjsk517x5rc0m044m3fnm0000gn/T/managed-iceberg-26065-0/local store/warehouse/objects/26065-1790702514478248000-6.parquet", + "profile": "local", + "profile_sha256": "d8b20caf7a1de92fd19928eee0690efca12458f671b51668f00f01e1ae5226d7", + "snapshot_id": "3670372088214088639", + "table": [ + "runtime", + "checkpoints" + ], + "table_uuid": "01a0ee2f-f167-73c1-bc65-8d64b9a01c46" + } + }, + "state": "running" + } + ], + "id": "26065-1790702406512034000-0", + "inspection": { + "metadata": { + "authoredGroups": [ + { + "id": "child", + "kind": "module", + "memberIds": [], + "member_key": "child", + "member_match": { + "debug_pattern": "\\bR_Module0_", + "propagate": true + }, + "name": "child", + "ports": [], + "provenance": { + "repository": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "revision": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6", + "source": null + } + }, + { + "id": "$inputs", + "kind": "module", + "memberIds": [], + "member_key": "$inputs", + "member_match": { + "debug_pattern": "^Input \\{ rel: \"R_Input_", + "propagate": false + }, + "name": "Inputs", + "ports": [], + "provenance": { + "repository": "processor_322879060d7cebd34152854cdfc8b52a", + "revision": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd", + "source": null + } + }, + { + "id": "$outputs", + "kind": "module", + "memberIds": [], + "member_key": "$outputs", + "member_match": { + "debug_pattern": "^InspectOutput \\{ rel: \"R_Output_", + "propagate": false + }, + "name": "Outputs", + "ports": [], + "provenance": { + "repository": "processor_322879060d7cebd34152854cdfc8b52a", + "revision": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd", + "source": null + } + } + ], + "schema_version": 1 + }, + "reason": "No native capture available; build driver may not support inspection", + "schema_version": 1, + "state": "missing" + }, + "instance": { + "build": { + "lowering_version": 2, + "native_sha256": "3f4046e61e7fbbf29a038150aec2babe5ef9808c16ebc46173f46c5ccc1057db", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "composition": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "generated_source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9", + "inputs": { + "source": "Input_source", + "unused": "Input_unused" + }, + "lowering_version": 2, + "nodes": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "outputs": { + "echo": "Output_echo" + }, + "relations": { + "Input_source": { + "fields": [ + "int", + "string" + ], + "input": "source", + "kind": "external_input", + "owner": null, + "scope": "" + }, + "Input_unused": { + "fields": [ + "string" + ], + "input": "unused", + "kind": "external_input", + "owner": null, + "scope": "" + }, + "Module0_echo": { + "kind": "processor_relation", + "node": "child", + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "relation": "echo", + "schema": { + "fields": [ + "int", + "string" + ], + "input": false + }, + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + }, + "Module0_source": { + "alias_of": "Input_source", + "kind": "processor_relation", + "node": "child", + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "relation": "source", + "schema": { + "fields": [ + "int", + "string" + ], + "input": true + }, + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + }, + "Module0_unused": { + "alias_of": "Input_unused", + "kind": "processor_relation", + "node": "child", + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "relation": "unused", + "schema": { + "fields": [ + "string" + ], + "input": true + }, + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + }, + "Output_echo": { + "fields": [ + "int", + "string" + ], + "from": { + "node": "child", + "relation": "echo" + }, + "kind": "external_output", + "output": "echo", + "owner": null, + "scope": "" + } + }, + "rules": [ + { + "head": "echo", + "kind": "processor_rule", + "name": null, + "node": "child", + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "rule": 0, + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + }, + { + "from": { + "node": "child", + "relation": "echo" + }, + "kind": "output_binding", + "output": "echo", + "owner": null, + "scope": "" + } + ] + }, + "health": "ready", + "instance_id": "26065-1790702406512034000-0", + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "program_version": 1, + "revision": 2, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "managed_processes": [ + { + "descendants_included": false, + "role": "native", + "sample": { + "cpu_basis": "ps_decaying_average", + "cpu_percent": 5.9, + "max_age_ms": 5000, + "pid": 34952, + "resident_bytes": 20774912, + "sampled_at_unix_ms": 1790702591738, + "scope": "process", + "state": "available" + } + } + ], + "owner": { + "kind": "embedded", + "memory_attribution": "shared_not_attributable", + "pid": 26065 + }, + "persistence": { + "checkpoints": [ + { + "availability": "present_unverified", + "error": null, + "receipt": { + "checkpoint_sha256": "bb81bae183e1ea1d576a000ab83b97ce38fba6d20778f09ba7891271e8963991", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "a925df9813475fef0d33760ad7d10819ebbbf77e6242c50c1192c98dceb062d2", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "26065-1790702406512034000-0" + }, + "program": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "Input_source" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "Output_echo" + } + ], + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "published_at_unix_ms": 1790702514186, + "receipt_id": "26065-1790702514127827000-4", + "schema_version": 1 + }, + "status": "published" + }, + { + "availability": "present_unverified", + "error": null, + "receipt": { + "checkpoint_sha256": "ab0cc8ddf9fa8cfb27ac8dbc34e49c1c1fc2f165312a0ab880f2ca7205464007", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "a925df9813475fef0d33760ad7d10819ebbbf77e6242c50c1192c98dceb062d2", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "26065-1790702406512034000-0" + }, + "program": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "Input_source" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "Output_echo" + } + ], + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "published_at_unix_ms": 1790702514410, + "receipt_id": "26065-1790702514292951000-5", + "schema_version": 1 + }, + "status": "published" + }, + { + "availability": "catalog_unverified", + "error": null, + "receipt": { + "checkpoint_sha256": "764a265aaee4b5b75ea1a6a7caa80ecff56478af791b448716c66fa7fc3070f9", + "format": "iceberg", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "a925df9813475fef0d33760ad7d10819ebbbf77e6242c50c1192c98dceb062d2", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "26065-1790702406512034000-0" + }, + "program": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "Input_source" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "Output_echo" + } + ], + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "published_at_unix_ms": 1790702515183, + "receipt_id": "26065-1790702514478248000-6", + "schema_version": 1, + "storage": { + "envelope_sha256": "b5737b4a612861c41891dbe117333cb29f3ee7241d18a57017106edf3830d641", + "object_sha256": "d94f8054da7c9aceef7cf37b9a558a62efa25d1c0a5f8a2d2e872af567cea94d", + "object_uri": "file:///private/var/folders/qq/pxfnjsk517x5rc0m044m3fnm0000gn/T/managed-iceberg-26065-0/local store/warehouse/objects/26065-1790702514478248000-6.parquet", + "profile": "local", + "profile_sha256": "d8b20caf7a1de92fd19928eee0690efca12458f671b51668f00f01e1ae5226d7", + "snapshot_id": "3670372088214088639", + "table": [ + "runtime", + "checkpoints" + ], + "table_uuid": "01a0ee2f-f167-73c1-bc65-8d64b9a01c46" + } + }, + "status": "published" + } + ], + "committed_revision": 2, + "coverage": "whole_pure_program_inputs", + "error": null, + "format": "json", + "iceberg": { + "admission_durability": "json", + "error": null, + "job": null, + "profile": "local", + "publication_ack": "catalog_visibility", + "published_revision": 2, + "schema_version": 1, + "status": "configured" + }, + "persisted_revision": null, + "restore_requested": { + "checkpoint_sha256": "764a265aaee4b5b75ea1a6a7caa80ecff56478af791b448716c66fa7fc3070f9", + "format": "iceberg", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "a925df9813475fef0d33760ad7d10819ebbbf77e6242c50c1192c98dceb062d2", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "26065-1790702406512034000-0" + }, + "program": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "Input_source" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "Output_echo" + } + ], + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "published_at_unix_ms": 1790702515183, + "receipt_id": "26065-1790702514478248000-6", + "schema_version": 1, + "storage": { + "envelope_sha256": "b5737b4a612861c41891dbe117333cb29f3ee7241d18a57017106edf3830d641", + "object_sha256": "d94f8054da7c9aceef7cf37b9a558a62efa25d1c0a5f8a2d2e872af567cea94d", + "object_uri": "file:///private/var/folders/qq/pxfnjsk517x5rc0m044m3fnm0000gn/T/managed-iceberg-26065-0/local store/warehouse/objects/26065-1790702514478248000-6.parquet", + "profile": "local", + "profile_sha256": "d8b20caf7a1de92fd19928eee0690efca12458f671b51668f00f01e1ae5226d7", + "snapshot_id": "3670372088214088639", + "table": [ + "runtime", + "checkpoints" + ], + "table_uuid": "01a0ee2f-f167-73c1-bc65-8d64b9a01c46" + } + }, + "restored_from": { + "checkpoint_sha256": "764a265aaee4b5b75ea1a6a7caa80ecff56478af791b448716c66fa7fc3070f9", + "format": "iceberg", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "a925df9813475fef0d33760ad7d10819ebbbf77e6242c50c1192c98dceb062d2", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "26065-1790702406512034000-0" + }, + "program": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "Input_source" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "Output_echo" + } + ], + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "published_at_unix_ms": 1790702515183, + "receipt_id": "26065-1790702514478248000-6", + "schema_version": 1, + "storage": { + "envelope_sha256": "b5737b4a612861c41891dbe117333cb29f3ee7241d18a57017106edf3830d641", + "object_sha256": "d94f8054da7c9aceef7cf37b9a558a62efa25d1c0a5f8a2d2e872af567cea94d", + "object_uri": "file:///private/var/folders/qq/pxfnjsk517x5rc0m044m3fnm0000gn/T/managed-iceberg-26065-0/local store/warehouse/objects/26065-1790702514478248000-6.parquet", + "profile": "local", + "profile_sha256": "d8b20caf7a1de92fd19928eee0690efca12458f671b51668f00f01e1ae5226d7", + "snapshot_id": "3670372088214088639", + "table": [ + "runtime", + "checkpoints" + ], + "table_uuid": "01a0ee2f-f167-73c1-bc65-8d64b9a01c46" + } + }, + "schema_version": 1, + "status": "configured", + "verification": "Receipt/object presence only; restore verifies integrity and pinned dependencies before compilation" + }, + "resources": { + "cpu_basis": "ps_decaying_average", + "cpu_percent": 5.9, + "max_age_ms": 5000, + "pid": 34952, + "resident_bytes": 20774912, + "sampled_at_unix_ms": 1790702591734, + "scope": "process", + "state": "available" + }, + "revision": 2, + "schema_version": 1, + "started_at_unix_ms": 1790702515278, + "state": "running", + "test": null, + "workers": [] + }, + "schema_version": 1, + "stage_catalog_invisible": true, + "staged": { + "availability": "catalog_unverified", + "error": null, + "receipt": { + "checkpoint_sha256": "764a265aaee4b5b75ea1a6a7caa80ecff56478af791b448716c66fa7fc3070f9", + "format": "iceberg", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "a925df9813475fef0d33760ad7d10819ebbbf77e6242c50c1192c98dceb062d2", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "26065-1790702406512034000-0" + }, + "program": { + "dependencies": { + "child": { + "processor_id": "processor_36ae64ccc6c1e3291a8a85993d33ada5", + "version": "sha256:a89fe744ac778f7e9dda58280f83fc68c132a3daa6351e7e1e5ec2aabb704bd6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_322879060d7cebd34152854cdfc8b52a", + "version": "sha256:3918cae10d0690e4e70d38297c559f70ca734eb7f156625eaaf1faa4442d42cd" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "Input_source" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "Output_echo" + } + ], + "source_sha256": "4477586936bf4c7db619a07523ae0e1c40692abc72968c9c60db38f59e9408b9" + }, + "published_at_unix_ms": null, + "receipt_id": "26065-1790702514478248000-6", + "schema_version": 1, + "storage": { + "envelope_sha256": "b5737b4a612861c41891dbe117333cb29f3ee7241d18a57017106edf3830d641", + "object_sha256": "d94f8054da7c9aceef7cf37b9a558a62efa25d1c0a5f8a2d2e872af567cea94d", + "object_uri": "file:///private/var/folders/qq/pxfnjsk517x5rc0m044m3fnm0000gn/T/managed-iceberg-26065-0/local store/warehouse/objects/26065-1790702514478248000-6.parquet", + "profile": "local", + "profile_sha256": "d8b20caf7a1de92fd19928eee0690efca12458f671b51668f00f01e1ae5226d7", + "snapshot_id": null, + "table": [ + "runtime", + "checkpoints" + ], + "table_uuid": "01a0ee2f-f167-73c1-bc65-8d64b9a01c46" + } + }, + "schema_version": 1, + "status": "staged" + }, + "unpublished_mutation_excluded": true, + "update_retraction": true + }, + "additional_passed_assertions": [ + "tampered object fails before native compilation", + "numeric snapshot IDs rejected; decimal string survives exact receipt validation", + "staged receipt is not restorable", + "duplicate frozen effect reservation after restore is rejected", + "pending publication cannot acknowledge restore before job completion", + "deadline/cancellation preserve retained receipt and newer lifecycle", + "missing Iceberg catalog does not disable JSON durable admission" + ], + "limits": [ + "Catalog visibility is distinct from JSON durable admission and is not power-loss certification.", + "One configured local profile and one staging/publication job per owner; restores use normal world startup.", + "No remote R2, provider call, active acceptance store, deployment or full HHMM acceptance is claimed.", + "Fixture object URIs identify temporary test state, removed after successful cleanup.", + "Runtime does not serialize independent world build drivers." + ] +} diff --git a/docs/evidence/managed-json-runtime.json b/docs/evidence/managed-json-runtime.json new file mode 100644 index 0000000..e55e50b --- /dev/null +++ b/docs/evidence/managed-json-runtime.json @@ -0,0 +1,290 @@ +{ + "schema_version": 1, + "recorded_date": "2026-09-29", + "scope": "Bounded managed JSON checkpoint and explicit asynchronous restore; local candidate only", + "status": "passed", + "host_rust": "1.94.1", + "native_ddlog": "1.2.3", + "native_rust": "1.65.0", + "source_sha256": { + "src/world_persistence.rs": "396fb90a987d1dad6f3cb344fd99e45f53a738d6093d6cc5ff9ed6eb850c2628", + "src/worlds.rs": "ffe00f120a8ec02751a2592b0b501d89582e8440a5a6fe2ae84a14a7aa23d266", + "src/instance.rs": "eaa68b6ba0865e01c7ad21b0236795268d7d29b489188964bc4e2178d8a749f9", + "src/checkpoint.rs": "4f05f81cf2ee596ef13042cd518826d4e162f66e33983103d2027cf53f064a3c", + "src/lib.rs": "2b37bf482a72528403a95a1fc7b948f84cba18b8b2107e38b74737327a249b69", + "src/bin/ddlog-worlds.rs": "fb11f63defe6175a8abd94f50a6996106263fc4dd864f418cebaeb5ecf48440d", + "tests/worlds/persistence.rs": "fca0082ba57d6559f32e7aa3f9b3ae55c79d547ce766eea6145b27381d220134", + "tests/worlds.rs": "65af9759a7a18e4178a826b293c910b217b8a708ad40d9982e05871b2be85556", + "tests/worlds_stdio.rs": "f166bdbf90909227a344fd0008718b8c468aed3a2595f5b8fb5b227860cfb8b0", + "tests/worlds_native.rs": "9ff364b1ecb4d86841db749e599c682b981f4937fe2edfabc3f6d1b17ae32d28", + "docs/managed-checkpoints.md": "dce90d2d25b1a514aca80c2b886dcf71a138a4e0fd21af812f53817e12f5958a", + "docs/worlds.md": "dddc22925d24113b35d352c95c6ade08ded317d7b67c64b58ebb604d06dbb967", + "docs/checkpoints.md": "bbb3729e22e99ae15734c2f5b93b48c6b40ff2f84c191b90cb8345789cace498", + "docs/specification.md": "b42a93070ff52b0553fb131cb5d2905e93b6a766103a09f55908f437e362a8b6" + }, + "focused_tests": { + "passed": 72, + "ignored": 2, + "features": [ + "mcp" + ], + "suites": [ + "fixture_hashes", + "lowering_v2", + "memory_runtime", + "processor_composition_registry", + "registered_requests", + "upstream_compatibility", + "worlds", + "worlds_stdio" + ], + "note": "Focused suites passed; new tests and additive persistence schema stamp were checked again. The two ignored tests are separate lowering measurement/native gates." + }, + "checks": [ + "cargo fmt --all --check", + "cargo clippy --locked --offline --features mcp --all-targets -- -D warnings", + "git diff --check" + ], + "native": { + "passed": 1, + "cases": 2, + "fresh_compilations": 6, + "mode": "fresh native compilation", + "test": "worlds_native::managed_json_restore_recomputes_public_state_and_preserves_process_capture", + "elapsed_seconds": 151.29, + "native_tested_world_persistence_sha256": "7c74b69f11c9263772fc585b12032685b402a74e289d407f9ce175b094b2bcca", + "subsequent_change": "Added schema_version:1 to status.persistence; verified by focused contract tests.", + "target": "/private/tmp/hhmm-x0-20260929/runtime-native-target", + "test_stores": "Created independently and removed after verification; retained receipts are test evidence, not available recovery objects.", + "evidence": [ + { + "receipt": { + "checkpoint_sha256": "eb18df800b74497e56d696b26b8f67bd75af2dcfe0b4fe14725fff25ff64d0df", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "70dffc7fe6e97473d0a94d834ec44d2eef4edd1e7372608598298cf0454676d9", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "bb4fd2460e4999911802b5dd9b3f8e258f0931e59f9a32cacb8b7bbd4bf765f2" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "42702-1790698922324295000-0" + }, + "program": { + "dependencies": {}, + "lowering_version": 2, + "processor": { + "processor_id": "processor_96e29a44ed01861aaaaec6ae8e7b0a61", + "version": "sha256:8764140061b44b98a2cd166e54f0eb044ea40f118e29ba7d43bbd019b6b4bbc6" + }, + "public_relations": [ + { + "fields": [ + "int", + "int" + ], + "input": true, + "name": "edge", + "physical": "edge" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "unused" + }, + { + "fields": [ + "int", + "int" + ], + "input": false, + "name": "reach", + "physical": "reach" + } + ], + "source_sha256": "bb4fd2460e4999911802b5dd9b3f8e258f0931e59f9a32cacb8b7bbd4bf765f2" + }, + "published_at_unix_ms": 1790698958040, + "receipt_id": "42702-1790698958006424000-2", + "schema_version": 1 + }, + "restored_build": { + "lowering_version": 2, + "native_sha256": "317123e6da3aa41179292884cd49e85b1070abf9580f54dd8b71be2f9d212f59", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "bb4fd2460e4999911802b5dd9b3f8e258f0931e59f9a32cacb8b7bbd4bf765f2" + }, + "restored_revision": 2, + "rows_before": [ + [ + 1, + 2 + ], + [ + 1, + 3 + ], + [ + 2, + 3 + ] + ], + "rows_after": [ + [ + 1, + 2 + ], + [ + 1, + 5 + ], + [ + 2, + 5 + ] + ], + "capture_state": "available" + }, + { + "receipt": { + "checkpoint_sha256": "922ba5c2a81b320d4305f01115e6bd83a1edf1609751e2151250aff202883c16", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "f256189a253d7fc61f742d7cf970c709c8f69efa457df19704bb7674189f64b0", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "b4852749ed9edce76f738a56c72bdf5ff5fa06dae5472130edb1f404ff69953c" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "42702-1790699009397731000-5" + }, + "program": { + "dependencies": { + "graph": { + "processor_id": "processor_96e29a44ed01861aaaaec6ae8e7b0a61", + "version": "sha256:8764140061b44b98a2cd166e54f0eb044ea40f118e29ba7d43bbd019b6b4bbc6" + } + }, + "lowering_version": 2, + "processor": { + "processor_id": "processor_d927730652b60ab47b40e822cc434269", + "version": "sha256:bcced04e2efad1746a10fffae73c8a61a507f6dcf4570da88e06b83d200317a1" + }, + "public_relations": [ + { + "fields": [ + "int", + "int" + ], + "input": true, + "name": "edge", + "physical": "Input_edge" + }, + { + "fields": [ + "string" + ], + "input": true, + "name": "unused", + "physical": "Input_unused" + }, + { + "fields": [ + "int", + "int" + ], + "input": false, + "name": "reach", + "physical": "Output_reach" + } + ], + "source_sha256": "b4852749ed9edce76f738a56c72bdf5ff5fa06dae5472130edb1f404ff69953c" + }, + "published_at_unix_ms": 1790699030561, + "receipt_id": "42702-1790699030514400000-7", + "schema_version": 1 + }, + "restored_build": { + "lowering_version": 2, + "native_sha256": "ebfa45bb67c623d2ac3ca6c2dfdc2364e9c1206ef715140c5906024924404284", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "b4852749ed9edce76f738a56c72bdf5ff5fa06dae5472130edb1f404ff69953c" + }, + "restored_revision": 2, + "rows_before": [ + [ + 1, + 2 + ], + [ + 1, + 3 + ], + [ + 2, + 3 + ] + ], + "rows_after": [ + [ + 1, + 2 + ], + [ + 1, + 5 + ], + [ + 2, + 5 + ] + ], + "capture_state": "available" + } + ] + }, + "provider_calls": 0, + "active_store_access": false, + "owner_replacement": false, + "commits_or_publication": false, + "integration_limits": [ + "HHMM/X0 application acceptance remains an integration gate; native tests here use a pure reachability program and a composition.", + "No worker supervisor, durable effect claims, record importer or Iceberg catalog configuration was added.", + "Build drivers execute concurrently across worlds. Shared native Cargo target program_cli copy can race; use per-project targets or serialize complete driver calls.", + "Checkpoint capture/publication is synchronous with a 64 MiB JSON bound. Restore compilation/replay is asynchronous.", + "Receipt availability is presence/type/size only until explicit restore validates integrity and pins." + ] +} diff --git a/docs/evidence/managed-workers-runtime.json b/docs/evidence/managed-workers-runtime.json new file mode 100644 index 0000000..1bf819f --- /dev/null +++ b/docs/evidence/managed-workers-runtime.json @@ -0,0 +1,612 @@ +{ + "schema_version": 1, + "recorded_date": "2026-09-29", + "scope": "Generic fenced durable admission and runtime-owned trusted external workers; local candidate only", + "status": "passed", + "source_sha256": { + "src/world_workers.rs": "c056602e755099dcd4153ad28e6f24e663650e4307280fe00f859ef6fae026cd", + "src/world_admission.rs": "86490f4f5f9e7348789f9cae96abcebc1b85cc0e4bf1ee924011e287b7c69925", + "src/world_persistence.rs": "81130ecc86fccb50289b87c5d39ef577d99ecabdcd4ab0f3006dd6192fb6b74e", + "src/worlds.rs": "1a8bec81cf8bae8f744fd1ef977e63a3eab9397472d3bb8aa0efdcc2d1e2d867", + "src/instance.rs": "a9c2dd6ad061608e23bc62ca984ea42c6174e75585e8dba7a18f497768f2f835", + "src/bin/ddlog-worlds.rs": "3cb8d794da92d1959f4ce5afaf07e5767104dc800949a51e3a43bfc0b18ad78e", + "src/processes.rs": "97fa41b7d31a5ccfadf1efb506776845c4926d4dd4f19adf2351480725ae206d", + "tests/worlds/admission.rs": "83e03ebdbf095bed49a1bf3fabbdb7d9442d9d8a20726003bd3bf639672f77dc", + "tests/worlds/workers.rs": "58dee1ee097a8554101f0d4a360d120a2efd90a25f4e986108aaeed70d8201ff", + "tests/workers_socket.rs": "115d0b33b032cfc2cb982a24048064089a5a6ed00b88cb2ae1caaeeddb53a114", + "tests/fixtures/managed_worker.py": "bda8767163996ff40eec485383fd5ebe68024beb6b2af8e2de1eb42a5890e264", + "docs/managed-workers.md": "76164737cfc2c7e6218b6ab873ba9d7f63108ac58a9ad237f0725a8273a8ceb4", + "docs/managed-checkpoints.md": "14e4d0682bc686c016c6b10be628f8c0208171e4708f09dc1e072e08cbd4508c" + }, + "validation": { + "fmt": "passed (owned Rust paths, skip_children=true)", + "clippy": "passed --offline --lib --bin ddlog-worlds --test worlds --test worlds_stdio --test workers_socket -- -D warnings", + "diff_check": "passed", + "focused_tests": { + "worlds": { + "passed": 35, + "seconds": 32.47 + }, + "worlds_stdio": { + "passed": 2, + "seconds": 2.04 + }, + "workers_socket_simulated": { + "passed": 1, + "seconds": 1.66 + }, + "workers_socket_native": { + "passed": 1, + "seconds": 46.95 + } + }, + "host_target": "/private/tmp/hhmm-x0-20260929/runtime-target", + "native_target": "/private/tmp/hhmm-x0-20260929/runtime-native-target", + "native_ddlog": "1.2.3", + "native_rust": "1.65.0", + "provider_calls": 0 + }, + "proof": { + "cleanup": "owner, worker descendant and native exited", + "competing_admissions": [ + { + "admission_key": "contender-0", + "applied_revision": null, + "effect_authorized": false, + "error": "Expected revision mismatch; request not applied", + "generation": 1, + "id": "12544-1790700361690018000-0", + "publication": "not_attempted", + "receipt": null, + "replayed": false, + "schema_version": 1, + "state": "not_applied" + }, + { + "admission_key": "contender-1", + "applied_revision": 4, + "effect_authorized": false, + "error": null, + "generation": 1, + "id": "12544-1790700361690018000-0", + "publication": "published", + "receipt": { + "checkpoint_sha256": "6ced754c27d0cabe466f29406f4f5d6cb23a0564a3d05fcd5030e7d0300ec0f3", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "c562d29968a76bc0aa6566764f3c03ab39972796226d3acdb238ce388e342cbc", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "generation": 1, + "program_version": 1, + "revision": 4, + "world_id": "12544-1790700361690018000-0" + }, + "program": { + "dependencies": {}, + "lowering_version": 2, + "processor": { + "processor_id": "processor_3c1b51e4abbd4a8b3e0ae8296aeed9b0", + "version": "sha256:746b2572edc03adc67a08467e65914b6823d3b410f049e1523106cc96bf51647" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "source" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "echo" + } + ], + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "published_at_unix_ms": 1790700384467, + "receipt_id": "12544-1790700384438324000-5", + "schema_version": 1 + }, + "replayed": false, + "schema_version": 1, + "state": "durable" + } + ], + "native": true, + "reserved": { + "admission_key": "attached.reserve", + "applied_revision": 2, + "effect_authorized": true, + "error": null, + "generation": 1, + "id": "12544-1790700361690018000-0", + "publication": "published", + "receipt": { + "checkpoint_sha256": "8b266759e37343ee7fb183ed96ef57ffdf279118129bfbf5911714a80e3e6da2", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "c562d29968a76bc0aa6566764f3c03ab39972796226d3acdb238ce388e342cbc", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "12544-1790700361690018000-0" + }, + "program": { + "dependencies": {}, + "lowering_version": 2, + "processor": { + "processor_id": "processor_3c1b51e4abbd4a8b3e0ae8296aeed9b0", + "version": "sha256:746b2572edc03adc67a08467e65914b6823d3b410f049e1523106cc96bf51647" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "source" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "echo" + } + ], + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "published_at_unix_ms": 1790700384109, + "receipt_id": "12544-1790700384074674000-3", + "schema_version": 1 + }, + "replayed": false, + "schema_version": 1, + "state": "durable" + }, + "restored": { + "checkpoints": [ + { + "availability": "present_unverified", + "error": null, + "receipt": { + "checkpoint_sha256": "8b266759e37343ee7fb183ed96ef57ffdf279118129bfbf5911714a80e3e6da2", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "c562d29968a76bc0aa6566764f3c03ab39972796226d3acdb238ce388e342cbc", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "12544-1790700361690018000-0" + }, + "program": { + "dependencies": {}, + "lowering_version": 2, + "processor": { + "processor_id": "processor_3c1b51e4abbd4a8b3e0ae8296aeed9b0", + "version": "sha256:746b2572edc03adc67a08467e65914b6823d3b410f049e1523106cc96bf51647" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "source" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "echo" + } + ], + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "published_at_unix_ms": 1790700384109, + "receipt_id": "12544-1790700384074674000-3", + "schema_version": 1 + }, + "status": "published" + }, + { + "availability": "present_unverified", + "error": null, + "receipt": { + "checkpoint_sha256": "07670bd090404a6c36e43c5afad0bf9dc867873e9b1b516e4dc80bf8efcb1eaf", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "c562d29968a76bc0aa6566764f3c03ab39972796226d3acdb238ce388e342cbc", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "generation": 1, + "program_version": 1, + "revision": 3, + "world_id": "12544-1790700361690018000-0" + }, + "program": { + "dependencies": {}, + "lowering_version": 2, + "processor": { + "processor_id": "processor_3c1b51e4abbd4a8b3e0ae8296aeed9b0", + "version": "sha256:746b2572edc03adc67a08467e65914b6823d3b410f049e1523106cc96bf51647" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "source" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "echo" + } + ], + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "published_at_unix_ms": 1790700384316, + "receipt_id": "12544-1790700384288326000-4", + "schema_version": 1 + }, + "status": "published" + }, + { + "availability": "present_unverified", + "error": null, + "receipt": { + "checkpoint_sha256": "6ced754c27d0cabe466f29406f4f5d6cb23a0564a3d05fcd5030e7d0300ec0f3", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "c562d29968a76bc0aa6566764f3c03ab39972796226d3acdb238ce388e342cbc", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "generation": 1, + "program_version": 1, + "revision": 4, + "world_id": "12544-1790700361690018000-0" + }, + "program": { + "dependencies": {}, + "lowering_version": 2, + "processor": { + "processor_id": "processor_3c1b51e4abbd4a8b3e0ae8296aeed9b0", + "version": "sha256:746b2572edc03adc67a08467e65914b6823d3b410f049e1523106cc96bf51647" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "source" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "echo" + } + ], + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "published_at_unix_ms": 1790700384467, + "receipt_id": "12544-1790700384438324000-5", + "schema_version": 1 + }, + "status": "published" + } + ], + "committed_revision": 2, + "coverage": "whole_pure_program_inputs", + "error": null, + "format": "json", + "persisted_revision": 2, + "restore_requested": { + "checkpoint_sha256": "8b266759e37343ee7fb183ed96ef57ffdf279118129bfbf5911714a80e3e6da2", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "c562d29968a76bc0aa6566764f3c03ab39972796226d3acdb238ce388e342cbc", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "12544-1790700361690018000-0" + }, + "program": { + "dependencies": {}, + "lowering_version": 2, + "processor": { + "processor_id": "processor_3c1b51e4abbd4a8b3e0ae8296aeed9b0", + "version": "sha256:746b2572edc03adc67a08467e65914b6823d3b410f049e1523106cc96bf51647" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "source" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "echo" + } + ], + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "published_at_unix_ms": 1790700384109, + "receipt_id": "12544-1790700384074674000-3", + "schema_version": 1 + }, + "restored_from": { + "checkpoint_sha256": "8b266759e37343ee7fb183ed96ef57ffdf279118129bfbf5911714a80e3e6da2", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "c562d29968a76bc0aa6566764f3c03ab39972796226d3acdb238ce388e342cbc", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "generation": 1, + "program_version": 1, + "revision": 2, + "world_id": "12544-1790700361690018000-0" + }, + "program": { + "dependencies": {}, + "lowering_version": 2, + "processor": { + "processor_id": "processor_3c1b51e4abbd4a8b3e0ae8296aeed9b0", + "version": "sha256:746b2572edc03adc67a08467e65914b6823d3b410f049e1523106cc96bf51647" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "source" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "echo" + } + ], + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "published_at_unix_ms": 1790700384109, + "receipt_id": "12544-1790700384074674000-3", + "schema_version": 1 + }, + "schema_version": 1, + "status": "configured", + "verification": "Receipt/object presence only; restore verifies integrity and pinned dependencies before compilation" + }, + "schema_version": 1, + "settled": { + "admission_key": "attached.settle", + "applied_revision": 3, + "effect_authorized": false, + "error": null, + "generation": 1, + "id": "12544-1790700361690018000-0", + "publication": "published", + "receipt": { + "checkpoint_sha256": "07670bd090404a6c36e43c5afad0bf9dc867873e9b1b516e4dc80bf8efcb1eaf", + "format": "json", + "origin": { + "build": { + "lowering_version": 2, + "native_sha256": "c562d29968a76bc0aa6566764f3c03ab39972796226d3acdb238ce388e342cbc", + "program_version": 1, + "runtime": { + "commit": "1f9f417824bac951d7fa644bba63b082759f94d6", + "crate_version": "0.1.0", + "dirty": true, + "schema_version": 1 + }, + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "generation": 1, + "program_version": 1, + "revision": 3, + "world_id": "12544-1790700361690018000-0" + }, + "program": { + "dependencies": {}, + "lowering_version": 2, + "processor": { + "processor_id": "processor_3c1b51e4abbd4a8b3e0ae8296aeed9b0", + "version": "sha256:746b2572edc03adc67a08467e65914b6823d3b410f049e1523106cc96bf51647" + }, + "public_relations": [ + { + "fields": [ + "int", + "string" + ], + "input": true, + "name": "source", + "physical": "source" + }, + { + "fields": [ + "int", + "string" + ], + "input": false, + "name": "echo", + "physical": "echo" + } + ], + "source_sha256": "14e803dbcf6e595bf20581dd1d925f4195dbbeff9b2709a6eb20ab76f4eb9f22" + }, + "published_at_unix_ms": 1790700384316, + "receipt_id": "12544-1790700384288326000-4", + "schema_version": 1 + }, + "replayed": false, + "schema_version": 1, + "state": "durable" + }, + "worker": { + "error": null, + "exit_code": 0, + "finished_at_unix_ms": 1790700384380, + "generation": 1, + "key": "attached", + "last_resources": { + "cpu_basis": "ps_decaying_average", + "cpu_percent": 0.0, + "max_age_ms": 5000, + "pid": 13657, + "resident_bytes": 32768, + "sampled_at_unix_ms": 1790700383928, + "scope": "process", + "state": "available" + }, + "outcome": { + "code": "fixture_ok", + "schema_version": 1, + "status": "completed" + }, + "pid": 13657, + "profile": "fixture", + "profile_sha256": "89258486f569109c562474c223a531ae19d3ddf35483badafdfd37d8f8d34630", + "request_sha256": "501e7f991db893fe06452a3f3a9b9306366867c6d113f85ab8419f30cd69841c", + "resources": { + "cpu_basis": "ps_decaying_average", + "cpu_percent": null, + "max_age_ms": 5000, + "pid": null, + "resident_bytes": null, + "sampled_at_unix_ms": 1790700384390, + "scope": "process", + "state": "missing" + }, + "schema_version": 1, + "started_at_unix_ms": 1790700383916, + "state": "completed", + "worker_id": "12544-1790700383916408000-2" + } + }, + "limits": [ + "Native receipts and process samples are historical fixture evidence; temporary test stores were removed after assertions.", + "Reservation authority is fresh-response-only. Lost responses, publication errors and uncertain outcomes never authorize automatic effect retry.", + "Restoring a checkpoint preceding an effect or explicitly starting fresh is not an exactly-once guarantee across that rollback.", + "Cleanup covers world/worker cancellation and graceful owner shutdown; uncatchable owner death cannot run cleanup. Recovery never adopts or signals stored PIDs.", + "Same-user workers are trusted, not a hostile-client sandbox; they must carry worker_id and use the fenced APIs.", + "Parent-owned registry/library importer, Python client, and native build-lock changes are separate integration work." + ] +} diff --git a/docs/evidence/runtime-final-candidate.json b/docs/evidence/runtime-final-candidate.json new file mode 100644 index 0000000..aa0efea --- /dev/null +++ b/docs/evidence/runtime-final-candidate.json @@ -0,0 +1,114 @@ +{ + "schema_version": 1, + "recorded_at_utc": "2026-09-29T17:33:07.119043+00:00", + "status": "passed", + "artifacts": { + "ddlog-worlds": { + "path": "/private/tmp/hhmm-x0-20260929/ddlog-worlds-final-iceberg-mcp", + "sha256": "2d17fb1c2fd2dab8c8d4d491a63b5814a2c6ce127e00efb210f95313ff37b891" + }, + "lemmalog-ddlog-mcp": { + "path": "/private/tmp/hhmm-x0-20260929/lemmalog-ddlog-mcp-final-iceberg-mcp", + "sha256": "7b3d7262400403c731395f29cd00e64c29ea06be84f27dd936e563389bda77e4" + }, + "python_wheel": { + "path": "/private/tmp/hhmm-x0-20260929/runtime-final-wheels/ddlog_runtime_client-0.1.0-py3-none-any.whl", + "sha256": "5836777c90c7d5e2425b2624b98b02b05998acf20938fcef238661bb6b4ac0a5" + } + }, + "host_rust": "1.95.0", + "features": [ + "iceberg", + "mcp" + ], + "source_manifest_sha256": "4835780aad62825768d09e455d1ae5ba5c0546a167dabc74fb18e55e3ff44837", + "source_sha256": { + "Cargo.lock": "2bda58218dea4e637857109e1539ec19c6931ba36176ea737a2004d158ac2212", + "Cargo.toml": "e806ad15e0818f3980b72d184a5c67963db63c05973d07e1acb6528d649823c1", + "build.rs": "7e9d9d32a756c4b043dd1eb7327e64dfa5ce97305ac01e2f6abacd21a3be20b8", + "crates/syntax/Cargo.toml": "05c7b2ff343d68623a5277675919b0b66e2f82501b1a02e402187a78208a0695", + "crates/syntax/LICENSE": "212da0f1b3ebea42880b338390b99defc8a12a0ac45d0acf6a228280b2cc8652", + "crates/syntax/README.md": "ca4d494d844c69a7040a711294521904a5dced8d6dafae9d944d89cb89c642d0", + "crates/syntax/src/ast.rs": "3240d236916e46bf9ca4844ed7117631e04da4c035cbf69586c0cdf223d99230", + "crates/syntax/src/lib.rs": "e84c428ed4a119a2761ee3fd9542cf7e68808240b5c6718d690975c43e770791", + "crates/syntax/tests/parser.rs": "f261fcd74484891197952cf9fcca8edba13acd009251daf752dc8d8791a17cde", + "python/ddlog_runtime_client/__init__.py": "793a45625f2adc2e6f96dd34b50fd73cacbbf46e31a41956de7b875fbe80afff", + "python/ddlog_runtime_client/transport.py": "818e5d02e4ecbaec85c9789ea85e0a989c1d10bce94276bfcee0670298220b72", + "python/ddlog_runtime_client/world.py": "95bce83368c4da56948b213bdfc640935cb220bf8397a0ae0f9aba007a50f99d", + "python/pyproject.toml": "cbed701c86b2dfdd24eaaae3262fa40a65a4edc9ab57d9d6ffc30dbb20bb5980", + "src/bin/ddlog-worlds.rs": "a7b3a9f61001bd1ee59212deb997ee506dbf84f7109d37704e056b78fae29049", + "src/bin/lemmalog-ddlog-mcp.rs": "b711edfbe92b16f269bb1c2105aa703b08b760a623358dd00f24daea25005cd0", + "src/bin/worlds_socket/mod.rs": "e754b7aa7e6739d2d13fa99a75b2be4b12377f57d11e27fd2d24b5c4a5ef541b", + "src/bounded.rs": "2b98ca0c05e99e3644dffe32c6be17ae5fc4d3419f44a0e47c50e2e26cf25ace", + "src/checkpoint.rs": "4f05f81cf2ee596ef13042cd518826d4e162f66e33983103d2027cf53f064a3c", + "src/composition.rs": "b8459d8b9ffc1a787df51e7c57fb1a704e0cb40e49f639c2cf866652b3cdf6de", + "src/host.rs": "46b2fbac1e9c817bd3995772cddc7628e716629b928cc88391f614c9338772d7", + "src/iceberg_checkpoint.rs": "2568ea8623b5943ea6715e2a6e2107c766e073256182bdd2ec306bd1ac128ea3", + "src/inspection.rs": "b3961aed7f1b22b10315eaf3cd7d44c6fed318f3a4b4371374606b831049e7f0", + "src/instance.rs": "a9c2dd6ad061608e23bc62ca984ea42c6174e75585e8dba7a18f497768f2f835", + "src/lib.rs": "2b37bf482a72528403a95a1fc7b948f84cba18b8b2107e38b74737327a249b69", + "src/lower.rs": "5f3a76f2e98ef4566a4f19e66b15b9595f358040d83b035202bd1e6b9d547d57", + "src/mcp.rs": "94a5e97b5ea49b606966565baf1b7bbaf6bebeb8dc0a546994505bf95bebd477", + "src/operations.rs": "3956034ded42033a547c2544680b466053a1bf6d719af75eafb8f75e2a201463", + "src/processes.rs": "97fa41b7d31a5ccfadf1efb506776845c4926d4dd4f19adf2351480725ae206d", + "src/registry.rs": "88476cf0b01734b41c96e4680ad8c314346bb4e7ae37bc9003b56768284112b2", + "src/rows.rs": "643e6c72260d11106e83b5c6014eed5dc0f527233fa52c71bff97d7051151173", + "src/source_inspection.rs": "3503b293f64cc6a9c724c2b688222162bc8e2c3e486fa7e0e9d645ee80a60972", + "src/star/lemmalog_star.dl": "185d130ea9974275a58bfbc155eac7779f500f5c663e0f39d88383fa5d1c52cb", + "src/star/lemmalog_star.rs": "b6b434bcaea589bee1bb1204700634390d2c2b7909c9158653aeb0a8726f185e", + "src/star.rs": "5cd7088587428bc7018bd0253c8ce46d6c599feb99388394073572045c91f3e2", + "src/telemetry.rs": "b978a57af9aab1a939807a2987924ddff0c99b4ae945e8715751117e741ee694", + "src/world_admission.rs": "86490f4f5f9e7348789f9cae96abcebc1b85cc0e4bf1ee924011e287b7c69925", + "src/world_library.rs": "2d70e30d9ae6a2ac9f3f613b705741220ff612dd19957b65d9281ae3f6f7077e", + "src/world_persistence.rs": "9283e01a4184e31d0dec228edbbe0d06671cf013c447d85626f9f2b69fb31802", + "src/world_storage.rs": "02c073e77af6c331af0fe6b53f38a7986ff0df3c703f9bba06213257161d6e94", + "src/world_workers.rs": "c9111c1a66410ff11fa9a23c2b624b1c4ecbc150501425b9a8f5fbd0d881739e", + "src/worlds.rs": "efbdd17101e2183e8dfec58a15bac9e36dc3d5c82efb62be6d2cb2a44b6d8365" + }, + "changes": [ + "Retain only validated failed worker outcomes/codes on nonzero exit; never promote a nonzero exit to completed.", + "Hold registry writer exclusion across import destination validation, initial current-pointer selection and publication.", + "SDK validates correlated admission identity/state and bounded reads; authority requires a fresh durable reservation; timing callbacks cannot mask outcomes.", + "Correct asynchronous test signaling/capture predicates and update the packaging Cargo fixture to the artifact-report contract.", + "Remove obsolete managed-iceberg-next.md; managed-iceberg.md remains the current managed Iceberg contract." + ], + "review": { + "world_library.rs": "Reviewed exact-record/pin/provenance/scenario prevalidation, association ownership and replay semantics; no additional change needed.", + "registry.rs": "Concurrent import/current-pointer race fixed in shared import_plan.", + "python_sdk": "Admission projection/identity and advisory timing defects fixed; fresh installed wheel tested." + }, + "validation": { + "fmt": "cargo +1.95.0 fmt --all -- --check: passed", + "clippy": "cargo +1.95.0 clippy --offline --workspace --all-features --all-targets -- -D warnings: passed", + "rust_workspace": { + "passed": 145, + "failed": 0, + "ignored": 8, + "log": "/private/tmp/hhmm-x0-20260929/runtime-final-rust-tests.log" + }, + "python_runtime": { + "passed": 80, + "skipped": 1, + "log": "/private/tmp/hhmm-x0-20260929/runtime-final-python-tests.log" + }, + "installed_python_client": { + "passed": 9, + "environment": "/private/tmp/hhmm-x0-20260929/runtime-final-client-venv" + }, + "packaging": "Wheel and sdist built offline with pinned build backend; wheel installed and tested outside source checkout.", + "build": "cargo +1.95.0 build --offline --features iceberg,mcp --bins: passed", + "diff_check": "passed" + }, + "native_evidence": { + "path": "docs/evidence/managed-iceberg-runtime.json", + "candidate_sha256": "10c832981c1a5a3fb150dccc04a9c57daa653bec34af086a31f4808b5772954a", + "scope": "Earlier isolated native composition proof has its own source manifest and candidate identity. Not relabeled as a native run of this final candidate; the final worker/registry/SDK pass uses workspace and subprocess regressions." + }, + "boundaries": { + "provider_calls": 0, + "active_stores_accessed": false, + "existing_owners_modified": false, + "commits_created": 0, + "previous_candidate_replaced": false + } +} diff --git a/docs/iceberg-checkpoints.md b/docs/iceberg-checkpoints.md index 962184b..bca3567 100644 --- a/docs/iceberg-checkpoints.md +++ b/docs/iceberg-checkpoints.md @@ -1,5 +1,9 @@ # Runtime-owned Iceberg checkpoints +For managed worlds, startup configuration, asynchronous operations and exact +public receipts, see [managed local Iceberg](managed-iceberg.md). This document +describes the shared Backend/catalog engine beneath that integration. + The optional `iceberg` feature stores the runtime's complete format-1 state in immutable Parquet and publishes it through an operator-provided Iceberg catalog. It requires Rust 1.95 and pins an Apache Iceberg git revision with Arrow/Parquet diff --git a/docs/library-artifacts.md b/docs/library-artifacts.md new file mode 100644 index 0000000..4171546 --- /dev/null +++ b/docs/library-artifacts.md @@ -0,0 +1,35 @@ +# Registered library artifacts + +`library_import` admits a publisher's exact immutable records, their saved test +scenarios, and display associations without compiling or creating a world. It is +the supported path for independent libraries to appear in a control plane. + +The request is `{artifact, revision, dry_run?}`. `revision` names publisher source +provenance explicitly. An artifact has `schema_version: 1`, `library`, `entries` +and `pins`: + +- `library`: `key`, `name`, `repository`, `package`, `package_version`. +- Each entry: `key`, `name`, `description`, repository-relative `source` paths, + `git_provenance`, the complete registry `record`, and `scenarios`. +- `pins`: entry key to exact `{processor_id, version}`. Each entry must agree with + its record and provenance; duplicate keys or pins are rejected. + +The library ID is stable for its key and repository. The registry verifies record +hashes, lowering versions and transitive composition dependencies before writing. +Scenario changes must match the actual public typed interface. A failed validation +writes nothing. Publication uses the existing immutable registry mechanism, +publishes exact-pin scenario sidecars, and makes the catalog association visible +last. An I/O failure can leave valid records or scenarios; repeating the same +artifact reconciles them. This is not a transaction spanning multiple repositories. + +Reimport preserves existing processor current pointers, historical versions and +world pins. A pin already associated with a different named library conflicts; +the importer never silently transfers its ownership. Definitions absent from a +later artifact are not erased. Catalog labels and scenario sidecars can change; +the immutable program content cannot. + +Replies contain `schema_version`, `dry_run`, `library_id`, exact `pins`, and the +record count. X0's `x0-library register` command produces this generic artifact; +the runtime has no X0-specific schemas or import branch. Other libraries can +publish the same contract. `ProcessorRegistry::import_records` also exposes the +record admission step independently of catalog association. diff --git a/docs/managed-checkpoints.md b/docs/managed-checkpoints.md new file mode 100644 index 0000000..8e5c8a2 --- /dev/null +++ b/docs/managed-checkpoints.md @@ -0,0 +1,161 @@ +# Managed JSON checkpoint contract (version 1) + +`ddlog-worlds` exposes `checkpoint` and `restore` through its existing stdio and +Unix-socket request dispatcher. No new owner, provider, or worker is created. +`src/world_persistence.rs` owns publication, receipt validation, availability and +world-record writes. `ProgramInstance` shares pure-program preparation between +install and restore, including exact registry validation and public-port admission. +`Backend` remains the authority for checkpoint format and acknowledged inputs. + +## Requests and receipts + +- `checkpoint {"id":""}` requires a healthy running instance, with no + pending lifecycle operation. It freezes the current committed revision and + returns the receipt object below only after the object and publication record + have been file-synced, renamed and directory-synced. This first slice performs + bounded synchronous JSON capture/publication (64 MiB format limit); no native + compilation or output scan is involved. There is no latency guarantee for disk IO. +- `restore {"id":"","receipt":}` requires + `created`, `stopped`, `failed` or `interrupted`, with no live instance or pending + start/restore. It verifies the stored receipt and checkpoint, records a new + generation as `starting`, and compiles/replays asynchronously into a fresh + Backend. Poll `status` or `inventory`; `running` confirms successful activation, + while `failed` carries the error. `stop` cancels this compiler/replay through the + same hosted process control as Start. A failed restore never activates partial + inputs. Test worlds restore without re-running scenarios. +- A different created world may restore a receipt from this owner's private store + only when its exact processor pin is identical. Other stores, client filesystem + paths, path traversal and symlinked objects/directories are rejected. +- `start {"id":"..."}` always starts fresh. It never selects a receipt implicitly. + The stored publication history remains visible after fresh Start. + +Library entrypoints are `WorldManager::checkpoint(id) -> Result` +and `WorldManager::restore_async(id, &receipt) -> Result`. +Requests retain the existing `{operation,args}` / `{ok,result|error}` envelope. +There is no automatic command retry or publication retry operation in this slice. +After an error or a lost response, inspect the retained publications; do not assume +that no file was published. Repeating Checkpoint makes a new immutable publication. + +A receipt is an opaque exact-match object, not a filename: + +```json +{ + "schema_version": 1, + "format": "json", + "receipt_id": "", + "program": { + "processor": {"processor_id": "processor_...", "version": "sha256:..."}, + "dependencies": {"": {"processor_id": "processor_...", "version": "sha256:..."}}, + "public_relations": [{"name": "edge", "input": true, "fields": ["int", "int"], "physical": "Input_edge"}], + "lowering_version": 2, + "source_sha256": "" + }, + "origin": { + "world_id": "", + "generation": 1, + "revision": 2, + "program_version": 1, + "build": { + "runtime": {"commit": "", "dirty": true, "crate_version": "0.1.0", "schema_version": 1}, + "native_sha256": "", + "source_sha256": "", + "lowering_version": 2, + "program_version": 1 + } + }, + "checkpoint_sha256": "", + "published_at_unix_ms": 1790000000000 +} +``` + +`dependencies` is empty for a plain program; a composition carries its exact +transitive dependency map. `public_relations` contains all public inputs/outputs +and their actual physical mapping. Origin identity and program identity are also +integrity-bound in the Backend checkpoint's application metadata. The caller +cannot substitute metadata or use the receipt to supply executable source. + +The immutable registry record and dependency closure are validated again. Restore +re-lowers the pin under the receipt's lowering version and compares complete source, +schemas, dependency identity and public interface before invoking the compiler. +Changing a dependency's current pointer does not change the pinned version. +Missing, corrupt or incompatible pins fail closed. Registered-operation programs +and imported native operators remain unsupported by JSON format 1; checkpoint +reports the existing format error. All inputs, including empty relations, survive. + +Restoration recompiles under the operator's current configured driver. The receipt +records the origin executable; it does not require byte-identical recompilation or +certify equivalence of arbitrary compiler versions. The new instance exposes its +own `instance.build`, generation and PID, plus the restored committed revision. +Its capture path and full-detail/rotation options belong to the new generation. +The digest detects corruption, not a malicious operator able to rewrite the entire +private store. Keep the configured native toolchain trusted and pinned. + +## Status, durability and compatibility + +`status.persistence` (also in summary inventory) contains `schema_version: 1` and: + +- `status: configured|error`, `format: json`, `coverage: whole_pure_program_inputs`, + and `error` for a checkpoint/store failure. `configured` means the managed JSON + store is enabled, not that every program is checkpointable or every receipt valid. +- `committed_revision`: current live revision, or null without a healthy instance. +- `persisted_revision`: highest retained published boundary for this generation, + or the boundary explicitly restored into it. Fresh Start does not inherit the + previous generation's persisted revision. Check individual availability before + presenting a receipt as a recovery choice. +- `checkpoints`: retained entries `{receipt,status,availability,error}`. Publication + phases are `staging`, `staged`, `published`, `failed`, `uncertain`; unreadable + entries are `unavailable` and retain their `receipt_id` plus diagnostic. Only + `published` accepts restore. Incomplete publication has no confirmed publication + time; it is never promoted on owner restart. +- `availability: present_unverified|missing_or_invalid` describes the object file. + An unreadable manifest has `availability: invalid`. Summary polling checks file + presence/type/size and reads bounded manifests; it does not hash checkpoint + bodies, query native rows, or resolve registry dependencies. `verification` + states that Restore performs those integrity and pin checks before compilation. +- `restore_requested`: the exact receipt selected for this generation, including + failed/cancelled attempts; `restored_from` is set only after successful replay. + Lifecycle history retains these identities even after a later fresh Start. + +The private layout is +`//checkpoints//{checkpoint.json,publication.json}`. +Clients cannot choose any of those paths. Receipt manifests are capped at 1 MiB. +The publication record is the authority for a durable receipt, independent of a +subsequent `world.json` write; a failed world-record write cannot erase publication +history. Publication errors after a possible rename are uncertain and retained +where storage permits. Missing objects and malformed manifests do not prevent +world inventory or lifecycle operations. + +Old `world.json` records deserialize with empty persistence state. Old Backend +checkpoint bytes and registry fixtures are unchanged and remain validated by the +existing library API. A raw historical Backend checkpoint has no managed receipt, +so it cannot be passed directly to this protocol. No historical world starts or +restores automatically. World records, registry records and the managed receipt +store must all be retained together; build caches are not a replacement for them. + +Coverage describes whole pure program inputs, not analytical storage per relation. +The [managed worker contract](managed-workers.md) adds generic durable admissions +and effect fences using these same receipts. Its optional effect metadata is +integrity-bound within the checkpoint; the receipt schema remains version 1. +The optional [managed Iceberg extension](managed-iceberg.md) reuses this format +and validation with separate asynchronous stage/publication operations. JSON +`persisted_revision` remains independent of Iceberg `published_revision`. +Neither contract adds automatic provider recovery. + +## Verification and integration risk + +`tests/worlds/persistence.rs` uses a simulated native transport to cover publication +failure, owner restart, tampering, missing objects/dependencies, private-path +rejection, composition admission, explicit fresh Start, failed build/replay, hosted +cancellation and observer options. `tests/worlds_stdio.rs` verifies the wire verbs. +`tests/worlds_native.rs::managed_json_restore_recomputes_public_state_and_preserves_process_capture` +requires `DDLOG_RUNTIME_NATIVE_BUILD`: it compiles both a pure program and a +composition, restores the exact public rows into a fresh owner, omits an unpublished +mutation, verifies empty inputs and a subsequent insertion/retraction, checks +process ownership/capture, and proves Start remains fresh. It makes no provider call. + +World start/restore workers do **not** serialize build-driver calls across worlds. +The bundled driver's `build-native-artifact.py` uses content-keyed native targets +and holds the key lock across input verification, Cargo build and publication. +Custom drivers sharing targets must prevent both generated-package freshness +collisions and build/copy races; a copy lock alone does not solve freshness. +Runtime acceptance uses a private native target separate from other lanes. diff --git a/docs/managed-iceberg.md b/docs/managed-iceberg.md new file mode 100644 index 0000000..fa75a31 --- /dev/null +++ b/docs/managed-iceberg.md @@ -0,0 +1,132 @@ +# Managed local Iceberg (version 1) + +Build the owner with `--features iceberg` using Rust 1.95. At startup pass +`--storage-profile /absolute/profile.json`. The trusted same-user regular file +must not be group/world writable and is loaded once: + +```json +{"schema_version":1,"profile":{"name":"local","root":"/absolute/private/iceberg","timeout_ms":30000}} +``` + +There is one profile, one cooperative catalog publisher and one pending staging +or publication job per owner. Restores use the existing world startup lifecycle. +The owner locks the profile root. It derives `catalog.sqlite`, +`warehouse/table` and immutable `warehouse/objects/.parquet` locations; +no request supplies a catalog path or object URI. Timeout defaults to 30 seconds +and must be 1–120000 ms. The local v3 unpartitioned table is `runtime.checkpoints` +with `commit.retry.num-retries=0`. Configuration is startup-only, and JSON/worker +operations still work without this feature or profile. + +## Wire + +The existing request/response and attachment envelopes remain unchanged. + +- `checkpoint_stage {id,expected_generation,expected_revision,profile}` requires + the expected healthy running world. It freezes the existing Backend checkpoint + bytes and ProgramInstance identity under the owner lock, saves publication + intent, then stages Parquet outside the lock. It returns + `{schema_version:1,id,receipt_id,status:"staging"}`. +- `checkpoint_status {id,receipt_id}` returns + `{schema_version:1,receipt,status,availability,error}`. Wait for `staged` before + requesting publication. Staging never makes a catalog snapshot visible. +- `checkpoint_publish {id,receipt}` requires the exact retained staged receipt + (or its exact published form). It returns the same ticket shape with + `status:"publishing"`; poll for `published`. An explicit retry reconciles that + same immutable object using catalog readback. It never commits input changes. +- `restore {id,receipt}` uses the exact **published** receipt. Existing stopped / + created / failed / interrupted gates apply. It returns a new starting generation + immediately. Catalog IO, validation and native compilation run off the owner + lock; failure never activates a partial backend. Poll the world's lifecycle. + Ordinary Start remains fresh. Cross-world restore requires the identical pin. + +World `status.persistence.checkpoints` includes the same entries. Iceberg states +are `staging`, `staged`, `publishing`, `published`, `uncertain` (invalid manifests +are `unavailable`). `availability:"catalog_unverified"` explicitly means status +has not queried the catalog/object. Restore and explicit publication reconciliation +verify real storage. A timeout/cancellation/error may leave uploaded bytes or a +committed snapshot, so the owner retains its exact receipt and error. Interrupted +staging/publication is marked uncertain at startup, never automatically adopted. +A stage interrupted before its staged receipt was retained may leave an orphan; +it cannot be implicitly restaged or published. + +## Exact Iceberg receipt + +JSON version-1 receipts are unchanged. The Iceberg receipt keeps the same +`schema_version:1`, `receipt_id`, `program`, `origin`, `checkpoint_sha256` and +`published_at_unix_ms` fields, with `format:"iceberg"` and this additional object: + +```json +{"storage":{ + "profile":"local", + "profile_sha256":"", + "table":["runtime","checkpoints"], + "table_uuid":"", + "object_uri":"file:///absolute/private/iceberg/warehouse/objects/.parquet", + "object_sha256":"", + "envelope_sha256":"", + "snapshot_id":null +}} +``` + +At `staged`, snapshot ID and publication time are null. At `published`, they name +exact catalog readback and its first confirmed publication time. `snapshot_id` +is a decimal **string**, never a JSON number: Iceberg's 64-bit IDs exceed browser +integer precision. Preserve the string exactly. Table UUID and +object/envelope hashes may be null during initial staging intent only. Keep the +entire returned receipt opaque/exact for publish and restore. The owner's private +manifest also retains the engine's Avro descriptor; it is never accepted from a +client. Profile identity prevents receipts from silently selecting another store. + +`checkpoint_sha256` remains the inner Backend state digest, while +`storage.envelope_sha256` hashes the entire encoded JSON envelope. These digests +are deliberately distinct. Program pins/dependencies, actual build/lowering, +public interfaces, origin revision and effect metadata use the same validation +and hosted fresh-backend restoration as managed JSON. +Capture ingestion keeps its existing asynchronous polling cadence; a world may +be running briefly before `inspection.state` becomes `available`. + +## Visibility and ownership + +`status.persistence.iceberg` has `schema_version:1`, `status` (`configured`, +`not_configured` or feature `unavailable`), `profile`, `publication_ack: +"catalog_visibility"`, `published_revision`, `error`, and `job` (null or +`{id,generation,receipt_id,cancel_requested}`). It also states +`admission_durability:"json"`. The root `persistence.persisted_revision` retains +its JSON durability meaning. JSON checkpoints and `admit_inputs` continue to use +file/directory-synced JSON independently; an Iceberg publication never grants +effect dispatch authority. + +Stop cancels a world's storage job and hosted restore; owner shutdown also cancels +storage work. Cancellation/deadlines do not roll back already dispatched IO. +Completion only records the original immutable publication and cannot replace a +newer world's lifecycle, revision, effects or restore identity. Native queries and +other worlds stay available while storage work waits. No hard latency guarantee +is made for synchronous filesystem calls, input freeze or operating-system IO. + +This slice is local SQLite/local FileIO, whole-program checkpoint transport with +the existing 64 MiB format and 128 MiB snapshot-scan limits. Catalog visibility is +not power-loss certification. No remote R2 proof, analytical per-relation tables, +extra Arrow buffering, distributed fencing, automatic garbage collection or +provider retry is implied. + +## Focused verification + +`tests/managed_iceberg.rs` uses real local SQLite, Parquet and catalog readback. +Its four default tests cover exact receipt reconciliation across owner restart, +unpublished mutation exclusion, tampered bytes, JSON effect-fence preservation, +catalog deadlines, pending-publication acknowledgment, cancellation across newer +generations, store locks, missing catalog and the executable's async wire. + +The opt-in `native_managed_iceberg_composition_recovery` test compiles a pinned +composition plus another native world, then restores into a fresh backend. It +checks public outputs, empty inputs, subsequent update/retraction, capture and +new PID ownership. Another world's status/query are measured while a real SQLite +exclusive lock blocks publication. The test uses only fresh temporary stores. + +Build the host test with Rust 1.95 and `--features iceberg` **before** sourcing +the native compiler environment. Invoke that compiled test executable with +`DDLOG_RUNTIME_NATIVE_BUILD` pointing to the supported build driver, an isolated +native `CARGO_TARGET_DIR`, and `--exact --ignored --nocapture`. Optional +`DDLOG_ICEBERG_EVIDENCE` records exact native receipts and measurements. This is +local composition recovery proof; the assembled HHMM/Observer acceptance and +remote storage gates remain separate. diff --git a/docs/managed-workers.md b/docs/managed-workers.md new file mode 100644 index 0000000..29f9639 --- /dev/null +++ b/docs/managed-workers.md @@ -0,0 +1,136 @@ +# Managed worker and durable admission contract (version 1) + +This is the generic runtime seam for an independently attached external worker. +Runtime owns admission, checkpoint publication and local child lifetime. The +worker owns application schemas, prompts, providers, tools and reconciliation. +There is no second daemon, automatic restart or automatic external-call retry. + +## Startup profiles and launch + +Launch the existing owner with `--listen ENDPOINT.json --worker-profiles PROFILES.json`. +The profile file is an absolute, same-user regular file, not group/world writable, +read once at startup; no protocol verb changes profiles. Its schema is: + +```json +{"schema_version":1,"profiles":{"example":{ + "argv":["/absolute/python","-m","application.worker"], + "env":{"APPLICATION_MODE":"fixture"}, + "config_ref":"/absolute/operator/config.json", + "allowed_processors":[{"processor_id":"processor_...","version":"sha256:..."}], + "max_concurrent_workers":4, + "timeout_ms":300000 +}}} +``` + +`env` defaults to empty, `config_ref` to null. The first argv element must be +absolute. Runtime never evaluates a shell or substitutes caller payload into argv +or env. The profile config reference is passed to the child, never read as provider +configuration by Runtime. Profiles have 1–64 concurrent children each, a 1 ms–1 hour +lifetime limit, and exact allowed processor pins. No profile means no worker launch. + +- `worker_start {id,expected_generation,profile,key,payload}`: requires a running + world and the profile's exact pin. `key` is a bounded opaque identifier, unique + across profiles for that world/generation. Identical requests return the existing + record without relaunching, even after completion/failure. Different payload or + profile for that key conflicts. Payload is at most 64 KiB JSON. +- `worker_status {id,generation?,worker_id?}`: returns retained records, optionally + selecting a generation/worker. `worker_stop {id,expected_generation,worker_id}` + cancels only that child process group. World Stop and owner shutdown cancel all + owned children, including descendants. Reopening never adopts an old PID. + +Stdin receives exactly one newline-terminated launch envelope then EOF: + +```json +{"schema_version":1,"world_id":"...","generation":1,"worker_id":"...", + "profile":"example","key":"run-123","processor":{"processor_id":"...","version":"..."}, + "owner_descriptor":"/absolute/ENDPOINT.json","config_ref":"/absolute/operator/config.json", + "payload":{}} +``` + +The child uses the existing descriptor/owner-incarnation socket protocol. It must +carry `worker_id` on read/admission requests to fence cancellation/late results. +Runtime accepts at most 16 KiB stdout containing one JSON outcome: +`{"schema_version":1,"status":"completed"|"failed","code":"optional_identifier"}`. +A zero exit plus a valid completed outcome establishes local completion only. +Nonzero exit, malformed/oversized output and timeouts fail. Stderr is discarded. +A valid bounded `status:"failed"` outcome and its code are retained even on a +nonzero exit; a nonzero exit can never establish `completed`. +Status exposes identity, profile hash, PID, resources, timestamps, state, exit code, +validated outcome code and generic errors; it never includes argv/env/config, +payload, stdout text or application prompts. States are starting/running/stopping, +completed/failed/stopped/interrupted. No worker or effect is retried automatically. +Process-group cleanup covers World Stop, worker cancellation and graceful owner +shutdown (including TERM/INT). An uncatchable owner kill or machine failure cannot +run cleanup; reopening marks unfinished records interrupted and never signals or +adopts recorded PIDs, which may have been reused. + +## Consistent reads and fenced durable writes + +`read_batch {id,expected_generation,expected_revision,worker_id?,queries:[ +{predicate,max_rows?,continuation?}]}` performs 1–16 public-relation reads under the +same owner lock, returning `{schema_version:1,id,generation,revision,results:[...]}`. +Total requested rows are at most 1000 and the response is capped at 4 MiB. Results +use the existing query_rows page shape and continuations. Incomplete pages are +explicit; the caller must not treat them as complete evidence. Every later batch +must use the same generation/revision. Input continuation tokens are additionally +fenced by those mandatory arguments. The write must carry that evidence revision. +Obtain the first generation/revision from `status`. A stale read fence can restart +the entire evidence read at a newly observed revision; discard earlier pages. +This read-only retry does not authorize mutation or an external-call retry. + +`admit_inputs {id,expected_generation,expected_revision,admission_key,changes, +worker_id?,effect?}` performs validation, one plain native input commit and managed +JSON checkpoint publication while holding the existing exclusive owner lock. +Changes use the existing public typed input contract; no output deltas are read. +The request is at most 256 KiB and 1000 changes. Keys are bounded identifiers. + +Result fields: `{schema_version:1,id,generation,admission_key,state,applied_revision, +receipt,error,replayed,effect_authorized,publication}`. Outcomes: + +- `durable`: native acknowledgment and durable publication succeeded. Receipt is + the exact managed checkpoint object; applied_revision names its committed inputs. +- `not_applied`: validation/fencing failed before a native transaction. +- `applied_but_unpublished`: native commit acknowledged but no durable publication + acknowledgment can be returned. `publication: uncertain` means files may exist; + this is never permission to call a provider. +- `uncertain`: native acknowledgment or final durable admission-record write was + lost. Reconcile explicitly. No automatic mutation/effect retry is allowed. + +`admission_status {id,generation,admission_key}` reads a retained outcome without +re-executing. Once an execution intent is recorded, repeating the exact admission +request is lookup-only; changing its contents conflicts. Preflight rejections +do not consume a key or promise a retained status record. All lookups return +`replayed:true,effect_authorized:false`. +Historical/pending records survive owner restart; uncompleted intent is uncertain. +Intent is persisted before mutation, without retaining caller changes/prompts. +At most 4096 admissions and 1024 effects are retained per world; exhausting either +fails before mutation and requires an explicit new-world/reconciliation decision. + +## Optional generic effect reservation and settlement + +For an external call use `effect:{key,phase:"reserve"}` on the admission that writes +its application's claim. The fresh response must be `durable` and +`effect_authorized:true` before the worker makes that call. The reservation key is +that admission's `admission_key`. Duplicate effect keys cannot reserve again, +including through another worker or after checkpoint restoration. Lost admission +responses never authorize automatic provider retry. + +Settle through another fenced admission with +`effect:{key,phase:"settle",reservation_key:""}` and the +application's result changes. The matching reservation must belong to this world, +generation and worker and must still be reserved. This prevents duplicate settlement +and results from old generations/stopped workers. Unrelated revisions may advance; +refresh consistent evidence and use the current revision, retaining the exact +reservation identity. No provider is recalled when CAS fails. + +Generic effect metadata is integrity-bound alongside inputs in Backend checkpoint +metadata, without changing the version-1 receipt shape. Restore retains reservations +and settlements but never grants dispatch authority or changes their origin fences. +Ordinary fresh Start clears execution state. Explicit restoration of a checkpoint +from before an effect is not an exactly-once guarantee; application/operator +reconciliation remains necessary. No generic retry/release/reassignment operation +is included in this slice. + +The trusted worker is not a hostile-client sandbox: same-user socket clients retain +operator access. The worker must use these fenced APIs and carry its worker ID; +legacy execute remains available to existing operator clients. diff --git a/docs/specification.md b/docs/specification.md index c125e6b..4a963f3 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -16,6 +16,8 @@ wire shapes and operational prerequisites. | Registered requests | Claim before completion; preserve late results with explicit freshness; identical completion is idempotent; conflicting completion fails; uncertain settlement never implies permission to repeat provider work | `tests/registered_requests.rs` | | Shared access | Connections share one owner; disconnect does not stop it; pinned versions and exported ports remain enforced | `tests/test_shared_host.py`, `tests/upstream_compatibility.rs` | | Checkpoints | Explicit pure-backend checkpoint; integrity-checked restore into a fresh backend; reconstruct outputs from acknowledged inputs | `tests/memory_runtime.rs`, [checkpoint contract](checkpoints.md) | +| Managed persistence | Durable exact JSON receipts; explicit asynchronous fresh-generation restore with pinned public admission and hosted cancellation | `tests/worlds/persistence.rs`, `tests/worlds_stdio.rs`, opt-in `tests/worlds_native.rs`, [managed contract](managed-checkpoints.md) | +| Managed workers and admission | Startup-only exact-pin profiles; owned bounded children; revision-fenced read batches and durable input admission; generic effect reservation/settlement with no replay authority | `tests/worlds/admission.rs`, `tests/worlds/workers.rs`, fake and opt-in native `tests/workers_socket.rs`, [worker contract](managed-workers.md) | | Compatibility | Preserve previously published definition hashes, native source fixtures, and MCP tool schemas unless a deliberate compatibility change is declared | `tests/upstream_compatibility.rs` | ## What an acknowledgment establishes diff --git a/docs/worlds.md b/docs/worlds.md index 2739dec..459a685 100644 --- a/docs/worlds.md +++ b/docs/worlds.md @@ -51,7 +51,22 @@ autostart. The operator chooses when to run this owner. gains synthesized **module groups** (below). - `start` takes `id`, records `starting`, then compiles/installs asynchronously. Poll `status`, `inspect`, or `inventory` to collect completion or failure. -- `stop` takes `id`. It cancels managed compiler/native process groups. A pending +- `checkpoint` takes `id` and returns an exact immutable managed JSON receipt after + durable publication. `restore` takes `id` and that `receipt` object and starts a + fresh generation asynchronously; ordinary Start never restores implicitly. See + the [managed checkpoint contract](managed-checkpoints.md) for receipt fields, + lifecycle gates, storage limits and failure/availability states. +- With the optional Iceberg feature and startup `--storage-profile`, + `checkpoint_stage`, `checkpoint_publish`, and `checkpoint_status` expose + asynchronous local catalog publication. Restore accepts its exact published + receipt. See [managed local Iceberg](managed-iceberg.md); JSON admission + durability and Iceberg catalog visibility remain separately reported. +- `worker_start`, `worker_status`, and `worker_stop` supervise trusted startup + profiles within this owner. `read_batch`, `admit_inputs`, and `admission_status` + provide consistent evidence and fenced durable input/effect boundaries. See the + [managed worker contract](managed-workers.md) for the exact version-1 schemas, + limits and explicit failure/replay semantics. +- `stop` takes `id`. It cancels managed compiler/native/worker process groups. A pending install reports `stopping` until its completion is collected, then `stopped`. - `inventory` `{processor_id?, version?, summary?}` lists worlds, optionally filtered by pin. `summary: true` returns each status **without** `inspection`, `instance` and @@ -59,7 +74,7 @@ autostart. The operator chooses when to run this owner. - `execute` takes world `id`, an inner `operation`, and inner `args`. Definition installation/registry mutations are rejected here; world definitions stay pinned. Beyond the existing program operations it offers: - - `instance_info` adds `revision`, `program_version`, `lowering_version` and `source_sha256` (sha256 of + - `instance_info` adds `revision`, `program_version`, `lowering_version`, `build` (runtime and activation-time native executable identity), and `source_sha256` (sha256 of the lowered `program.dl`) while the instance is healthy. - `apply_changes` adds `revision`. - `relations` `{}` → `{revision, relations: [{name, input, fields, count}]}` over the @@ -118,8 +133,10 @@ resources, managed processes, instance info and inspection: library, its generated copy carries the `large-star`, `small-star` and `minimum-label` phase regions. That patch is applied to the generated project at build time; `src/star/lemmalog_star.rs` and the registry content hashes are unchanged by it. -- `persistence`: `{status: "not_configured", reason: "world checkpoints are not wired"}` - in this slice; nothing claims durable world state. +- `persistence`: managed JSON publication history, file availability, committed and + persisted revisions, and exact `restore_requested` / `restored_from` receipts. + Presence is labeled `present_unverified`; Restore verifies integrity and exact + pins before compilation. See [managed checkpoints](managed-checkpoints.md). - `test`: for test worlds, `{phase: building|applying|done|failed, scenario_index, results: [{name, passed, revision, expected, observed, missing, unexpected, error}], passed: bool|null, error}`, persisted in `world.json`. `expected`, `observed`, diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..cfe61b0 --- /dev/null +++ b/python/README.md @@ -0,0 +1,43 @@ +# Runtime attachment client + +`ddlog-runtime-client` is the shared, dependency-free Python client for the native +world owner. Observer and product workers use the same transport and admission +contract. It contains no HTTP server, process supervisor, provider implementation, +product schemas, or automatic mutation retry. + +```python +from ddlog_runtime_client import AttachedRuntimeClient, ManagedWorldHost + +client = AttachedRuntimeClient('/absolute/environment/owner/endpoint.json') +try: + reply = client.request('status', {'id': world_id}) + if not reply['ok']: + raise RuntimeError(reply['error']) + status = reply['result'] + host = ManagedWorldHost(client, world_id, status['generation']) + evidence = host.snapshot(['input_relation', 'output_relation']) +finally: + client.close() # Detaches; never stops the owner or its worlds. +``` + +The endpoint descriptor identifies one owner incarnation. Each request uses a +bounded connection and checks response identity. A timeout means the outcome may +be unknown; it does not close the owner's input or kill its process. The optional +timing callback receives content-free request timing. + +`ManagedWorldHost.snapshot` reads complete public relations at one world generation +and committed revision. It pages through the owner's `read_batch`, rejects stale +or inconsistent evidence and enforces row/byte limits. It does not return a partial +table as a complete snapshot. A supervised worker passes its `worker_id` to the +constructor, so runtime cancellation fences subsequent reads and admissions. + +`compare_apply` carries the snapshot's generation/revision and an admission key to +`admit_inputs`. Its result keeps native application, durable publication, replay, +and external-effect authority separate. Only a fresh durable effect reservation +with `effect_authorized: true` authorizes an external call. Neither a repeated +request nor an admission-status lookup grants that authority. See +[managed workers](../docs/managed-workers.md) for exact request shapes and limits. + +Build this wheel before installing Observer or a product worker. Release all three +against their recorded source commits; a source-tree `PYTHONPATH` is a development +convenience, not the installed deployment contract. diff --git a/python/ddlog_runtime_client/__init__.py b/python/ddlog_runtime_client/__init__.py new file mode 100644 index 0000000..afd4e1f --- /dev/null +++ b/python/ddlog_runtime_client/__init__.py @@ -0,0 +1,5 @@ +"""Generic clients; no library-specific rules, provider, HTTP or process owner.""" +from .transport import AttachedRuntimeClient +from .world import ManagedWorldHost + +__all__ = ['AttachedRuntimeClient', 'ManagedWorldHost'] diff --git a/python/ddlog_runtime_client/transport.py b/python/ddlog_runtime_client/transport.py new file mode 100644 index 0000000..d6e9f1f --- /dev/null +++ b/python/ddlog_runtime_client/transport.py @@ -0,0 +1,99 @@ +"""One bounded socket exchange per request; attachment never owns the runtime.""" +import json +import math +import socket +import time +import uuid +from pathlib import Path + + +class AttachedRuntimeClient: + """One connection per request to an independently owned local runtime. + + A lost reply is never retried. Closing this client cannot close the runtime. + The descriptor's incarnation fences reconnections to a replacement owner. + """ + + def __init__(self, descriptor, timeout=60, *, on_timing=None): + self.on_timing = on_timing + with Path(descriptor).open('rb') as stream: + data = stream.read(65537) + if len(data) > 65536: + raise ValueError('Runtime endpoint descriptor exceeds 64 KiB') + self.descriptor = json.loads(data) + if not isinstance(self.descriptor, dict) or self.descriptor.get('schema_version') != 1: + raise ValueError('Unsupported runtime endpoint descriptor') + self.socket_path = self.descriptor.get('socket') + self.owner = self.descriptor.get('owner_incarnation') + if (not isinstance(self.socket_path, str) or not Path(self.socket_path).is_absolute() + or not isinstance(self.owner, str) or not 1 <= len(self.owner) <= 128): + raise ValueError('Invalid runtime endpoint identity or socket path') + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError('Runtime timeout must be finite and positive') + self.timeout = timeout + self.closed = False + + def request(self, operation, args=None, *, expected_generation=None, expected_revision=None): + if self.closed: + raise RuntimeError('Runtime attachment is closed; reconnect explicitly.') + request_id = uuid.uuid4().hex + message = {'schema_version': 1, 'request_id': request_id, + 'owner_incarnation': self.owner, 'operation': operation, 'args': args or {}} + for name, value in (('expected_generation', expected_generation), ('expected_revision', expected_revision)): + if value is not None: + if type(value) is not int or value < 0: + raise ValueError(f'{name} must be a nonnegative integer') + message[name] = value + wire = json.dumps(message).encode() + b'\n' + if len(wire) > 1024 * 1024: + raise RuntimeError('Runtime request exceeds 1 MiB; not sent.') + sent = False + started = time.perf_counter() + timing = {} + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + deadline = time.monotonic() + self.timeout + connection.settimeout(self.timeout) + connection.connect(self.socket_path) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError('connection deadline exceeded') + connection.settimeout(remaining) + sent = True # sendall can fail after a partial write. + connection.sendall(wire) + data = bytearray() + while not data.endswith(b'\n'): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError('deadline exceeded') + connection.settimeout(remaining) + chunk = connection.recv(4096) + if not chunk: + raise RuntimeError('Runtime closed its response stream') + data.extend(chunk) + if len(data) > 16 * 1024 * 1024: + raise RuntimeError('Runtime response exceeds 16 MiB') + if data.count(b'\n') != 1: + raise RuntimeError('Invalid runtime response framing') + reply = json.loads(data) + if (not isinstance(reply, dict) or type(reply.get('ok')) is not bool + or reply.get('request_id') != request_id + or reply.get('owner_incarnation') != self.owner): + raise RuntimeError('Runtime response identity or framing mismatch') + timing = reply.get("timing") if isinstance(reply.get("timing"), dict) else {} + return reply + except (OSError, ValueError, RuntimeError) as error: + outcome = 'outcome unknown; no retry' if sent else 'not sent' + raise RuntimeError(f'Runtime request {request_id}: {outcome}: {error}') from error + finally: + if self.on_timing is not None: + # Instrumentation cannot erase an acknowledged mutation result + # or mask an uncertain transport outcome with a callback error. + try: + self.on_timing(request_id, (time.perf_counter()-started)*1000, + timing.get('queue_ms'), timing.get('execution_ms')) + except Exception: + pass + + def close(self): + self.closed = True diff --git a/python/ddlog_runtime_client/world.py b/python/ddlog_runtime_client/world.py new file mode 100644 index 0000000..3fb1a9a --- /dev/null +++ b/python/ddlog_runtime_client/world.py @@ -0,0 +1,101 @@ +"""Translate generic owner contracts into a fenced, bounded worker access port.""" +from copy import deepcopy +import json +import uuid + + +class ManagedWorldHost: + def __init__(self, client, world_id, generation, *, worker_id=None, max_rows=10000, max_bytes=4*1024*1024): + if not isinstance(world_id, str) or not world_id or type(generation) is not int or generation < 1: + raise ValueError('A managed host requires an exact world/generation at startup') + if type(max_rows) is not int or not 1 <= max_rows <= 10000: + raise ValueError('max_rows must be within 1..10000') + if type(max_bytes) is not int or not 1 <= max_bytes <= 4*1024*1024: + raise ValueError('max_bytes must be within 1..4194304') + self.client, self.worker_id = client, worker_id + self.identity = {'world_id': world_id, 'generation': generation} + self.max_rows, self.max_bytes = max_rows, max_bytes + + def _args(self, **values): + return {'id':self.identity['world_id'], 'expected_generation':self.identity['generation'], + **({'worker_id':self.worker_id} if self.worker_id else {}), **values} + + def _call(self, operation, args): + response = self.client.request(operation, args) + if not response.get('ok'): + raise RuntimeError(response.get('error') or 'Runtime rejected the request') + return response['result'] + + def snapshot(self, relations): + if (not isinstance(relations, list) or len(relations) > 128 + or any(not isinstance(name, str) or not name for name in relations) + or len(set(relations)) != len(relations)): + raise ValueError('Snapshot requires at most 128 unique relation names') + status = self._call('status', {'id':self.identity['world_id']}) + revision = status.get('revision') + if (status.get('id') != self.identity['world_id'] + or status.get('generation') != self.identity['generation'] or status.get('state') != 'running' + or type(revision) is not int or revision < 0): + raise RuntimeError('Worker world/generation is no longer running') + result, total_rows, total_bytes = {}, 0, 0 + # Bound each batch to the owner's 1000-row limit. The same revision is + # mandatory across every page and batch, so a concurrent write rejects + # the whole observation rather than mixing revisions. + pending = [(name, None) for name in relations] + result = {name: [] for name in relations} + while pending: + selected, pending = pending[:16], pending[16:] + page_size = min(62, self.max_rows - total_rows + 1) + queries = [{'predicate':name, 'max_rows':page_size, 'continuation':continuation} + for name, continuation in selected] + batch = self._call('read_batch', self._args(expected_revision=revision, queries=queries)) + if (batch.get('generation') != self.identity['generation'] or batch.get('revision') != revision + or batch.get('id') != self.identity['world_id'] or len(batch.get('results', [])) != len(selected)): + raise RuntimeError('Inconsistent snapshot batch identity') + for (name, continuation), page in zip(selected, batch['results']): + if page.get('revision') != revision or not isinstance(page.get('rows'), list): + raise RuntimeError('Inconsistent snapshot relation revision') + total_bytes += len(json.dumps(page['rows']).encode()) + total_rows += len(page['rows']) + if total_rows > self.max_rows or total_bytes > self.max_bytes: + raise RuntimeError('Snapshot exceeds worker row/byte bound; no partial evidence returned') + result[name].extend(page['rows']) + if page.get('complete') is True: continue + next_page = page.get('continuation') + if next_page is None or next_page == continuation: + raise RuntimeError('Incomplete snapshot page has no advancing continuation') + pending.append((name, next_page)) + return {**self.identity, 'revision':revision, 'complete':True, 'relations':result} + + def rows(self, name): + return self.snapshot([name])['relations'][name] + + def compare_apply(self, expected, changes, evidence): + if (any(expected.get(k) != v for k,v in self.identity.items()) + or type(expected.get('revision')) is not int or expected['revision'] < 0): + raise ValueError('Mutation evidence does not match the worker identity/revision') + evidence = deepcopy(evidence) + args = self._args(expected_revision=expected['revision'], changes=changes, + admission_key=evidence.get('admission_key') or uuid.uuid4().hex) + if evidence.get('effect') is not None: args['effect'] = evidence['effect'] + publication = self._call('admit_inputs', args) + phase = publication.get('state') + if (publication.get('id') != self.identity['world_id'] + or publication.get('generation') != self.identity['generation'] + or publication.get('admission_key') != args['admission_key'] + or phase not in ('not_applied', 'durable', 'applied_but_unpublished', 'uncertain')): + raise RuntimeError('Inconsistent admission identity or state; outcome unknown; no retry') + durable = phase == 'durable' + if durable and (not isinstance(publication.get('receipt'), dict) + or type(publication.get('applied_revision')) is not int + or publication['applied_revision'] < expected['revision']): + raise RuntimeError('Invalid durable admission boundary; outcome unknown; no retry') + authority = (durable and publication.get('effect_authorized') is True + and publication.get('replayed') is False + and (args.get('effect') or {}).get('phase') == 'reserve') + return {**self.identity, 'applied': None if phase == 'uncertain' else phase in ('durable','applied_but_unpublished'), + 'durable':durable, 'revision':publication.get('applied_revision'), + 'receipt':publication.get('receipt'), 'error':publication.get('error'), + 'admission_key':publication.get('admission_key'), + 'effect_authorized':authority, + 'replayed':publication.get('replayed'), 'publication':publication} diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..21224f7 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools==83.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "ddlog-runtime-client" +version = "0.1.0" +description = "Bounded attachment and durable admission client for the native DDLog world owner" +requires-python = ">=3.11" +dependencies = [] + +[tool.setuptools] +packages = ["ddlog_runtime_client"] diff --git a/scripts/build-ddlog.sh b/scripts/build-ddlog.sh index 4b4ce77..18f7317 100755 --- a/scripts/build-ddlog.sh +++ b/scripts/build-ddlog.sh @@ -4,6 +4,9 @@ set -eu runtime_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) source_path=$1 output_path=$2 +# Resolve before chdir; the helper atomically publishes Cargo's reported artifact. +source_path=$(python3 -c 'import os,sys; print(os.path.abspath(sys.argv[1]))' "$source_path") +output_path=$(python3 -c 'import os,sys; print(os.path.abspath(sys.argv[1]))' "$output_path") : "${DDLOG_HOME:?Set DDLOG_HOME to the DDlog distribution}" cd "$(dirname "$source_path")" "$DDLOG_HOME/bin/ddlog" -i "$(basename "$source_path")" @@ -32,8 +35,7 @@ fi export CARGO_PROFILE_DEV_DEBUG=0 export CARGO_INCREMENTAL=0 if [ "${DDLOG_OFFLINE:-0}" = 1 ]; then - "${DDLOG_CARGO:-cargo}" build --offline --locked --bin program_cli + python3 "$runtime_root/scripts/build-native-artifact.py" "$output_path" "${DDLOG_CARGO:-cargo}" build --offline --locked --bin program_cli else - "${DDLOG_CARGO:-cargo}" build --bin program_cli + python3 "$runtime_root/scripts/build-native-artifact.py" "$output_path" "${DDLOG_CARGO:-cargo}" build --bin program_cli fi -cp "${CARGO_TARGET_DIR:-target}/debug/program_cli" "$output_path" diff --git a/scripts/build-native-artifact.py b/scripts/build-native-artifact.py new file mode 100644 index 0000000..1573a8f --- /dev/null +++ b/scripts/build-native-artifact.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Prevent identical DDlog Cargo package names from sharing generated artifacts. + +Cargo 1.65 may treat a different generated workspace as fresh in a shared target. +Each absolute project identity and generated-source/config/lock content key gets +its own target, including dependencies. Cargo's reported executable is the only +publication source; an old debug/program_cli is never a fallback. + +This is a generated-project isolation boundary, not a hermetic build system. +External locked dependencies and the operator's pinned toolchain use Cargo's +normal validation/freshness rules. We do not traverse vendor trees, SDKs or Rust +sysroots, or include unrelated environment values (such as credentials) in keys. +A project lock covers two bounded source fingerprints, compilation and copy. +""" +import fcntl +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import time +import tomllib +import uuid + + +VERSION = 2 + + +def digest_file(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(chunk) + return digest.hexdigest() + + +def digest_tree(root, excluded=()): + """Hash names, symlink targets, executable bits and all file contents.""" + excluded = {Path(p).resolve() for p in excluded} + digest = hashlib.sha256() + directories, files = set(), {} + + def add(value): + data = json.dumps(value, ensure_ascii=True, separators=(',', ':')).encode() + digest.update(len(data).to_bytes(8, 'big')) + digest.update(data) + + def visit(path, relative): + if path.is_symlink(): + add(['link', relative, os.readlink(path)]) + if not path.exists(): + add(['dangling', relative]) + return + resolved = path.resolve(strict=True) + if resolved in excluded: + return + if path.is_dir(): + if resolved in directories: + # Aliases and parent links can otherwise expand exponentially. + # Hash every physical tree once, retaining all alias identities. + add(['directory-reference', relative, str(resolved)]) + return + directories.add(resolved) + add(['directory', relative]) + for child in sorted(path.iterdir()): + # Cargo/libtool output and VCS administration are not sources. + if child.name not in ('target', '.git'): + visit(child, relative + '/' + child.name) + elif path.is_file(): + if resolved not in files: + files[resolved] = digest_file(path) + add(['file', relative, path.stat().st_mode & 0o111, files[resolved]]) + else: + raise ValueError(f'Unsupported native input: {path}') + + root = Path(root) + if not root.exists() and not root.is_symlink(): + add(['missing']) # Also fence creation of previously absent Cargo configs. + else: + visit(root, '.') + return digest.hexdigest() + + +def capture(command, env): + result = subprocess.run(command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode: + raise ValueError(f'Native input discovery failed ({command[0]}): {result.stderr.strip()}') + return result.stdout.strip() + + +def executable(name, env): + found = shutil.which(name, path=env.get('PATH')) + if found is None: + raise ValueError(f'Native build tool is unavailable: {name}') + # Proxies such as rustup dispatch on the invoked basename. Preserve it for + # version queries; resolve symlinks only when recording the tool's path. + return str(Path(found).absolute()) + + +def input_roots(command, env, project): + """Generated workspace, effective Cargo config and declared tool identities.""" + roots = {project} + config_paths = [parent / '.cargo' / name for parent in (project, *project.parents) + for name in ('config', 'config.toml')] + cargo_home = Path(env.get('CARGO_HOME', Path.home() / '.cargo')) + config_paths += [cargo_home / name for name in ('config', 'config.toml')] + roots.update(config_paths) + rustc = env.get('RUSTC') or env.get('CARGO_BUILD_RUSTC') + for path in config_paths: + if path.is_file(): + rustc = rustc or tomllib.loads(path.read_text()).get('build', {}).get('rustc') + cargo, rustc = executable(command[0], env), executable(rustc or 'rustc', env) + versions = {'cargo_path': str(Path(cargo).resolve()), 'cargo': capture([cargo, '--version', '--verbose'], env), + 'rustc_path': str(Path(rustc).resolve()), 'rustc': capture([rustc, '--version', '--verbose'], env), + 'platform': sys.platform, 'machine': os.uname().machine} + # Resolving the default Apple SDK is cheap; hashing its whole tree is not. + if sys.platform == 'darwin': + versions['sdk_path'] = env.get('SDKROOT') or capture(['xcrun', '--show-sdk-path'], env) + return roots, versions + + +def build_environment(env): + """Only supported code-generation/link configuration, never the whole env.""" + names = {'RUSTC', 'RUSTC_WRAPPER', 'RUSTC_WORKSPACE_WRAPPER', 'RUSTFLAGS', + 'CARGO_ENCODED_RUSTFLAGS', 'CARGO_BUILD_TARGET', 'CARGO_BUILD_RUSTC', + 'CARGO_BUILD_RUSTFLAGS', 'CC', 'CXX', 'AR', 'LD', 'RANLIB', + 'CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'LDFLAGS', 'SDKROOT', + 'MACOSX_DEPLOYMENT_TARGET', 'DEVELOPER_DIR', 'LIBRARY_PATH', + 'CPATH', 'CPLUS_INCLUDE_PATH', 'PKG_CONFIG_PATH'} + return {name: value for name, value in env.items() if name in names + or name.startswith('CARGO_PROFILE_') + or (name.startswith('CARGO_TARGET_') and name.endswith(('_LINKER', '_RUSTFLAGS')))} + + +def fingerprint(roots, versions, command, env, excluded): + started = time.perf_counter() + # Even supported build flags can contain private values; persist only a hash. + environment = hashlib.sha256(json.dumps(build_environment(env), sort_keys=True).encode()).hexdigest() + inputs = {'schema_version': VERSION, 'command': command, 'build_environment_sha256': environment, + 'tool_versions': versions, + 'inputs': {str(p): digest_tree(p, excluded) for p in sorted(roots)}} + key = hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest() + print(f'generated-project fingerprint: {time.perf_counter() - started:.3f}s', flush=True) + return key, inputs + + +def project_lock(base, project): + return base / 'ddlog-project-locks' / (hashlib.sha256(str(project).encode()).hexdigest() + '.lock') + + +def cargo_build(command, env, target): + artifacts = set() + invocation = [*command, '--target-dir', str(target), '--message-format=json-render-diagnostics'] + with subprocess.Popen(invocation, env=env, stdout=subprocess.PIPE, text=True) as process: + for line in process.stdout: + try: + event = json.loads(line) + except ValueError: + print(line, end='', flush=True) + continue + if (event.get('reason') == 'compiler-artifact' and event.get('target', {}).get('name') == 'program_cli' + and 'bin' in event['target'].get('kind', []) and event.get('executable')): + artifacts.add(Path(event['executable']).resolve()) + status = process.wait() + if status: + return status, None + if len(artifacts) != 1: + raise ValueError('Cargo did not report exactly one program_cli executable; refusing stale artifact fallback') + artifact = artifacts.pop() + if target.resolve() not in artifact.parents or not artifact.is_file(): + raise ValueError('Cargo executable is outside the isolated target or missing') + return 0, artifact + + +def publish(artifact, output): + descriptor, staging = tempfile.mkstemp(prefix='.' + output.name + '-', dir=output.parent) + os.close(descriptor) + try: + shutil.copy2(artifact, staging) + with open(staging, 'rb') as stream: + os.fsync(stream.fileno()) + os.replace(staging, output) + finally: + if os.path.exists(staging): + os.unlink(staging) + + +def build(output, command): + project = Path.cwd().resolve() + output = Path(output).absolute() + base = Path(os.environ.get('CARGO_TARGET_DIR', 'target')).resolve() + cache = base / 'ddlog-content-v2' + cache.mkdir(parents=True, exist_ok=True) + env = dict(os.environ) + lock_path = project_lock(base, project) + lock_path.parent.mkdir(exist_ok=True) + lock = os.open(lock_path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + try: + fcntl.flock(lock, fcntl.LOCK_EX) + # Read only after the project lock; no redundant pre-lock fingerprint. + roots, versions = input_roots(command, env, project) + key, inputs = fingerprint(roots, versions, command, env, (base, output)) + bucket = cache / key + bucket.mkdir(exist_ok=True) + (bucket / 'inputs.json').write_text(json.dumps(inputs, sort_keys=True, indent=2) + '\n') + target = bucket / 'target' + build_env = dict(env, CARGO_TARGET_DIR=str(target)) + print(f'ddlog-runtime native content key: {key}\ntarget: {target}', flush=True) + status, artifact = cargo_build(command, build_env, target) + if status: + return status + if fingerprint(roots, versions, command, env, (base, output))[0] != key: + target.rename(bucket / ('changed-inputs-' + uuid.uuid4().hex)) + raise ValueError('Generated project inputs changed during compilation; artifact quarantined, not published') + publish(artifact, output) + print(f'published native sha256: {digest_file(output)}', flush=True) + return 0 + finally: + os.close(lock) + + +if __name__ == '__main__': + try: + raise SystemExit(build(sys.argv[1], sys.argv[2:])) + except (ValueError, OSError, KeyError) as error: + print(f'build-native-artifact.py: {error}', file=sys.stderr) + raise SystemExit(1) diff --git a/src/bin/ddlog-worlds.rs b/src/bin/ddlog-worlds.rs index 38d4be0..c8cc714 100644 --- a/src/bin/ddlog-worlds.rs +++ b/src/bin/ddlog-worlds.rs @@ -40,6 +40,9 @@ fn request(manager: &mut WorldManager, request: Value) -> Result "library_create" => { manager.create_library(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?) } + "library_import" => { + manager.library_import(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?) + } "create" => { let definition: WorldDefinition = serde_json::from_value(args.clone()).map_err(|e| e.to_string())?; @@ -47,6 +50,31 @@ fn request(manager: &mut WorldManager, request: Value) -> Result manager.status(&id) } "start" => manager.start_async(id()?), + "checkpoint" => manager.checkpoint(id()?), + "checkpoint_stage" => manager + .checkpoint_stage(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?), + "checkpoint_publish" => manager + .checkpoint_publish(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?), + "checkpoint_status" => manager + .checkpoint_status(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?), + "restore" => manager.restore_async(id()?, &args["receipt"]), + "worker_start" => { + manager.worker_start(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?) + } + "worker_status" => { + manager.worker_status(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?) + } + "worker_stop" => { + manager.worker_stop(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?) + } + "read_batch" => { + manager.read_batch(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?) + } + "admit_inputs" => { + manager.admit_inputs(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?) + } + "admission_status" => manager + .admission_status(serde_json::from_value(args.clone()).map_err(|e| e.to_string())?), "stop" => manager.stop(id()?), "status" | "inspect" => manager.status(id()?), "register" => { @@ -107,17 +135,41 @@ static STOP: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new( fn main() -> Result<(), Box> { let args: Vec<_> = std::env::args().skip(1).collect(); - let socket_mode = args.len() == 5 && args[3] == "--listen"; - if args.len() != 3 && !socket_mode { + if args.len() < 3 || (args.len() - 3) % 2 != 0 { return Err( - "Usage: ddlog-worlds REGISTRY BUILD_ROOT BUILD_DRIVER [--listen ENDPOINT.json]".into(), + "Usage: ddlog-worlds REGISTRY BUILD_ROOT BUILD_DRIVER [--listen ENDPOINT.json [--worker-profiles PROFILES.json]] [--storage-profile PROFILE.json]".into(), ); } + let mut endpoint = None; + let mut profiles = None; + let mut storage_profile = None; + for pair in args[3..].chunks_exact(2) { + match pair[0].as_str() { + "--listen" if endpoint.is_none() => endpoint = Some(std::path::Path::new(&pair[1])), + "--worker-profiles" if profiles.is_none() => { + profiles = Some(std::path::Path::new(&pair[1])) + } + "--storage-profile" if storage_profile.is_none() => { + storage_profile = Some(std::path::Path::new(&pair[1])) + } + _ => return Err("Unknown or duplicate owner option".into()), + } + } + if profiles.is_some() && endpoint.is_none() { + return Err("Worker profiles require --listen".into()); + } + let socket_mode = endpoint.is_some(); let mut manager = WorldManager::new( args[0].clone().into(), args[1].clone().into(), args[2].clone().into(), )?; + if let Some(profiles) = profiles { + manager.configure_workers(profiles, endpoint.unwrap().to_path_buf())?; + } + if let Some(profile) = storage_profile { + manager.configure_storage(profile)?; + } #[cfg(unix)] { use std::sync::atomic::Ordering; @@ -142,8 +194,7 @@ fn main() -> Result<(), Box> { } if socket_mode { #[cfg(unix)] - return worlds_socket::serve(manager, std::path::Path::new(&args[4]), &STOP) - .map_err(Into::into); + return worlds_socket::serve(manager, endpoint.unwrap(), &STOP).map_err(Into::into); #[cfg(not(unix))] return Err("Independent owner attachment requires Unix".into()); } diff --git a/src/checkpoint.rs b/src/checkpoint.rs index 492081a..60583ca 100644 --- a/src/checkpoint.rs +++ b/src/checkpoint.rs @@ -11,27 +11,27 @@ use std::path::Path; use std::sync::atomic::{AtomicU64, Ordering}; const FORMAT_VERSION: u32 = 1; -const MAX_BYTES: u64 = 64 * 1024 * 1024; +pub(crate) const MAX_BYTES: u64 = 64 * 1024 * 1024; static TEMP_ID: AtomicU64 = AtomicU64::new(0); #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] -struct State { +pub(crate) struct State { format_version: u32, - source: String, - schemas: BTreeMap, + pub(crate) source: String, + pub(crate) schemas: BTreeMap, inputs: BTreeMap>>, - revision: u64, - program_version: u64, + pub(crate) revision: u64, + pub(crate) program_version: u64, /// Opaque application payload, integrity-bound but never interpreted by runtime. - metadata: Value, + pub(crate) metadata: Value, /// Lowering the source was generated under; omitted (and 1) for every /// checkpoint written before version 2 existed, so their digests hold. #[serde( default = "default_lowering_version", skip_serializing_if = "is_version_1" )] - lowering_version: u32, + pub(crate) lowering_version: u32, } fn default_lowering_version() -> u32 { 1 @@ -57,6 +57,16 @@ fn decode_checkpoint(bytes: &[u8]) -> Result { } Ok(checkpoint) } +/// Inspect a managed checkpoint with the same validation as Backend restore. +pub(crate) fn inspect(bytes: &[u8]) -> Result<(String, State)> { + if bytes.len() as u64 > MAX_BYTES { + return Err("Checkpoint exceeds 64 MiB limit".into()); + } + let checkpoint = decode_checkpoint(bytes)?; + checkpoint.state.validate()?; + LoweringOptions::for_version(checkpoint.state.lowering_version)?; + Ok((checkpoint.sha256, checkpoint.state)) +} #[cfg(feature = "iceberg")] pub(crate) fn validate_encoded(bytes: &[u8]) -> Result<()> { if bytes.len() as u64 > MAX_BYTES { @@ -270,6 +280,13 @@ impl Backend { let facts = checkpoint.state.validate()?; // Stage in a separate owner; errors leave this backend untouched. let mut candidate = Backend::new(self.root.clone(), self.driver.clone()); + // Restore is still owned by the hosting world. In particular, Stop must + // cancel both compilation and replay, and the new generation keeps its + // own observer capture rather than inheriting ambient process settings. + candidate.control = self.control.clone(); + candidate.inspection_log = self.inspection_log.clone(); + candidate.observer = self.observer.clone(); + candidate.lowering_version = checkpoint.state.lowering_version; candidate.attempt = self.attempt; candidate.facts = facts; candidate.schema = checkpoint.state.schemas.clone(); diff --git a/src/iceberg_checkpoint.rs b/src/iceberg_checkpoint.rs index 3319dc9..ac63f2f 100644 --- a/src/iceberg_checkpoint.rs +++ b/src/iceberg_checkpoint.rs @@ -116,74 +116,8 @@ impl Backend { object_uri: &str, metadata: Value, ) -> Result { - validate_table(table)?; - if !id_valid(publication_id) { - return Err("Invalid checkpoint publication ID".into()); - } let checkpoint = self.checkpoint_bytes(metadata)?; - let digest = hash(&checkpoint); - if table - .file_io() - .new_input(object_uri) - .map_err(err)? - .exists() - .await - .map_err(err)? - { - return Err( - "Object already exists; retry with the retained staged receipt, never overwrite" - .into(), - ); - } - let arrays: Vec = vec![ - Arc::new(StringArray::from(vec![publication_id])), - Arc::new(StringArray::from(vec![digest.as_str()])), - Arc::new(LargeBinaryArray::from(vec![checkpoint.as_slice()])), - ]; - let batch = RecordBatch::try_new( - Arc::new(schema_to_arrow_schema(&schema()?).map_err(err)?), - arrays, - ) - .map_err(err)?; - let mut writer = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - table.metadata().current_schema().clone(), - ) - .build(table.file_io().new_output(object_uri).map_err(err)?) - .await - .map_err(err)?; - writer.write(&batch).await.map_err(err)?; - let builders = writer.close().await.map_err(err)?; - if builders.len() != 1 { - return Err("Unexpected checkpoint file inventory".into()); - } - let bytes = table - .file_io() - .new_input(object_uri) - .map_err(err)? - .read() - .await - .map_err(err)?; - if bytes.len() > MAX_BYTES + 1024 * 1024 { - return Err("Parquet checkpoint exceeds transport limit".into()); - } - let mut receipt = StagedCheckpoint { - publication_id: publication_id.into(), - table_uuid: table.metadata().uuid().to_string(), - checkpoint_sha256: digest, - object_uri: object_uri.into(), - object_sha256: hash(&bytes), - descriptor_avro: Vec::new(), - }; - validate_parquet(bytes.clone(), &receipt)?; - write_data_files_to_avro( - &mut receipt.descriptor_avro, - vec![descriptor(table, &receipt.object_uri, bytes.len())?], - table.metadata().default_partition_type(), - FormatVersion::V3, - ) - .map_err(err)?; - Ok(receipt) + stage_bytes(table, publication_id, object_uri, &checkpoint).await } /// Read the explicitly selected publication through a pinned Iceberg scan, @@ -194,13 +128,111 @@ impl Backend { ident: &TableIdent, receipt: &StagedCheckpoint, ) -> Result { - let table = catalog.load_table(ident).await.map_err(err)?; - let snapshot = find_snapshot(&table, receipt)?.ok_or("Checkpoint not catalog-visible")?; - let bytes = read_checkpoint(&table, snapshot, receipt).await?; + let bytes = load_bytes(catalog, ident, receipt).await?; self.restore_checkpoint_bytes(&bytes) } } +/// Stage an already frozen format-1 snapshot. No native instance or owner lock +/// is held while FileIO/catalog work runs. Uses the same single Arrow payload. +pub async fn stage_bytes( + table: &Table, + publication_id: &str, + object_uri: &str, + checkpoint: &[u8], +) -> Result { + validate_table(table)?; + if !id_valid(publication_id) { + return Err("Invalid checkpoint publication ID".into()); + } + crate::checkpoint::validate_encoded(checkpoint)?; + let digest = hash(checkpoint); + if table + .file_io() + .new_input(object_uri) + .map_err(err)? + .exists() + .await + .map_err(err)? + { + return Err( + "Object already exists; retry with the retained staged receipt, never overwrite".into(), + ); + } + let arrays: Vec = vec![ + Arc::new(StringArray::from(vec![publication_id])), + Arc::new(StringArray::from(vec![digest.as_str()])), + Arc::new(LargeBinaryArray::from(vec![checkpoint])), + ]; + let batch = RecordBatch::try_new( + Arc::new(schema_to_arrow_schema(&schema()?).map_err(err)?), + arrays, + ) + .map_err(err)?; + let mut writer = ParquetWriterBuilder::new( + WriterProperties::builder().build(), + table.metadata().current_schema().clone(), + ) + .build(table.file_io().new_output(object_uri).map_err(err)?) + .await + .map_err(err)?; + writer.write(&batch).await.map_err(err)?; + let builders = writer.close().await.map_err(err)?; + if builders.len() != 1 { + return Err("Unexpected checkpoint file inventory".into()); + } + let bytes = table + .file_io() + .new_input(object_uri) + .map_err(err)? + .read() + .await + .map_err(err)?; + if bytes.len() > MAX_BYTES + 1024 * 1024 { + return Err("Parquet checkpoint exceeds transport limit".into()); + } + let mut receipt = StagedCheckpoint { + publication_id: publication_id.into(), + table_uuid: table.metadata().uuid().to_string(), + checkpoint_sha256: digest, + object_uri: object_uri.into(), + object_sha256: hash(&bytes), + descriptor_avro: Vec::new(), + }; + validate_parquet(bytes.clone(), &receipt)?; + write_data_files_to_avro( + &mut receipt.descriptor_avro, + vec![descriptor(table, &receipt.object_uri, bytes.len())?], + table.metadata().default_partition_type(), + FormatVersion::V3, + ) + .map_err(err)?; + Ok(receipt) +} + +/// Verify the exact catalog-visible object and return its validated envelope. +/// Managed worlds subsequently validate their ProgramInstance pin/interface. +pub async fn load_bytes( + catalog: &dyn Catalog, + ident: &TableIdent, + receipt: &StagedCheckpoint, +) -> Result> { + load_publication(catalog, ident, receipt) + .await + .map(|(_, bytes)| bytes) +} + +/// Pinned snapshot identity together with the validated encoded checkpoint. +pub async fn load_publication( + catalog: &dyn Catalog, + ident: &TableIdent, + receipt: &StagedCheckpoint, +) -> Result<(i64, Vec)> { + let table = catalog.load_table(ident).await.map_err(err)?; + let snapshot = find_snapshot(&table, receipt)?.ok_or("Checkpoint not catalog-visible")?; + Ok((snapshot, read_checkpoint(&table, snapshot, receipt).await?)) +} + fn find_snapshot(table: &Table, receipt: &StagedCheckpoint) -> Result> { validate_table(table)?; if !id_valid(&receipt.publication_id) diff --git a/src/instance.rs b/src/instance.rs index 8bd1a91..776f705 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,10 +1,12 @@ //! Program state and semantic operations, independent of MCP and connections. use crate::composition::CompositionResolution; -use crate::registry::{GitProvenance, ProcessorDefinition, ProcessorRegistry, ProcessorVersion}; +use crate::registry::{ + GitProvenance, ProcessorDefinition, ProcessorReference, ProcessorRegistry, ProcessorVersion, +}; use crate::{AgentProgram, Backend, LoweringOptions, Operation, Schema}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; /// One relation addressable through a world's public tools. `physical` is the /// generated native relation name; every other field is the public contract. @@ -76,6 +78,100 @@ pub fn public_relations(record: &ProcessorVersion) -> Result } } +/// Identity of the actual pure program build, independent of native execution. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ProgramIdentity { + pub processor: ProcessorReference, + pub dependencies: BTreeMap, + pub public_relations: Vec, + pub lowering_version: u32, + pub source_sha256: String, +} + +/// The same lowering and public admission mapping serve install and restore. +/// Preparing this plan validates pins/dependencies without starting a compiler. +struct PreparedProgram { + source: String, + schemas: BTreeMap, + interface: Option, + composition: Option, + identity: ProgramIdentity, +} +impl PreparedProgram { + fn new( + registry: &ProcessorRegistry, + record: &ProcessorVersion, + version: u32, + ) -> Result { + let options = LoweringOptions::for_version(version)?; + let (source, schemas, interface, composition) = match &record.definition { + ProcessorDefinition::Composition(definition) => { + let compiled = + registry.compile_composition_versioned(&definition.composition, version)?; + let interface = PublicInterface { + inputs: compiled.resolution.inputs.clone(), + outputs: compiled.resolution.outputs.clone(), + }; + ( + compiled.source, + compiled.schemas, + Some(interface), + Some(compiled.resolution), + ) + } + ProcessorDefinition::Program(definition) => { + if definition.operation.is_some() { + return Err( + "Registered-operation programs do not support managed checkpoints".into(), + ); + } + let schemas = serde_json::from_value(definition.schemas.clone()) + .map_err(|e| e.to_string())?; + let exports = definition + .interface + .as_ref() + .map(|i| i.outputs.iter().cloned().collect()); + let source = crate::lower_with_options( + &definition.rules, + &schemas, + &definition.operators, + &options, + exports.as_ref(), + )?; + let interface = definition + .interface + .as_ref() + .map(PublicInterface::from_program); + (source, schemas, interface, None) + } + }; + let mut actual = record.clone(); + actual.composition = composition.clone(); + use sha2::{Digest, Sha256}; + let identity = ProgramIdentity { + processor: ProcessorReference { + processor_id: record.processor_id.clone(), + version: record.version.clone(), + }, + dependencies: composition + .as_ref() + .map(|c| c.dependencies.clone()) + .unwrap_or_default(), + public_relations: public_relations(&actual)?, + lowering_version: version, + source_sha256: format!("{:x}", Sha256::digest(source.as_bytes())), + }; + Ok(Self { + source, + schemas, + interface, + composition, + identity, + }) + } +} + /// Continuation for paged reads of retained input facts. Output continuations /// are the native-bound [`crate::QueryCursor`]; both are opaque to callers. #[derive(Serialize, Deserialize)] @@ -142,6 +238,7 @@ impl ProgramInstance { "processor":self.processor,"composition":self.composition, "revision":live.then(|| self.backend.revision()), "program_version":live.then_some(self.backend.version), + "build":live.then(|| self.backend.build_identity()), "source_sha256":live.then(|| self.backend.source_sha256()), "lowering_version":live.then(|| self.backend.lowering_version()), })); @@ -348,6 +445,14 @@ impl ProgramInstance { if !(1..=crate::MAX_QUERY_ROWS).contains(&max_rows) { return Err(format!("max_rows must be in 1..={}", crate::MAX_QUERY_ROWS)); } + let max_bytes = match a.get("max_bytes") { + None | Some(Value::Null) => crate::MAX_QUERY_BYTES, + Some(value) => usize::try_from(value.as_u64().ok_or("max_bytes must be an integer")?) + .map_err(|e| e.to_string())?, + }; + if !(2..=crate::MAX_QUERY_BYTES).contains(&max_bytes) { + return Err("Invalid query byte limit".into()); + } let relation = self .public_relations()? .into_iter() @@ -355,6 +460,9 @@ impl ProgramInstance { .ok_or_else(|| format!("Unknown public relation {predicate}"))?; let continuation = a.get("continuation").filter(|value| !value.is_null()); if relation.input { + if self.backend.health() != "ready" { + return Err("Input query requires a healthy initialized runtime".into()); + } let offset = match continuation { Some(cursor) => { let cursor: InputCursor = @@ -371,27 +479,43 @@ impl ProgramInstance { } None => 0, }; - let inputs = self.backend.export_inputs()?; - let all = inputs - .get(&relation.physical) - .ok_or("Retained input schema missing")?; - if offset > all.len() { + let all = self + .backend + .facts + .iter() + .filter(|((name, _), _)| name == &relation.physical) + .map(|(_, row)| row); + let total = all.clone().count(); + if offset > total { return Err("Continuation offset exceeds the selected relation".into()); } - let rows: Vec> = all.iter().skip(offset).take(max_rows).cloned().collect(); + let mut rows = Vec::new(); + let mut bytes = 2; + for row in all.skip(offset).take(max_rows) { + let size = serde_json::to_vec(row).map_err(|e| e.to_string())?.len() + + usize::from(!rows.is_empty()); + if bytes + size > max_bytes { + if rows.is_empty() { + return Err("Input row exceeds query byte limit".into()); + } + break; + } + bytes += size; + rows.push(row.clone()); + } let end = offset + rows.len(); - let complete = end >= all.len(); + let complete = end >= total; let revision = self.backend.revision(); return Ok(json!({ "predicate":predicate,"revision":revision,"fields":relation.fields,"rows":rows, - "total":all.len(),"complete":complete, + "total":total,"complete":complete, "continuation":(!complete).then(|| json!(InputCursor{kind:"input".into(),revision,predicate:predicate.into(),offset:end})), })); } let query = crate::BoundedQuery { filters: BTreeMap::new(), max_rows, - max_bytes: crate::MAX_QUERY_BYTES, + max_bytes, continuation: continuation .map(|cursor| serde_json::from_value(cursor.clone()).map_err(|e| e.to_string())) .transpose()?, @@ -495,6 +619,95 @@ impl ProgramInstance { serde_json::to_value(record).map_err(|e| e.to_string()) } + pub(crate) fn prepare_admission_changes(&self, changes: &Value) -> Result { + if self.agent.is_some() { + return Err("Durable input admission requires a pure program".into()); + } + let changes = self + .interface + .as_ref() + .map(|i| i.changes(changes)) + .unwrap_or_else(|| Ok(changes.clone()))?; + for change in changes.as_array().ok_or("Expected changes array")? { + if !matches!(change["op"].as_str(), Some("insert" | "delete")) { + return Err("Expected insert or delete".into()); + } + self.backend.fact( + string(change, "predicate")?, + change["values"].as_array().ok_or("Expected input values")?, + )?; + } + Ok(changes) + } + + fn install_pure(&mut self, record: &ProcessorVersion, version: u32) -> Result { + let prepared = PreparedProgram::new(self.registry.as_ref().unwrap(), record, version)?; + let result = self.backend.install_source( + prepared.source, + prepared.schemas, + LoweringOptions::for_version(version)?, + )?; + self.interface = prepared.interface; + self.composition = prepared.composition; + Ok(result) + } + + pub(crate) fn checkpoint_identity(&self) -> Result { + let record = self + .record + .as_ref() + .ok_or("Checkpoint requires a pinned instance")?; + let registry = self + .registry + .as_ref() + .ok_or("Processor registry is not configured")?; + let verified = registry.get(&record.processor_id, Some(&record.version))?; + let prepared = PreparedProgram::new(registry, &verified, self.backend.lowering_version())?; + if self.agent.is_some() + || prepared.source != self.backend.active_source + || prepared.schemas != self.backend.schema + { + return Err("Installed program does not match its pinned checkpoint identity".into()); + } + Ok(prepared.identity) + } + + /// Validate the complete checkpoint against its exact pinned lowering before + /// Backend compiles anything; activate public admission only after replay. + pub(crate) fn restore_pinned( + &mut self, + bytes: &[u8], + identity: &ProgramIdentity, + ) -> Result<(), String> { + if self.processor.is_some() || self.agent.is_some() { + return Err("Restore requires a fresh pinned instance".into()); + } + let registry = self + .registry + .as_ref() + .ok_or("Processor registry is not configured")?; + registry.ensure_active(&identity.processor.processor_id)?; + let record = registry.get( + &identity.processor.processor_id, + Some(&identity.processor.version), + )?; + let prepared = PreparedProgram::new(registry, &record, identity.lowering_version)?; + let (_, state) = crate::checkpoint::inspect(bytes)?; + if &prepared.identity != identity + || state.source != prepared.source + || state.schemas != prepared.schemas + || state.lowering_version != identity.lowering_version + { + return Err("Checkpoint does not match pinned program, dependencies, public interfaces or lowering".into()); + } + self.backend.restore_checkpoint_bytes(bytes)?; + self.interface = prepared.interface; + self.composition = prepared.composition; + self.processor = Some(json!(identity.processor)); + self.record = Some(record); + Ok(()) + } + fn install_processor(&mut self, a: &Value) -> Result { if self.backend.health() != "uninitialized" || self.agent.is_some() { return Err("Select a processor only in a fresh instance".into()); @@ -512,82 +725,43 @@ impl ProgramInstance { a.get("version").map(|_| string(a, "version")).transpose()?, )?; let pinned = record.clone(); - // An explicit `lowering_version` argument overrides the instance - // default for this build only; the pinned record stays as registered. let lowering_version = lowering_version(a, self.backend.lowering_version())?; - let lowering = LoweringOptions::for_version(lowering_version)?; self.backend.set_lowering_version(lowering_version)?; - let mut result = match record.definition { - ProcessorDefinition::Composition(definition) => { - let compiled = self - .registry + let mut result = if let ProcessorDefinition::Program(definition) = &record.definition { + if let Some(binding) = &definition.operation { + if !definition.operators.is_empty() { + return Err("Typed operators cannot be combined with a registered operation; put the operator in a separate pure program".into()); + } + let operation = self + .operations + .get(&binding.name) + .ok_or("Pinned operation is not registered on this host")? + .clone(); + if operation.version != binding.version + || operation.description != binding.description + { + return Err( + "Pinned operation definition does not match this host registry".into(), + ); + } + let (agent, result) = AgentProgram::install( + &mut self.backend, + &binding.name, + operation, + &definition.rules, + definition.schemas.clone(), + )?; + self.agent = Some(agent); + self.interface = definition + .interface .as_ref() - .unwrap() - .compile_composition_versioned(&definition.composition, lowering_version)?; - let result = - self.backend - .install_source(compiled.source, compiled.schemas, lowering)?; - self.interface = Some(PublicInterface { - inputs: compiled.resolution.inputs.clone(), - outputs: compiled.resolution.outputs.clone(), - }); - self.composition = Some(compiled.resolution); - result - } - ProcessorDefinition::Program(definition) => { - let result = if let Some(binding) = definition.operation { - if !definition.operators.is_empty() { - return Err("Typed operators cannot be combined with a registered operation; put the operator in a separate pure program".into()); - } - let operation = self - .operations - .get(&binding.name) - .ok_or("Pinned operation is not registered on this host")? - .clone(); - if operation.version != binding.version - || operation.description != binding.description - { - return Err( - "Pinned operation definition does not match this host registry".into(), - ); - } - let (agent, result) = AgentProgram::install( - &mut self.backend, - &binding.name, - operation, - &definition.rules, - definition.schemas, - )?; - self.agent = Some(agent); - result - } else { - let schema: BTreeMap = - serde_json::from_value(definition.schemas).map_err(|e| e.to_string())?; - let exports: Option> = definition - .interface - .as_ref() - .map(|interface| interface.outputs.iter().cloned().collect()); - self.backend.install_program( - &definition.rules, - schema, - &definition.operators, - exports.as_ref(), - )? - }; - self.interface = definition.interface.map(|interface| PublicInterface { - inputs: interface - .inputs - .into_iter() - .map(|name| (name.clone(), name)) - .collect(), - outputs: interface - .outputs - .into_iter() - .map(|name| (name.clone(), name)) - .collect(), - }); + .map(PublicInterface::from_program); result + } else { + self.install_pure(&record, lowering_version)? } + } else { + self.install_pure(&record, lowering_version)? }; self.processor = Some(json!({"processor_id":record.processor_id,"version":record.version})); self.record = Some(pinned); @@ -606,6 +780,18 @@ struct PublicInterface { outputs: BTreeMap, } impl PublicInterface { + fn from_program(interface: &crate::composition::ProgramInterface) -> Self { + let ports = |names: &[String]| { + names + .iter() + .map(|name| (name.clone(), name.clone())) + .collect() + }; + Self { + inputs: ports(&interface.inputs), + outputs: ports(&interface.outputs), + } + } fn changes(&self, changes: &Value) -> Result { let mut mapped = changes.as_array().ok_or("Expected changes array")?.clone(); for change in &mut mapped { diff --git a/src/lib.rs b/src/lib.rs index 0905e0b..a14c429 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -209,6 +209,7 @@ pub struct Backend { revision: u64, attempt: u64, active_source: String, + active_binary_sha256: String, failed: bool, control: processes::ProcessControl, inspection_log: Option, @@ -230,6 +231,7 @@ impl Backend { revision: 0, attempt: 0, active_source: String::new(), + active_binary_sha256: String::new(), failed: false, control: processes::ProcessControl::default(), inspection_log: None, @@ -238,6 +240,12 @@ impl Backend { active_lowering: LoweringOptions::VERSION_1, } } + /// Provenance captured at activation, never re-read from a mutable build file. + pub(crate) fn build_identity(&self) -> Value { + json!({"runtime":runtime_info(),"native_sha256":self.active_binary_sha256, + "source_sha256":self.source_sha256(),"lowering_version":self.lowering_version(), + "program_version":self.version}) + } /// Capture options for the native child; only meaningful with an /// inspection log. Takes effect at the next install. pub fn set_observer(&mut self, observer: ObserverOptions) { @@ -398,6 +406,11 @@ impl Backend { dir.join("build.log").display() )); } + use sha2::{Digest, Sha256}; + let mut binary_file = std::fs::File::open(&binary).map_err(|e| e.to_string())?; + let mut binary_digest = Sha256::new(); + std::io::copy(&mut binary_file, &mut binary_digest).map_err(|e| e.to_string())?; + let binary_sha256 = format!("{:x}", binary_digest.finalize()); let mut runtime = Runtime::start( &binary, &self.control, @@ -419,6 +432,7 @@ impl Backend { self.failed = false; self.schema = schema; self.active_source = source; + self.active_binary_sha256 = binary_sha256; self.active_lowering = lowering; self.version = version; self.revision = revision; diff --git a/src/registry.rs b/src/registry.rs index a7a9d10..70e8222 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -854,9 +854,115 @@ impl ProcessorRegistry { for id in roots { plan.load_processor(self, &id, &mut Vec::new())?; } + self.import_plan(plan, dry_run) + } + + /// Import a saved collection of exact records without a source registry. + /// Supplied order selects the initial current version of a new processor: + /// its first record wins, just as with repeated `import_version` calls. + /// Existing current pointers never move. All records and their transitive + /// dependency closure are validated before publication; writes use the + /// same immutable publication path as directory imports. + pub fn import_records( + &self, + records: Vec, + dry_run: bool, + ) -> Result { + if records.is_empty() || records.len() > 4096 { + return Err("Import requires between 1 and 4096 exact records".into()); + } + let mut plan = ImportPlan { + source: PathBuf::new(), + processors: BTreeMap::new(), + order: Vec::new(), + report: ImportReport::default(), + }; + for record in records { + verify_envelope(&record, &record.processor_id, &record.version)?; + let source = plan + .processors + .entry(record.processor_id.clone()) + .or_insert_with(|| SourceProcessor { + current: Current { + format_version: FORMAT_VERSION, + processor_id: record.processor_id.clone(), + version: record.version.clone(), + lineage: record.lineage.clone(), + }, + records: BTreeMap::new(), + }); + if source + .records + .insert(record.version.clone(), record) + .is_some() + { + return Err("Duplicate processor/version in record import".into()); + } + } + fn visit( + plan: &mut ImportPlan, + key: (String, String), + stack: &mut Vec<(String, String)>, + ) -> Result<()> { + if plan.order.contains(&key) { + return Ok(()); + } + if stack.contains(&key) || stack.len() >= 128 { + return Err("Cyclic or excessively nested record import".into()); + } + let Some(record) = plan + .processors + .get(&key.0) + .and_then(|p| p.records.get(&key.1)) + else { + // An omitted dependency must exist at the destination; the + // shared validator below checks it and its full closure. + return Ok(()); + }; + let children = match &record.definition { + ProcessorDefinition::Program(_) => Vec::new(), + ProcessorDefinition::Composition(definition) => definition + .composition + .nodes + .values() + .map(|pin| (pin.processor_id.clone(), pin.version.clone())) + .collect(), + }; + stack.push(key.clone()); + for child in children { + visit(plan, child, stack)?; + } + stack.pop(); + plan.order.push(key); + Ok(()) + } + let keys: Vec<_> = plan + .processors + .iter() + .flat_map(|(id, p)| { + p.records + .keys() + .map(move |version| (id.clone(), version.clone())) + }) + .collect(); + for key in keys { + visit(&mut plan, key, &mut Vec::new())?; + } + self.import_plan(plan, dry_run) + } + + fn import_plan(&self, mut plan: ImportPlan, dry_run: bool) -> Result { if !plan.report.errors.is_empty() { return Ok(plan.report); } + // Keep destination validation and current-pointer selection in the same + // writer exclusion interval as publication. Otherwise another importer + // can create a current pointer after we classified it as absent. + let _lock = if dry_run { + None + } else { + Some(UpdateLock::acquire(&self.root)?) + }; let reader = |reference: &ProcessorReference| -> Result { match plan .processors @@ -919,7 +1025,6 @@ impl ProcessorRegistry { } return Ok(plan.report); } - let lock = UpdateLock::acquire(&self.root)?; for (processor_id, version) in &plan.order { let source = &plan.processors[processor_id]; let record = &source.records[version]; @@ -954,7 +1059,6 @@ impl ProcessorRegistry { true, )?; } - drop(lock); Ok(plan.report) } diff --git a/src/world_admission.rs b/src/world_admission.rs new file mode 100644 index 0000000..ae99db0 --- /dev/null +++ b/src/world_admission.rs @@ -0,0 +1,424 @@ +//! Owner-serialized typed admission, durable boundaries and generic effect fences. +//! No external effect is executed here, and a replay never grants dispatch authority. +use super::{persist, timestamp, workers, ScenarioChange, World, WorldManager}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::collections::BTreeMap; + +type Result = std::result::Result; +const MAX_ADMISSIONS: usize = 4096; +const MAX_EFFECTS: usize = 1024; + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EffectRequest { + pub key: String, + pub phase: String, + #[serde(default)] + pub reservation_key: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AdmitInputs { + pub id: String, + pub expected_generation: u64, + pub expected_revision: u64, + pub admission_key: String, + pub changes: Vec, + #[serde(default)] + pub worker_id: Option, + #[serde(default)] + pub effect: Option, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AdmissionQuery { + pub id: String, + pub generation: u64, + pub admission_key: String, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReadBatch { + pub id: String, + pub expected_generation: u64, + pub expected_revision: u64, + #[serde(default)] + pub worker_id: Option, + pub queries: Vec, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReadQuery { + pub predicate: String, + #[serde(default = "default_rows")] + pub max_rows: usize, + #[serde(default)] + pub continuation: Option, +} +fn default_rows() -> usize { + 100 +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Effect { + world_id: String, + generation: u64, + worker_id: Option, + reservation_key: String, + state: String, + settlement_key: Option, + revision: u64, +} +pub(super) type Effects = BTreeMap; +#[derive(Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct AdmissionState { + records: BTreeMap, + pub effects: Effects, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Record { + generation: u64, + admission_key: String, + request_sha256: String, + effect: Option, + state: String, + applied_revision: Option, + receipt: Option, + error: Option, + publication: String, + at_unix_ms: u128, +} +impl Record { + fn result(&self, id: &str, replayed: bool, authority: bool) -> Value { + json!({"schema_version":1,"id":id,"generation":self.generation,"admission_key":self.admission_key, + "state":self.state,"applied_revision":self.applied_revision,"receipt":self.receipt,"error":self.error, + "publication":self.publication,"replayed":replayed, + "effect_authorized":authority && !replayed && self.state=="durable" && self.effect.as_ref().is_some_and(|e|e.phase=="reserve")}) + } +} +fn record_key(generation: u64, key: &str) -> String { + format!("{generation}/{key}") +} +fn fence(world: &World, generation: u64, revision: u64, worker_id: Option<&str>) -> Result<()> { + if world.state != "running" || world.pending.is_some() || world.generation != generation { + return Err("Expected running generation mismatch; request not applied".into()); + } + let instance = world + .instance + .as_ref() + .ok_or("World has no live instance")?; + if instance.backend.revision() != revision { + return Err("Expected revision mismatch; request not applied".into()); + } + workers::check_worker(world, worker_id) +} +fn validate_effect(world: &World, request: &AdmitInputs) -> Result<()> { + let Some(effect) = &request.effect else { + return Ok(()); + }; + workers::token(&effect.key)?; + match effect.phase.as_str() { + "reserve" => { + if effect.reservation_key.is_some() { + return Err("Reservation must not supply a reservation_key".into()); + } + if world.admission.effects.contains_key(&effect.key) { + return Err( + "Effect already reserved or settled; automatic redispatch is forbidden".into(), + ); + } + if world.admission.effects.len() >= MAX_EFFECTS { + return Err("World effect retention limit reached".into()); + } + } + "settle" => { + let key = effect + .reservation_key + .as_ref() + .ok_or("Settlement requires reservation_key")?; + let prior = world + .admission + .effects + .get(&effect.key) + .ok_or("Effect has no reservation")?; + if prior.state != "reserved" + || prior.world_id != request.id + || prior.generation != request.expected_generation + || prior.worker_id != request.worker_id + || &prior.reservation_key != key + { + return Err( + "Effect settlement fence mismatch; duplicate/late settlement is forbidden" + .into(), + ); + } + } + _ => return Err("Effect phase must be reserve or settle".into()), + } + Ok(()) +} +fn apply_effect(world: &mut World, request: &AdmitInputs, revision: u64) { + if let Some(effect) = &request.effect { + if effect.phase == "reserve" { + world.admission.effects.insert( + effect.key.clone(), + Effect { + world_id: request.id.clone(), + generation: request.expected_generation, + worker_id: request.worker_id.clone(), + reservation_key: request.admission_key.clone(), + state: "reserved".into(), + settlement_key: None, + revision, + }, + ); + } else if let Some(prior) = world.admission.effects.get_mut(&effect.key) { + prior.state = "settled".into(); + prior.settlement_key = Some(request.admission_key.clone()); + prior.revision = revision; + } + } +} +pub(super) fn validate_effects(value: Option<&Value>) -> Result { + let effects: Effects = match value { + None => BTreeMap::new(), + Some(v) => { + serde_json::from_value(v.clone()).map_err(|_| "Invalid checkpoint effect metadata")? + } + }; + if effects.len() > MAX_EFFECTS { + return Err("Checkpoint effect metadata exceeds limit".into()); + } + for (key, effect) in &effects { + workers::token(key)?; + workers::token(&effect.reservation_key)?; + workers::token(&effect.world_id)?; + if effect.generation == 0 + || effect.revision == 0 + || !matches!( + effect.state.as_str(), + "reserved" | "settled" | "unpublished" | "uncertain" + ) + { + return Err("Invalid checkpoint effect fence".into()); + } + } + Ok(effects) +} +pub(super) fn recover(world: &mut World) { + for record in world.admission.records.values_mut() { + if matches!(record.state.as_str(), "pending" | "applied") { + record.state = "uncertain".into(); + record.error = + Some("Owner exited before a durable admission outcome was recorded".into()); + record.publication = "uncertain".into(); + record.receipt = None; + world.persistence_dirty = true; + } + } +} +impl WorldManager { + pub fn read_batch(&mut self, request: ReadBatch) -> Result { + self.ensure_starting_allowed()?; + self.status_with(&request.id, true)?; + let world = self.worlds.get_mut(&request.id).ok_or("Unknown world")?; + fence( + world, + request.expected_generation, + request.expected_revision, + request.worker_id.as_deref(), + )?; + if request.queries.is_empty() + || request.queries.len() > 16 + || request + .queries + .iter() + .any(|q| q.max_rows == 0 || q.max_rows > 1000) + || request.queries.iter().map(|q| q.max_rows).sum::() > 1000 + { + return Err("Read batch requires 1–16 queries and at most 1000 requested rows".into()); + } + let mut results = Vec::new(); + let mut bytes = 0; + for query in request.queries { + let page=world.instance.as_mut().unwrap().execute("query_rows",&json!({"predicate":query.predicate,"max_rows":query.max_rows,"continuation":query.continuation,"max_bytes":crate::MAX_QUERY_BYTES}))?; + bytes += serde_json::to_vec(&page).map_err(|e| e.to_string())?.len(); + if bytes > crate::MAX_QUERY_BYTES { + return Err("Read batch exceeds 4 MiB; reduce page sizes".into()); + } + results.push(page); + } + let result = json!({"schema_version":1,"id":request.id,"generation":world.generation,"revision":request.expected_revision,"results":results}); + if serde_json::to_vec(&result) + .map_err(|e| e.to_string())? + .len() + > crate::MAX_QUERY_BYTES + { + return Err("Read batch exceeds 4 MiB; reduce page sizes".into()); + } + Ok(result) + } + pub fn admission_status(&mut self, request: AdmissionQuery) -> Result { + let world = self.worlds.get(&request.id).ok_or("Unknown world")?; + let record = world + .admission + .records + .get(&record_key(request.generation, &request.admission_key)) + .ok_or("Unknown admission")?; + Ok(record.result(&request.id, true, false)) + } + pub fn admit_inputs(&mut self, request: AdmitInputs) -> Result { + workers::token(&request.admission_key)?; + if request.changes.len() > 1000 + || serde_json::to_vec(&request) + .map_err(|e| e.to_string())? + .len() + > 256 * 1024 + { + return Err("Admission exceeds 1000 changes or 256 KiB".into()); + } + let key = record_key(request.expected_generation, &request.admission_key); + let hash = workers::digest(&request)?; + let world = self.worlds.get(&request.id).ok_or("Unknown world")?; + if let Some(record) = world.admission.records.get(&key) { + if record.request_sha256 != hash { + return Err( + "Admission key already identifies different contents; request not applied" + .into(), + ); + } + return Ok(record.result(&request.id, true, false)); + } + let mut record = Record { + generation: request.expected_generation, + admission_key: request.admission_key.clone(), + request_sha256: hash, + effect: request.effect.clone(), + state: "not_applied".into(), + applied_revision: None, + receipt: None, + error: None, + publication: "not_attempted".into(), + at_unix_ms: timestamp(), + }; + let prepared = (|| { + self.ensure_starting_allowed()?; + self.status_with(&request.id, true)?; + let world = self.worlds.get(&request.id).ok_or("Unknown world")?; + fence( + world, + request.expected_generation, + request.expected_revision, + request.worker_id.as_deref(), + )?; + if world.admission.records.len() >= MAX_ADMISSIONS { + return Err("World admission retention limit reached".into()); + } + validate_effect(world, &request)?; + let instance = world.instance.as_ref().unwrap(); + // Reject unsupported checkpoint programs before a mutation. The + // later publication still has an independent failure outcome. + instance.checkpoint_identity()?; + instance.backend.checkpoint_bytes(Value::Null)?; + instance.prepare_admission_changes( + &serde_json::to_value(&request.changes).map_err(|e| e.to_string())?, + ) + })(); + let changes = match prepared { + Ok(c) => c, + Err(e) => { + record.error = Some(e); + return Ok(record.result(&request.id, false, false)); + } + }; + let world = self.worlds.get_mut(&request.id).unwrap(); + record.state = "pending".into(); + world.admission.records.insert(key.clone(), record.clone()); + world.persistence_dirty = true; + if let Err(error) = persist(&self.build_root, &request.id, world) { + record.state = "not_applied".into(); + record.error = Some(error); + world.admission.records.insert(key, record.clone()); + world.persistence_dirty = true; + return Ok(record.result(&request.id, false, false)); + } + match world + .instance + .as_mut() + .unwrap() + .backend + .apply_without_deltas(&changes) + { + Ok(_) => { + let revision = world.instance.as_ref().unwrap().backend.revision(); + record.applied_revision = Some(revision); + record.state = "applied".into(); + apply_effect(world, &request, revision); + world.admission.records.insert(key.clone(), record.clone()); + world.persistence_dirty = true; + } + Err(_) => { + record.state = "uncertain".into(); + record.error = + Some("Native input acknowledgment failed; reconcile explicitly".into()); + world.admission.records.insert(key, record.clone()); + world.persistence_dirty = true; + let _ = persist(&self.build_root, &request.id, world); + return Ok(record.result(&request.id, false, false)); + } + } + match self.checkpoint(&request.id) { + Ok(receipt) => { + record.state = "durable".into(); + record.publication = "published".into(); + record.receipt = Some(receipt); + } + Err(error) => { + record.state = "applied_but_unpublished".into(); + record.publication = "uncertain".into(); + record.error = Some(error); + if let Some(effect) = &request.effect { + if let Some(effect) = self + .worlds + .get_mut(&request.id) + .unwrap() + .admission + .effects + .get_mut(&effect.key) + { + effect.state = "unpublished".into(); + } + } + } + } + let world = self.worlds.get_mut(&request.id).unwrap(); + workers::poll(world); + let authority = workers::check_worker(world, request.worker_id.as_deref()).is_ok() + && !self + .shutdown + .stopped + .load(std::sync::atomic::Ordering::SeqCst); + world.admission.records.insert(key.clone(), record.clone()); + world.persistence_dirty = true; + if let Err(error) = persist(&self.build_root, &request.id, world) { + record.state = "uncertain".into(); + record.publication = "uncertain".into(); + record.error = Some(error); + record.receipt = None; + if let Some(effect) = request + .effect + .as_ref() + .and_then(|e| world.admission.effects.get_mut(&e.key)) + { + effect.state = "uncertain".into(); + } + world.admission.records.insert(key, record.clone()); + world.persistence_dirty = true; + } + Ok(record.result(&request.id, false, authority)) + } +} diff --git a/src/world_library.rs b/src/world_library.rs new file mode 100644 index 0000000..3c4814f --- /dev/null +++ b/src/world_library.rs @@ -0,0 +1,200 @@ +//! Saved library artifacts: exact registry admission followed by catalog and +//! scenario publication. No compilation, world creation, or library-specific logic. +use super::*; +use sha2::{Digest, Sha256}; + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LibraryArtifact { + pub schema_version: u32, + pub library: ArtifactLibrary, + pub entries: Vec, + pub pins: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactLibrary { + pub key: String, + pub name: String, + pub repository: String, + pub package: String, + pub package_version: String, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactEntry { + pub key: String, + pub name: String, + pub description: String, + pub source: Vec, + pub git_provenance: Option, + pub record: ProcessorVersion, + pub scenarios: Vec, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LibraryImportRequest { + pub artifact: LibraryArtifact, + /// Explicit source/build provenance supplied by the publisher, never + /// inferred from package_version or a directory name. + pub revision: String, + #[serde(default)] + pub dry_run: bool, +} + +fn bounded(value: &str, label: &str) -> Result<(), String> { + if value.trim().is_empty() || value.len() > 4096 { + return Err(format!("{label} must contain 1–4096 bytes")); + } + Ok(()) +} + +impl WorldManager { + /// A replayable artifact import. Validation failure writes nothing. An I/O + /// failure may leave valid immutable records or saved scenarios; repeating + /// the same artifact reconciles them. Existing world pins are never changed. + pub fn library_import(&self, request: LibraryImportRequest) -> Result { + let artifact = &request.artifact; + if artifact.schema_version != 1 + || artifact.entries.is_empty() + || artifact.entries.len() > 4096 + { + return Err("Library artifact requires schema_version 1 and 1–4096 entries".into()); + } + let library = &artifact.library; + for (label, value) in [ + ("Library key", &library.key), + ("Library name", &library.name), + ("Repository", &library.repository), + ("Package", &library.package), + ("Package version", &library.package_version), + ("Revision", &request.revision), + ] { + bounded(value, label)?; + } + let id = format!( + "{:x}", + Sha256::digest( + serde_json::to_vec(&json!({ + "schema_version":1,"key":library.key,"repository":library.repository + })) + .map_err(|e| e.to_string())? + ) + ); + let mut catalog = self.stored_catalog()?; + let mut keys = BTreeSet::new(); + let mut pins = BTreeSet::new(); + for entry in &artifact.entries { + bounded(&entry.key, "Entry key")?; + validate_name_description(&entry.name, &entry.description)?; + if !keys.insert(entry.key.clone()) { + return Err("Duplicate library entry key".into()); + } + let record = &entry.record; + if !pins.insert((record.processor_id.clone(), record.version.clone())) { + return Err("A library artifact cannot name the same pin twice".into()); + } + let pin = ProcessorReference { + processor_id: record.processor_id.clone(), + version: record.version.clone(), + }; + if artifact.pins.get(&entry.key) != Some(&pin) + || entry.git_provenance != record.git_provenance + { + return Err(format!( + "Library entry {} pin or provenance does not match its exact record", + entry.key + )); + } + for source in &entry.source { + bounded(source, "Source path")?; + if std::path::Path::new(source).is_absolute() + || source.split('/').any(|p| p == "..") + { + return Err("Library source locations must be repository-relative".into()); + } + } + validate_scenarios(&public_relations(record)?, &entry.scenarios)?; + for existing in catalog["libraries"].as_array().unwrap() { + if existing["id"] != id + && existing["id"] != UNASSIGNED_LIBRARY + && existing["processors"].as_array().unwrap().iter().any(|p| { + p["processor_id"] == record.processor_id && p["version"] == record.version + }) + { + return Err(format!("Pin for {} already belongs to another library; reconcile its association explicitly", entry.key)); + } + } + } + if keys.len() != artifact.pins.len() { + return Err("Library pins contain unknown keys".into()); + } + let registry = self.registry()?; + let records: Vec<_> = artifact.entries.iter().map(|e| e.record.clone()).collect(); + let checked = registry.import_records(records.clone(), true)?; + if !checked.errors.is_empty() { + return Err(format!( + "Library admission failed: {}", + serde_json::to_string(&checked.errors).map_err(|e| e.to_string())? + )); + } + if request.dry_run { + return Ok( + json!({"schema_version":1,"dry_run":true,"library_id":id,"pins":artifact.pins,"records":checked.imported.len()}), + ); + } + let imported = registry.import_records(records, false)?; + if !imported.errors.is_empty() { + return Err(format!( + "Library record publication incomplete: {}", + serde_json::to_string(&imported.errors).map_err(|e| e.to_string())? + )); + } + // Scenarios are exact-pin sidecars. Catalog publication comes last so + // a newly visible library has all its saved scenarios in place. + for entry in &artifact.entries { + self.scenarios_set( + &entry.record.processor_id, + &entry.record.version, + entry.scenarios.clone(), + )?; + } + let libraries = catalog["libraries"].as_array_mut().unwrap(); + for existing in libraries.iter_mut() { + existing["processors"].as_array_mut().unwrap().retain(|p| { + !pins.contains(&( + p["processor_id"].as_str().unwrap_or("").into(), + p["version"].as_str().unwrap_or("").into(), + )) + }); + } + let index = if let Some(index) = libraries.iter().position(|l| l["id"] == id) { + index + } else { + libraries.push(json!({"id":id,"registered_at_unix_ms":timestamp(),"processors":[]})); + libraries.len() - 1 + }; + let saved = &mut libraries[index]; + saved["key"] = json!(library.key); + saved["name"] = json!(library.name); + saved["repository"] = json!(library.repository); + saved["revision"] = json!(request.revision); + saved["package"] = json!(library.package); + saved["package_version"] = json!(library.package_version); + saved["definitions"] = json!("registered_exact_records"); + saved["provenance"] = json!("publisher_supplied"); + for entry in &artifact.entries { + saved["processors"].as_array_mut().unwrap().push(json!({ + "key":entry.key,"processor_id":entry.record.processor_id,"version":entry.record.version, + "name":entry.name,"description":entry.description,"source":entry.source + })); + } + atomic_json(&self.build_root.join("libraries.json"), &catalog)?; + Ok( + json!({"schema_version":1,"dry_run":false,"library_id":id,"pins":artifact.pins,"records":imported.imported.len()}), + ) + } +} diff --git a/src/world_persistence.rs b/src/world_persistence.rs new file mode 100644 index 0000000..1b3838c --- /dev/null +++ b/src/world_persistence.rs @@ -0,0 +1,507 @@ +//! Durable world records and managed checkpoint publications. Receipt paths are always +//! derived from private owner state; clients supply an exact receipt, never a path. +use super::{timestamp, World, WorldManager}; +use crate::instance::ProgramIdentity; +use crate::registry::ProcessorRegistry; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +type Result = std::result::Result; +const MAX_RECEIPT_BYTES: u64 = 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Origin { + pub world_id: String, + pub generation: u64, + pub revision: u64, + pub program_version: u64, + /// Provenance of the actual origin executable, not a promise that a rebuilt + /// executable is byte-identical under another native toolchain. + pub build: Value, +} +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Receipt { + pub schema_version: u32, + pub format: String, + pub receipt_id: String, + pub program: ProgramIdentity, + pub origin: Origin, + pub checkpoint_sha256: String, + pub published_at_unix_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, +} +impl Receipt { + pub fn metadata(&self) -> Value { + json!({"world_checkpoint":{"schema_version":self.schema_version, + "receipt_id":self.receipt_id,"program":self.program,"origin":self.origin}}) + } + pub fn validate(&self) -> Result<()> { + component(&self.receipt_id)?; + component(&self.origin.world_id)?; + if self.schema_version != 1 + || !matches!(self.format.as_str(), "json" | "iceberg") + || (self.format == "json" && self.storage.is_some()) + || (self.format == "iceberg" && self.storage.is_none()) + || self.origin.generation == 0 + || self.origin.program_version == 0 + || self.origin.revision < self.origin.program_version + || self.checkpoint_sha256.len() != 64 + || !self + .checkpoint_sha256 + .bytes() + .all(|c| c.is_ascii_hexdigit()) + || self.published_at_unix_ms == Some(0) + { + return Err("Invalid managed checkpoint receipt".into()); + } + Ok(()) + } +} + +/// Additive defaults keep old world records readable. Publications themselves +/// live separately so a failed world.json write cannot erase a durable receipt. +#[derive(Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Persistence { + pub checkpoint_error: Option, + pub restore_requested: Option, + pub restored_from: Option, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Publication { + pub status: String, + pub receipt: Receipt, + pub error: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub staged: Option, +} + +/// Validated bytes are owned by the asynchronous start thread. There is no +/// second filesystem read between verification and replay. +pub(super) struct Restore { + pub receipt: Receipt, + pub source: RestoreSource, +} +pub(super) enum RestoreSource { + Json(Vec), + #[cfg(feature = "iceberg")] + Iceberg(super::storage::Read), +} +impl Restore { + pub fn install( + self, + instance: &mut crate::ProgramInstance, + ) -> Result { + #[cfg(not(feature = "iceberg"))] + let RestoreSource::Json(bytes) = self.source; + #[cfg(feature = "iceberg")] + let bytes = match self.source { + RestoreSource::Json(bytes) => bytes, + #[cfg(feature = "iceberg")] + RestoreSource::Iceberg(read) => read.load(instance.backend.control.clone())?, + }; + let effects = validate_snapshot(&self.receipt, &bytes)?; + instance.restore_pinned(&bytes, &self.receipt.program)?; + Ok(effects) + } +} +pub(super) fn validate_snapshot( + receipt: &Receipt, + bytes: &[u8], +) -> Result { + let (digest, state) = crate::checkpoint::inspect(bytes)?; + if digest != receipt.checkpoint_sha256 + || state.metadata["world_checkpoint"] != receipt.metadata()["world_checkpoint"] + || state + .metadata + .as_object() + .is_none_or(|m| m.keys().any(|k| k != "world_checkpoint" && k != "effects")) + || state.revision != receipt.origin.revision + || state.program_version != receipt.origin.program_version + { + return Err("Checkpoint digest, origin or metadata does not match receipt".into()); + } + super::admission::validate_effects(state.metadata.get("effects")) +} + +pub(super) fn component(id: &str) -> Result<()> { + if id.is_empty() + || id.len() > 128 + || !id.bytes().all(|b| b.is_ascii_digit() || b == b'-') + || !id.bytes().any(|b| b.is_ascii_digit()) + { + return Err("Invalid relative world/receipt ID".into()); + } + Ok(()) +} +pub(super) fn private_dir(path: &Path) -> Result<()> { + // Reuse the existing same-user, non-symlink, owner-private directory contract. + ProcessorRegistry::open(path.to_path_buf()).map(|_| ()) +} +pub(super) fn existing_dir(path: &Path) -> Result<()> { + let meta = fs::symlink_metadata(path).map_err(|e| e.to_string())?; + if !meta.is_dir() || meta.file_type().is_symlink() { + return Err("Managed persistence directory is missing or is a symlink".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if meta.uid() != unsafe { libc::geteuid() } || meta.mode() & 0o077 != 0 { + return Err("Managed persistence directory must be owner-private".into()); + } + } + Ok(()) +} +pub(super) fn checkpoint_root(root: &Path, id: &str) -> Result { + component(id)?; + let world = root.join(id); + // World directories in older stores inherit the private build root's + // protection, so check their type but do not impose a new permission format. + if !fs::symlink_metadata(&world) + .map_err(|e| e.to_string())? + .is_dir() + { + return Err("Invalid origin world directory".into()); + } + Ok(world.join("checkpoints")) +} +pub(super) fn receipt_dir(root: &Path, receipt: &Receipt) -> Result { + receipt.validate()?; + let checkpoints = checkpoint_root(root, &receipt.origin.world_id)?; + existing_dir(&checkpoints)?; + let dir = checkpoints.join(&receipt.receipt_id); + existing_dir(&dir)?; + Ok(dir) +} +pub(super) fn read_bounded(path: &Path, max: u64) -> Result> { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + let file = options.open(path).map_err(|e| e.to_string())?; + let meta = file.metadata().map_err(|e| e.to_string())?; + if !meta.is_file() || meta.len() > max { + return Err("Invalid or oversized managed persistence file".into()); + } + let mut bytes = Vec::new(); + file.take(max + 1) + .read_to_end(&mut bytes) + .map_err(|e| e.to_string())?; + if bytes.len() as u64 > max { + return Err("Managed persistence file exceeds size limit".into()); + } + Ok(bytes) +} +pub(super) fn read_publication(dir: &Path) -> Result { + let publication: Publication = serde_json::from_slice(&read_bounded( + &dir.join("publication.json"), + MAX_RECEIPT_BYTES, + )?) + .map_err(|e| e.to_string())?; + publication.receipt.validate()?; + if !matches!( + publication.status.as_str(), + "staging" | "staged" | "publishing" | "published" | "failed" | "uncertain" + ) { + return Err("Invalid checkpoint publication state".into()); + } + if publication.status == "published" && publication.receipt.published_at_unix_ms.is_none() { + return Err("Published receipt has no publication time".into()); + } + Ok(publication) +} + +impl WorldManager { + /// Synchronous bounded snapshot/publication (64 MiB JSON format limit). + /// A receipt is returned only after both checkpoint and publication fsync. + pub fn checkpoint(&mut self, id: &str) -> Result { + self.status_with(id, true)?; + let result = self.publish_checkpoint(id); + let world = self.worlds.get_mut(id).ok_or("Unknown world")?; + world.persistence.checkpoint_error = result.as_ref().err().cloned(); + world.persistence_dirty = true; + // Receipt publication is independently durable. A world-record error + // cannot turn it into a returned success or discard its history. + persist(&self.build_root, id, world)?; + result + } + fn publish_checkpoint(&self, id: &str) -> Result { + let world = self.worlds.get(id).ok_or("Unknown world")?; + let (receipt, bytes) = freeze(world, id, "json")?; + let checkpoints = checkpoint_root(&self.build_root, id)?; + private_dir(&checkpoints)?; + let dir = checkpoints.join(&receipt.receipt_id); + fs::create_dir(&dir).map_err(|e| e.to_string())?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&dir, fs::Permissions::from_mode(0o700)) + .map_err(|e| e.to_string())?; + } + File::open(&checkpoints) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string())?; + File::open(self.build_root.join(id)) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string())?; + let mut publication = Publication { + status: "staging".into(), + receipt, + error: None, + staged: None, + }; + let manifest = dir.join("publication.json"); + let write_publication = |p: &Publication| write_record(&manifest, p); + write_publication(&publication)?; + let result = (|| { + atomic_bytes(&dir.join("checkpoint.json"), &bytes)?; + publication.status = "staged".into(); + write_publication(&publication)?; + publication.status = "published".into(); + publication.receipt.published_at_unix_ms = Some(timestamp()); + write_publication(&publication) + })(); + if let Err(error) = result { + publication.status = if dir.join("checkpoint.json").exists() { + "uncertain" + } else { + "failed" + } + .into(); + publication.receipt.published_at_unix_ms = None; + publication.error = Some(error.clone()); + let _ = write_publication(&publication); + return Err(format!( + "Checkpoint publication failed; inspect persistence before retrying: {error}" + )); + } + Ok(json!(publication.receipt)) + } + + /// Restore is explicit and asynchronous. The immutable receipt must be + /// published in this owner's private store; another world may use it only + /// with the identical processor pin. Ordinary Start never consults it. + pub fn restore_async(&mut self, id: &str, receipt: &Value) -> Result { + self.ensure_starting_allowed()?; + self.status_with(id, true)?; + let world = self.worlds.get(id).ok_or("Unknown world")?; + if !matches!( + world.state.as_str(), + "created" | "stopped" | "failed" | "interrupted" + ) || world.instance.is_some() + || world.pending.is_some() + { + return Err("Restore requires a created, stopped, failed or interrupted world with no live/pending instance".into()); + } + let receipt: Receipt = + serde_json::from_value(receipt.clone()).map_err(|e| e.to_string())?; + if receipt.program.processor != world.definition.processor { + return Err("Restore receipt processor pin does not match target world".into()); + } + let dir = receipt_dir(&self.build_root, &receipt)?; + let publication = read_publication(&dir)?; + if publication.status != "published" || publication.receipt != receipt { + return Err("Restore requires the exact immutable published receipt".into()); + } + let source = if receipt.format == "json" { + let bytes = read_bounded(&dir.join("checkpoint.json"), crate::checkpoint::MAX_BYTES)?; + validate_snapshot(&receipt, &bytes)?; + RestoreSource::Json(bytes) + } else { + self.iceberg_restore_source(&publication)? + }; + self.start_with(id, true, Some(Restore { receipt, source })) + } +} + +/// Shared identity and input freeze. Storage adapters never reconstruct inputs. +pub(super) fn freeze(world: &World, id: &str, format: &str) -> Result<(Receipt, Vec)> { + if world.state != "running" || world.pending.is_some() { + return Err("Checkpoint requires a running world without pending lifecycle work".into()); + } + let instance = world.instance.as_ref().ok_or("World is not running")?; + let program = instance.checkpoint_identity()?; + let mut receipt = Receipt { + schema_version: 1, + format: format.into(), + storage: None, + receipt_id: crate::bounded::owner_identity(), + origin: Origin { + world_id: id.into(), + generation: world.generation, + revision: instance.backend.revision(), + program_version: instance.backend.version, + build: instance.backend.build_identity(), + }, + program, + checkpoint_sha256: String::new(), + published_at_unix_ms: None, + }; + // Backend owns format validation and the acknowledged input snapshot, + // including empty inputs and format limits. No output/provider replay. + let mut metadata = receipt.metadata(); + if !world.admission.effects.is_empty() { + metadata["effects"] = + serde_json::to_value(&world.admission.effects).map_err(|e| e.to_string())?; + } + let bytes = instance.backend.checkpoint_bytes(metadata)?; + receipt.checkpoint_sha256 = crate::checkpoint::inspect(&bytes)?.0; + Ok((receipt, bytes)) +} +pub(super) fn write_record(path: &Path, p: &Publication) -> Result<()> { + let value = serde_json::to_value(p).map_err(|e| e.to_string())?; + if serde_json::to_vec(&value).map_err(|e| e.to_string())?.len() as u64 > MAX_RECEIPT_BYTES { + return Err("Managed receipt exceeds 1 MiB limit".into()); + } + atomic_json(path, &value) +} + +/// Availability is a cheap filesystem observation, never a claim of verified +/// contents or valid registry dependencies. Restore verifies both before launch. +pub(super) fn status(root: &Path, id: &str, world: &World, revision: &Value) -> Value { + let mut checkpoints = Vec::new(); + let entries = (|| -> Result<_> { + let path = checkpoint_root(root, id)?; + match fs::symlink_metadata(&path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.to_string()), + Ok(_) => (), + } + existing_dir(&path)?; + Ok(Some(fs::read_dir(path).map_err(|e| e.to_string())?)) + })(); + let mut error = world.persistence.checkpoint_error.clone(); + match entries { + Err(e) => error = Some(e), + Ok(None) => (), + Ok(Some(entries)) => { + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(e) => { + error = Some(e.to_string()); + continue; + } + }; + let result = (|| -> Result { + let name = entry.file_name().to_string_lossy().into_owned(); + component(&name)?; + existing_dir(&entry.path())?; + let publication = read_publication(&entry.path())?; + if publication.receipt.receipt_id != name + || publication.receipt.origin.world_id != id + { + return Err("Receipt location/origin mismatch".into()); + } + Ok(publication) + })(); + match result { + Err(e) => checkpoints.push(json!({"receipt_id":entry.file_name().to_string_lossy(),"status":"unavailable","availability":"invalid","error":e})), + Ok(p) => { + let present = fs::symlink_metadata(entry.path().join("checkpoint.json")) + .is_ok_and(|m| m.is_file() && m.len() <= crate::checkpoint::MAX_BYTES); + checkpoints.push(json!({"receipt":p.receipt,"status":p.status,"error":p.error, + "availability":if p.receipt.format == "iceberg" {"catalog_unverified"} else if present {"present_unverified"} else {"missing_or_invalid"}})); + } + } + } + } + } + checkpoints.sort_by_key(|p| p["receipt"]["published_at_unix_ms"].as_u64().unwrap_or(0)); + let persisted_revision = checkpoints + .iter() + .filter(|p| { + p["status"] == "published" + && p["availability"] == "present_unverified" + && p["receipt"]["origin"]["generation"] == world.generation + }) + .filter_map(|p| p["receipt"]["origin"]["revision"].as_u64()) + .chain( + world + .persistence + .restored_from + .iter() + .filter(|r| r.format == "json") + .map(|r| r.origin.revision), + ) + .max(); + json!({"schema_version":1,"status":if error.is_some() {"error"} else {"configured"}, "format":"json", + "coverage":"whole_pure_program_inputs", "committed_revision":revision, + "persisted_revision":persisted_revision,"checkpoints":checkpoints,"error":error, + "restore_requested":world.persistence.restore_requested,"restored_from":world.persistence.restored_from, + "verification":"Receipt/object presence only; restore verifies integrity and pinned dependencies before compilation"}) +} + +pub(super) fn persist_if_changed( + root: &std::path::Path, + id: &str, + world: &mut World, +) -> Result<()> { + if world.persistence_dirty + || world.history.last().is_none_or(|last| { + last["state"] != world.state + || last["generation"] != world.generation + || last["error"] != json!(world.error) + }) + { + persist(root, id, world)?; + } + Ok(()) +} +pub(super) fn persist(root: &std::path::Path, id: &str, world: &mut World) -> Result<()> { + let mut history = world.history.clone(); + if history.last().is_none_or(|last| { + last["state"] != world.state + || last["generation"] != world.generation + || last["error"] != json!(world.error) + }) { + history.push(json!({"at_unix_ms":timestamp(),"state":world.state,"generation":world.generation,"error":world.error,"restore_requested":world.persistence.restore_requested,"restored_from":world.persistence.restored_from})); + } + let path = root.join(id).join("world.json"); + let mut record = serde_json::to_value(&*world).map_err(|e| e.to_string())?; + record["history"] = json!(history); + atomic_json(&path, &record)?; + world.history = history; + world.persistence_dirty = false; + Ok(()) +} + +pub(super) fn atomic_json(path: &Path, value: &Value) -> Result<()> { + atomic_bytes(path, &serde_json::to_vec(value).map_err(|e| e.to_string())?) +} +fn atomic_bytes(path: &Path, bytes: &[u8]) -> Result<()> { + let parent = path.parent().ok_or("Missing persistence parent")?; + let temp = path.with_extension("json.tmp"); + let mut options = OpenOptions::new(); + options.create(true).truncate(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(&temp).map_err(|e| e.to_string())?; + let result = (|| -> std::io::Result<()> { + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + fs::rename(&temp, path)?; + File::open(parent)?.sync_all() + })(); + if let Err(error) = result { + let _ = fs::remove_file(temp); + return Err(format!( + "Durable publication failed (target may exist after rename): {error}" + )); + } + Ok(()) +} diff --git a/src/world_storage.rs b/src/world_storage.rs new file mode 100644 index 0000000..26c696f --- /dev/null +++ b/src/world_storage.rs @@ -0,0 +1,728 @@ +//! One trusted local Iceberg profile, off-lock storage jobs, and exact receipts. +//! Catalog visibility is independent of the JSON durable-admission policy. +use super::{persistence, World, WorldManager}; +#[cfg(feature = "iceberg")] +use persistence::Receipt; +use persistence::{Publication, RestoreSource}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::path::Path; +#[cfg(feature = "iceberg")] +use std::path::PathBuf; +type Result = std::result::Result; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Location { + pub profile: String, + pub profile_sha256: String, + pub table: Vec, + pub table_uuid: Option, + pub object_uri: String, + pub object_sha256: Option, + pub envelope_sha256: Option, + /// Decimal text preserves all 64 bits through browser JSON round trips. + pub snapshot_id: Option, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointStage { + pub id: String, + pub expected_generation: u64, + pub expected_revision: u64, + pub profile: String, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointPublish { + pub id: String, + pub receipt: Value, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CheckpointQuery { + pub id: String, + pub receipt_id: String, +} +fn entry(p: &Publication, dir: &Path) -> Value { + let present = std::fs::symlink_metadata(dir.join("checkpoint.json")) + .is_ok_and(|m| m.is_file() && m.len() <= crate::checkpoint::MAX_BYTES); + json!({"schema_version":1,"receipt":p.receipt,"status":p.status,"error":p.error, + "availability":if p.receipt.format=="iceberg" {"catalog_unverified"} else if present {"present_unverified"} else {"missing_or_invalid"}}) +} +fn observe_interrupted(entry: &mut Value) { + if entry["receipt"]["format"] == "iceberg" + && matches!(entry["status"].as_str(), Some("staging" | "publishing")) + { + entry["status"] = json!("uncertain"); + } +} +impl WorldManager { + pub fn checkpoint_status(&mut self, request: CheckpointQuery) -> Result { + self.poll_storage(); + if !self.worlds.contains_key(&request.id) { + return Err("Unknown world".into()); + } + persistence::component(&request.receipt_id)?; + let root = persistence::checkpoint_root(&self.build_root, &request.id)?; + persistence::existing_dir(&root)?; + let dir = root.join(&request.receipt_id); + persistence::existing_dir(&dir)?; + let p = persistence::read_publication(&dir)?; + if p.receipt.origin.world_id != request.id || p.receipt.receipt_id != request.receipt_id { + return Err("Receipt location mismatch".into()); + } + let mut result = entry(&p, &dir); + self.storage_configuration.observe_entry(&mut result); + Ok(result) + } + pub(super) fn poll_storage(&mut self) { + self.storage_configuration.poll(); + } +} +#[cfg(not(feature = "iceberg"))] +#[derive(Default)] +pub(super) struct Configuration { + _private: (), +} +#[cfg(not(feature = "iceberg"))] +impl Configuration { + pub fn observe_entry(&self, entry: &mut Value) { + observe_interrupted(entry); + } + pub fn cancel(&mut self, _: &str) {} + pub fn poll(&mut self) {} + pub fn status(&self, _: &World, _: &Value) -> Value { + json!({"schema_version":1,"status":"unavailable","reason":"Build with the iceberg feature", + "profile":null,"publication_ack":"catalog_visibility","admission_durability":"json", + "published_revision":null,"error":null,"job":null}) + } +} +#[cfg(not(feature = "iceberg"))] +impl WorldManager { + pub fn configure_storage(&mut self, _: &Path) -> Result<()> { + Err("Build with the iceberg feature".into()) + } + pub fn checkpoint_stage(&mut self, _: CheckpointStage) -> Result { + Err("Build with the iceberg feature".into()) + } + pub fn checkpoint_publish(&mut self, _: CheckpointPublish) -> Result { + Err("Build with the iceberg feature".into()) + } + pub(super) fn iceberg_restore_source(&self, _: &Publication) -> Result { + Err("Build with the iceberg feature".into()) + } +} +#[cfg(feature = "iceberg")] +pub(super) use enabled::{Configuration, Read}; +#[cfg(feature = "iceberg")] +mod enabled { + use super::*; + use crate::iceberg_checkpoint::{self, StagedCheckpoint}; + use crate::processes::ProcessControl; + use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation, TableIdent}; + use iceberg_catalog_sql::{SqlBindStyle, SqlCatalogBuilder}; + use std::collections::HashMap; + use std::fs::{self, File}; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, + }; + use std::time::Duration; + + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct FileConfig { + schema_version: u32, + profile: ProfileConfig, + } + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct ProfileConfig { + name: String, + root: PathBuf, + #[serde(default = "timeout")] + timeout_ms: u64, + } + fn timeout() -> u64 { + 30_000 + } + struct Profile { + name: String, + root: PathBuf, + sha256: String, + timeout_ms: u64, + _lock: File, + } + impl Profile { + fn uri(&self, path: &str) -> String { + // This pinned LocalFs FileIO strips file:// literally; percent-encoding + // would select a different directory. SQLite below does use URI syntax. + format!( + "file://{}", + self.root.join("warehouse").join(path).display() + ) + } + fn location(&self, id: &str) -> Location { + Location { + profile: self.name.clone(), + profile_sha256: self.sha256.clone(), + table: vec!["runtime".into(), "checkpoints".into()], + table_uuid: None, + object_uri: self.uri(&format!("objects/{id}.parquet")), + object_sha256: None, + envelope_sha256: None, + snapshot_id: None, + } + } + fn check(&self, receipt: &Receipt) -> Result<()> { + let location = receipt + .storage + .as_ref() + .ok_or("Missing Iceberg storage identity")?; + let expected = self.location(&receipt.receipt_id); + if receipt.format != "iceberg" + || location.profile != expected.profile + || location.profile_sha256 != expected.profile_sha256 + || location.table != expected.table + || location.object_uri != expected.object_uri + { + return Err( + "Iceberg receipt does not match the configured local profile/location".into(), + ); + } + Ok(()) + } + } + fn encoded_path(path: &Path) -> String { + path.to_string_lossy() + .bytes() + .map(|b| { + if b.is_ascii_alphanumeric() || b"/-_.~".contains(&b) { + (b as char).to_string() + } else { + format!("%{b:02X}") + } + }) + .collect() + } + fn ident() -> TableIdent { + TableIdent::from_strs(["runtime", "checkpoints"]).expect("constant table identity") + } + async fn catalog(profile: &Profile, create: bool) -> Result> { + let db = profile.root.join("catalog.sqlite"); + if let Ok(meta) = fs::symlink_metadata(&db) { + if !meta.is_file() { + return Err("Invalid local catalog file".into()); + } + } + if !create && !db.is_file() { + return Err("Local Iceberg catalog is missing".into()); + } + let catalog = SqlCatalogBuilder::default() + .uri(format!( + "sqlite://{}?mode={}", + encoded_path(&db), + if create { "rwc" } else { "rw" } + )) + .warehouse_location(profile.uri("")) + .sql_bind_style(SqlBindStyle::QMark) + .prop("pool.max-connections", "1") + .with_storage_factory(Arc::new(iceberg::io::LocalFsStorageFactory)) + .load("managed-checkpoints", HashMap::new()) + .await + .map_err(|e| e.to_string())?; + if create + && !catalog + .table_exists(&ident()) + .await + .map_err(|e| e.to_string())? + { + let namespace = NamespaceIdent::new("runtime".into()); + if !catalog + .namespace_exists(&namespace) + .await + .map_err(|e| e.to_string())? + { + catalog + .create_namespace(&namespace, HashMap::new()) + .await + .map_err(|e| e.to_string())?; + } + catalog + .create_table( + &namespace, + TableCreation::builder() + .name("checkpoints".into()) + .location(profile.uri("table")) + .schema(iceberg_checkpoint::schema()?) + .format_version(iceberg::spec::FormatVersion::V3) + .properties(HashMap::from([( + "commit.retry.num-retries".into(), + "0".into(), + )])) + .build(), + ) + .await + .map_err(|e| e.to_string())?; + } + Ok(Arc::new(catalog)) + } + fn run( + timeout_ms: u64, + cancelled: impl Fn() -> bool, + future: impl std::future::Future>, + ) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| e.to_string())?; + let result=runtime.block_on(async { + tokio::select! { + result=tokio::time::timeout(Duration::from_millis(timeout_ms),future)=>result.map_err(|_|"Storage deadline elapsed; publication may have occurred".to_string())?, + _=async {loop {if cancelled(){break;}tokio::time::sleep(Duration::from_millis(10)).await;}}=>Err("Storage operation cancelled; publication may have occurred".into()), + } + }); + // Cancellation does not roll back already dispatched filesystem/SQLite IO. + // Do not wait indefinitely for blocking IO during owner shutdown. + runtime.shutdown_timeout(Duration::from_millis(100)); + result + } + struct Job { + world: String, + generation: u64, + receipt_id: String, + phase: String, + manifest: PathBuf, + cancel: Arc, + receiver: mpsc::Receiver>, + thread: Option>, + } + impl Drop for Job { + fn drop(&mut self) { + self.cancel.store(true, Ordering::SeqCst); + } + } + #[derive(Default)] + pub(crate) struct Configuration { + profile: Option>, + job: Option, + error: Option, + uncertain: Option<(String, String)>, + } + impl Configuration { + pub fn observe_entry(&self, entry: &mut Value) { + // An interrupted job or failed manifest write cannot promise progress. + // The active job, if any, overrides this below until completion. + observe_interrupted(entry); + if self.uncertain.as_ref().is_some_and(|(world, receipt)| { + entry["receipt"]["origin"]["world_id"] == *world + && entry["receipt"]["receipt_id"] == *receipt + }) { + entry["status"] = json!("uncertain"); + entry["error"] = json!(self.error); + } + if let Some(job) = &self.job { + if entry["receipt"]["origin"]["world_id"] == job.world + && entry["receipt"]["receipt_id"] == job.receipt_id + { + // A renamed manifest is visible before its final fsync/send. + // Do not acknowledge completion until the job itself finished. + entry["status"] = json!(job.phase); + } + } + } + pub fn cancel(&mut self, id: &str) { + if let Some(job) = &self.job { + if job.world == id { + job.cancel.store(true, Ordering::SeqCst); + } + } + } + pub fn poll(&mut self) { + let result = self.job.as_ref().map(|j| j.receiver.try_recv()); + let result = match result { + Some(Ok(r)) => Some(r), + Some(Err(mpsc::TryRecvError::Disconnected)) => Some(Err( + "Storage worker exited without an outcome; inspect retained receipt".into(), + )), + _ => None, + }; + if let Some(result) = result { + if let Some(mut job) = self.job.take() { + if let Some(thread) = job.thread.take() { + let _ = thread.join(); + } + if let Err(error) = &result { + self.uncertain = Some((job.world.clone(), job.receipt_id.clone())); + if let Some(dir) = job.manifest.parent() { + if let Ok(mut p) = persistence::read_publication(dir) { + p.status = "uncertain".into(); + p.error = Some(error.clone()); + let _ = persistence::write_record(&job.manifest, &p); + } + } + } + } + self.error = result.err(); + } + } + pub fn status(&self, world: &World, persistence: &Value) -> Value { + let published = persistence["checkpoints"] + .as_array() + .into_iter() + .flatten() + .filter(|p| { + p["receipt"]["format"] == "iceberg" + && p["status"] == "published" + && p["receipt"]["origin"]["generation"] == world.generation + }) + .filter_map(|p| p["receipt"]["origin"]["revision"].as_u64()) + .chain( + world + .persistence + .restored_from + .iter() + .filter(|r| r.format == "iceberg") + .map(|r| r.origin.revision), + ) + .max(); + json!({"schema_version":1,"status":if self.profile.is_some(){"configured"}else{"not_configured"},"profile":self.profile.as_ref().map(|p|&p.name), + "publication_ack":"catalog_visibility","admission_durability":"json","published_revision":published,"error":self.error, + "job":self.job.as_ref().map(|j|json!({"id":j.world,"generation":j.generation,"receipt_id":j.receipt_id,"cancel_requested":j.cancel.load(Ordering::SeqCst)}))}) + } + fn configured(&self) -> Result> { + self.profile + .clone() + .ok_or("No local Iceberg profile configured".into()) + } + } + pub(crate) struct Read { + profile: Arc, + receipt: StagedCheckpoint, + snapshot: i64, + } + impl Read { + pub fn load(self, control: ProcessControl) -> Result> { + run(self.profile.timeout_ms, || control.stopped(), async { + let catalog = catalog(&self.profile, false).await?; + let (snapshot, bytes) = + iceberg_checkpoint::load_publication(catalog.as_ref(), &ident(), &self.receipt) + .await?; + if snapshot != self.snapshot { + return Err("Published snapshot identity mismatch".into()); + } + Ok(bytes) + }) + } + } + fn staged(p: &Publication, profile: &Profile) -> Result { + profile.check(&p.receipt)?; + let staged: StagedCheckpoint = serde_json::from_value( + p.staged + .clone() + .ok_or("No retained staged receipt; staging cannot be retried implicitly")?, + ) + .map_err(|e| e.to_string())?; + let location = p.receipt.storage.as_ref().unwrap(); + if staged.publication_id != p.receipt.receipt_id + || Some(&staged.table_uuid) != location.table_uuid.as_ref() + || staged.object_uri != location.object_uri + || Some(&staged.object_sha256) != location.object_sha256.as_ref() + || Some(&staged.checkpoint_sha256) != location.envelope_sha256.as_ref() + { + return Err("Managed and staged storage identities differ".into()); + } + Ok(staged) + } + fn stage_form(mut r: Receipt) -> Receipt { + r.published_at_unix_ms = None; + if let Some(s) = &mut r.storage { + s.snapshot_id = None; + } + r + } + enum Work { + Stage(Vec), + Publish(StagedCheckpoint), + } + impl WorldManager { + pub fn configure_storage(&mut self, path: &Path) -> Result<()> { + if self.storage_configuration.profile.is_some() + || self + .worlds + .values() + .any(|w| w.instance.is_some() || w.pending.is_some()) + { + return Err("Storage profile is startup-only".into()); + } + if !path.is_absolute() { + return Err("Storage profile file must be absolute".into()); + } + use std::os::unix::fs::MetadataExt; + let meta = fs::symlink_metadata(path).map_err(|e| e.to_string())?; + if !meta.is_file() + || meta.uid() != unsafe { libc::geteuid() } + || meta.mode() & 0o022 != 0 + { + return Err( + "Storage profile must be a same-user regular file, not group/world writable" + .into(), + ); + } + let config: FileConfig = + serde_json::from_slice(&persistence::read_bounded(path, 1024 * 1024)?) + .map_err(|e| e.to_string())?; + if config.schema_version != 1 + || !config.profile.root.is_absolute() + || !(1..=120_000).contains(&config.profile.timeout_ms) + { + return Err("Invalid local storage profile version, root or timeout".into()); + } + super::super::workers::token(&config.profile.name)?; + persistence::private_dir(&config.profile.root)?; + let root = fs::canonicalize(config.profile.root).map_err(|e| e.to_string())?; + let lock = super::super::lock_owner(&root)?; + persistence::private_dir(&root.join("warehouse"))?; + persistence::private_dir(&root.join("warehouse/objects"))?; + let sha256 = super::super::workers::digest( + &json!({"name":config.profile.name,"root":root,"kind":"local_sqlite"}), + )?; + self.storage_configuration.profile = Some(Arc::new(Profile { + name: config.profile.name, + root, + sha256, + timeout_ms: config.profile.timeout_ms, + _lock: lock, + })); + // An interrupted intent is never auto-published or adopted on restart. + for id in self.worlds.keys() { + let root = persistence::checkpoint_root(&self.build_root, id)?; + if !root.exists() { + continue; + } + persistence::existing_dir(&root)?; + for item in fs::read_dir(root).map_err(|e| e.to_string())? { + let dir = item.map_err(|e| e.to_string())?.path(); + if persistence::existing_dir(&dir).is_err() { + continue; + } + if let Ok(mut p) = persistence::read_publication(&dir) { + if p.receipt.format == "iceberg" + && matches!(p.status.as_str(), "staging" | "publishing") + { + p.status = "uncertain".into(); + p.error=Some("Previous owner exited during storage operation; reconcile the retained receipt explicitly".into()); + persistence::write_record(&dir.join("publication.json"), &p)?; + } + } + } + } + Ok(()) + } + pub fn checkpoint_stage(&mut self, request: CheckpointStage) -> Result { + self.ensure_starting_allowed()?; + self.status_with(&request.id, true)?; + if self.storage_configuration.job.is_some() { + return Err("One storage operation is already pending".into()); + } + let profile = self.storage_configuration.configured()?; + if profile.name != request.profile { + return Err("Unknown storage profile".into()); + } + let world = self.worlds.get(&request.id).ok_or("Unknown world")?; + if world.generation != request.expected_generation + || world + .instance + .as_ref() + .is_none_or(|i| i.backend.revision() != request.expected_revision) + { + return Err("Checkpoint generation/revision fence mismatch".into()); + } + let (mut receipt, bytes) = persistence::freeze(world, &request.id, "iceberg")?; + receipt.storage = Some(profile.location(&receipt.receipt_id)); + let root = persistence::checkpoint_root(&self.build_root, &request.id)?; + persistence::private_dir(&root)?; + let dir = root.join(&receipt.receipt_id); + persistence::private_dir(&dir)?; + File::open(root) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string())?; + File::open(self.build_root.join(&request.id)) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string())?; + self.launch_storage( + profile, + Publication { + status: "staging".into(), + receipt, + error: None, + staged: None, + }, + dir, + Work::Stage(bytes), + ) + } + pub fn checkpoint_publish(&mut self, request: CheckpointPublish) -> Result { + self.ensure_starting_allowed()?; + self.poll_storage(); + if self.storage_configuration.job.is_some() { + return Err("One storage operation is already pending".into()); + } + let profile = self.storage_configuration.configured()?; + let receipt: Receipt = + serde_json::from_value(request.receipt).map_err(|e| e.to_string())?; + if !self.worlds.contains_key(&request.id) || receipt.origin.world_id != request.id { + return Err("Publication must name its origin world".into()); + } + let dir = persistence::receipt_dir(&self.build_root, &receipt)?; + let mut p = persistence::read_publication(&dir)?; + if p.receipt != receipt && stage_form(p.receipt.clone()) != receipt { + return Err("Publication requires the exact retained receipt".into()); + } + if !matches!( + p.status.as_str(), + "staged" | "uncertain" | "published" | "publishing" + ) { + return Err("Publication is not ready for explicit reconciliation".into()); + } + let staged = staged(&p, &profile)?; + p.status = "publishing".into(); + p.error = None; + self.launch_storage(profile, p, dir, Work::Publish(staged)) + } + fn launch_storage( + &mut self, + profile: Arc, + mut p: Publication, + dir: PathBuf, + work: Work, + ) -> Result { + persistence::write_record(&dir.join("publication.json"), &p)?; + let ticket = json!({"schema_version":1,"id":p.receipt.origin.world_id,"receipt_id":p.receipt.receipt_id,"status":p.status}); + let cancel = Arc::new(AtomicBool::new(false)); + let stop = cancel.clone(); + let owner = self.shutdown.stopped.clone(); + let (sender, receiver) = mpsc::channel(); + let mut job = Job { + world: p.receipt.origin.world_id.clone(), + generation: p.receipt.origin.generation, + receipt_id: p.receipt.receipt_id.clone(), + phase: p.status.clone(), + manifest: dir.join("publication.json"), + cancel, + receiver, + thread: None, + }; + job.thread = Some( + std::thread::Builder::new() + .name("world-storage".into()) + .spawn(move || { + let outcome = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run( + profile.timeout_ms, + || stop.load(Ordering::SeqCst) || owner.load(Ordering::SeqCst), + async { + match work { + Work::Stage(bytes) => { + let catalog = catalog(&profile, true).await?; + let table = catalog + .load_table(&ident()) + .await + .map_err(|e| e.to_string())?; + let staged = iceberg_checkpoint::stage_bytes( + &table, + &p.receipt.receipt_id, + &p.receipt.storage.as_ref().unwrap().object_uri, + &bytes, + ) + .await?; + let location = p.receipt.storage.as_mut().unwrap(); + location.table_uuid = + Some(staged.table_uuid.clone()); + location.object_sha256 = + Some(staged.object_sha256.clone()); + location.envelope_sha256 = + Some(staged.checkpoint_sha256.clone()); + p.staged = Some( + serde_json::to_value(staged) + .map_err(|e| e.to_string())?, + ); + p.status = "staged".into(); + } + Work::Publish(staged) => { + let catalog = catalog(&profile, false).await?; + let published = iceberg_checkpoint::publish( + catalog.as_ref(), + &ident(), + &staged, + ) + .await?; + let location = p.receipt.storage.as_mut().unwrap(); + let snapshot = published.snapshot_id.to_string(); + if location + .snapshot_id + .as_ref() + .is_some_and(|s| *s != snapshot) + { + return Err( + "Publication snapshot identity changed" + .into(), + ); + } + location.snapshot_id = Some(snapshot); + p.receipt + .published_at_unix_ms + .get_or_insert_with(super::super::timestamp); + p.status = "published".into(); + } + } + Ok(()) + }, + ) + })) + .unwrap_or_else(|_| { + Err("Storage worker panicked; reconcile retained receipt".into()) + }); + if let Err(error) = &outcome { + p.status = "uncertain".into(); + p.error = Some(error.clone()); + } + let saved = persistence::write_record(&dir.join("publication.json"), &p); + let _ = sender.send(saved.and(outcome)); + }) + .map_err(|e| e.to_string())?, + ); + self.storage_configuration.error = None; + self.storage_configuration.uncertain = None; + self.storage_configuration.job = Some(job); + Ok(ticket) + } + pub(in crate::worlds) fn iceberg_restore_source( + &self, + p: &Publication, + ) -> Result { + if self.storage_configuration.job.as_ref().is_some_and(|job| { + job.world == p.receipt.origin.world_id && job.receipt_id == p.receipt.receipt_id + }) { + return Err("Publication is still pending; poll checkpoint_status".into()); + } + let profile = self.storage_configuration.configured()?; + let staged = staged(p, &profile)?; + let snapshot = p + .receipt + .storage + .as_ref() + .and_then(|s| s.snapshot_id.as_deref()) + .ok_or("Missing published Iceberg snapshot")? + .parse::() + .map_err(|_| "Invalid published Iceberg snapshot")?; + Ok(RestoreSource::Iceberg(Read { + profile, + receipt: staged, + snapshot, + })) + } + } +} diff --git a/src/world_workers.rs b/src/world_workers.rs new file mode 100644 index 0000000..dd9ad9a --- /dev/null +++ b/src/world_workers.rs @@ -0,0 +1,711 @@ +//! Trusted external children share their world's existing process control. +//! Application payload/configuration is delivered only to stdin, never status. +use super::{persist, sample_process, timestamp, World, WorldManager}; +use crate::processes::{Group, ProcessControl}; +use crate::registry::ProcessorReference; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::collections::BTreeMap; +use std::fs::OpenOptions; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, +}; +use std::time::{Duration, Instant}; + +type Result = std::result::Result; +const MAX_PAYLOAD: usize = 64 * 1024; +const MAX_OUTPUT: usize = 16 * 1024; +const MAX_WORKERS: usize = 4096; + +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Profile { + argv: Vec, + #[serde(default)] + env: BTreeMap, + #[serde(default)] + config_ref: Option, + allowed_processors: Vec, + max_concurrent_workers: usize, + timeout_ms: u64, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Profiles { + schema_version: u32, + profiles: BTreeMap, +} +#[derive(Default)] +pub(super) struct Configuration { + profiles: Option>, + descriptor: Option, +} +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerStart { + pub id: String, + pub expected_generation: u64, + pub profile: String, + pub key: String, + pub payload: Value, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerQuery { + pub id: String, + #[serde(default)] + pub generation: Option, + #[serde(default)] + pub worker_id: Option, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerStop { + pub id: String, + pub expected_generation: u64, + pub worker_id: String, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Worker { + worker_id: String, + generation: u64, + key: String, + profile: String, + profile_sha256: String, + request_sha256: String, + pub(super) pid: Option, + state: String, + error: Option, + started_at_unix_ms: u128, + finished_at_unix_ms: Option, + exit_code: Option, + outcome: Option, + last_resources: Value, + #[serde(skip)] + live: Option, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Outcome { + schema_version: u32, + status: String, + #[serde(default)] + code: Option, +} +struct Completion { + state: String, + error: Option, + exit_code: Option, + outcome: Option, +} +struct Live { + cancelled: Arc, + receiver: mpsc::Receiver, + thread: Option>, +} +impl Drop for Live { + fn drop(&mut self) { + self.cancelled.store(true, Ordering::SeqCst); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +pub(super) fn token(value: &str) -> Result<()> { + if value.is_empty() + || value.len() > 128 + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_.:".contains(&b)) + { + return Err( + "Identifier must contain 1–128 ASCII letters, digits, dash, dot, underscore or colon" + .into(), + ); + } + Ok(()) +} +pub(super) fn digest(value: &impl Serialize) -> Result { + use sha2::{Digest, Sha256}; + let canonical = serde_json::to_value(value).map_err(|e| e.to_string())?; + Ok(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&canonical).map_err(|e| e.to_string())?) + )) +} +fn load_profiles(path: &Path) -> Result> { + if !path.is_absolute() { + return Err("Worker profiles require an absolute operator path".into()); + } + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + let file = options.open(path).map_err(|e| e.to_string())?; + let metadata = file.metadata().map_err(|e| e.to_string())?; + if !metadata.is_file() || metadata.len() > 1024 * 1024 { + return Err("Invalid worker profile file".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if metadata.uid() != unsafe { libc::geteuid() } || metadata.mode() & 0o022 != 0 { + return Err("Worker profiles must be same-user and not group/world writable".into()); + } + } + let mut bytes = Vec::new(); + file.take(1024 * 1024 + 1) + .read_to_end(&mut bytes) + .map_err(|e| e.to_string())?; + if bytes.len() > 1024 * 1024 { + return Err("Worker profiles exceed 1 MiB".into()); + } + let profiles: Profiles = + serde_json::from_slice(&bytes).map_err(|_| "Invalid worker profile schema")?; + if profiles.schema_version != 1 || profiles.profiles.len() > 64 { + return Err("Unsupported worker profile version or count".into()); + } + for (name, profile) in &profiles.profiles { + token(name)?; + if profile.argv.is_empty() + || profile.argv.len() > 64 + || !Path::new(&profile.argv[0]).is_absolute() + || profile + .argv + .iter() + .any(|a| a.len() > 16384 || a.contains('\0')) + || profile.allowed_processors.is_empty() + || profile.allowed_processors.len() > 256 + || !(1..=64).contains(&profile.max_concurrent_workers) + || !(1..=3_600_000).contains(&profile.timeout_ms) + || profile + .config_ref + .as_ref() + .is_some_and(|p| !p.is_absolute()) + || profile.env.len() > 64 + || profile.env.iter().any(|(k, v)| { + k.is_empty() + || k.len() > 128 + || k.contains('=') + || k.contains('\0') + || v.contains('\0') + || v.len() > 16384 + }) + { + return Err("Invalid worker profile bounds, executable or configuration".into()); + } + } + Ok(profiles.profiles) +} + +impl WorldManager { + /// Trusted embedding/startup API only; there is deliberately no wire equivalent. + pub fn configure_workers(&mut self, profiles: &Path, descriptor: PathBuf) -> Result<()> { + if self.worker_configuration.profiles.is_some() + || self + .worlds + .values() + .any(|w| w.instance.is_some() || w.pending.is_some()) + { + return Err("Worker profiles are startup-only".into()); + } + if !descriptor.is_absolute() { + return Err("Worker endpoint must be absolute".into()); + } + self.worker_configuration = Configuration { + profiles: Some(load_profiles(profiles)?), + descriptor: Some(descriptor), + }; + Ok(()) + } + pub fn worker_start(&mut self, request: WorkerStart) -> Result { + self.ensure_starting_allowed()?; + token(&request.key)?; + token(&request.profile)?; + if serde_json::to_vec(&request.payload) + .map_err(|e| e.to_string())? + .len() + > MAX_PAYLOAD + { + return Err("Worker payload exceeds 64 KiB".into()); + } + self.status_with(&request.id, true)?; + for world in self.worlds.values_mut() { + poll(world); + } + let world = self.worlds.get(&request.id).ok_or("Unknown world")?; + if world.state != "running" + || world.generation != request.expected_generation + || world.pending.is_some() + { + return Err("Worker launch requires the expected running generation".into()); + } + let request_sha256 = digest(&request)?; + if let Some(worker) = world + .workers + .iter() + .find(|w| w.generation == request.expected_generation && w.key == request.key) + { + if worker.request_sha256 != request_sha256 { + return Err("Worker key already names a different request".into()); + } + let mut status = worker.status(); + status["replayed"] = json!(true); + return Ok(status); + } + if world.workers.len() >= MAX_WORKERS { + return Err("World worker history limit reached".into()); + } + let profile = self + .worker_configuration + .profiles + .as_ref() + .and_then(|p| p.get(&request.profile)) + .ok_or("Worker profile is not configured")? + .clone(); + if !profile + .allowed_processors + .contains(&world.definition.processor) + { + return Err("Worker profile does not allow this exact processor pin".into()); + } + let count = self + .worlds + .values() + .flat_map(|w| &w.workers) + .filter(|w| w.profile == request.profile && w.live.is_some()) + .count(); + if count >= profile.max_concurrent_workers { + return Err("Worker profile concurrency limit reached".into()); + } + let worker_id = crate::bounded::owner_identity(); + let mut envelope = serde_json::to_vec(&json!({"schema_version":1,"world_id":request.id, + "generation":request.expected_generation,"worker_id":worker_id,"key":request.key, + "profile":request.profile,"processor":world.definition.processor,"owner_descriptor":self.worker_configuration.descriptor, + "config_ref":profile.config_ref,"payload":request.payload})).map_err(|e| e.to_string())?; + if envelope.len() > 128 * 1024 { + return Err("Worker launch envelope exceeds 128 KiB".into()); + } + envelope.push(b'\n'); + let control = self + .shutdown + .controls + .lock() + .map_err(|_| "Shutdown lock poisoned")? + .get(&request.id) + .cloned() + .ok_or("World has no process owner")?; + let world = self.worlds.get_mut(&request.id).unwrap(); + world.workers.push(Worker { + worker_id: worker_id.clone(), + generation: request.expected_generation, + key: request.key, + profile: request.profile, + profile_sha256: digest(&profile)?, + request_sha256, + pid: None, + state: "starting".into(), + error: None, + started_at_unix_ms: timestamp(), + finished_at_unix_ms: None, + exit_code: None, + outcome: None, + last_resources: sample_process(None), + live: None, + }); + world.persistence_dirty = true; + // Reserve the key durably before a child can be created. + if let Err(error) = persist(&self.build_root, &request.id, world) { + let worker = world.workers.last_mut().unwrap(); + worker.state = "failed".into(); + worker.error = Some("Cannot persist worker launch intent".into()); + worker.finished_at_unix_ms = Some(timestamp()); + world.persistence_dirty = true; + return Err(error); + } + let launch = spawn(&profile, control, envelope); + let worker = world.workers.last_mut().unwrap(); + match launch { + Ok((pid, live)) => { + worker.pid = Some(pid); + worker.last_resources = sample_process(Some(pid)); + worker.live = Some(live); + worker.state = "running".into(); + } + Err(_) => { + worker.state = "failed".into(); + worker.error = Some("Worker process could not be launched".into()); + worker.finished_at_unix_ms = Some(timestamp()); + } + } + world.persistence_dirty = true; + if let Err(error) = persist(&self.build_root, &request.id, world) { + let worker = world.workers.last_mut().unwrap(); + drop(worker.live.take()); + worker.state = "failed".into(); + worker.error = Some("Cannot persist worker launch; child cancelled".into()); + worker.finished_at_unix_ms = Some(timestamp()); + world.persistence_dirty = true; + return Err(error); + } + let mut status = world.workers.last().unwrap().status(); + status["replayed"] = json!(false); + Ok(status) + } + pub fn worker_status(&mut self, query: WorkerQuery) -> Result { + self.status_with(&query.id, true)?; + let world = self.worlds.get(&query.id).ok_or("Unknown world")?; + let workers: Vec<_> = world + .workers + .iter() + .filter(|w| { + query.generation.is_none_or(|g| g == w.generation) + && query.worker_id.as_ref().is_none_or(|id| id == &w.worker_id) + }) + .map(Worker::status) + .collect(); + if query.worker_id.is_some() && workers.is_empty() { + return Err("Unknown worker".into()); + } + Ok(json!({"schema_version":1,"id":query.id,"workers":workers})) + } + pub fn worker_stop(&mut self, request: WorkerStop) -> Result { + self.status_with(&request.id, true)?; + let world = self.worlds.get_mut(&request.id).ok_or("Unknown world")?; + if world.generation != request.expected_generation { + return Err("Worker stop generation mismatch".into()); + } + let worker = world + .workers + .iter_mut() + .find(|w| { + w.worker_id == request.worker_id && w.generation == request.expected_generation + }) + .ok_or("Unknown worker in generation")?; + worker.stop(); + world.persistence_dirty = true; + persist(&self.build_root, &request.id, world)?; + Ok(world + .workers + .iter() + .find(|w| w.worker_id == request.worker_id) + .unwrap() + .status()) + } +} +impl Worker { + fn complete(&mut self, result: Completion) { + self.state = result.state; + self.error = result.error; + self.exit_code = result.exit_code; + self.outcome = result.outcome; + self.finished_at_unix_ms = Some(timestamp()); + } + fn stop(&mut self) { + if let Some(live) = &self.live { + live.cancelled.store(true, Ordering::SeqCst); + self.state = "stopping".into(); + } + } + fn status(&self) -> Value { + let mut status = serde_json::to_value(self).unwrap(); + status["schema_version"] = json!(1); + status["resources"] = sample_process(self.live.as_ref().and(self.pid)); + status + } +} +pub(super) fn statuses(world: &World) -> Value { + json!(world.workers.iter().map(Worker::status).collect::>()) +} +pub(super) fn is_worker_pid(world: &World, pid: u32) -> bool { + world + .workers + .iter() + .any(|w| w.live.is_some() && w.pid == Some(pid)) +} +pub(super) fn check_worker(world: &World, worker_id: Option<&str>) -> Result<()> { + if let Some(id) = worker_id { + let worker = world + .workers + .iter() + .find(|w| w.worker_id == id && w.generation == world.generation) + .ok_or("Worker identity/generation mismatch")?; + if worker.state != "running" + || worker + .live + .as_ref() + .is_none_or(|l| l.cancelled.load(Ordering::SeqCst)) + { + return Err("Worker no longer has active admission authority".into()); + } + } + Ok(()) +} +pub(super) fn stop(world: &mut World) { + for worker in &mut world.workers { + worker.stop(); + } + world.persistence_dirty = true; +} +pub(super) fn shutdown(world: &mut World) { + poll(world); + stop(world); + for worker in &mut world.workers { + if let Some(mut live) = worker.live.take() { + if let Some(thread) = live.thread.take() { + let _ = thread.join(); + } + let result = live.receiver.try_recv().unwrap_or_else(|_| Completion { + state: "stopped".into(), + error: Some("Worker stopped without a retained outcome".into()), + exit_code: None, + outcome: None, + }); + worker.complete(result); + } + } +} +pub(super) fn recover(world: &mut World) { + for worker in &mut world.workers { + if matches!(worker.state.as_str(), "starting" | "running" | "stopping") { + worker.state = "interrupted".into(); + worker.error = + Some("Previous owner exited; worker was not adopted or restarted".into()); + worker.finished_at_unix_ms = Some(timestamp()); + world.persistence_dirty = true; + } + } +} +pub(super) fn poll(world: &mut World) { + for worker in &mut world.workers { + let completion = worker.live.as_ref().map(|l| l.receiver.try_recv()); + let result = match completion { + Some(Ok(result)) => Some(result), + Some(Err(mpsc::TryRecvError::Disconnected)) => Some(Completion { + state: "failed".into(), + error: Some("Worker monitor exited without an outcome".into()), + exit_code: None, + outcome: None, + }), + _ => None, + }; + if let Some(result) = result { + drop(worker.live.take()); + worker.complete(result); + world.persistence_dirty = true; + } + } +} + +/// Drop also covers monitor panics or failure to create its thread. A dropped +/// std::process::Child alone neither kills nor reaps the process. +struct OwnedChild { + child: Child, + group: Group, +} +impl Drop for OwnedChild { + fn drop(&mut self) { + self.group.kill(); + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +#[cfg(unix)] +fn nonblocking(file: &impl std::os::fd::AsRawFd) -> Result<()> { + let fd = file.as_raw_fd(); + let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + if flags < 0 || unsafe { libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0 { + return Err("Cannot configure worker pipes".into()); + } + Ok(()) +} +fn spawn(profile: &Profile, control: ProcessControl, envelope: Vec) -> Result<(u32, Live)> { + #[cfg(not(unix))] + { + let _ = (profile, control, envelope); + Err("Managed workers require Unix".into()) + } + #[cfg(unix)] + { + let mut command = Command::new(&profile.argv[0]); + command + .args(&profile.argv[1..]) + .envs(&profile.env) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + crate::processes::separate_group(&mut command, &control); + let child = command.spawn().map_err(|e| e.to_string())?; + let pid = child.id(); + let group = control.track(pid); + let mut owned = OwnedChild { child, group }; + let input = owned.child.stdin.take().ok_or("Worker stdin unavailable")?; + let output = owned + .child + .stdout + .take() + .ok_or("Worker stdout unavailable")?; + if nonblocking(&input) + .and_then(|_| nonblocking(&output)) + .is_err() + { + return Err("Cannot configure worker pipes".into()); + } + let cancelled = Arc::new(AtomicBool::new(false)); + let stop = cancelled.clone(); + let timeout = Duration::from_millis(profile.timeout_ms); + let (sender, receiver) = mpsc::channel(); + let thread = std::thread::Builder::new() + .name("world-worker".into()) + .spawn(move || { + let result = monitor( + &mut owned.child, + &owned.group, + &control, + &stop, + input, + output, + envelope, + timeout, + ); + stop.store(true, Ordering::SeqCst); + drop(owned); + let _ = sender.send(result); + }) + .map_err(|_| "Cannot create worker monitor")?; + Ok(( + pid, + Live { + cancelled, + receiver, + thread: Some(thread), + }, + )) + } +} +#[cfg(unix)] +#[allow(clippy::too_many_arguments)] +fn monitor( + child: &mut Child, + group: &Group, + control: &ProcessControl, + cancelled: &AtomicBool, + input: std::process::ChildStdin, + mut output: std::process::ChildStdout, + envelope: Vec, + timeout: Duration, +) -> Completion { + let started = Instant::now(); + let mut offset = 0; + let mut bytes = Vec::new(); + let mut input = Some(input); + let failure = |state: &str, error: &str| Completion { + state: state.into(), + error: Some(error.into()), + exit_code: None, + outcome: None, + }; + loop { + if cancelled.load(Ordering::SeqCst) || control.stopped() { + return failure("stopped", "Worker cancelled by its owner"); + } + if started.elapsed() >= timeout { + return failure("failed", "Worker lifetime limit exceeded"); + } + if let Some(writer) = input.as_mut() { + match writer.write(&envelope[offset..]) { + Ok(0) => return failure("failed", "Worker did not accept launch envelope"), + Ok(count) => offset += count, + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => (), + Err(_) => return failure("failed", "Worker launch pipe failed"), + } + if offset < envelope.len() && started.elapsed() > Duration::from_secs(5) { + return failure("failed", "Worker launch envelope timed out"); + } + } + if offset == envelope.len() { + drop(input.take()); + } + if drain(&mut output, &mut bytes).is_err() { + return failure( + "failed", + "Worker stdout exceeded limit or could not be read", + ); + } + match child.try_wait() { + Ok(Some(status)) => { + group.kill(); + if drain(&mut output, &mut bytes).is_err() { + return failure( + "failed", + "Worker stdout exceeded limit or could not be read", + ); + } + let mut result = Completion { + state: "failed".into(), + error: None, + exit_code: status.code(), + outcome: None, + }; + if !status.success() { + result.error = Some("Worker exited unsuccessfully".into()); + } + match serde_json::from_slice::(&bytes) { + Ok(outcome) + if outcome.schema_version == 1 + && matches!(outcome.status.as_str(), "completed" | "failed") + && (status.success() || outcome.status == "failed") + && outcome + .code + .as_ref() + .is_none_or(|c| c.len() <= 64 && token(c).is_ok()) => + { + result.state = outcome.status.clone(); + if result.state == "failed" { + result + .error + .get_or_insert_with(|| "Worker reported failure".into()); + } + result.outcome = Some(outcome); + } + _ => { + result.error.get_or_insert_with(|| { + "Worker returned an invalid bounded outcome".into() + }); + } + } + return result; + } + Err(_) => return failure("failed", "Cannot observe worker exit"), + Ok(None) => std::thread::sleep(Duration::from_millis(10)), + } + } +} +#[cfg(unix)] +fn drain(output: &mut impl Read, bytes: &mut Vec) -> Result<()> { + loop { + let mut buffer = [0; 4096]; + match output.read(&mut buffer) { + Ok(0) => return Ok(()), + Ok(count) => { + if bytes.len() + count > MAX_OUTPUT { + return Err("Worker stdout limit".into()); + } + bytes.extend_from_slice(&buffer[..count]); + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => return Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => (), + Err(_) => return Err("Worker stdout read failed".into()), + } + } +} diff --git a/src/worlds.rs b/src/worlds.rs index fac16b7..e360030 100644 --- a/src/worlds.rs +++ b/src/worlds.rs @@ -18,6 +18,22 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; +#[path = "world_persistence.rs"] +mod persistence; +use persistence::{atomic_json, persist, persist_if_changed}; +#[path = "world_admission.rs"] +mod admission; +#[path = "world_storage.rs"] +mod storage; +#[path = "world_workers.rs"] +mod workers; +pub use admission::{AdmissionQuery, AdmitInputs, EffectRequest, ReadBatch, ReadQuery}; +pub use storage::{CheckpointPublish, CheckpointQuery, CheckpointStage}; +pub use workers::{WorkerQuery, WorkerStart, WorkerStop}; +#[path = "world_library.rs"] +mod library; +pub use library::{ArtifactEntry, ArtifactLibrary, LibraryArtifact, LibraryImportRequest}; + pub const SCHEMA_VERSION: u32 = 1; /// Implicit library holding every registered definition without an explicit one. pub const UNASSIGNED_LIBRARY: &str = "unassigned"; @@ -73,7 +89,11 @@ pub struct WorldDefinition { } /// Message from a start thread: a live instance, a completed test run whose /// instance was dropped, or the install/test failure. -type StartOutcome = Result, String>; +struct Started { + instance: ProgramInstance, + effects: admission::Effects, +} +type StartOutcome = Result, String>; /// The capture tailer of one generation. `live` tells it whether an instance /// (or a pending start) still exists; without one it exits after idle polls. /// Dropping a tailer stops and joins its thread, so no thread outlives its @@ -119,6 +139,14 @@ struct World { #[serde(default, skip_serializing_if = "Option::is_none")] capture_generation: Option, history: Vec, + #[serde(default)] + persistence: persistence::Persistence, + #[serde(skip)] + persistence_dirty: bool, + #[serde(default)] + admission: admission::AdmissionState, + #[serde(default)] + workers: Vec, #[serde(skip)] pending: Option>, #[serde(skip)] @@ -187,6 +215,8 @@ pub struct WorldManager { build_root: PathBuf, driver: PathBuf, worlds: BTreeMap, + worker_configuration: workers::Configuration, + storage_configuration: storage::Configuration, shutdown: WorldShutdown, _owner_lock: std::fs::File, } @@ -237,6 +267,8 @@ impl WorldManager { ) { return Err("Invalid persisted world state".into()); } + workers::recover(&mut world); + admission::recover(&mut world); if matches!(world.state.as_str(), "starting" | "running" | "stopping") { world.state = "interrupted".into(); world.error = @@ -252,6 +284,7 @@ impl WorldManager { .join("native-events.jsonl"), )); } + persist_if_changed(&build_root, &id, &mut world)?; worlds.insert(id, world); } Ok(Self { @@ -259,6 +292,8 @@ impl WorldManager { build_root, driver, worlds, + worker_configuration: workers::Configuration::default(), + storage_configuration: storage::Configuration::default(), _owner_lock: owner_lock, shutdown: WorldShutdown::default(), }) @@ -699,6 +734,10 @@ impl WorldManager { reader: None, capture_generation: None, history: vec![], + persistence: persistence::Persistence::default(), + persistence_dirty: false, + admission: admission::AdmissionState::default(), + workers: Vec::new(), pending: None, stop_requested: false, test: None, @@ -760,7 +799,7 @@ impl WorldManager { purpose: "test".into(), scenarios, })?; - match self.start_with(&id, request.keep_world) { + match self.start_with(&id, request.keep_world, None) { Ok(status) => Ok(status), Err(error) => { // A test world that never started is not inventory: the reply @@ -810,9 +849,14 @@ impl WorldManager { /// Publish starting state before compiling; inventory and stop remain available. /// A restarted test world keeps its instance after re-running its scenarios. pub fn start_async(&mut self, id: &str) -> Result { - self.start_with(id, true) + self.start_with(id, true, None) } - fn start_with(&mut self, id: &str, keep_world: bool) -> Result { + fn start_with( + &mut self, + id: &str, + keep_world: bool, + restore: Option, + ) -> Result { let registry = self.registry()?; let controls = self.shutdown.controls.clone(); let mut controls = controls.lock().map_err(|_| "Shutdown lock poisoned")?; @@ -821,6 +865,7 @@ impl WorldManager { if world.instance.is_some() || world.pending.is_some() { return Err("World already owns an instance; stop it before restarting".into()); } + workers::shutdown(world); if let Some(mut tailer) = world.tailer.take() { tailer.stop(); } @@ -833,7 +878,12 @@ impl WorldManager { let metadata = self.world_metadata(&record)?; let world = self.worlds.get_mut(id).ok_or("Unknown world")?; world.metadata = metadata; - world.generation += 1; + world.generation = world + .generation + .checked_add(1) + .ok_or("World generation exhausted")?; + world.persistence.restore_requested = restore.as_ref().map(|r| r.receipt.clone()); + world.persistence.restored_from = None; world.stop_requested = false; let mut backend = Backend::new( self.build_root.join(id).join(world.generation.to_string()), @@ -875,7 +925,7 @@ impl WorldManager { .map_err(|error| format!("Cannot select the world lowering: {error}"))?; world.state = "starting".into(); world.error = None; - let scenarios = if world.definition.purpose == "test" { + let scenarios = if restore.is_none() && world.definition.purpose == "test" { world.definition.scenarios.clone() } else { Vec::new() @@ -907,10 +957,15 @@ impl WorldManager { let (sender, receiver) = std::sync::mpsc::channel(); world.pending = Some(receiver); std::thread::spawn(move || { - let installed = instance.execute( - "processor_install", - &json!({"processor_id":pin.processor_id,"version":pin.version}), - ); + let installed = match restore { + Some(restore) => restore.install(&mut instance), + None => instance + .execute( + "processor_install", + &json!({"processor_id":pin.processor_id,"version":pin.version}), + ) + .map(|_| admission::Effects::default()), + }; let result: StartOutcome = match (installed, progress) { (Err(error), Some(progress)) => { let mut status = progress.lock().unwrap(); @@ -920,11 +975,11 @@ impl WorldManager { Err(error) } (Err(error), None) => Err(error), - (Ok(_), None) => Ok(Some(instance)), - (Ok(_), Some(progress)) => { + (Ok(effects), None) => Ok(Some(Started { instance, effects })), + (Ok(effects), Some(progress)) => { run_scenarios(&mut instance, &scenarios, &progress); if keep_world { - Ok(Some(instance)) + Ok(Some(Started { instance, effects })) } else { drop(instance); Ok(None) @@ -937,7 +992,9 @@ impl WorldManager { self.status(id) } pub fn stop(&mut self, id: &str) -> Result { + self.storage_configuration.cancel(id); let world = self.worlds.get_mut(id).ok_or("Unknown world")?; + workers::stop(world); if let Some(instance) = world.instance.take() { #[cfg(unix)] instance.backend.control.stop(); @@ -990,7 +1047,9 @@ impl WorldManager { self.status_with(id, false) } fn status_with(&mut self, id: &str, summary: bool) -> Result { + self.poll_storage(); let world = self.worlds.get_mut(id).ok_or("Unknown world")?; + workers::poll(world); let completion = world.pending.as_ref().map(|receiver| receiver.try_recv()); match completion { Some(Ok(result)) => { @@ -1000,8 +1059,11 @@ impl WorldManager { world.state = "stopped".into(); } else { match result { - Ok(Some(instance)) => { - world.instance = Some(instance); + Ok(Some(started)) => { + world.instance = Some(started.instance); + world.admission.effects = started.effects; + world.persistence.restored_from = + world.persistence.restore_requested.clone(); world.state = "running".into(); world.error = None; } @@ -1098,10 +1160,19 @@ impl WorldManager { "state":world.state,"generation":world.generation,"error":world.error,"history":world.history, "owner":{"kind":"embedded","pid":std::process::id(),"memory_attribution":"shared_not_attributable"}, "resources":sample_process(pid),"started_at_unix_ms":started_at,"build":build,"revision":revision, - "persistence":{"status":"not_configured","reason":"world checkpoints are not wired"}, + "persistence":persistence::status(&self.build_root, id, world, &revision), + "workers":workers::statuses(world), "test":world.test}); + if let Some(entries) = status["persistence"]["checkpoints"].as_array_mut() { + for entry in entries { + self.storage_configuration.observe_entry(entry); + } + } + status["persistence"]["iceberg"] = self + .storage_configuration + .status(world, &status["persistence"]); if !summary { - let processes = self.shutdown.controls.lock().map_err(|_|"Shutdown lock poisoned")?.get(id).map(|control|control.tracked_pids()).unwrap_or_default().into_iter().map(|process|json!({"role":if Some(process)==pid {"native"} else {"compiler_or_startup"},"sample":sample_process(Some(process)),"descendants_included":false})).collect::>(); + let processes = self.shutdown.controls.lock().map_err(|_|"Shutdown lock poisoned")?.get(id).map(|control|control.tracked_pids()).unwrap_or_default().into_iter().map(|process|json!({"role":if Some(process)==pid {"native"} else if workers::is_worker_pid(world, process) {"worker"} else {"compiler_or_startup"},"sample":sample_process(Some(process)),"descendants_included":false})).collect::>(); status["managed_processes"] = json!(processes); status["instance"] = instance_info; // A recovered world reads its last capture once; nothing tails it. @@ -1444,6 +1515,7 @@ impl Drop for WorldManager { fn drop(&mut self) { self.shutdown.stop_all(); for (id, world) in self.worlds.iter_mut() { + workers::shutdown(world); if let Some(instance) = world.instance.take() { #[cfg(unix)] instance.backend.control.stop(); @@ -1454,6 +1526,7 @@ impl Drop for WorldManager { if let Some(mut tailer) = world.tailer.take() { tailer.stop(); } + let _ = persist_if_changed(&self.build_root, id, world); } } } @@ -1547,66 +1620,3 @@ fn lock_owner(root: &std::path::Path) -> Result { } Ok(file) } -fn persist_if_changed(root: &std::path::Path, id: &str, world: &mut World) -> Result<(), String> { - if world.history.last().is_none_or(|last| { - last["state"] != world.state - || last["generation"] != world.generation - || last["error"] != json!(world.error) - }) { - persist(root, id, world)?; - } - Ok(()) -} -fn persist(root: &std::path::Path, id: &str, world: &mut World) -> Result<(), String> { - let mut history = world.history.clone(); - if history.last().is_none_or(|last| { - last["state"] != world.state - || last["generation"] != world.generation - || last["error"] != json!(world.error) - }) { - history.push(json!({"at_unix_ms":timestamp(),"state":world.state,"generation":world.generation,"error":world.error})); - } - let path = root.join(id).join("world.json"); - let temp = path.with_extension("json.tmp"); - let mut record = serde_json::to_value(&*world).map_err(|e| e.to_string())?; - record["history"] = json!(history); - let bytes = serde_json::to_vec_pretty(&record).map_err(|e| e.to_string())?; - let mut options = std::fs::OpenOptions::new(); - options.write(true).create(true).truncate(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600).custom_flags(libc::O_NOFOLLOW); - } - let mut file = options.open(&temp).map_err(|e| e.to_string())?; - use std::io::Write; - file.write_all(&bytes) - .and_then(|_| file.sync_all()) - .map_err(|e| e.to_string())?; - std::fs::rename(&temp, &path).map_err(|e| e.to_string())?; - std::fs::File::open(root.join(id)) - .and_then(|f| f.sync_all()) - .map_err(|e| e.to_string())?; - world.history = history; - Ok(()) -} - -fn atomic_json(path: &std::path::Path, value: &Value) -> Result<(), String> { - let temp = path.with_extension("json.tmp"); - let mut options = std::fs::OpenOptions::new(); - options.create(true).truncate(true).write(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600).custom_flags(libc::O_NOFOLLOW); - } - let mut file = options.open(&temp).map_err(|e| e.to_string())?; - use std::io::Write; - file.write_all(&serde_json::to_vec_pretty(value).map_err(|e| e.to_string())?) - .and_then(|_| file.sync_all()) - .map_err(|e| e.to_string())?; - std::fs::rename(&temp, path).map_err(|e| e.to_string())?; - std::fs::File::open(path.parent().ok_or("Missing catalog parent")?) - .and_then(|file| file.sync_all()) - .map_err(|e| e.to_string()) -} diff --git a/tests/fixtures/managed_worker.py b/tests/fixtures/managed_worker.py new file mode 100644 index 0000000..24ed067 --- /dev/null +++ b/tests/fixtures/managed_worker.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Trusted test worker only; no provider, prompt, or application integration.""" +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time + +launch = json.load(sys.stdin) # also proves owner closes stdin after the envelope +root = Path(os.environ["FIXTURE_ROOT"]) +mode = launch["payload"].get("mode", "complete") +(root / (launch["key"] + ".launch.json")).write_text(json.dumps(launch)) +if mode in ("hold", "descendant"): + if mode == "descendant": + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"]) + (root / (launch["key"] + ".child")).write_text(str(child.pid)) + time.sleep(120) +elif mode == "oversized": + print("x" * 20000, flush=True) + time.sleep(120) +elif mode == "invalid": + print('{"prompt":"never expose this output"}', flush=True) + sys.exit(0) +elif mode == "exit": + sys.exit(9) +elif mode in ("failed_exit", "completed_exit", "invalid_exit", "unknown_exit"): + outcome = {"schema_version": 1, "status": "failed", "code": "configuration_invalid"} + if mode == "completed_exit": + outcome["status"] = "completed" + elif mode == "invalid_exit": + outcome["code"] = "never expose private output" + elif mode == "unknown_exit": + outcome["prompt"] = "never expose private output" + print(json.dumps(outcome), flush=True) + sys.exit(9) +elif mode == "attached": + descriptor = json.loads(Path(launch["owner_descriptor"]).read_text()) + sequence = 0 + + def call(operation, args): + global sequence + sequence += 1 + request = dict(schema_version=1, request_id=str(sequence), + owner_incarnation=descriptor["owner_incarnation"], + operation=operation, args=args) + with socket.socket(socket.AF_UNIX) as client: + client.settimeout(15) + client.connect(descriptor["socket"]) + client.sendall(json.dumps(request).encode() + b"\n") + response = json.loads(client.makefile("rb").readline(16 * 1024 * 1024)) + assert response["ok"], response + return response["result"] + + common = dict(id=launch["world_id"], expected_generation=launch["generation"], + worker_id=launch["worker_id"]) + observed = call("status", {"id":launch["world_id"]}) + revision = observed["revision"] + evidence = call("read_batch", dict(common, expected_revision=revision, + queries=[{"predicate":"source"},{"predicate":"echo"}])) + assert evidence["revision"] == revision and all(p["complete"] for p in evidence["results"]) + claim = dict(common, expected_revision=revision, admission_key=launch["key"]+".reserve", + changes=[{"op":"insert","predicate":"source","values":[10,"reserved"]}], + effect={"key":launch["key"],"phase":"reserve"}) + reserved = call("admit_inputs", claim) + assert reserved["state"] == "durable" and reserved["effect_authorized"] + replay = call("admit_inputs", claim) + assert replay["replayed"] and not replay["effect_authorized"] + (root / (launch["key"] + ".reserved.json")).write_text(json.dumps(reserved)) + # Simulates time outside the owner, without performing any external effect. + release = root / (launch["key"] + ".release") + while not release.exists(): + time.sleep(.01) + settled = call("admit_inputs", dict(common, expected_revision=reserved["applied_revision"], + admission_key=launch["key"]+".settle", + changes=[{"op":"delete","predicate":"source","values":[10,"reserved"]}, + {"op":"insert","predicate":"source","values":[20,"settled"]}], + effect={"key":launch["key"],"phase":"settle","reservation_key":claim["admission_key"]})) + assert settled["state"] == "durable" and not settled["effect_authorized"] + (root / (launch["key"] + ".settled.json")).write_text(json.dumps(settled)) +print(json.dumps({"schema_version":1,"status":"completed","code":"fixture_ok"}), flush=True) diff --git a/tests/library_import.rs b/tests/library_import.rs new file mode 100644 index 0000000..8b7d57d --- /dev/null +++ b/tests/library_import.rs @@ -0,0 +1,190 @@ +#![cfg(unix)] +//! Saved-artifact admission: no compiler or execution process is used. +use ddlog_runtime::registry::{ProcessorDefinition, ProcessorRegistry, ProcessorVersion}; +use ddlog_runtime::worlds::{InventoryQuery, LibraryImportRequest, WorldManager}; +use serde_json::{json, Value}; +use std::{fs, path::PathBuf}; + +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "library-import-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&path).unwrap(); + Self(path) + } + fn registry(&self, name: &str) -> ProcessorRegistry { + ProcessorRegistry::open(self.0.join(name)).unwrap() + } + fn manager(&self) -> WorldManager { + WorldManager::new( + self.0.join("destination"), + self.0.join("worlds"), + "/usr/bin/false".into(), + ) + .unwrap() + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn records(registry: &ProcessorRegistry) -> (ProcessorVersion, ProcessorVersion) { + let leaf: ProcessorDefinition = serde_json::from_value(json!({ + "rules":"out(X) :- source(X).", "schemas":{"source":{"input":true,"fields":["int"]},"out":{"input":false,"fields":["int"]}}, + "interface":{"inputs":["source"],"outputs":["out"]} + })).unwrap(); + let child = registry.create(leaf, None).unwrap(); + let parent = registry.create(serde_json::from_value(json!({"composition":{ + "nodes":{"child":{"processor_id":child.processor_id,"version":child.version}}, + "inputs":{"input":{"fields":["int"],"targets":[{"node":"child","relation":"source"}]}}, + "bindings":[],"outputs":{"result":{"node":"child","relation":"out"}} + }})).unwrap(), None).unwrap(); + (child, parent) +} +fn artifact(child: &ProcessorVersion, parent: &ProcessorVersion) -> Value { + json!({"revision":"explicit-test-revision","artifact":{ + "schema_version":1,"library":{"key":"test","name":"Test","repository":"test://source","package":"test","package_version":"1"}, + "entries":[ + {"key":"parent","name":"Parent","description":"Composed program","source":["programs/parent.dl"],"git_provenance":null,"record":parent, + "scenarios":[{"name":"one","description":"One cumulative change","changes":[{"op":"insert","predicate":"input","values":[1]}],"expect":{"result":[[1]]}}]}, + {"key":"child","name":"Child","description":"Leaf","source":[],"git_provenance":null,"record":child,"scenarios":[]} + ], + "pins":{"parent":{"processor_id":parent.processor_id,"version":parent.version},"child":{"processor_id":child.processor_id,"version":child.version}} + }}) +} +fn request(value: Value) -> LibraryImportRequest { + serde_json::from_value(value).unwrap() +} + +#[test] +fn imported_records_validate_full_closure_before_writes_and_keep_current() { + let fixture = Fixture::new(); + let source = fixture.registry("source"); + let (child, parent) = records(&source); + let destination = fixture.registry("destination"); + let missing = destination + .import_records(vec![parent.clone()], false) + .unwrap(); + assert!(!missing.errors.is_empty()); + assert_eq!( + fs::read_dir(fixture.0.join("destination")).unwrap().count(), + 0 + ); + let report = destination + .import_records(vec![parent.clone(), child.clone()], true) + .unwrap(); + assert!(report.errors.is_empty()); + assert_eq!( + fs::read_dir(fixture.0.join("destination")).unwrap().count(), + 0 + ); + let report = destination + .import_records(vec![parent.clone(), child.clone()], false) + .unwrap(); + assert!(report.errors.is_empty()); + assert_eq!(report.imported[0].record, child); + assert_eq!(destination.get(&parent.processor_id, None).unwrap(), parent); + let mut changed = child.definition.clone(); + if let ProcessorDefinition::Program(p) = &mut changed { + p.rules.push('\n'); + } + let newer = destination + .publish(&child.processor_id, changed, &child.version, None) + .unwrap(); + destination + .import_records(vec![child.clone(), parent], false) + .unwrap(); + assert_eq!(destination.get(&child.processor_id, None).unwrap(), newer); +} + +#[test] +fn library_import_is_idempotent_and_creates_no_world_or_build() { + let fixture = Fixture::new(); + let (child, parent) = records(&fixture.registry("source")); + let mut manager = fixture.manager(); + let value = artifact(&child, &parent); + let mut dry = value.clone(); + dry["dry_run"] = json!(true); + assert_eq!( + manager.library_import(request(dry)).unwrap()["dry_run"], + true + ); + assert_eq!( + manager.libraries().unwrap()["libraries"] + .as_array() + .unwrap() + .len(), + 1 + ); + let first = manager.library_import(request(value.clone())).unwrap(); + assert_eq!(manager.library_import(request(value)).unwrap(), first); + assert_eq!( + manager.inventory(&InventoryQuery::default()).unwrap()["worlds"], + json!([]) + ); + let libraries = manager.libraries().unwrap(); + let library = libraries["libraries"] + .as_array() + .unwrap() + .iter() + .find(|l| l["id"] == first["library_id"]) + .unwrap(); + assert_eq!(library["processors"].as_array().unwrap().len(), 2); + assert_eq!( + manager + .scenarios_get(&parent.processor_id, &parent.version) + .unwrap()["scenarios"][0]["name"], + "one" + ); + assert_eq!( + manager + .registry() + .unwrap() + .get(&parent.processor_id, Some(&parent.version)) + .unwrap(), + parent + ); +} + +#[test] +fn invalid_scenarios_pins_provenance_and_records_leave_no_publication() { + let fixture = Fixture::new(); + let (child, parent) = records(&fixture.registry("source")); + let manager = fixture.manager(); + let baseline = artifact(&child, &parent); + for changed in [0, 1, 2, 3, 4] { + let mut value = baseline.clone(); + match changed { + 0 => { + value["artifact"]["entries"][0]["scenarios"][0]["changes"][0]["values"] = + json!(["wrong type"]) + } + 1 => value["artifact"]["pins"]["parent"]["version"] = json!(child.version), + 2 => { + value["artifact"]["entries"][0]["git_provenance"] = + json!({"repository":"wrong","revision":"wrong"}) + } + 3 => { + value["artifact"]["entries"][1]["record"]["definition"]["rules"] = + json!("out(X) :- source(X), X > 0.") + } + _ => value["artifact"]["entries"][0]["source"] = json!(["../../outside"]), + } + assert!(manager.library_import(request(value)).is_err()); + assert_eq!( + fs::read_dir(fixture.0.join("destination")).unwrap().count(), + 0 + ); + assert!(!fixture.0.join("worlds/libraries.json").exists()); + assert!(!fixture.0.join("worlds/scenarios").exists()); + } +} diff --git a/tests/managed_iceberg.rs b/tests/managed_iceberg.rs new file mode 100644 index 0000000..0821437 --- /dev/null +++ b/tests/managed_iceberg.rs @@ -0,0 +1,609 @@ +#![cfg(all(unix, feature = "iceberg"))] +//! Real SQLite/Parquet/catalog IO. Native DDLog proof is a separate opt-in test. +use ddlog_runtime::{ + registry::{ProcessorDefinition, ProcessorReference}, + worlds::{WorldDefinition, WorldManager}, +}; +use iceberg::{Catalog, CatalogBuilder, TableIdent}; +use iceberg_catalog_sql::{SqlBindStyle, SqlCatalogBuilder}; +use serde_json::{json, Value}; +use std::{ + collections::HashMap, + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; +static NEXT: AtomicU64 = AtomicU64::new(0); +struct Fixture { + root: PathBuf, + driver: PathBuf, +} +impl Fixture { + fn new(native: bool) -> Self { + let root = std::env::temp_dir().join(format!( + "managed-iceberg-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::SeqCst) + )); + fs::create_dir(&root).unwrap(); + let driver = if native { + std::env::var_os("DDLOG_RUNTIME_NATIVE_BUILD") + .expect("native build driver") + .into() + } else { + let template = + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/memory_fake_runtime.py"); + let driver = root.join("build.py"); + fs::write(&driver,format!("#!/usr/bin/env python3\nimport sys\nfrom pathlib import Path\nsource=Path({}).read_text().replace('__CONTROL__',{})\nPath(sys.argv[2]).write_text(source)\nPath(sys.argv[2]).chmod(0o700)\n",json!(template),json!(serde_json::to_string(&root).unwrap()))).unwrap(); + fs::set_permissions(&driver, fs::Permissions::from_mode(0o700)).unwrap(); + driver + }; + let f = Self { root, driver }; + f.profile(30000); + f + } + fn storage(&self) -> PathBuf { + self.root.join("local store") + } + fn profile(&self, timeout: u64) { + fs::write(self.root.join("profile.json"),serde_json::to_vec(&json!({"schema_version":1,"profile":{"name":"local","root":self.storage(),"timeout_ms":timeout}})).unwrap()).unwrap(); + fs::set_permissions( + self.root.join("profile.json"), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + } + fn manager(&self) -> WorldManager { + let mut manager = WorldManager::new( + self.root.join("registry"), + self.root.join("worlds"), + self.driver.clone(), + ) + .unwrap(); + manager + .configure_storage(&self.root.join("profile.json")) + .unwrap(); + manager + } + fn record(&self, receipt: &Value) -> PathBuf { + self.root + .join("worlds") + .join(receipt["origin"]["world_id"].as_str().unwrap()) + .join("checkpoints") + .join(receipt["receipt_id"].as_str().unwrap()) + .join("publication.json") + } +} +impl Drop for Fixture { + fn drop(&mut self) { + if !std::thread::panicking() { + let _ = fs::remove_dir_all(&self.root); + } + } +} +fn make(manager: &mut WorldManager, composition: bool) -> String { + let definition:ProcessorDefinition=serde_json::from_value(json!({"rules":"echo(N,S) :- source(N,S).","schemas":{"source":{"input":true,"fields":["int","string"]},"unused":{"input":true,"fields":["string"]},"echo":{"input":false,"fields":["int","string"]}},"interface":{"inputs":["source","unused"],"outputs":["echo"]}})).unwrap(); + let leaf = manager + .registry() + .unwrap() + .create(definition, None) + .unwrap(); + let leaf = ProcessorReference { + processor_id: leaf.processor_id, + version: leaf.version, + }; + let pin = if composition { + let definition=serde_json::from_value(json!({"composition":{"nodes":{"child":leaf},"bindings":[], + "inputs":{"source":{"fields":["int","string"],"targets":[{"node":"child","relation":"source"}]},"unused":{"fields":["string"],"targets":[{"node":"child","relation":"unused"}]}},"outputs":{"echo":{"node":"child","relation":"echo"}}}})).unwrap(); + let record = manager + .registry() + .unwrap() + .create(definition, None) + .unwrap(); + ProcessorReference { + processor_id: record.processor_id, + version: record.version, + } + } else { + leaf + }; + manager + .create(WorldDefinition { + label: "Iceberg recovery".into(), + processor: pin, + purpose: "instance".into(), + scenarios: vec![], + }) + .unwrap() +} +fn change(m: &mut WorldManager, id: &str, op: &str, values: Value) { + m.execute( + id, + "apply_changes", + &json!({"changes":[{"op":op,"predicate":"source","values":values}]}), + ) + .unwrap(); +} +fn rows(m: &mut WorldManager, id: &str, name: &str) -> Value { + m.execute(id, "query_rows", &json!({"predicate":name})) + .unwrap()["rows"] + .clone() +} +fn stage(m: &mut WorldManager, id: &str, revision: u64) -> Value { + m.checkpoint_stage( + serde_json::from_value( + json!({"id":id,"expected_generation":1,"expected_revision":revision,"profile":"local"}), + ) + .unwrap(), + ) + .unwrap() +} +fn publication(m: &mut WorldManager, ticket: &Value) -> Value { + let deadline = Instant::now() + Duration::from_secs(35); + loop { + let p = m + .checkpoint_status( + serde_json::from_value( + json!({"id":ticket["id"],"receipt_id":ticket["receipt_id"]}), + ) + .unwrap(), + ) + .unwrap(); + if !matches!(p["status"].as_str().unwrap(), "staging" | "publishing") { + return p; + } + assert!(Instant::now() < deadline, "{p}"); + std::thread::sleep(Duration::from_millis(10)); + } +} +fn publish(m: &mut WorldManager, id: &str, receipt: &Value) -> Value { + let ticket = m + .checkpoint_publish(serde_json::from_value(json!({"id":id,"receipt":receipt})).unwrap()) + .unwrap(); + publication(m, &ticket) +} +fn running(m: &mut WorldManager, id: &str) -> Value { + let deadline = Instant::now() + Duration::from_secs(300); + loop { + let status = m.status(id).unwrap(); + if status["state"] != "starting" { + return status; + } + assert!(Instant::now() < deadline, "{status}"); + std::thread::sleep(Duration::from_millis(30)); + } +} +fn snapshots(f: &Fixture) -> Vec { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let catalog = SqlCatalogBuilder::default() + .uri(format!( + "sqlite://{}?mode=rw", + f.storage().join("catalog.sqlite").display() + )) + .warehouse_location(format!("file://{}/warehouse", f.storage().display())) + .sql_bind_style(SqlBindStyle::QMark) + .with_storage_factory(Arc::new(iceberg::io::LocalFsStorageFactory)) + .load("managed-checkpoints", HashMap::new()) + .await + .unwrap(); + let table = catalog + .load_table(&TableIdent::from_strs(["runtime", "checkpoints"]).unwrap()) + .await + .unwrap(); + table + .metadata() + .snapshots() + .map(|s| s.snapshot_id()) + .collect() + }) +} +fn read(path: &Path) -> Value { + serde_json::from_slice(&fs::read(path).unwrap()).unwrap() +} +fn write(path: &Path, value: &Value) { + fs::write(path, serde_json::to_vec(value).unwrap()).unwrap(); +} +fn exercise(native: bool) { + let f = Fixture::new(native); + let mut m = f.manager(); + let id = make(&mut m, native); + let second = make(&mut m, false); + m.start(&id).unwrap(); + m.start(&second).unwrap(); + let admission=m.admit_inputs(serde_json::from_value(json!({"id":id,"expected_generation":1,"expected_revision":1,"admission_key":"before-iceberg", + "changes":[{"op":"insert","predicate":"source","values":[-7,"saved\n\"é"]}],"effect":{"key":"frozen-effect","phase":"reserve"}})).unwrap()).unwrap(); + assert_eq!(admission["state"], "durable"); + let origin = m.status(&id).unwrap(); + let json_receipt = m.checkpoint(&id).unwrap(); + assert!(json_receipt.get("storage").is_none()); + let ticket = stage(&mut m, &id, 2); + let staged = publication(&mut m, &ticket); + assert_eq!(staged["status"], "staged", "{staged}"); + let receipt = &staged["receipt"]; + assert_eq!(receipt["storage"]["snapshot_id"], Value::Null); + assert_eq!(receipt["published_at_unix_ms"], Value::Null); + assert!(snapshots(&f).is_empty()); + assert_ne!( + receipt["checkpoint_sha256"], + receipt["storage"]["envelope_sha256"] + ); + assert_eq!(receipt["origin"]["revision"], 2); + if native { + assert!(!receipt["program"]["dependencies"] + .as_object() + .unwrap() + .is_empty()); + } + change(&mut m, &id, "insert", json!([9, "unpublished"])); + let staged_record = read(&f.record(receipt)); + // Block real catalog IO while another native/simulated world stays usable. + let lock = CatalogLock::new(&f); + let pending = m + .checkpoint_publish(serde_json::from_value(json!({"id":id,"receipt":receipt})).unwrap()) + .unwrap(); + let before = Instant::now(); + assert_eq!(m.status(&second).unwrap()["state"], "running"); + let status_ms = before.elapsed().as_secs_f64() * 1000.0; + let before = Instant::now(); + assert_eq!(rows(&mut m, &second, "echo"), json!([])); + let query_ms = before.elapsed().as_secs_f64() * 1000.0; + assert!( + status_ms < 1000.0 && query_ms < 1000.0, + "status={status_ms}, query={query_ms}" + ); + drop(lock); + let published = publication(&mut m, &pending); + assert_eq!(published["status"], "published", "{published}"); + let snapshot = published["receipt"]["storage"]["snapshot_id"] + .as_str() + .unwrap() + .parse::() + .unwrap(); + assert_eq!(snapshots(&f), vec![snapshot]); + assert_eq!( + publish(&mut m, &id, &published["receipt"])["receipt"], + published["receipt"] + ); + // Model a lost final readback/manifest write after a real catalog commit. + // The immutable staged receipt remains sufficient for explicit reconciliation. + let mut uncertain = staged_record; + uncertain["status"] = json!("publishing"); + write(&f.record(receipt), &uncertain); + m.stop(&id).unwrap(); + m.stop(&second).unwrap(); + drop(m); + let mut m = f.manager(); + let recovered = publication(&mut m, &ticket); + assert_eq!(recovered["status"], "uncertain"); + assert_eq!(recovered["receipt"], *receipt); + let published = publish(&mut m, &id, receipt); + assert_eq!(published["status"], "published", "{published}"); + assert_eq!(snapshots(&f), vec![snapshot]); + assert_eq!( + published["receipt"]["storage"]["snapshot_id"], + snapshot.to_string() + ); + for (pointer, value) in [ + ("/storage/object_uri", json!("file:///etc/passwd")), + ("/storage/snapshot_id", json!(1)), + ("/storage/snapshot_id", json!("1")), + ("/checkpoint_sha256", json!("0".repeat(64))), + ] { + let mut altered = published["receipt"].clone(); + *altered.pointer_mut(pointer).unwrap() = value; + assert!(m.restore_async(&id, &altered).is_err()); + } + assert!( + m.restore_async(&id, receipt).is_err(), + "staged is not a restore receipt" + ); + m.restore_async(&id, &published["receipt"]).unwrap(); + let restored = running(&mut m, &id); + assert_eq!(restored["state"], "running", "{restored}"); + assert_eq!(restored["generation"], 2); + let duplicate=m.admit_inputs(serde_json::from_value(json!({"id":id,"expected_generation":2,"expected_revision":2,"admission_key":"late-retry","changes":[],"effect":{"key":"frozen-effect","phase":"reserve"}})).unwrap()).unwrap(); + assert_eq!(duplicate["state"], "not_applied"); + assert_eq!(duplicate["effect_authorized"], false); + + assert_eq!(restored["revision"], 2); + assert_ne!(restored["resources"]["pid"], origin["resources"]["pid"]); + assert_eq!( + restored["persistence"]["restored_from"], + published["receipt"] + ); + assert_eq!(restored["persistence"]["persisted_revision"], Value::Null); + assert_eq!(restored["persistence"]["iceberg"]["published_revision"], 2); + assert_eq!(rows(&mut m, &id, "source"), json!([[-7, "saved\n\"é"]])); + assert_eq!(rows(&mut m, &id, "unused"), json!([])); + if native { + assert_eq!(rows(&mut m, &id, "echo"), json!([[-7, "saved\n\"é"]])); + } + change(&mut m, &id, "delete", json!([-7, "saved\n\"é"])); + change(&mut m, &id, "insert", json!([8, "after restore"])); + assert_eq!(rows(&mut m, &id, "source"), json!([[8, "after restore"]])); + if native { + assert_eq!(rows(&mut m, &id, "echo"), json!([[8, "after restore"]])); + } + if native { + // Capture ingestion has its own 100 ms tailer, independent of native + // transaction acknowledgment. Require eventual capture, not a race. + let deadline = Instant::now() + Duration::from_secs(5); + let captured = loop { + let status = m.status(&id).unwrap(); + if status["inspection"]["state"] == "available" + && status["inspection"]["activity"]["totals"]["timely"] + .as_u64() + .unwrap_or(0) + > 0 + { + break status; + } + assert!( + Instant::now() < deadline, + "Capture did not arrive: {}", + status["inspection"] + ); + std::thread::sleep(Duration::from_millis(20)); + }; + if let Some(path) = std::env::var_os("DDLOG_ICEBERG_EVIDENCE") { + write( + Path::new(&path), + &json!({"schema_version":1,"native":true,"staged":staged,"published":published,"restore":restored,"capture_state":captured["inspection"]["state"],"capture_generation":captured["generation"],"catalog_snapshot_id":snapshot.to_string(),"stage_catalog_invisible":true,"ambiguous_record_reconciled":true,"unpublished_mutation_excluded":true,"update_retraction":true,"public_composition":true,"provider_calls":0,"blocked_catalog_status_ms":status_ms,"blocked_catalog_query_ms":query_ms}), + ); + } + } + m.stop(&id).unwrap(); + let object = f.storage().join("warehouse/objects").join(format!( + "{}.parquet", + receipt["receipt_id"].as_str().unwrap() + )); + fs::write(&object, "tampered").unwrap(); + m.restore_async(&id, &published["receipt"]).unwrap(); + let failed = running(&mut m, &id); + assert_eq!(failed["state"], "failed"); + assert!(!f.root.join("worlds").join(&id).join("3/build-1").exists()); +} +#[test] +fn managed_local_iceberg_recovery_and_exact_receipts() { + exercise(false); +} +#[test] +#[ignore = "requires DDLOG_RUNTIME_NATIVE_BUILD; fresh isolated toolchain execution"] +fn native_managed_iceberg_composition_recovery() { + exercise(true); +} + +/// A real exclusive SQLite lock forces a pending storage job, without a fake +/// catalog or production fault switches. Only this fresh fixture DB is touched. +struct CatalogLock { + release: Option>, + thread: Option>, +} +impl CatalogLock { + fn new(f: &Fixture) -> Self { + let path = f.storage().join("catalog.sqlite"); + let (ready, wait) = std::sync::mpsc::channel(); + let (release, rx) = std::sync::mpsc::channel(); + let thread = std::thread::spawn(move || { + use sqlx::{Connection, Executor}; + tokio::runtime::Runtime::new().unwrap().block_on(async { + let mut conn = sqlx::SqliteConnection::connect_with( + &sqlx::sqlite::SqliteConnectOptions::new().filename(path), + ) + .await + .unwrap(); + conn.execute("BEGIN EXCLUSIVE").await.unwrap(); + ready.send(()).unwrap(); + rx.recv().unwrap(); + conn.execute("ROLLBACK").await.unwrap(); + conn.close().await.unwrap(); + }); + }); + wait.recv_timeout(Duration::from_secs(5)).unwrap(); + Self { + release: Some(release), + thread: Some(thread), + } + } +} +impl Drop for CatalogLock { + fn drop(&mut self) { + if let Some(release) = self.release.take() { + let _ = release.send(()); + } + if let Some(t) = self.thread.take() { + t.join().unwrap(); + } + } +} +#[test] +fn storage_deadline_and_stop_preserve_responsiveness_and_exact_retry() { + let f = Fixture::new(false); + f.profile(500); + let mut m = f.manager(); + let id = make(&mut m, false); + let other = make(&mut m, false); + m.start(&id).unwrap(); + m.start(&other).unwrap(); + let ticket = stage(&mut m, &id, 1); + let staged = publication(&mut m, &ticket); + assert_eq!(staged["status"], "staged", "{staged}"); + let lock = CatalogLock::new(&f); + let before = Instant::now(); + let publish = m + .checkpoint_publish( + serde_json::from_value(json!({"id":id,"receipt":staged["receipt"]})).unwrap(), + ) + .unwrap(); + assert!(before.elapsed() < Duration::from_secs(1)); + let before = Instant::now(); + assert_eq!(m.status(&other).unwrap()["state"], "running"); + assert!(before.elapsed() < Duration::from_secs(1)); + assert_eq!(rows(&mut m, &other, "source"), json!([])); + let uncertain = publication(&mut m, &publish); + assert_eq!(uncertain["status"], "uncertain"); + assert_eq!(uncertain["receipt"], staged["receipt"]); + assert!(uncertain["error"].as_str().unwrap().contains("deadline")); + drop(lock); + let published = publish_receipt(&mut m, &id, &staged["receipt"]); + assert_eq!(published["status"], "published", "{published}"); + let target = m + .create(WorldDefinition { + label: "Pending publication restore".into(), + processor: serde_json::from_value(published["receipt"]["program"]["processor"].clone()) + .unwrap(), + purpose: "instance".into(), + scenarios: vec![], + }) + .unwrap(); + let record = read(&f.record(&published["receipt"])); + let lock = CatalogLock::new(&f); + let pending = m + .checkpoint_publish( + serde_json::from_value(json!({"id":id,"receipt":published["receipt"]})).unwrap(), + ) + .unwrap(); + // Model the rename-to-fsync window: a complete manifest alone must not + // acknowledge completion while its storage job is still in flight. + write(&f.record(&published["receipt"]), &record); + let status = m + .checkpoint_status( + serde_json::from_value(json!({"id":id,"receipt_id":pending["receipt_id"]})).unwrap(), + ) + .unwrap(); + assert_eq!(status["status"], "publishing"); + assert!(m + .restore_async(&target, &published["receipt"]) + .unwrap_err() + .contains("pending")); + drop(lock); + assert_eq!(publication(&mut m, &pending)["status"], "published"); + // Cancellation and a newer generation cannot install an old job's state. + let lock = CatalogLock::new(&f); + let ticket = stage(&mut m, &id, 1); + let before = Instant::now(); + m.stop(&id).unwrap(); + assert!(before.elapsed() < Duration::from_secs(1)); + m.start(&id).unwrap(); + let uncertain = publication(&mut m, &ticket); + assert_eq!(uncertain["status"], "uncertain"); + assert_eq!(m.status(&id).unwrap()["generation"], 2); + assert_eq!(m.status(&id).unwrap()["state"], "running"); + drop(lock); +} +fn publish_receipt(m: &mut WorldManager, id: &str, receipt: &Value) -> Value { + publish(m, id, receipt) +} +#[test] +fn profiles_lock_stores_and_missing_catalog_is_truthful() { + let f = Fixture::new(false); + let mut m = f.manager(); + let id = make(&mut m, false); + m.start(&id).unwrap(); + let ticket = stage(&mut m, &id, 1); + let staged = publication(&mut m, &ticket); + assert_eq!(staged["status"], "staged", "{staged}"); + let mut other = WorldManager::new( + f.root.join("other-registry"), + f.root.join("other-worlds"), + f.driver.clone(), + ) + .unwrap(); + assert!(other + .configure_storage(&f.root.join("profile.json")) + .is_err()); + let db = f.storage().join("catalog.sqlite"); + fs::rename(&db, db.with_extension("saved")).unwrap(); + let uncertain = publish(&mut m, &id, &staged["receipt"]); + assert_eq!(uncertain["status"], "uncertain"); + assert!(uncertain["error"].as_str().unwrap().contains("missing")); + assert_eq!(m.status(&id).unwrap()["state"], "running"); + assert_eq!( + m.status(&id).unwrap()["persistence"]["iceberg"]["admission_durability"], + "json" + ); + // JSON admission is unaffected by unavailable Iceberg storage. + let admitted=m.admit_inputs(serde_json::from_value(json!({"id":id,"expected_generation":1,"expected_revision":1,"admission_key":"json-independent","changes":[],"effect":{"key":"external","phase":"reserve"}})).unwrap()).unwrap(); + assert_eq!(admitted["state"], "durable"); + assert_eq!(admitted["receipt"]["format"], "json"); + assert_eq!(admitted["effect_authorized"], true); +} + +#[test] +fn storage_startup_flag_and_async_wire_are_reusable() { + use std::io::{BufRead, BufReader, Write}; + use std::process::{Command, Stdio}; + let f = Fixture::new(false); + let mut m = f.manager(); + let id = make(&mut m, false); + drop(m); + let mut child = Command::new(env!("CARGO_BIN_EXE_ddlog-worlds")) + .arg(f.root.join("registry")) + .arg(f.root.join("worlds")) + .arg(&f.driver) + .arg("--storage-profile") + .arg(f.root.join("profile.json")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + let mut input = child.stdin.take().unwrap(); + let mut output = BufReader::new(child.stdout.take().unwrap()); + let mut call = |operation: &str, args: Value| -> Value { + writeln!(input, "{}", json!({"operation":operation,"args":args})).unwrap(); + input.flush().unwrap(); + let mut line = String::new(); + output.read_line(&mut line).unwrap(); + let value: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(value["ok"], true, "{value}"); + value["result"].clone() + }; + call("start", json!({"id":id})); + let deadline = Instant::now() + Duration::from_secs(20); + while call("status", json!({"id":id}))["state"] == "starting" { + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } + let ticket = call( + "checkpoint_stage", + json!({"id":id,"expected_generation":1,"expected_revision":1,"profile":"local"}), + ); + assert_eq!(ticket["status"], "staging"); + let query = json!({"id":id,"receipt_id":ticket["receipt_id"]}); + let staged = loop { + let value = call("checkpoint_status", query.clone()); + if value["status"] != "staging" { + break value; + } + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + }; + assert_eq!(staged["status"], "staged", "{staged}"); + assert_eq!( + call( + "checkpoint_publish", + json!({"id":id,"receipt":staged["receipt"]}) + )["status"], + "publishing" + ); + let published = loop { + let value = call("checkpoint_status", query.clone()); + if value["status"] != "publishing" { + break value; + } + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + }; + assert_eq!(published["status"], "published", "{published}"); + assert!(published["receipt"]["storage"]["snapshot_id"].is_string()); + call("stop", json!({"id":id})); + drop(input); + assert!(child.wait().unwrap().success()); +} diff --git a/tests/test_native_artifact.py b/tests/test_native_artifact.py new file mode 100644 index 0000000..e0c4367 --- /dev/null +++ b/tests/test_native_artifact.py @@ -0,0 +1,316 @@ +"""Content isolation unit tests and opt-in, real DDlog compile/query regression. + +Native: source the supported toolchain env, then set DDLOG_NATIVE_ISOLATION=1. +DDLOG_NATIVE_TEST_ROOT may name a fresh private directory to retain logs/evidence. +No owner, existing world, HTTP server or provider is used. +""" +from contextlib import chdir +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +HELPER = ROOT / 'scripts/build-native-artifact.py' +spec = importlib.util.spec_from_file_location('native_artifact', HELPER) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class ArtifactOwnership(unittest.TestCase): + def test_discovery_is_bounded_to_generated_project_and_cargo_configuration(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + project = root / 'project' + (project / '.cargo').mkdir(parents=True) + (project / '.cargo/config.toml').write_text('[build]\nrustc="fixture-rustc"\n') + commands = [] + def capture(command, env): + commands.append(command) + self.assertEqual(command[1:], ['--version', '--verbose']) + return 'fixture-version' + env = {'CARGO_HOME': str(root / 'cargo-home'), 'PATH': '/fixture'} + with patch.object(module, 'capture', capture), \ + patch.object(module, 'executable', side_effect=lambda name, env: str(root / name)), \ + patch.object(module.sys, 'platform', 'linux'): + roots, versions = module.input_roots(['cargo', 'build', '--locked', '--offline'], env, project) + self.assertIn(project, roots) + self.assertTrue(all(p == project or p.name in ('config', 'config.toml') for p in roots)) + self.assertEqual(len(commands), 2) # No cargo metadata or sysroot/vendor/SDK traversal. + self.assertEqual(versions['rustc_path'], str(root / 'fixture-rustc')) + self.assertEqual(versions['rustc'], 'fixture-version') + + def test_rustup_symlinks_query_selected_tools_and_separate_toolchain_keys(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + project, binaries = root / 'project', root / 'bin' + project.mkdir() + binaries.mkdir() + proxy = binaries / 'rustup' + proxy.write_text('''#!/bin/sh +case "$1 $2" in '--version --verbose') ;; *) exit 2 ;; esac +case "${0##*/}" in + cargo|rustc) printf '%s fixture-%s\\n' "${0##*/}" "$RUSTUP_TOOLCHAIN" ;; + *) printf 'rustup fixture-manager\\n' ;; +esac +''') + proxy.chmod(0o755) + for name in ('cargo', 'rustc'): + (binaries / name).symlink_to(proxy) + for explicit in (False, True): + with self.subTest(explicit_tool_paths=explicit): + cargo = str(binaries / 'cargo') if explicit else 'cargo' + rustc = str(binaries / 'rustc') if explicit else 'rustc' + command = [cargo, 'build', '--locked', '--offline'] + env = {'PATH': str(binaries), 'RUSTC': rustc, + 'CARGO_HOME': str(root / 'cargo-home')} + keys = [] + for toolchain in ('1.94.1', '1.95.0'): + env['RUSTUP_TOOLCHAIN'] = toolchain + with patch.object(module.sys, 'platform', 'linux'): + roots, versions = module.input_roots(command, env, project) + for name in ('cargo', 'rustc'): + self.assertEqual(versions[name], f'{name} fixture-{toolchain}') + self.assertEqual(versions[name + '_path'], str(proxy)) + keys.append(module.fingerprint(roots, versions, command, env, ())[0]) + self.assertNotEqual(keys[0], keys[1]) + + def test_key_covers_project_sources_lock_configuration_and_relevant_flags_only(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + paths = ['program/src/lib.rs', 'program/types/child/lib.rs', 'program/Cargo.lock', + 'program/.cargo/config.toml', 'cargo-home/config.toml'] + for name in paths: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('original') + roots = {root / 'program', root / 'cargo-home/config.toml'} + env = {'RUSTFLAGS': '-C opt-level=0', 'PRIVATE_CREDENTIAL': 'must-not-be-recorded'} + def fingerprint(): + return module.fingerprint(roots, {'rustc': 'fixture'}, ['cargo', 'build'], env, ()) + original, inputs = fingerprint() + self.assertNotIn('must-not-be-recorded', json.dumps(inputs)) + for name in paths: + with self.subTest(input=name): + path = root / name + before = path.stat() + path.write_text('modified') + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) + self.assertNotEqual(fingerprint()[0], original) + path.write_text('original') + self.assertEqual(fingerprint()[0], original) + env.update(PRIVATE_CREDENTIAL='rotated-secret', UNRELATED_WORLD_GENERATION='2') + self.assertEqual(fingerprint()[0], original) + env['RUSTFLAGS'] = '-C opt-level=1' + self.assertNotEqual(fingerprint()[0], original) + env['RUSTFLAGS'] = '-C opt-level=0' + self.assertNotEqual(module.fingerprint(roots, {'rustc': 'changed'}, + ['cargo', 'build'], env, ())[0], original) + self.assertNotEqual(module.fingerprint(roots, {'rustc': 'fixture'}, + ['cargo', 'build', '--release'], env, ())[0], original) + (root / 'program/target').mkdir() + (root / 'program/target/ignored-output').write_text('not an input') + self.assertEqual(fingerprint()[0], original) + # A different generated project's equal package names never share artifacts. + import shutil + shutil.copytree(root / 'program', root / 'other') + self.assertNotEqual(module.fingerprint({root / 'other', root / 'cargo-home/config.toml'}, + {'rustc': 'fixture'}, ['cargo', 'build'], env, ())[0], original) + + def test_project_symlink_aliases_do_not_recurse_forever(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'dependency').write_text('first') + (root / 'project').mkdir() + (root / 'project/link').symlink_to(root / 'dependency') + first = module.digest_tree(root / 'project') + (root / 'dependency').write_text('other') + self.assertNotEqual(module.digest_tree(root / 'project'), first) + (root / 'project/cycle').symlink_to(root / 'project', target_is_directory=True) + cyclic = module.digest_tree(root / 'project') + self.assertEqual(module.digest_tree(root / 'project'), cyclic) + + def fixture(self, root): + project = root / 'project' + project.mkdir() + (project / 'input.rs').write_text('alpha') + compiler = root / 'compiler.py' + compiler.write_text('''import json,os,sys,time +from pathlib import Path +source=Path('input.rs'); value=source.read_text() +target=Path(os.environ['CARGO_TARGET_DIR']) +assert str(target)==sys.argv[sys.argv.index('--target-dir')+1] +target.mkdir(parents=True,exist_ok=True) +marker=target/'active' +fd=os.open(marker,os.O_CREAT|os.O_EXCL|os.O_WRONLY,0o600) +try: + artifact=target/'debug/program_cli'; artifact.parent.mkdir(exist_ok=True) + artifact.write_text(value) + time.sleep(.05) + if value=='mutate': source.write_text('changed') + if value=='failure': sys.exit(9) + if value!='missing': + print(json.dumps({'reason':'compiler-artifact','target':{'name':'program_cli','kind':['bin']},'executable':str(artifact)})) +finally: + os.close(fd); marker.unlink() +''') + return project, [sys.executable, str(compiler)] + + def test_exact_artifact_failure_and_changed_inputs_never_publish_stale_binary(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + project, command = self.fixture(root) + base, output = root / 'cache', root / 'output' + with chdir(project), patch.dict(os.environ, {'CARGO_TARGET_DIR': str(base)}), \ + patch.object(module, 'input_roots', return_value=({project}, {})): + self.assertEqual(module.build(output, command), 0) + self.assertEqual(output.read_text(), 'alpha') + for value in ('beta', 'alpha'): + (project / 'input.rs').write_text(value) + self.assertEqual(module.build(output, command), 0) + self.assertEqual(output.read_text(), value) + self.assertEqual(len(list((base / 'ddlog-content-v2').iterdir())), 2) + (project / 'input.rs').write_text('failure') + self.assertEqual(module.build(output, command), 9) + self.assertEqual(output.read_text(), 'alpha') + (project / 'input.rs').write_text('missing') + with self.assertRaisesRegex(ValueError, 'refusing stale artifact'): + module.build(output, command) + (project / 'input.rs').write_text('mutate') + with self.assertRaisesRegex(ValueError, 'quarantined'): + module.build(output, command) + self.assertEqual(output.read_text(), 'alpha') + self.assertEqual(len(list(base.glob('ddlog-content-v2/*/changed-inputs-*'))), 1) + + def test_key_lock_covers_atomic_output_publication(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + project, command = self.fixture(root) + base = root / 'cache' + original = module.shutil.copy2 + def check_copy(source, destination): + lock = module.project_lock(base, project.resolve()) + probe = subprocess.run([sys.executable, '-c', '''import fcntl,sys +with open(sys.argv[1],'r') as lock: + try: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + except BlockingIOError: sys.exit(7) +''', str(lock)], timeout=5) + self.assertEqual(probe.returncode, 7) + self.assertEqual((root / 'output').read_text(), 'old') + return original(source, destination) + (root / 'output').write_text('old') + with chdir(project), patch.dict(os.environ, {'CARGO_TARGET_DIR': str(base)}), \ + patch.object(module, 'input_roots', return_value=({project}, {})), \ + patch.object(module.shutil, 'copy2', check_copy): + self.assertEqual(module.build(root / 'output', command), 0) + self.assertEqual((root / 'output').read_text(), 'alpha') + + def test_concurrent_identical_inputs_share_key_and_keep_lock_through_copy(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + project, command = self.fixture(root) + # Mock discovery only; both separate helper processes use the real + # fingerprint, locking, artifact parsing and publication code. + bootstrap = '''import importlib.util,sys +from pathlib import Path +s=importlib.util.spec_from_file_location('artifact',sys.argv[1]);m=importlib.util.module_from_spec(s);s.loader.exec_module(m) +m.input_roots=lambda *args: ({Path.cwd()}, {}) +sys.exit(m.build(sys.argv[2],sys.argv[3:])) +''' + env = dict(os.environ, CARGO_TARGET_DIR=str(root / 'cache')) + children = [subprocess.Popen([sys.executable, '-c', bootstrap, str(HELPER), + str(root / name), *command], cwd=project, env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) for name in ('one', 'two')] + try: + for child in children: + stdout, stderr = child.communicate(timeout=15) + self.assertEqual(child.returncode, 0, (stdout + stderr).decode()) + finally: + for child in children: + if child.poll() is None: + child.kill() + child.wait() + self.assertEqual((root / 'one').read_text(), 'alpha') + self.assertEqual((root / 'two').read_text(), 'alpha') + self.assertEqual(len(list((root / 'cache/ddlog-content-v2').iterdir())), 1) + + +@unittest.skipUnless(os.environ.get('DDLOG_NATIVE_ISOLATION') == '1', 'opt-in native DDlog toolchain') +class NativeIsolation(unittest.TestCase): + def test_simultaneous_and_sequential_programs_query_their_own_answers(self): + retained = os.environ.get('DDLOG_NATIVE_TEST_ROOT') + temporary = None + if retained: + root = Path(retained).resolve() + root.mkdir(parents=True, exist_ok=False) + else: + temporary = tempfile.TemporaryDirectory(prefix='ddlog-native-isolation-') + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + env = dict(os.environ, CARGO_TARGET_DIR=str(root / 'native-target'), + CARGO_HOME=str(root / 'cargo-home')) + (root / 'cargo-home').mkdir() + programs = [('alpha', 'Alpha', 7), ('beta', 'Beta', 107)] + for name, relation, answer in programs: + project = root / name + project.mkdir() + (project / 'program.dl').write_text( + 'input relation Seed(value: signed<64>)\n' + f'output relation {relation}(value: signed<64>)\n' + f'{relation}(v) :- Seed(x), var v = x + {answer - 7}.\n') + # Both sources predate either native Cargo build, reproducing the stale + # freshness condition. The two generated packages have identical names. + builds = [] + try: + for name, _, _ in programs: + project = root / name + log = (root / f'{name}-concurrent.log').open('w') + child = subprocess.Popen([str(ROOT / 'scripts/build-ddlog.sh'), + str(project / 'program.dl'), str(project / 'program_cli')], env=env, stdout=log, stderr=log) + builds.append((child, log)) + for child, log in builds: + self.assertEqual(child.wait(timeout=600), 0, f'See retained log {log.name}') + finally: + for child, log in builds: + if child.poll() is None: + child.kill() + child.wait() + log.close() + evidence = [] + def query(name, relation, answer, phase): + binary = root / name / 'program_cli' + result = subprocess.run([str(binary)], input=f'start;\ninsert Seed(7);\ncommit;\ndump {relation};\nexit;\n', + text=True, capture_output=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f'{relation}{{.value = {answer}}}', result.stdout) + self.assertNotIn('Unknown', result.stdout + result.stderr) + evidence.append({'program': name, 'phase': phase, 'stdout': result.stdout, + 'stderr': result.stderr, 'source_sha256': module.digest_file(root / name / 'program.dl'), + 'binary_sha256': module.digest_file(binary)}) + for name, relation, answer in programs: + query(name, relation, answer, 'concurrent') + self.assertNotEqual(evidence[0]['binary_sha256'], evidence[1]['binary_sha256']) + # Interleave warm builds in reverse order; neither may inherit its neighbor. + for name, relation, answer in reversed(programs): + project = root / name + with (root / f'{name}-sequential.log').open('w') as log: + result = subprocess.run([str(ROOT / 'scripts/build-ddlog.sh'), + str(project / 'program.dl'), str(project / 'program_cli')], env=env, + stdout=log, stderr=log, timeout=600) + self.assertEqual(result.returncode, 0, f'See {log.name}') + query(name, relation, answer, 'sequential') + buckets = list((root / 'native-target/ddlog-content-v2').iterdir()) + self.assertEqual(len(buckets), 2, 'Identical repeated build inputs must reuse their content namespace') + (root / 'evidence.json').write_text(json.dumps({'passed': True, 'queries': evidence, + 'content_keys': sorted(p.name for p in buckets)}, indent=2) + '\n') + print(f'Native isolation evidence: {root}', flush=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_native_packaging.py b/tests/test_native_packaging.py index 8a9a9b7..58c20b2 100644 --- a/tests/test_native_packaging.py +++ b/tests/test_native_packaging.py @@ -5,6 +5,7 @@ import os from pathlib import Path import subprocess +import sys import tempfile import tomllib import unittest @@ -53,7 +54,20 @@ def test_driver_selects_lock_and_preserves_explicit_override(self): ('mkdir -p program_ddlog/types/lemmalog_star\n' if star else '')) compiler.chmod(0o755) cargo = root / 'cargo' - cargo.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > args\nmkdir -p target/debug\nprintf binary > target/debug/program_cli\n') + cargo.write_text(f'''#!{sys.executable} +import json +from pathlib import Path +import sys +if '--version' in sys.argv: + print('fixture-tool 1.0') +else: + Path({str(root / 'build-args')!r}).write_text('\\n'.join(sys.argv[1:])) + target = Path(sys.argv[sys.argv.index('--target-dir') + 1]) + artifact = target / 'debug/program_cli' + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_text('binary') + print(json.dumps({{'reason':'compiler-artifact', 'target':{{'name':'program_cli','kind':['bin']}}, 'executable':str(artifact)}})) +''') cargo.chmod(0o755) # Simulated pinned generator output needed by the native hook installer. project = root / 'program_ddlog' @@ -68,14 +82,14 @@ def test_driver_selects_lock_and_preserves_explicit_override(self): (root / 'override.lock').write_text('explicit override\n[[package]]\nname = "types__lemmalog_star"\n') env = {k: v for k, v in os.environ.items() if not k.startswith(('DDLOG_', 'CARGO_'))} env.update(DDLOG_HOME=str(root), DDLOG_CARGO=str(cargo), DDLOG_OFFLINE='1', - DDLOG_LOCK_DIR=str(ROOT / 'native')) + DDLOG_LOCK_DIR=str(ROOT / 'native'), RUSTC=str(cargo)) if override: env['DDLOG_CARGO_LOCK'] = str(root / 'override.lock') subprocess.run([str(ROOT / 'scripts/build-ddlog.sh'), str(root / 'program.dl'), str(root / 'output')], env=env, check=True, capture_output=True) expected = root / 'override.lock' if override else ROOT / 'native' / ('star.Cargo.lock' if star else 'program.Cargo.lock') self.assertEqual((root / 'program_ddlog/Cargo.lock').read_bytes(), expected.read_bytes()) - self.assertIn('--locked', (root / 'program_ddlog/args').read_text()) + self.assertIn('--locked', (root / 'build-args').read_text()) self.assertEqual((root / 'output').read_text(), 'binary') self.assertEqual((project / 'differential_datalog/src/observer.rs').read_bytes(), (ROOT / 'native/observer.rs').read_bytes()) marker = json.loads((project / 'observer-install.json').read_text()) diff --git a/tests/test_python_transport.py b/tests/test_python_transport.py new file mode 100644 index 0000000..c1b2a8c --- /dev/null +++ b/tests/test_python_transport.py @@ -0,0 +1,57 @@ +"""Response identity, no retry, and advisory timing for the shared attachment.""" +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'python')) +from ddlog_runtime_client import AttachedRuntimeClient + + +class TransportTests(unittest.TestCase): + def exchange(self, modify=lambda reply: None, callback=None, disconnect=False): + with tempfile.TemporaryDirectory() as directory: + descriptor = Path(directory) / 'endpoint.json' + descriptor.write_text(json.dumps({'schema_version':1, 'socket':'/fixture.sock', + 'owner_incarnation':'owner'})) + client = AttachedRuntimeClient(descriptor, on_timing=callback) + connection = MagicMock() + def receive(_): + if disconnect: + return b'' + request = json.loads(connection.sendall.call_args.args[0]) + reply = {'ok':True, 'request_id':request['request_id'], + 'owner_incarnation':'owner', 'result':{'acknowledged':True}} + modify(reply) + return json.dumps(reply).encode() + b'\n' + connection.recv.side_effect = receive + with patch('ddlog_runtime_client.transport.socket.socket') as factory: + factory.return_value.__enter__.return_value = connection + try: + return client.request('admit_inputs', {'id':'world'}) + finally: + self.assertEqual(factory.call_count, 1) + self.assertEqual(connection.sendall.call_count, 1) + + def test_advisory_timing_cannot_discard_acknowledgment(self): + def broken_callback(*_): + raise RuntimeError('telemetry unavailable') + reply = self.exchange(lambda reply: reply.update(timing='malformed'), broken_callback) + self.assertTrue(reply['result']['acknowledged']) + + def test_mismatched_response_identity_is_unknown_and_not_retried(self): + for key in ['request_id', 'owner_incarnation']: + with self.subTest(key=key), self.assertRaisesRegex(RuntimeError, 'outcome unknown; no retry'): + self.exchange(lambda reply: reply.update({key:'different'})) + + def test_timing_failure_does_not_mask_lost_reply(self): + def broken_callback(*_): + raise ValueError('metrics failure') + with self.assertRaisesRegex(RuntimeError, 'outcome unknown; no retry.*closed'): + self.exchange(callback=broken_callback, disconnect=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_python_world_client.py b/tests/test_python_world_client.py new file mode 100644 index 0000000..459ff33 --- /dev/null +++ b/tests/test_python_world_client.py @@ -0,0 +1,111 @@ +"""Public Python worker client: consistent evidence and no retry of writes.""" +import copy +import importlib +from pathlib import Path +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'python')) +from ddlog_runtime_client import ManagedWorldHost + + +class Client: + def __init__(self): + self.calls = []; self.revision = 4; self.generation = 2; self.rows = {}; self.reject_page = False + self.publication = {'state':'durable','applied_revision':5,'receipt':{'exact':True}, + 'effect_authorized':True,'replayed':False} + def request(self, operation, args): + self.calls.append((operation, copy.deepcopy(args))) + if operation == 'status': + return {'ok':True,'result':{'id':'world','generation':self.generation,'revision':self.revision,'state':'running'}} + if operation == 'read_batch': + if self.reject_page and args['queries'][0]['continuation']: + return {'ok':False,'error':'Expected revision mismatch; request not applied'} + results = [] + for query in args['queries']: + rows = self.rows.get(query['predicate'], []) + start = query.get('continuation') or 0; end = start + query['max_rows'] + results.append({'revision':self.revision,'rows':rows[start:end], + 'complete':end>=len(rows),'continuation':end if end Self { + let endpoint = root.join("owner.json"); + let child = Command::new(env!("CARGO_BIN_EXE_ddlog-worlds")) + .arg(root.join("registry")) + .arg(root.join("worlds")) + .arg(driver) + .arg("--listen") + .arg(&endpoint) + .arg("--worker-profiles") + .arg(root.join("profiles.json")) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .unwrap(); + let mut owner = Self { child, endpoint }; + let deadline = Instant::now() + Duration::from_secs(10); + while !owner.endpoint.exists() { + assert!( + owner.child.try_wait().unwrap().is_none(), + "owner exited before publication" + ); + assert!(Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } + owner + } + fn call(&self, operation: &str, args: Value) -> Value { + call(&self.endpoint, operation, args) + } + fn status(&self, id: &str) -> Value { + self.call("status", json!({"id":id})) + } + fn wait(&self, id: &str, done: impl Fn(&Value) -> bool) -> Value { + let deadline = Instant::now() + Duration::from_secs(180); + loop { + let status = self.status(id); + if done(&status) { + return status; + } + assert!(Instant::now() < deadline, "{status}"); + std::thread::sleep(Duration::from_millis(20)); + } + } + fn worker(&self, id: &str, generation: u64, key: &str, mode: &str) -> Value { + self.call("worker_start",json!({"id":id,"expected_generation":generation,"profile":"fixture","key":key,"payload":{"mode":mode}})) + } +} +impl Drop for Owner { + fn drop(&mut self) { + unsafe { + libc::kill(self.child.id() as i32, libc::SIGTERM); + } + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if self.child.try_wait().unwrap().is_some() { + break; + } + if Instant::now() >= deadline { + let _ = self.child.kill(); + let _ = self.child.wait(); + panic!("isolated owner shutdown exceeded deadline"); + } + std::thread::sleep(Duration::from_millis(10)); + } + } +} +fn call(endpoint: &Path, operation: &str, args: Value) -> Value { + let descriptor: Value = serde_json::from_slice(&fs::read(endpoint).unwrap()).unwrap(); + let mut stream = UnixStream::connect(descriptor["socket"].as_str().unwrap()).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(20))) + .unwrap(); + let mut bytes=serde_json::to_vec(&json!({"schema_version":1,"request_id":"fixture-client","owner_incarnation":descriptor["owner_incarnation"],"operation":operation,"args":args})).unwrap(); + bytes.push(b'\n'); + stream.write_all(&bytes).unwrap(); + let mut line = String::new(); + BufReader::new(stream).read_line(&mut line).unwrap(); + let response: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(response["ok"], true, "{response}"); + response["result"].clone() +} +fn file(path: &Path, owner: &Owner, id: &str) { + let deadline = Instant::now() + Duration::from_secs(15); + while !path.exists() { + let status = owner.status(id); + assert!( + !status["workers"] + .as_array() + .unwrap() + .iter() + .any(|w| w["state"] == "failed"), + "{status}" + ); + assert!( + Instant::now() < deadline, + "waiting for {}: {status}", + path.display() + ); + std::thread::sleep(Duration::from_millis(20)); + } +} +fn gone(pid: i32) { + let deadline = Instant::now() + Duration::from_secs(5); + while unsafe { libc::kill(pid, 0) } == 0 { + assert!(Instant::now() < deadline, "owned test pid {pid} survived"); + std::thread::sleep(Duration::from_millis(10)); + } +} +fn exercise(native: bool) { + let root = + std::env::temp_dir().join(format!("wks-{}-{}", std::process::id(), u8::from(native))); + fs::create_dir(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + let registry = ProcessorRegistry::open(root.join("registry")).unwrap(); + let definition:ProcessorDefinition=serde_json::from_value(json!({"rules":"echo(N,S) :- source(N,S).","schemas":{"source":{"input":true,"fields":["int","string"]},"echo":{"input":false,"fields":["int","string"]}}})).unwrap(); + let record = registry.create(definition, None).unwrap(); + let pin = json!({"processor_id":record.processor_id,"version":record.version}); + let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + fs::write(root.join("profiles.json"),serde_json::to_vec(&json!({"schema_version":1,"profiles":{"fixture":{ + "argv":["/usr/bin/env","python3",manifest.join("tests/fixtures/managed_worker.py")], + "env":{"FIXTURE_ROOT":root},"allowed_processors":[pin],"max_concurrent_workers":2,"timeout_ms":30000 + }}})).unwrap()).unwrap(); + fs::set_permissions( + root.join("profiles.json"), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + let driver = if native { + PathBuf::from( + std::env::var_os("DDLOG_RUNTIME_NATIVE_BUILD").expect("Configure native build driver"), + ) + } else { + let path = root.join("build.py"); + fs::write(&path,format!("#!/usr/bin/env python3\nimport sys\nfrom pathlib import Path\nsource=Path({}).read_text().replace('__CONTROL__',{})\nPath(sys.argv[2]).write_text(source)\nPath(sys.argv[2]).chmod(0o700)\n",json!(manifest.join("tests/fixtures/memory_fake_runtime.py")),json!(serde_json::to_string(&root).unwrap()))).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); + path + }; + let owner = Owner::launch(&root, &driver); + let id=owner.call("create",json!({"label":"bounded worker fixture","processor":pin,"purpose":"instance","scenarios":[]}))["id"].as_str().unwrap().to_string(); + owner.call("start", json!({"id":id})); + let started = owner.wait(&id, |s| s["state"] != "starting"); + assert_eq!(started["state"], "running", "{started}"); + let worker = owner.worker(&id, 1, "attached", "attached"); + // Every call has already closed its socket. No HTTP/client process owns this worker. + file(&root.join("attached.reserved.json"), &owner, &id); + let reserved: Value = + serde_json::from_slice(&fs::read(root.join("attached.reserved.json")).unwrap()).unwrap(); + assert_eq!(reserved["effect_authorized"], true); + assert_eq!(reserved["applied_revision"], 2); + assert_eq!(owner.status(&id)["workers"][0]["state"], "running"); + assert_eq!( + owner.status(&id)["resources"]["pid"], + started["resources"]["pid"] + ); + let historical = owner.call( + "admission_status", + json!({"id":id,"generation":1,"admission_key":"attached.reserve"}), + ); + assert_eq!(historical["effect_authorized"], false); + fs::write(root.join("attached.release"), "").unwrap(); + let finished = owner.wait(&id, |s| s["workers"][0]["state"] != "running"); + assert_eq!(finished["workers"][0]["state"], "completed", "{finished}"); + let settled: Value = + serde_json::from_slice(&fs::read(root.join("attached.settled.json")).unwrap()).unwrap(); + assert_eq!(settled["applied_revision"], 3); + assert_eq!(settled["state"], "durable"); + let evidence=owner.call("read_batch",json!({"id":id,"expected_generation":1,"expected_revision":3,"queries":[{"predicate":"source"},{"predicate":"echo"}]})); + assert_eq!(evidence["results"][0]["rows"], json!([[20, "settled"]])); + assert_eq!(evidence["results"][1]["rows"], json!([[20, "settled"]])); + // Two independently attached writers race the same evidence revision. + let requests:Vec<_>=(0..2).map(|n|{let endpoint=owner.endpoint.clone();let id=id.clone();std::thread::spawn(move||call(&endpoint,"admit_inputs",json!({"id":id,"expected_generation":1,"expected_revision":3,"admission_key":format!("contender-{n}"),"changes":[]})))}).collect(); + let results: Vec<_> = requests.into_iter().map(|r| r.join().unwrap()).collect(); + assert_eq!( + results.iter().filter(|r| r["state"] == "durable").count(), + 1 + ); + assert_eq!( + results + .iter() + .filter(|r| r["state"] == "not_applied") + .count(), + 1 + ); + let duplicate = owner.worker(&id, 1, "attached", "attached"); + assert_eq!(duplicate["worker_id"], worker["worker_id"]); + assert_eq!(duplicate["replayed"], true); + owner.call("stop", json!({"id":id})); + drop(owner); + let owner = Owner::launch(&root, &driver); + assert_eq!(owner.status(&id)["workers"][0]["state"], "completed"); + owner.call("restore", json!({"id":id,"receipt":reserved["receipt"]})); + let restored = owner.wait(&id, |s| s["state"] != "starting"); + assert_eq!(restored["state"], "running", "{restored}"); + assert_eq!(restored["generation"], 2); + assert_eq!(restored["revision"], 2); + let duplicate=owner.call("admit_inputs",json!({"id":id,"expected_generation":2,"expected_revision":2,"admission_key":"restored-duplicate","changes":[],"effect":{"key":"attached","phase":"reserve"}})); + assert_eq!(duplicate["state"], "not_applied"); + assert_eq!(duplicate["effect_authorized"], false); + let old=owner.call("admit_inputs",json!({"id":id,"expected_generation":2,"expected_revision":2,"admission_key":"late","worker_id":worker["worker_id"],"changes":[]})); + assert_eq!(old["state"], "not_applied"); + let evidence=owner.call("read_batch",json!({"id":id,"expected_generation":2,"expected_revision":2,"queries":[{"predicate":"echo"}]})); + assert_eq!(evidence["results"][0]["rows"], json!([[10, "reserved"]])); + // Real owner SIGTERM must kill worker descendants as well as the native process. + let dying = owner.worker(&id, 2, "owner-exit", "descendant"); + file(&root.join("owner-exit.child"), &owner, &id); + let descendant = fs::read_to_string(root.join("owner-exit.child")) + .unwrap() + .parse::() + .unwrap(); + let native_pid = owner.status(&id)["resources"]["pid"].as_u64().unwrap() as i32; + drop(owner); + gone(dying["pid"].as_u64().unwrap() as i32); + gone(descendant); + gone(native_pid); + if native { + if let Some(path) = std::env::var_os("DDLOG_WORKER_EVIDENCE") { + fs::write(path,serde_json::to_vec_pretty(&json!({"schema_version":1,"native":true,"reserved":reserved,"settled":settled,"restored":restored["persistence"],"competing_admissions":results,"worker":finished["workers"][0],"cleanup":"owner, worker descendant and native exited"})).unwrap()).unwrap(); + } + } + fs::remove_dir_all(root).unwrap(); +} +#[test] +fn attached_worker_admission_replay_restore_and_owner_shutdown() { + exercise(false); +} +#[test] +#[ignore = "requires operator-configured DDLOG_RUNTIME_NATIVE_BUILD toolchain"] +fn native_attached_worker_admission_replay_restore_and_owner_shutdown() { + exercise(true); +} diff --git a/tests/worlds.rs b/tests/worlds.rs index d8c7e1d..705b056 100644 --- a/tests/worlds.rs +++ b/tests/worlds.rs @@ -680,10 +680,9 @@ fn inventory_filters_by_pin_and_summary_skips_live_instance_calls() { for absent in ["inspection", "instance", "managed_processes"] { assert!(running.get(absent).is_none(), "{absent} present in summary"); } - assert_eq!( - running["persistence"], - json!({"status":"not_configured","reason":"world checkpoints are not wired"}) - ); + assert_eq!(running["persistence"]["status"], "configured"); + assert_eq!(running["persistence"]["checkpoints"], json!([])); + assert_eq!(running["persistence"]["persisted_revision"], Value::Null); assert_eq!(running["build"], Value::Null); assert_eq!(running["resources"]["state"], "available"); let full = manager.status(&b).unwrap(); @@ -1115,6 +1114,9 @@ fn activity_is_tailed_rotation_keeps_topology_and_captures_are_retained_once() { append_capture(&f, &id, 1, &topology()); let available = wait_until(&mut manager, &id, |s| { s["inspection"]["state"] == "available" + && s["inspection"]["activity"]["complete"] == true + && s["inspection"]["activity"]["totals"]["operators"] == 2 + && s["inspection"]["activity"]["totals"]["channels"] == 1 }); assert_eq!(available["inspection"]["unresolved_channels"], 0); assert_eq!(available["inspection"]["activity"]["complete"], true); @@ -1731,3 +1733,11 @@ fn composition_worlds_synthesize_module_groups_that_propagate_and_nest() { Value::Null ); } + +#[path = "worlds/persistence.rs"] +mod persistence; + +#[path = "worlds/admission.rs"] +mod admission; +#[path = "worlds/workers.rs"] +mod workers; diff --git a/tests/worlds/admission.rs b/tests/worlds/admission.rs new file mode 100644 index 0000000..42329c2 --- /dev/null +++ b/tests/worlds/admission.rs @@ -0,0 +1,236 @@ +//! Fences and failure classification with a simulated native transport. +use super::*; +fn admit(manager: &mut WorldManager, request: Value) -> Value { + manager + .admit_inputs(serde_json::from_value(request).unwrap()) + .unwrap() +} +fn request(id: &str, generation: u64, revision: u64, key: &str) -> Value { + json!({"id":id,"expected_generation":generation,"expected_revision":revision,"admission_key":key, + "changes":[{"op":"insert","predicate":"source","values":[revision,"private input"]}]}) +} +fn read( + manager: &mut WorldManager, + id: &str, + generation: u64, + revision: u64, +) -> Result { + manager.read_batch( + serde_json::from_value( + json!({"id":id,"expected_generation":generation,"expected_revision":revision, + "queries":[{"predicate":"source","max_rows":1},{"predicate":"echo"}]}), + ) + .unwrap(), + ) +} +#[test] +fn cas_reads_effects_replay_and_restore_fence_all_mutations() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + let initial = read(&mut manager, &id, 1, 1).unwrap(); + assert_eq!(initial["results"][0]["rows"], json!([])); + let mut claim = request(&id, 1, 1, "claim"); + claim["effect"] = json!({"key":"effect","phase":"reserve"}); + let reserved = admit(&mut manager, claim.clone()); + assert_eq!(reserved["state"], "durable"); + assert_eq!(reserved["effect_authorized"], true); + assert_eq!(reserved["receipt"]["origin"]["revision"], 2); + let duplicate = admit(&mut manager, claim.clone()); + assert_eq!(duplicate["receipt"], reserved["receipt"]); + assert_eq!(duplicate["effect_authorized"], false); + assert_eq!(duplicate["replayed"], true); + claim["changes"][0]["values"][0] = json!(99); + assert!(manager + .admit_inputs(serde_json::from_value(claim).unwrap()) + .is_err()); + assert!(read(&mut manager, &id, 1, 1).is_err()); + let stale = admit(&mut manager, request(&id, 1, 1, "stale")); + assert_eq!(stale["state"], "not_applied"); + let mut second = request(&id, 1, 2, "second-claim"); + second["effect"] = json!({"key":"effect","phase":"reserve"}); + assert_eq!(admit(&mut manager, second)["state"], "not_applied"); + let mut settle = request(&id, 1, 2, "settle"); + settle["effect"] = json!({"key":"effect","phase":"settle","reservation_key":"wrong"}); + assert_eq!(admit(&mut manager, settle.clone())["state"], "not_applied"); + settle["effect"]["reservation_key"] = json!("claim"); + assert_eq!(admit(&mut manager, settle.clone())["state"], "durable"); + settle["admission_key"] = json!("duplicate-settlement"); + settle["expected_revision"] = json!(3); + assert_eq!(admit(&mut manager, settle)["state"], "not_applied"); + let page = read(&mut manager, &id, 1, 3).unwrap(); + assert_eq!(page["results"][0]["total"], 2); + assert_eq!(page["results"][0]["complete"], false); + assert_eq!(page["results"][1]["revision"], 3); + assert_eq!( + admit(&mut manager, request(&id, 1, 3, "advance"))["state"], + "durable" + ); + assert!(manager + .read_batch( + serde_json::from_value( + json!({"id":id,"expected_generation":1,"expected_revision":4, + "queries":[{"predicate":"source","continuation":page["results"][0]["continuation"]}]}) + ) + .unwrap() + ) + .is_err()); + let commands = fs::read_to_string(f.root.join("commands")).unwrap(); + assert!( + !commands.contains("commit dump_changes"), + "admission must never consume unbounded deltas" + ); + manager.stop(&id).unwrap(); + drop(manager); + let mut manager = f.manager(); + let historical = manager + .admission_status( + serde_json::from_value(json!({"id":id,"generation":1,"admission_key":"claim"})) + .unwrap(), + ) + .unwrap(); + assert_eq!(historical["receipt"], reserved["receipt"]); + assert_eq!(historical["effect_authorized"], false); + manager.restore_async(&id, &reserved["receipt"]).unwrap(); + wait_until(&mut manager, &id, |s| s["state"] == "running"); + let mut retry = request(&id, 2, 2, "new-claim"); + retry["effect"] = json!({"key":"effect","phase":"reserve"}); + assert_eq!(admit(&mut manager, retry.clone())["state"], "not_applied"); + retry["effect"] = json!({"key":"effect","phase":"settle","reservation_key":"claim"}); + assert_eq!(admit(&mut manager, retry)["state"], "not_applied"); + manager.stop(&id).unwrap(); + manager.start(&id).unwrap(); + let mut fresh = request(&id, 3, 1, "fresh"); + fresh["effect"] = json!({"key":"effect","phase":"reserve"}); + assert_eq!(admit(&mut manager, fresh)["effect_authorized"], true); +} +#[test] +fn publication_and_native_ack_failures_never_authorize_effects() { + for lose_ack in [false, true] { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + if lose_ack { + fs::write(f.root.join("die_on_commit"), "").unwrap(); + } else { + fs::write( + f.root.join("worlds").join(&id).join("checkpoints"), + "block publication", + ) + .unwrap(); + } + let mut claim = request(&id, 1, 1, "claim"); + claim["effect"] = json!({"key":"effect","phase":"reserve"}); + let failed = admit(&mut manager, claim.clone()); + assert_eq!( + failed["state"], + if lose_ack { + "uncertain" + } else { + "applied_but_unpublished" + } + ); + assert_eq!(failed["receipt"], Value::Null); + assert_eq!(failed["effect_authorized"], false); + assert_eq!( + failed["applied_revision"], + if lose_ack { Value::Null } else { json!(2) } + ); + assert_eq!(admit(&mut manager, claim)["effect_authorized"], false); + if !lose_ack { + assert_eq!( + read(&mut manager, &id, 1, 2).unwrap()["results"][0]["total"], + 1 + ); + fs::remove_file(f.root.join("worlds").join(&id).join("checkpoints")).unwrap(); + let mut settle = request(&id, 1, 2, "settle"); + settle["effect"] = json!({"key":"effect","phase":"settle","reservation_key":"claim"}); + assert_eq!(admit(&mut manager, settle)["state"], "not_applied"); + } + drop(manager); + let mut manager = f.manager(); + let persisted = manager + .admission_status( + serde_json::from_value(json!({"id":id,"generation":1,"admission_key":"claim"})) + .unwrap(), + ) + .unwrap(); + assert_eq!(persisted["state"], failed["state"]); + assert_eq!(persisted["effect_authorized"], false); + } +} +#[test] +fn input_validation_limits_and_intent_persistence_precede_native_commit() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + let before = fs::read(f.root.join("commands")).unwrap(); + let mut invalid = request(&id, 1, 1, "bad"); + invalid["changes"] + .as_array_mut() + .unwrap() + .push(json!({"op":"insert","predicate":"echo","values":[2,"derived"]})); + assert_eq!(admit(&mut manager, invalid)["state"], "not_applied"); + assert_eq!(fs::read(f.root.join("commands")).unwrap(), before); + let queries = vec![json!({"predicate":"source","max_rows":100}); 11]; + assert!(manager + .read_batch( + serde_json::from_value( + json!({"id":id,"expected_generation":1,"expected_revision":1,"queries":queries}) + ) + .unwrap() + ) + .is_err()); + // Block atomic replacement of world.json, but not status: a clean status + // performs no write. No native mutation may precede durable intent. + let path = f.root.join("worlds").join(&id).join("world.json"); + fs::remove_file(&path).unwrap(); + fs::create_dir(&path).unwrap(); + let failed = admit(&mut manager, request(&id, 1, 1, "intent-failure")); + assert_eq!(failed["state"], "not_applied"); + assert_eq!(fs::read(f.root.join("commands")).unwrap(), before); + fs::remove_dir(path).unwrap(); +} +#[test] +fn input_page_byte_limit_and_crashed_intents_are_honest() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + admit(&mut manager, request(&id, 1, 1, "one")); + let page = manager + .execute( + &id, + "query_rows", + &json!({"predicate":"source","max_bytes":64}), + ) + .unwrap(); + assert_eq!(page["rows"].as_array().unwrap().len(), 1); + assert!(manager + .execute( + &id, + "query_rows", + &json!({"predicate":"source","max_bytes":2}) + ) + .is_err()); + drop(manager); + let path = f.root.join("worlds").join(&id).join("world.json"); + let mut record: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + record["admission"]["records"]["1/one"]["state"] = json!("pending"); + fs::write(path, serde_json::to_vec(&record).unwrap()).unwrap(); + let mut manager = f.manager(); + let result = manager + .admission_status( + serde_json::from_value(json!({"id":id,"generation":1,"admission_key":"one"})).unwrap(), + ) + .unwrap(); + assert_eq!(result["state"], "uncertain"); + assert_eq!(result["receipt"], Value::Null); +} diff --git a/tests/worlds/persistence.rs b/tests/worlds/persistence.rs new file mode 100644 index 0000000..a355a6c --- /dev/null +++ b/tests/worlds/persistence.rs @@ -0,0 +1,516 @@ +//! Managed persistence contract under a simulated transport, not native DDLog proof. +use super::*; +use sha2::{Digest, Sha256}; +use std::path::Path; + +fn change(manager: &mut WorldManager, id: &str, op: &str, predicate: &str, values: Value) { + manager + .execute( + id, + "apply_changes", + &json!({"changes":[{"op":op,"predicate":predicate,"values":values}]}), + ) + .unwrap(); +} +fn rows(manager: &mut WorldManager, id: &str, predicate: &str) -> Value { + manager + .execute(id, "query_rows", &json!({"predicate":predicate})) + .unwrap()["rows"] + .clone() +} +fn receipt_dir(f: &Fixture, receipt: &Value) -> PathBuf { + f.root + .join("worlds") + .join(receipt["origin"]["world_id"].as_str().unwrap()) + .join("checkpoints") + .join(receipt["receipt_id"].as_str().unwrap()) +} +fn read_json(path: &Path) -> Value { + serde_json::from_slice(&fs::read(path).unwrap()).unwrap() +} +fn write_json(path: &Path, value: &Value) { + fs::write(path, serde_json::to_vec(value).unwrap()).unwrap(); +} +fn restore(manager: &mut WorldManager, id: &str, receipt: &Value) -> Value { + manager.restore_async(id, receipt).unwrap(); + let status = wait_until(manager, id, |s| s["state"] != "starting"); + assert_eq!(status["state"], "running", "{status}"); + status +} + +#[test] +fn published_boundary_survives_restart_and_restore_is_explicit() { + let f = Fixture::new(); + let mut manager = f.manager(); + let mut def = echo_definition(); + def["schemas"]["unused"] = json!({"input":true,"fields":["string"]}); + def["interface"] = json!({"inputs":["source","unused"],"outputs":["echo"]}); + let record = manager + .registry() + .unwrap() + .create(serde_json::from_value(def).unwrap(), None) + .unwrap(); + let def = WorldDefinition { + label: "checkpoint".into(), + processor: ProcessorReference { + processor_id: record.processor_id, + version: record.version, + }, + purpose: "instance".into(), + scenarios: vec![], + }; + let id = manager.create(def.clone()).unwrap(); + assert!(manager.checkpoint(&id).is_err()); + manager.start(&id).unwrap(); + change( + &mut manager, + &id, + "insert", + "source", + json!([-7, "saved\n\"é"]), + ); + let receipt = manager.checkpoint(&id).unwrap(); + assert_eq!(receipt["origin"]["revision"], 2); + assert_eq!(receipt["program"]["lowering_version"], 2); + assert_eq!( + receipt["origin"]["build"]["native_sha256"] + .as_str() + .unwrap() + .len(), + 64 + ); + let object = read_json(&receipt_dir(&f, &receipt).join("checkpoint.json")); + assert_eq!(object["state"]["inputs"]["unused"], json!([])); + change( + &mut manager, + &id, + "insert", + "source", + json!([9, "unpublished"]), + ); + assert!(manager.restore_async(&id, &receipt).is_err()); + let status = manager.status(&id).unwrap(); + assert_eq!(status["persistence"]["schema_version"], 1); + assert_eq!(status["persistence"]["committed_revision"], 3); + assert_eq!(status["persistence"]["persisted_revision"], 2); + assert_eq!(status["persistence"]["checkpoints"][0]["receipt"], receipt); + // Model an abrupt owner exit with its last durable running record. No + // process is adopted when that record and receipt are reopened. + let record_path = f.root.join("worlds").join(&id).join("world.json"); + let running_record = fs::read(&record_path).unwrap(); + drop(manager); + fs::write(record_path, running_record).unwrap(); + let mut manager = f.manager(); + let status = manager.status(&id).unwrap(); + assert_eq!(status["state"], "interrupted"); + assert_eq!(status["persistence"]["checkpoints"][0]["receipt"], receipt); + assert_eq!(status["resources"]["pid"], Value::Null); + let status = restore(&mut manager, &id, &receipt); + assert_eq!(status["generation"], 2); + assert_eq!(status["revision"], 2); + assert_eq!(status["persistence"]["restored_from"], receipt); + assert_eq!(status["instance"]["processor"], json!(def.processor)); + assert_eq!(rows(&mut manager, &id, "echo"), json!([[-7, "saved\n\"é"]])); + assert_eq!(rows(&mut manager, &id, "unused"), json!([])); + change( + &mut manager, + &id, + "delete", + "source", + json!([-7, "saved\n\"é"]), + ); + change(&mut manager, &id, "insert", "source", json!([8, "new"])); + assert_eq!(rows(&mut manager, &id, "echo"), json!([[8, "new"]])); + // A created world with the same pin can explicitly use the original receipt. + let target = manager.create(def).unwrap(); + assert_eq!(restore(&mut manager, &target, &receipt)["generation"], 1); + assert_eq!( + rows(&mut manager, &target, "source"), + json!([[-7, "saved\n\"é"]]) + ); + manager.stop(&id).unwrap(); + let fresh = manager.start(&id).unwrap(); + assert_eq!(fresh["generation"], 3); + assert_eq!(fresh["revision"], 1); + assert_eq!(fresh["persistence"]["restored_from"], Value::Null); + assert_eq!(fresh["persistence"]["persisted_revision"], Value::Null); + assert_eq!(rows(&mut manager, &id, "source"), json!([])); + manager.stop(&id).unwrap(); + manager.stop(&target).unwrap(); + drop(manager); + let mut manager = f.manager(); + let status = manager.status(&id).unwrap(); + assert!(status["history"] + .as_array() + .unwrap() + .iter() + .any(|h| h["restored_from"] == receipt)); + assert_eq!(status["persistence"]["checkpoints"][0]["receipt"], receipt); +} + +#[test] +fn receipt_tampering_missing_objects_and_paths_fail_closed() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + let receipt = manager.checkpoint(&id).unwrap(); + manager.stop(&id).unwrap(); + for (pointer, value) in [ + ("/receipt_id", json!("../../world.json")), + ("/receipt_id", json!("/tmp/checkpoint.json")), + ("/origin/world_id", json!("../outside")), + ("/origin/revision", json!(99)), + ("/origin/build/native_sha256", json!("tampered")), + ("/program/lowering_version", json!(1)), + ("/program/public_relations/0/physical", json!("private")), + ( + "/program/dependencies", + json!({"fake":{"processor_id":"fake","version":"fake"}}), + ), + ("/checkpoint_sha256", json!("0".repeat(64))), + ] { + let mut changed = receipt.clone(); + *changed.pointer_mut(pointer).unwrap() = value; + assert!(manager.restore_async(&id, &changed).is_err(), "{pointer}"); + assert_eq!(manager.status(&id).unwrap()["generation"], 1); + } + assert!(manager + .restore_async(&id, &json!("/tmp/checkpoint.json")) + .is_err()); + let dir = receipt_dir(&f, &receipt); + let path = dir.join("checkpoint.json"); + let bytes = fs::read(&path).unwrap(); + fs::write(&path, b"corrupt").unwrap(); + assert!(manager.restore_async(&id, &receipt).is_err()); + fs::remove_file(&path).unwrap(); + assert_eq!( + manager.status(&id).unwrap()["persistence"]["checkpoints"][0]["availability"], + "missing_or_invalid" + ); + assert!(manager.restore_async(&id, &receipt).is_err()); + let outside = f.root.join("outside.json"); + fs::write(&outside, &bytes).unwrap(); + std::os::unix::fs::symlink(&outside, &path).unwrap(); + assert!(manager.restore_async(&id, &receipt).is_err()); + fs::remove_file(&path).unwrap(); + fs::write(&path, bytes).unwrap(); + let other = definition(&manager); + let other = manager.create(other).unwrap(); + assert!(manager + .restore_async(&other, &receipt) + .unwrap_err() + .contains("pin")); + let mut manifest = read_json(&dir.join("publication.json")); + manifest["status"] = json!("staged"); + write_json(&dir.join("publication.json"), &manifest); + assert!(manager.restore_async(&id, &receipt).is_err()); + drop(manager); + let mut manager = f.manager(); + assert_eq!( + manager.status(&id).unwrap()["persistence"]["checkpoints"][0]["status"], + "staged" + ); + assert!(manager.restore_async(&id, &receipt).is_err()); +} + +#[test] +fn forged_lowered_source_is_rejected_before_compilation_even_with_valid_digest() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + let mut receipt = manager.checkpoint(&id).unwrap(); + manager.stop(&id).unwrap(); + let dir = receipt_dir(&f, &receipt); + let path = dir.join("checkpoint.json"); + let mut checkpoint = read_json(&path); + let source = checkpoint["state"]["source"].as_str().unwrap().to_owned(); + checkpoint["state"]["source"] = json!(format!("{source}\n// changed source\n")); + checkpoint["sha256"] = json!(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&checkpoint["state"]).unwrap()) + )); + receipt["checkpoint_sha256"] = checkpoint["sha256"].clone(); + write_json(&path, &checkpoint); + let mut publication = read_json(&dir.join("publication.json")); + publication["receipt"] = receipt.clone(); + write_json(&dir.join("publication.json"), &publication); + manager.restore_async(&id, &receipt).unwrap(); + let status = wait_until(&mut manager, &id, |s| s["state"] != "starting"); + assert_eq!(status["state"], "failed"); + assert!(status["error"].as_str().unwrap().contains("pinned")); + assert!(!f.root.join("worlds").join(id).join("2/build-1").exists()); + assert_eq!(status["persistence"]["restored_from"], Value::Null); +} + +#[test] +fn composition_restore_retains_public_mapping_exact_dependencies_and_admission() { + let f = Fixture::new(); + let mut manager = f.manager(); + let mut leaf = echo_definition(); + leaf["interface"] = json!({"inputs":["source"],"outputs":["echo"]}); + let leaf = manager + .registry() + .unwrap() + .create(serde_json::from_value(leaf).unwrap(), None) + .unwrap(); + let leaf = ProcessorReference { + processor_id: leaf.processor_id, + version: leaf.version, + }; + let composition: ProcessorDefinition = serde_json::from_value(json!({"composition":{ + "nodes":{"child":leaf},"inputs":{"in":{"fields":["int","string"],"targets":[{"node":"child","relation":"source"}]}}, + "bindings":[],"outputs":{"out":{"node":"child","relation":"echo"}}}})).unwrap(); + let record = manager + .registry() + .unwrap() + .create(composition, None) + .unwrap(); + let id = manager + .create(WorldDefinition { + label: "composition".into(), + processor: ProcessorReference { + processor_id: record.processor_id, + version: record.version, + }, + purpose: "instance".into(), + scenarios: vec![], + }) + .unwrap(); + manager.start(&id).unwrap(); + change(&mut manager, &id, "insert", "in", json!([1, "retained"])); + let receipt = manager.checkpoint(&id).unwrap(); + assert!(!receipt["program"]["dependencies"] + .as_object() + .unwrap() + .is_empty()); + manager.stop(&id).unwrap(); + // Current-version changes do not replace the dependency's exact version. + let mut next = echo_definition(); + next["interface"] = json!({"inputs":["source"],"outputs":["echo"]}); + next["rules"] = json!("echo(N,S) :- source(N,S). echo(N,S) :- source(N,S), N > 0."); + manager + .registry() + .unwrap() + .publish( + &leaf.processor_id, + serde_json::from_value(next).unwrap(), + &leaf.version, + None, + ) + .unwrap(); + let status = restore(&mut manager, &id, &receipt); + assert_eq!(status["instance"]["composition"]["lowering_version"], 2); + assert_eq!(rows(&mut manager, &id, "in"), json!([[1, "retained"]])); + assert!(manager + .execute(&id, "query_rows", &json!({"predicate":"Module0_source"})) + .is_err()); + assert!(manager + .execute( + &id, + "apply_changes", + &json!({"changes":[{"op":"insert","predicate":"Input_in","values":[2,"private"]}]}) + ) + .is_err()); + change(&mut manager, &id, "delete", "in", json!([1, "retained"])); + assert_eq!(rows(&mut manager, &id, "in"), json!([])); + change(&mut manager, &id, "insert", "in", json!([2, "new"])); + assert_eq!(rows(&mut manager, &id, "in"), json!([[2, "new"]])); + manager.stop(&id).unwrap(); + let path = f + .root + .join("registry") + .join(&leaf.processor_id) + .join("versions") + .join(format!( + "{}.json", + leaf.version.strip_prefix("sha256:").unwrap() + )); + fs::remove_file(path).unwrap(); + assert!(manager.restore_async(&id, &receipt).is_err()); + assert_eq!(manager.status(&id).unwrap()["generation"], 2); +} + +#[test] +fn failed_publication_preserves_prior_receipt_and_live_state() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + let receipt = manager.checkpoint(&id).unwrap(); + change( + &mut manager, + &id, + "insert", + "source", + json!([1, "unpublished"]), + ); + let checkpoints = f.root.join("worlds").join(&id).join("checkpoints"); + let retained = f.root.join("retained"); + fs::rename(&checkpoints, &retained).unwrap(); + fs::write(&checkpoints, "blocked").unwrap(); + assert!(manager.checkpoint(&id).is_err()); + assert_eq!( + rows(&mut manager, &id, "source"), + json!([[1, "unpublished"]]) + ); + fs::remove_file(&checkpoints).unwrap(); + fs::rename(retained, &checkpoints).unwrap(); + assert_eq!( + manager.status(&id).unwrap()["persistence"]["checkpoints"][0]["receipt"], + receipt + ); + // A failure after receipt publication must not lose the durable receipt. + let temp = f.root.join("worlds").join(&id).join("world.json.tmp"); + fs::create_dir(&temp).unwrap(); + assert!(manager.checkpoint(&id).is_err()); + fs::remove_dir(temp).unwrap(); + drop(manager); + let mut manager = f.manager(); + let status = manager.status(&id).unwrap(); + let checkpoints = status["persistence"]["checkpoints"].as_array().unwrap(); + assert_eq!(checkpoints.len(), 2); + assert_eq!(checkpoints[1]["status"], "published"); + restore(&mut manager, &id, &checkpoints[1]["receipt"]); + assert_eq!( + rows(&mut manager, &id, "source"), + json!([[1, "unpublished"]]) + ); +} + +#[test] +fn restore_build_and_replay_failures_are_retryable_explicitly() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + let receipt = manager.checkpoint(&id).unwrap(); + manager.stop(&id).unwrap(); + for flag in ["reject_build", "fail_replay"] { + fs::write(f.root.join(flag), "").unwrap(); + manager.restore_async(&id, &receipt).unwrap(); + let failed = wait_until(&mut manager, &id, |s| s["state"] != "starting"); + assert_eq!(failed["state"], "failed"); + assert_eq!(failed["persistence"]["restored_from"], Value::Null); + assert_eq!(failed["managed_processes"], json!([])); + fs::remove_file(f.root.join(flag)).unwrap(); + } + assert_eq!(restore(&mut manager, &id, &receipt)["generation"], 4); +} + +#[test] +fn restore_retains_hosted_compiler_cancellation_and_observer_options() { + let f = Fixture::new(); + let driver = f.root.join("build.py"); + let mut script = fs::read_to_string(&driver).unwrap(); + script = script.replace("import sys", "import sys, os, time, subprocess"); + script = script.replace("if (root/'reject_build')", "if (root/'hold_build').exists():\n child=subprocess.Popen([sys.executable,'-c','import time; time.sleep(3600)'])\n (root/'compiler.pending').write_text(__import__('json').dumps({'pid':os.getpid(),'child':child.pid,'pgid':os.getpgrp()}))\n os.replace(root/'compiler.pending', root/'compiler.json')\n while True: time.sleep(0.01)\nif (root/'reject_build')"); + let injected = "(control/'observer.json').write_text(json.dumps({'detail':os.environ.get('DDLOG_OBSERVER_DETAIL'),'rotate':os.environ.get('DDLOG_OBSERVER_ROTATE'),'file':os.environ.get('DDLOG_OBSERVER_FILE'),'pid':os.getpid(),'pgid':os.getpgrp()}))\nfacts, staged = {}, {}"; + script = script.replace("Path(sys.argv[2]).write_text(source)", &format!("source=source.replace('facts, staged = {{}}, {{}}', {})\nPath(sys.argv[2]).write_text(source)", json!(injected))); + fs::write(&driver, script).unwrap(); + let mut manager = f.manager(); + let def = definition(&manager); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + let receipt = manager.checkpoint(&id).unwrap(); + manager.stop(&id).unwrap(); + fs::write(f.root.join("hold_build"), "").unwrap(); + let beginning = std::time::Instant::now(); + manager.restore_async(&id, &receipt).unwrap(); + assert!(beginning.elapsed() < std::time::Duration::from_secs(2)); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !f.root.join("compiler.json").exists() { + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + let compiler = read_json(&f.root.join("compiler.json")); + assert_eq!(compiler["pid"], compiler["pgid"]); + assert!(manager.restore_async(&id, &receipt).is_err()); + manager.stop(&id).unwrap(); + let stopped = wait_until(&mut manager, &id, |s| s["state"] == "stopped"); + assert_eq!(stopped["managed_processes"], json!([])); + assert_eq!(stopped["persistence"]["restored_from"], Value::Null); + assert_eq!( + unsafe { libc::kill(compiler["pid"].as_i64().unwrap() as i32, 0) }, + -1 + ); + let child = compiler["child"].as_i64().unwrap() as i32; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while unsafe { libc::kill(child, 0) } == 0 { + assert!( + std::time::Instant::now() < deadline, + "compiler descendant survived cancellation" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + fs::remove_file(f.root.join("hold_build")).unwrap(); + let status = restore(&mut manager, &id, &receipt); + let observed = read_json(&f.root.join("observer.json")); + assert_eq!(observed["detail"], "full"); + assert_eq!(observed["rotate"], "1"); + assert_eq!(observed["pid"], observed["pgid"]); + assert_eq!(observed["pid"], status["resources"]["pid"]); + assert_eq!( + observed["file"], + json!(f + .root + .join("worlds") + .join(&id) + .join("3/native-events.jsonl")) + ); + assert_eq!(status["managed_processes"][0]["role"], "native"); + let pid = observed["pid"].as_i64().unwrap() as i32; + manager.shutdown_handle().stop_all(); + wait_until(&mut manager, &id, |s| s["state"] == "failed"); + assert_eq!(unsafe { libc::kill(pid, 0) }, -1); +} + +#[test] +fn old_world_records_default_to_empty_persistence_and_unsupported_imports_fail_explicitly() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let id = manager.create(def).unwrap(); + drop(manager); + let path = f.root.join("worlds").join(&id).join("world.json"); + let mut old = read_json(&path); + old.as_object_mut().unwrap().remove("persistence"); + write_json(&path, &old); + let mut manager = f.manager(); + let status = manager.status(&id).unwrap(); + assert_eq!(status["state"], "created"); + assert_eq!(status["persistence"]["checkpoints"], json!([])); + let star = manager.registry().unwrap().create(serde_json::from_value(json!({ + "rules":"", "schemas":{ + "vertices":{"input":true,"fields":["int"]}, + "edges":{"input":true,"fields":["int","int"]}, + "labels":{"input":false,"fields":["int","int"]}}, + "operators":[{"type":"large_small_star","vertices":"vertices","edges":"edges","output":"labels"}], + "interface":{"inputs":["vertices","edges"],"outputs":["labels"]} + })).unwrap(),None).unwrap(); + let id = manager + .create(WorldDefinition { + label: "unsupported checkpoint".into(), + processor: ProcessorReference { + processor_id: star.processor_id, + version: star.version, + }, + purpose: "instance".into(), + scenarios: vec![], + }) + .unwrap(); + manager.start(&id).unwrap(); + assert!(manager + .checkpoint(&id) + .unwrap_err() + .contains("imported native operators")); + let status = manager.status(&id).unwrap(); + assert_eq!(status["state"], "running"); + assert_eq!(status["persistence"]["status"], "error"); + assert_eq!(status["persistence"]["checkpoints"], json!([])); +} diff --git a/tests/worlds/workers.rs b/tests/worlds/workers.rs new file mode 100644 index 0000000..456ce68 --- /dev/null +++ b/tests/worlds/workers.rs @@ -0,0 +1,273 @@ +//! Process ownership, profile validation and cancellation without any provider. +use super::*; +fn profiles(f: &Fixture, pin: &ProcessorReference, timeout: u64) -> PathBuf { + let path = f.root.join("profiles.json"); + fs::write(&path,serde_json::to_vec(&json!({"schema_version":1,"profiles":{"fixture":{ + "argv":["/usr/bin/env","python3",PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/managed_worker.py")], + "env":{"FIXTURE_ROOT":f.root},"config_ref":f.root.join("configuration.json"), + "allowed_processors":[pin],"max_concurrent_workers":1,"timeout_ms":timeout + }}})).unwrap()).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + path +} +fn start(manager: &mut WorldManager, id: &str, key: &str, mode: &str) -> Result { + manager.worker_start( + serde_json::from_value( + json!({"id":id,"expected_generation":1,"profile":"fixture","key":key, + "payload":{"mode":mode,"private":"not in status"}}), + ) + .unwrap(), + ) +} +fn wait(manager: &mut WorldManager, id: &str, worker: &Value) -> Value { + let status = wait_until(manager, id, |s| { + s["workers"].as_array().unwrap().iter().any(|w| { + w["worker_id"] == worker["worker_id"] + && !matches!( + w["state"].as_str().unwrap(), + "running" | "starting" | "stopping" + ) + }) + }); + status["workers"] + .as_array() + .unwrap() + .iter() + .find(|w| w["worker_id"] == worker["worker_id"]) + .unwrap() + .clone() +} +fn wait_file(path: &std::path::Path) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !path.exists() { + assert!(std::time::Instant::now() < deadline, "{}", path.display()); + std::thread::sleep(std::time::Duration::from_millis(10)); + } +} +#[test] +fn startup_profiles_exact_pins_keys_and_safe_retained_outcomes() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let path = profiles(&f, &def.processor, 5000); + manager + .configure_workers(&path, f.root.join("owner.json")) + .unwrap(); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + assert!(manager + .configure_workers(&path, f.root.join("owner.json")) + .is_err()); + let worker = start(&mut manager, &id, "run", "complete").unwrap(); + let pid = worker["pid"].as_u64().unwrap() as i32; + let complete = wait(&mut manager, &id, &worker); + assert_eq!(complete["state"], "completed"); + assert_eq!(complete["exit_code"], 0); + assert_eq!(complete["outcome"]["code"], "fixture_ok"); + assert_eq!(unsafe { libc::kill(pid, 0) }, -1); + let text = complete.to_string(); + assert!(!text.contains("not in status")); + assert!(!text.contains("configuration.json")); + assert!(!text.contains("argv")); + let envelope: Value = + serde_json::from_slice(&fs::read(f.root.join("run.launch.json")).unwrap()).unwrap(); + assert_eq!(envelope["worker_id"], worker["worker_id"]); + assert_eq!( + envelope["owner_descriptor"], + json!(f.root.join("owner.json")) + ); + assert_eq!( + start(&mut manager, &id, "run", "complete").unwrap()["replayed"], + true + ); + assert!(start(&mut manager, &id, "run", "hold").is_err()); + let wrong = definition(&manager); + let wrong = manager.create(wrong).unwrap(); + manager.start(&wrong).unwrap(); + assert!(start(&mut manager, &wrong, "wrong-pin", "complete") + .unwrap_err() + .contains("pin")); + drop(manager); + let mut manager = f.manager(); + let status = manager + .worker_status( + serde_json::from_value(json!({"id":id,"worker_id":worker["worker_id"]})).unwrap(), + ) + .unwrap(); + assert_eq!(status["workers"][0]["state"], "completed"); + assert_eq!(status["workers"][0]["pid"], pid); +} +#[test] +fn cancellation_capacity_and_world_shutdown_reap_worker_descendants() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let path = profiles(&f, &def.processor, 30000); + manager + .configure_workers(&path, f.root.join("owner.json")) + .unwrap(); + let id = manager.create(def.clone()).unwrap(); + manager.start(&id).unwrap(); + let other = manager.create(def).unwrap(); + manager.start(&other).unwrap(); + let worker = start(&mut manager, &id, "run", "descendant").unwrap(); + wait_file(&f.root.join("run.child")); + let child = fs::read_to_string(f.root.join("run.child")) + .unwrap() + .parse::() + .unwrap(); + assert!(start(&mut manager, &other, "other", "hold") + .unwrap_err() + .contains("concurrency")); + let status = manager.status(&id).unwrap(); + assert!(status["managed_processes"] + .as_array() + .unwrap() + .iter() + .any(|p| p["role"] == "worker" && p["sample"]["pid"] == worker["pid"])); + manager + .worker_stop( + serde_json::from_value( + json!({"id":id,"expected_generation":1,"worker_id":worker["worker_id"]}), + ) + .unwrap(), + ) + .unwrap(); + let late = manager + .admit_inputs( + serde_json::from_value( + json!({"id":id,"expected_generation":1,"expected_revision":1, + "admission_key":"late","worker_id":worker["worker_id"],"changes":[]}), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(late["state"], "not_applied"); + assert_eq!(wait(&mut manager, &id, &worker)["state"], "stopped"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while unsafe { libc::kill(child, 0) } == 0 { + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + assert_eq!( + manager.status(&id).unwrap()["state"], + "running", + "worker_stop must not kill native" + ); + let worker = start(&mut manager, &id, "stop-world", "hold").unwrap(); + manager.stop(&id).unwrap(); + assert_eq!(wait(&mut manager, &id, &worker)["state"], "stopped"); + let worker = start(&mut manager, &other, "owner-exit", "hold").unwrap(); + drop(manager); + assert_eq!( + unsafe { libc::kill(worker["pid"].as_u64().unwrap() as i32, 0) }, + -1 + ); + let mut manager = f.manager(); + assert_eq!( + manager.status(&other).unwrap()["workers"][0]["state"], + "stopped" + ); +} +#[test] +fn malformed_oversized_nonzero_and_timeout_outcomes_fail_boundedly() { + for (mode, timeout) in [ + ("invalid", 5000), + ("oversized", 5000), + ("exit", 5000), + ("hold", 200), + ] { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let path = profiles(&f, &def.processor, timeout); + manager + .configure_workers(&path, f.root.join("owner.json")) + .unwrap(); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + let worker = start(&mut manager, &id, "run", mode).unwrap(); + let failed = wait(&mut manager, &id, &worker); + assert_eq!(failed["state"], "failed"); + assert!(!failed.to_string().contains("never expose")); + assert_eq!( + start(&mut manager, &id, "run", mode).unwrap()["replayed"], + true + ); + assert_eq!( + unsafe { libc::kill(worker["pid"].as_u64().unwrap() as i32, 0) }, + -1 + ); + } +} +#[test] +fn nonzero_exit_retains_only_valid_failure_outcomes_across_restart() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let path = profiles(&f, &def.processor, 5000); + manager + .configure_workers(&path, f.root.join("owner.json")) + .unwrap(); + let id = manager.create(def).unwrap(); + manager.start(&id).unwrap(); + let mut expected = Vec::new(); + for mode in [ + "failed_exit", + "completed_exit", + "invalid_exit", + "unknown_exit", + ] { + let worker = start(&mut manager, &id, mode, mode).unwrap(); + let failed = wait(&mut manager, &id, &worker); + assert_eq!(failed["state"], "failed"); + assert_eq!(failed["exit_code"], 9); + if mode == "failed_exit" { + assert_eq!( + failed["outcome"], + json!({"schema_version":1,"status":"failed","code":"configuration_invalid"}) + ); + } else { + assert!(failed["outcome"].is_null()); + } + assert!(!failed.to_string().contains("never expose")); + assert_eq!( + start(&mut manager, &id, mode, mode).unwrap()["replayed"], + true + ); + expected.push(failed["outcome"].clone()); + } + drop(manager); + let mut manager = f.manager(); + let status = manager.status(&id).unwrap(); + for (worker, outcome) in status["workers"].as_array().unwrap().iter().zip(expected) { + assert_eq!(worker["state"], "failed"); + assert_eq!(worker["outcome"], outcome); + assert_eq!(worker["exit_code"], 9); + } +} + +#[test] +fn unsafe_or_mutable_profiles_and_unknown_launch_fields_are_rejected() { + let f = Fixture::new(); + let mut manager = f.manager(); + let def = definition(&manager); + let path = profiles(&f, &def.processor, 5000); + fs::set_permissions(&path, fs::Permissions::from_mode(0o666)).unwrap(); + assert!(manager + .configure_workers(&path, f.root.join("owner.json")) + .is_err()); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + let link = f.root.join("symlink.json"); + std::os::unix::fs::symlink(&path, &link).unwrap(); + assert!(manager + .configure_workers(&link, f.root.join("owner.json")) + .is_err()); + let mut value: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + value["profiles"]["fixture"]["argv"][0] = json!("relative"); + fs::write(&path, serde_json::to_vec(&value).unwrap()).unwrap(); + assert!(manager + .configure_workers(&path, f.root.join("owner.json")) + .is_err()); + assert!(serde_json::from_value::(json!({"id":"world","expected_generation":1,"profile":"fixture","key":"run","payload":{},"argv":["/bin/sh"]})).is_err()); +} diff --git a/tests/worlds_native.rs b/tests/worlds_native.rs index ba7f5f1..a36b99d 100644 --- a/tests/worlds_native.rs +++ b/tests/worlds_native.rs @@ -336,3 +336,182 @@ fn registered_worlds_expose_native_graph_and_metadata_then_stop() { drop(recovered); std::fs::remove_dir_all(root).unwrap(); } + +#[test] +#[ignore = "requires DDLOG_RUNTIME_NATIVE_BUILD and its operator-configured native toolchain"] +fn managed_json_restore_recomputes_public_state_and_preserves_process_capture() { + let root = std::env::temp_dir().join(format!( + "world-checkpoint-native-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir(&root).unwrap(); + let driver: std::path::PathBuf = std::env::var_os("DDLOG_RUNTIME_NATIVE_BUILD") + .expect("Configure native build driver") + .into(); + let mut manager = + WorldManager::new(root.join("registry"), root.join("worlds"), driver.clone()).unwrap(); + let leaf = manager + .registry() + .unwrap() + .create( + serde_json::from_value(json!({ + "rules":"reach(X,Y) :- edge(X,Y). reach(X,Z) :- reach(X,Y), edge(Y,Z).", + "schemas":{ + "edge":{"input":true,"fields":["int","int"]}, + "unused":{"input":true,"fields":["string"]}, + "reach":{"input":false,"fields":["int","int"]}}, + "interface":{"inputs":["edge","unused"],"outputs":["reach"]} + })) + .unwrap(), + None, + ) + .unwrap(); + let composed = manager.registry().unwrap().create(serde_json::from_value(json!({"composition":{ + "nodes":{"graph":{"processor_id":leaf.processor_id,"version":leaf.version}}, + "inputs":{ + "edge":{"fields":["int","int"],"targets":[{"node":"graph","relation":"edge"}]}, + "unused":{"fields":["string"],"targets":[{"node":"graph","relation":"unused"}]}}, + "bindings":[],"outputs":{"reach":{"node":"graph","relation":"reach"}}} + })).unwrap(),None).unwrap(); + let mut evidence = Vec::new(); + for record in [leaf, composed] { + let id = manager + .create(world( + "Native managed recovery", + &serde_json::to_value(&record).unwrap(), + )) + .unwrap(); + let start = manager.start(&id).unwrap(); + let original_pid = start["resources"]["pid"].as_u64().unwrap() as i32; + manager + .execute( + &id, + "apply_changes", + &json!({"changes":[ + {"op":"insert","predicate":"edge","values":[1,2]}, + {"op":"insert","predicate":"edge","values":[2,3]}]}), + ) + .unwrap(); + let expected = manager + .execute(&id, "query_rows", &json!({"predicate":"reach"})) + .unwrap()["rows"] + .clone(); + assert_eq!(expected.as_array().unwrap().len(), 3); + let receipt = manager.checkpoint(&id).unwrap(); + manager + .execute( + &id, + "apply_changes", + &json!({"changes":[{"op":"insert","predicate":"edge","values":[3,4]}]}), + ) + .unwrap(); + assert_eq!( + manager + .execute(&id, "query_rows", &json!({"predicate":"reach"})) + .unwrap()["total"], + 6 + ); + manager.stop(&id).unwrap(); + assert_eq!(unsafe { libc::kill(original_pid, 0) }, -1); + drop(manager); + manager = + WorldManager::new(root.join("registry"), root.join("worlds"), driver.clone()).unwrap(); + assert_eq!( + manager.status(&id).unwrap()["persistence"]["checkpoints"][0]["receipt"], + receipt + ); + let admission = manager.restore_async(&id, &receipt).unwrap(); + assert_eq!(admission["state"], "starting"); + let restored = wait_until(&mut manager, &id, "restored native instance", |s| { + s["state"] != "starting" + }); + assert_eq!(restored["state"], "running", "{restored}"); + assert_eq!(restored["generation"], 2); + assert_eq!(restored["revision"], 2); + assert_eq!(restored["persistence"]["restored_from"], receipt); + assert_eq!( + restored["instance"]["source_sha256"], + receipt["program"]["source_sha256"] + ); + let pid = restored["resources"]["pid"].as_u64().unwrap() as i32; + assert_ne!(pid, original_pid); + assert_eq!(unsafe { libc::getpgid(pid) }, pid); + assert_eq!(restored["managed_processes"][0]["role"], "native"); + assert_eq!( + manager + .execute(&id, "query_rows", &json!({"predicate":"reach"})) + .unwrap()["rows"], + expected + ); + assert_eq!( + manager + .execute(&id, "query_rows", &json!({"predicate":"unused"})) + .unwrap()["rows"], + json!([]) + ); + assert!(manager + .execute(&id, "query_rows", &json!({"predicate":"Module0_reach"})) + .is_err()); + manager + .execute( + &id, + "apply_changes", + &json!({"changes":[ + {"op":"delete","predicate":"edge","values":[2,3]}, + {"op":"insert","predicate":"edge","values":[2,5]}]}), + ) + .unwrap(); + let mut after = manager + .execute(&id, "query_rows", &json!({"predicate":"reach"})) + .unwrap()["rows"] + .as_array() + .unwrap() + .clone(); + after.sort_by_key(Value::to_string); + assert_eq!( + after, + json!([[1, 2], [1, 5], [2, 5]]).as_array().unwrap().clone() + ); + let captured = wait_until(&mut manager, &id, "restored generation capture", |s| { + s["inspection"]["state"] == "available" + && s["inspection"]["activity"]["totals"]["timely"] + .as_u64() + .unwrap_or(0) + > 0 + }); + assert!(root + .join("worlds") + .join(&id) + .join("2/native-events.jsonl") + .is_file()); + evidence.push(json!({"receipt":receipt,"restored_instance":restored["instance"], + "rows_before":expected,"rows_after":after,"capture_state":captured["inspection"]["state"], + "provider_calls":0})); + manager.stop(&id).unwrap(); + assert_eq!(unsafe { libc::kill(pid, 0) }, -1); + let fresh = manager.start(&id).unwrap(); + assert_eq!(fresh["revision"], 1); + assert_eq!(fresh["persistence"]["restored_from"], Value::Null); + assert_eq!( + manager + .execute(&id, "query_rows", &json!({"predicate":"reach"})) + .unwrap()["rows"], + json!([]) + ); + manager.stop(&id).unwrap(); + } + drop(manager); + if let Some(path) = std::env::var_os("DDLOG_RUNTIME_NATIVE_EVIDENCE") { + std::fs::write( + path, + serde_json::to_vec_pretty(&json!({"mode":"fresh native compilation","cases":evidence})) + .unwrap(), + ) + .unwrap(); + } + std::fs::remove_dir_all(root).unwrap(); +} diff --git a/tests/worlds_stdio.rs b/tests/worlds_stdio.rs index f2d9413..b1614ee 100644 --- a/tests/worlds_stdio.rs +++ b/tests/worlds_stdio.rs @@ -162,7 +162,8 @@ fn stdio_protocol_covers_every_control_plane_verb() { let created = owner.call("create", json!({"label":"Echo world","processor":pin})); let id = created["id"].as_str().unwrap().to_string(); assert_eq!(created["state"], "created"); - assert_eq!(created["persistence"]["status"], "not_configured"); + assert_eq!(created["persistence"]["schema_version"], 1); + assert_eq!(created["persistence"]["status"], "configured"); assert_eq!(owner.call("start", json!({"id":id}))["state"], "starting"); let running = owner.wait(&id); assert_eq!(running["state"], "running"); @@ -245,3 +246,40 @@ fn stdio_protocol_covers_every_control_plane_verb() { let status = owner.child.wait().unwrap(); assert!(status.success()); } + +#[test] +fn checkpoint_and_explicit_restore_protocol_preserves_receipt_and_revision() { + let mut owner = Owner::spawn(); + let record = owner.call( + "register", + json!({"name":"Checkpoint echo", "definition":{ + "rules":"echo(N,S) :- source(N,S).", "schemas":{ + "source":{"input":true,"fields":["int","string"]}, + "echo":{"input":false,"fields":["int","string"]}}}}), + ); + let created = owner.call("create",json!({"label":"restore", "processor":{"processor_id":record["processor_id"],"version":record["version"]}})); + let id = created["id"].as_str().unwrap(); + owner.call("start", json!({"id":id})); + assert_eq!(owner.wait(id)["state"], "running"); + owner.call("execute",json!({"id":id,"operation":"apply_changes","args":{"changes":[{"op":"insert","predicate":"source","values":[1,"saved"]}]}})); + let receipt = owner.call("checkpoint", json!({"id":id})); + assert_eq!(receipt["origin"]["revision"], 2); + assert!(!owner + .fail("restore", json!({"id":id,"receipt":receipt})) + .is_empty()); + owner.call("stop", json!({"id":id})); + owner.call("restore", json!({"id":id,"receipt":receipt})); + let status = owner.wait(id); + assert_eq!(status["state"], "running"); + assert_eq!(status["generation"], 2); + assert_eq!(status["revision"], 2); + assert_eq!(status["persistence"]["restored_from"], receipt); + assert_eq!( + owner.call( + "execute", + json!({"id":id,"operation":"query_rows","args":{"predicate":"echo"}}) + )["rows"], + json!([[1, "saved"]]) + ); + owner.call("stop", json!({"id":id})); +}