diff --git a/.github/workflows/immunopeptidomics-reference.yml b/.github/workflows/immunopeptidomics-reference.yml new file mode 100644 index 0000000..76e7263 --- /dev/null +++ b/.github/workflows/immunopeptidomics-reference.yml @@ -0,0 +1,99 @@ +name: Immunopeptidomics Reference + +on: + pull_request: + push: + branches: + - main + - 'reference/**' + workflow_dispatch: + +permissions: + contents: read + +jobs: + reference: + name: Synthetic scientific workflow and signed evidence + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Check out source + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Rust + uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 + with: + toolchain: 1.91.0 + targets: wasm32-unknown-unknown + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: | + python/requirements.txt + python/requirements-immunopeptidomics.txt + + - name: Cache Rust artifacts + uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 + + - name: Install the declared Python environment + run: | + python -m pip install -r python/requirements-immunopeptidomics.txt + python -m pip check + + - name: Test scientific semantics and failure handling + run: | + cargo test --locked --manifest-path examples/wasm-services/Cargo.toml -p immunopeptidomics + python -W error -m unittest python.tests.test_froglet_fl_evidence examples.test_immunopeptidomics_demo -v + python -W error integrations/openclaw/froglet/test/openshell-qualification-backend.test.py -v + + - name: Build the node and scientific Wasm module + run: | + cargo build --locked --release -p froglet --bin froglet-node + cargo build --locked --release --target wasm32-unknown-unknown --manifest-path examples/wasm-services/Cargo.toml -p immunopeptidomics + + - name: Run real local providers through NVIDIA FLARE components + run: >- + python examples/immunopeptidomics_demo.py + --binary target/release/froglet-node + --module examples/wasm-services/target/wasm32-unknown-unknown/release/immunopeptidomics.wasm + --output "${{ runner.temp }}/immunopeptidomics-success.json" + + - name: Refuse incomplete output when a required provider is unavailable + run: | + set +e + python examples/immunopeptidomics_demo.py \ + --binary target/release/froglet-node \ + --module examples/wasm-services/target/wasm32-unknown-unknown/release/immunopeptidomics.wasm \ + --unavailable-site site_b \ + --output "$RUNNER_TEMP/immunopeptidomics-failure.json" + reference_status=$? + set -e + test "$reference_status" -eq 1 + python - <<'PY' + import json, os + from pathlib import Path + report = json.loads((Path(os.environ['RUNNER_TEMP']) / 'immunopeptidomics-failure.json').read_text()) + assert report['status'] == 'failed' + assert report['phase'] == 'collect_site_counts' + assert report['aggregate'] is None + assert set(report['workers']) == {'site_a'} + assert report['evidence']['missing_participants'] == ['site_b'] + assert report['evidence']['output'] is None + PY + + - name: Record installed dependency versions + if: always() + run: python -m pip freeze > "${{ runner.temp }}/immunopeptidomics-dependencies.txt" + + - name: Retain reference evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: immunopeptidomics-reference-evidence + path: | + ${{ runner.temp }}/immunopeptidomics-*.json + ${{ runner.temp }}/immunopeptidomics-dependencies.txt + if-no-files-found: warn diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bca765..c71a43a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ Lightning rail and MCP registry distribution. ## [Unreleased] +### Added + +- Synthetic immunopeptidomics reference: two provider-local SQLite datasets, NVIDIA FLARE task components, a bounded Wasm count workload, explicit denominators/missingness, signed evidence and incomplete-result rejection. +- Standalone OpenShell plugin qualification recipe and disposable local backend/probe; corrected directory-upload destinations in NemoClaw setup instructions and doctor output. + ## [0.4.7-beta.1] - Prepared 2026-10-03 This section records the beta candidate at preparation, before immutable diff --git a/docs/IMMUNOPEPTIDOMICS_REFERENCE.md b/docs/IMMUNOPEPTIDOMICS_REFERENCE.md new file mode 100644 index 0000000..1e5801f --- /dev/null +++ b/docs/IMMUNOPEPTIDOMICS_REFERENCE.md @@ -0,0 +1,135 @@ +# Synthetic immunopeptidomics with NVIDIA FLARE and Froglet + +This reference asks a narrow question: among samples in a selected cohort whose +recorded HLA genotype contains the requested allele, how many accepted, available +assay results detected an exact candidate peptide? + +It uses two synthetic sites, separate provider-local SQLite databases, and a +small Wasm computation. Installed NVIDIA FLARE Executor/DXO components dispatch +tasks from one Python process to real local Froglet nodes. It does not provision +a FLARE server/client network or connect to HLA-Compass production. + +## Reproduce the reference + +Use Python 3.11 and Rust 1.91.0 with `wasm32-unknown-unknown`. Install the optional +Python environment separately from your other applications: + +```bash +python3.11 -m venv .venv +source .venv/bin/activate +python -m pip install -r python/requirements-immunopeptidomics.txt +python -m pip check +rustup target add wasm32-unknown-unknown +cargo build --locked --release -p froglet --bin froglet-node +cargo build --locked --release --target wasm32-unknown-unknown \ + --manifest-path examples/wasm-services/Cargo.toml -p immunopeptidomics +python examples/immunopeptidomics_demo.py \ + --binary target/release/froglet-node \ + --module examples/wasm-services/target/wasm32-unknown-unknown/release/immunopeptidomics.wasm \ + --output /tmp/immunopeptidomics-success.json +``` + +Use an exact published commit for repeatable source selection and record +`git rev-parse HEAD`. The report includes source, binary, module, fixture and +provider database hashes, plus the installed FLARE version. The direct framework +dependencies are pinned; retain `python -m pip freeze` alongside the report to +record the complete installed environment. Execution requires FLARE 2.9.0 and +does not silently substitute another framework or an arithmetic-only path. + +The default provider execution allowance is 30 seconds, including cold Wasm +compilation; `--execution-timeout-secs` accepts 1–300. Providers retain bounded +admission and execution limits. Successful runs remove temporary node state; +use `--keep-state` to retain it. Execution failures retain state and write a +failure report when `--output` is supplied. Invalid preflight arguments and +missing dependencies fail before node startup. + +## Scientific contract and expected result + +The [fixture contract](../examples/fixtures/immunopeptidomics/README.md), +[schema](../examples/fixtures/immunopeptidomics/schema.sql), and +[hand-labelled expectations](../examples/fixtures/immunopeptidomics/expected.json) +are part of the example. All samples, labels and detection calls are invented. + +The query is an exact unmodified peptide sequence, an exact two-field class-I +HLA allele, a sample-class token and an organ token. HLA selection describes the +sample genotype; it does not identify the presenting allele or prove binding. +Known target alleles establish eligibility. If the target is absent, only +complete typing at the requested locus establishes nonmatching; incomplete or +missing typing remains unknown. + +Only eligible samples with accepted, completed and available assay results form +the detection denominator. A sample with an explicitly empty accepted detection +set is measured but not detected. Failed or unavailable assays remain missing, +even if partial observation rows exist. Duplicate observations count once. + +| Synthetic count | Site A | Site B | Pooled | +|---|---:|---:|---:| +| Selected cohort | 8 | 8 | 16 | +| Genotype eligible | 5 | 6 | 11 | +| Measured eligible | 3 | 5 | 8 | +| Candidate detected | 2 | 3 | 5 | +| Candidate not detected | 1 | 2 | 3 | +| Eligible assay missing | 2 | 1 | 3 | +| Genotype unknown | 2 | 1 | 3 | +| Genotype nonmatching | 1 | 1 | 2 | + +The expected detection fraction is **5/8**, obtained by summing counts, not +averaging site percentages. When no eligible sample is measured, the fraction +is null rather than zero. The unit is a specimen; real integrations must address +duplicate specimens and technical replicates before pooling. This example does +not deduplicate donors across sites or establish population prevalence. + +## Execution and evidence + +1. The coordinator declares the required provider identities, module and exact + request/workload hashes. +2. FLARE components pass each query to a site executor, which requests a signed + Froglet computation under the provider's granted host-compute offer. +3. The Wasm module uses fixed parameterized read-only SQL through the local + `cohort` handle. Only counts, query metadata and interpretation labels are + returned; no sample rows, genotype lists or observation lists are returned. +4. The requester verifies each five-artifact chain and the result's canonical + JSON hash, validates the query/count contract, and compares the synthetic + output with an independent Python oracle. +5. A repeated submission must recover the same signed deal and result. +6. The aggregate is promoted only after every declared site has contributed and + the pooled result matches the independent oracle. + +Each worker's report retains its signed artifacts and result for offline +rechecking. A valid signature establishes a recorded commitment and its chain; +it does not prove physical dataset identity or scientific correctness. The +completeness state is in memory, with no durable restart protocol or distributed +rollback. + +## Failure and regression checks + +```bash +cargo test --locked --manifest-path examples/wasm-services/Cargo.toml \ + -p immunopeptidomics +python -W error -m unittest python.tests.test_froglet_fl_evidence \ + examples.test_immunopeptidomics_demo -v +python examples/immunopeptidomics_demo.py \ + --binary target/release/froglet-node \ + --module examples/wasm-services/target/wasm32-unknown-unknown/release/immunopeptidomics.wasm \ + --unavailable-site site_b --output /tmp/immunopeptidomics-failure.json +``` + +The last command intentionally exits with failure. Its JSON must retain Site A's +verified contribution, mark Site B missing, and contain no promoted aggregate. +An unavailable site is not interpreted as a site with zero detections. The +dedicated `Immunopeptidomics Reference` CI workflow runs both live paths and +retains the reports and dependency inventory. + +## Interpretation and deployment boundary + +Non-detection is not biological absence. This example consumes accepted peptide +calls; it does not reprocess spectra, validate false-discovery thresholds, +harmonize intensities, establish assay comparability, predict immunogenicity, or +assess clinical safety. It has no GPU requirement or GPU performance claim. + +Aggregate counts can disclose information. The example has no small-cell +suppression, differential privacy, secure aggregation, or independently enforced +query budget. Use only the synthetic fixtures for this reference. A real data +deployment needs a defined scientific/export contract and disclosure policy. + +Agent runtime qualification is separate; see [OpenShell/NemoClaw](NEMOCLAW.md). diff --git a/docs/NEMOCLAW.md b/docs/NEMOCLAW.md index da08a43..e702edc 100644 --- a/docs/NEMOCLAW.md +++ b/docs/NEMOCLAW.md @@ -46,15 +46,23 @@ Supported plugin keys are the same as OpenClaw: Use documented OpenShell/NemoClaw commands only. -Stage the plugin: +Stage the plugin and its shared library, preserving the repository's relative +directory layout. The plugin imports `integrations/shared/froglet-lib`; uploading +only the plugin directory leaves those imports unresolved. For a directory +upload, OpenShell retains the source directory's basename inside the destination, +so pass its parent directory as the destination: ```bash openshell sandbox upload my-node \ /absolute/path/to/froglet/integrations/openclaw/froglet \ - /sandbox/froglet/integrations/openclaw/froglet + /sandbox/froglet/integrations/openclaw +openshell sandbox upload my-node \ + /absolute/path/to/froglet/integrations/shared/froglet-lib \ + /sandbox/froglet/integrations/shared ``` -Stage the Froglet control token: +Stage the Froglet control token. Unlike a directory upload, this unfiltered +single-file upload names the destination file: ```bash openshell sandbox upload my-node \ @@ -64,11 +72,21 @@ openshell sandbox upload my-node \ If the host uses a private CA, stage that CA as well. +The token in this example is readable inside the sandbox. This staging flow +does not implement credential substitution outside the agent workload. + ## Verification -Current status: source-level shared OpenClaw/NemoClaw plugin checks pass, but -real NemoClaw verification is not complete until a live NemoClaw/OpenShell -sandbox runs the plugin and calls one Froglet action. +Full NemoClaw-managed onboarding and plugin execution are not yet verified. +The [standalone OpenShell qualification](OPENSHELL_QUALIFICATION.md) is a separate +target: it uses OpenShell 0.1.7, while +[NemoClaw 0.0.142 pins OpenShell 0.0.116](https://github.com/NVIDIA/NemoClaw/blob/v0.0.142/nemoclaw-blueprint/blueprint.yaml#L7). +Do not treat a standalone OpenShell result as verification of that NemoClaw stack. + +The plugin doctor checks the staged entrypoint, manifest and shared dependency +files without executing plugin code. An error produces a nonzero exit status; +a skipped runtime check remains a warning. Passing these checks does not verify +OpenShell policy enforcement or replace a real tool invocation in the sandbox. ```bash nemoclaw my-node status @@ -89,7 +107,10 @@ Minimum completion evidence: ```bash openshell sandbox upload my-node \ /absolute/path/to/froglet/integrations/openclaw/froglet \ - /sandbox/froglet/integrations/openclaw/froglet + /sandbox/froglet/integrations/openclaw +openshell sandbox upload my-node \ + /absolute/path/to/froglet/integrations/shared/froglet-lib \ + /sandbox/froglet/integrations/shared openshell sandbox upload my-node \ /absolute/path/to/froglet-control.token \ /sandbox/.openclaw/froglet-control.token diff --git a/docs/OPENSHELL_QUALIFICATION.md b/docs/OPENSHELL_QUALIFICATION.md new file mode 100644 index 0000000..d00b751 --- /dev/null +++ b/docs/OPENSHELL_QUALIFICATION.md @@ -0,0 +1,182 @@ +# Froglet plugin qualification with standalone NVIDIA OpenShell + +This is a reproducible **plugin/runtime boundary test**. It imports the actual +OpenClaw plugin, registers its `froglet` tool, and executes two real HTTP-backed +actions inside OpenShell against a disposable Froglet node. It does not run an +OpenClaw gateway, a language model, a scientific workload, or managed NemoClaw +onboarding. + +The observed platform is Apple Silicon macOS with Docker Desktop, OpenShell +**0.1.7**, Node **22.14.0** inside the workload, and Froglet built from public +commit `c6c5dc53f38a2c485f95cd6776f32f4e7b33785b`. The supplied preparation helper +deliberately rejects other host platforms; Linux is not yet verified by this +recipe. + +## What the check establishes + +1. Under an explicit baseline policy, `list_local_services` and + `get_spend_status` both succeed against the same test node. +2. After removing only the spend-read rule and waiting for the policy to load, + service listing still succeeds. The identical spend read receives an + OpenShell HTTP 403 `policy_denied` response. +3. OpenShell records an L7 REST denial. The HTTPS adapter's access log records + no corresponding request reaching Froglet after the policy change. + +The workload is non-root. Its policy requires Landlock enforcement, disables +operator-requested host bind mounts, and grants `/usr/local/bin/node` only the specified HTTPS +endpoint and REST paths. Gateway authentication uses mTLS. Endpoint and +interception certificates remain verified. A separate test CA signs a server +leaf certificate; this is the certificate arrangement qualified here. + +The two Froglet bearer tokens are generated by the disposable local node and +copied into the workload. **The plugin can read them.** This test does not +implement OpenShell credential substitution, establish secret invisibility, or +test protection against inference from permitted responses. + +## Version boundary + +Standalone OpenShell and NemoClaw must be qualified separately. As checked on +2026-11-09, maintained NemoClaw **v0.0.142** pins OpenShell **0.0.116**, while this +test uses standalone OpenShell **0.1.7**. Do not describe this result as a full +NemoClaw deployment. The existing NemoClaw adapter profile and doctor remain +useful setup components; managed onboarding and agent inference are not yet +verified here. + +Official references: + +- [OpenShell 0.1.7 release](https://github.com/NVIDIA/OpenShell/releases/tag/v0.1.7) +- [Gateway configuration](https://docs.nvidia.com/openshell/v0.1.7/how-it-works/gateways/configuration) +- [Policy schema](https://docs.nvidia.com/openshell/v0.1.7/how-it-works/policies/schema) +- [NemoClaw v0.0.142 version constraints](https://github.com/NVIDIA/NemoClaw/blob/v0.0.142/nemoclaw-blueprint/blueprint.yaml) +- [NemoClaw prerequisites](https://docs.nvidia.com/nemoclaw/latest/user-guide/openclaw/get-started/prerequisites) + +## Prerequisites and pinned tools + +Use a fresh terminal on Apple Silicon macOS. Start the existing Docker Desktop +installation and confirm `docker info` succeeds. Build Froglet from the revision +being qualified, for example `cargo build --locked --release --bin froglet-node`. +Python 3.10 or later and OpenSSL must be on PATH. No model download, inference +provider, paid API, production token, or production data is needed. + +From this repository's root, install the official binaries into a temporary +directory. The digests below are the published v0.1.7 macOS arm64 release hashes: + +```sh +repo_root="$PWD" +task_dir=$(mktemp -d "${TMPDIR:-/tmp}/froglet-openshell.XXXXXX") +mkdir "$task_dir/bin" +curl -fLsS https://github.com/NVIDIA/OpenShell/releases/download/v0.1.7/openshell-aarch64-apple-darwin.tar.gz -o "$task_dir/cli.tar.gz" +curl -fLsS https://github.com/NVIDIA/OpenShell/releases/download/v0.1.7/openshell-gateway-aarch64-apple-darwin.tar.gz -o "$task_dir/gateway.tar.gz" +printf '%s %s\n' c5899a2fe6b40c371e7d660bd8983cefb766d04658bd9193b12a80525b12c742 "$task_dir/cli.tar.gz" | shasum -a 256 -c - +printf '%s %s\n' fe47a385c16c10cde8905bf1bf627e4a9fbf4a81bcf44b388dec4d22618fbb48 "$task_dir/gateway.tar.gz" | shasum -a 256 -c - +tar -xzf "$task_dir/cli.tar.gz" -C "$task_dir/bin" +tar -xzf "$task_dir/gateway.tar.gz" -C "$task_dir/bin" +export PATH="$task_dir/bin:$PATH" +``` + +Check each command succeeds before continuing. This leaves Homebrew services and +ordinary OpenShell configuration untouched. Ports 18670 and 19443 must be free; +the recipe does not stop an existing service occupying either port. + +## Prepare the real local backend and gateway + +The helper generates private task-only certificates, writes the gateway and +policy configuration, starts a real Froglet node with payments disabled, and +exposes a loopback HTTPS adapter. The adapter forwards GET requests to that node +and records only method, path, and upstream status; it does not mock API results. + +```sh +python3 integrations/openclaw/froglet/scripts/openshell-qualification-backend.py \ + --work-dir "$task_dir" --repo "$repo_root" \ + --binary "$repo_root/target/release/froglet-node" \ + > "$task_dir/backend.log" 2>&1 & +backend_pid=$! +``` + +Wait for `$task_dir/ready.json` and check the helper is still running. If it exits, +inspect `backend.log` and stop; do not continue with partial setup. + +```sh +. "$task_dir/env.sh" +openshell-gateway --config "$task_dir/config/gateway.toml" \ + > "$task_dir/gateway.log" 2>&1 & +gateway_pid=$! +openshell gateway add https://127.0.0.1:18670 --local --name froglet-qualification +openshell status +``` + +Wait until status reports **Connected**, authenticated with mTLS, version +**0.1.7**. A successful `gateway add` alone does not establish connectivity. + +## Run the baseline and denial checks + +```sh +docker build -t froglet-openshell-qualification:node22.14.0 "$task_dir/image" +openshell sandbox create --name froglet-qual \ + --from froglet-openshell-qualification:node22.14.0 \ + --policy "$task_dir/config/policy-baseline.yaml" \ + --no-auto-providers --detach -- /bin/sleep infinity +openshell sandbox get froglet-qual --output json > "$task_dir/evidence/baseline-state.json" +``` + +Require phase `Ready` and `configuration_admission.state` equal to `accepted`. +For directory uploads, the destination is the **parent** directory: OpenShell +retains the source basename. Single-file token uploads name the final file. + +```sh +openshell sandbox upload froglet-qual "$repo_root/integrations/openclaw/froglet" /sandbox/froglet/integrations/openclaw +openshell sandbox upload froglet-qual "$repo_root/integrations/shared/froglet-lib" /sandbox/froglet/integrations/shared +openshell sandbox upload froglet-qual "$task_dir/stage/provider.token" /sandbox/provider.token +openshell sandbox upload froglet-qual "$task_dir/stage/runtime.token" /sandbox/runtime.token +openshell sandbox exec froglet-qual --no-login-shell -- node \ + /sandbox/froglet/integrations/openclaw/froglet/scripts/qualify-openshell-plugin.mjs baseline \ + > "$task_dir/evidence/baseline.json" +openshell policy set froglet-qual --policy "$task_dir/config/policy-restricted.yaml" +openshell policy list froglet-qual +openshell sandbox get froglet-qual --output json > "$task_dir/evidence/restricted-state.json" +``` + +Wait for the new policy to show `Loaded`. Require its configuration admission to +be `accepted`, and verify the active policy contains only the service-listing +REST rule. A pending revision is not evidence that the restriction is active. + +```sh +openshell sandbox exec froglet-qual --no-login-shell -- node \ + /sandbox/froglet/integrations/openclaw/froglet/scripts/qualify-openshell-plugin.mjs restricted \ + > "$task_dir/evidence/restricted.json" +openshell logs froglet-qual -n 500 > "$task_dir/evidence/openshell.log" +``` + +The probe exits nonzero on any unexpected outcome. It requires both baseline +reads to succeed, the restricted listing to succeed, and the restricted spend +read to include both `403` and `policy_denied`. Also inspect the independent +evidence: + +- `openshell.log` must contain `DENIED GET` for `/v1/runtime/spend` with `engine:l7`. + Log delivery can lag; collect it again if the event has not arrived yet. +- `backend-access.jsonl` must contain two successful service-listing requests and + only one spend request, from the baseline. An extra request from a separate + diagnostic must be identified rather than silently counted as a test result. +- Preserve the source revision, CLI versions, accepted policies, and image + digests with the result. Retain failed attempts as diagnostics rather than + presenting them as successful qualification runs. + +## Cleanup and evidence handling + +After saving the nonsecret evidence, delete only this test's workload, then stop +the two processes started above. The helper stops its disposable Froglet child. + +```sh +openshell sandbox delete froglet-qual +kill "$gateway_pid" "$backend_pid" +wait "$gateway_pid" "$backend_pid" 2>/dev/null || true +``` + +The task directory contains private test keys and token files. Publish selected +files from `evidence/`, not the entire directory. Remove the private task +directory after retaining the needed evidence. Docker caches remain available; +do not prune unrelated images, volumes, or containers to clean up this test. + +The result is limited to the named plugin actions and policy on the tested +platform. It does not establish universal rollback, privacy of allowed model +updates, scientific validity, GPU acceleration, or customer deployment. diff --git a/docs/README.md b/docs/README.md index d200d19..d30d323 100644 --- a/docs/README.md +++ b/docs/README.md @@ -45,6 +45,7 @@ Public launch entry points: ## Companion Docs +- [IMMUNOPEPTIDOMICS_REFERENCE.md](IMMUNOPEPTIDOMICS_REFERENCE.md): synthetic peptide detection counts through FLARE components and signed Froglet computations - [ADAPTERS.md](ADAPTERS.md): transport and execution adapter boundaries - [API_ERRORS.md](API_ERRORS.md): API error reference — status codes and error shapes - [CONFIDENTIAL.md](CONFIDENTIAL.md): TEE and encrypted execution extension @@ -73,4 +74,5 @@ Public launch entry points: - [MCP_DIRECTORY_SUBMISSIONS.md](MCP_DIRECTORY_SUBMISSIONS.md): MCP directory submission tracking - [OPENCLAW.md](OPENCLAW.md): OpenClaw (Claude) integration - [NEMOCLAW.md](NEMOCLAW.md): NemoClaw (local agent) integration +- [OPENSHELL_QUALIFICATION.md](OPENSHELL_QUALIFICATION.md): reproducible allow/deny check for the Froglet plugin in standalone OpenShell - [ROLE_TOOL_ARCHITECTURE.md](ROLE_TOOL_ARCHITECTURE.md): role and tool architecture diff --git a/examples/README.md b/examples/README.md index 55be3d8..be9c10f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,16 @@ # Examples +## Synthetic immunopeptidomics with NVIDIA FLARE + +[immunopeptidomics_demo.py](immunopeptidomics_demo.py) sends an exact peptide/HLA +cohort query through installed FLARE components to two local Froglet providers. +A small Wasm module reads separate local SQLite fixtures and returns counts with +explicit measured denominators and missingness. Signed evidence, retries and +refusal of incomplete aggregates are checked. All data is synthetic. + +See [the reproducible reference](../docs/IMMUNOPEPTIDOMICS_REFERENCE.md) for the +pinned environment, build/run commands, scientific boundaries and failure test. + ## Local Three-Role Stack ```bash diff --git a/examples/fixtures/immunopeptidomics/README.md b/examples/fixtures/immunopeptidomics/README.md new file mode 100644 index 0000000..5525d2c --- /dev/null +++ b/examples/fixtures/immunopeptidomics/README.md @@ -0,0 +1,59 @@ +# Synthetic immunopeptidomics fixtures + +These files contain **invented data only**: 20 samples at two simulated sites, synthetic tissue labels and synthetic peptide detections. The candidate sequence is an example string with no asserted biological property. HLA names are standard allele labels; their use does not make the samples real. + +- `fixture.json`: one query and four local SQLite tables per site. Each site contains exactly `samples`, `sample_hla_typing`, `sample_hla_typing_status` and `sample_peptide_detections`. Site labels are top-level metadata. +- `schema.sql`: the local table contract. It is not a production HLA-Compass database schema or data connector. +- `expected.json`: independently hand-labelled expected counts and sample membership lists for test review. Membership lists are synthetic development evidence and must not be included in provider results. + +## What is counted + +The query asks: among specimens in the exact sample-class/organ cohort whose recorded genotype includes the requested HLA allele, how many accepted available assay results contain the exact candidate sequence? + +The unit is a **sample/specimen**, not a patient, donor, MS injection, technical replicate or peptide-spectrum match. These synthetic samples are distinct across sites. The example does not implement cross-site donor linkage or technical-replicate consolidation. + +Input accepts an uppercase unmodified peptide sequence and an exact two-field class-I `HLA-A`, `HLA-B` or `HLA-C` name. It does not normalize aliases, truncate allele resolution, remove expression suffixes, match peptide variants or interpret PTM notation. `healthy_tissue` and `synthetic_organ_a` are exact synthetic metadata tokens, not an asserted universal vocabulary. + +## HLA stratum and assay availability + +HLA typing belongs to the sample. If the requested allele is recorded, that sample belongs to the stratum even when other typing information is incomplete. If it is not recorded, `complete` typing at the requested locus permits a nonmatching classification; `incomplete`, `missing` or absent completeness metadata produces an unknown classification. Here `missing` means no typing-completeness assessment is available. It does not override an actually recorded allele. + +This is a **sample-genotype filter**. It does not establish that the selected allele presented the peptide or assign an observed peptide to one allele in a multiallelic sample. + +`assay_status` is an explicit data-export assertion: + +- `measured`: an accepted, completed assay's observation set is available in this local snapshot. An explicitly empty observation set is valid. Do not derive this status merely from the existence of one peptide row. +- `not_measured`: no accepted completed result is available because the assay was not performed or its result is unavailable. +- `failed`: the assay did not meet local acceptance criteria. Retained partial detection rows do not become accepted detections. + +Eligible samples partition into measured, not measured and failed. The latter two contribute to missingness, not to the measured denominator. Among measured eligible samples, an exact candidate observation contributes one detected sample. Repeated observations never increase that sample's contribution. Absence from the accepted observation set means **not detected in that assay**, not biological absence. + +## Hand-checked result + +| Count | Site A | Site B | Sum | +|---|---:|---:|---:| +| Cohort samples | 8 | 8 | 16 | +| Unknown genotype stratum | 2 | 1 | 3 | +| Nonmatching genotype stratum | 1 | 1 | 2 | +| Eligible samples | 5 | 6 | 11 | +| Measured eligible samples | 3 | 5 | 8 | +| Detected | 2 | 3 | 5 | +| Not detected | 1 | 2 | 3 | +| Not measured | 1 | 1 | 2 | +| Failed | 1 | 0 | 1 | +| Missing | 2 | 1 | 3 | + +The pooled detection fraction is **5/8 = 0.625**. Pool by summing counts; the unweighted mean of the site fractions is 19/30 and is deliberately different. Missingness is 3/11 among eligible samples. If the measured denominator is zero, the detection fraction is null, not zero. A missing/refusing site is not an empty cohort and cannot silently contribute zero to an all-sites result. + +The fixture exercises duplicate detection rows, an accepted empty observation set, incomplete HLA typing both with and without the requested allele, missing typing, failed/unavailable assays, and samples outside the requested tissue/class cohort. A1's duplicate candidate observation counts once; A6/A7 detections do not enter the requested stratum's numerator. + +## Limits + +This workload counts previously accepted observations. It does not process spectra, choose false-discovery thresholds, validate local assay acceptance, harmonize intensity measurements or establish assay comparability across laboratories. Summing counts is arithmetic; it does not turn the fraction into population prevalence, immunogenicity, toxicity or therapeutic safety evidence. + +Only aggregate service results are intended to leave each simulated site. This does not establish statistical privacy: small-cell suppression, adaptive-query limits and differential privacy are not supplied by the counting example. Real deployment requires its own data-access and result-release policy. + +Primary research explaining why these distinctions matter: + +- [Sensitive Immunopeptidomics by Leveraging Available Large-Scale Multi-HLA Spectral Libraries, Data-Independent Acquisition, and MS/MS Prediction](https://pmc.ncbi.nlm.nih.gov/articles/PMC8724634/) examines multiallelic samples and sensitivity dependent on measurement methods. +- [Deep learning boosts sensitivity of mass spectrometry-based immunopeptidomics](https://www.nature.com/articles/s41467-021-23713-9) shows that peptide identification changes with analytical sensitivity; non-detection is not a biological-absence guarantee. diff --git a/examples/fixtures/immunopeptidomics/expected.json b/examples/fixtures/immunopeptidomics/expected.json new file mode 100644 index 0000000..54c13d0 --- /dev/null +++ b/examples/fixtures/immunopeptidomics/expected.json @@ -0,0 +1,129 @@ +{ + "sites": { + "site_a": { + "cohort_total": 8, + "hla_unknown": 2, + "hla_nonmatching": 1, + "eligible": 5, + "measured": 3, + "detected": 2, + "not_detected": 1, + "not_measured": 1, + "failed": 1, + "missing": 2 + }, + "site_b": { + "cohort_total": 8, + "hla_unknown": 1, + "hla_nonmatching": 1, + "eligible": 6, + "measured": 5, + "detected": 3, + "not_detected": 2, + "not_measured": 1, + "failed": 0, + "missing": 1 + } + }, + "aggregate": { + "cohort_total": 16, + "hla_unknown": 3, + "hla_nonmatching": 2, + "eligible": 11, + "measured": 8, + "detected": 5, + "not_detected": 3, + "not_measured": 2, + "failed": 1, + "missing": 3 + }, + "detection_fraction": { + "numerator": 5, + "denominator": 8, + "value": 0.625 + }, + "missing_fraction": { + "numerator": 3, + "denominator": 11 + }, + "expected_member_ids_for_development_only": { + "site_a": { + "eligible": [ + "A1", + "A2", + "A3", + "A4", + "A5" + ], + "measured": [ + "A1", + "A2", + "A3" + ], + "detected": [ + "A1", + "A3" + ], + "not_detected": [ + "A2" + ], + "not_measured": [ + "A4" + ], + "failed": [ + "A5" + ], + "hla_nonmatching": [ + "A6" + ], + "hla_unknown": [ + "A7", + "A8" + ], + "outside_cohort": [ + "A9", + "A10" + ] + }, + "site_b": { + "eligible": [ + "B1", + "B2", + "B3", + "B4", + "B5", + "B6" + ], + "measured": [ + "B1", + "B2", + "B3", + "B4", + "B5" + ], + "detected": [ + "B1", + "B4", + "B5" + ], + "not_detected": [ + "B2", + "B3" + ], + "not_measured": [ + "B6" + ], + "failed": [], + "hla_nonmatching": [ + "B7" + ], + "hla_unknown": [ + "B8" + ], + "outside_cohort": [ + "B9", + "B10" + ] + } + } +} diff --git a/examples/fixtures/immunopeptidomics/fixture.json b/examples/fixtures/immunopeptidomics/fixture.json new file mode 100644 index 0000000..2913d1c --- /dev/null +++ b/examples/fixtures/immunopeptidomics/fixture.json @@ -0,0 +1,409 @@ +{ + "schema_version": "immunopeptidomics.synthetic_fixture.v1", + "data_origin": "synthetic", + "description": "Invented samples and detections for a class-I sample-genotype-stratified analytics example; not clinical evidence. Sample class and organ are synthetic exact-match vocabulary.", + "query": { + "candidate_peptide": "ACDEFGHIK", + "hla_allele": "HLA-A*02:01", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a" + }, + "sites": { + "site_a": { + "samples": [ + { + "sample_id": "A1", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "A2", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "A3", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "A4", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "not_measured" + }, + { + "sample_id": "A5", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "failed" + }, + { + "sample_id": "A6", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "A7", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "A8", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "A9", + "sample_class": "tumor_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "A10", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_b", + "assay_status": "measured" + } + ], + "sample_hla_typing": [ + { + "sample_id": "A1", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "A1", + "allele_name": "HLA-A*24:02" + }, + { + "sample_id": "A2", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "A3", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "A4", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "A5", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "A6", + "allele_name": "HLA-A*24:02" + }, + { + "sample_id": "A7", + "allele_name": "HLA-A*24:02" + }, + { + "sample_id": "A9", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "A10", + "allele_name": "HLA-A*02:01" + } + ], + "sample_hla_typing_status": [ + { + "sample_id": "A1", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "A2", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "A3", + "locus": "HLA-A", + "status": "incomplete" + }, + { + "sample_id": "A4", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "A5", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "A6", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "A7", + "locus": "HLA-A", + "status": "incomplete" + }, + { + "sample_id": "A8", + "locus": "HLA-A", + "status": "missing" + }, + { + "sample_id": "A9", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "A10", + "locus": "HLA-A", + "status": "complete" + } + ], + "sample_peptide_detections": [ + { + "sample_id": "A1", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "A1", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "A1", + "sequence": "KLMNPQRST" + }, + { + "sample_id": "A2", + "sequence": "KLMNPQRST" + }, + { + "sample_id": "A3", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "A6", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "A7", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "A8", + "sequence": "KLMNPQRST" + }, + { + "sample_id": "A9", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "A10", + "sequence": "ACDEFGHIK" + } + ] + }, + "site_b": { + "samples": [ + { + "sample_id": "B1", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "B2", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "B3", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "B4", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "B5", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "B6", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "not_measured" + }, + { + "sample_id": "B7", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "B8", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "B9", + "sample_class": "tumor_tissue", + "organ": "synthetic_organ_a", + "assay_status": "measured" + }, + { + "sample_id": "B10", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_b", + "assay_status": "measured" + } + ], + "sample_hla_typing": [ + { + "sample_id": "B1", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "B2", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "B3", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "B4", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "B5", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "B6", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "B7", + "allele_name": "HLA-A*24:02" + }, + { + "sample_id": "B8", + "allele_name": "HLA-A*24:02" + }, + { + "sample_id": "B9", + "allele_name": "HLA-A*02:01" + }, + { + "sample_id": "B10", + "allele_name": "HLA-A*02:01" + } + ], + "sample_hla_typing_status": [ + { + "sample_id": "B1", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "B2", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "B3", + "locus": "HLA-A", + "status": "incomplete" + }, + { + "sample_id": "B4", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "B5", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "B6", + "locus": "HLA-A", + "status": "incomplete" + }, + { + "sample_id": "B7", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "B8", + "locus": "HLA-A", + "status": "incomplete" + }, + { + "sample_id": "B9", + "locus": "HLA-A", + "status": "complete" + }, + { + "sample_id": "B10", + "locus": "HLA-A", + "status": "complete" + } + ], + "sample_peptide_detections": [ + { + "sample_id": "B1", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "B3", + "sequence": "KLMNPQRST" + }, + { + "sample_id": "B4", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "B5", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "B7", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "B8", + "sequence": "KLMNPQRST" + }, + { + "sample_id": "B9", + "sequence": "ACDEFGHIK" + }, + { + "sample_id": "B10", + "sequence": "ACDEFGHIK" + } + ] + } + }, + "site_labels": { + "site_a": "synthetic_site_a", + "site_b": "synthetic_site_b" + } +} diff --git a/examples/fixtures/immunopeptidomics/schema.sql b/examples/fixtures/immunopeptidomics/schema.sql new file mode 100644 index 0000000..0974ebf --- /dev/null +++ b/examples/fixtures/immunopeptidomics/schema.sql @@ -0,0 +1,22 @@ +PRAGMA foreign_keys = ON; + +CREATE TABLE samples ( + sample_id TEXT PRIMARY KEY, + sample_class TEXT NOT NULL, + organ TEXT NOT NULL, + assay_status TEXT NOT NULL CHECK (assay_status IN ('measured', 'not_measured', 'failed')) +); +CREATE TABLE sample_hla_typing ( + sample_id TEXT NOT NULL REFERENCES samples(sample_id), + allele_name TEXT NOT NULL +); +CREATE TABLE sample_hla_typing_status ( + sample_id TEXT NOT NULL REFERENCES samples(sample_id), + locus TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('complete', 'incomplete', 'missing')), + PRIMARY KEY (sample_id, locus) +); +CREATE TABLE sample_peptide_detections ( + sample_id TEXT NOT NULL REFERENCES samples(sample_id), + sequence TEXT NOT NULL +); diff --git a/examples/immunopeptidomics_demo.py b/examples/immunopeptidomics_demo.py new file mode 100644 index 0000000..e876f6a --- /dev/null +++ b/examples/immunopeptidomics_demo.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +"""Synthetic immunopeptidomics counts over two local Froglet providers. + +NVIDIA FLARE Executor/DXO components run in this Python process; Froglet nodes +execute the signed Wasm requests against separate provider-local SQLite files. +This is a federated analytics reference, not a provisioned FLARE deployment, +biological validation, differential-privacy guarantee, or proof of data identity. +""" +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import closing +import hashlib +import importlib.metadata +import json +import os +from pathlib import Path +import re +import secrets +import shutil +import signal +import sqlite3 +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO / "python" / "froglet-verify")) +sys.path.insert(0, str(REPO)) +import froglet_verify +from python.froglet_fl_evidence import RoundEvidence +from examples import a2a_compute_demo as runtime + +SCHEMA = "froglet.immunopeptidomics.counts.v1" +OPERATION = "immunopeptidomics_candidate_detection" +CAPABILITY = "db.sqlite.query.read.cohort" +FIXTURE = REPO / "examples/fixtures/immunopeptidomics/fixture.json" +COUNT_FIELDS = ("cohort_total", "hla_unknown", "hla_nonmatching", "eligible", "measured", + "detected", "not_detected", "not_measured", "failed", "missing") +INTERPRETATION = { + "hla_selection": "recorded_genotype_not_peptide_restriction", + "not_detected": "not_detected_in_measured_assay_not_biological_absence", + "denominator": "measured_eligible_samples", +} +TABLES = { + "samples": ("sample_id", "sample_class", "organ", "assay_status"), + "sample_hla_typing": ("sample_id", "allele_name"), + "sample_hla_typing_status": ("sample_id", "locus", "status"), + "sample_peptide_detections": ("sample_id", "sequence"), +} + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def validate_query(query): + require(type(query) is dict and set(query) == { + "candidate_peptide", "hla_allele", "sample_class", "organ"}, "Invalid query fields") + require(all(type(value) is str and value for value in query.values()), + "Query fields must be nonempty strings") + require(re.fullmatch(r"[ACDEFGHIKLMNPQRSTVWY]{1,128}", query["candidate_peptide"]), + "Expected an unmodified uppercase peptide sequence") + require(all(query[key].strip() and len(query[key].encode("utf-8")) <= 128 + for key in ("sample_class", "organ")), "Cohort labels must contain 1..128 UTF-8 bytes") + require(re.fullmatch(r"HLA-[ABC]\*[0-9]{2,3}:[0-9]{2,3}", query["hla_allele"]), + "Expected an exact two-field HLA class-I allele") + + +def oracle(site, query): + """Independent set-based calculation from fixture rows, without SQL or Wasm.""" + validate_query(query) + require(type(site) is dict and set(site) == set(TABLES), "Invalid fixture tables") + for table, columns in TABLES.items(): + require(type(site[table]) is list and all( + type(row) is dict and set(row) == set(columns) + and all(type(value) is str and value for value in row.values()) + for row in site[table]), f"Invalid {table} fixture rows") + sample_ids = [row["sample_id"] for row in site["samples"]] + require(len(sample_ids) == len(set(sample_ids)), "Duplicate fixture sample ID") + require(all(row["sample_id"] in sample_ids for table in TABLES if table != "samples" + for row in site[table]), "Orphan fixture observation") + statuses = {} + for row in site["sample_hla_typing_status"]: + key = (row["sample_id"], row["locus"]) + require(key not in statuses, "Duplicate HLA typing status") + require(row["status"] in {"complete", "incomplete", "missing"}, "Invalid typing status") + statuses[key] = row["status"] + alleles = {(row["sample_id"], row["allele_name"]) for row in site["sample_hla_typing"]} + detections = {(row["sample_id"], row["sequence"]) for row in site["sample_peptide_detections"]} + counts = dict.fromkeys(COUNT_FIELDS, 0) + locus = query["hla_allele"].split("*", 1)[0] + for row in site["samples"]: + require(row["assay_status"] in {"measured", "not_measured", "failed"}, "Invalid assay status") + if (row["sample_class"], row["organ"]) != (query["sample_class"], query["organ"]): + continue + counts["cohort_total"] += 1 + sample_id = row["sample_id"] + if (sample_id, query["hla_allele"]) not in alleles: + counts["hla_nonmatching" if statuses.get((sample_id, locus)) == "complete" + else "hla_unknown"] += 1 + continue + counts["eligible"] += 1 + if row["assay_status"] == "measured": + counts["measured"] += 1 + counts["detected" if (sample_id, query["candidate_peptide"]) in detections + else "not_detected"] += 1 + else: + counts[row["assay_status"]] += 1 + counts["missing"] += 1 + return {"schema_version": SCHEMA, "query": dict(query), "counts": counts, + "interpretation": dict(INTERPRETATION)} + + +def validate_result(result, query): + require(type(result) is dict and set(result) == { + "schema_version", "query", "counts", "interpretation"}, "Invalid aggregate result fields") + require(result["schema_version"] == SCHEMA and result["query"] == query + and result["interpretation"] == INTERPRETATION, "Result semantics differ from query") + counts = result["counts"] + require(type(counts) is dict and set(counts) == set(COUNT_FIELDS) + and all(type(value) is int and value >= 0 for value in counts.values()), + "Counts must be nonnegative integers") + require(counts["cohort_total"] == counts["eligible"] + counts["hla_unknown"] + counts["hla_nonmatching"] + and counts["eligible"] == counts["measured"] + counts["missing"] + and counts["measured"] == counts["detected"] + counts["not_detected"] + and counts["missing"] == counts["not_measured"] + counts["failed"], + "Count denominators are inconsistent") + + +def aggregate_results(results, query): + require(bool(results), "No site results") + counts = dict.fromkeys(COUNT_FIELDS, 0) + for result in results: + validate_result(result, query) + for key in COUNT_FIELDS: + counts[key] += result["counts"][key] + return {"schema_version": SCHEMA, "query": dict(query), "counts": counts, + "interpretation": dict(INTERPRETATION)} + + +def detection_fraction(result): + counts = result["counts"] + return {"numerator": counts["detected"], "denominator": counts["measured"], + "value": counts["detected"] / counts["measured"] if counts["measured"] else None} + + +def load_fixture(path): + fixture = json.loads(Path(path).read_text(encoding="utf-8")) + require(type(fixture) is dict and type(fixture.get("sites")) is dict + and len(fixture["sites"]) == 2, "Reference fixture must contain exactly two sites") + require(all(re.fullmatch(r"[a-z][a-z0-9_]{0,31}", name) for name in fixture["sites"]), + "Invalid fixture site name") + validate_query(fixture["query"]) + for site in fixture["sites"].values(): + oracle(site, fixture["query"]) + return fixture + + +def provision_sqlite(node, site): + path = node.data / "cohort.sqlite" + with closing(sqlite3.connect(path)) as connection: + connection.executescript(""" + PRAGMA foreign_keys=ON; + CREATE TABLE samples (sample_id TEXT PRIMARY KEY, sample_class TEXT NOT NULL, + organ TEXT NOT NULL, assay_status TEXT NOT NULL + CHECK (assay_status IN ('measured','not_measured','failed'))); + CREATE TABLE sample_hla_typing (sample_id TEXT REFERENCES samples(sample_id), allele_name TEXT NOT NULL); + CREATE TABLE sample_hla_typing_status (sample_id TEXT REFERENCES samples(sample_id), + locus TEXT NOT NULL, status TEXT NOT NULL CHECK(status IN ('complete','incomplete','missing')), + PRIMARY KEY(sample_id,locus)); + CREATE TABLE sample_peptide_detections (sample_id TEXT REFERENCES samples(sample_id), sequence TEXT NOT NULL); + """) + for table, columns in TABLES.items(): + connection.executemany(f"INSERT INTO {table} VALUES ({','.join('?' for _ in columns)})", + [tuple(row[column] for column in columns) for row in site[table]]) + connection.commit() + path.chmod(0o600) + policy = node.data / "wasm-policy.toml" + policy.write_text("[sqlite]\nmax_queries_per_execution = 1\nmax_rows_per_query = 1000\n" + "max_result_bytes = 262144\n\n[sqlite.handles.cohort]\npath = " + + json.dumps(str(path.resolve())) + "\n", encoding="utf-8") + policy.chmod(0o600) + node.env["FROGLET_WASM_POLICY_PATH"] = str(policy) + return path + + +def build_submission(module, query): + validate_query(query) + return {"schema_version": "froglet/v1", "submission_type": "wasm_submission", + "workload": {"schema_version": "froglet/v1", "workload_kind": "compute.wasm.v1", + "abi_version": "froglet.wasm.host_json.v1", "module_format": "application/wasm", + "module_hash": hashlib.sha256(module).hexdigest(), + "input_format": "application/json+jcs", "input_hash": froglet_verify.payload_hash(query), + "requested_capabilities": [CAPABILITY]}, + "module_bytes_hex": module.hex(), "input": dict(query)} + + +def http_json(method, url, token, payload=None): + data = json.dumps(payload).encode() if payload is not None else None + request = urllib.request.Request(url, data=data, method=method, + headers={"Authorization": "Bearer " + token, "Content-Type": "application/json"}) + try: + with runtime.HTTP.open(request, timeout=10) as response: + content = response.read(4 * 1024 * 1024 + 1) + except urllib.error.HTTPError as error: + raise RuntimeError(f"Runtime request failed with HTTP {error.code}") from error + require(len(content) <= 4 * 1024 * 1024, "Runtime response exceeds reference limit") + return json.loads(content) + + +def submit(coordinator, provider_id, origin, submission, idempotency_key): + token = (coordinator.data / "runtime/auth.token").read_text().strip() + response = http_json("POST", coordinator.url + "/v1/runtime/deals", token, { + "provider": {"provider_id": provider_id, "provider_url": origin}, + "offer_id": "execute.compute.host", "kind": "wasm", "max_price_sats": 0, + "idempotency_key": idempotency_key, "submission": submission}) + deal = response["deal"] + runtime_ms = deal["quote"]["payload"]["execution_limits"]["max_runtime_ms"] + deadline = time.monotonic() + max(30, runtime_ms / 1000 + 15) + while deal["status"] not in {"succeeded", "failed", "rejected", "canceled"}: + require(time.monotonic() < deadline, "Runtime receipt polling timed out") + time.sleep(0.05) + deal = http_json("GET", coordinator.url + "/v1/runtime/deals/" + + urllib.parse.quote(deal["deal_id"], safe=""), token)["deal"] + require(deal["status"] == "succeeded", "Signed computation did not succeed: " + + str(deal.get("receipt", {}).get("payload", {}).get("failure_code", deal["status"]))) + return deal + + +def artifact_bundle(origin, deal): + artifacts = {name: deal[name] for name in ("quote", "deal", "receipt")} + for name in ("descriptor", "offer"): + wrapped = runtime.get_json(origin + "/v1/artifacts/" + deal["quote"]["payload"][name + "_hash"]) + document = wrapped["document"] + artifacts[name] = json.loads(document) if isinstance(document, str) else document + return {"deal_id": deal["deal_id"], "result_hash": deal["result_hash"], "signed_artifacts": artifacts} + + +def verify_artifact_chain_offline(evidence, result): + """Verify signatures, chain links, successful execution and exact JCS result.""" + artifacts = evidence["signed_artifacts"] + require(set(artifacts) == {"descriptor", "offer", "quote", "deal", "receipt"}, + "Expected a free five-artifact chain") + chain = froglet_verify.validate_full_chain(**artifacts) + require(chain.get("valid") is True, "Offline signed-artifact verification failed") + receipt, quote, deal = artifacts["receipt"], artifacts["quote"], artifacts["deal"] + result_hash = froglet_verify.payload_hash(result) + require(result_hash == evidence["result_hash"] == receipt["payload"].get("result_hash"), + "Result differs from signed Receipt hash") + require(evidence["deal_id"] == deal["hash"], "Deal identifier differs from signed Deal") + require(receipt["payload"]["deal_state"] == "succeeded" + and receipt["payload"]["execution_state"] == "succeeded", "Receipt is not successful") + return {"offline_chain_verified": True, "artifact_count": 5, + **{name + "_hash": artifact["hash"] for name, artifact in artifacts.items()}, + "invoice_bundle_hash": None, "result_hash": result_hash, + "provider_id": receipt["signer"], "requester_id": deal["signer"], + "workload_hash": quote["payload"]["workload_hash"], + "module_hash": receipt["payload"]["executor"].get("module_hash"), + "deal_state": receipt["payload"]["deal_state"], + "execution_state": receipt["payload"]["execution_state"]} + + +def nvflare_components(): + """Fail explicitly: this NVIDIA reference has no substitute execution mode.""" + try: + from nvflare.apis.dxo import DXO, DataKind, from_shareable + from nvflare.apis.executor import Executor + from nvflare.apis.fl_context import FLContextManager + from nvflare.apis.signal import Signal + version = importlib.metadata.version("nvflare") + except (ImportError, importlib.metadata.PackageNotFoundError) as error: + raise RuntimeError("NVIDIA FLARE is required; install python/requirements-immunopeptidomics.txt") from error + require(version == "2.9.0", "This reference requires the tested NVIDIA FLARE 2.9.0") + return DXO, DataKind, from_shareable, Executor, FLContextManager, Signal + + +def dispatch_nvflare(site_names, query, worker): + """Call installed FLARE APIs; worker performs real Froglet transport in run().""" + DXO, DataKind, from_shareable, Executor, FLContextManager, Signal = nvflare_components() + + class SiteExecutor(Executor): + def __init__(self, name): + super().__init__() + self.site_name = name + + def execute(self, task_name, shareable, fl_ctx, abort_signal): + require(task_name == OPERATION and not abort_signal.triggered, "Invalid or aborted FLARE task") + request = from_shareable(shareable) + require(request.data_kind == DataKind.APP_DEFINED and request.data == query, + "FLARE task differs from declared query") + return DXO(DataKind.APP_DEFINED, data=worker(self.site_name, request.data)).to_shareable() + + def execute(name): + manager = FLContextManager(engine=None, identity_name=name, job_id="froglet-immunopeptidomics") + with manager.new_context() as context: + response = SiteExecutor(name).execute(OPERATION, DXO(DataKind.APP_DEFINED, data=dict(query)).to_shareable(), + context, Signal()) + result = from_shareable(response) + require(result.data_kind == DataKind.APP_DEFINED, "Invalid FLARE response kind") + return result.data + + records, failures = {}, {} + with ThreadPoolExecutor(max_workers=len(site_names)) as pool: + pending = {pool.submit(execute, name): name for name in site_names} + for future in as_completed(pending): + name = pending[future] + try: + records[name] = future.result() + except Exception as error: + failures[name] = {"type": type(error).__name__, "message": str(error)} + return records, failures + + +def stage_records(evidence, records, failures, *, run_id, module_hash, workload_hashes, fixture_sites): + for name, failure in failures.items(): + evidence.record_failure(name, failure["type"] + ": " + failure["message"]) + for name, record in records.items(): + audit = verify_artifact_chain_offline(record, record["result"]) + arguments = {"run_id": run_id, "round_id": 0, "operation": OPERATION, + "module_hash": module_hash, "workload_hash": workload_hashes[name], + "deal_id": record["deal_id"], "verification": audit, + "dataset_commitment": froglet_verify.payload_hash(fixture_sites[name])} + require(evidence.stage(name, **arguments), "Contribution unexpectedly already staged") + require(evidence.stage(name, **arguments) is False, "Identical staged retry was not idempotent") + record["offline_verification"] = audit + + +def write_report(path, report): + if path is None: + return + path = Path(path) + encoded = json.dumps(report, indent=2, allow_nan=False) + "\n" + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(prefix="." + path.name + ".", dir=path.parent) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(encoded) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def source_hashes(): + paths = [Path(__file__), REPO / "examples/a2a_compute_demo.py", REPO / "examples/research_profile.py", + REPO / "python/froglet_fl_evidence.py", REPO / "python/requirements-immunopeptidomics.txt", + REPO / "examples/wasm-services/Cargo.toml", REPO / "examples/wasm-services/Cargo.lock"] + paths += sorted((REPO / "python/froglet-verify/froglet_verify").glob("*.py")) + paths += sorted((REPO / "examples/wasm-services/immunopeptidomics").rglob("*.rs")) + paths += [REPO / "examples/wasm-services/immunopeptidomics/Cargo.toml"] + return {str(path.relative_to(REPO)): sha256(path) for path in paths} + + +def run(binary, module_path, fixture_path=FIXTURE, output=None, keep_state=False, + execution_timeout_secs=30, unavailable_site=None): + require(type(execution_timeout_secs) is int and 1 <= execution_timeout_secs <= 300, + "Execution timeout must be an integer from 1 through 300") + nvflare_components() + fixture = load_fixture(fixture_path) + names, query = sorted(fixture["sites"]), fixture["query"] + require(unavailable_site is None or unavailable_site in names, "Unknown unavailable site") + module = Path(module_path).read_bytes() + require(module.startswith(b"\0asm\x01\0\0\0") and len(module) <= 1024 * 1024, + "Expected a Wasm v1 module no larger than 1 MiB") + module_hash = hashlib.sha256(module).hexdigest() + submission = build_submission(module, query) + workload_hashes = {name: froglet_verify.payload_hash(submission["workload"]) for name in names} + run_id = "immunopeptidomics-" + secrets.token_hex(12) + root = Path(tempfile.mkdtemp(prefix="froglet-immunopeptidomics-")).resolve() + root.chmod(0o700) + coordinator = runtime.Node(Path(binary), root, "coordinator", "runtime") + nodes = {name: runtime.Node(Path(binary), root, name, "provider") for name in names} + for node in nodes.values(): + node.env.update({"FROGLET_EXECUTION_TIMEOUT_SECS": str(execution_timeout_secs), + "FROGLET_PROVIDER_MAX_TOTAL_RUNTIME_MS": str(10 * execution_timeout_secs * 1000)}) + succeeded, evidence = False, None + report = {"schema_version": "froglet.immunopeptidomics.reference_run.v1", "run_id": run_id, + "status": "running", "query": query, "execution_timeout_secs": execution_timeout_secs, + "source_sha256": source_hashes(), "fixture_sha256": sha256(fixture_path), + "binary_sha256": sha256(binary), "module_sha256": module_hash, + "framework": {"name": "NVIDIA FLARE", "version": importlib.metadata.version("nvflare"), + "components": ["Executor", "DXO", "Shareable", "FLContext"], + "scope": "in-process FLARE components; live localhost Froglet providers"}, + "limitations": ["All fixtures are synthetic; no biological validation.", + "Recorded sample HLA genotype does not establish peptide restriction.", + "Denominators count synthetic specimens, not unique donors or technical replicates.", + "Non-detection is not biological absence; missing assays are excluded from the measured denominator.", + "Small aggregate counts are disclosed; no differential privacy or secure aggregation.", + "Hashes bind local files and signed reported values, not physical dataset identity.", + "Completeness state is in memory; no durable resume or distributed rollback."], + "workers": {}, "aggregate": None} + phase = "setup" + try: + requester_id = coordinator.identity() + identities = {name: node.identity() for name, node in nodes.items()} + require(len({requester_id, *identities.values()}) == len(names) + 1, "Node identities are not independent") + evidence = RoundEvidence(run_id=run_id, round_id=0, operation=OPERATION, + module_hash=module_hash, participants=identities, workload_hashes=workload_hashes) + provider_config, origins = [], {} + report["provider_fixture_hashes"] = {} + for name, node in nodes.items(): + database = provision_sqlite(node, fixture["sites"][name]) + report["provider_fixture_hashes"][name] = {"rows_jcs_sha256": froglet_verify.payload_hash(fixture["sites"][name]), + "sqlite_sha256": sha256(database)} + node.start() + origin = origins[name] = node.url + offer = next(item for item in runtime.get_json(origin + "/v1/provider/offers")["offers"] + if item["payload"]["offer_id"] == "execute.compute.host" + and item["payload"]["settlement_method"] == "none") + require(offer["payload"]["price_schedule"] == {"base_fee_msat": 0, "success_fee_msat": 0}, + "Reference requires a free local compute offer") + node.stop() + node.env["FROGLET_LISTEN_ADDR"] = origin.removeprefix("http://") + token = secrets.token_urlsafe(36) + config = node.data / "a2a.json" + runtime.private_json(config, {"clients": [{"requester_id": requester_id, "token": token, + "offer_hashes": [offer["hash"]]}]}) + node.env["FROGLET_A2A_CONFIG_PATH"] = str(config) + provider_config.append({"provider_url": origin, "token": token, "allow_loopback": True}) + config = coordinator.data / "a2a.json" + runtime.private_json(config, {"providers": provider_config}) + coordinator.env["FROGLET_A2A_CONFIG_PATH"] = str(config) + for name, node in nodes.items(): + node.start() + require(node.url == origins[name], "Provider origin changed on restart") + coordinator.start() + if unavailable_site: + nodes[unavailable_site].stop() + phase = "collect_site_counts" + + def worker(name, request): + require(request == query, "Worker query changed") + key = run_id + ":" + name + deal = submit(coordinator, identities[name], origins[name], submission, key) + result = deal["result"] + validate_result(result, query) + record = dict(artifact_bundle(origins[name], deal), result=result) + verify_artifact_chain_offline(record, result) + require(result == oracle(fixture["sites"][name], query), "Site result differs from independent oracle") + replay = submit(coordinator, identities[name], origins[name], submission, key) + require(all(replay[field] == deal[field] for field in ("deal_id", "deal", "quote", "receipt", "result_hash", "result")), + "Idempotent submission changed signed result") + record["idempotent_replay_same_deal"] = True + return record + + records, failures = dispatch_nvflare(names, query, worker) + report["workers"], report["worker_failures"] = records, failures + stage_records(evidence, records, failures, run_id=run_id, module_hash=module_hash, + workload_hashes=workload_hashes, fixture_sites=fixture["sites"]) + require(evidence.snapshot()["status"] == "ready", "Missing or failed site: aggregate not promoted") + phase = "verify_and_promote" + reduced = aggregate_results([records[name]["result"] for name in names], query) + pooled = {table: [row for name in names for row in fixture["sites"][name][table]] for table in TABLES} + require(reduced == oracle(pooled, query), "Pooled independent oracle disagrees with reduction") + evidence.promote(reduced) + report.update(status="succeeded", aggregate=reduced, detection_fraction=detection_fraction(reduced), + evidence=evidence.snapshot(), independent_oracle_matches=True) + write_report(output, report) + succeeded = True + return report + except BaseException as error: + report.update(status="failed", phase=phase, aggregate=None, + error={"type": type(error).__name__, "message": str(error)}, + evidence=evidence.snapshot() if evidence else None, retained_state_directory=str(root)) + write_report(output, report) + raise + finally: + coordinator.stop() + for node in nodes.values(): + node.stop() + if succeeded and not keep_state: + shutil.rmtree(root) + else: + print(f"Retained demo state directory: {root}", file=sys.stderr) + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--no-build", action="store_true") + parser.add_argument("--binary", type=Path) + parser.add_argument("--module", type=Path) + parser.add_argument("--fixture", type=Path, default=FIXTURE) + parser.add_argument("--output", type=Path) + parser.add_argument("--keep-state", action="store_true") + parser.add_argument("--execution-timeout-secs", type=int, default=30, + help="Provider budget including cold compilation, 1..300 seconds (default30)") + parser.add_argument("--unavailable-site", help="Stop this synthetic provider before dispatch to test incomplete output") + args = parser.parse_args(argv) + if bool(args.binary) != bool(args.module): + parser.error("--binary and --module must be supplied together") + if not 1 <= args.execution_timeout_secs <= 300: + parser.error("--execution-timeout-secs must be between 1 and 300") + return args + + +def main(argv=None): + args = parse_args(argv) + nvflare_components() + binary = args.binary.resolve() if args.binary else REPO / "target/debug/froglet-node" + module = args.module.resolve() if args.module else REPO / "examples/wasm-services/target/wasm32-unknown-unknown/release/immunopeptidomics.wasm" + if not args.no_build and not args.binary: + commands = [ + ["cargo", "build", "--locked", "-p", "froglet", "--bin", "froglet-node", "--target-dir", str(REPO / "target")], + ["cargo", "build", "--locked", "--release", "--target", "wasm32-unknown-unknown", "-p", "immunopeptidomics", + "--manifest-path", str(REPO / "examples/wasm-services/Cargo.toml"), + "--target-dir", str(REPO / "examples/wasm-services/target")]] + for command in commands: + subprocess.run(command, cwd=REPO, env=runtime.clean_environment(), check=True) + require(binary.is_file() and module.is_file(), "Missing node binary or Wasm module") + report = run(binary, module, args.fixture, args.output, args.keep_state, + args.execution_timeout_secs, args.unavailable_site) + print(json.dumps(report, indent=2, allow_nan=False)) + return 0 + + +if __name__ == "__main__": + def interrupted(_signum, _frame): + raise KeyboardInterrupt + signal.signal(signal.SIGTERM, interrupted) + try: + raise SystemExit(main()) + except KeyboardInterrupt: + print("Reference run interrupted; started nodes stopped.", file=sys.stderr) + raise SystemExit(130) + except (OSError, RuntimeError, ValueError, KeyError, StopIteration, subprocess.SubprocessError) as error: + print(f"Reference run failed: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/examples/test_immunopeptidomics_demo.py b/examples/test_immunopeptidomics_demo.py new file mode 100644 index 0000000..d899c46 --- /dev/null +++ b/examples/test_immunopeptidomics_demo.py @@ -0,0 +1,218 @@ +"""Reference semantics and failure gates; live provider checks are in the CLI. + +The optional FLARE test uses installed components with a stub worker; the live +CLI always requires FLARE and executes real signed provider-local Wasm deals. +""" +import copy +from contextlib import closing +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +from examples import immunopeptidomics_demo as demo +from python.froglet_fl_evidence import EvidenceError, RoundEvidence + + +class ScientificOracleTests(unittest.TestCase): + def setUp(self): + self.fixture = demo.load_fixture(demo.FIXTURE) + self.query = self.fixture["query"] + self.site = self.fixture["sites"]["site_a"] + + def test_hand_counted_fixture_and_weighted_denominator(self): + a = demo.oracle(self.site, self.query) + b = demo.oracle(self.fixture["sites"]["site_b"], self.query) + self.assertEqual(a["counts"], dict(zip(demo.COUNT_FIELDS, [8, 2, 1, 5, 3, 2, 1, 1, 1, 2]))) + self.assertEqual(b["counts"], dict(zip(demo.COUNT_FIELDS, [8, 1, 1, 6, 5, 3, 2, 1, 0, 1]))) + result = demo.aggregate_results([a, b], self.query) + self.assertEqual(result["counts"], dict(zip(demo.COUNT_FIELDS, [16, 3, 2, 11, 8, 5, 3, 2, 1, 3]))) + self.assertEqual(demo.detection_fraction(result), {"numerator": 5, "denominator": 8, "value": 0.625}) + self.assertNotEqual(0.625, (2 / 3 + 3 / 5) / 2) + + def test_duplicate_detections_and_hla_rows_do_not_multiply_samples(self): + changed = copy.deepcopy(self.site) + changed["sample_peptide_detections"] *= 3 + changed["sample_hla_typing"] *= 2 + self.assertEqual(demo.oracle(changed, self.query), demo.oracle(self.site, self.query)) + + def test_incomplete_nonmatching_hla_is_unknown_and_recorded_target_is_eligible(self): + # A3 has the target with incomplete typing. A7 lacks it with incomplete + # typing, whereas A6 lacks it with complete typing at the requested locus. + baseline = demo.oracle(self.site, self.query)["counts"] + changed = copy.deepcopy(self.site) + for row in changed["sample_hla_typing_status"]: + if row["sample_id"] == "A7": + row["status"] = "complete" + elif row["sample_id"] == "A3": + row["status"] = "missing" + actual = demo.oracle(changed, self.query)["counts"] + self.assertEqual(actual["eligible"], baseline["eligible"]) + self.assertEqual(actual["detected"], baseline["detected"]) + self.assertEqual(actual["hla_unknown"], baseline["hla_unknown"] - 1) + self.assertEqual(actual["hla_nonmatching"], baseline["hla_nonmatching"] + 1) + + def test_missing_typing_row_is_unknown_at_requested_locus(self): + changed = copy.deepcopy(self.site) + changed["sample_hla_typing_status"] = [row for row in changed["sample_hla_typing_status"] + if row["sample_id"] != "A6"] + changed["sample_hla_typing_status"].append({"sample_id": "A6", "locus": "HLA-B", "status": "complete"}) + actual = demo.oracle(changed, self.query)["counts"] + self.assertEqual(actual["hla_unknown"], 3) + self.assertEqual(actual["hla_nonmatching"], 0) + + def test_unmeasured_and_failed_observations_never_become_detected(self): + changed = copy.deepcopy(self.site) + for sample in ("A4", "A5"): + changed["sample_peptide_detections"].append({"sample_id": sample, "sequence": self.query["candidate_peptide"]}) + self.assertEqual(demo.oracle(changed, self.query), demo.oracle(self.site, self.query)) + for row in changed["samples"]: + if row["assay_status"] == "measured": + row["assay_status"] = "not_measured" + result = demo.oracle(changed, self.query) + self.assertEqual(result["counts"]["detected"], 0) + self.assertEqual(demo.detection_fraction(result), {"numerator": 0, "denominator": 0, "value": None}) + + def test_empty_cohort_and_exact_peptide_matching(self): + query = dict(self.query, organ="Synthetic_organ_a") + self.assertEqual(set(demo.oracle(self.site, query)["counts"].values()), {0}) + query = dict(self.query, candidate_peptide="ACDEFGHIL") + result = demo.oracle(self.site, query) + self.assertEqual(result["counts"]["measured"], 3) + self.assertEqual(result["counts"]["detected"], 0) + self.assertEqual(result["counts"]["not_detected"], 3) + + def test_malformed_inputs_and_duplicate_samples_rejected(self): + for query in ({}, dict(self.query, op="extra"), dict(self.query, hla_allele="A0201"), + dict(self.query, hla_allele="HLA-A*02:01:01"), + dict(self.query, candidate_peptide="ACDEFGHIK[mod]"), dict(self.query, organ=" ")): + with self.subTest(query=query), self.assertRaises(RuntimeError): + demo.oracle(self.site, query) + for table in ("samples", "sample_hla_typing_status"): + changed = copy.deepcopy(self.site) + changed[table].append(dict(changed[table][0])) + with self.subTest(table=table), self.assertRaises(RuntimeError): + demo.oracle(changed, self.query) + + def test_result_rejects_denominator_corruption_and_sample_level_fields(self): + for mutate in (lambda result: result["counts"].update(measured=99), + lambda result: result.update(sample_ids=["A1"]), + lambda result: result["counts"].update(detected=True), + lambda result: result.update(interpretation={})): + result = demo.oracle(self.site, self.query) + mutate(result) + with self.assertRaises(RuntimeError): + demo.validate_result(result, self.query) + + def test_sqlite_contains_only_own_site_rows_and_read_policy(self): + with tempfile.TemporaryDirectory() as folder: + node = SimpleNamespace(data=Path(folder), env={}) + database = demo.provision_sqlite(node, self.site) + with closing(sqlite3.connect(database)) as connection: + ids = {row[0] for row in connection.execute("SELECT sample_id FROM samples")} + self.assertEqual(ids, {"A" + str(i) for i in range(1, 11)}) + self.assertEqual(connection.execute("SELECT COUNT(*) FROM sample_peptide_detections WHERE sample_id='A1' AND sequence='ACDEFGHIK'").fetchone()[0], 2) + policy = Path(node.env["FROGLET_WASM_POLICY_PATH"]).read_text() + self.assertIn("[sqlite.handles.cohort]", policy) + self.assertIn("max_queries_per_execution = 1", policy) + self.assertNotIn("[http]", policy) + + def test_submission_commits_query_and_module_without_fixture_rows(self): + module = b"\0asm\x01\0\0\0" + submission = demo.build_submission(module, self.query) + self.assertEqual(submission["input"], self.query) + self.assertEqual(submission["workload"]["requested_capabilities"], [demo.CAPABILITY]) + changed = demo.build_submission(module, dict(self.query, organ="synthetic_organ_b")) + self.assertNotEqual(demo.froglet_verify.payload_hash(submission["workload"]), + demo.froglet_verify.payload_hash(changed["workload"])) + self.assertNotIn("samples", json.dumps(submission)) + + +class OfflineEvidenceTests(unittest.TestCase): + def setUp(self): + fixture = json.loads((demo.REPO / "conformance/kernel_v1.json").read_text()) + self.artifacts = {name: copy.deepcopy(fixture["artifacts"]["free_" + name if name != "descriptor" else name]["artifact"]) + for name in ("descriptor", "offer", "quote", "deal", "receipt")} + self.result = fixture["result"] + self.evidence = {"signed_artifacts": self.artifacts, "deal_id": self.artifacts["deal"]["hash"], + "result_hash": self.artifacts["receipt"]["payload"]["result_hash"]} + + def test_real_signed_free_conformance_chain_verifies(self): + audit = demo.verify_artifact_chain_offline(self.evidence, self.result) + self.assertTrue(audit["offline_chain_verified"]) + self.assertEqual(audit["artifact_count"], 5) + + def test_tampered_signature_and_result_fail(self): + changed = copy.deepcopy(self.evidence) + signature = changed["signed_artifacts"]["receipt"]["signature"] + changed["signed_artifacts"]["receipt"]["signature"] = ("1" if signature[0] == "0" else "0") + signature[1:] + with self.assertRaisesRegex(RuntimeError, "verification failed"): + demo.verify_artifact_chain_offline(changed, self.result) + with self.assertRaisesRegex(RuntimeError, "Result differs"): + demo.verify_artifact_chain_offline(self.evidence, {"tampered": True}) + + def test_missing_participant_blocks_promotion_and_retry_is_idempotent(self): + audit = demo.verify_artifact_chain_offline(self.evidence, self.result) + module_hash = audit["module_hash"] + workload = audit["workload_hash"] + evidence = RoundEvidence(run_id="reference-test", round_id=0, operation=demo.OPERATION, + module_hash=module_hash, participants={"site_a": audit["provider_id"], "site_b": "f" * 64}, + workload_hashes={"site_a": workload, "site_b": workload}) + record = dict(self.evidence, result=self.result) + demo.stage_records(evidence, {"site_a": record}, {"site_b": {"type": "ConnectionError", "message": "unavailable"}}, + run_id="reference-test", module_hash=module_hash, + workload_hashes={"site_a": workload, "site_b": workload}, fixture_sites={"site_a": {}}) + snapshot = evidence.snapshot() + self.assertEqual(len(snapshot["contributions"]), 1) + self.assertEqual(snapshot["missing_participants"], ["site_b"]) + self.assertIsNone(snapshot["output"]) + with self.assertRaises(EvidenceError): + evidence.promote({"counts": "incomplete"}) + + def test_atomic_report_and_cli_dependency_failure(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "report.json" + demo.write_report(path, {"status": "succeeded"}) + demo.write_report(path, {"status": "failed", "aggregate": None}) + self.assertEqual(json.loads(path.read_text()), {"status": "failed", "aggregate": None}) + self.assertEqual(list(path.parent.iterdir()), [path]) + with mock.patch.object(demo.importlib.metadata, "version", return_value="999"): + if importlib.util.find_spec("nvflare"): + with self.assertRaisesRegex(RuntimeError, "2.9.0"): + demo.nvflare_components() + + def test_cli_without_framework_dependency_fails_explicitly(self): + # -S excludes installed site-packages while allowing this reference's + # zero-dependency verifier and committed runtime helper to import. + result = subprocess.run([sys.executable, "-I", "-S", "-B", str(Path(demo.__file__)), "--no-build"], + capture_output=True, text=True, timeout=15) + self.assertEqual(result.returncode, 1) + self.assertIn("NVIDIA FLARE is required", result.stderr) + self.assertNotIn("Traceback", result.stderr) + + +@unittest.skipUnless(importlib.util.find_spec("nvflare"), "NVIDIA FLARE not installed; live CLI requires it") +class InstalledFlareTests(unittest.TestCase): + def test_two_executor_dxo_paths_and_site_failure_are_observable(self): + fixture = demo.load_fixture(demo.FIXTURE) + seen = [] + def worker(name, query): + seen.append(name) + if name == "site_b": + raise ConnectionError("synthetic unavailable worker") + return demo.oracle(fixture["sites"][name], query) + records, failures = demo.dispatch_nvflare(sorted(fixture["sites"]), fixture["query"], worker) + self.assertEqual(set(seen), {"site_a", "site_b"}) + self.assertEqual(set(records), {"site_a"}) + self.assertEqual(failures["site_b"]["type"], "ConnectionError") + self.assertEqual(records["site_a"]["counts"]["detected"], 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/wasm-services/Cargo.lock b/examples/wasm-services/Cargo.lock index cb27d8b..824dcee 100644 --- a/examples/wasm-services/Cargo.lock +++ b/examples/wasm-services/Cargo.lock @@ -10,6 +10,14 @@ version = "0.1.0" name = "fibonacci" version = "0.1.0" +[[package]] +name = "immunopeptidomics" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "itoa" version = "1.0.18" diff --git a/examples/wasm-services/Cargo.toml b/examples/wasm-services/Cargo.toml index 0ccbf38..7be8c25 100644 --- a/examples/wasm-services/Cargo.toml +++ b/examples/wasm-services/Cargo.toml @@ -1,7 +1,7 @@ # Sample froglet.wasm.run_json.v1 services. A standalone workspace with a release profile tuned for small WebAssembly # modules; the site build compiles it (see docs-site/scripts/build-playground.mjs), and the playground publishes them. [workspace] -members = ["adder", "fibonacci", "ontology-check"] +members = ["adder", "fibonacci", "ontology-check", "immunopeptidomics"] resolver = "2" [profile.release] diff --git a/examples/wasm-services/immunopeptidomics/Cargo.toml b/examples/wasm-services/immunopeptidomics/Cargo.toml new file mode 100644 index 0000000..2bd6113 --- /dev/null +++ b/examples/wasm-services/immunopeptidomics/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "immunopeptidomics" +version = "0.1.0" +edition = "2021" +publish = false +license = "Apache-2.0" +description = "Synthetic provider-local peptide detection counts through the Froglet host JSON ABI" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +serde = { version = "=1.0.228", features = ["derive"] } +serde_json = "=1.0.149" diff --git a/examples/wasm-services/immunopeptidomics/README.md b/examples/wasm-services/immunopeptidomics/README.md new file mode 100644 index 0000000..a610b2c --- /dev/null +++ b/examples/wasm-services/immunopeptidomics/README.md @@ -0,0 +1,94 @@ +# Synthetic immunopeptidomics counts + +This small `froglet.wasm.host_json.v1` module reads the provider's local SQLite +handle `cohort`. Its sole requested capability is +`db.sqlite.query.read.cohort`. It is independent of `fl-compute` and requires no +GPU, model training, or protocol changes. + +The input has exactly four fields, for example: + +```json +{ + "candidate_peptide": "ACDEFGHIK", + "hla_allele": "HLA-A*02:01", + "sample_class": "healthy_tissue", + "organ": "synthetic_organ_a" +} +``` + +The supported query is deliberately narrow: an uppercase, unmodified peptide +using the twenty standard amino-acid letters and a two-field HLA-A, HLA-B, or +HLA-C allele (two or three digits in each field). Matching is exact. The module +does not normalize identifiers, infer HLA restriction, translate allele +resolution, remove allele suffixes, or interpret modified peptides. + +The four provider-local tables are defined by the +[fixture schema](../../fixtures/immunopeptidomics/schema.sql). They +store samples, recorded HLA alleles, typing completeness by locus, and peptide +detections. The fixture and schema contain only synthetic examples. A measured +assay means that the assay completed, passed the fixture's inclusion rules, +and its accepted observation set is available. A missing observation export +must not be labeled measured. This does not imply exhaustive detection of +presented peptides. + +For samples matching the requested sample class and organ: + +- A recorded requested allele makes the sample eligible, even when typing is + incomplete or its completeness assessment is missing. +- Without that allele, complete typing at its locus means nonmatching; + incomplete or missing typing means unknown. Missing completeness assessment + does not mean no alleles were recorded. +- Only measured eligible samples form the detection denominator. Each sample + contributes at most once, regardless of duplicate peptide observations. +- Failed and not-measured eligible samples remain missing. Their partial or + stale detection rows cannot turn them into measured positives or negatives. + +The result has `schema_version: "froglet.immunopeptidomics.counts.v1"`, the +original `query`, an `interpretation` object, and these integer `counts`: + +```text +cohort_total, hla_unknown, hla_nonmatching, eligible, measured, +detected, not_detected, not_measured, failed, missing +``` + +The identities are `cohort_total = hla_unknown + hla_nonmatching + eligible`, +`eligible = measured + missing`, `measured = detected + not_detected`, and +`missing = not_measured + failed`. A presenter may compute `detected / measured`; +when measured is zero, the rate is undefined, not zero. These are sample counts, +not patient prevalence. Pooling requires disjoint sample sets; this module +does not deduplicate across sites. + +The interpretation strings are: + +```json +{ + "hla_selection": "recorded_genotype_not_peptide_restriction", + "not_detected": "not_detected_in_measured_assay_not_biological_absence", + "denominator": "measured_eligible_samples" +} +``` + +Only counts and query metadata leave the guest. No sample identifiers or local +observation records are returned. Aggregate counts are not, by themselves, a +privacy guarantee; the example does not implement disclosure thresholds or +differential privacy. + +The fixed SQL uses parameters and existence predicates. Duplicate sample IDs, +duplicate completeness rows, malformed host responses, and invalid statuses +produce `invalid_local_data`. Invalid requests produce `invalid_request` before +any host call. A host policy/query error traps in the runtime; it must be treated +as a failed execution, never a successful zero-count result. The SQLite host +rejects excessive row counts or output bytes rather than returning partial rows. + +From the repository root: + +```sh +cargo test --locked -p immunopeptidomics --manifest-path examples/wasm-services/Cargo.toml +cargo build --locked --release --target wasm32-unknown-unknown -p immunopeptidomics --manifest-path examples/wasm-services/Cargo.toml +``` + +The Wasm artifact is +`examples/wasm-services/target/wasm32-unknown-unknown/release/immunopeptidomics.wasm`. +Native tests exercise aggregation, missingness, strict validation, host request +shape, and record exclusion from outputs. They do not replace a live signed +execution against the provider runtime. diff --git a/examples/wasm-services/immunopeptidomics/src/lib.rs b/examples/wasm-services/immunopeptidomics/src/lib.rs new file mode 100644 index 0000000..9083a09 --- /dev/null +++ b/examples/wasm-services/immunopeptidomics/src/lib.rs @@ -0,0 +1,397 @@ +//! Narrow synthetic immunopeptidomics example using provider-local SQLite. +//! +//! A recorded genotype selects a sample stratum; it does not establish peptide +//! restriction. Absence from a measured assay is not evidence of biological +//! absence. Only counts and requester-supplied query metadata are returned. + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::collections::BTreeSet; + +const SCHEMA: &str = "froglet.immunopeptidomics.counts.v1"; +const COLUMNS: [&str; 6] = [ + "sample_id", + "assay_status", + "has_requested_allele", + "typing_status", + "typing_status_rows", + "has_exact_peptide", +]; + +// One row per local sample, with existence checks rather than observation joins. +// Duplicate detections or typing observations cannot multiply the denominator. +// Every comparison is explicitly binary: this example does no normalization. +const SQL: &str = "SELECT s.sample_id AS sample_id, s.assay_status AS assay_status, \ +EXISTS(SELECT 1 FROM sample_hla_typing h WHERE h.sample_id = s.sample_id COLLATE BINARY AND h.allele_name = ?2 COLLATE BINARY) AS has_requested_allele, \ +(SELECT CASE WHEN COUNT(*) = 0 THEN 'missing' ELSE MIN(t.status) END FROM sample_hla_typing_status t WHERE t.sample_id = s.sample_id COLLATE BINARY AND t.locus = ?5 COLLATE BINARY) AS typing_status, \ +(SELECT COUNT(*) FROM sample_hla_typing_status t WHERE t.sample_id = s.sample_id COLLATE BINARY AND t.locus = ?5 COLLATE BINARY) AS typing_status_rows, \ +EXISTS(SELECT 1 FROM sample_peptide_detections p WHERE p.sample_id = s.sample_id COLLATE BINARY AND p.sequence = ?1 COLLATE BINARY) AS has_exact_peptide \ +FROM samples s WHERE s.sample_class = ?3 COLLATE BINARY AND s.organ = ?4 COLLATE BINARY ORDER BY s.sample_id COLLATE BINARY"; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Query { + candidate_peptide: String, + hla_allele: String, + sample_class: String, + organ: String, +} + +impl Query { + fn locus(&self) -> Option<&str> { + let (locus, allele) = self.hla_allele.split_once('*')?; + if !matches!(locus, "HLA-A" | "HLA-B" | "HLA-C") { + return None; + } + let fields = allele.split(':').collect::>(); + if fields.len() != 2 + || fields.iter().any(|field| { + !(2..=3).contains(&field.len()) || !field.bytes().all(|b| b.is_ascii_digit()) + }) + { + return None; + } + Some(locus) + } + + fn valid(&self) -> bool { + !self.candidate_peptide.is_empty() + && self.candidate_peptide.len() <= 128 + && self + .candidate_peptide + .bytes() + .all(|b| b"ACDEFGHIKLMNPQRSTVWY".contains(&b)) + && self.locus().is_some() + && [&self.sample_class, &self.organ] + .iter() + .all(|value| !value.trim().is_empty() && value.len() <= 128) + } +} + +#[derive(Debug, Default, Serialize, PartialEq, Eq)] +struct Counts { + cohort_total: u64, + hla_unknown: u64, + hla_nonmatching: u64, + eligible: u64, + measured: u64, + detected: u64, + not_detected: u64, + not_measured: u64, + failed: u64, + missing: u64, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct HostRows { + columns: Vec, + rows: Vec>, +} + +fn flag(value: &Value) -> Result { + match value.as_u64() { + Some(0) => Ok(false), + Some(1) => Ok(true), + _ => Err("invalid_local_data"), + } +} + +fn aggregate(response: &[u8]) -> Result { + let data: HostRows = serde_json::from_slice(response).map_err(|_| "invalid_local_data")?; + if data.columns != COLUMNS { + return Err("invalid_local_data"); + } + let mut counts = Counts::default(); + let mut ids = BTreeSet::new(); + for row in &data.rows { + if row.len() != COLUMNS.len() { + return Err("invalid_local_data"); + } + let id = row[0].as_str().ok_or("invalid_local_data")?; + if id.is_empty() || !ids.insert(id) { + return Err("invalid_local_data"); + } + let assay = row[1].as_str().ok_or("invalid_local_data")?; + if !matches!(assay, "measured" | "not_measured" | "failed") { + return Err("invalid_local_data"); + } + let has_allele = flag(&row[2])?; + let typing_status = row[3].as_str().ok_or("invalid_local_data")?; + if !matches!(typing_status, "complete" | "incomplete" | "missing") { + return Err("invalid_local_data"); + } + let typing_rows = row[4].as_u64().ok_or("invalid_local_data")?; + if typing_rows > 1 || (typing_rows == 0 && typing_status != "missing") { + return Err("invalid_local_data"); + } + let detected = flag(&row[5])?; + counts.cohort_total += 1; + if !has_allele { + if typing_status == "complete" { + counts.hla_nonmatching += 1; + } else { + counts.hla_unknown += 1; + } + continue; + } + counts.eligible += 1; + match assay { + "measured" => { + counts.measured += 1; + if detected { + counts.detected += 1; + } else { + counts.not_detected += 1; + } + } + "not_measured" => counts.not_measured += 1, + "failed" => counts.failed += 1, + _ => unreachable!(), + } + } + counts.missing = counts.not_measured + counts.failed; + Ok(counts) +} + +fn respond_with_host(request: &[u8], host: F) -> String +where + F: FnOnce(&[u8]) -> Result, &'static str>, +{ + // Derived structs accept positional arrays; require an object first. + if !matches!( + serde_json::from_slice::(request), + Ok(Value::Object(_)) + ) { + return json!({"error": "invalid_request"}).to_string(); + } + let query = match serde_json::from_slice::(request) { + Ok(query) if query.valid() => query, + _ => return json!({"error": "invalid_request"}).to_string(), + }; + let envelope = json!({ + "op": "db.query", + "request": { + "handle": "cohort", + "sql": SQL, + "params": [query.candidate_peptide, query.hla_allele, query.sample_class, query.organ, query.locus().expect("validated locus")], + }, + }); + let request_bytes = serde_json::to_vec(&envelope).expect("host request is JSON data"); + let result = host(&request_bytes).and_then(|bytes| aggregate(&bytes)); + match result { + Ok(counts) => json!({ + "schema_version": SCHEMA, + "query": query, + "counts": counts, + "interpretation": { + "hla_selection": "recorded_genotype_not_peptide_restriction", + "not_detected": "not_detected_in_measured_assay_not_biological_absence", + "denominator": "measured_eligible_samples", + }, + }) + .to_string(), + Err(code) => json!({"error": code}).to_string(), + } +} + +#[cfg(target_arch = "wasm32")] +#[link(wasm_import_module = "froglet_host")] +extern "C" { + fn call_json(pointer: i32, len: i32) -> i64; +} + +#[cfg(target_arch = "wasm32")] +fn invoke_host(request: &[u8]) -> Result, &'static str> { + let len = i32::try_from(request.len()).map_err(|_| "host_query_failed")?; + let packed = unsafe { call_json(request.as_ptr() as i32, len) } as u64; + let pointer = (packed >> 32) as u32 as *const u8; + let len = (packed & 0xffff_ffff) as u32 as usize; + if pointer.is_null() || len == 0 { + return Err("host_query_failed"); + } + Ok(unsafe { std::slice::from_raw_parts(pointer, len) }.to_vec()) +} + +#[cfg(not(target_arch = "wasm32"))] +fn invoke_host(_request: &[u8]) -> Result, &'static str> { + Err("host_query_failed") +} + +/// Evaluate the request; native execution deliberately cannot access a database. +pub fn respond(request: &[u8]) -> String { + respond_with_host(request, invoke_host) +} + +#[cfg(target_arch = "wasm32")] +#[no_mangle] +pub extern "C" fn alloc(len: i32) -> i32 { + let mut buffer = Vec::::with_capacity(len as u32 as usize); + let pointer = buffer.as_mut_ptr(); + std::mem::forget(buffer); + pointer as i32 +} + +#[cfg(target_arch = "wasm32")] +#[no_mangle] +pub extern "C" fn run(pointer: i32, len: i32) -> i64 { + let request = + unsafe { std::slice::from_raw_parts(pointer as u32 as *const u8, len as u32 as usize) }; + let bytes = respond(request).into_bytes(); + let packed = ((bytes.as_ptr() as u32 as i64) << 32) | bytes.len() as i64; + std::mem::forget(bytes); + packed +} + +#[cfg(test)] +mod tests { + use super::*; + + const REQUEST: &[u8] = br#"{"candidate_peptide":"ACDEFGHIK","hla_allele":"HLA-A*02:01","sample_class":"healthy_tissue","organ":"synthetic_organ_a"}"#; + + fn response(rows: Vec) -> Vec { + serde_json::to_vec(&json!({"columns": COLUMNS, "rows": rows})).unwrap() + } + + #[test] + fn denominator_and_missingness_have_distinct_meanings() { + let rows = vec![ + json!(["d", "measured", 1, "complete", 1, 1]), + json!(["n", "measured", 1, "complete", 1, 0]), + json!(["u", "not_measured", 1, "complete", 1, 1]), + json!(["f", "failed", 1, "complete", 1, 1]), + json!(["x", "measured", 0, "complete", 1, 1]), + json!(["m", "measured", 0, "missing", 0, 1]), + json!(["i", "measured", 0, "incomplete", 1, 1]), + ]; + assert_eq!( + aggregate(&response(rows)).unwrap(), + Counts { + cohort_total: 7, + hla_unknown: 2, + hla_nonmatching: 1, + eligible: 4, + measured: 2, + detected: 1, + not_detected: 1, + not_measured: 1, + failed: 1, + missing: 2, + } + ); + } + + #[test] + fn recorded_allele_is_sufficient_despite_incomplete_or_missing_typing() { + let counts = aggregate(&response(vec![ + json!(["a", "measured", 1, "incomplete", 1, 1]), + json!(["b", "measured", 1, "missing", 0, 0]), + ])) + .unwrap(); + assert_eq!(counts.eligible, 2); + assert_eq!(counts.measured, 2); + assert_eq!(counts.detected, 1); + assert_eq!(counts.hla_unknown, 0); + } + + #[test] + fn empty_cohort_is_valid_and_zero_denominator_is_explicit() { + assert_eq!(aggregate(&response(vec![])).unwrap(), Counts::default()); + let counts = + aggregate(&response(vec![json!(["a", "failed", 1, "complete", 1, 0])])).unwrap(); + assert_eq!(counts.measured, 0); + assert_eq!(counts.missing, 1); + } + + #[test] + fn malformed_local_facts_fail_closed_without_echoing_records() { + let valid = json!(["private-sample", "measured", 1, "complete", 1, 1]); + for rows in [ + vec![valid.clone(), valid.clone()], + vec![json!(["", "measured", 1, "complete", 1, 1])], + vec![json!(["private-sample", "unknown", 1, "complete", 1, 1])], + vec![json!([ + "private-sample", + "measured", + true, + "complete", + 1, + 1 + ])], + vec![json!(["private-sample", "measured", 1, "unknown", 1, 1])], + vec![json!(["private-sample", "measured", 1, "complete", 2, 1])], + vec![json!(["private-sample", "measured", 1, "complete", 0, 1])], + vec![json!(["private-sample", "measured", 1, "complete", 1, 2])], + ] { + let text = respond_with_host(REQUEST, |_| Ok(response(rows))); + assert_eq!(text, r#"{"error":"invalid_local_data"}"#); + assert!(!text.contains("private-sample")); + } + for bytes in [ + br#"{"columns":[],"rows":[]}"#.to_vec(), + br#"{"columns":[],"rows":[],"truncated":true}"#.to_vec(), + b"not JSON".to_vec(), + ] { + assert!(aggregate(&bytes).is_err()); + } + } + + #[test] + fn request_is_bound_to_fixed_sql_and_output_contains_only_counts_and_metadata() { + let output = respond_with_host(REQUEST, |request| { + let value: Value = serde_json::from_slice(request).unwrap(); + assert_eq!(value["op"], "db.query"); + assert_eq!(value["request"]["handle"], "cohort"); + assert_eq!(value["request"]["sql"], SQL); + assert_eq!( + value["request"]["params"], + json!([ + "ACDEFGHIK", + "HLA-A*02:01", + "healthy_tissue", + "synthetic_organ_a", + "HLA-A" + ]) + ); + Ok(response(vec![json!([ + "private-sample", + "measured", + 1, + "complete", + 1, + 1 + ])])) + }); + let value: Value = serde_json::from_str(&output).unwrap(); + assert_eq!(value["counts"]["detected"], 1); + assert_eq!( + value["interpretation"]["denominator"], + "measured_eligible_samples" + ); + assert!(!output.contains("private-sample")); + assert_eq!(value.as_object().unwrap().len(), 4); + } + + #[test] + fn malformed_requests_never_call_host() { + for input in [ + b"null".as_slice(), b"[]".as_slice(), b"{}".as_slice(), + br#"{"candidate_peptide":"ACDEFGHIK","hla_allele":"HLA-A*02:01","sample_class":"healthy_tissue","organ":"a","rows":[]}"#, + br#"{"candidate_peptide":"ACDEFGHIK","candidate_peptide":"AAAA","hla_allele":"HLA-A*02:01","sample_class":"healthy_tissue","organ":"a"}"#, + br#"{"candidate_peptide":"acdefghik","hla_allele":"HLA-A*02:01","sample_class":"healthy_tissue","organ":"a"}"#, + br#"{"candidate_peptide":"ACDEFGHIK","hla_allele":"A0201","sample_class":"healthy_tissue","organ":"a"}"#, + br#"{"candidate_peptide":"ACDEFGHIK","hla_allele":"HLA-A*02","sample_class":"healthy_tissue","organ":"a"}"#, + ] { + assert_eq!(respond_with_host(input, |_| panic!("must not access host")), r#"{"error":"invalid_request"}"#); + } + } + + #[test] + fn host_failures_do_not_become_zero_counts() { + assert_eq!( + respond_with_host(REQUEST, |_| Err("host_query_failed")), + r#"{"error":"host_query_failed"}"# + ); + assert_eq!(respond(REQUEST), r#"{"error":"host_query_failed"}"#); + } +} diff --git a/integrations/openclaw/froglet/scripts/doctor.mjs b/integrations/openclaw/froglet/scripts/doctor.mjs index 0feb7e1..907fc8f 100644 --- a/integrations/openclaw/froglet/scripts/doctor.mjs +++ b/integrations/openclaw/froglet/scripts/doctor.mjs @@ -1,4 +1,4 @@ -import { access, readFile } from "node:fs/promises" +import { access, readFile, stat } from "node:fs/promises" import { constants as fsConstants } from "node:fs" import path from "node:path" import process from "node:process" @@ -85,6 +85,40 @@ async function pathReadable(targetPath) { } } +export async function checkPluginSource(pluginPath) { + try { + if (!(await stat(pluginPath)).isDirectory()) { + return buildCheck("plugin_source", "error", `Plugin path is not a directory: ${pluginPath}`) + } + } catch { + return buildCheck("plugin_source", "error", `Plugin directory is missing or unreadable: ${pluginPath}`) + } + + const sharedPath = path.resolve(pluginPath, "../../shared/froglet-lib") + const requiredFiles = [ + ...["index.js", "package.json", "openclaw.plugin.json", "lib/config.js", "lib/shared.js", "lib/froglet-tool.js"] + .map((file) => path.join(pluginPath, file)), + ...["package.json", "shared.js", "tool-contract.js", "tool-dispatch.js", "froglet-client.js", "summarize.js", "marketplace-publish.js", "url-safety.js"] + .map((file) => path.join(sharedPath, file)) + ] + const missingFiles = (await Promise.all(requiredFiles.map(async (file) => { + try { + return (await stat(file)).isFile() && await pathReadable(file) ? null : file + } catch { + return file + } + }))).filter((file) => file !== null) + + return buildCheck( + "plugin_source", + missingFiles.length === 0 ? "ok" : "error", + missingFiles.length === 0 + ? `Plugin files and shared dependencies are present: ${pluginPath}` + : `Plugin staging is incomplete: ${missingFiles.length} required file(s) missing or unreadable`, + { plugin_path: pluginPath, shared_path: sharedPath, missing_files: missingFiles } + ) +} + function getPluginPath(document) { const paths = document?.plugins?.load?.paths if (!Array.isArray(paths) || paths.length === 0) { @@ -144,7 +178,11 @@ export async function checkApis( } } -function manualCommands(target, config) { +function shellQuote(value) { + return `'${String(value).replace(/'/g, "'\\''")}'` +} + +function manualCommands(target, config, pluginPath) { const providerUrl = config.providerUrl const runtimeUrl = config.runtimeUrl const tokenPath = config.providerAuthTokenPath @@ -152,8 +190,9 @@ function manualCommands(target, config) { return [ "nemoclaw status", "openshell sandbox get ", - "openshell sandbox upload /absolute/path/to/froglet /sandbox/froglet/integrations/openclaw/froglet", - `openshell sandbox upload /absolute/path/to/froglet-control.token ${tokenPath}`, + `openshell sandbox upload /absolute/path/to/froglet/integrations/openclaw/froglet ${shellQuote(path.dirname(pluginPath))}`, + `openshell sandbox upload /absolute/path/to/froglet/integrations/shared/froglet-lib ${shellQuote(path.resolve(pluginPath, "../../shared"))}`, + `openshell sandbox upload /absolute/path/to/froglet-control.token ${shellQuote(tokenPath)}`, "nemoclaw connect", `TOKEN=$(cat ${tokenPath})`, `curl -H "Authorization: Bearer $TOKEN" ${providerUrl}/health`, @@ -218,16 +257,7 @@ export async function main() { const checks = [] checks.push(buildCheck("config", "ok", "plugin config parsed", { hostProduct: pluginConfig.hostProduct })) - const pluginExists = await pathReadable(pluginPath) - checks.push( - buildCheck( - "plugin_source", - pluginExists ? "ok" : "error", - pluginExists - ? `Plugin path exists: ${pluginPath}` - : `Plugin path is missing: ${pluginPath}` - ) - ) + checks.push(await checkPluginSource(pluginPath)) const tokenReadable = await pathReadable(pluginConfig.providerAuthTokenPath) const runtimeTokenReadable = await pathReadable(pluginConfig.runtimeAuthTokenPath) @@ -259,22 +289,26 @@ export async function main() { "runtime_health", "warning", "Runtime check skipped", - { manual_commands: manualCommands(target, pluginConfig) } + { manual_commands: manualCommands(target, pluginConfig, pluginPath) } ) ) } + const status = overallStatus(checks) process.stdout.write( `${JSON.stringify( { target, - overall_status: overallStatus(checks), + overall_status: status, checks }, null, 2 )}\n` ) + if (status === "error") { + process.exitCode = 1 + } } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { diff --git a/integrations/openclaw/froglet/scripts/openshell-qualification-backend.py b/integrations/openclaw/froglet/scripts/openshell-qualification-backend.py new file mode 100644 index 0000000..1c2e6f3 --- /dev/null +++ b/integrations/openclaw/froglet/scripts/openshell-qualification-backend.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Prepare a disposable real Froglet/TLS endpoint for the documented macOS test. + +Requires the pinned OpenShell 0.1.7 CLI and gateway on PATH, Docker Desktop, +OpenSSL, and a built Froglet node. No inference provider or production token is +used. This helper does not start OpenShell, install software, or change Docker +settings. See docs/OPENSHELL_QUALIFICATION.md for the complete test and cleanup. +""" +from __future__ import annotations + +import argparse +import atexit +import http.server +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import signal +import ssl +import subprocess +import sys +import time +import urllib.error +import urllib.request + + +def wait_for_provider(node, timeout=30): + """Wait independently for the dual-role provider listener and healthy API.""" + deadline = time.monotonic() + timeout + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + while time.monotonic() < deadline: + if node.process.poll() is not None: + raise RuntimeError(f"Froglet exited before provider readiness; inspect {node.log}") + match = re.search(r"Local API Gateway: (http://127\.0\.0\.1:\d+)", + node.log.read_text(encoding="utf-8", errors="replace")) + if match: + provider = match.group(1) + try: + with opener.open(provider + "/health", timeout=2) as response: + if json.load(response).get("status") == "ok": + return provider + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError): + pass + time.sleep(0.1) + raise RuntimeError(f"Froglet provider readiness timed out; inspect {node.log}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work-dir", type=Path, required=True) + parser.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[4]) + parser.add_argument("--binary", type=Path) + args = parser.parse_args() + if sys.platform != "darwin": + parser.error("This recipe is qualified on macOS/Docker Desktop only; Linux is not yet verified") + root = args.work_dir.resolve() + root.mkdir(mode=0o700, parents=True, exist_ok=True) + root.chmod(0o700) + if (root / "tls").exists() or (root / "test-node").exists(): + parser.error("Use a new work directory; existing private state is never overwritten") + repo = args.repo.resolve() + binary = (args.binary or repo / "target/release/froglet-node").resolve() + if not binary.is_file(): + parser.error("Build froglet-node first or supply --binary") + for command, expected in [("openshell", "openshell 0.1.7"), + ("openshell-gateway", "openshell-gateway 0.1.7")]: + version = subprocess.check_output([command, "--version"], text=True).strip() + if version != expected: + parser.error(f"Expected {expected}; found {version}") + docker_host = subprocess.check_output( + ["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], + text=True).strip() + if os.environ.get("DOCKER_HOST"): + docker_host = os.environ["DOCKER_HOST"] + if not docker_host.startswith("unix://"): + parser.error("Only a local Unix Docker socket is accepted; remote engines are outside this test") + subprocess.run(["docker", "info", "--format", "{{.ServerVersion}}"], check=True, + stdout=subprocess.DEVNULL) + for directory in ["tls", "config", "state", "data", "stage", "evidence", "image"]: + (root / directory).mkdir(mode=0o700) + environment = {key: value for key, value in os.environ.items() + if not key.startswith("OPENSHELL_")} + environment.update({ + "XDG_CONFIG_HOME": str(root / "config"), + "XDG_STATE_HOME": str(root / "state"), + "XDG_DATA_HOME": str(root / "data"), + "OPENSHELL_LOCAL_TLS_DIR": str(root / "tls/gateway"), + }) + (root / "env.sh").write_text("\n".join( + f"export {key}={shlex.quote(environment[key])}" + for key in ["XDG_CONFIG_HOME", "XDG_STATE_HOME", "XDG_DATA_HOME", "OPENSHELL_LOCAL_TLS_DIR"] + ) + "\n") + + def checked(argv): + subprocess.run(argv, check=True, env=environment, stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE) + + checked(["openshell-gateway", "generate-certs", "--output-dir", str(root / "tls/gateway"), + "--server-san", "host.docker.internal", "--server-san", "localhost", + "--server-san", "127.0.0.1"]) + (root / "config/ca.cnf").write_text("""[req] +distinguished_name=dn +x509_extensions=extensions +prompt=no +[dn] +CN=Froglet disposable qualification CA +[extensions] +basicConstraints=critical,CA:TRUE +keyUsage=critical,digitalSignature,keyCertSign +""") + (root / "config/leaf.cnf").write_text("""basicConstraints=critical,CA:FALSE +keyUsage=critical,digitalSignature,keyEncipherment +extendedKeyUsage=serverAuth +subjectAltName=DNS:host.openshell.internal,DNS:host.docker.internal,DNS:localhost,IP:127.0.0.1 +""") + checked(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "2", + "-config", str(root / "config/ca.cnf"), "-keyout", str(root / "tls/backend-ca.key"), + "-out", str(root / "tls/backend-ca.crt")]) + checked(["openssl", "req", "-new", "-newkey", "rsa:2048", "-nodes", + "-subj", "/CN=Froglet qualification endpoint", "-keyout", str(root / "tls/backend.key"), + "-out", str(root / "tls/backend.csr")]) + checked(["openssl", "x509", "-req", "-in", str(root / "tls/backend.csr"), + "-CA", str(root / "tls/backend-ca.crt"), "-CAkey", str(root / "tls/backend-ca.key"), + "-CAcreateserial", "-out", str(root / "tls/backend.crt"), "-days", "2", + "-extfile", str(root / "config/leaf.cnf")]) + for key in (root / "tls").glob("*.key"): + key.chmod(0o600) + # json.dumps quotes paths as TOML basic strings without shell interpolation. + quote = lambda path: json.dumps(str(path)) + gateway_tls = root / "tls/gateway" + (root / "config/gateway.toml").write_text(f"""[openshell] +version = 2 +[openshell.gateway] +name = "froglet-qualification" +bind_address = "127.0.0.1:18670" +compute_driver = "docker" +guest_tls_ca = {quote(gateway_tls / 'ca.crt')} +[openshell.gateway.tls] +cert_path = {quote(gateway_tls / 'server/tls.crt')} +key_path = {quote(gateway_tls / 'server/tls.key')} +client_ca_path = {quote(gateway_tls / 'ca.crt')} +[openshell.gateway.gateway_jwt] +signing_key_path = {quote(gateway_tls / 'jwt/signing.pem')} +public_key_path = {quote(gateway_tls / 'jwt/public.pem')} +kid_path = {quote(gateway_tls / 'jwt/kid')} +gateway_id = "froglet-qualification" +[openshell.gateway.auth] +allow_unauthenticated_users = false +[openshell.gateway.mtls_auth] +enabled = true +[openshell.supervisor.network] +additional_ca_bundle = {quote(root / 'tls/backend-ca.crt')} +[openshell.drivers.docker] +socket_path = {quote(docker_host.removeprefix('unix://'))} +image_pull_policy = "if_not_present" +sandbox_label = "froglet-qualification" +supervisor_network_mode = "bridge" +grpc_endpoint = "https://host.docker.internal:18670" +enable_bind_mounts = false +sandbox_pids_limit = 2048 +""") + checked(["openshell-gateway", "config", "preflight", "--path", str(root / "config/gateway.toml")]) + (root / "image/Dockerfile").write_text("""FROM node:22.14.0-bookworm-slim@sha256:1c18d9ab3af4585870b92e4dbc5cac5a0dc77dd13df1a5905cea89fc720eb05b +WORKDIR /sandbox +RUN chown node:node /sandbox +USER node +CMD ["sleep", "infinity"] +""") + policy = { + "version": 1, + "filesystem_policy": {"include_workdir": True, + "read_only": ["/usr", "/lib", "/etc", "/proc", "/dev/urandom", "/var/log"], + "read_write": ["/tmp", "/dev/null"]}, + "landlock": {"compatibility": "hard_requirement"}, + "process": {"run_as_user": "1000", "run_as_group": "1000"}, + "network_policies": {"froglet_read_qualification": { + "endpoints": [{"host": "host.openshell.internal", "port": 19443, + "protocol": "rest", "enforcement": "enforce", "rules": [ + {"allow": {"method": "GET", "path": "/v1/provider/services"}}, + {"allow": {"method": "GET", "path": "/v1/runtime/spend"}}]}], + "binaries": [{"path": "/usr/local/bin/node"}]}} + } + for mode in ["baseline", "restricted"]: + (root / f"config/policy-{mode}.yaml").write_text(json.dumps(policy, indent=2) + "\n") + policy["network_policies"]["froglet_read_qualification"]["endpoints"][0]["rules"] = [ + {"allow": {"method": "GET", "path": "/v1/provider/services"}}] + sys.path.insert(0, str(repo / "examples")) + from a2a_compute_demo import Node + node = Node(binary, root, "test-node", "dual") + atexit.register(node.stop) + node.identity() + node.start() + provider = wait_for_provider(node) + runtime = node.url + for source, target in [(node.data / "runtime/froglet-control.token", "provider.token"), + (node.data / "runtime/auth.token", "runtime.token")]: + shutil.copyfile(source, root / "stage" / target) + (root / "stage" / target).chmod(0o600) + audit = root / "evidence/backend-access.jsonl" + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + upstream = runtime if self.path.startswith("/v1/runtime/") else provider + request = urllib.request.Request(upstream + self.path, + headers={"Authorization": self.headers.get("Authorization", "")}) + try: + response = opener.open(request, timeout=15) + except urllib.error.HTTPError as error: + response = error + payload = response.read() + with audit.open("a") as output: + output.write(json.dumps({"method": "GET", "path": self.path, + "upstream_status": response.status}) + "\n") + self.send_response(response.status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 19443), Handler) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(root / "tls/backend.crt", root / "tls/backend.key") + server.socket = context.wrap_socket(server.socket, server_side=True) + (root / "ready.json").write_text(json.dumps({ + "provider": provider, "runtime": runtime, "binary": str(binary), + "api_kind": "real_froglet_node", "credentials": "disposable_test_node", + }, indent=2) + "\n") + signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) + signal.signal(signal.SIGINT, lambda *_: sys.exit(0)) + try: + print("Disposable Froglet endpoint ready; private state must not be published", flush=True) + server.serve_forever() + finally: + server.server_close() + node.stop() + + +if __name__ == "__main__": + main() diff --git a/integrations/openclaw/froglet/scripts/qualify-openshell-plugin.mjs b/integrations/openclaw/froglet/scripts/qualify-openshell-plugin.mjs new file mode 100644 index 0000000..f5f1a35 --- /dev/null +++ b/integrations/openclaw/froglet/scripts/qualify-openshell-plugin.mjs @@ -0,0 +1,38 @@ +import assert from "node:assert/strict" +import register from "../index.js" + +// Runs the actual registered plugin action, with real HTTP transport. This is a +// plugin/runtime boundary check, not an OpenClaw gateway or model-inference test. +const mode = process.argv[2] +assert.ok(["baseline", "restricted"].includes(mode), "expected baseline or restricted") +let tool +register({ + config: { + hostProduct: "openclaw", + baseUrl: "https://host.openshell.internal:19443", + providerAuthTokenPath: "/sandbox/provider.token", + runtimeAuthTokenPath: "/sandbox/runtime.token", + requestTimeoutMs: 10000 + }, + registerTool: value => { tool = value }, + logger: { info() {} } +}) +assert.equal(tool?.name, "froglet") +const results = [] +for (const action of ["list_local_services", "get_spend_status"]) { + const result = await tool.execute(`qualification-${mode}-${action}`, { action }) + const text = result.content?.filter(item => item.type === "text") + .map(item => item.text).join("\n") ?? "" + results.push({ action, isError: result.isError === true, text }) +} +console.log(JSON.stringify({ mode, registered_tool: tool.name, results }, null, 2)) +assert.equal(results[0].isError, false, "service listing must succeed") +assert.match(results[0].text, /services: \d+/) +if (mode === "baseline") { + assert.equal(results[1].isError, false, "spend read must succeed under baseline policy") + assert.match(results[1].text, /spend_budget_msat:/) +} else { + assert.equal(results[1].isError, true, "spend read must be denied under restricted policy") + assert.match(results[1].text, /403/) + assert.match(results[1].text, /policy_denied/) +} diff --git a/integrations/openclaw/froglet/test/config-profiles.test.mjs b/integrations/openclaw/froglet/test/config-profiles.test.mjs index 7ad129f..30d2e9e 100644 --- a/integrations/openclaw/froglet/test/config-profiles.test.mjs +++ b/integrations/openclaw/froglet/test/config-profiles.test.mjs @@ -1,12 +1,18 @@ import assert from "node:assert/strict" -import { readFile } from "node:fs/promises" +import { cp, mkdir, mkdtemp, readFile, rm } from "node:fs/promises" +import { execFile } from "node:child_process" +import os from "node:os" import path from "node:path" +import process from "node:process" import test from "node:test" import { fileURLToPath } from "node:url" +import { promisify } from "node:util" import register from "../index.js" +import { checkPluginSource } from "../scripts/doctor.mjs" const pluginRoot = fileURLToPath(new URL("..", import.meta.url)) +const execFileAsync = promisify(execFile) const exampleFiles = [ "openclaw.config.example.json", "openclaw.config.nemoclaw.example.json", @@ -42,3 +48,58 @@ test("checked-in examples parse and register the single froglet tool", async () ) } }) + +test("documented OpenShell directory uploads preserve the plugin and shared-library layout", async () => { + const stagingRoot = await mkdtemp(path.join(os.tmpdir(), "froglet-staging-")) + try { + const repoRoot = path.resolve(pluginRoot, "../../..") + const documentation = await readFile(path.join(repoRoot, "docs/NEMOCLAW.md"), "utf8") + const directoryUploads = [...documentation.matchAll( + /openshell sandbox upload my-node \\\n \/absolute\/path\/to\/froglet\/(integrations\/(?:openclaw\/froglet|shared\/froglet-lib)) \\\n (\/sandbox\/[^\s]+)/g + )] + assert.equal(directoryUploads.length, 4) + assert.deepEqual(directoryUploads.slice(0, 2).map((match) => match.slice(1)), + directoryUploads.slice(2).map((match) => match.slice(1))) + async function uploadDirectory(match) { + const source = path.join(repoRoot, match[1]) + const destination = path.join(stagingRoot, match[2].slice(1)) + // OpenShell extracts a directory archive into DEST, retaining its basename. + const uploadedPath = path.join(destination, path.basename(source)) + await mkdir(destination, { recursive: true }) + await cp(source, uploadedPath, { recursive: true }) + return uploadedPath + } + const stagedPlugin = path.join(stagingRoot, "sandbox/froglet/integrations/openclaw/froglet") + assert.equal(await uploadDirectory(directoryUploads[0]), stagedPlugin) + const importAndRegister = ` + import { pathToFileURL } from 'node:url'; + const { default: register } = await import(pathToFileURL(process.argv[1]).href); + const names = []; + register({ + config: { hostProduct: 'nemoclaw', baseUrl: 'https://node.example', authTokenPath: '/sandbox/froglet.token' }, + registerTool(tool) { names.push(tool.name); }, + logger: { info() {} } + }); + process.stdout.write(JSON.stringify(names)); + ` + const args = ["--input-type=module", "-e", importAndRegister, path.join(stagedPlugin, "index.js")] + + await assert.rejects(execFileAsync(process.execPath, args), (error) => { + assert.match(error.stderr, /ERR_MODULE_NOT_FOUND/) + assert.match(error.stderr, /shared\/froglet-lib/) + return true + }) + const incomplete = await checkPluginSource(stagedPlugin) + assert.equal(incomplete.status, "error") + assert.ok(incomplete.details.missing_files.some((file) => file.includes("shared/froglet-lib"))) + + const stagedShared = path.join(stagingRoot, "sandbox/froglet/integrations/shared/froglet-lib") + assert.equal(await uploadDirectory(directoryUploads[1]), stagedShared) + const complete = await checkPluginSource(stagedPlugin) + assert.equal(complete.status, "ok") + const result = await execFileAsync(process.execPath, args) + assert.deepEqual(JSON.parse(result.stdout), ["froglet"]) + } finally { + await rm(stagingRoot, { recursive: true, force: true }) + } +}) diff --git a/integrations/openclaw/froglet/test/doctor.test.mjs b/integrations/openclaw/froglet/test/doctor.test.mjs index 603f5c2..c7f9481 100644 --- a/integrations/openclaw/froglet/test/doctor.test.mjs +++ b/integrations/openclaw/froglet/test/doctor.test.mjs @@ -1,5 +1,5 @@ import assert from "node:assert/strict" -import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import os from "node:os" import path from "node:path" import process from "node:process" @@ -9,12 +9,23 @@ import { fileURLToPath } from "node:url" import { checkApis, runtimeHealthCheck } from "../scripts/doctor.mjs" const doctorPath = fileURLToPath(new URL("../scripts/doctor.mjs", import.meta.url)) +const pluginRoot = fileURLToPath(new URL("..", import.meta.url)) -async function writeConfig(tempDir, fileName, pluginConfig) { +async function stagePlugin(tempDir) { + const pluginPath = path.join(tempDir, "integrations/openclaw/froglet") + const sharedPath = path.join(tempDir, "integrations/shared/froglet-lib") + await mkdir(path.dirname(pluginPath), { recursive: true }) + await mkdir(path.dirname(sharedPath), { recursive: true }) + await cp(pluginRoot, pluginPath, { recursive: true }) + await cp(path.resolve(pluginRoot, "../../shared/froglet-lib"), sharedPath, { recursive: true }) + return pluginPath +} + +async function writeConfig(tempDir, fileName, pluginConfig, pluginPath = path.join(tempDir, "plugin")) { const configPath = path.join(tempDir, fileName) const document = { plugins: { - load: { paths: [path.join(tempDir, "plugin")] }, + load: { paths: [pluginPath] }, entries: { froglet: { enabled: true, @@ -52,7 +63,7 @@ test("doctor validates the new dual-URL config shape", async () => { try { const tokenPath = path.join(tempDir, "froglet.token") await writeFile(tokenPath, "froglet-token\n", "utf8") - await writeFile(path.join(tempDir, "plugin"), "ok\n", "utf8") + const pluginPath = await stagePlugin(tempDir) const configPath = await writeConfig(tempDir, "openclaw.json", { hostProduct: "openclaw", providerUrl: "http://127.0.0.1:8080", @@ -62,7 +73,7 @@ test("doctor validates the new dual-URL config shape", async () => { requestTimeoutMs: 1000, defaultSearchLimit: 10, maxSearchLimit: 50 - }) + }, pluginPath) const result = await runDoctor(["--config", configPath, "--target", "openclaw"]) assert.equal(result.code, 0) const payload = JSON.parse(result.stdout) @@ -77,7 +88,7 @@ test("doctor validates legacy baseUrl/authTokenPath config as fallback", async ( try { const tokenPath = path.join(tempDir, "froglet.token") await writeFile(tokenPath, "froglet-token\n", "utf8") - await writeFile(path.join(tempDir, "plugin"), "ok\n", "utf8") + const pluginPath = await stagePlugin(tempDir) const configPath = await writeConfig(tempDir, "openclaw-legacy.json", { hostProduct: "openclaw", baseUrl: "http://127.0.0.1:8080", @@ -85,12 +96,62 @@ test("doctor validates legacy baseUrl/authTokenPath config as fallback", async ( requestTimeoutMs: 1000, defaultSearchLimit: 10, maxSearchLimit: 50 - }) + }, pluginPath) const result = await runDoctor(["--config", configPath, "--target", "openclaw"]) assert.equal(result.code, 0) const payload = JSON.parse(result.stdout) - // auth_token check may be warning (plugin dir doesn't exist) but must not be error - assert.ok(["ok", "warning"].includes(payload.overall_status)) + assert.equal(payload.overall_status, "warning") + assert.equal(payload.checks.find((check) => check.id === "plugin_source").status, "ok") + } finally { + await rm(tempDir, { recursive: true, force: true }) + } +}) + +test("doctor rejects a readable file as a plugin and exits nonzero", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "froglet-doctor-")) + try { + const tokenPath = path.join(tempDir, "froglet.token") + await writeFile(tokenPath, "froglet-token\n", "utf8") + await writeFile(path.join(tempDir, "plugin"), "not a plugin directory\n", "utf8") + const configPath = await writeConfig(tempDir, "invalid-plugin.json", { + hostProduct: "nemoclaw", + baseUrl: "https://node.example", + authTokenPath: tokenPath + }) + const result = await runDoctor(["--config", configPath, "--target", "nemoclaw"]) + assert.equal(result.code, 1) + const payload = JSON.parse(result.stdout) + assert.equal(payload.overall_status, "error") + assert.equal(payload.checks.find((check) => check.id === "plugin_source").status, "error") + assert.equal(payload.checks.find((check) => check.id === "runtime_health").status, "warning") + } finally { + await rm(tempDir, { recursive: true, force: true }) + } +}) + +test("doctor rejects missing credentials and keeps skipped-runtime status explicit", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "froglet-doctor-")) + try { + const pluginPath = await stagePlugin(tempDir) + const configPath = await writeConfig(tempDir, "missing-token.json", { + hostProduct: "nemoclaw", + baseUrl: "https://node.example", + authTokenPath: path.join(tempDir, "missing.token") + }, pluginPath) + const result = await runDoctor(["--config", configPath, "--target", "nemoclaw", "--check-runtime"]) + assert.equal(result.code, 1) + const payload = JSON.parse(result.stdout) + assert.equal(payload.checks.find((check) => check.id === "auth_token").status, "error") + const runtimeCheck = payload.checks.find((check) => check.id === "runtime_health") + assert.equal(runtimeCheck.status, "warning") + assert.equal(runtimeCheck.summary, "Runtime check skipped") + const uploadCommands = runtimeCheck.details.manual_commands.filter((command) => command.startsWith("openshell sandbox upload")) + assert.equal(uploadCommands.length, 3) + assert.deepEqual(uploadCommands, [ + `openshell sandbox upload /absolute/path/to/froglet/integrations/openclaw/froglet '${path.dirname(pluginPath)}'`, + `openshell sandbox upload /absolute/path/to/froglet/integrations/shared/froglet-lib '${path.resolve(pluginPath, "../../shared")}'`, + `openshell sandbox upload /absolute/path/to/froglet-control.token '${path.join(tempDir, "missing.token")}'` + ]) } finally { await rm(tempDir, { recursive: true, force: true }) } diff --git a/integrations/openclaw/froglet/test/openshell-qualification-backend.test.py b/integrations/openclaw/froglet/test/openshell-qualification-backend.test.py new file mode 100644 index 0000000..1d42d13 --- /dev/null +++ b/integrations/openclaw/froglet/test/openshell-qualification-backend.test.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Deterministic provider-readiness regression tests; no Docker or network.""" +import importlib.util +import io +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import MagicMock, patch + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts/openshell-qualification-backend.py" +SPEC = importlib.util.spec_from_file_location("qualification_backend", SCRIPT) +BACKEND = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(BACKEND) + + +class ProviderReadinessTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.log = Path(self.directory.name) / "node.log" + self.log.write_text("Local Runtime API: http://127.0.0.1:8001\n") + self.node = SimpleNamespace(log=self.log, process=MagicMock()) + self.node.process.poll.return_value = None + + def test_waits_for_delayed_provider_log_and_then_healthy_response(self): + opener = MagicMock() + opener.open.side_effect = [io.StringIO('{"status":"starting"}'), + io.StringIO('{"status":"ok"}')] + sleeps = [] + + def advance(delay): + sleeps.append(delay) + if len(sleeps) == 1: + with self.log.open("a") as output: + output.write("Local API Gateway: http://127.0.0.1:8002\n") + + with patch.object(BACKEND.time, "monotonic", side_effect=[0, 0, 1, 2]), \ + patch.object(BACKEND.time, "sleep", side_effect=advance), \ + patch.object(BACKEND.urllib.request, "build_opener", return_value=opener): + self.assertEqual(BACKEND.wait_for_provider(self.node), "http://127.0.0.1:8002") + self.assertEqual(len(sleeps), 2) + self.assertEqual(opener.open.call_count, 2) + opener.open.assert_called_with("http://127.0.0.1:8002/health", timeout=2) + + def test_missing_provider_listener_times_out(self): + with patch.object(BACKEND.time, "monotonic", side_effect=[0, 0, 31]), \ + patch.object(BACKEND.time, "sleep"), \ + patch.object(BACKEND.urllib.request, "build_opener") as opener: + with self.assertRaisesRegex(RuntimeError, "provider readiness timed out"): + BACKEND.wait_for_provider(self.node) + opener.return_value.open.assert_not_called() + + def test_node_exit_does_not_leave_readiness_wait_running(self): + self.node.process.poll.return_value = 1 + with patch.object(BACKEND.time, "sleep") as sleep: + with self.assertRaisesRegex(RuntimeError, "exited before provider readiness"): + BACKEND.wait_for_provider(self.node) + sleep.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/python/froglet_fl_evidence.py b/python/froglet_fl_evidence.py new file mode 100644 index 0000000..92fc814 --- /dev/null +++ b/python/froglet_fl_evidence.py @@ -0,0 +1,257 @@ +"""Completeness records for the federated-learning reference example. + +This application helper consumes the output of the example's existing offline +artifact verifier. It does not verify signatures, authenticate datasets, prove +correct computation, or change Froglet protocol/settlement semantics. Callers +must extract workload hashes from the verified Quote and declare expected +workloads before collecting contributions. Run/round labels are application +metadata, not additional signed protocol fields. + +The record is in-memory. Exporting a snapshot does not implement durable +recovery or a distributed commit protocol. +""" + +from __future__ import annotations + +from collections.abc import Mapping +import json +import re +from typing import Any + + +class EvidenceError(ValueError): + """A contribution or publication conflicts with the declared example run.""" + + +def _text(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise EvidenceError(f"{field} must be a non-empty string") + return value + + +def _hash(value: object, field: str) -> str: + if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None: + raise EvidenceError(f"{field} must be a 64-character lowercase hexadecimal string") + return value + + +def _round(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise EvidenceError("round_id must be a non-negative integer") + return value + + +def _json_copy_bytes(value: object) -> str: + """Freeze JSON data without accepting lossy key/type coercions or NaN.""" + def check(item: object) -> None: + if item is None or type(item) in (str, bool, int, float): + return + if type(item) is list: + for child in item: + check(child) + return + if type(item) is dict: + for key, child in item.items(): + if not isinstance(key, str): + raise EvidenceError("output object keys must be strings") + check(child) + return + raise EvidenceError("output must contain only JSON values") + + try: + check(value) + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + except (ValueError, TypeError, RecursionError) as error: + raise EvidenceError(f"output is not finite JSON data: {error}") from error + + +class RoundEvidence: + """Stage one verified contribution per declared participant before publication. + + ``workload_hashes`` binds each participant to its expected request, including + the input/model state committed by that request. Reusing the same expected + hashes across rounds does not create a new cryptographic round binding. + An identical retry is harmless; replacing accepted evidence is forbidden. + A successful retry can resolve a recorded failure without erasing its reason. + """ + + def __init__( + self, + *, + run_id: str, + round_id: int, + operation: str, + module_hash: str, + participants: Mapping[str, str], + workload_hashes: Mapping[str, str], + ) -> None: + self._run_id = _text(run_id, "run_id") + self._round_id = _round(round_id) + self._operation = _text(operation, "operation") + self._module_hash = _hash(module_hash, "module_hash") + if not isinstance(participants, Mapping) or not participants: + raise EvidenceError("participants must be a non-empty mapping") + self._participants = { + _text(name, "participant_id"): _hash(provider, "provider_id") + for name, provider in participants.items() + } + if len(set(self._participants.values())) != len(self._participants): + raise EvidenceError("participants must have distinct provider identities") + if not isinstance(workload_hashes, Mapping) or set(workload_hashes) != set( + self._participants + ): + raise EvidenceError("workload_hashes must cover exactly the declared participants") + self._workload_hashes = { + name: _hash(workload_hashes[name], "workload_hash") + for name in self._participants + } + self._contributions: dict[str, str] = {} + self._failures: dict[str, list[str]] = {} + self._output: str | None = None + + def _participant(self, participant_id: str) -> None: + if not isinstance(participant_id, str) or participant_id not in self._participants: + raise EvidenceError("unexpected participant") + + def stage( + self, + participant_id: str, + *, + run_id: str, + round_id: int, + operation: str, + module_hash: str, + workload_hash: str, + deal_id: str, + verification: Mapping[str, Any], + dataset_commitment: str | None = None, + ) -> bool: + """Stage already-verified evidence; return False for an identical retry. + + ``verification`` must come from ``verify_artifact_chain_offline``, not + from an untrusted network response asserting that verification passed. + The verifier must return ``workload_hash`` from the verified Quote and + ``module_hash`` from the verified Receipt executor. This helper compares + those values to the declaration; it does not rerun the cryptography. + """ + self._participant(participant_id) + if ( + _text(run_id, "run_id") != self._run_id + or _round(round_id) != self._round_id + or _text(operation, "operation") != self._operation + or _hash(module_hash, "module_hash") != self._module_hash + ): + raise EvidenceError("contribution does not match the declared run/round/workload") + if _hash(workload_hash, "workload_hash") != self._workload_hashes[participant_id]: + raise EvidenceError("contribution workload hash differs from the declared request") + if not isinstance(verification, Mapping) or verification.get("offline_chain_verified") is not True: + raise EvidenceError("offline artifact verification is required") + if verification.get("deal_state") != "succeeded" or verification.get("execution_state") != "succeeded": + raise EvidenceError("a verified successful deal and execution are required") + if _hash(verification.get("workload_hash"), "verified workload_hash") != workload_hash: + raise EvidenceError("verified workload hash differs from the declared request") + if _hash(verification.get("module_hash"), "verified module_hash") != self._module_hash: + raise EvidenceError("verified module hash differs from the declared module") + provider_id = _hash(verification.get("provider_id"), "verified provider_id") + if provider_id != self._participants[participant_id]: + raise EvidenceError("verified provider differs from the declared participant") + artifact_count = verification.get("artifact_count") + if type(artifact_count) is not int or artifact_count not in (5, 6): + raise EvidenceError("verified artifact_count must be 5 or 6") + hashes = { + name: _hash(verification.get(name), f"verified {name}") + for name in ( + "descriptor_hash", "offer_hash", "quote_hash", "deal_hash", + "receipt_hash", "result_hash", "requester_id", + ) + } + invoice_hash = verification.get("invoice_bundle_hash") + if artifact_count == 6: + invoice_hash = _hash(invoice_hash, "verified invoice_bundle_hash") + elif invoice_hash is not None: + raise EvidenceError("five-artifact evidence must not contain an invoice bundle") + if dataset_commitment is not None: + raw_commitment = dataset_commitment.removeprefix("sha256:") if isinstance(dataset_commitment, str) else dataset_commitment + dataset_commitment = "sha256:" + _hash(raw_commitment, "dataset_commitment") + contribution = { + "participant_id": participant_id, + "provider_id": provider_id, + "run_id": self._run_id, + "round_id": self._round_id, + "operation": self._operation, + "module_hash": self._module_hash, + "workload_hash": workload_hash, + "deal_id": _text(deal_id, "deal_id"), + "artifact_count": artifact_count, + "deal_state": "succeeded", + "execution_state": "succeeded", + "invoice_bundle_hash": invoice_hash, + "dataset_commitment": dataset_commitment, + **hashes, + } + encoded = _json_copy_bytes(contribution) + previous = self._contributions.get(participant_id) + if previous is not None: + if previous != encoded: + raise EvidenceError("conflicting contribution for an already staged participant") + return False + if self._output is not None: + raise EvidenceError("published round evidence is immutable") + # A single signed receipt cannot stand in for two independent participants. + for item in self._contributions.values(): + if json.loads(item)["receipt_hash"] == hashes["receipt_hash"]: + raise EvidenceError("receipt is already assigned to another participant") + self._contributions[participant_id] = encoded + return True + + def record_failure(self, participant_id: str, reason: str) -> None: + """Retain an unsuccessful attempt; it does not count as a contribution.""" + self._participant(participant_id) + reason = _text(reason, "failure reason") + if self._output is not None: + raise EvidenceError("published round evidence is immutable") + if participant_id in self._contributions: + raise EvidenceError("cannot replace a staged contribution with failure") + history = self._failures.setdefault(participant_id, []) + if reason not in history: + history.append(reason) + + def snapshot(self) -> dict[str, Any]: + """Return detached JSON data; an unpromoted snapshot never contains output.""" + missing = sorted(set(self._participants) - set(self._contributions)) + failed = [name for name in missing if self._failures.get(name)] + return { + "schema_version": "froglet.fl.reference_evidence.v1", + "run_id": self._run_id, + "round_id": self._round_id, + "operation": self._operation, + "module_hash": self._module_hash, + "participants": dict(sorted(self._participants.items())), + "expected_workload_hashes": dict(sorted(self._workload_hashes.items())), + "status": "promoted" if self._output is not None else "ready" if not missing else "incomplete", + "missing_participants": missing, + "failed_participants": failed, + "failure_history": {name: list(reasons) for name, reasons in sorted(self._failures.items())}, + "contributions": [json.loads(self._contributions[name]) for name in sorted(self._contributions)], + "output": json.loads(self._output) if self._output is not None else None, + "evidence_boundary": ( + "Application completeness record using caller-verified signed artifacts. " + "Does not independently verify signatures, dataset identity, computation, " + "output correctness, privacy or distributed rollback." + ), + } + + def promote(self, output: dict[str, Any]) -> dict[str, Any]: + """Publish a detached output record only after every contribution is staged.""" + if type(output) is not dict: + raise EvidenceError("output must be a JSON object") + encoded = _json_copy_bytes(output) + if self._output is not None: + if encoded != self._output: + raise EvidenceError("published output is immutable") + return self.snapshot() + missing = sorted(set(self._participants) - set(self._contributions)) + if missing: + raise EvidenceError(f"cannot promote incomplete round; missing participants: {', '.join(missing)}") + self._output = encoded + return self.snapshot() diff --git a/python/requirements-immunopeptidomics.txt b/python/requirements-immunopeptidomics.txt new file mode 100644 index 0000000..93fe913 --- /dev/null +++ b/python/requirements-immunopeptidomics.txt @@ -0,0 +1,5 @@ +# Reference environment tested with Python 3.11; install in a dedicated venv. +-r requirements.txt +nvflare==2.9.0 +numpy==2.3.5 +cryptography==46.0.7 diff --git a/python/tests/test_froglet_fl_evidence.py b/python/tests/test_froglet_fl_evidence.py new file mode 100644 index 0000000..d1f10b8 --- /dev/null +++ b/python/tests/test_froglet_fl_evidence.py @@ -0,0 +1,229 @@ +"""Failure and publication behavior of the FL example completeness record.""" + +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path +import sys +import unittest + +REPO = Path(__file__).resolve().parents[2] +if str(REPO / "python") not in sys.path: + sys.path.insert(0, str(REPO / "python")) + +from froglet_fl_evidence import EvidenceError, RoundEvidence # noqa: E402 + + +class RoundEvidenceTests(unittest.TestCase): + def setUp(self) -> None: + self.providers = {"silo_a": "a" * 64, "silo_b": "b" * 64} + self.workloads = {"silo_a": "1" * 64, "silo_b": "2" * 64} + self.context = { + "run_id": "synthetic-reference-run", + "round_id": 0, + "operation": "map_shard_stats", + "module_hash": "c" * 64, + } + self.ledger = RoundEvidence( + **self.context, participants=self.providers, workload_hashes=self.workloads + ) + + def contribution(self, name: str) -> dict: + marker = "3" if name == "silo_a" else "4" + audit = { + "offline_chain_verified": True, + "artifact_count": 5, + "deal_state": "succeeded", + "execution_state": "succeeded", + "provider_id": self.providers[name], + "requester_id": "d" * 64, + "descriptor_hash": "5" * 64, + "offer_hash": "6" * 64, + "quote_hash": "7" * 64, + "deal_hash": "8" * 64, + "receipt_hash": marker * 64, + "result_hash": "9" * 64, + "invoice_bundle_hash": None, + "module_hash": self.context["module_hash"], + "workload_hash": self.workloads[name], + } + return { + **self.context, + "workload_hash": self.workloads[name], + "deal_id": f"deal-{name}", + "verification": audit, + } + + def stage_all(self) -> None: + for name in self.providers: + self.ledger.stage(name, **self.contribution(name)) + + def test_missing_contribution_blocks_output_even_after_one_success(self) -> None: + self.ledger.stage("silo_a", **self.contribution("silo_a")) + with self.assertRaisesRegex(EvidenceError, "incomplete.*silo_b"): + self.ledger.promote({"count": 20}) + snapshot = self.ledger.snapshot() + self.assertEqual(snapshot["status"], "incomplete") + self.assertEqual(snapshot["missing_participants"], ["silo_b"]) + self.assertIsNone(snapshot["output"]) + + def test_failure_stays_visible_and_successful_retry_resolves_only_active_failure(self) -> None: + self.ledger.record_failure("silo_b", "provider timeout") + self.ledger.record_failure("silo_b", "provider timeout") + self.ledger.record_failure("silo_b", "access refused") + failed = self.ledger.snapshot() + self.assertEqual(failed["failed_participants"], ["silo_b"]) + self.assertEqual(failed["failure_history"]["silo_b"], ["provider timeout", "access refused"]) + with self.assertRaisesRegex(EvidenceError, "incomplete"): + self.ledger.promote({}) + self.stage_all() + promoted = self.ledger.promote({"count": 20}) + self.assertEqual(promoted["failed_participants"], []) + self.assertEqual(promoted["failure_history"], failed["failure_history"]) + + def test_identical_retry_is_idempotent_before_and_after_promotion(self) -> None: + args = self.contribution("silo_a") + self.assertTrue(self.ledger.stage("silo_a", **args)) + self.assertFalse(self.ledger.stage("silo_a", **deepcopy(args))) + self.ledger.stage("silo_b", **self.contribution("silo_b")) + published = self.ledger.promote({"weights": [1, 2], "count": 20}) + self.assertFalse(self.ledger.stage("silo_a", **args)) + self.assertEqual(published, self.ledger.promote({"count": 20, "weights": [1, 2]})) + self.assertEqual(len(published["contributions"]), 2) + + def test_conflicting_retry_preserves_the_accepted_contribution(self) -> None: + args = self.contribution("silo_a") + self.ledger.stage("silo_a", **args) + before = self.ledger.snapshot() + for field in ("receipt_hash", "result_hash", "deal_hash", "quote_hash"): + changed = deepcopy(args) + changed["verification"][field] = "e" * 64 + with self.subTest(field=field), self.assertRaisesRegex(EvidenceError, "conflicting"): + self.ledger.stage("silo_a", **changed) + self.assertEqual(before, self.ledger.snapshot()) + + def test_cross_run_round_operation_and_module_are_rejected(self) -> None: + changes = {"run_id": "other-run", "round_id": 1, "operation": "fl_local_sgd", "module_hash": "f" * 64} + for field, value in changes.items(): + args = self.contribution("silo_a") + args[field] = value + with self.subTest(field=field), self.assertRaisesRegex(EvidenceError, "declared run/round/workload"): + self.ledger.stage("silo_a", **args) + self.assertEqual(self.ledger.snapshot()["contributions"], []) + + def test_relabeling_old_verified_workload_or_module_is_rejected(self) -> None: + for field in ("workload_hash", "module_hash"): + args = self.contribution("silo_a") + args["verification"][field] = "f" * 64 + with self.subTest(field=field), self.assertRaisesRegex(EvidenceError, "verified .* hash differs"): + self.ledger.stage("silo_a", **args) + args = self.contribution("silo_a") + args["workload_hash"] = "f" * 64 + args["verification"]["workload_hash"] = "f" * 64 + with self.assertRaisesRegex(EvidenceError, "declared request"): + self.ledger.stage("silo_a", **args) + + def test_unexpected_participant_wrong_provider_and_reused_receipt_are_rejected(self) -> None: + with self.assertRaisesRegex(EvidenceError, "unexpected participant"): + self.ledger.stage("silo_c", **self.contribution("silo_a")) + with self.assertRaisesRegex(EvidenceError, "unexpected participant"): + self.ledger.record_failure("silo_c", "timeout") + wrong_provider = self.contribution("silo_a") + wrong_provider["verification"]["provider_id"] = self.providers["silo_b"] + with self.assertRaisesRegex(EvidenceError, "provider differs"): + self.ledger.stage("silo_a", **wrong_provider) + self.ledger.stage("silo_a", **self.contribution("silo_a")) + duplicate = self.contribution("silo_b") + duplicate["verification"]["receipt_hash"] = "3" * 64 + with self.assertRaisesRegex(EvidenceError, "receipt is already assigned"): + self.ledger.stage("silo_b", **duplicate) + + def test_declared_identity_and_workload_coverage_are_required(self) -> None: + for providers, workloads in ( + ({}, {}), + (self.providers, {"silo_a": "1" * 64}), + (self.providers, {**self.workloads, "silo_c": "3" * 64}), + ({"silo_a": "a" * 64, "silo_b": "a" * 64}, self.workloads), + ): + with self.subTest(providers=providers, workloads=workloads), self.assertRaises(EvidenceError): + RoundEvidence(**self.context, participants=providers, workload_hashes=workloads) + with self.assertRaisesRegex(EvidenceError, "round_id"): + RoundEvidence(**{**self.context, "round_id": True}, participants=self.providers, workload_hashes=self.workloads) + + def test_unverified_malformed_or_inconsistent_audits_are_rejected(self) -> None: + invalid_fields = ( + ("offline_chain_verified", False), ("offline_chain_verified", 1), + ("deal_state", "canceled"), ("deal_state", "failed"), + ("execution_state", "failed"), ("execution_state", None), + ("receipt_hash", "ABC"), ("result_hash", None), + ("artifact_count", 7), ("artifact_count", True), + ("invoice_bundle_hash", "f" * 64), + ) + for field, value in invalid_fields: + args = self.contribution("silo_a") + args["verification"][field] = value + with self.subTest(field=field, value=value), self.assertRaises(EvidenceError): + self.ledger.stage("silo_a", **args) + args = self.contribution("silo_a") + args["verification"]["artifact_count"] = 6 + with self.assertRaisesRegex(EvidenceError, "invoice_bundle_hash"): + self.ledger.stage("silo_a", **args) + args["verification"]["invoice_bundle_hash"] = "f" * 64 + self.assertTrue(self.ledger.stage("silo_a", **args)) + + def test_inputs_and_exported_snapshots_cannot_mutate_retained_evidence(self) -> None: + args = self.contribution("silo_a") + self.ledger.stage("silo_a", **args) + self.providers["silo_a"] = "f" * 64 + self.workloads["silo_a"] = "f" * 64 + args["verification"]["result_hash"] = "f" * 64 + snapshot = self.ledger.snapshot() + snapshot["participants"]["silo_a"] = "f" * 64 + snapshot["contributions"][0]["result_hash"] = "f" * 64 + self.assertEqual(self.ledger.snapshot()["participants"]["silo_a"], "a" * 64) + self.assertEqual(self.ledger.snapshot()["contributions"][0]["result_hash"], "9" * 64) + self.ledger.stage("silo_b", **self.contribution("silo_b")) + output = {"nested": {"counts": [10, 20]}} + published = self.ledger.promote(output) + output["nested"]["counts"].append(30) + published["output"]["nested"]["counts"].append(40) + self.assertEqual(self.ledger.snapshot()["output"], {"nested": {"counts": [10, 20]}}) + + def test_publication_cannot_be_replaced_by_changed_output_or_failure(self) -> None: + self.stage_all() + published = self.ledger.promote({"count": 20}) + with self.assertRaisesRegex(EvidenceError, "immutable"): + self.ledger.promote({"count": 21}) + with self.assertRaisesRegex(EvidenceError, "immutable"): + self.ledger.record_failure("silo_a", "late timeout") + self.assertEqual(self.ledger.snapshot(), published) + + def test_failed_attempt_cannot_overwrite_staged_success(self) -> None: + self.ledger.stage("silo_a", **self.contribution("silo_a")) + with self.assertRaisesRegex(EvidenceError, "cannot replace"): + self.ledger.record_failure("silo_a", "late failure") + self.assertEqual(self.ledger.snapshot()["failure_history"], {}) + + def test_invalid_output_is_not_promoted(self) -> None: + self.stage_all() + for output in ({"x": float("nan")}, {"x": float("inf")}, {1: "bad-key"}, {"x": (1, 2)}, []): + with self.subTest(output=output), self.assertRaises(EvidenceError): + self.ledger.promote(output) + cyclic = {} + cyclic["self"] = cyclic + with self.assertRaises(EvidenceError): + self.ledger.promote(cyclic) + self.assertEqual(self.ledger.snapshot()["status"], "ready") + self.assertIsNone(self.ledger.snapshot()["output"]) + + def test_dataset_commitment_is_optional_declared_metadata(self) -> None: + args = self.contribution("silo_a") + self.ledger.stage("silo_a", **args, dataset_commitment="e" * 64) + self.assertFalse(self.ledger.stage("silo_a", **args, dataset_commitment="sha256:" + "e" * 64)) + entry = self.ledger.snapshot()["contributions"][0] + self.assertEqual(entry["dataset_commitment"], "sha256:" + "e" * 64) + self.assertIn("dataset identity", self.ledger.snapshot()["evidence_boundary"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/strict_checks.sh b/scripts/strict_checks.sh index 377eff9..95c3d9d 100755 --- a/scripts/strict_checks.sh +++ b/scripts/strict_checks.sh @@ -128,6 +128,7 @@ if command -v node >/dev/null 2>&1; then node --check integrations/openclaw/froglet/scripts/doctor.mjs node --test integrations/openclaw/froglet/test/plugin.test.js \ integrations/openclaw/froglet/test/*.test.mjs + python3 -W error integrations/openclaw/froglet/test/openshell-qualification-backend.test.py -v echo "[strict] MCP server checks" node --check integrations/mcp/froglet/server.js @@ -163,7 +164,7 @@ if rustup target list --installed 2>/dev/null | grep -q wasm32-unknown-unknown; echo "[strict] ontology checker and independent oracle unit checks" CARGO_INCREMENTAL=0 RUSTFLAGS="$strict_rustflags" \ cargo test --locked --manifest-path examples/wasm-services/Cargo.toml -p ontology-check - python3 -W error -m unittest examples.test_a2a_compute_demo examples.test_research_profile examples.test_go_subset_catalog examples.test_qualify_ontology_capacity examples.test_generated_program_replay -v + python3 -W error -m unittest examples.test_a2a_compute_demo examples.test_immunopeptidomics_demo examples.test_research_profile examples.test_go_subset_catalog examples.test_qualify_ontology_capacity examples.test_generated_program_replay -v if [[ -z "$qualification_node" && "${FROGLET_REQUIRE_SOFTWARE_CHECKS:-0}" == 1 ]]; then # Standalone release CI has no package gate to supply its candidate.