From f179c8a535b4bfb2109191cf42979a82f3bfa35f Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 5 Nov 2026 21:50:29 +0200 Subject: [PATCH 1/7] build: local tooling directories are ignored by the gitignore that travels .superpowers/ was matched only by .git/info/exclude, which is local to one clone and is never pushed. On a fresh clone or a second checkout its contents would have appeared as ordinary untracked files, one 'git add .' away from being committed. The repository's other local-only directories are already listed in the committed .gitignore; this puts .superpowers/ beside them. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e6ba671e..c4c32b6d 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ htmlcov/ debug.py /scripts/ .private/ +.superpowers/ _old_docs/ .mcp.json .claude/ From 86185a29fce9de4b7e0d8f0d08f7eb93ee0844e1 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 5 Nov 2026 21:50:36 +0200 Subject: [PATCH 2/7] fix: a verification record named a machine, and one named another checkout Fourteen tracked verification and benchmark records under audit-docs/ embedded absolute paths from the machine that produced them. One went further: assay_oracle_verification_2026_10_20.json recorded a second project's working tree as its pythonpath and as the path of the package under test. None of this identifies an artifact -- the sha256 recorded beside each path does that -- so the paths are now written relative to the repository. The four sites that produced them are fixed too, otherwise the next run of any of these tools would write the absolute paths back: duplex_compile_benchmark, duplex_family_benchmark and original_end_stress each recorded extension_path from Path(_engine.__file__).resolve(). They now record it relative to the repository root, falling back to the basename for an extension installed outside it. After this, no tracked file in the repository names a path on anyone's machine. --- .../assay_oracle_verification_2026_10_20.json | 12 ++--- .../biological_metadata_audit_2026_10_20.json | 4 +- ..._metadata_fix_verification_2026_10_20.json | 4 +- ...oupled_duplex_verification_2026_10_20.json | 2 +- ...periment_compile_benchmark_2026_10_20.json | 38 +++++++------- .../duplex_family_benchmark_2026_10_20.json | 14 +++--- ...ex_projection_verification_2026_10_20.json | 4 +- .../durable_family_benchmark_2026_10_21.json | 50 +++++++++---------- ...family_replay_verification_2026_10_21.json | 4 +- ...original_coding_end_stress_2026_10_20.json | 2 +- ...al_coding_end_verification_2026_10_20.json | 2 +- ...ive_p_support_verification_2026_10_20.json | 4 +- ...uctivity_indel_cross_locus_2026_10_20.json | 2 +- ..._indel_repair_verification_2026_10_20.json | 2 +- .../duplex_compile_benchmark.py | 19 ++++++- .../duplex_family_benchmark.py | 17 ++++++- .../original_end_stress.py | 16 +++++- 17 files changed, 120 insertions(+), 76 deletions(-) diff --git a/audit-docs/assay_oracle_verification_2026_10_20.json b/audit-docs/assay_oracle_verification_2026_10_20.json index b8f1989c..304ad2b3 100644 --- a/audit-docs/assay_oracle_verification_2026_10_20.json +++ b/audit-docs/assay_oracle_verification_2026_10_20.json @@ -4,8 +4,8 @@ "scope": "All S0-S12 dev suites rebuilt with the final local engine, then alignair_assay.testing.assert_oracle_perfect was applied to every truth set; donor genotypes were supplied whenever donors.json exists. This verifies the external assay truth/scoring oracle rule and records its exclusions/warnings. It is not proof that every simulator metadata field or biological distribution is correct.", "run": { "directory": "/tmp/genairr_assay_fixed_final", - "python": "/home/thomas/Desktop/AlignAIR/.venv/bin/python", - "pythonpath": "/home/thomas/Desktop/GenAIRR/src:/home/thomas/Desktop/AlignAIR/.claude/worktrees/dockerhub-registry/src", + "python": "/.venv/bin/python", + "pythonpath": "/src:/src", "build_command": "python -m alignair_assay build --suites S0,S1,S2,S3,S4,S5,S6,S7,S8,S9,S10,S11,S12 --run-dir /tmp/genairr_assay_fixed_final --split dev --scale 0.02 --workers 1", "build_log": "/tmp/genairr_external_assay_final.log", "verification_script": "/tmp/verify_genairr_external_assay.py", @@ -20,20 +20,20 @@ "source": { "genairr": { "genairr_version": "1.0.0", - "genairr_source": "/home/thomas/Desktop/GenAIRR/src/GenAIRR", + "genairr_source": "/src/GenAIRR", "genairr_git_commit": "1ededa676e8904430f7ea0bdcf03092de4888a41", "genairr_git_dirty": true, "genairr_source_sha256": "0678200704dddd82b43e09b2e676fdf9a5fa90cd7c4d91b0084d1833782aa778", "branch": "fix/biological-metadata-correctness", "source_project_version": "2.3.1", - "loaded_package": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/__init__.py", - "loaded_engine": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "loaded_package": "/src/GenAIRR/__init__.py", + "loaded_engine": "/src/GenAIRR/_engine.abi3.so", "loaded_engine_sha256": "bcb885e05b763f7c160cfa08c5a1043b9779867f72b9c09b945ba81d61831443", "rust_source_sha256": "d6ddd23999e3ab05012d55af720a2cbee85d8a33ab9888b759e025244a5075f8", "metadata_version_note": "The external AlignAIR virtual environment has stale installed GenAIRR distribution metadata (1.0.0). PYTHONPATH loads this checkout's package and rebuilt native extension (source project version 2.3.1); paths and hashes above identify them." }, "assay": { - "path": "/home/thomas/Desktop/AlignAIR/.claude/worktrees/dockerhub-registry/src/alignair_assay", + "path": "/src/alignair_assay", "git_commit": "8cf24ca29a89df3f8617c99f51fdbc0fa1a6c77e", "git_dirty": false, "python_source_sha256": "fc69139307ce83ece3a4f9a7fbca482c0cc12b2ae49b82ae98f329ff607b7ab2", diff --git a/audit-docs/biological_metadata_audit_2026_10_20.json b/audit-docs/biological_metadata_audit_2026_10_20.json index 57e27367..116fc6ac 100644 --- a/audit-docs/biological_metadata_audit_2026_10_20.json +++ b/audit-docs/biological_metadata_audit_2026_10_20.json @@ -2,8 +2,8 @@ "audit_date": "2026-10-20", "commit": "1ededa676e8904430f7ea0bdcf03092de4888a41", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", - "package": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/__init__.py", - "extension": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "package": "/src/GenAIRR/__init__.py", + "extension": "/src/GenAIRR/_engine.abi3.so", "checks_run": 19, "discrepancies_reproduced": 19, "confirmed_defect_probes_reproduced": 16, diff --git a/audit-docs/biological_metadata_fix_verification_2026_10_20.json b/audit-docs/biological_metadata_fix_verification_2026_10_20.json index 7d62dd43..a5cc8478 100644 --- a/audit-docs/biological_metadata_fix_verification_2026_10_20.json +++ b/audit-docs/biological_metadata_fix_verification_2026_10_20.json @@ -4,8 +4,8 @@ "branch": "fix/biological-metadata-correctness", "tracked_source_modified": true, "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", - "package": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/__init__.py", - "extension": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "package": "/src/GenAIRR/__init__.py", + "extension": "/src/GenAIRR/_engine.abi3.so", "checks_run": 19, "discrepancies_reproduced": 2, "confirmed_defect_probes_reproduced": 1, diff --git a/audit-docs/coupled_duplex_verification_2026_10_20.json b/audit-docs/coupled_duplex_verification_2026_10_20.json index a467d74f..56018459 100644 --- a/audit-docs/coupled_duplex_verification_2026_10_20.json +++ b/audit-docs/coupled_duplex_verification_2026_10_20.json @@ -22,7 +22,7 @@ }, "extension": { "build_log": "/tmp/genairr_coupled_release_build.log", - "path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "path": "/src/GenAIRR/_engine.abi3.so", "sha256": "a59585ffc7e20db37e858d3ec1be13484b12d4367453afcd04cc72421187a302" }, "local_model_unchanged": "experimental_duplex_repair_v1", diff --git a/audit-docs/duplex_experiment_compile_benchmark_2026_10_20.json b/audit-docs/duplex_experiment_compile_benchmark_2026_10_20.json index 4827fd27..c93c8b8f 100644 --- a/audit-docs/duplex_experiment_compile_benchmark_2026_10_20.json +++ b/audit-docs/duplex_experiment_compile_benchmark_2026_10_20.json @@ -3,7 +3,7 @@ "branch": "fix/biological-metadata-correctness", "commit": "6413a6bbf321a492f6c79f679b71506dabf101ae", "dirty": true, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "6e45f13f74a170caa8f9fb861192350659586b315812deb2701fc0a9ad1909d4", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", @@ -25,7 +25,7 @@ "compile_peak_increase_kib": 832, "compile_seconds": 0.0009993479907279834, "distinct_histories_sampled": 8, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000007", "model_digest": "sha256:c257de0b03b439eb6cd5fca7fd9c633c12829e5134bd95be4909485c957ed9c6", @@ -51,7 +51,7 @@ "compile_peak_increase_kib": 960, "compile_seconds": 0.001016237001749687, "distinct_histories_sampled": 8, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000007", "model_digest": "sha256:c257de0b03b439eb6cd5fca7fd9c633c12829e5134bd95be4909485c957ed9c6", @@ -77,7 +77,7 @@ "compile_peak_increase_kib": 832, "compile_seconds": 0.0010169219895033166, "distinct_histories_sampled": 8, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000000", "model_digest": "sha256:c257de0b03b439eb6cd5fca7fd9c633c12829e5134bd95be4909485c957ed9c6", @@ -103,7 +103,7 @@ "compile_peak_increase_kib": 1728, "compile_seconds": 0.0032104130077641457, "distinct_histories_sampled": 32, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000030", "model_digest": "sha256:d53f6a07e73d86006c0f3efafac6f8991bf9f610fe99bd43c5f3fb858ee554a1", @@ -129,7 +129,7 @@ "compile_peak_increase_kib": 1728, "compile_seconds": 0.00322621499071829, "distinct_histories_sampled": 32, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000030", "model_digest": "sha256:d53f6a07e73d86006c0f3efafac6f8991bf9f610fe99bd43c5f3fb858ee554a1", @@ -155,7 +155,7 @@ "compile_peak_increase_kib": 1600, "compile_seconds": 0.004085260006831959, "distinct_histories_sampled": 32, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000003", "model_digest": "sha256:d53f6a07e73d86006c0f3efafac6f8991bf9f610fe99bd43c5f3fb858ee554a1", @@ -181,7 +181,7 @@ "compile_peak_increase_kib": 4544, "compile_seconds": 0.012196705996757373, "distinct_histories_sampled": 128, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000120", "model_digest": "sha256:8ae42bb7db72df5b22b1a764d44e24d83579a24d5fe2ef36b36e06ecd0cdcf55", @@ -207,7 +207,7 @@ "compile_peak_increase_kib": 4544, "compile_seconds": 0.012122743006329983, "distinct_histories_sampled": 128, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000122", "model_digest": "sha256:8ae42bb7db72df5b22b1a764d44e24d83579a24d5fe2ef36b36e06ecd0cdcf55", @@ -233,7 +233,7 @@ "compile_peak_increase_kib": 4544, "compile_seconds": 0.011953912995522842, "distinct_histories_sampled": 128, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000012", "model_digest": "sha256:8ae42bb7db72df5b22b1a764d44e24d83579a24d5fe2ef36b36e06ecd0cdcf55", @@ -259,7 +259,7 @@ "compile_peak_increase_kib": 960, "compile_seconds": 0.001114522005082108, "distinct_histories_sampled": 8, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000002", "model_digest": "sha256:d2cdb7434de5271ab3b73ebd9196671672f54da9d9143caa076b371df52dd3fd", @@ -285,7 +285,7 @@ "compile_peak_increase_kib": 960, "compile_seconds": 0.0011133459920529276, "distinct_histories_sampled": 8, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000005", "model_digest": "sha256:d2cdb7434de5271ab3b73ebd9196671672f54da9d9143caa076b371df52dd3fd", @@ -311,7 +311,7 @@ "compile_peak_increase_kib": 832, "compile_seconds": 0.0011279000027570873, "distinct_histories_sampled": 8, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000000", "model_digest": "sha256:d2cdb7434de5271ab3b73ebd9196671672f54da9d9143caa076b371df52dd3fd", @@ -337,7 +337,7 @@ "compile_peak_increase_kib": 1728, "compile_seconds": 0.00369748700177297, "distinct_histories_sampled": 32, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000026", "model_digest": "sha256:03ffd2e6189ca098690f4148b1ae7babb8e3c6279124d4354938313812733dcf", @@ -363,7 +363,7 @@ "compile_peak_increase_kib": 1728, "compile_seconds": 0.0036066010070499033, "distinct_histories_sampled": 32, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000029", "model_digest": "sha256:03ffd2e6189ca098690f4148b1ae7babb8e3c6279124d4354938313812733dcf", @@ -389,7 +389,7 @@ "compile_peak_increase_kib": 1728, "compile_seconds": 0.0036449659965001047, "distinct_histories_sampled": 32, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000000", "model_digest": "sha256:03ffd2e6189ca098690f4148b1ae7babb8e3c6279124d4354938313812733dcf", @@ -415,7 +415,7 @@ "compile_peak_increase_kib": 4544, "compile_seconds": 0.013903794999350794, "distinct_histories_sampled": 128, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000122", "model_digest": "sha256:9e9bec8822e3e6bbbf82831f0ee9dcbf6f48b2f0402a15a0a399fd2ae3a627d3", @@ -441,7 +441,7 @@ "compile_peak_increase_kib": 4544, "compile_seconds": 0.013659849995747209, "distinct_histories_sampled": 128, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000125", "model_digest": "sha256:9e9bec8822e3e6bbbf82831f0ee9dcbf6f48b2f0402a15a0a399fd2ae3a627d3", @@ -467,7 +467,7 @@ "compile_peak_increase_kib": 4544, "compile_seconds": 0.013803929003188387, "distinct_histories_sampled": 128, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "first_record": { "history_id": "history-000008", "model_digest": "sha256:9e9bec8822e3e6bbbf82831f0ee9dcbf6f48b2f0402a15a0a399fd2ae3a627d3", diff --git a/audit-docs/duplex_family_benchmark_2026_10_20.json b/audit-docs/duplex_family_benchmark_2026_10_20.json index c8032829..e95fe183 100644 --- a/audit-docs/duplex_family_benchmark_2026_10_20.json +++ b/audit-docs/duplex_family_benchmark_2026_10_20.json @@ -4,7 +4,7 @@ "branch": "fix/biological-metadata-correctness", "commit": "a75db8cf75443d29e0031c40bb6d8994adcadc49", "dirty": true, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "e3edd39a59b3c0743d25ce03a407f65738eb4b13ef36637f152ecfd8658b08f6", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", @@ -53,7 +53,7 @@ "branch": "fix/biological-metadata-correctness", "commit": "a75db8cf75443d29e0031c40bb6d8994adcadc49", "dirty": true, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "e3edd39a59b3c0743d25ce03a407f65738eb4b13ef36637f152ecfd8658b08f6", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", @@ -116,7 +116,7 @@ "branch": "fix/biological-metadata-correctness", "commit": "a75db8cf75443d29e0031c40bb6d8994adcadc49", "dirty": true, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "e3edd39a59b3c0743d25ce03a407f65738eb4b13ef36637f152ecfd8658b08f6", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", @@ -179,7 +179,7 @@ "branch": "fix/biological-metadata-correctness", "commit": "a75db8cf75443d29e0031c40bb6d8994adcadc49", "dirty": true, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "e3edd39a59b3c0743d25ce03a407f65738eb4b13ef36637f152ecfd8658b08f6", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", @@ -242,7 +242,7 @@ "branch": "fix/biological-metadata-correctness", "commit": "a75db8cf75443d29e0031c40bb6d8994adcadc49", "dirty": true, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "e3edd39a59b3c0743d25ce03a407f65738eb4b13ef36637f152ecfd8658b08f6", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", @@ -305,7 +305,7 @@ "branch": "fix/biological-metadata-correctness", "commit": "a75db8cf75443d29e0031c40bb6d8994adcadc49", "dirty": true, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "e3edd39a59b3c0743d25ce03a407f65738eb4b13ef36637f152ecfd8658b08f6", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", @@ -368,7 +368,7 @@ "branch": "fix/biological-metadata-correctness", "commit": "a75db8cf75443d29e0031c40bb6d8994adcadc49", "dirty": true, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "e3edd39a59b3c0743d25ce03a407f65738eb4b13ef36637f152ecfd8658b08f6", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", diff --git a/audit-docs/duplex_projection_verification_2026_10_20.json b/audit-docs/duplex_projection_verification_2026_10_20.json index ac8c2c68..68404d38 100644 --- a/audit-docs/duplex_projection_verification_2026_10_20.json +++ b/audit-docs/duplex_projection_verification_2026_10_20.json @@ -2,7 +2,7 @@ "baseline_commit": "c480374e0fffb8991c03a113017578847992e7c5", "branch": "fix/biological-metadata-correctness", "commands": [ - "VIRTUAL_ENV=/home/thomas/Desktop/GenAIRR/.venv .venv/bin/maturin develop --release", + "VIRTUAL_ENV=/.venv .venv/bin/maturin develop --release", "cargo test --manifest-path engine_rs/Cargo.toml -q", "cargo test --release --manifest-path engine_rs/Cargo.toml -q", ".venv/bin/python -m pytest -q tests/test_duplex_projection.py tests/test_coupled_duplex_repair.py tests/test_experimental_duplex_repair.py tests/test_coding_end_models.py tests/test_metadata_np_estimator_regressions.py tests/test_np_length_estimation_contract.py tests/test_np_length_estimation_implementation.py tests/test_np_base_model_estimation_contract.py tests/test_np_base_model_estimation_implementation.py tests/test_p_nucleotide_length_estimation_contract.py tests/test_p_nucleotide_length_estimation_implementation.py tests/test_trim_distribution_estimation_contract.py tests/test_trim_distribution_estimation_implementation.py tests/test_productivity_assessment.py tests/test_productivity_trusted_landmarks.py", @@ -22,7 +22,7 @@ }, "extension": { "build_log": "/tmp/genairr_duplex_projection_build_verified.log", - "path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "path": "/src/GenAIRR/_engine.abi3.so", "sha256": "3b601a4d6df473e8c45bc3b31da1be8148a67899d594f248e823e49e40f9950c" }, "model": "duplex_airr_projection_v1", diff --git a/audit-docs/durable_family_benchmark_2026_10_21.json b/audit-docs/durable_family_benchmark_2026_10_21.json index 6af79e07..f066de65 100644 --- a/audit-docs/durable_family_benchmark_2026_10_21.json +++ b/audit-docs/durable_family_benchmark_2026_10_21.json @@ -7,7 +7,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -196,7 +196,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -270,7 +270,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -314,7 +314,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -376,7 +376,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -430,7 +430,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -503,7 +503,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -547,7 +547,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -609,7 +609,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -663,7 +663,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -738,7 +738,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -782,7 +782,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -844,7 +844,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -898,7 +898,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -971,7 +971,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -1015,7 +1015,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -1077,7 +1077,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -1131,7 +1131,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -1210,7 +1210,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -1254,7 +1254,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -1316,7 +1316,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -1370,7 +1370,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -1444,7 +1444,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -1488,7 +1488,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" @@ -1550,7 +1550,7 @@ "source_sha256": "bb602141a97971330c88db8e6457df8328ffe573a13cd9960946ff06ea7aa8f3", "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": "894741a18ceac70c5ca4b295a07eb59bc1f6fa9e9baba1b0c35a72c94060d5e9", - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "a31be980a932d7333714f2ddf51c78ff9296943450d2d105313638bcff482244", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]", "platform": "Linux-7.0.0-38-generic-x86_64-with-glibc2.39" diff --git a/audit-docs/durable_family_replay_verification_2026_10_21.json b/audit-docs/durable_family_replay_verification_2026_10_21.json index d57dbf5c..de5b7995 100644 --- a/audit-docs/durable_family_replay_verification_2026_10_21.json +++ b/audit-docs/durable_family_replay_verification_2026_10_21.json @@ -52,7 +52,7 @@ "status": 0, "summary": "1569 passed, 103 warnings in 355.83s (0:05:55)", "command": [ - "/home/thomas/Desktop/GenAIRR/.venv/bin/python", + "/.venv/bin/python", "-m", "pytest", "tests/test_airr_record_validator.py", @@ -210,7 +210,7 @@ "status": 0, "summary": "1569 passed, 68 warnings in 219.92s (0:03:39)", "command": [ - "/home/thomas/Desktop/GenAIRR/.venv/bin/python", + "/.venv/bin/python", "-m", "pytest", "tests/test_allele_usage_estimation_implementation.py", diff --git a/audit-docs/original_coding_end_stress_2026_10_20.json b/audit-docs/original_coding_end_stress_2026_10_20.json index e583c85e..e0e004e2 100644 --- a/audit-docs/original_coding_end_stress_2026_10_20.json +++ b/audit-docs/original_coding_end_stress_2026_10_20.json @@ -158,7 +158,7 @@ "replay_equal": true } ], - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "guarded_rejections": [ { "config": "mouse_igk", diff --git a/audit-docs/original_coding_end_verification_2026_10_20.json b/audit-docs/original_coding_end_verification_2026_10_20.json index 8c53948f..2a72cc76 100644 --- a/audit-docs/original_coding_end_verification_2026_10_20.json +++ b/audit-docs/original_coding_end_verification_2026_10_20.json @@ -3,7 +3,7 @@ "branch": "fix/biological-metadata-correctness", "model": "original_coding_end_v1", "external_dependencies_added": false, - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "extension_sha256": "0a364e94191e001517166c07dea8d9343d8ee01abdb16586a6ac7f86f985de25", "python": { "passed": 2636, diff --git a/audit-docs/productive_p_support_verification_2026_10_20.json b/audit-docs/productive_p_support_verification_2026_10_20.json index a2cfc34c..f20babb8 100644 --- a/audit-docs/productive_p_support_verification_2026_10_20.json +++ b/audit-docs/productive_p_support_verification_2026_10_20.json @@ -925,8 +925,8 @@ "environment": { "concurrent_workload": "Root agent running full Python and Rust test suites; timings are observational, not isolated benchmarks.", "engine_mtime_utc": "2026-10-20T07:01:17.103705+00:00", - "engine_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", - "package_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/__init__.py", + "engine_path": "/src/GenAIRR/_engine.abi3.so", + "package_path": "/src/GenAIRR/__init__.py", "python": "3.12.3 (main, Aug 31 2026, 10:18:26) [GCC 13.3.0]" }, "method": { diff --git a/audit-docs/productivity_indel_cross_locus_2026_10_20.json b/audit-docs/productivity_indel_cross_locus_2026_10_20.json index 6a063fd8..e9316d28 100644 --- a/audit-docs/productivity_indel_cross_locus_2026_10_20.json +++ b/audit-docs/productivity_indel_cross_locus_2026_10_20.json @@ -158,7 +158,7 @@ "replay_equal": true } ], - "extension_path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "extension_path": "/src/GenAIRR/_engine.abi3.so", "guarded_rejections": [ { "config": "mouse_igk", diff --git a/audit-docs/productivity_indel_repair_verification_2026_10_20.json b/audit-docs/productivity_indel_repair_verification_2026_10_20.json index 3a81aee1..69b37c8d 100644 --- a/audit-docs/productivity_indel_repair_verification_2026_10_20.json +++ b/audit-docs/productivity_indel_repair_verification_2026_10_20.json @@ -53,7 +53,7 @@ "extension": { "build_command": ".venv/bin/maturin develop --release", "build_log": "/tmp/genairr_correctness_release_build_final.log", - "path": "/home/thomas/Desktop/GenAIRR/src/GenAIRR/_engine.abi3.so", + "path": "/src/GenAIRR/_engine.abi3.so", "sha256": "03b21c9769157a78eb4797477be6acfb02265acdd85d58ef8b362950ac8d0e78" }, "fixed_precursors": { diff --git a/tools/metadata_biology_audit/duplex_compile_benchmark.py b/tools/metadata_biology_audit/duplex_compile_benchmark.py index 259a1cff..e98de09c 100644 --- a/tools/metadata_biology_audit/duplex_compile_benchmark.py +++ b/tools/metadata_biology_audit/duplex_compile_benchmark.py @@ -26,6 +26,21 @@ ROOT = Path(__file__).resolve().parents[2] +def _relative_extension_path() -> str: + """The extension's path relative to the repo, so a record names no machine. + + An absolute path identifies the author's filesystem, not the artifact; the + sha256 beside this field is what identifies the binary. + """ + extension = Path(_engine.__file__).resolve() + try: + return str(extension.relative_to(ROOT)) + except ValueError: + return extension.name + + + + def _rss_kib(): value = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss return value / 1024 if platform.system() == "Darwin" else value @@ -94,7 +109,7 @@ def _worker(rows, contexts, records, seed): "distinct_histories_sampled": len(counts), "sampled_records": sum(counts.values()), "max_preflight_operations": model.max_preflight_operations, "max_preflight_bases": model.max_preflight_bases, - "extension_path": str(Path(_engine.__file__).resolve()), "first_record": first, + "extension_path": _relative_extension_path(), "first_record": first, } @@ -118,7 +133,7 @@ def git(*args): "dirty": bool(git("status", "--porcelain")), "source_sha256": digest.hexdigest(), "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", - "extension_path": str(extension), + "extension_path": _relative_extension_path(), "extension_sha256": hashlib.sha256(extension.read_bytes()).hexdigest(), "python": sys.version, "platform": platform.platform(), } diff --git a/tools/metadata_biology_audit/duplex_family_benchmark.py b/tools/metadata_biology_audit/duplex_family_benchmark.py index 8a41ae49..9966f3dd 100644 --- a/tools/metadata_biology_audit/duplex_family_benchmark.py +++ b/tools/metadata_biology_audit/duplex_family_benchmark.py @@ -26,6 +26,21 @@ ROOT = Path(__file__).resolve().parents[2] + + +def _relative_extension_path() -> str: + """The extension's path relative to the repo, so a record names no machine. + + An absolute path identifies the author's filesystem, not the artifact; the + sha256 beside this field is what identifies the binary. + """ + extension = Path(_engine.__file__).resolve() + try: + return str(extension.relative_to(ROOT)) + except ValueError: + return extension.name + + PARAMETERS = { "n_clones": 2, "n_max": 32, @@ -72,7 +87,7 @@ def git(*args): "source_sha256": digest.hexdigest(), "source_hash_scope": "engine_rs/src/**/*.rs,src/GenAIRR/**/*.py,engine_rs/Cargo.toml,Cargo.lock", "benchmark_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), - "extension_path": str(extension), + "extension_path": _relative_extension_path(), "extension_sha256": hashlib.sha256(extension.read_bytes()).hexdigest(), "python": sys.version, "platform": platform.platform(), diff --git a/tools/metadata_biology_audit/original_end_stress.py b/tools/metadata_biology_audit/original_end_stress.py index d4b75ea1..23ededcd 100644 --- a/tools/metadata_biology_audit/original_end_stress.py +++ b/tools/metadata_biology_audit/original_end_stress.py @@ -12,6 +12,8 @@ import GenAIRR as ga from GenAIRR import _engine +ROOT = Path(__file__).resolve().parents[2] + CONFIGS = [f"{species}_{locus}" for species in ("human", "mouse") for locus in ("igh", "igk", "igl", "tcra", "tcrb", "tcrd", "tcrg")] COMPLEMENT = str.maketrans("ACGTacgtNn", "TGCAtgcaNn") @@ -61,6 +63,18 @@ def check(record, reference): assert len(origins) == max(-outcome, 0) +def _relative_extension_path() -> str: + """The extension's path relative to the repo, so a record names no machine. + + An absolute path identifies the author's filesystem, not the artifact; the + sha256 beside this field is what identifies the binary. + """ + extension = Path(_engine.__file__).resolve() + try: + return str(extension.relative_to(ROOT)) + except ValueError: + return extension.name + def main(): parser = argparse.ArgumentParser() parser.add_argument("--records", type=int, default=30) @@ -130,7 +144,7 @@ def main(): results.append(item) print(json.dumps(item), flush=True) artifact = {"seed": args.seed, "records": sum(r["records"] for r in results), - "errors": 0, "profiles": results, "extension_path": _engine.__file__, + "errors": 0, "profiles": results, "extension_path": _relative_extension_path(), "guarded_rejections": rejections, "productive_indel_count": 2 if args.productive_indels else 0, "reference_policy": "allow curatable reference annotations; productive support still requires valid retained anchors and coding sequence", From e67ac9741b8e5a28088aa75b6e9fb927b6ae64d2 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 5 Nov 2026 21:50:45 +0200 Subject: [PATCH 3/7] docs: the build files no longer point at a document nobody can read requirements-docs.txt and two Makefile comments referenced docs/docs_mkdocs_migration_plan.md. docs/ is gitignored, so that document is not in the repository and never was: a reader following the reference finds nothing. requirements-docs.txt also gave its own install path as docs/requirements-docs.txt, which does not exist either; the file is at the repository root. The Makefile's docs comment additionally said the contributor-only markdowns live at docs/*.md -- they are at audit-docs/ -- and that 'docs-build' should be run in CI 'once Phase 6 cutover lands'. It already is, by .github/workflows/deploy-docs.yml, on every docs change. --- Makefile | 12 ++++++------ requirements-docs.txt | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 7df247b8..db145172 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ help: @echo " make validate-full Tier 2: correctness + performance budgets" @echo " make validate-release Tier 3: full + wheel build + golden trace compat" @echo "" - @echo "Documentation (MkDocs Material — see docs/docs_mkdocs_migration_plan.md):" + @echo "Documentation (MkDocs Material — source at site_docs/):" @echo " make docs-install Install docs-build deps from requirements-docs.txt" @echo " make docs-build mkdocs build --strict (fails on broken links / nav drift)" @echo " make docs-serve mkdocs serve (live-reload local preview on :8000)" @@ -78,15 +78,15 @@ validate-release: validate-full $(PYTHON) -m pytest tests/components/io/test_trace_file_compat.py -q maturin build --release -# --- Docs (MkDocs Material — Phase 1 onwards) --------------------- +# --- Docs (MkDocs Material) --------------------------------------- # # The docs source lives at site_docs/. The contributor-only audit/ -# design markdowns at docs/*.md stay rendered on GitHub and are -# linked from the future Architecture section of the site. +# design markdowns at audit-docs/*.md are rendered on GitHub and are +# linked from the site's Architecture section. # # `docs-build` uses `--strict` so broken nav links / missing files -# / dead anchors fail the build. Run this in CI once Phase 6 -# cutover lands. +# / dead anchors fail the build. CI runs it on every docs change +# (.github/workflows/deploy-docs.yml). DOCS_REQ ?= requirements-docs.txt diff --git a/requirements-docs.txt b/requirements-docs.txt index 4555bea3..4fbe197d 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,9 +1,8 @@ # Documentation-build dependencies for the MkDocs Material site. # -# Phase 1 of the migration plan in `docs/docs_mkdocs_migration_plan.md`. # Install via: # -# pip install -r docs/requirements-docs.txt +# pip install -r requirements-docs.txt # # Or via `make docs-install`. # From e6708689c5d7a87cf86199b725dbacce4cba7dbe Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 5 Nov 2026 21:51:01 +0200 Subject: [PATCH 4/7] docs: every example and every number re-measured against the code 422 commits since 2.3.1 moved what the documentation describes, and the documentation did not follow. Each claim here was checked by running it, not by reading the code around it. Examples that could not work as written: the README's custom RefDataConfig block had a literal '.' in a nucleotide string and an anchor past the end of a 21-nt sequence; two species examples raise on a quarantined catalogue entry without .allow_curatable_refdata(); a paired-end example asked for a 450-nt insert on a molecule that is 352-436 nt; an NpBaseModelSpec example passed a set where a 4x4 matrix belongs; a sequencing-errors block printed the previous block's record because it never rebound its variable. Claims that were false rather than stale: productive_only() was documented as refusing TCR references, and it does not; S5F was attributed to Shapiro rather than Yaari et al. (2013), and described as concentrating in the CDRs, which its kernel has no way to do; a paired-end window was said to clip to what is available, when it raises; sequence case was documented inverted. Quantities were re-counted rather than adjusted: 40 names in __all__, 106 catalogue entries of which 80 are quarantined, nine performance workloads, six estimators, over 120 validated AIRR fields. n_max and lambda_base are described as the engine now defines them: n_max is the population a lineage settles at under Beverton-Holt, not a ceiling, and lambda_base is the Poisson mean for a founder-baseline cell in an empty germinal center rather than the mean once damping or selection applies. A wrong-length affinity target is refused, not scored over its prefix. The determinism promise is stated with its scope: same seed and same determinism_version gives byte-identical output, checked by the reproducibility goldens on Linux, macOS and Windows across Python 3.10-3.13, with aarch64 named as shipping a wheel that nothing compares. It previously promised platform independence with no version and no scope at all. expand_clones()'s DeprecationWarning appeared on no page; genotypes appeared nowhere in the README. Both do now. CONTRIBUTING.md said GenAIRR ships two PyPI packages and that a release updates setup.py; it ships one wheel and there is no setup.py. Thirty-seven unwritten pages carried a placeholder linking to a gitignored internal plan, so every one of those links was a 404 on the published site. Each now points at the page that actually covers the material. --- CONTRIBUTING.md | 35 +-- README.md | 211 +++++++++++++----- mkdocs.yml | 55 +++-- site_docs/architecture/adding-a-pass.md | 12 +- site_docs/architecture/audit-index.md | 12 +- site_docs/architecture/engine.md | 12 +- site_docs/architecture/validation-matrix.md | 12 +- site_docs/cartridges/build-from-fasta.md | 12 +- site_docs/cartridges/empirical-models.md | 12 +- .../cartridges/estimators/allele-usage.md | 12 +- site_docs/cartridges/estimators/index.md | 12 +- .../cartridges/estimators/np-base-model.md | 12 +- site_docs/cartridges/estimators/np-length.md | 12 +- .../estimators/p-nucleotide-length.md | 12 +- .../estimators/trim-distributions.md | 12 +- site_docs/cartridges/rules-and-identity.md | 12 +- .../cartridges/v-subregion-annotations.md | 12 +- site_docs/concepts/airr-record.md | 32 +++ site_docs/concepts/contracts.md | 12 +- site_docs/concepts/live-call.md | 12 +- site_docs/concepts/persistent-ir.md | 12 +- site_docs/concepts/pipeline.md | 12 +- site_docs/concepts/reference-cartridge.md | 29 ++- site_docs/demo.md | 41 ++-- .../getting-started/first-airr-record.md | 49 ++-- site_docs/getting-started/installation.md | 15 +- ...esson-1-vdj-recombination-from-bases-up.md | 12 +- .../lesson-2-the-pipeline-scrubber.md | 12 +- .../lesson-3-s5f-mutation-isnt-uniform.md | 12 +- ...lesson-4-sequencing-artifacts-dialed-in.md | 12 +- .../lesson-5-the-ground-truth-payoff.md | 12 +- site_docs/getting-started/quick-start.md | 4 +- site_docs/guides/biology-map.md | 12 +- site_docs/guides/build-config.md | 12 +- site_docs/guides/build-reference-cartridge.md | 9 +- site_docs/guides/cartridge-manifest-report.md | 48 ++-- site_docs/guides/clonal-families.md | 14 +- site_docs/guides/clonal-lineage.md | 7 +- site_docs/guides/corruption-sequencing.md | 21 +- site_docs/guides/estimate-cartridge-models.md | 54 +++-- site_docs/guides/experiment-builder.md | 13 +- site_docs/guides/export.md | 14 +- site_docs/guides/genotype-priors.md | 11 +- site_docs/guides/genotype.md | 24 +- site_docs/guides/index.md | 33 ++- site_docs/guides/junction-additions.md | 16 +- site_docs/guides/paired-end-fastq.md | 38 ++-- site_docs/guides/productive.md | 14 +- site_docs/guides/recombination-editing.md | 8 +- site_docs/guides/recombination-junction.md | 8 +- site_docs/guides/shm-targeting.md | 30 ++- site_docs/guides/streaming.md | 12 +- site_docs/guides/trace-replay.md | 35 ++- site_docs/guides/tune-corruption.md | 12 +- site_docs/guides/v-usage.md | 12 +- site_docs/guides/with-metadata.md | 12 +- site_docs/learn.md | 2 +- site_docs/reference/experiment.md | 6 +- site_docs/reference/index.md | 28 ++- .../reference/reference-cartridge-builder.md | 4 + site_docs/reference/simulation-result.md | 8 +- site_docs/stylesheets/extra.css | 15 +- site_docs/validation/audit-realism.md | 12 +- site_docs/validation/benchmark-aligner.md | 12 +- site_docs/validation/compare-shm.md | 12 +- site_docs/validation/index.md | 19 +- site_docs/validation/replay.md | 12 +- site_docs/validation/reproduce.md | 12 +- site_docs/validation/smoke-test.md | 12 +- site_docs/validation/trace-introspection.md | 12 +- site_docs/validation/two-layer-model.md | 12 +- site_docs/validation/validate-records.md | 29 ++- 72 files changed, 762 insertions(+), 661 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 04478886..eab9c0ea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,11 +38,11 @@ make test-rust # cargo test │ ├── passes/ # Concrete passes (recombination, mutation, corruption) │ ├── contract/ # Contract trait + filter/verify implementations │ ├── dist.rs # Distribution trait + concrete distributions -│ └── python/ # PyO3 wrappers exposed as `genairr_engine` +│ └── python/ # PyO3 wrappers exposed as `GenAIRR._engine` └── src/GenAIRR/ ├── experiment.py # Experiment DSL (public entry point) ├── dataconfig/ # DataConfig + species metadata - ├── data/ # 106+ built-in DataConfig pickles + ├── data/ # the 106 built-in DataConfig pickles └── utilities/ # Helpers, IMGT regions ``` @@ -55,27 +55,34 @@ cd engine_rs && cargo test # Rust tests (unit + integration) ## Release Process -GenAIRR ships two PyPI packages: - -- **`genairr_engine`** — Rust kernel built per OS/arch via maturin (abi3-py39, so one wheel per platform covers Python 3.9+). -- **`GenAIRR`** — pure Python (single universal wheel + sdist). +GenAIRR ships as **one** PyPI package. `maturin` builds the Rust kernel +(`engine_rs/`) and the Python wrappers (`src/GenAIRR/`) into a single wheel per +OS/arch; the kernel is exposed as the private submodule `GenAIRR._engine`. The +extension is `abi3-py39`, so one wheel per platform covers Python 3.9+. GitHub Actions handles CI and publishing: -1. **Tests** run on every push/PR (Linux, macOS, Windows × Python 3.10–3.12) plus a separate `cargo test` job. -2. **Wheels** are built when a tag is pushed: maturin builds engine wheels for Linux x86_64+aarch64, macOS x86_64+arm64, Windows x64; setuptools builds a universal Python wheel. +1. **Tests** run on every push/PR (Linux, macOS, Windows × Python 3.10–3.13) + plus a separate `cargo test` job. +2. **Wheels** are built when a tag is pushed, for Linux x86_64+aarch64, macOS + x86_64+arm64 and Windows x64, alongside a maturin sdist. Each built wheel is + installed and smoke-run on Python 3.9 and 3.13 on all three platforms before + anything is published. 3. **Publishing** to PyPI happens automatically on GitHub Release or version tag. ### Creating a Release -1. Update the version in `setup.py` and `engine_rs/Cargo.toml` (keep them in sync). -2. Update `CHANGELOG.md`. -3. Commit, tag, and push: +1. Update `CHANGELOG.md` and `MIGRATING.md`. If the release changes what a seed + produces, bump `DETERMINISM_VERSION` in `engine_rs/src/determinism.rs`, add + its row to the table in that file's header, and regenerate the goldens. +2. Tag and push — the release workflow derives the version from the tag name and + rewrites `pyproject.toml` and `engine_rs/Cargo.toml` itself, so there is no + version to edit by hand: ```bash - git tag v1.0.0 - git push origin v1.0.0 + git tag v2.4.0 + git push origin v2.4.0 ``` -4. Create a GitHub Release from the tag. +3. Create a GitHub Release from the tag. ### Prerequisites diff --git a/README.md b/README.md index 71f6005d..ab5250e4 100644 --- a/README.md +++ b/README.md @@ -57,20 +57,21 @@ result = ( len(result) # 1000 rec = result[0] -rec["sequence"] # 'gaggtgcagctggtggagtctgggggaggc...' (nucleotide) -rec["sequence_aa"] # 'EVQLVESGGGLVQPGGSLRLSCSAS...' (translated) +rec["sequence"] # 'caggtgcagctggtgcagtctggggctgaggtg...' (nucleotide) +rec["sequence_aa"] # 'QVQLVQSGAEVKKPGASVKVSCKASGYTFTSYY...' (translated) rec["locus"] # 'IGH' -rec["v_call"] # 'IGHVF10-G38*04' (generating allele) +rec["v_call"] # 'IGHVF6-G23*01' (generating allele) rec["recalled_v_call"] # post-mutation inference; comma-separated when tied -rec["d_call"] # 'IGHD2-15*01' -rec["j_call"] # 'IGHJ2*01' -rec["junction_aa"] # 'CVKDDGNRGYCSGGSCYGRCCALDYWYFDLW' +rec["d_call"] # 'IGHD1-20*01' +rec["j_call"] # 'IGHJ1*01' +rec["junction_aa"] # 'CARCRTRLTRAYHITGTAHFQHW' rec["productive"] # True rec["v_identity"] # 1.0 (matches/total over the V segment) rec["n_mutations"] # 0 +rec["determinism_version"] # 4 (which sampling model drew this record) # Export in any of the standard formats. TSV/FASTA/FASTQ are dependency-free; -# to_dataframe() needs pandas (pip install GenAIRR[all]). +# to_dataframe() needs pandas (pip install pandas). result.to_tsv("repertoire.tsv") # AIRR-spec TSV (50+ columns) result.to_fasta("sequences.fasta") # FASTA with v_call/j_call in the headers result.to_fastq("sequences.fastq") # FASTQ with illumina-shaped quality scores @@ -96,11 +97,11 @@ result = ( result.to_paired_fastq("reads_R1.fastq", "reads_R2.fastq") ``` -Headers use the AIRR record's `sequence_id` with the universally-portable `/1` and `/2` suffix (`@seq0/1` and `@seq0/2`) — no `|`-pipe metadata that some aligners (STAR < 2.7) split on. R1 / R2 sequences are written verbatim from the AIRR `r1_sequence` / `r2_sequence` fields (R2 is already reverse-complemented at projection time; the writer does NOT apply a second flip). Default `overwrite=False` refuses to clobber existing files; pass `overwrite=True` to replace. Quality strings use the same pluggable models as `to_fastq` — `quality="illumina"` (default trapezoid shape, applied per read) or `quality="constant"` with `q=...`. See [`docs/fastq_export_design.md`](audit-docs/fastq_export_design.md). +Headers use the AIRR record's `sequence_id` with the universally-portable `/1` and `/2` suffix (`@seq0/1` and `@seq0/2`) — no `|`-pipe metadata that some aligners (STAR < 2.7) split on. R1 / R2 sequences are written verbatim from the AIRR `r1_sequence` / `r2_sequence` fields (R2 is already reverse-complemented at projection time; the writer does NOT apply a second flip). Default `overwrite=False` refuses to clobber existing files; pass `overwrite=True` to replace. Quality strings use the same pluggable models as `to_fastq` — `quality="illumina"` (default trapezoid shape, applied per read) or `quality="constant"` with `q=...`. The requested `insert_size` has to fit inside the molecule: on human IGH a record is roughly 350–440 nt, so an insert of 450 leaves the R2 window off the end of the sequence and `validate_records` reports `ReadWindowOutOfBounds`. See [`audit-docs/fastq_export_design.md`](audit-docs/fastq_export_design.md). `Experiment.on(...)` accepts **a config-name string** (e.g. `"human_igh"`, `"mouse_tcrb"`), **a `DataConfig`** loaded from the bundled species pickles, or **a `RefDataConfig`** for [custom reference data](#custom-reference-data). `.productive_only()` is the constraint-aware bundle — covered in the next section. Drop it to sample without the productivity constraint; records then report true, false, or unresolved productivity according to their observed evidence. -> See the full walkthrough in the docs: [Quick Start](https://mutejester.github.io/GenAIRR/lesson-1.html) · [Interpreting Results](https://mutejester.github.io/GenAIRR/concept-airr-record.html) +> See the full walkthrough in the docs: [Quick Start](https://mutejester.github.io/GenAIRR/getting-started/quick-start.html) · [Your first AIRR record](https://mutejester.github.io/GenAIRR/getting-started/first-airr-record.html) · [Interpreting Results](https://mutejester.github.io/GenAIRR/concepts/airr-record.html) --- @@ -134,24 +135,23 @@ result = ( .ambiguous_base_calls(count=(0, 2)) # 4. Stamp arbitrary metadata onto every record. .with_metadata(experiment_id="exp001", tissue="peripheral_blood") - # The founder is drawn under the productive() constraint. Accumulated - # SHM and corruption passes can still introduce stop codons / - # frameshifts in individual cells, so expect ~75% productive under - # this SHM + corruption load — that mirrors real wet-lab data, where a - # productive B-cell can sequence as a non-productive read. + # The constraint is carried, not just applied to the founder: each + # lineage division and each corruption pass draws only substitutions, + # indels and masks that keep the cell productive, so every emitted + # record is productive by construction. .productive_only().run_records(seed=42) ) -len(result) # 415 (sampled live cells across 50 trees) -sum(1 for r in result if r["productive"]) # 316 (~76% under this SHM + corruption load) +len(result) # 440 (sampled live cells across 50 trees) +sum(1 for r in result if r["productive"]) # 440 (all of them — see the note above) len(result.lineage_trees) # 50 (one ground-truth tree per clone) # Same clone, different cells — shared founder V(D)J, # independent SHM + errors: result[0]["clone_id"], result[1]["clone_id"] # (0, 0) -result[0]["v_call"], result[1]["v_call"] # both 'IGHVF10-G51*05' -result[0]["n_mutations"], result[1]["n_mutations"] # (14, 11) — independent SHM -result[0]["n_pcr_errors"], result[1]["n_pcr_errors"] # (3, 0) — independent errors +result[0]["v_call"], result[1]["v_call"] # both 'IGHVF6-G21*11' +result[0]["n_mutations"], result[1]["n_mutations"] # (11, 17) — independent SHM +result[0]["n_pcr_errors"], result[1]["n_pcr_errors"] # (1, 0) — independent errors # Custom metadata propagated: result[0]["experiment_id"], result[0]["tissue"] # ('exp001', 'peripheral_blood') @@ -168,7 +168,7 @@ GenAIRR models clonal structure three ways. Pick by what you need: |---|---|---| | **`clonal_lineage`** | BCR affinity-maturation **trees** — generation-synchronous birth–death, per-division S5F SHM, optional affinity selection, sample + genotype-collapse | B-cell lineage trees with **ground truth** (Newick/FASTA per clone), benchmarking lineage/clone inference, ML training data | | **`clonal_repertoire`** | Non-tree **abundance** repertoires — heavy-tailed clone sizes (power-law/lognormal) + unexpanded-singleton fraction, reads through library-prep, collapsed to `duplicate_count` | **TCR** repertoires (no SHM) and flat-BCR abundance; clone-calling benchmarks | -| `expand_clones` *(deprecated)* | Fixed-size **star**: one founder + `per_clone` independent descendants | legacy; kept working, but prefer the two above | +| `expand_clones` *(deprecated)* | Fixed-size **star**: one founder + `per_clone` independent descendants | legacy; still works and still the only way to get an exact per-clone count, but it raises a `DeprecationWarning` — prefer the two above | ```python import GenAIRR as ga @@ -205,9 +205,9 @@ Other feature flags worth knowing: | `.restrict_alleles(v=[...], d=[...], j=[...])` | Restrict allele sampling to a specific subset — useful for benchmarking against a known repertoire. | | `.mutate(model="uniform", rate=0.03)` | Use a uniform-rate mutation model instead of S5F. | | `.mutate(model="s5f", rate=0.03, segment_rates={"V": 1.0, "D": 0.2, "J": 0.5, "NP": 0.0})` | Restrict SHM targeting by biological region class. Buckets are `"V"`, `"D"`, `"J"`, `"NP"` (the NP entry covers both NP1 and NP2). Omitted buckets default to `1.0`; `0.0` disables a region entirely so sites in it drop out of proposal support before contract admissibility. Default (no kwarg) is byte-identical to the pre-slice engine. Composes with `productive_only()`; replay reproduces exactly when the same rate vector is used. The rate vector itself is **not** part of cartridge identity / Rust `content_hash` — it's a per-experiment parameter (audit's documented v1 boundary). | -| `.mutate(model="s5f", rate=0.03, segment_rates={"V": 1.0, "NP": 0.0}, v_subregion_rates={"CDR": 2.0, "FWR": 0.5})` | Refine V-segment SHM targeting by IMGT subregion on top of `segment_rates`. Accepted keys are the five canonical labels `"FWR1"` / `"CDR1"` / `"FWR2"` / `"CDR2"` / `"FWR3"` plus the two aliases `"FWR"` (expands to FWR1 / FWR2 / FWR3) and `"CDR"` (expands to CDR1 / CDR2). **Alias expansion runs first, then explicit labels override** — `{"FWR": 0.5, "FWR2": 2.0}` resolves to `FWR1=0.5, FWR2=2.0, FWR3=0.5, CDR1=1.0, CDR2=1.0`. The V-site weight becomes `base × segment_rate(V) × v_subregion_rate(label)`. Non-V sites and V alleles without subregion annotations receive identity factor `1.0`, so the kwarg composes cleanly with mixed cartridges and with `productive_only()`. Default (no kwarg, `{}`, or explicit all-ones) is byte-identical to the pre-slice engine. Requires the cartridge to carry V-subregion annotations — the bundled human OGRDB cartridges do; calling `v_subregion_rates` against an unannotated cartridge raises `ValueError` at builder time. Like `segment_rates`, the rate vector is part of the plan signature (replay against a different vector fails before consuming choices) but **not** part of `refdata_content_hash`. See [`docs/v_subregion_shm_rate_design.md`](audit-docs/v_subregion_shm_rate_design.md). | -| **`n_mutations` / `n_v_mutations` / `n_d_mutations` / `n_j_mutations` / `n_np_mutations`** (AIRR fields) | `n_mutations` is the **global biological SHM** counter (uniform + S5F substitutions only, sourced from `Simulation.mutation_count`). The four per-segment fields **partition** it by carried event segment — `n_v + n_d + n_j + n_np == n_mutations` by construction, enforced by the validator's `MutationCountSumMismatch` cross-check. NP1 and NP2 events roll into `n_np_mutations` (matches the `segment_rates` DSL grouping). **PCR / quality / N-corruption / receptor revision / D inversion / contaminant are intentionally excluded** — observation-stage and recombination-stage changes do not count as SHM. PCR + quality artefacts surface separately as `n_pcr_errors` / `n_quality_errors`. See [`docs/mutation_provenance_audit.md`](audit-docs/mutation_provenance_audit.md). | -| **`n_fwr1_mutations` / `n_cdr1_mutations` / `n_fwr2_mutations` / `n_cdr2_mutations` / `n_fwr3_mutations` / `n_v_unannotated_mutations`** (AIRR fields) | **Per-V-subregion SHM counters** — a partition of **`n_v_mutations`** (not of the global `n_mutations`). The five canonical IMGT labels bucket V SHM events by the assigned V allele's `FWR1` / `CDR1` / `FWR2` / `CDR2` / `FWR3` interval; `n_v_unannotated_mutations` catches everything else under V. Three "unannotated" cases: (1) the user's cartridge has at least one V allele without subregion annotations (legacy / hand-authored), (2) the SHM event lands in the **V-side CDR3 contribution stretch** between `FWR3.end` and `len(V_allele.seq)` — the five canonical labels deliberately stop at FWR3, so junctional V-tail SHM goes here, (3) an SHM pass runs after an indel pass and mutates an inserted V base (non-canonical pass order). On bundled human OGRDB cartridges the unannotated bucket is a small but non-zero baseline (case 2 — V-tail / CDR3 contribution). Partition invariant: `n_fwr1 + n_cdr1 + n_fwr2 + n_cdr2 + n_fwr3 + n_v_unannotated == n_v_mutations` on every record, enforced by the validator's six `NMutationsMismatch` checks plus the `VSubregionMutationCountSumMismatch` cross-field invariant. Two-bucket aggregates (`n_cdr_mutations` / `n_fwr_mutations`) are deliberately NOT exposed — derive them downstream as `df["n_cdr_mutations"] = df["n_cdr1_mutations"] + df["n_cdr2_mutations"]`. See [`docs/v_subregion_mutation_counters_audit.md`](audit-docs/v_subregion_mutation_counters_audit.md). | +| `.mutate(model="s5f", rate=0.03, segment_rates={"V": 1.0, "NP": 0.0}, v_subregion_rates={"CDR": 2.0, "FWR": 0.5})` | Refine V-segment SHM targeting by IMGT subregion on top of `segment_rates`. Accepted keys are the five canonical labels `"FWR1"` / `"CDR1"` / `"FWR2"` / `"CDR2"` / `"FWR3"` plus the two aliases `"FWR"` (expands to FWR1 / FWR2 / FWR3) and `"CDR"` (expands to CDR1 / CDR2). **Alias expansion runs first, then explicit labels override** — `{"FWR": 0.5, "FWR2": 2.0}` resolves to `FWR1=0.5, FWR2=2.0, FWR3=0.5, CDR1=1.0, CDR2=1.0`. The V-site weight becomes `base × segment_rate(V) × v_subregion_rate(label)`. Non-V sites and V alleles without subregion annotations receive identity factor `1.0`, so the kwarg composes cleanly with mixed cartridges and with `productive_only()`. Default (no kwarg, `{}`, or explicit all-ones) is byte-identical to the pre-slice engine. Requires the cartridge to carry V-subregion annotations — the bundled human OGRDB cartridges do; calling `v_subregion_rates` against an unannotated cartridge raises `ValueError` at builder time. Like `segment_rates`, the rate vector is part of the plan signature (replay against a different vector fails before consuming choices) but **not** part of `refdata_content_hash`. See [`audit-docs/v_subregion_shm_rate_design.md`](audit-docs/v_subregion_shm_rate_design.md). | +| **`n_mutations` / `n_v_mutations` / `n_d_mutations` / `n_j_mutations` / `n_np_mutations`** (AIRR fields) | `n_mutations` is the **global biological SHM** counter (uniform + S5F substitutions only, sourced from `Simulation.mutation_count`). The four per-segment fields **partition** it by carried event segment — `n_v + n_d + n_j + n_np == n_mutations` by construction, enforced by the validator's `MutationCountSumMismatch` cross-check. NP1 and NP2 events roll into `n_np_mutations` (matches the `segment_rates` DSL grouping). **PCR / quality / N-corruption / receptor revision / D inversion / contaminant are intentionally excluded** — observation-stage and recombination-stage changes do not count as SHM. PCR + quality artefacts surface separately as `n_pcr_errors` / `n_quality_errors`. See [`audit-docs/mutation_provenance_audit.md`](audit-docs/mutation_provenance_audit.md). | +| **`n_fwr1_mutations` / `n_cdr1_mutations` / `n_fwr2_mutations` / `n_cdr2_mutations` / `n_fwr3_mutations` / `n_v_unannotated_mutations`** (AIRR fields) | **Per-V-subregion SHM counters** — a partition of **`n_v_mutations`** (not of the global `n_mutations`). The five canonical IMGT labels bucket V SHM events by the assigned V allele's `FWR1` / `CDR1` / `FWR2` / `CDR2` / `FWR3` interval; `n_v_unannotated_mutations` catches everything else under V. Three "unannotated" cases: (1) the user's cartridge has at least one V allele without subregion annotations (legacy / hand-authored), (2) the SHM event lands in the **V-side CDR3 contribution stretch** between `FWR3.end` and `len(V_allele.seq)` — the five canonical labels deliberately stop at FWR3, so junctional V-tail SHM goes here, (3) an SHM pass runs after an indel pass and mutates an inserted V base (non-canonical pass order). On bundled human OGRDB cartridges the unannotated bucket is a small but non-zero baseline (case 2 — V-tail / CDR3 contribution). Partition invariant: `n_fwr1 + n_cdr1 + n_fwr2 + n_cdr2 + n_fwr3 + n_v_unannotated == n_v_mutations` on every record, enforced by the validator's six `NMutationsMismatch` checks plus the `VSubregionMutationCountSumMismatch` cross-field invariant. Two-bucket aggregates (`n_cdr_mutations` / `n_fwr_mutations`) are deliberately NOT exposed — derive them downstream as `df["n_cdr_mutations"] = df["n_cdr1_mutations"] + df["n_cdr2_mutations"]`. See [`audit-docs/v_subregion_mutation_counters_audit.md`](audit-docs/v_subregion_mutation_counters_audit.md). | | `.invert_d(prob=0.05)` | Heavy-chain only. With probability `prob`, sample the D allele in reverse-complement orientation (V(D)J inversion event). Records the decision under `sample_allele.d.inverted`; the assembled D bytes are the WC complement of the original allele; AIRR records carry `d_inverted: bool` for provenance. | | `.receptor_revision(prob=0.05)` | Heavy-chain only. With probability `prob`, replace V after initial recombination with a different germline allele (B-cell receptor revision). Records `receptor_revision.applied` (+ replacement allele id / 3' trim on `True`); the V slice in the assembled pool is rewritten; AIRR records carry `receptor_revision_applied: bool` and `original_v_call: str` (the pre-revision V; empty when no revision happened). `v_call` continues to report the post-revision identity. | | `.paired_end(r1_length=150, insert_size=300)` | Both VDJ and VJ. Add Illumina-style paired-end read-layout projection to every record: AIRR fields `r1_sequence` (forward window) + `r2_sequence` (reverse-complement of the 3' window) + `r1_start/end` / `r2_start/end` / `insert_size` / `read_layout="paired_end"`. Optional `r2_length` (defaults to `r1_length`). Each length accepts `int` / `(low, high)` / empirical `[(value, weight)]`. Trace records `paired_end.r1_length` / `.r2_length` / `.insert_size`; AIRR builder reads them back to populate the windows. Projection-only — no IR mutation, no live-call invalidation, runs after end-loss + rev-comp. | @@ -215,6 +215,59 @@ Other feature flags worth knowing: --- +## Per-individual genotypes + +A `Genotype` is one subject's diploid germline complement — which V/D/J alleles +they carry, on which chromosome, with which genes deleted or duplicated. Attach +one and recombination becomes **haplotype-phased**: the V, D and J of a +rearrangement are drawn from a single chromosome, so the linkage that +haplotype-inference methods exploit is actually there in the data. + +```python +import GenAIRR as ga +import GenAIRR.data as gdata +from GenAIRR.genotype import Genotype + +cfg = gdata.HUMAN_IGH_OGRDB +g = ( + Genotype.from_dataconfig(cfg) + .complete_from_reference("homozygous_first_reference") + .heterozygous("IGHVF1-G1", "IGHVF1-G1*01", "IGHVF1-G1*02") + .delete_gene("IGHVF3-G7", haplotype="both") + .with_subject("DONOR01") +) + +result = ( + ga.Experiment.on(cfg) + .with_genotype(g) # recombination is now haplotype-phased + .recombine() + .run_records(n=1000, seed=7) +) + +result[0]["subject_id"] # 'DONOR01' — on every record +result[0]["haplotype"] # 0 or 1 — which chromosome this read used + +# The genotype leaves the library with the records it explains: +result.to_genotype_tsv("genotype_truth.tsv") +# subject_id locus segment gene allele chromosome novel gene_state +``` + +`to_genotype_tsv()` writes one row per carried allele, so a dataset you hand to +someone else arrives with the truth it is meant to be scored against. A deleted +gene keeps its row with the literal allele `(none)`; a run with no genotype, or a +genotype that leaves reference genes unspecified, raises rather than writing a +file that looks complete and is not. Whether a given read drew a **novel** allele +is a join of `truth_v_call` against that table on `(subject_id, allele)` — +`run_records(..., expose_provenance=True)` adds the `truth_*_call` columns. + +`run_cohort(genotypes, n_per_subject=...)` runs many subjects in one call and +namespaces their records; `CohortResult.to_genotype_tsv(path)` writes the whole +cohort's truth table. + +> Docs: [Genotypes](https://mutejester.github.io/GenAIRR/guides/genotype.html) · [Genotype sampling & population priors](https://mutejester.github.io/GenAIRR/guides/genotype-priors.html) · [Genotype cohorts](https://mutejester.github.io/GenAIRR/guides/genotype-cohorts.html) · [Benchmarking genotype inference](https://mutejester.github.io/GenAIRR/guides/genotype-benchmarking.html) + +--- + ## Constraint-aware sampling GenAIRR's signature feature is **constraint-aware sampling**: contracts that prune the candidate distribution at sample time, not retries after the fact. The canonical bundle is `productive()` (preserved coding frame, no stops across the observed coding sequence, and intact V/J anchors): @@ -248,7 +301,7 @@ preserved; impossible support raises an error even in permissive mode instead of dropping requested additions. This conditions each local draw, rather than sampling from a globally conditioned joint probability law. -> Docs: [Productive sequences](https://mutejester.github.io/GenAIRR/guide-productive.html) +> Docs: [Sample only productive sequences](https://mutejester.github.io/GenAIRR/guides/productive.html) ### Strict vs permissive mode @@ -270,28 +323,47 @@ except ga.StrictSamplingError as e: ## Reproducibility +The promise is narrow and worth stating exactly: **one seed and one +determinism version give one answer.** Two runs at the same +`GenAIRR.DETERMINISM_VERSION` agree byte for byte; two runs at different +versions promise each other nothing — not "mostly the same", not +"statistically equivalent". Every exported record carries the number as a +`determinism_version` column, and so does every trace file, so a dataset says +which sampling model drew it. + ```python import GenAIRR as ga -# Same seed → byte-identical records across runs and platforms. +ga.DETERMINISM_VERSION # 4 — the installed engine's sampling model + +# Same seed + same determinism version → byte-identical records. a = ga.Experiment.on("human_igh").recombine().run_records(n=100, seed=42) b = ga.Experiment.on("human_igh").recombine().run_records(n=100, seed=42) assert a[0]["sequence"] == b[0]["sequence"] +assert a[0]["determinism_version"] == ga.DETERMINISM_VERSION -# `n` runs use seeds [seed, seed+1, ..., seed+n-1] so consecutive -# batches stitch together by offsetting the starting seed. +# `n` runs use seeds [seed, seed+1, ..., seed+n-1], so a seed names a +# *record*, not a run: consecutive batches stitch together by offsetting +# the starting seed by at least `n`. batch_a = ga.Experiment.on("human_igh").recombine().run_records(n=100, seed=0) batch_b = ga.Experiment.on("human_igh").recombine().run_records(n=100, seed=100) # batch_a[50] is byte-equal to a one-off run at seed=50. +# Offsetting by less than `n` overlaps: seeds 0, 1, 2 are not three +# independent replicates, they are one dataset read from three +# starting points. ``` -> Docs: [Reproducibility](https://mutejester.github.io/GenAIRR/guide-reproduce.html) +The reproducibility goldens run in CI on **Linux, macOS and Windows across +Python 3.10–3.13** and assert that the same seed reproduces the same bytes on +each. Linux aarch64 ships a wheel but is not covered by those goldens. + +> Docs: [Trace, replay, reproducibility](https://mutejester.github.io/GenAIRR/guides/trace-replay.html) --- ## Validation posture -GenAIRR's release readiness rests on five guardrails, each backed by +GenAIRR's release readiness rests on seven guardrails, each backed by an audit doc and a golden-test file. Every PR runs the full suite; every release additionally runs the golden trace compatibility fixtures so on-disk formats stay byte-stable. @@ -300,7 +372,7 @@ fixtures so on-disk formats stay byte-stable. - **Provenance correctness** — indels, end-loss, allele-call ambiguity, junction fields, and per-pass event ledger each have isolated-scenario tests. - **Reproducibility** — every audit slice includes trace-replay round-trips that reproduce sequence, AIRR coordinates, and per-pass event counts. - **Distribution invariants** — Monte-Carlo tests (±5σ) prove the constrained samplers draw from `natural_weight × admissibility`, with explicit negative controls against renormalization bugs. -- **Performance budgets** — wall-time regression guards on seven representative workloads catch ~10× slowdowns before they ship. +- **Performance budgets** — wall-time regression guards on nine representative workloads catch ~10× slowdowns before they ship. - **Postcondition validator** — `result.validate_records(refdata)` runs the engine's own truth oracle over every projected AIRR record (re-derives counters, junction, allele tie-set, structural coords from outcome state). Returns a `ValidationReport` you can `assert` on as a one-line CI guard. **This is the public output-correctness contract.** - **Live-call cache parity** — `outcome.check_live_call_cache_parity(refdata)` compares the cached `SegmentLiveCall` on the final simulation against a from-scratch recompute, per V/D/J. **This is the internal cache-correctness check on the state that feeds projection.** @@ -322,7 +394,7 @@ for outcome in result.outcomes: **Troubleshooting rule.** If a CI run has both layers failing on the same batch, fix the **parity** divergence first: a stale cache leaks into projection and produces spurious validator failures downstream. Once parity is green, rerun the validator — remaining failures then point at a genuine projection-layer bug rather than cache fallout. The navigable index — guarantees → audit docs → test files — lives in -[`docs/validation_matrix.md`](audit-docs/validation_matrix.md). Use it to +[`audit-docs/validation_matrix.md`](audit-docs/validation_matrix.md). Use it to locate the right test for a change you're making, or to scope a follow-up that touches an open drift item in any audit's §6. @@ -369,7 +441,8 @@ for batch in range(10): | `result.to_csv(path, *, airr_strict=False)` | — | Comma-separated; keeps Python record conventions unless `airr_strict=True`. | | `result.to_fasta(path, *, prefix="seq")` | — | FASTA. Headers include `v_call` and `j_call`. | | `result.to_fastq(path, *, quality="illumina", **kw)` | — | FASTQ. Quality models: `"illumina"` (smoothed trapezoid) or `"constant"`. | -| `result.to_dataframe(*, airr_strict=False)` | `pandas.DataFrame` | One row per record. Requires pandas (`pip install GenAIRR[all]`). | +| `result.to_dataframe(*, airr_strict=False)` | `pandas.DataFrame` | One row per record. Requires pandas (`pip install pandas`). | +| `result.to_genotype_tsv(path)` | — | The genotype truth sidecar: one row per carried allele. Raises when the run carried no genotype. | | `result.outcomes` | `list[Outcome] \| None` | The underlying `Outcome` objects, for advanced introspection (see below). | Each record dict has 50+ AIRR fields. The most commonly used: @@ -385,9 +458,11 @@ Each record dict has 50+ AIRR fields. The most commonly used: | `productive` | `True` / `False` / `None` | In-frame junction AND no stop codons AND anchors preserved. `None` when undefined (e.g. junction not present). | | `v_identity` / `d_identity` / `j_identity` | `0.987` | Fraction from 0 to 1: matching bases divided by alignment columns, including gaps. | | `v_cigar` / `d_cigar` / `j_cigar` | `'17D279M'` | Local per-segment M/I/D runs in Python dicts; strict exports add leading query `S` and reference `N` offsets when nonzero. | -| `n_mutations` / `n_pcr_errors` / `n_quality_errors` / `n_indels` | `4` / `0` / `2` / `1` | Per-record error counts from the trace. | +| `n_mutations` / `n_pcr_errors` / `n_quality_errors` / `n_ambiguous_bases` / `n_indels` | `4` / `0` / `2` / `3` / `1` | Per-record event counts from the trace. `n_quality_errors` comes from `sequencing_errors()`; `ambiguous_base_calls()` reports on `n_ambiguous_bases`. | +| `n_observable_mutations` | `3` | The SHM substitutions a mutation caller can **recover from `sequence`**. Below `n_mutations`: a mutation on an untemplated junction base has no germline base to differ from, a second hit can restore the germline base, and a position lost to end loss is gone. **Score a caller against this field, not `n_mutations`.** | +| `determinism_version` | `4` | Which sampling model drew the row. Two datasets with different numbers are not comparable. | -The full schema (plus the `*_sequence_start/end`, `*_alignment_start/end`, `*_germline_start/end` coordinate fields, `vj_in_frame`, `stop_codon`, `rev_comp`, and others) is documented at [Interpreting Results](https://mutejester.github.io/GenAIRR/concept-airr-record.html). +The full schema (plus the `*_sequence_start/end`, `*_alignment_start/end`, `*_germline_start/end` coordinate fields, `vj_in_frame`, `stop_codon`, `rev_comp`, and others) is documented at [The AIRR record](https://mutejester.github.io/GenAIRR/concepts/airr-record.html). ### Advanced: full pipeline state via `Outcome` @@ -407,7 +482,7 @@ Each `Simulation` exposes `len(sim)` (pool length), `sim.bases() → bytes`, `si `.run_records(...)` also exposes these via `result.outcomes[i]` — so you can have both the AIRR records *and* the deep introspection from a single call. -> Docs: [Simulation Pipeline](https://mutejester.github.io/GenAIRR/concept-pipeline.html) · [Ground-truth Contracts](https://mutejester.github.io/GenAIRR/concept-contracts.html) · [Interpreting Results](https://mutejester.github.io/GenAIRR/concept-airr-record.html) +> Docs: [The Experiment builder](https://mutejester.github.io/GenAIRR/guides/experiment-builder.html) · [Trace, replay, reproducibility](https://mutejester.github.io/GenAIRR/guides/trace-replay.html) · [The AIRR record](https://mutejester.github.io/GenAIRR/concepts/airr-record.html) --- @@ -442,12 +517,24 @@ Salmon, Sheep, Trout, Zebrafish. ```python import GenAIRR as ga -ga.Experiment.on("mouse_igh").recombine().run_records(n=500) ga.Experiment.on("rabbit_tcrb").recombine().run_records(n=500) -ga.Experiment.on("rhesus_igk").recombine().run_records(n=500) + +# 80 of the 106 bundled catalogues carry unresolved anchor or +# pseudogene/ORF findings, so strict compilation quarantines them. Opt in +# explicitly to simulate from one — the records are then drawn from a +# catalogue whose landmarks the project has not been able to confirm. +ga.Experiment.on("mouse_igh").allow_curatable_refdata().recombine().run_records(n=500) +ga.Experiment.on("rhesus_igk").allow_curatable_refdata().recombine().run_records(n=500) ``` -> Docs: [Learn GenAIRR](https://mutejester.github.io/GenAIRR/learn.html) · [Reference (Configs)](https://mutejester.github.io/GenAIRR/reference.html) +The 26 catalogues that compile strictly today are the three human OGRDB +cartridges (`human_igh`, `human_igk`, `human_igl`), `HUMAN_IGH_EXTENDED`, and +22 IMGT catalogues whose anchors all resolved. Everything else — including +every mouse catalogue and `human_tcrb` — needs `allow_curatable_refdata()`. +The compile error lists the findings allele by allele, so you can also filter +the catalogue or curate it with `Experiment.curate_refdata(...)` instead. + +> Docs: [Choose your path](https://mutejester.github.io/GenAIRR/learn.html) · [Reference evidence and quarantine](https://mutejester.github.io/GenAIRR/cartridges/reference-evidence.html) · [API Reference](https://mutejester.github.io/GenAIRR/reference/index.html) --- @@ -458,19 +545,30 @@ For non-builtin alleles (custom IMGT pulls, in-house references, etc.) you can b ```python import GenAIRR as ga +# Real bytes, so the example runs as written. Substitute your own. +igk = ga.HUMAN_IGK_OGRDB +v1, v2 = [a for g in igk.v_alleles.values() for a in g][:2] +j1 = next(a for g in igk.j_alleles.values() for a in g) + cfg = ga.RefDataConfig.vj() -cfg.add_v_allele("v_custom*01", "v_custom", b"GAAGTACAGCTGGTGCAG...", anchor=288) -cfg.add_v_allele("v_custom*02", "v_custom", b"GAAGTACAGCTAGTGCAG...", anchor=288) -cfg.add_j_allele("j_custom*01", "j_custom", b"TGGGGCCAAGGG...", anchor=10) +cfg.add_v_allele("v_custom*01", "v_custom", v1.ungapped_seq.upper().encode(), anchor=v1.anchor) +cfg.add_v_allele("v_custom*02", "v_custom", v2.ungapped_seq.upper().encode(), anchor=v2.anchor) +cfg.add_j_allele("j_custom*01", "j_custom", j1.ungapped_seq.upper().encode(), anchor=j1.anchor) -result = ga.Experiment.on(cfg).recombine().run_records(n=100, seed=42) +result = ( + ga.Experiment.on(cfg) + .recombine(np1_lengths=[(n, 1.0) for n in range(7)]) # no empirical NP model + .trim(enabled=False) # no trim distributions + .run_records(n=100, seed=42) +) +result[0]["v_call"], result[0]["junction_aa"] # ('v_custom*01', 'CLQHNSYPPWRFTF') ``` -`RefDataConfig.vdj()` builds a heavy-chain-shaped refdata (with a D pool); `add_d_allele(...)` populates it. Anchors are 0-based offsets of the V Cys / J W or F codon's first base, used to keep the junction frame-aligned during recombination. +`RefDataConfig.vdj()` builds a heavy-chain-shaped refdata (with a D pool); `add_d_allele(...)` populates it. Anchors are 0-based offsets of the V Cys / J W or F codon's first base, used to keep the junction frame-aligned during recombination — the engine checks that the codon at the anchor really translates to Cys (V) or Trp/Phe (J) and refuses the reference otherwise. Sequence bytes must be A/C/G/T/N; a raw `RefDataConfig` carries no empirical NP-length or trim distributions, so pass them explicitly (as above) or accept the two `UserWarning`s that say the engine fell back to a uniform NP length and a no-op trim. -For non-human or otherwise non-standard references — custom J anchor amino acids, extended sequence alphabets, hand-authored NP-length or trim distributions, pseudogene-aware allele filtering — configure a **reference cartridge** instead of relying on the built-in locus defaults. The cartridge has four typed planes (identity, catalogue, rules, empirical models) plus an explicit curation policy. See [`docs/reference_cartridge.md`](audit-docs/reference_cartridge.md) for the model, the authoring surface (`ReferenceRulesSpec`, `ReferenceEmpiricalModels`, `Experiment.curate_refdata`), and end-to-end examples. +For non-human or otherwise non-standard references — custom J anchor amino acids, extended sequence alphabets, hand-authored NP-length or trim distributions, pseudogene-aware allele filtering — configure a **reference cartridge** instead of relying on the built-in locus defaults. The cartridge has four typed planes (identity, catalogue, rules, empirical models) plus an explicit curation policy. See [`audit-docs/reference_cartridge.md`](audit-docs/reference_cartridge.md) for the model, the authoring surface (`ReferenceRulesSpec`, `ReferenceEmpiricalModels`, `Experiment.curate_refdata`), and end-to-end examples. -Cartridges can also carry **V-region substructure annotations** — per-V-allele IMGT `FWR1` / `CDR1` / `FWR2` / `CDR2` / `FWR3` intervals derived from the IMGT-gapped reference sequence. The bundled OGRDB cartridges populate them at load time for all human IGH / IGK / IGL V alleles; user cartridges can override the dict explicitly. Subregion intervals are inspectable on the bridged `RefDataConfig`, surfaced in `DataConfig.cartridge_manifest()["models"]["shm"]["v_subregion_support"]`, and folded into `refdata_content_hash` so two cartridges that differ only in subregion boundaries hash differently. **They also drive SHM targeting** via `Experiment.mutate(v_subregion_rates={…})` — see the targeted-SHM example in the pipeline-steps table above. Per-region AIRR mutation counters (`n_cdr1_mutations` etc.) are still deferred — that's a separate future slice. See [`docs/v_region_substructure_audit.md`](audit-docs/v_region_substructure_audit.md) and [`docs/v_subregion_shm_rate_design.md`](audit-docs/v_subregion_shm_rate_design.md) for the audits and per-slice scope. +Cartridges can also carry **V-region substructure annotations** — per-V-allele IMGT `FWR1` / `CDR1` / `FWR2` / `CDR2` / `FWR3` intervals derived from the IMGT-gapped reference sequence. The bundled OGRDB cartridges populate them at load time for all human IGH / IGK / IGL V alleles; user cartridges can override the dict explicitly. Subregion intervals are inspectable on the bridged `RefDataConfig`, surfaced in `DataConfig.cartridge_manifest()["models"]["shm"]["v_subregion_support"]`, and folded into `refdata_content_hash` so two cartridges that differ only in subregion boundaries hash differently. **They also drive SHM targeting** via `Experiment.mutate(v_subregion_rates={…})` — see the targeted-SHM example in the pipeline-steps table above — and they bucket the per-region AIRR mutation counters (`n_cdr1_mutations` …) and their recoverable counterparts (`n_cdr1_observable_mutations` …), both described in the pipeline-steps table. See [`audit-docs/v_region_substructure_audit.md`](audit-docs/v_region_substructure_audit.md) and [`audit-docs/v_subregion_shm_rate_design.md`](audit-docs/v_subregion_shm_rate_design.md) for the audits and per-slice scope. ### Typed NP base models (uniform / empirical / Markov) @@ -504,7 +602,7 @@ cfg.reference_models = ReferenceEmpiricalModels( Both rows must cover every canonical from-base — the Python validator rejects partial matrices at construction time. Plan signatures fold the full Markov payload (first-base row + 4 transition rows), so replay against a different matrix fails the signature gate before any choice is consumed; same-cartridge + same-seed replay is byte-identical. Productive-only sampling composes through the existing admit-mask intersection. -**Legacy `NP_transitions` / `NP_first_bases` are NOT auto-lifted yet.** The bundled cartridges still carry those dicts as documented orphan fields, and the manifest reports `legacy_fallback=False`. Authors who want Markov today populate `ReferenceEmpiricalModels.np_bases` explicitly; an opt-in auto-lift slice is a separate cartridge-migration decision. See [`docs/np_markov_base_generator_design.md`](audit-docs/np_markov_base_generator_design.md) and [`docs/junction_n_addition_audit.md`](audit-docs/junction_n_addition_audit.md) for the full design. +**Legacy `NP_transitions` / `NP_first_bases` are NOT auto-lifted yet.** The bundled cartridges still carry those dicts as documented orphan fields, and the manifest reports `legacy_fallback=False`. Authors who want Markov today populate `ReferenceEmpiricalModels.np_bases` explicitly; an opt-in auto-lift slice is a separate cartridge-migration decision. See [`audit-docs/np_markov_base_generator_design.md`](audit-docs/np_markov_base_generator_design.md) and [`audit-docs/junction_n_addition_audit.md`](audit-docs/junction_n_addition_audit.md) for the full design. ### Templated P-nucleotide (palindromic) additions @@ -548,7 +646,7 @@ p_v_3_length, p_d_5_length, p_d_3_length, p_j_5_length …each counting the number of templated P-bytes emitted at that V(D)J coding-end junction side. Zero on records from cartridges without a typed P-plane. The validator's `PLengthMismatch` issue kind catches downstream record tampering; cache parity is unaffected (P-bytes don't move structural region boundaries). -**Legacy `p_nucleotide_length_probs` is NOT auto-lifted yet.** The bundled cartridges still carry the orphan dict; the manifest reports `legacy_fallback=False`. v1 ships *lengths-only* — per-base P strings (`p_v_3`, ...) and the aggregate `n_p_nucleotides` are deferred. See [`docs/p_nucleotide_design.md`](audit-docs/p_nucleotide_design.md) for the full design. +**Legacy `p_nucleotide_length_probs` is NOT auto-lifted yet.** The bundled cartridges still carry the orphan dict; the manifest reports `legacy_fallback=False`. Per-base P strings (`p_v_3`, …) and an aggregate `n_p_nucleotides` are still not exported, but each NP region now reports how its bases arose: `np1_n_tdt` / `np1_n_p` / `np1_n_inserted` (and the `np2_*` triple) partition `np1_length` into TdT N additions, templated P bases and bases an indel inserted. See [`audit-docs/p_nucleotide_design.md`](audit-docs/p_nucleotide_design.md) for the full design. ### Build a cartridge from FASTA @@ -595,9 +693,9 @@ builder = ( cfg = builder.build() ``` -Recombination on the resulting cartridge **uses the estimated weights by default** — `ga.Experiment.on(cfg).recombine()` now samples alleles according to the typed plane unless an explicit `recombine(v_allele_weights=..., d_allele_weights=..., j_allele_weights=...)` kwarg is passed (kwarg > cartridge plane > uniform). Ambiguity policy is `"fractional"` (splits credit across tie-set entries), `"truth_first"` (uses the first call only), or `"reject"` (drops ambiguous rows into `report.rejected`). Unknown alleles, missing-D-on-VDJ rows, and below-`min_count` drops all surface as structured entries on the build report. Per-segment weights are normalised to sum to 1.0; the `manifest["models"]["allele_usage"]` block surfaces `available` / `nonempty_segments` / `legacy_gene_use_dict_present` / `in_plan_signature=False` (inherited soft gap from `v_allele_weights`). See [`docs/allele_usage_estimation_design.md`](audit-docs/allele_usage_estimation_design.md). +Recombination on the resulting cartridge **uses the estimated weights by default** — `ga.Experiment.on(cfg).recombine()` now samples alleles according to the typed plane unless an explicit `recombine(v_allele_weights=..., d_allele_weights=..., j_allele_weights=...)` kwarg is passed (kwarg > cartridge plane > uniform). Ambiguity policy is `"fractional"` (splits credit across tie-set entries), `"truth_first"` (uses the first call only), or `"reject"` (drops ambiguous rows into `report.rejected`). Unknown alleles, missing-D-on-VDJ rows, and below-`min_count` drops all surface as structured entries on the build report. Per-segment weights are normalised to sum to 1.0; the `manifest["models"]["allele_usage"]` block surfaces `available` / `nonempty_segments` / `legacy_gene_use_dict_present` / `in_plan_signature=False` (inherited soft gap from `v_allele_weights`). See [`audit-docs/allele_usage_estimation_design.md`](audit-docs/allele_usage_estimation_design.md). -**Remaining estimators stay deferred.** `estimate_trim_distributions` / `estimate_np_length_distributions` / `estimate_np_base_model` / `estimate_p_nucleotide_lengths` / `estimate_shm_rates` are scoped to follow-up slices — each will append new stage entries to the build report without changing the facade. See [`docs/reference_cartridge_authoring_audit.md`](audit-docs/reference_cartridge_authoring_audit.md) for the full design. +**The other estimators have landed since.** `estimate_trim_distributions`, `estimate_np_length_distributions`, `estimate_np_base_model`, `estimate_p_nucleotide_lengths` and `estimate_genotype_priors` all ship alongside `estimate_allele_usage` and append their own stage entries to the build report; `set_genotype_priors` attaches a population model you already have. Only `estimate_shm_rates` is still deferred. See [`audit-docs/reference_cartridge_authoring_audit.md`](audit-docs/reference_cartridge_authoring_audit.md) for the full design and [Estimate models from data](https://mutejester.github.io/GenAIRR/guides/estimate-cartridge-models.html) for the per-estimator walkthrough. --- @@ -606,7 +704,7 @@ Recombination on the resulting cartridge **uses the estimated weights by default - **Rust simulation kernel** — persistent IR with full revision history, addressed-trace introspection, `cargo test`-grade unit coverage. - **Constraint-aware sampling** — contracts prune candidate distributions at sample time so productive sequences come out of the engine by construction; no retry loops. - **Strict-mode opt-in** — surface unsatisfiable plans as `StrictSamplingError` instead of silently relaxing the bundle. -- **Deterministic seeds** — same seed reproduces every byte of the pool and every entry of the trace, across runs and platforms. +- **Deterministic seeds** — one seed and one `determinism_version` reproduce every byte of the pool and every entry of the trace; the goldens check that on Linux, macOS and Windows across Python 3.10–3.13. - **Full revision history** — `outcome.revision(i)` exposes the IR after each pass for fine-grained debugging. - **Addressed trace** — every random draw is keyed by a hierarchical string (`"np.np1.bases[3]"`) and survives end-to-end into the returned `Outcome`. - **23 species, 106 configs** — built-in IMGT + OGRDB reference pickles ship with the wheel. @@ -698,10 +796,11 @@ A quick smoke-test prompt to verify the install: *"List the available GenAIRR co The full documentation site is at **[mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/)**. Useful starting points: -- **Learn** — [Lesson 1: V(D)J recombination](https://mutejester.github.io/GenAIRR/lesson-1.html) · [Lesson 2: Pipeline scrubber](https://mutejester.github.io/GenAIRR/lesson-2.html) · [Lesson 3: S5F SHM](https://mutejester.github.io/GenAIRR/lesson-3.html) · [Lesson 4: Sequencing artifacts](https://mutejester.github.io/GenAIRR/lesson-4.html) · [Lesson 5: Ground-truth payoff](https://mutejester.github.io/GenAIRR/lesson-5.html) -- **Concepts** — [Simulation Pipeline](https://mutejester.github.io/GenAIRR/concept-pipeline.html) · [Persistent IR](https://mutejester.github.io/GenAIRR/concept-persistent-ir.html) · [Contracts](https://mutejester.github.io/GenAIRR/concept-contracts.html) · [AIRR Record](https://mutejester.github.io/GenAIRR/concept-airr-record.html) · [Live Calls](https://mutejester.github.io/GenAIRR/concept-live-call.html) -- **Guides** — [Build a config](https://mutejester.github.io/GenAIRR/guide-build-config.html) · [Productive sampling](https://mutejester.github.io/GenAIRR/guide-productive.html) · [Clonal lineage trees](https://mutejester.github.io/GenAIRR/guides/clonal-lineage.html) · [Clonal repertoires](https://mutejester.github.io/GenAIRR/guides/clonal-repertoire.html) · [Export](https://mutejester.github.io/GenAIRR/guide-export.html) · [Replay](https://mutejester.github.io/GenAIRR/guide-replay.html) · [Reproduce a dataset](https://mutejester.github.io/GenAIRR/guide-reproduce.html) · [Compare SHM models](https://mutejester.github.io/GenAIRR/guide-compare-shm.html) · [Tune corruption](https://mutejester.github.io/GenAIRR/guide-tune-corruption.html) -- **Reference** — [API + Configs + AIRR fields](https://mutejester.github.io/GenAIRR/reference.html) +- **Getting started** — [Installation](https://mutejester.github.io/GenAIRR/getting-started/installation.html) · [Quick start](https://mutejester.github.io/GenAIRR/getting-started/quick-start.html) · [Your first AIRR record](https://mutejester.github.io/GenAIRR/getting-started/first-airr-record.html) · [Export the results](https://mutejester.github.io/GenAIRR/getting-started/export-results.html) +- **Concepts** — [AIRR Record](https://mutejester.github.io/GenAIRR/concepts/airr-record.html) · [Reference Cartridge](https://mutejester.github.io/GenAIRR/concepts/reference-cartridge.html) +- **Guides** — [The Experiment builder](https://mutejester.github.io/GenAIRR/guides/experiment-builder.html) · [Productive sampling](https://mutejester.github.io/GenAIRR/guides/productive.html) · [Clonal lineage trees](https://mutejester.github.io/GenAIRR/guides/clonal-lineage.html) · [Clonal repertoires](https://mutejester.github.io/GenAIRR/guides/clonal-repertoire.html) · [Genotypes](https://mutejester.github.io/GenAIRR/guides/genotype.html) · [Paired-end reads and FASTQ](https://mutejester.github.io/GenAIRR/guides/paired-end-fastq.html) · [Corruption + sequencing artefacts](https://mutejester.github.io/GenAIRR/guides/corruption-sequencing.html) · [Targeted SHM rates](https://mutejester.github.io/GenAIRR/guides/shm-targeting.html) · [Trace, replay, reproducibility](https://mutejester.github.io/GenAIRR/guides/trace-replay.html) +- **Validation** — [validate_records](https://mutejester.github.io/GenAIRR/validation/validate-records.html) · [Validation overview](https://mutejester.github.io/GenAIRR/validation/index.html) +- **Reference** — [API + Configs + AIRR fields](https://mutejester.github.io/GenAIRR/reference/index.html) --- @@ -717,8 +816,8 @@ If GenAIRR is useful in your research, please cite: Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and guidelines. If you're adding a pass, a contract, or any code that mutates the simulation IR: -- **Read first**: [docs/engine_architecture.md](audit-docs/engine_architecture.md) — codifies the invariants (contracts constrain support, trace = choices, events = consequences, live-call refresh follows events) and lists the anti-patterns CI catches. -- **Then copy from**: [docs/adding_a_pass.md](audit-docs/adding_a_pass.md) — minimal pass template, three required test patterns with `passes::test_support` helpers, and a crib sheet of which existing pass to model the new one on. +- **Read first**: [audit-docs/engine_architecture.md](audit-docs/engine_architecture.md) — codifies the invariants (contracts constrain support, trace = choices, events = consequences, live-call refresh follows events) and lists the anti-patterns CI catches. +- **Then copy from**: [audit-docs/adding_a_pass.md](audit-docs/adding_a_pass.md) — minimal pass template, three required test patterns with `passes::test_support` helpers, and a crib sheet of which existing pass to model the new one on. ## License diff --git a/mkdocs.yml b/mkdocs.yml index 2cae6713..3b5ebdbf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,7 +7,7 @@ # `tests/acceptance/test_docs_mkdocs_migration_contract.py` guard structure. # # To build: -# pip install -e ".[docs]" +# pip install -r requirements-docs.txt # mkdocs build --strict # # Existing live URLs at `mutejester.github.io/GenAIRR/.html` @@ -22,16 +22,16 @@ repo_name: MuteJester/GenAIRR repo_url: https://github.com/MuteJester/GenAIRR edit_uri: edit/master/site_docs/ -# Source directory deliberately separate from `docs/` so the -# 39 contributor-only audit/design markdowns at `docs/*.md` -# stay untouched and continue to render on GitHub. The -# user-facing site lives at `site_docs/`. +# Source directory deliberately separate from `audit-docs/` so the +# contributor-only audit and design markdowns there stay untouched +# and continue to render on GitHub. The user-facing site lives at +# `site_docs/`. docs_dir: site_docs -# Emit `.html` URLs (NOT `/` clean-URLs). This is -# the URL contract Slice 1 committed to preserve via the -# `redirects` plugin — every existing `mutejester.github.io/GenAIRR/.html` -# remains resolvable post-cutover. With `use_directory_urls: false`, +# Emit `.html` URLs (NOT `/` clean-URLs). This is the +# URL contract the `redirects` plugin preserves — every existing +# `mutejester.github.io/GenAIRR/.html` stays resolvable. +# With `use_directory_urls: false`, # the `redirects` plugin's `'old.md': 'new.md'` entries emit a # redirect file at `.html` pointing at `.html`, which # is exactly what we need. @@ -73,10 +73,10 @@ theme: icon: repo: fontawesome/brands/github -# GenAIRR theme layer (Phase 1.5) — design tokens + component -# overrides that prevent the site from looking like default MkDocs -# Material. See site_docs/stylesheets/extra.css for the full -# stylesheet (sourced from website/styles.css). +# GenAIRR theme layer — design tokens + component overrides that +# prevent the site from looking like default MkDocs Material. See +# site_docs/stylesheets/extra.css for the full stylesheet (sourced +# from website/styles.css). extra_css: - stylesheets/extra.css @@ -110,27 +110,23 @@ markdown_extensions: anchor_linenums: true - pymdownx.inlinehilite -# Plugins — Phase 1 of the migration (deps installable via +# Plugins (deps installable via # `pip install -r requirements-docs.txt`). # # - `search` is provided by mkdocs core and enabled by default. -# - `redirects` carries the live-URL → MkDocs-route map per -# docs/docs_mkdocs_migration_plan.md §3.1. Every existing -# website/*.html path stays resolvable after the Phase 6 -# cutover. -# - `mkdocstrings` is enabled but currently unused; the -# auto-generated API reference page is Phase 5 work. The -# plugin sits idle until a `:::` directive is added to a -# markdown source. +# - `redirects` carries the live-URL → MkDocs-route map, so every +# existing `website/*.html` path stays resolvable. +# - `mkdocstrings` drives the generated API-reference pages under +# `site_docs/reference/`. # - `mkdocs-jupyter` is intentionally NOT enabled here. It's -# listed in `requirements-docs.txt` so Phase 4 (notebook -# tutorials) can flip it on without a separate dep install. +# listed in `requirements-docs.txt` so notebook tutorials can +# flip it on without a separate dep install. plugins: - search - redirects: redirect_maps: - # Live-URL preservation per docs/docs_mkdocs_migration_plan.md §3.1. - # `mkdocs-redirects` treats the LEFT key as a fake source path: it + # Live-URL preservation. `mkdocs-redirects` treats the LEFT + # key as a fake source path: it # generates a redirect HTML file at the URL MkDocs would have # produced for that path. With `use_directory_urls: false`, `foo.md` # → `foo.html`. So `'learn.md': 'getting-started/index.md'` emits @@ -179,11 +175,10 @@ plugins: # Navigation — curated path from beginner to contributor. # -# Placeholder pages are intentionally NOT in nav. They stay on disk +# Unwritten pages are intentionally NOT in nav. They stay on disk # (the `redirects` plugin needs them as targets for live-URL -# preservation, see §3.1 of docs/docs_mkdocs_migration_plan.md), -# but the user-visible nav lists only real content. As each -# placeholder is migrated to real content, add it back to nav here. +# preservation), but the user-visible nav lists only real content. +# As each one is written, add it back to nav here. nav: - Home: index.md - Demo: demo.md diff --git a/site_docs/architecture/adding-a-pass.md b/site_docs/architecture/adding-a-pass.md index f23c2a46..31164c0f 100644 --- a/site_docs/architecture/adding-a-pass.md +++ b/site_docs/architecture/adding-a-pass.md @@ -3,12 +3,6 @@ _Linked from `docs/adding_a_pass.md`._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 5** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Architecture](https://mutejester.github.io/GenAIRR/architecture/index.html). diff --git a/site_docs/architecture/audit-index.md b/site_docs/architecture/audit-index.md index 6bbb4136..d1fd67e4 100644 --- a/site_docs/architecture/audit-index.md +++ b/site_docs/architecture/audit-index.md @@ -3,12 +3,6 @@ _Auto-generated catalogue of every `docs/*audit.md` and `docs/*design.md`._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 5** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Architecture](https://mutejester.github.io/GenAIRR/architecture/index.html). diff --git a/site_docs/architecture/engine.md b/site_docs/architecture/engine.md index c2c9310e..466783d6 100644 --- a/site_docs/architecture/engine.md +++ b/site_docs/architecture/engine.md @@ -3,12 +3,6 @@ _Linked from `docs/engine_architecture.md`._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 5** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Architecture](https://mutejester.github.io/GenAIRR/architecture/index.html). diff --git a/site_docs/architecture/validation-matrix.md b/site_docs/architecture/validation-matrix.md index d5446a41..5fd94a2d 100644 --- a/site_docs/architecture/validation-matrix.md +++ b/site_docs/architecture/validation-matrix.md @@ -3,12 +3,6 @@ _Linked from `docs/validation_matrix.md`._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 5** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Validation & Debugging](https://mutejester.github.io/GenAIRR/validation/index.html). diff --git a/site_docs/cartridges/build-from-fasta.md b/site_docs/cartridges/build-from-fasta.md index 544fa3ad..540ff12a 100644 --- a/site_docs/cartridges/build-from-fasta.md +++ b/site_docs/cartridges/build-from-fasta.md @@ -1,14 +1,8 @@ # Build a cartridge from FASTA -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 4** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Build a reference cartridge](https://mutejester.github.io/GenAIRR/guides/build-reference-cartridge.html). New authoring also preserves source-header declarations and carries optional sequence-bound landmark, coding-phase and coding-boundary evidence. Missing diff --git a/site_docs/cartridges/empirical-models.md b/site_docs/cartridges/empirical-models.md index 9206cf71..3a74fecb 100644 --- a/site_docs/cartridges/empirical-models.md +++ b/site_docs/cartridges/empirical-models.md @@ -1,11 +1,5 @@ # Empirical models -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 4** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Estimate models from data](https://mutejester.github.io/GenAIRR/guides/estimate-cartridge-models.html). diff --git a/site_docs/cartridges/estimators/allele-usage.md b/site_docs/cartridges/estimators/allele-usage.md index 1ae09c22..32414633 100644 --- a/site_docs/cartridges/estimators/allele-usage.md +++ b/site_docs/cartridges/estimators/allele-usage.md @@ -3,12 +3,6 @@ _Rewritten from `docs/allele_usage_estimation_design.md`._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 4** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Estimate models from data](https://mutejester.github.io/GenAIRR/guides/estimate-cartridge-models.html). diff --git a/site_docs/cartridges/estimators/index.md b/site_docs/cartridges/estimators/index.md index 383ddf14..bf40b133 100644 --- a/site_docs/cartridges/estimators/index.md +++ b/site_docs/cartridges/estimators/index.md @@ -3,12 +3,6 @@ _Hub for the 5 cartridge estimators._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 4** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Estimate models from data](https://mutejester.github.io/GenAIRR/guides/estimate-cartridge-models.html). diff --git a/site_docs/cartridges/estimators/np-base-model.md b/site_docs/cartridges/estimators/np-base-model.md index d60f89ab..f2a4c203 100644 --- a/site_docs/cartridges/estimators/np-base-model.md +++ b/site_docs/cartridges/estimators/np-base-model.md @@ -3,12 +3,6 @@ _Rewritten from `docs/np_base_model_estimation_design.md`._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 4** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Estimate models from data](https://mutejester.github.io/GenAIRR/guides/estimate-cartridge-models.html). diff --git a/site_docs/cartridges/estimators/np-length.md b/site_docs/cartridges/estimators/np-length.md index c3204a93..bc9ba2ef 100644 --- a/site_docs/cartridges/estimators/np-length.md +++ b/site_docs/cartridges/estimators/np-length.md @@ -3,12 +3,6 @@ _Rewritten from `docs/np_length_estimation_design.md`._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 4** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Estimate models from data](https://mutejester.github.io/GenAIRR/guides/estimate-cartridge-models.html). diff --git a/site_docs/cartridges/estimators/p-nucleotide-length.md b/site_docs/cartridges/estimators/p-nucleotide-length.md index 77dfc7af..31d6018b 100644 --- a/site_docs/cartridges/estimators/p-nucleotide-length.md +++ b/site_docs/cartridges/estimators/p-nucleotide-length.md @@ -3,12 +3,6 @@ _Rewritten from `docs/p_nucleotide_length_estimation_design.md`._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 4** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Estimate models from data](https://mutejester.github.io/GenAIRR/guides/estimate-cartridge-models.html). diff --git a/site_docs/cartridges/estimators/trim-distributions.md b/site_docs/cartridges/estimators/trim-distributions.md index f0a69ac4..3d74c62d 100644 --- a/site_docs/cartridges/estimators/trim-distributions.md +++ b/site_docs/cartridges/estimators/trim-distributions.md @@ -3,12 +3,6 @@ _Rewritten from `docs/trim_distribution_estimation_design.md`._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 4** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Estimate models from data](https://mutejester.github.io/GenAIRR/guides/estimate-cartridge-models.html). diff --git a/site_docs/cartridges/rules-and-identity.md b/site_docs/cartridges/rules-and-identity.md index 2af56a89..6d4eff82 100644 --- a/site_docs/cartridges/rules-and-identity.md +++ b/site_docs/cartridges/rules-and-identity.md @@ -1,11 +1,5 @@ # Rules + identity -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 4** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Reference Cartridge (concept)](https://mutejester.github.io/GenAIRR/concepts/reference-cartridge.html). diff --git a/site_docs/cartridges/v-subregion-annotations.md b/site_docs/cartridges/v-subregion-annotations.md index 79cef838..693def9d 100644 --- a/site_docs/cartridges/v-subregion-annotations.md +++ b/site_docs/cartridges/v-subregion-annotations.md @@ -3,12 +3,6 @@ _Rewritten from `docs/v_region_substructure_audit.md`._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 4** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Targeted SHM rates](https://mutejester.github.io/GenAIRR/guides/shm-targeting.html). diff --git a/site_docs/concepts/airr-record.md b/site_docs/concepts/airr-record.md index d6512012..827b8036 100644 --- a/site_docs/concepts/airr-record.md +++ b/site_docs/concepts/airr-record.md @@ -35,6 +35,9 @@ programmatically as `result.to_dataframe().columns`. The canonical groups, in the order they appear on the record: +0. `determinism_version` - the record's first key: which sampling + model drew it (see [Trace, replay, + reproducibility](../guides/trace-replay.md)) 1. AIRR metadata (sequence + length + locus) 2. V / D / J calls + alignment coordinates + trim 3. Junction @@ -50,6 +53,8 @@ The canonical groups, in the order they appear on the record: 13. Read layout (paired-end) 14. Clonal stamping (when a clonal workflow runs) 15. Truth columns (when `expose_provenance=True`) +16. Genotype stamping (`subject_id`, `haplotype`) when a genotype + or a cohort run is attached The remaining sections walk through each. @@ -123,6 +128,15 @@ benchmarking workflows. | `productivity_policy` | str | `canonical_observed_vdj_v1` | | `productivity_scope` | str | `observed_coding_span` | | `productivity_reasons` | list[str] | Stable diagnostic codes, or `["accepted"]`; JSON in TSV/CSV cells | +| `coding_frame_start`, `coding_frame_end` | int | The reading frame the assessment used: `sequence[coding_frame_start:coding_frame_end]` is the coding span | +| `v_anchor_accepted`, `j_anchor_accepted` | bool or null | Whether each conserved anchor codon was accepted; `None` when an ambiguous base or an insertion inside the codon leaves it unresolved | +| `j_tail_in_frame` | bool or null | Whether the J tail joins the constant region in frame - the fifth predicate of `productive`, and the one a record can fail on its own | + +`productive` is the conjunction of **five** exported predicates under +three-valued logic: `v_anchor_accepted`, `j_anchor_accepted`, +`vj_in_frame`, `stop_codon` being `False`, and `j_tail_in_frame`. Every +one of them is on the record, so you can recompute the verdict and see +which predicate a disagreement comes from. The policy requires original V/J landmark provenance, V Cys and J Phe/Trp residue classes, V/J and J-tail frame agreement, and definitely stop-free @@ -448,6 +462,24 @@ code that assumes they're always present; check with `"clone_id" in rec`. the absent case explicitly. See [Clonal simulation overview](../guides/clonal-families.md). +## Genotype fields + +When an experiment carries a genotype (`.with_genotype(...)`), or when +records come from `run_cohort(...)`, two more columns are stamped on +every record: + +| Field | Type | Meaning | +|---|---|---| +| `subject_id` | str | The genotype's subject label, carried from `Genotype.with_subject(...)` or auto-assigned (`subject_0`, `subject_1`, …) by `run_cohort` | +| `haplotype` | int | `0` or `1` - which chromosome this rearrangement's V, D and J were all drawn from | + +Whether a given read drew a **novel** allele is deliberately not a +record column: it is a join of `truth_v_call` (from +`expose_provenance=True`) against the genotype truth sidecar that +`result.to_genotype_tsv(path)` writes, on `(subject_id, allele)`. The +record already carries both halves. See +[Genotypes](../guides/genotype.md). + ## Validation The records page composes cleanly with the validation surface: diff --git a/site_docs/concepts/contracts.md b/site_docs/concepts/contracts.md index 636595a5..471a5e56 100644 --- a/site_docs/concepts/contracts.md +++ b/site_docs/concepts/contracts.md @@ -1,11 +1,5 @@ # Contracts -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Sample only productive sequences](https://mutejester.github.io/GenAIRR/guides/productive.html). diff --git a/site_docs/concepts/live-call.md b/site_docs/concepts/live-call.md index 3e59e2e6..84bff3f2 100644 --- a/site_docs/concepts/live-call.md +++ b/site_docs/concepts/live-call.md @@ -1,11 +1,5 @@ # Live Calls -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Architecture](https://mutejester.github.io/GenAIRR/architecture/index.html). diff --git a/site_docs/concepts/persistent-ir.md b/site_docs/concepts/persistent-ir.md index 236f81a0..722e7784 100644 --- a/site_docs/concepts/persistent-ir.md +++ b/site_docs/concepts/persistent-ir.md @@ -1,11 +1,5 @@ # Persistent IR -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Architecture](https://mutejester.github.io/GenAIRR/architecture/index.html). diff --git a/site_docs/concepts/pipeline.md b/site_docs/concepts/pipeline.md index 8b538a4d..0c0adf82 100644 --- a/site_docs/concepts/pipeline.md +++ b/site_docs/concepts/pipeline.md @@ -1,11 +1,5 @@ # The Simulation Pipeline -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [The Experiment builder](https://mutejester.github.io/GenAIRR/guides/experiment-builder.html). diff --git a/site_docs/concepts/reference-cartridge.md b/site_docs/concepts/reference-cartridge.md index a7303fdc..31305a8b 100644 --- a/site_docs/concepts/reference-cartridge.md +++ b/site_docs/concepts/reference-cartridge.md @@ -57,7 +57,8 @@ differently; the trace files for their runs are not confusable. cfg.metadata.species # Species.HUMAN cfg.metadata.chain_type # ChainType.BCR_HEAVY cfg.metadata.reference_set # "OGRDB V8" -cfg.name # "human_igh" +cfg.name # "Heavy Chain" - the cartridge's own display + # name, not the "human_igh" string shortcut ``` The cartridge validator cross-checks `chain_type` against the locus @@ -137,7 +138,12 @@ cfg.reference_models = ReferenceEmpiricalModels( "NP1": NpBaseModelSpec( kind="markov", first_base={"A": 0.25, "C": 0.25, "G": 0.25, "T": 0.25}, - transitions={...}, + transitions={ + "A": {"A": 0.0, "C": 0.3, "G": 0.4, "T": 0.3}, + "C": {"A": 0.3, "C": 0.0, "G": 0.3, "T": 0.4}, + "G": {"A": 0.4, "C": 0.3, "G": 0.0, "T": 0.3}, + "T": {"A": 0.3, "C": 0.4, "G": 0.3, "T": 0.0}, + }, ), }, p_nucleotide_lengths={ @@ -271,7 +277,8 @@ introspection: manifest = cfg.cartridge_manifest() manifest["identity"] # name, species, locus, reference_set, source -manifest["catalogue"]["v_allele_count"] # int +manifest["catalogue"]["v_count"] # int (also d_count / j_count / + # c_count / functional_status_counts) manifest["rules"] # anchor expectations, allowed bases manifest["models"]["allele_usage"] # available / nonempty_segments / soft-gap flag manifest["models"]["trim_models"] # keys / source / plan-signature flag @@ -337,8 +344,9 @@ explicitly use `allow_curatable_refdata`. Three layers, in the order they fire: 1. **Validation** describes the catalogue against the cartridge's - rules. Returns a list of issue dicts each tagged with severity - (`"fatal"` or `"curatable"`). + rules. On the engine-side `RefDataConfig`, `validate()` returns a + list of issue dicts each tagged with severity (`"fatal"` or + `"curatable"`). 2. **Curation** selects which alleles participate. Re-runs validation on the curated cartridge. Fatal structural issues are NOT fixed by curation - they still surface. @@ -348,10 +356,13 @@ Three layers, in the order they fire: strict mode. ```python -# Direct authoring -issues = cfg.validate() -cfg.validate_strict() # raises on any issue -cfg.validate_with_mode(mode="allow_curatable") # raises only on Fatal +# Direct authoring. The three mode-aware entry points live on the +# engine-side bridge object, not on the Python DataConfig: +# DataConfig.validate() raises DataConfigError and returns nothing. +refdata = ga.dataconfig_to_refdata(cfg) +issues = refdata.validate() # list[dict], [] when clean +refdata.validate_strict() # raises on any issue +refdata.validate_with_mode(mode="allow_curatable") # raises only on Fatal # Through Experiment ga.Experiment.on(cfg).curate_refdata("functional_anchors_only").compile() diff --git a/site_docs/demo.md b/site_docs/demo.md index ee02b1d4..0ee539d2 100644 --- a/site_docs/demo.md +++ b/site_docs/demo.md @@ -98,8 +98,7 @@ df = result.to_dataframe() ```text len(result) = 1000 -df.shape = (1000, 102) -elapsed = 2.0s +df.shape = (1000, 128) ``` @@ -112,26 +111,26 @@ elapsed = 2.0s
row 0 sequence - CVKDDGNRGYCSGDSCYGHCCALDYWYFDLW + CARCRTRLTRAYHITGTAHFQHW
Observed fields -
v_callIGHVF10-G38*04
-
d_callIGHD2-15*01
-
j_callIGHJ2*01
+
v_callIGHVF6-G23*01
+
d_callIGHD1-20*01
+
j_callIGHJ1*01
Ground truth -
truth_v_callIGHVF10-G38*04
-
truth_d_callIGHD2-15*01
-
truth_j_callIGHJ2*01
+
truth_v_callIGHVF6-G23*01
+
truth_d_callIGHD1-20*01
+
truth_j_callIGHJ1*01
Record state
productiveTrue
-
n_mutations10
-
columns102
+
n_mutations14
+
columns128
@@ -173,7 +172,7 @@ print(f"With productive_only(): {constrained_productive:.1%}")
Output
```text -Without productive_only(): 18.5% +Without productive_only(): 19.0% With productive_only(): 100.0% ``` @@ -182,8 +181,8 @@ With productive_only(): 100.0%
Without productive_only()
-
-
18.5%
+
+
19.0%
With productive_only()
@@ -212,7 +211,7 @@ pipeline.
Trace file - 12,996 bytes + 14,247 bytes
@@ -258,7 +257,7 @@ replayed = compiled.simulator.replay_from_trace_file(tf, strict=False)
Output
```text -trace size = 12,996 bytes +trace size = 14,247 bytes byte-identical fields = sequence, V/D/J calls, junction_aa, n_mutations ``` @@ -280,7 +279,9 @@ compiled_modified.simulator.replay_from_trace_file(tf, strict=False)
Output
```text -ValueError: replay_from_trace_file: pass plan signature mismatch. +ValueError: replay_from_trace_file: trace-file pass plan signature mismatch: + expected: sample_allele.v()|...|mutate.s5f(rate=0.03) + actual: sample_allele.v()|...|mutate.s5f(rate=0.05) ```
@@ -291,7 +292,9 @@ ValueError: replay_from_trace_file: pass plan signature mismatch. One parent recombination can fork into many independently mutated descendants. This demo uses legacy `expand_clones` for the simple -fixed-size star shape; for new clone benchmarks, see +fixed-size star shape - it is deprecated and raises a +`DeprecationWarning`, but it is still the only surface that gives an +exact per-clone count. For new clone benchmarks, see [`clonal_lineage`](guides/clonal-lineage.md) for BCR trees and [`clonal_repertoire`](guides/clonal-repertoire.md) for TCR / abundance repertoires. @@ -315,7 +318,7 @@ result = ( records = 500 parents = 10 per clone = 50 -clone 0 SHM counts = 4, 14, 8, 7, 5, ... +clone 0 SHM counts = 10, 7, 4, 13, 14, ... validate_families() = ok ``` diff --git a/site_docs/getting-started/first-airr-record.md b/site_docs/getting-started/first-airr-record.md index 56f73272..89dd9532 100644 --- a/site_docs/getting-started/first-airr-record.md +++ b/site_docs/getting-started/first-airr-record.md @@ -9,7 +9,7 @@ real record so you can read your own output confidently.

```python import GenAIRR as ga -result = ga.Experiment.on("human_igh").recombine().run_records(n=1, seed=0) +result = ga.Experiment.on("human_igh").recombine().run_records(n=1, seed=5) rec = result[0] print(rec) ``` @@ -20,15 +20,15 @@ keys; we'll cover them by category below): ```python { "sequence_id": "seq0", - "sequence": "gaggtgcagctggtg...", - "sequence_aa": "EVQLVESGGGLVQPG...", + "sequence": "gaagtgcagctggtg...", + "sequence_aa": "EVQLVESGGVVVQPG...", "locus": "IGH", - "v_call": "IGHVF10-G38*04", - "d_call": "IGHD2-15*01", - "j_call": "IGHJ2*01", - "junction": "TGTGCGAGAGATGATGGAAATAGAGGCTACTGCAGT...", - "junction_aa": "CARDDGNRGYCSGGSCYGRCCALDYW", - "junction_length": 78, + "v_call": "IGHVF10-G42*04", + "d_call": "IGHD2-2*02", + "j_call": "IGHJ1*01", + "junction": "tgtgcaaaagatATCTTTAACGAatattgtagtagtacc...", + "junction_aa": "CAKDIFNEYCSSTSCYFQHW", + "junction_length": 60, "productive": True, "vj_in_frame": True, "stop_codon": False, @@ -41,8 +41,8 @@ keys; we'll cover them by category below): ## Sequence ```python -rec["sequence"] # 'gaggtgcagctggtg…' - assembled nucleotide -rec["sequence_aa"] # 'EVQLVESGGGLVQPG…' - codon-rail translation +rec["sequence"] # 'gaagtgcagctggtg…' - assembled nucleotide +rec["sequence_aa"] # 'EVQLVESGGVVVQPG…' - codon-rail translation rec["sequence_id"] # 'seq0' - auto-assigned; carries to FASTQ headers rec["locus"] # 'IGH' - derived from v_call / j_call ``` @@ -55,9 +55,9 @@ translation - stops emit `*`, ambiguous codons emit `X`. ## V / D / J calls ```python -rec["v_call"] # 'IGHVF10-G38*04' -rec["d_call"] # 'IGHD2-15*01' -rec["j_call"] # 'IGHJ2*01' +rec["v_call"] # 'IGHVF10-G42*04' +rec["d_call"] # 'IGHD2-2*02' +rec["j_call"] # 'IGHJ1*01' ``` Each standard call names the generating allele used by the corresponding @@ -74,9 +74,9 @@ calls when `expose_provenance=True`. ## Junction + productivity ```python -rec["junction"] # 'TGTGCGAGAGATGAT…' - V Cys through J W/F + 3 -rec["junction_aa"] # 'CARDDGNRGYCSGGSCYGRCCALDYW' -rec["junction_length"] # 78 +rec["junction"] # 'tgtgcaaaagatATCTTT…' - V Cys through J W/F + 3 +rec["junction_aa"] # 'CAKDIFNEYCSSTSCYFQHW' +rec["junction_length"] # 60 rec["productive"] # True rec["vj_in_frame"] # True rec["stop_codon"] # False @@ -147,7 +147,7 @@ When `productive_only()` is in the pipeline, every record carries ```python rec["coding_frame_start"] # 0 (first base of the reading frame) -rec["coding_frame_end"] # 316 (one past the last base assessed) +rec["coding_frame_end"] # 376 (one past the last base assessed) rec["j_tail_in_frame"] # True (the fifth predicate of `productive`) rec["v_anchor_accepted"] # True (the first two) rec["j_anchor_accepted"] # True @@ -223,20 +223,21 @@ its germline, and none of them moves this count — those are reported on ## Paired-end fields (when requested) ```python -rec["read_layout"] # 'single' by default; 'paired_end' after .paired_end(...) +rec["read_layout"] # '' by default; 'paired_end' after .paired_end(...) rec["r1_sequence"] # '' unless paired_end is in the pipeline rec["r2_sequence"] # '' -rec["r1_start"] # 0 -rec["r1_end"] # 0 -rec["r2_start"] # 0 -rec["r2_end"] # 0 +rec["r1_start"] # None +rec["r1_end"] # None +rec["r2_start"] # None +rec["r2_end"] # None rec["insert_size"] # 0 ``` These fields are always present on the record dict for schema stability, but they're populated only when `Experiment.paired_end( r1_length=..., r2_length=..., insert_size=...)` is in the pipeline. -Otherwise they carry their zero / empty defaults. +Otherwise the four window coordinates are `None`, `read_layout` and the +two sequences are empty strings, and `insert_size` is `0`. ## The full schema diff --git a/site_docs/getting-started/installation.md b/site_docs/getting-started/installation.md index 9dddb50f..57f75e2b 100644 --- a/site_docs/getting-started/installation.md +++ b/site_docs/getting-started/installation.md @@ -6,8 +6,10 @@ required on any supported platform.

## Requirements -- **Python 3.9 or newer** (3.9, 3.10, 3.11, 3.12, and 3.13 are - officially supported and CI-tested). +- **Python 3.9 or newer.** The test suite runs in CI on Python + 3.10, 3.11, 3.12 and 3.13, on Linux, macOS and Windows. Python 3.9 + is supported and its wheels are smoke-tested on all three platforms, + but the suite is not run against it. - One of the following platforms with a pre-built wheel: - Linux x86_64 or aarch64 - macOS Intel or Apple Silicon @@ -36,6 +38,7 @@ After install, a one-liner confirms everything is in place: import GenAIRR as ga print(ga.__version__) +print(ga.DETERMINISM_VERSION) # which sampling model this engine ships ``` If this prints a version string, the Python API loaded and the @@ -60,10 +63,10 @@ pip install GenAIRR[viz] # graphviz pip install GenAIRR[mcp] # fastmcp (for the MCP server) ``` -`pandas` is needed for `result.to_dataframe()`; it ships under -`[all]`. The core simulator works without any extras - every export -format except `to_dataframe` is implemented in pure Python on the -standard library. +`pandas` is needed for `result.to_dataframe()` and is **not** part of +any extra - install it yourself (`pip install pandas`). The core +simulator works without any extras: every export format except +`to_dataframe` is implemented in pure Python on the standard library. ## Documentation build (optional, for contributors) diff --git a/site_docs/getting-started/lesson-1-vdj-recombination-from-bases-up.md b/site_docs/getting-started/lesson-1-vdj-recombination-from-bases-up.md index 24d6bc84..4512eaad 100644 --- a/site_docs/getting-started/lesson-1-vdj-recombination-from-bases-up.md +++ b/site_docs/getting-started/lesson-1-vdj-recombination-from-bases-up.md @@ -1,11 +1,5 @@ # Lesson 1 - V(D)J recombination, from bases up -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Quick start](https://mutejester.github.io/GenAIRR/getting-started/quick-start.html). diff --git a/site_docs/getting-started/lesson-2-the-pipeline-scrubber.md b/site_docs/getting-started/lesson-2-the-pipeline-scrubber.md index 4156633f..b91b95a2 100644 --- a/site_docs/getting-started/lesson-2-the-pipeline-scrubber.md +++ b/site_docs/getting-started/lesson-2-the-pipeline-scrubber.md @@ -1,11 +1,5 @@ # Lesson 2 - The pipeline scrubber -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [The Experiment builder](https://mutejester.github.io/GenAIRR/guides/experiment-builder.html). diff --git a/site_docs/getting-started/lesson-3-s5f-mutation-isnt-uniform.md b/site_docs/getting-started/lesson-3-s5f-mutation-isnt-uniform.md index cddfbc61..d66812db 100644 --- a/site_docs/getting-started/lesson-3-s5f-mutation-isnt-uniform.md +++ b/site_docs/getting-started/lesson-3-s5f-mutation-isnt-uniform.md @@ -1,11 +1,5 @@ # Lesson 3 - S5F: mutation isn't uniform -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Targeted SHM rates](https://mutejester.github.io/GenAIRR/guides/shm-targeting.html). diff --git a/site_docs/getting-started/lesson-4-sequencing-artifacts-dialed-in.md b/site_docs/getting-started/lesson-4-sequencing-artifacts-dialed-in.md index 54771d31..cc549adf 100644 --- a/site_docs/getting-started/lesson-4-sequencing-artifacts-dialed-in.md +++ b/site_docs/getting-started/lesson-4-sequencing-artifacts-dialed-in.md @@ -1,11 +1,5 @@ # Lesson 4 - Sequencing artifacts, dialed in -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Corruption + sequencing artefacts](https://mutejester.github.io/GenAIRR/guides/corruption-sequencing.html). diff --git a/site_docs/getting-started/lesson-5-the-ground-truth-payoff.md b/site_docs/getting-started/lesson-5-the-ground-truth-payoff.md index 620a994a..b8c15f8c 100644 --- a/site_docs/getting-started/lesson-5-the-ground-truth-payoff.md +++ b/site_docs/getting-started/lesson-5-the-ground-truth-payoff.md @@ -1,11 +1,5 @@ # Lesson 5 - The ground-truth payoff -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Your first AIRR record](https://mutejester.github.io/GenAIRR/getting-started/first-airr-record.html). diff --git a/site_docs/getting-started/quick-start.md b/site_docs/getting-started/quick-start.md index 3b3051e7..ec2d294e 100644 --- a/site_docs/getting-started/quick-start.md +++ b/site_docs/getting-started/quick-start.md @@ -28,8 +28,8 @@ print(len(result)) # 1000 rec = result[0] print(rec["v_call"], rec["d_call"], - rec["j_call"]) # 'IGHVF10-G38*04 IGHD2-15*01 IGHJ2*01' -print(rec["junction_aa"]) # 'CVKDDGNRGYCSGGSCYGRCCALDYWYFDLW' + rec["j_call"]) # 'IGHVF6-G23*01 IGHD1-20*01 IGHJ1*01' +print(rec["junction_aa"]) # 'CARCRTRLTRAYHITGTAHFQHW' print(rec["productive"]) # True ``` diff --git a/site_docs/guides/biology-map.md b/site_docs/guides/biology-map.md index 666be808..f0d34c3b 100644 --- a/site_docs/guides/biology-map.md +++ b/site_docs/guides/biology-map.md @@ -30,24 +30,24 @@ better starting point. | **Allele restriction** | `.restrict_alleles(v=..., d=..., j=...)` | Constraint-aware sampling | Sampled `v_call`/`d_call`/`j_call` restricted to the listed alleles | [Recombination + junction biology](recombination-junction.md) | | **Somatic hypermutation (SHM)** | `.mutate(model="s5f", rate=...)` or `.mutate(model="uniform", count=...)` | Biology - descendant phase | `n_mutations`, `mutation_rate`, `n_v_mutations`, `n_d_mutations`, `n_j_mutations`, `n_np_mutations` | [SHM and mutation targeting](shm-targeting.md) | | **Targeted SHM** | `.mutate(..., segment_rates=..., v_subregion_rates=...)` | Biology - descendant phase | Same plus the six V-subregion counters (`n_fwr1_mutations` … `n_v_unannotated_mutations`) | [SHM and mutation targeting](shm-targeting.md) | -| **PCR substitution errors** | `.pcr_amplify(count=...)` or `.pcr_amplify(rate=...)` | Library / sequencing artefact - descendant | `n_pcr_errors`, lowercase corruption markers in `sequence` | [Corruption + sequencing artefacts](corruption-sequencing.md) | +| **PCR substitution errors** | `.pcr_amplify(count=...)` or `.pcr_amplify(rate=...)` | Library / sequencing artefact - descendant | `n_pcr_errors` (the substituted bytes are written uppercase, like every engine-written base) | [Corruption + sequencing artefacts](corruption-sequencing.md) | | **Sequencing errors** | `.sequencing_errors(count=...)` or `.sequencing_errors(rate=...)` | Library / sequencing artefact - descendant | `n_quality_errors`, lowercase corruption markers in `sequence` | [Corruption + sequencing artefacts](corruption-sequencing.md) | -| **Ambiguous base calls (N)** | `.ambiguous_base_calls(count=...)` | Library / sequencing artefact - descendant | `n_quality_errors`, `N` characters in `sequence` | [Corruption + sequencing artefacts](corruption-sequencing.md) | +| **Ambiguous base calls (N)** | `.ambiguous_base_calls(count=...)` | Library / sequencing artefact - descendant | `n_ambiguous_bases`, `N` characters in `sequence` (it moves neither error counter) | [Corruption + sequencing artefacts](corruption-sequencing.md) | | **Polymerase indels** | `.polymerase_indels(count=..., insertion_prob=0.5)` | Library / sequencing artefact - descendant | `n_indels`, `n_v_indels`, `n_d_indels`, `n_j_indels` (NP indels count toward total only) | [Corruption + sequencing artefacts](corruption-sequencing.md) | | **End-loss (5′ / 3′)** | `.end_loss_5prime(length=...)`, `.end_loss_3prime(length=...)` (or `primer_trim_*prime` aliases) | Library / sequencing artefact - descendant | `end_loss_5_length`, `end_loss_3_length` | [Corruption + sequencing artefacts](corruption-sequencing.md) | | **Random strand orientation** | `.random_strand_orientation(prob=0.5)` | Read layout - descendant | `rev_comp` | [Corruption + sequencing artefacts](corruption-sequencing.md) | | **Paired-end layout** | `.paired_end(r1_length=..., insert_size=...)` | Read layout - descendant | `read_layout`, `r1_sequence`, `r2_sequence`, `r1_start`, `r1_end`, `r2_start`, `r2_end`, `insert_size` | [Paired-end reads and FASTQ](paired-end-fastq.md) | | **BCR lineage trees** | `.clonal_lineage(...)` | BCR affinity-maturation tree | `clone_id`, `lineage_*`, `duplicate_count`, `result.lineage_trees` | [Clonal lineage trees](clonal-lineage.md) | | **TCR / flat-BCR clone-size repertoires** | `.clonal_repertoire(...)` | Non-tree clonal abundance | `clone_id`, `duplicate_count` | [Clonal repertoires](clonal-repertoire.md) | -| **Legacy fixed-size clonal stars** | `.expand_clones(n_clones=..., per_clone=...)` | Ancestor / descendant fork | `clone_id`, `parent_id`, `result.parents` | [Clonal simulation overview](clonal-families.md) | +| **Legacy fixed-size clonal stars** | `.expand_clones(n_clones=..., per_clone=...)` *(deprecated - raises a `DeprecationWarning`)* | Ancestor / descendant fork | `clone_id`, `parent_id`, `result.parents` | [Clonal simulation overview](clonal-families.md) | | **Contamination** | `.contaminate(prob=...)` | Library / sequencing artefact - descendant | `is_contaminant` | [Experiment builder](experiment-builder.md) | | **Sample metadata** | `.with_metadata(**fields)` | Bookkeeping - post-run | Arbitrary user-stamped columns | [Experiment builder](experiment-builder.md) | ## Stage ordering -GenAIRR's pipeline splits into five conceptual stages. The -engine enforces the ordering at compile time - any out-of-order -call raises `ValueError`. +GenAIRR's pipeline splits into five conceptual stages. The DSL +enforces the ordering on the method call itself, before `compile()` - +an out-of-order call raises `ValueError` where you wrote it. **1. Recombination.** `.recombine()` runs the V(D)J join, trim-and-fill, NP-region generation, and P-nucleotide diff --git a/site_docs/guides/build-config.md b/site_docs/guides/build-config.md index d5740dec..01575f57 100644 --- a/site_docs/guides/build-config.md +++ b/site_docs/guides/build-config.md @@ -1,11 +1,5 @@ # Build a config for your locus or species -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Build a reference cartridge](https://mutejester.github.io/GenAIRR/guides/build-reference-cartridge.html) or [Reference Cartridge Authoring](https://mutejester.github.io/GenAIRR/cartridges/index.html). diff --git a/site_docs/guides/build-reference-cartridge.md b/site_docs/guides/build-reference-cartridge.md index c9256b83..b1c9755a 100644 --- a/site_docs/guides/build-reference-cartridge.md +++ b/site_docs/guides/build-reference-cartridge.md @@ -212,7 +212,12 @@ models = ReferenceEmpiricalModels( "NP1": NpBaseModelSpec( kind="markov", first_base={"A": 0.25, "C": 0.25, "G": 0.25, "T": 0.25}, - transitions={...}, + transitions={ + "A": {"A": 0.0, "C": 0.3, "G": 0.4, "T": 0.3}, + "C": {"A": 0.3, "C": 0.0, "G": 0.3, "T": 0.4}, + "G": {"A": 0.4, "C": 0.3, "G": 0.0, "T": 0.3}, + "T": {"A": 0.3, "C": 0.4, "G": 0.3, "T": 0.0}, + }, ), }, p_nucleotide_lengths={ @@ -345,7 +350,7 @@ authored cartridge: manifest = cfg.cartridge_manifest() manifest["identity"] # species, locus, reference_set, name, source -manifest["catalogue"]["v_allele_count"] # post-curation count +manifest["catalogue"]["v_count"] # post-curation count manifest["rules"]["allowed_bases"] # what the alphabet is manifest["models"]["allele_usage"] # available + nonempty segments + soft-gap flag manifest["models"]["trim_models"] # keys + source + plan-signature flag diff --git a/site_docs/guides/cartridge-manifest-report.md b/site_docs/guides/cartridge-manifest-report.md index 45ea9871..db86e09c 100644 --- a/site_docs/guides/cartridge-manifest-report.md +++ b/site_docs/guides/cartridge-manifest-report.md @@ -53,16 +53,23 @@ the present state and the immutable history of how you got here. ## Reading the manifest `cfg.cartridge_manifest()` returns a single nested dict with -nine top-level keys: +twelve top-level keys: ```python manifest = cfg.cartridge_manifest() sorted(manifest) -# ['catalogue', 'curation', 'dropped_allele_fields', 'errors', +# ['catalogue', 'checksum_scheme', 'curation', 'dropped_allele_fields', # 'hashes', 'identity', 'models', 'orphan_dataconfig_fields', -# 'rules', 'schema_version'] +# 'reference_audit', 'reference_evidence', 'rules', 'schema_version'] ``` +`checksum_scheme` names the checksum algorithm the artifact was +stamped with (`pickle4_roundtrip_v2` on cartridges built by the +current builder, `pickle4_v1` on archived ones). +`reference_audit` and `reference_evidence` carry the catalogue's +quarantine findings and its sequence-bound landmark evidence - see +[Reference evidence and quarantine](../cartridges/reference-evidence.md). + The remaining sections walk through each. ### `identity` @@ -134,8 +141,14 @@ manifest["models"] # "np_length_models": {...}, # "np_base_models": {...}, # "p_nucleotide_models": {...}, -# "np_length_keys": ["NP1", "NP2"], -# "trim_keys": ["v_trim_3", "d_trim_5", "d_trim_3", "j_trim_5"], +# "genotype_priors": {...}, +# } +# The trim and NP length keys live inside their own blocks, as plane +# keys rather than DataConfig field names: +# manifest["models"]["trim_models"]["keys"] # ["D_3","D_5","J_5","V_3"] +# manifest["models"]["np_length_models"]["keys"] # ["NP1","NP2"] +# Both are [] on a cartridge with no typed plane (HUMAN_IGH_OGRDB included). +# { # "legacy_np_lengths_present": False, # "legacy_trim_dicts_present": False # } @@ -186,8 +199,8 @@ surface. ```python manifest["hashes"] # { -# "data_config_checksum": "sha256:e3b0c44...", -# "refdata_content_hash": "sha256:6a1f8b2..." +# "data_config_checksum": "a66c87e1...", # bare hex, no prefix +# "refdata_content_hash": "sha256:a98f4dad..." # prefixed # } ``` @@ -260,9 +273,11 @@ One entry per builder method that ran. Each carries: - `inferred` - what the stage computed (counts, model summaries, fallback flags) - `warnings` - stage-level warnings (e.g. P-naive zero-fraction) -- `replaced` - `True` when an estimator overwrote a prior typed - plane (idempotency marker; see - [Estimate cartridge models from real data](estimate-cartridge-models.md)) + +Those four are the whole stage dict. An estimator's idempotency +marker is `inputs["replaced"]` - `True` when the estimator +overwrote a prior typed plane - not a fifth top-level key; see +[Estimate cartridge models from real data](estimate-cartridge-models.md). ### `warnings` @@ -316,12 +331,13 @@ assert manifest["catalogue"]["j_count"] >= 5 # Models - every plane you need must be attached assert manifest["models"]["allele_usage"]["available"] -assert manifest["models"]["trim_models"]["available"] -assert manifest["models"]["np_length_models"]["available"] +assert manifest["models"]["trim_models"]["keys"] +assert manifest["models"]["np_length_models"]["keys"] assert manifest["models"]["np_base_models"]["models"] -# No deferred errors -assert not manifest.get("errors") +# Only `allele_usage` and `genotype_priors` carry an `available` +# flag. The trim / NP-length blocks report their plane `keys` +# instead, which are empty when no typed plane is attached. # No rejected rows above an acceptable threshold (records-driven build) assert len(report.rejected) < 0.01 * len(input_records) @@ -364,8 +380,8 @@ be filtered to known alleles. **Zero coverage for a model plane.** An estimator stage that ran on zero rows (all rows rejected). The cartridge silently falls back to defaults for that plane; the build report's stage -entry stamps `records_used: 0`. Pin the count in CI to catch -this. +entry stamps `inputs["record_count"]`. Pin that count in CI to +catch this. ## Hashes and reproducibility diff --git a/site_docs/guides/clonal-families.md b/site_docs/guides/clonal-families.md index 427e26cb..107642a4 100644 --- a/site_docs/guides/clonal-families.md +++ b/site_docs/guides/clonal-families.md @@ -15,10 +15,12 @@ truth the simulator created.

| **BCR lineage reconstruction / affinity maturation** | [`clonal_lineage(...)`](clonal-lineage.md) | Generation-synchronous B-cell tree, per-division S5F SHM, optional sequence-distance selection, final live-cell sampling | AIRR records with `clone_id`, `lineage_*`, `duplicate_count`; one `LineageTree` per clone | | **TCR clone-size / abundance benchmark** | [`clonal_repertoire(...)`](clonal-repertoire.md) | One rearranged T cell copied to a heavy-tailed clone size; no SHM; optional per-read technical noise | AIRR records with `clone_id` and AIRR `duplicate_count` | | **Flat BCR abundance without genealogy** | [`clonal_repertoire(...)`](clonal-repertoire.md) | One BCR rearrangement copied to a heavy-tailed size; optional flat post-fork SHM per copy | AIRR records with `clone_id` and `duplicate_count` | -| **Legacy fixed-size star** | `expand_clones(...)` | One parent rearrangement and a fixed `per_clone` descendant count | AIRR records with `clone_id`, `parent_id`; parent `Outcome`s on `result.parents` | +| **Legacy fixed-size star** | `expand_clones(...)` *(deprecated - raises a `DeprecationWarning`)* | One parent rearrangement and a fixed `per_clone` descendant count | AIRR records with `clone_id`, `parent_id`; parent `Outcome`s on `result.parents` | -`expand_clones` is still supported for old scripts, but new clone-related -benchmarks should usually start with `clonal_lineage` or `clonal_repertoire`. +`expand_clones` is still supported for old scripts and is still the only +surface that gives an exact per-clone count, but it raises a +`DeprecationWarning`; new clone-related benchmarks should start with +`clonal_lineage` or `clonal_repertoire`. Those two encode the distinction AIRR users usually care about: BCR lineages are mutation trees, while TCR clones are abundance groups with technical read noise. @@ -89,8 +91,10 @@ node_table = tree.to_node_table_tsv() What happens: 1. `recombine()` creates one naive BCR founder per clone. -2. The Rust lineage engine grows a tree for `max_generations`, with live-cell - carrying capacity `n_max`. +2. The Rust lineage engine grows a tree for `max_generations`. `n_max` is the + live-cell population each generation **settles at** under Beverton-Holt + regulation, not a hard cap: a generation may overshoot it and the next is + regulated back below replacement. 3. Each child receives per-division S5F SHM at `rate`. 4. If selection is enabled, offspring rates are modulated by a BLOSUM62 sequence-distance proxy, not a physical antigen-binding model. diff --git a/site_docs/guides/clonal-lineage.md b/site_docs/guides/clonal-lineage.md index 3beeff24..81012596 100644 --- a/site_docs/guides/clonal-lineage.md +++ b/site_docs/guides/clonal-lineage.md @@ -47,7 +47,8 @@ result = ( n_max=300, # living population each generation settles at n_sample=30, # cells sampled per family at the end rate=0.01, # per-base S5F SHM rate, per division - lambda_base=1.6, # mean offspring per cell per generation + lambda_base=1.6, # Poisson mean for a founder-baseline cell + # in an empty germinal center selection_strength=10.0, # 0 = neutral; >0 = affinity selection ) .run_records(seed=0) @@ -390,10 +391,10 @@ seed draws a different repertoire rather than a reshuffling of the same one. | `n_max` | 1000 | **Per-generation LIVING-population carrying capacity** - the population each generation settles at. A generation may overshoot it; the next is regulated back. It is **not** a cap on total cells per clone - the tree contains far more nodes across generations | | `n_sample` | 50 | Cells sampled per family at the end; records per clone ≤ this (genotype-collapsed) | | `rate` | 0.05 | Per-base S5F SHM rate, per division | -| `lambda_base` | 1.5 | Mean offspring per cell per generation | +| `lambda_base` | 1.5 | Poisson mean offspring count for a **founder-baseline cell in an empty germinal center**. The mean a cell actually draws against is this damped by the live population (see `n_max`) and, under selection, scaled by the cell's fitness relative to the founder - so it is **not** the mean offspring count once either is active | | `selection_strength` | 0.0 | `0` = neutral drift (`fitness ≡ 1`); set `> 0` for affinity selection. Note `0` makes selection neutral but does **not** force `lineage_affinity` to 0 - affinities are still computed/reported whenever a `target_aa` is supplied | | `beta` | 1.0 | Affinity steepness in `exp(−beta·distance)` | -| `target_aa` | `None` | Target amino-acid sequence (a full translated receptor) used as the selection target (BLOSUM62-weighted distance, position-wise; only the overlapping prefix is scored when lengths differ). A **sequence-distance proxy**, not a biophysical antigen. `None` ⇒ auto "mature" target | +| `target_aa` | `None` | Target amino-acid sequence (a full translated receptor) used as the selection target (BLOSUM62-weighted distance, position-wise). It must be **exactly as long as the receptor's protein** or the run is refused - a wrong-length target used to be scored over the overlapping prefix and reported a number that was not a distance to it. A **sequence-distance proxy**, not a biophysical antigen. `None` ⇒ auto "mature" target | | `mature_substitutions` | 5 | aa substitutions for the auto target | | `s5f_model` | `"hh_s5f"` | Bundled S5F kernel | | `allow_extinction` | `False` | `False` ⇒ condition each clone on survival (retry extinct founders with fresh deterministic sub-seeds), so you reliably get `n_clones` families. `True` ⇒ accept extinction and skip extinct clones, producing fewer families | diff --git a/site_docs/guides/corruption-sequencing.md b/site_docs/guides/corruption-sequencing.md index 6fc64b01..9c230245 100644 --- a/site_docs/guides/corruption-sequencing.md +++ b/site_docs/guides/corruption-sequencing.md @@ -10,7 +10,7 @@ artefact profile of the sequencer you're benchmarking against.

## What counts as observation-stage corruption -Seven passes belong in the observation-stage block. Each models a +Eight passes belong in the observation-stage block. Each models a distinct artefact mechanism, each has its own AIRR provenance, and each is independently tunable: @@ -23,6 +23,7 @@ each is independently tunable: | `.end_loss_5prime(...)` | 5' adapter / primer loss | `end_loss_5_length` | | `.end_loss_3prime(...)` | 3' adapter / primer loss | `end_loss_3_length` | | `.random_strand_orientation(...)` | Strand flip during library prep | `rev_comp` | +| `.contaminate(...)` | Replaces the record with an unrelated background sequence | `is_contaminant` | Plus `.paired_end(...)`, which isn't corruption - it's a read-layout projection - but it sits in the same descendant-phase block of the @@ -87,16 +88,19 @@ result = ( .run_records(n=100, seed=1) ) +rec = result[0] rec["n_quality_errors"] # int - number of sequencer base-call errors ``` `sequencing_errors` models sequencer-quality miscalls. The pass **lowercases the affected bytes** in the assembled sequence as a -visible quality marker. So `sequence` may end up with mixed-case -bytes: uppercase = clean, lowercase = sequencer-flagged -low-quality position. Downstream pipelines that want to mask -quality-flagged bases can do -`sequence.upper()` to normalise. +visible quality marker. Case is not a clean/dirty flag, though: +germline reference bases are already lowercase, while N/P additions, +SHM substitutions and PCR errors are written uppercase, so a record +with no corruption at all is mostly lowercase. Case marks what the +engine wrote, not what is wrong; read the counters +(`n_quality_errors`, `n_pcr_errors`, `n_ambiguous_bases`) for that, +and `sequence.upper()` to normalise. ## N corruption / ambiguous bases @@ -263,7 +267,7 @@ A few facts to know: ## Ordering with clonal workflows -**All seven corruption passes are descendant-phase.** They model +**All eight corruption passes are descendant-phase.** They model per-read artefacts - every emitted clone member or observed lineage cell gets independent PCR errors, independent indel events, independent end-loss draws, etc. @@ -430,8 +434,7 @@ on) already reflect the post-flip orientation. R2 is also already reverse-complemented at projection time (that's the paired-end-output convention). Applying any further flips downstream produces wrong reads. The validator's -`PairedEndWindowMismatch` check enforces the per-record R2 RC -state. +`ReadSequenceMismatch` check enforces the per-record R2 RC state. **Looking for an `n_ns` counter.** The field is spelled `n_ambiguous_bases`. It counts positions the pass actually masked, diff --git a/site_docs/guides/estimate-cartridge-models.md b/site_docs/guides/estimate-cartridge-models.md index e814e49a..8755ac2c 100644 --- a/site_docs/guides/estimate-cartridge-models.md +++ b/site_docs/guides/estimate-cartridge-models.md @@ -19,15 +19,16 @@ records ──estimator──► cfg.reference_models. ``` The output is still a normal `DataConfig`. The cartridge's -identity (`cfg.identity`), allele catalogue (`cfg.alleles`), -rules (`cfg.rules`), and any models you didn't estimate stay +identity (`cfg.metadata`), allele catalogue +(`cfg.v_alleles` / `cfg.d_alleles` / `cfg.j_alleles`), rules +(`cfg.reference_rules`), and any models you didn't estimate stay exactly as they were. Estimators are **typed-plane mutators**: they touch one plane each, leave the rest alone, and stamp an auditable trail on the build report. ## Which estimators ship today -Five estimators are available in v1: +Six estimators ship today. Five populate an empirical plane: | Estimator | Plane it populates | Required columns | Optional columns | |---|---|---|---| @@ -37,6 +38,11 @@ Five estimators are available in v1: | `estimate_np_base_model` | `reference_models.np_bases` | `np1` | `np2` (required on VDJ) | | `estimate_p_nucleotide_lengths` | `reference_models.p_nucleotide_lengths` | `p_v_3_length`, `p_j_5_length` | `p_d_5_length`, `p_d_3_length` (required on VDJ) | +The sixth, `estimate_genotype_priors(genotypes, ...)`, populates the +cartridge's genotype-prior plane from observed `Genotype` objects +(`set_genotype_priors(model)` attaches one you already have); see +[Genotype sampling & population priors](genotype-priors.md). + !!! warning "`estimate_shm_rates` is deferred" SHM-model estimation is not in v1. Calling `builder.estimate_shm_rates(...)` raises `AttributeError`. SHM @@ -211,10 +217,12 @@ report.checksum_at_build_time # str | None - schema_sha256 stamped on cfg report.to_dict() # JSON-clean dict for CI artifacts ``` -`stages` carries an `inputs` and `inferred` block per stage. For -estimator stages you'll find counts of `records_seen`, -`records_used`, `replaced` (idempotency marker), and per-key -diagnostic dicts. +Each stage dict has exactly four keys: `stage`, `inputs`, +`inferred` and `warnings`. For estimator stages `inputs` carries +`record_count`, `source`, `min_count`, `replaced` (the idempotency +marker) and the estimator's own options; `inferred` carries the +computed per-key weights plus diagnostics such as `skipped` and +`below_min_count`. ### The cartridge manifest @@ -223,23 +231,27 @@ manifest = cfg.cartridge_manifest() manifest["models"] # { # "has_reference_models": True, -# "allele_usage": {"v": 320, "d": 30, "j": 6, ...}, -# "trim_models": {"v_trim_3": {"support_size": 18, ...}, ...}, -# "np_length_models": {"np1": {...}, "np2": {...}}, -# "np_base_models": {"first_base": {...}, "transition": {...}}, -# "p_nucleotide_models": {"p_v_3_length": {...}, ...}, +# "allele_usage": {"available": True, "segments": {...}, +# "nonempty_segments": [...], "source": ..., +# "legacy_gene_use_dict_present": ..., +# "legacy_fallback": ..., "in_plan_signature": ...}, +# "trim_models": {"keys": ["D_3", "D_5", "J_5", "V_3"], "source": ..., +# "in_plan_signature": ..., "legacy_...": ...}, +# "np_length_models": {"keys": ["NP1", "NP2"], ...}, +# "np_base_models": {"models": {"NP1": {"kind": "markov"}, +# "NP2": {"kind": "markov"}}, +# "supported_kinds": [...], "deferred_kinds": [...], ...}, +# "p_nucleotide_models": {"length_keys": ["D_3", "D_5", "J_5", "V_3"], +# "supported_ends": [...], ...}, # "shm": {"v_subregion_support": {"available": True, ...}, ...}, -# "trim_keys": ["v_trim_3", "d_trim_5", ...], -# "np_length_keys": ["np1", "np2"], -# "legacy_trim_dicts_present": False, -# "legacy_np_lengths_present": False, +# "genotype_priors": {"available": ..., "model_id": ..., ...}, # } ``` `manifest["models"]` is the canonical place to verify what got -estimated. Each top-level key is `None` when its plane is -empty, so `manifest["models"]["allele_usage"] is None` after a -build that skipped `estimate_allele_usage` is normal. +estimated. No block is ever `None`: a plane you did not estimate +reports an empty `keys` / `models` list, or `"available": False` +on the two blocks that carry that flag. ## Common mistakes @@ -278,8 +290,8 @@ then raises `ValueError` instead of overwriting. **Markov NP-base estimation with `pseudocount=0` on tiny inputs.** If a from-base never appears in the observed sequences, the engine can't form a transition row for it, and `ValueError` -fires before spec construction. Either use the default -`kind="markov", pseudocount=1.0`, or drop to +fires before spec construction. Either raise `pseudocount` above +its default of `0.0` (`kind="markov"` is the default), or drop to `kind="empirical_first_base"` if the from-base coverage isn't representative. diff --git a/site_docs/guides/experiment-builder.md b/site_docs/guides/experiment-builder.md index 0f2ae3bf..824e6707 100644 --- a/site_docs/guides/experiment-builder.md +++ b/site_docs/guides/experiment-builder.md @@ -58,14 +58,14 @@ result = ( .end_loss_5prime(length=(0, 3)) .end_loss_3prime(length=(0, 3)) .random_strand_orientation(prob=0.5) - .paired_end(r1_length=150, insert_size=(250, 450)) + .paired_end(r1_length=150, insert_size=(250, 350)) .run_records(n=100, seed=1, validate_records=True) ) print(len(result)) # 100 -print(result[0]["productive"], result[0]["v_call"]) # True 'IGHVF10-G38*04' +print(result[0]["productive"], result[0]["v_call"]) # True 'IGHVF10-G51*02' print(result[0]["read_layout"]) # 'paired_end' -print(result[0]["r1_sequence"][:30]) # 'gaggtgcagctg...' +print(result[0]["r1_sequence"][:30]) # 'Gctgaggagacggtgaccagggtgccctgg' ``` That's a finished simulation: every record is productive, has @@ -234,8 +234,11 @@ result = exp.run_records(n=1000, seed=42, validate_records=True) Two flags carry most of the reproducibility story: -- **`seed=`** - same seed → byte-identical output across runs and - platforms. `n` consecutive seeded records use seeds +- **`seed=`** - the same seed, run against the same + `GenAIRR.DETERMINISM_VERSION`, gives byte-identical output; a different + determinism version promises nothing at all. The goldens check this on + Linux, macOS and Windows across Python 3.10-3.13. `n` consecutive + seeded records use seeds `[seed, seed+1, …, seed+n-1]`, so batches stitch together if you offset the starting seed **by at least `n`**. Offset by less and they overlap: `n=1000, seed=0` and `n=1000, seed=1` share 999 records. See diff --git a/site_docs/guides/export.md b/site_docs/guides/export.md index 4a9fcb73..c0b321ed 100644 --- a/site_docs/guides/export.md +++ b/site_docs/guides/export.md @@ -1,11 +1,5 @@ -# Export to AIRR TSV, Parquet, or FASTA +# Export to AIRR TSV, FASTA, or FASTQ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Export the results](https://mutejester.github.io/GenAIRR/getting-started/export-results.html). diff --git a/site_docs/guides/genotype-priors.md b/site_docs/guides/genotype-priors.md index 2efe9785..aa6bf0b2 100644 --- a/site_docs/guides/genotype-priors.md +++ b/site_docs/guides/genotype-priors.md @@ -57,7 +57,11 @@ excludes an allele), and unspecified genes fall back to uniform. population source (e.g. VDJbase) for realistic per-gene frequencies; the default is uniform within each gene. `allele_frequencies="usage_as_prior"` is an explicit opt-in that reuses the cartridge's recombination `allele_usage` as a - frequency proxy - convenient but biologically approximate. + frequency proxy - convenient but biologically approximate, and it requires a + cartridge that actually carries a typed `reference_models.allele_usage` + plane. The bundled cartridges (`HUMAN_IGH_OGRDB` included) do not, so on + those it raises `ValueError`; estimate the plane first with + `ReferenceCartridgeBuilder.estimate_allele_usage(...)`. ### Loading population frequencies @@ -126,8 +130,11 @@ Given a list of observed `Genotype` objects (e.g. one per donor), estimate a prior directly: ```python +# Each genotype needs a subject_id: the default +# subject_id_policy="require_unique" refuses a list it cannot check for +# duplicate subjects. Pass subject_id_policy="allow_duplicates" to opt out. model = PopulationGenotypeModel.from_genotypes( - [g_donor1, g_donor2, g_donor3], + [g_donor1.with_subject("D1"), g_donor2.with_subject("D2"), g_donor3.with_subject("D3")], cfg=cfg, model_id="cohort-est", source="my-cohort", ) # or, attaching to a builder in one chained step: diff --git a/site_docs/guides/genotype.md b/site_docs/guides/genotype.md index fd4df706..7b03850a 100644 --- a/site_docs/guides/genotype.md +++ b/site_docs/guides/genotype.md @@ -9,9 +9,9 @@ presence/absence, zygosity, and gene deletion. With no genotype attached the engine is byte-for-byte unchanged. This page explains exactly what a genotype is, how the engine samples from it, and how to build one - nothing here is a black box. Three companion pages cover the rest: -sampling & population priors, -cohorts (many subjects at once), and -benchmarking genotype inference.

+sampling & population priors, +cohorts (many subjects at once), and +benchmarking genotype inference.

## What a genotype is (and why it matters) @@ -143,8 +143,10 @@ ground-truth benchmark. `Genotype.permissive(cfg)` is a separate, explicitly **non-diploid** fallback: unspecified genes are left to sample over *all* their reference alleles without phasing. It exists for the "I only want to constrain a few genes" case - it is -**not** a biological genotype, and it is labelled as such in `to_table()` and -`repr`. +**not** a biological genotype. Each row of `to_table()` carries +`permissive: True` to say so - note that an untouched permissive genotype has +no rows yet, so the label appears only once you specify a gene, and `repr()` +carries no label at all. In both modes, feasibility (e.g. `productive_only`) is applied the same way the non-genotype path applies it: candidates are filtered to the feasible set, and the @@ -486,7 +488,11 @@ you can judge whether it fits your study): ## Backward compatibility -The genotype machinery is purely additive. An experiment with **no** genotype -attached produces byte-identical output to previous releases (pinned by a -checksum test). Attaching a genotype is the only thing that switches recombination -onto the phased path. +The genotype machinery is purely additive **within one determinism version**: +an experiment with no genotype attached draws exactly what it drew before +genotypes existed, at the same `determinism_version`, and attaching a genotype is +the only thing that switches recombination onto the phased path. It is *not* +byte-identical to releases 2.3.1 and earlier - those predate +`GenAIRR.DETERMINISM_VERSION`, which is now `4`, and each increment moved what a +given seed produces for every pipeline. See +[Trace, replay, reproducibility](trace-replay.md). diff --git a/site_docs/guides/index.md b/site_docs/guides/index.md index 27c2a250..ca38d43d 100644 --- a/site_docs/guides/index.md +++ b/site_docs/guides/index.md @@ -10,6 +10,14 @@ specific biological mechanisms or pipeline patterns.

control panel. Every pipeline goes through this builder; this is the one guide every user should read. +## Constraints + +- **[Sample only productive sequences](productive.md)** - what + `productive_only()` guarantees, the named policy, and strict + vs permissive failure. +- **[Biology map](biology-map.md)** - every biological mechanism, + the DSL call that turns it on, and the fields it writes. + ## Biology mechanisms - **[Recombination + junction biology](recombination-junction.md)** @@ -29,6 +37,22 @@ specific biological mechanisms or pipeline patterns.

ground-truth exports. - **[Clonal repertoires](clonal-repertoire.md)** - TCR and flat-BCR abundance models with heavy-tailed clone sizes and `duplicate_count`. +- **[Retained coding end calibration](coding-end-calibration.md)** + signed net coding-end outcomes, fitting and held-out scoring. +- **[Experimental duplex repair](experimental-duplex-repair.md)** + authored two-strand coding-end opening, resection and ligation. + +## Per-individual genotypes + +- **[Genotypes](genotype.md)** - one subject's diploid germline + complement, haplotype-phased recombination, novel alleles, and + the genotype truth sidecar. +- **[Genotype sampling & population priors](genotype-priors.md)** + draw genotypes from a per-gene population model. +- **[Genotype cohorts](genotype-cohorts.md)** - many subjects in + one run, namespaced records, one truth table. +- **[Benchmarking genotype inference](genotype-benchmarking.md)** + scoring a genotype caller against planted truth. ## Library + sequencing @@ -41,7 +65,14 @@ specific biological mechanisms or pipeline patterns.

## Reproducibility - **[Trace, replay, reproducibility](trace-replay.md)** - what - a trace contains, how to replay it, every failure mode. + a trace contains, how to replay it, every failure mode, and + what `determinism_version` promises. + +## Portable references + +- **[Portable local references](portable-references.md)** - build + a reference from your own V/D/J FASTA plus an annotation + sidecar, and carry it between machines. ## Cartridge authoring (separate section) diff --git a/site_docs/guides/junction-additions.md b/site_docs/guides/junction-additions.md index ba6d5433..b5499641 100644 --- a/site_docs/guides/junction-additions.md +++ b/site_docs/guides/junction-additions.md @@ -59,8 +59,10 @@ A few rules to keep in mind when you reason about the geometry: ## N length models `ReferenceEmpiricalModels.np_lengths` controls how many bases each -NP region holds. Keys are `NP1` (always) and `NP2` (VDJ only - VJ -cartridges reject `NP2` keys at attach time): +NP region holds. Keys are `NP1` (always) and `NP2` (VDJ only). Note that an `NP2` +key on a VJ cartridge is **accepted** and then has nothing to apply +to - only the D-side keys of `trims` and `p_nucleotide_lengths` are +chain-gated - so check the chain yourself before authoring one: ```python from GenAIRR import EmpiricalDistributionSpec, ReferenceEmpiricalModels @@ -329,10 +331,12 @@ distribution and emit the ≥95%-zero warning. Use this estimator only against data that genuinely carries P-length annotations. **Expecting `NP2` on a VJ cartridge.** VJ chains have only one NP -region. Attempting to author `np_lengths["NP2"]` or -`p_nucleotide_lengths["D_5"]` / `["D_3"]` on a VJ cartridge -raises `ValueError` at validate time; the engine never gets the -chance to silently lose your spec. +region. `p_nucleotide_lengths["D_5"]` / `["D_3"]` and the D-side +`trims` keys do raise `ValueError` at validate time on a VJ +cartridge. `np_lengths["NP2"]` / `np_bases["NP2"]` do **not** - the +key is accepted and silently has no NP2 to apply to, so a spec +authored there is lost without a warning. Check the chain type +before authoring NP2. ## Where to go next diff --git a/site_docs/guides/paired-end-fastq.md b/site_docs/guides/paired-end-fastq.md index c8bfd4cc..c93614c9 100644 --- a/site_docs/guides/paired-end-fastq.md +++ b/site_docs/guides/paired-end-fastq.md @@ -82,9 +82,9 @@ default: | Field | Default (no paired-end) | |---|---| -| `read_layout` | `'single'` | +| `read_layout` | `''` (empty string) | | `r1_sequence`, `r2_sequence` | `''` (empty string) | -| `r1_start`, `r1_end`, `r2_start`, `r2_end` | `0` | +| `r1_start`, `r1_end`, `r2_start`, `r2_end` | `None` | | `insert_size` | `0` | Code that may receive either shape can safely check @@ -162,9 +162,15 @@ few interactions are worth knowing: - **End-loss happens before read layout.** When `.end_loss_5prime(...)` and `.end_loss_3prime(...)` are in the pipeline, they shorten the molecule first; `paired_end` then - slices R1/R2 windows over what's left. A 300-bp insert request - against a 290-bp post-end-loss molecule clips to what's - available - no error, just a shorter insert. + slices R1/R2 windows over what's left. **The insert is not clipped + to fit.** A requested insert longer than the surviving molecule + leaves the record with `read_layout='paired_end'` but + `insert_size=0`, all four window coordinates `None` and both read + sequences empty: `validate_records` then reports + `ReadWindowOutOfBounds`, and `to_paired_fastq` raises on the empty + `r1_sequence`. Size `insert_size` against the shortest molecule the + pipeline can produce - on human IGH that is about 350 nt before end + loss. - **Random strand orientation flips before R1/R2 windows.** When `.random_strand_orientation(prob=0.5)` is in the pipeline and the record draws a strand flip, the molecule's sequence is @@ -182,7 +188,8 @@ few interactions are worth knowing: `paired_end` is a descendant-phase pass - R1/R2 windows are per-read, so each emitted clone member gets its own independent layout -draw. It is fully supported after legacy `expand_clones(...)`, and +draw. It is fully supported after legacy `expand_clones(...)` - which is +deprecated and raises a `DeprecationWarning` - and accepted after `clonal_repertoire(...)` with one important caveat: `clonal_repertoire` collapses identical assembled sequences into one record with `duplicate_count`, and FASTQ export does not expand that @@ -236,8 +243,9 @@ validity, run `validate_records` on the `SimulationResult` before exporting: ```python +refdata = exp.compile().refdata # a RefDataConfig, not a DataConfig report = result.validate_records(refdata) -assert report, report.summary() +assert report, report.summary() # summary() is a dict of issue counts result.to_paired_fastq("R1.fastq", "R2.fastq") ``` @@ -252,7 +260,9 @@ A handful of issues that show up repeatedly with paired-end. are per-read, so `paired_end` is descendant-phase. Move `.paired_end(...)` after `clonal_repertoire(...)` or legacy `expand_clones(...)`. For exact per-copy paired FASTQ depth today, -use legacy `expand_clones`; `clonal_repertoire` is abundance-collapsed +use legacy `expand_clones` - it is deprecated and raises a +`DeprecationWarning`, but it is still the only surface that gives an +exact per-copy count; `clonal_repertoire` is abundance-collapsed and exposes copy number through `duplicate_count`. `clonal_lineage(...)` currently rejects `.paired_end(...)` even after the fork; paired-end lineage output is a future addition. @@ -269,13 +279,15 @@ is already reverse-complemented at AIRR-projection time. Reading it and applying another `reverse_complement()` would flip it back to the forward orientation and the FASTQ would no longer match what a sequencer actually emits. Trust the field as -written; the validator's `PairedEndWindowMismatch { side: R2 }` -check enforces it. +written; the validator's `ReadSequenceMismatch` check enforces it +(the other paired-end issue kinds are +`PairedEndFieldWithoutLayout`, `ReadWindowOutOfBounds`, +`ReadInsertSizeMismatch` and `ReadLayoutMismatch`). **Expecting FASTQ export without `.paired_end()` in the -pipeline.** `to_paired_fastq` raises on records whose -`read_layout == "single"` (or whose `r1_sequence` / `r2_sequence` -are empty). For single-end FASTQ output, use `to_fastq(...)` +pipeline.** `to_paired_fastq` raises on records whose `read_layout` is not +`"paired_end"` (it is `''` without the pass), and on records whose +`r1_sequence` / `r2_sequence` are empty. For single-end FASTQ output, use `to_fastq(...)` instead. ## Where to go next diff --git a/site_docs/guides/productive.md b/site_docs/guides/productive.md index 053873ce..882d0f1e 100644 --- a/site_docs/guides/productive.md +++ b/site_docs/guides/productive.md @@ -72,11 +72,15 @@ Leave it off when you specifically want the unconstrained mix (including non-productive rearrangements) - e.g. to study out-of-frame rates or to stress-test a tool on non-productive input. -!!! warning "Immunoglobulin (BCR) only" - The productive contract's anchor checks assume BCR semantics, so it is an - immunoglobulin feature. TCR refdata accepts the call but raises a - `ValueError` at compile time - don't add `productive_only()` to a TCR - pipeline. +!!! note "TCR loci too" + `productive_only()` works on TCR cartridges as well as immunoglobulin + ones: the anchor classes it requires (V Cys, J Phe/Trp) are the same + landmarks a TCR cartridge declares. The `ValueError` a TCR pipeline is + likely to hit first is unrelated to productivity - it is the + curatable-reference gate, because most bundled TCR catalogues carry + unresolved anchor findings. Pass `allow_curatable_refdata()` (or + `run_records(..., allow_curatable_refdata=True)`) and the productive + constraint applies normally. The constraint is implemented as a bundle of engine contracts (anchor preservation, junction frame, no-junction-stop); the contributor-facing details diff --git a/site_docs/guides/recombination-editing.md b/site_docs/guides/recombination-editing.md index f2c0ce48..bd3d01c6 100644 --- a/site_docs/guides/recombination-editing.md +++ b/site_docs/guides/recombination-editing.md @@ -329,9 +329,11 @@ the replacement allele happened to land on the same V). **Treating `d_cigar` coordinates as reversed.** Under `d_inverted=True`, the CIGAR ops still use the forward allele's coordinate space - `d_germline_start/end` are forward-allele -coordinates. The match-rate `d_identity` drops because the -inverted pool bytes don't match the forward allele bytes, but -the coordinate system stays stable. If your downstream pipeline +coordinates, and the coordinate system stays stable. Inversion on +its own does **not** lower `d_identity`: the engine aligns the +inverted reference in its assembled orientation, so an unmutated +inverted D reports `d_identity == 1.0`, exactly as a forward one +does. If your downstream pipeline re-RCs the bytes before comparing, do it on `rec["sequence"][d_sequence_start:d_sequence_end]` - don't remap the coordinates. diff --git a/site_docs/guides/recombination-junction.md b/site_docs/guides/recombination-junction.md index 13f6ba98..f0d1a606 100644 --- a/site_docs/guides/recombination-junction.md +++ b/site_docs/guides/recombination-junction.md @@ -173,9 +173,11 @@ the fields without branching first; the values are just zero or empty string when D and NP2 don't exist. Authoring a VJ cartridge with D-end keys on `trims` or -`p_nucleotide_lengths`, or with `NP2` on `np_lengths`, raises -`ValueError` at the cartridge attach time - the validator catches -the impossible shape before the engine gets the chance. +`p_nucleotide_lengths` raises `ValueError` at cartridge attach time - +the validator catches the impossible shape before the engine gets the +chance. `NP2` on `np_lengths` / `np_bases` is **not** gated that way: +the key is accepted and then has no NP2 region to apply to, so a VJ +cartridge authored with one silently loses it. ## Allele calls and ambiguity diff --git a/site_docs/guides/shm-targeting.md b/site_docs/guides/shm-targeting.md index d850dd85..8659e72e 100644 --- a/site_docs/guides/shm-targeting.md +++ b/site_docs/guides/shm-targeting.md @@ -98,10 +98,14 @@ events per record. Use whichever shape matches your intent. ### `model="s5f"` -The canonical Shapiro 5-mer model. Each site's mutability is -weighted by its 5-base context, so hot spots concentrate in the -CDRs (where SHM actually accumulates in vivo) rather than scattering -uniformly across the V region. +The S5F 5-mer model of Yaari et al. (2013). Each site's mutability is +weighted by its 5-base context, so the V region's mutation density is +uneven rather than flat. It is **context** weighting and nothing else - +the kernel has no FWR/CDR awareness. Measured over 2,000 human IGH +records at `count=(10, 20)`, S5F puts 22.9% of V mutations in CDR1+CDR2 +against 16.5% for the uniform model, while FWR3 alone still carries 42% +under either. If you want mutations concentrated in the CDRs, ask for it +with `v_subregion_rates` (below). ```python exp.mutate(model="s5f", rate=0.03) @@ -116,7 +120,10 @@ cartridge through the manifest: manifest = cfg.cartridge_manifest() manifest["models"]["shm"] # full SHM model inventory manifest["models"]["shm"]["v_subregion_support"] -# {"available": True, "annotated_v_count": 320, "total_v_count": 320} +# {"available": True, "labels": ["FWR1", "CDR1", "FWR2", "CDR2", "FWR3"], +# "annotated_v_count": 198, "total_v_count": 198, +# "derivation": "bridge_imgt_gapped_seq", "in_content_hash": True} +# (the numbers are HUMAN_IGH_OGRDB's; every cartridge reports its own) ``` The `v_subregion_support.available` flag tells you whether the @@ -350,6 +357,9 @@ Two diagnostics are most informative: one value in [0, 1] and the shape is sensitive to whether the model concentrates mutations in motifs or scatters them. +The two plotting blocks below need `matplotlib` and `seaborn`, neither +of which GenAIRR depends on; `pip install matplotlib seaborn` first. + ```python import matplotlib.pyplot as plt import seaborn as sns @@ -400,11 +410,11 @@ They never do. `n_mutations` is biology only. Use library-and-sequencing artefacts; use `n_mutations` only for SHM. **Setting every rate to zero.** `segment_rates={"V": 0, "D": 0, -"J": 0, "NP": 0}` doesn't produce zero mutations with a "no -sites available" error - it makes the proposal support empty, so -the pass draws nothing. Every record will have `n_mutations: 0`. -If you wanted that, fine; if you didn't, check which buckets you -zeroed. +"J": 0, "NP": 0}` is refused at the `mutate()` call with +`ValueError: segment_rates: at least one bucket must have a positive +rate` - a configuration that zeroes every biological segment would make +the SHM pass a deterministic no-op, which is better said than silently +done. Drop the pass instead, or leave one bucket positive. **Using `v_subregion_rates` on an unannotated cartridge.** Raises `ValueError` at builder time. The bundled OGRDB cartridges diff --git a/site_docs/guides/streaming.md b/site_docs/guides/streaming.md index 957dc9a4..d851b8ff 100644 --- a/site_docs/guides/streaming.md +++ b/site_docs/guides/streaming.md @@ -1,11 +1,5 @@ # Stream millions of sequences without OOM -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [The Experiment builder](https://mutejester.github.io/GenAIRR/guides/experiment-builder.html). diff --git a/site_docs/guides/trace-replay.md b/site_docs/guides/trace-replay.md index 7dc73b49..1dea0928 100644 --- a/site_docs/guides/trace-replay.md +++ b/site_docs/guides/trace-replay.md @@ -33,6 +33,12 @@ existed. A trace file carries the same number, and replaying one whose number disagrees with the installed engine fails rather than quietly reproducing something else. +**Where it is checked.** The reproducibility goldens run in CI on Linux, +macOS and Windows across Python 3.10-3.13 and assert that the same seed +reproduces the same bytes on each. Linux aarch64 ships a wheel but is not +covered by those goldens, so on that architecture the promise is the +engine's intent rather than something measured on every commit. + **One stream per pass.** Each pass draws from its own random substream, derived from the run seed and the pass's identity rather than taken from a stream the pipeline shares. Two consequences worth @@ -188,8 +194,9 @@ strongest reproducibility check you can author with one trace. ## What the trace contains -Version 4 records the annotation policy and active contract context alongside -the existing signatures: +Fresh traces are written at schema version **5**, which adds +`determinism_version` to the annotation policy and active-contract context +schema 4 introduced: | Field | Purpose | |---|---| @@ -217,9 +224,11 @@ its payload. You ship the trace + the cartridge name; the engine re-derives everything else from the recorded choices. Both seed reruns and choice replay check reference content, address vocabulary, -annotation policy and active contracts. New schema-4 files require explicit -policy/context fields. Earlier schemas remain readable with their documented -historical limitations. +annotation policy, active contracts and determinism version. Files written at +schema 4 or later require explicit policy/context fields; schema 5 adds the +determinism stamp, and a file written before it reports `0`, meaning +"unknown". Earlier schemas remain readable with their documented historical +limitations. Constrained indels now use `natural_sequential_v1`, whose probabilities follow the actual sequence after each event. Changed sampler signatures intentionally @@ -232,7 +241,11 @@ historical productivity annotations. ## What can make replay fail Replay raises `ValueError` (never `KeyError`, never a custom -exception) with one of these distinct shapes: +exception) with one of these distinct shapes. The names below are the +gates' names, not tokens inside the message: the `ValueError` text is +English prose that describes the mismatch (and, for the signature gates, +prints the expected and actual signatures). Match on the exception type +and read the message; do not grep for the snake_case name. ### Validation phase - checked before any choices are consumed @@ -375,10 +388,12 @@ loud on purpose. Use **rerun** if you want to see what would change; use **replay** when you want to verify nothing did. **Using a different cartridge.** Replaying a `HUMAN_IGH_OGRDB` -trace against `HUMAN_IGH_EXTENDED` fires -`refdata_signature_mismatch`. The two cartridges have different -catalogues, different allele names at the same index - replay -would silently corrupt. The gate catches it. +trace against `HUMAN_IGH_EXTENDED` is refused. In practice the pass-plan +signature gate fires first, because a cartridge's trim and NP-length +distributions are baked into the resolved pass plan, so the error names the +plan rather than the refdata. Either way the two cartridges have different +catalogues and different allele names at the same index - replay would +silently corrupt, and the gate catches it. **Confusing replay with rerun.** Replay consumes recorded values; rerun re-draws. Calling `replay_from_trace_file` and expecting diff --git a/site_docs/guides/tune-corruption.md b/site_docs/guides/tune-corruption.md index 215a7e7a..056ea9a2 100644 --- a/site_docs/guides/tune-corruption.md +++ b/site_docs/guides/tune-corruption.md @@ -1,11 +1,5 @@ # Tune corruption rates to match your sequencer -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Corruption + sequencing artefacts](https://mutejester.github.io/GenAIRR/guides/corruption-sequencing.html). diff --git a/site_docs/guides/v-usage.md b/site_docs/guides/v-usage.md index 2907f728..abff3d10 100644 --- a/site_docs/guides/v-usage.md +++ b/site_docs/guides/v-usage.md @@ -1,11 +1,5 @@ # Match an empirical V-gene usage distribution -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Estimate models from data](https://mutejester.github.io/GenAIRR/guides/estimate-cartridge-models.html). diff --git a/site_docs/guides/with-metadata.md b/site_docs/guides/with-metadata.md index b1857841..e361d3b4 100644 --- a/site_docs/guides/with-metadata.md +++ b/site_docs/guides/with-metadata.md @@ -1,11 +1,5 @@ # Attach sample metadata to records -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [The Experiment builder](https://mutejester.github.io/GenAIRR/guides/experiment-builder.html). diff --git a/site_docs/learn.md b/site_docs/learn.md index fb3ae8f2..732ba8c9 100644 --- a/site_docs/learn.md +++ b/site_docs/learn.md @@ -11,7 +11,7 @@ The fastest path from `pip install` to a productive heavy-chain batch. 1. **[Quick start](getting-started/quick-start.md)** - 5 minutes, - 100 productive heavy chains, AIRR records returned. + 1,000 productive heavy chains, AIRR records returned. 2. **[Your first AIRR record](getting-started/first-airr-record.md)** field-by-field tour of what the engine emits. 3. **[The Experiment builder](guides/experiment-builder.md)** - diff --git a/site_docs/reference/experiment.md b/site_docs/reference/experiment.md index 437f8b2c..96397f95 100644 --- a/site_docs/reference/experiment.md +++ b/site_docs/reference/experiment.md @@ -22,9 +22,11 @@ The methods you'll reach for in most real pipelines: | `.mutate(...)` | Apply biological SHM (uniform or S5F) on top of recombination | | `.clonal_lineage(...)` | Grow BCR affinity-maturation lineage trees | | `.clonal_repertoire(...)` | Generate TCR / flat-BCR abundance repertoires with clone sizes and `duplicate_count` | -| `.expand_clones(...)` | Legacy fixed-size clonal star model | +| `.expand_clones(...)` | Legacy fixed-size clonal star model; deprecated and raises a `DeprecationWarning` | +| `.with_genotype(genotype)` | Draw V/D/J from one subject's diploid complement, haplotype-phased | | `.paired_end(...)` | Project assembled sequences as paired R1 / R2 reads | | `.run_records(...)` | Compile + run + return a `SimulationResult` | +| `.run_cohort(genotypes, ...)` | Run many genotyped subjects in one call; returns a `CohortResult` | For the full pipeline-stage map (constraints, trim overrides, all the corruption methods, compile reuse, strict mode), see @@ -50,8 +52,10 @@ pilot for the wider generated-reference effort. - clonal_lineage - clonal_repertoire - expand_clones + - with_genotype - paired_end - run_records + - run_cohort - run - compile - describe diff --git a/site_docs/reference/index.md b/site_docs/reference/index.md index ce5d4294..32d69513 100644 --- a/site_docs/reference/index.md +++ b/site_docs/reference/index.md @@ -27,7 +27,7 @@ below shows what those auto-rendered pages look like. ## Top-level imports -35 names ship in `GenAIRR.__all__`. Grouped by what they do: +40 names ship in `GenAIRR.__all__`. Grouped by what they do: ### Simulation entry points @@ -38,6 +38,7 @@ below shows what those auto-rendered pages look like. | `SimulationResult` | Records + outcomes + parents wrapper | [Quick start](../getting-started/quick-start.md) | | `set_seed`, `get_seed`, `reset_seed` | Thread-local seed management | [Validation hub](../validation/index.md#trace-and-replay) | | `__version__` | Resolved via `importlib.metadata`; falls back to `"0.0.0"` for editable installs | - | +| `DETERMINISM_VERSION` | Which sampling model this engine implements (`4`). Stamped on every record as `determinism_version` | [Trace, replay, reproducibility](../guides/trace-replay.md) | ### Reference data - cartridges and bridges @@ -46,11 +47,14 @@ below shows what those auto-rendered pages look like. | `DataConfig` | The Python-side cartridge dataclass | [Reference cartridge](../concepts/reference-cartridge.md) | | `RefDataConfig` | The engine-side refdata bridge object | [Reference cartridge](../concepts/reference-cartridge.md) | | `dataconfig_to_refdata` | Bridge function - `DataConfig → RefDataConfig` | [`validate_records`](../validation/validate-records.md) | +| `Reference` | Portable local germline reference built from V/D/J FASTA plus a versioned annotation sidecar | [Portable local references](../guides/portable-references.md) | +| `ReferenceArtifactError` | Raised when a reference artifact cannot be read or written | [Portable local references](../guides/portable-references.md) | +| `ReferenceCapabilityError` | Raised when a run needs evidence the reference does not carry | [Portable local references](../guides/portable-references.md) | | `ConfigInfo` | Identity metadata | [Reference cartridge](../concepts/reference-cartridge.md) | | `ChainType` | `BCR_HEAVY` / `BCR_LIGHT_KAPPA` / ... | - | | `Species` | `HUMAN` / `MOUSE` / ... | - | | `Productivity` | Productive / NonProductive enumeration | - | -| `list_configs` | Enumerate the 100+ bundled cartridges | - | +| `list_configs` | Enumerate the 106 bundled cartridges | - | | `DataConfigError` | DataConfig validation failure | - | ### Bundled cartridges (lazy-loaded) @@ -70,9 +74,12 @@ cfg = ga.HUMAN_IGH_OGRDB # pickled cartridge - loaded on first access | `HUMAN_TCRB_IMGT` | Human TRB, IMGT | The five top-level bundled cartridges above are the most commonly -used. The full 100+-cartridge catalogue stays reachable via +used. The full 106-cartridge catalogue stays reachable via `ga.list_configs()` and `GenAIRR.data.` (lazy attribute -access). +access). 80 of the 106 carry unresolved anchor or pseudogene/ORF +findings and are quarantined under strict compilation - see +[Reference evidence and quarantine](../cartridges/reference-evidence.md) +for the list and for `allow_curatable_refdata()`. ### Reference cartridge authoring @@ -86,6 +93,7 @@ access). | `EmpiricalDistributionSpec` | `[(value, weight), ...]` shape | [Reference cartridge](../concepts/reference-cartridge.md) | | `NpBaseModelSpec` | NP-base sampling model (uniform / empirical / Markov) | [Junction N/P additions](../guides/junction-additions.md) | | `AlleleUsageSpec` | Per-segment allele-usage weights | [Reference cartridge](../concepts/reference-cartridge.md) | +| `CodingEndOutcomeModel` | Signed net coding-end outcome model, with fitting and held-out scoring | [Retained coding end calibration](../guides/coding-end-calibration.md) | ### Validation reports and exceptions @@ -111,12 +119,14 @@ is called. | **Recombination** | `.recombine()`, `.invert_d(prob=...)`, `.receptor_revision(prob=...)` | Ancestor-phase mechanisms | | **Constraints** | `.productive_only()`, `.restrict_alleles(v=..., d=..., j=...)` | Constrain the sample space | | **Biological mutation** | `.mutate(model="s5f"\|"uniform", rate=..., count=..., segment_rates=..., v_subregion_rates=...)` | The only pass that increments `n_mutations` | -| **Clonal structure** | `.clonal_lineage(...)`, `.clonal_repertoire(...)`, legacy `.expand_clones(...)` | BCR trees, TCR / flat-BCR abundance repertoires, or fixed-size star families | +| **Clonal structure** | `.clonal_lineage(...)`, `.clonal_repertoire(...)`, legacy `.expand_clones(...)` | BCR trees, TCR / flat-BCR abundance repertoires, or fixed-size star families. `expand_clones` is deprecated and raises a `DeprecationWarning` | +| **Genotype** | `.with_genotype(genotype)` | Haplotype-phased recombination from one subject's diploid complement; see [Genotypes](../guides/genotype.md) | | **Trims override** | `.trim(v_3=..., d_5=..., d_3=..., j_5=..., enabled=...)` | Per-experiment trim distribution overrides | | **Library / sequencer artefacts** | `.pcr_amplify(...)`, `.polymerase_indels(...)`, `.ambiguous_base_calls(...)`, `.sequencing_errors(...)`, `.end_loss_5prime(...)`, `.end_loss_3prime(...)` | All descendant-phase | | **Read layout** | `.paired_end(r1_length=..., r2_length=..., insert_size=...)`, `.random_strand_orientation(prob=...)` | Per-read projection | | **Bookkeeping** | `.with_metadata(...)`, `.contaminate(prob=...)` | Stamp metadata; inject background contaminants | | **Run** | `.run_records(n=..., seed=..., validate_records=..., strict=..., expose_provenance=...)`, `.run(...)`, `.stream(...)`, `.stream_records(...)` | Compile + run + project | +| **Run a cohort** | `.run_cohort(genotypes, *, n_per_subject=1, seed=0, counts=None, strict=False, expose_provenance=False)` | Many subjects in one call; returns a `CohortResult`. See [Genotype cohorts](../guides/genotype-cohorts.md) | | **Compile reuse** | `.compile()` | Compile once, reuse across many batches | | **Compile mode** | `.allow_curatable_refdata()`, `.curate_refdata(policy)` | Cartridge-validation mode at compile time | @@ -133,7 +143,7 @@ record dicts plus the underlying `Outcome` objects. | **List-like access** | `len(result)`, `result[i]`, `result[a:b]`, `for rec in result:` | One AIRR dict per element | | **Underlying state** | `.records`, `.outcomes`, `.parents`, `.lineage_trees` | `outcomes` is `None` when built from records only; `parents` exists only for legacy `expand_clones`; `lineage_trees` exists on lineage results | | **Validation** | `.validate_records(refdata)`, `.validate_families()`, `.validate_families_with_parents(refdata)` | See [Validation hub](../validation/index.md) | -| **Export** | `.to_tsv(path, *, airr_strict=True)`, `.to_csv(path, *, airr_strict=False)`, `.to_fasta(path, *, prefix="seq")`, `.to_fastq(path, *, quality="illumina", **kw)`, `.to_paired_fastq(r1, r2, *, quality="illumina", overwrite=False, **kw)`, `.to_dataframe(*, airr_strict=False)` | TSV defaults to AIRR coordinate/CIGAR conversion and `T` / `F` booleans; see [Export the results](../getting-started/export-results.md) | +| **Export** | `.to_tsv(path, *, airr_strict=True)`, `.to_csv(path, *, airr_strict=False)`, `.to_fasta(path, *, prefix="seq")`, `.to_fastq(path, *, quality="illumina", **kw)`, `.to_paired_fastq(r1, r2, *, quality="illumina", overwrite=False, **kw)`, `.to_dataframe(*, airr_strict=False)`, `.to_genotype_tsv(path)` | TSV defaults to AIRR coordinate/CIGAR conversion and `T` / `F` booleans; `to_genotype_tsv` writes the genotype truth sidecar and raises when the run carried no genotype, or when its genotypes carry no `subject_id`. See [Export the results](../getting-started/export-results.md) | | **Construction** | `SimulationResult.from_outcomes(outcomes, refdata, *, id_prefix="seq", expose_provenance=False)` | Build a result from Rust `Outcome` objects + refdata; `expose_provenance=True` injects `truth_*_call` columns | ## Reports and exceptions @@ -145,7 +155,7 @@ report = result.validate_records(refdata) report.ok # bool - every record passed report.count # int - total records validated report.failures # list[dict] - per-record failure entries with structured issues -report.summary() # str - histogram of issue kinds +report.summary() # dict[str, int] - issue kind -> count ({} when ok) bool(report) # == report.ok ``` @@ -233,10 +243,6 @@ A few surfaces deliberately don't appear on this page: - **Experimental / private helpers.** Anything under `GenAIRR._*` is private; signatures and behaviour can change between releases without notice. -- **Pre-engine documentation.** The historical `_old_docs/` - directory in the repository carries an earlier docs system - that predates the Rust engine; it's preserved for reference - but not migrated into the current site. For the full per-symbol docstring inventory, the **API Reference** nav section carries `mkdocstrings`-driven pages for diff --git a/site_docs/reference/reference-cartridge-builder.md b/site_docs/reference/reference-cartridge-builder.md index 5d2ccfea..b1a96ec3 100644 --- a/site_docs/reference/reference-cartridge-builder.md +++ b/site_docs/reference/reference-cartridge-builder.md @@ -30,6 +30,8 @@ for the estimator deep dive, see | `.estimate_np_length_distributions(records)` | Estimate NP1 / NP2 length distributions | | `.estimate_np_base_model(records, kind=...)` | Estimate NP-base model (Markov or empirical first-base) | | `.estimate_p_nucleotide_lengths(records)` | Estimate palindromic-insertion length distributions | +| `.estimate_genotype_priors(genotypes, ...)` | Estimate a per-gene population genotype prior from observed genotypes | +| `.set_genotype_priors(model)` | Attach a `PopulationGenotypeModel` you already have | | `.build()` | Finalise, stamp checksum, return the `DataConfig` | | `.report()` | Get the `CartridgeBuildReport` with stages / warnings / rejections | @@ -56,5 +58,7 @@ for source requirements, conservative anchor inference and legacy compatibility. - estimate_np_length_distributions - estimate_np_base_model - estimate_p_nucleotide_lengths + - estimate_genotype_priors + - set_genotype_priors - build - report diff --git a/site_docs/reference/simulation-result.md b/site_docs/reference/simulation-result.md index e79cf32b..5fd2568d 100644 --- a/site_docs/reference/simulation-result.md +++ b/site_docs/reference/simulation-result.md @@ -12,7 +12,7 @@ and export helpers below.

## Common methods -The eight methods you'll reach for in real pipelines: +The methods you'll reach for in real pipelines: | Method | Purpose | |---|---| @@ -22,14 +22,16 @@ The eight methods you'll reach for in real pipelines: | `.to_dataframe(*, airr_strict=False)` | Return a pandas DataFrame with canonical AIRR columns | | `.to_tsv(path, *, airr_strict=True)` | Write TSV with AIRR coordinates, CIGAR placement offsets, and `T` / `F` booleans | | `.to_csv(path, *, airr_strict=False)` | Write CSV with Python record conventions by default | -| `.to_genotype_tsv(path)` | Write the run's genotype ground truth as a sidecar, one row per carried allele, joined to the records on `subject_id` + allele. Raises if the run carried no genotype | +| `.to_genotype_tsv(path)` | Write the run's genotype ground truth as a sidecar, one row per carried allele, joined to the records on `subject_id` + allele. Raises if the run carried no genotype, if its genotypes carry no `subject_id`, or if a permissive genotype leaves reference genes unspecified | | `.to_fasta(path, *, prefix="seq")` | Write assembled sequences as FASTA | | `.to_fastq(...)` / `.to_paired_fastq(...)` | Write FASTQ; paired-end requires `read_layout="paired_end"` | +| `.to_genotype_tsv(path)` on a `CohortResult` | The same sidecar for every subject of a cohort run | `clonal_repertoire` returns ordinary `SimulationResult` records with `clone_id` and `duplicate_count`. `clonal_lineage` returns `SimulationResultWithLineages`, a subclass that adds `.lineage_trees`. -Legacy `expand_clones` returns `SimulationResult` with `.parents`. +Legacy `expand_clones` returns `SimulationResult` with `.parents`; the +call itself is deprecated and raises a `DeprecationWarning`. Record dicts, default CSV, and default DataFrames use 0-based, half-open biological coordinates and local per-segment CIGARs. diff --git a/site_docs/stylesheets/extra.css b/site_docs/stylesheets/extra.css index 21aea647..95296979 100644 --- a/site_docs/stylesheets/extra.css +++ b/site_docs/stylesheets/extra.css @@ -5,13 +5,10 @@ * "bench-science palette v2" design language from the existing * hand-rolled site at website/styles.css. * - * Sources: - * - website/styles.css — 3132-line canonical design system - * extracted by the Phase 1.5 inventory workflow. - * - docs/docs_mkdocs_migration_plan.md §5.2 — Phase 1.5 - * contract. + * Source: + * - website/styles.css — the 3132-line canonical design system. * - * Goals (per docs_mkdocs_migration_plan.md §5.2.2): + * Goals: * - Preserve IBM Plex triad (Sans / Mono / Serif). * - Three-color semantic system: GREEN accent, COBALT info, * PLUM archive — each with -ink (deep) + -soft (tint). @@ -890,9 +887,9 @@ code, kbd, pre, samp, color: var(--ink); } -/* Migration-status banner: when a page carries the "Migration placeholder" - admonition, give it a deliberate visual to signal "this isn't done yet" - so users don't mistake it for finished content. */ +/* Unwritten-page banner: when a page carries the "has not been written + yet" admonition, give it a deliberate visual so readers don't mistake + it for finished content. */ .md-typeset .admonition.info:has(> .admonition-title:first-child) { /* Default styling is fine — the eyebrow label tells the story. */ } diff --git a/site_docs/validation/audit-realism.md b/site_docs/validation/audit-realism.md index 7bd2b906..3446cbaa 100644 --- a/site_docs/validation/audit-realism.md +++ b/site_docs/validation/audit-realism.md @@ -1,11 +1,5 @@ # Audit a simulation for biological realism -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Validation & Debugging](https://mutejester.github.io/GenAIRR/validation/index.html). diff --git a/site_docs/validation/benchmark-aligner.md b/site_docs/validation/benchmark-aligner.md index 29ab60e6..b7528cbe 100644 --- a/site_docs/validation/benchmark-aligner.md +++ b/site_docs/validation/benchmark-aligner.md @@ -1,11 +1,5 @@ # Benchmark an aligner against truth -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Validation & Debugging](https://mutejester.github.io/GenAIRR/validation/index.html). diff --git a/site_docs/validation/compare-shm.md b/site_docs/validation/compare-shm.md index 4c3f5f93..d3990451 100644 --- a/site_docs/validation/compare-shm.md +++ b/site_docs/validation/compare-shm.md @@ -1,11 +1,5 @@ # Compare two SHM models -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Targeted SHM rates](https://mutejester.github.io/GenAIRR/guides/shm-targeting.html). diff --git a/site_docs/validation/index.md b/site_docs/validation/index.md index df7a0db6..e88d1dc7 100644 --- a/site_docs/validation/index.md +++ b/site_docs/validation/index.md @@ -1,6 +1,6 @@ # Validation & reproducibility -

GenAIRR's outputs are rich - 50+ AIRR fields per +

GenAIRR's outputs are rich - over 120 fields per record, full event ledgers per outcome, recoverable traces per seed. Validation answers one question across all of it: does what the record reports agree with what the engine actually did? This @@ -115,6 +115,10 @@ hot loops. | Clonal batch is consistently stamped with `clone_id`; truth V/D/J calls are invariant within clone when truth columns are present | `validate_families` | | Legacy `expand_clones` descendants agree with their parent `Outcome` | `validate_families_with_parents` | +`expand_clones(...)` is deprecated and raises a `DeprecationWarning`; +`validate_families` and `validate_families_with_parents` continue to +cover it. + The three layers are independent - `validate_records` doesn't require a clonal structure; `validate_families` doesn't require a refdata. They compose for the strongest possible gate: @@ -137,8 +141,14 @@ assert families, families.summary() GenAIRR's reproducibility model rests on two facts: -- **Same seed + same plan + same cartridge → byte-identical output.** - Across runs, machines, and platforms. The `seed=` argument on +- **Same seed + same plan + same cartridge + same determinism + version → byte-identical output.** `GenAIRR.DETERMINISM_VERSION` + (currently `4`) names the sampling model the installed engine + implements, and every record carries it as a `determinism_version` + column; two runs at different versions promise each other nothing. + The reproducibility goldens check this in CI on Linux, macOS and + Windows across Python 3.10-3.13; Linux aarch64 ships a wheel but is + not covered by them. The `seed=` argument on `run_records(...)` is the canonical surface; `n` records use seeds `[seed, seed+1, …, seed+n-1]`, so batches stitch together if you offset the starting seed **by at least `n`**. Offsetting by @@ -261,7 +271,8 @@ except ga.StrictSamplingError as e: The `reason` field is a stable lowercase code - common values: `"empty_admissible_support"`, `"support_unavailable"`, -`"missing_allele.V.42"`, `"contract_violation.NoStopCodonInJunction"`. +`"missing_allele.v.42"` (the whole code is lowercased), +`"contract_violation.NoStopCodonInJunction"`. The recommended posture: **leave strict off in production**, where the sentinel fallback keeps your batch flowing; **turn strict on diff --git a/site_docs/validation/replay.md b/site_docs/validation/replay.md index d6814a3d..0280a561 100644 --- a/site_docs/validation/replay.md +++ b/site_docs/validation/replay.md @@ -1,11 +1,5 @@ # Replay a simulation with one knob changed -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Trace, replay, reproducibility](https://mutejester.github.io/GenAIRR/guides/trace-replay.html). diff --git a/site_docs/validation/reproduce.md b/site_docs/validation/reproduce.md index b443ae1a..c55aa5a2 100644 --- a/site_docs/validation/reproduce.md +++ b/site_docs/validation/reproduce.md @@ -1,11 +1,5 @@ # Reproduce a published dataset -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Trace, replay, reproducibility](https://mutejester.github.io/GenAIRR/guides/trace-replay.html). diff --git a/site_docs/validation/smoke-test.md b/site_docs/validation/smoke-test.md index f316c739..43f60659 100644 --- a/site_docs/validation/smoke-test.md +++ b/site_docs/validation/smoke-test.md @@ -1,11 +1,5 @@ # Smoke-test a config change -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Installation](https://mutejester.github.io/GenAIRR/getting-started/installation.html). diff --git a/site_docs/validation/trace-introspection.md b/site_docs/validation/trace-introspection.md index abba92e1..a1d16f06 100644 --- a/site_docs/validation/trace-introspection.md +++ b/site_docs/validation/trace-introspection.md @@ -1,11 +1,5 @@ # Introspect the trace -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 3** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Trace, replay, reproducibility](https://mutejester.github.io/GenAIRR/guides/trace-replay.html). diff --git a/site_docs/validation/two-layer-model.md b/site_docs/validation/two-layer-model.md index c0989766..28da66fc 100644 --- a/site_docs/validation/two-layer-model.md +++ b/site_docs/validation/two-layer-model.md @@ -3,12 +3,6 @@ _Rewritten from `docs/validation_matrix.md` §1.0._ -!!! info "Migration placeholder" - This page is a placeholder. Content lands in **Phase 5** - of the migration plan documented in - [`docs/docs_mkdocs_migration_plan.md`](https://github.com/MuteJester/GenAIRR/blob/master/docs/docs_mkdocs_migration_plan.md). - - Until then, see the corresponding live page on the existing - documentation site at - [mutejester.github.io/GenAIRR](https://mutejester.github.io/GenAIRR/) - or the source markdown referenced above (if any). +!!! info "This page has not been written yet" + There is no content here yet. What you are looking for is most + likely on [Validation & Debugging](https://mutejester.github.io/GenAIRR/validation/index.html). diff --git a/site_docs/validation/validate-records.md b/site_docs/validation/validate-records.md index fa137afa..665c3289 100644 --- a/site_docs/validation/validate-records.md +++ b/site_docs/validation/validate-records.md @@ -47,15 +47,16 @@ result = ( report = result.validate_records(refdata) assert report # ValidationReport is truthy iff every record passed -print(report.summary()) # Histogram of issue kinds (empty when ok) +print(report.summary()) # dict[str, int] of issue kinds (empty when ok) ``` `refdata` is the cartridge in its engine-side `RefDataConfig` shape. When you bound the `Experiment` to a string like `"human_igh"` or a `DataConfig`, the bridge produces a `RefDataConfig` for the engine; you can obtain the same object via `ga.dataconfig_to_refdata(cfg)` -on the Python side, or read it off -`exp.compile().simulator.refdata` after a compile. +on the Python side, or read it off `exp.compile().refdata` after a +compile. Passing a `DataConfig` straight in raises `TypeError` - the +validator wants the engine-side shape. ## Runtime validation @@ -87,7 +88,7 @@ report = result.validate_records(refdata) report.ok # True / False - short-circuit gate for CI report.count # int - total records validated report.failures # list[dict] - one entry per failing record -report.summary() # str - single-line histogram of issue kinds +report.summary() # dict[str, int] - issue kind -> count ({} when ok) bool(report) # equivalent to report.ok assert report, report.summary() # the canonical CI one-liner @@ -100,8 +101,8 @@ When `ok` is `False`, each entry of `failures` carries: "record_index": 17, "sequence_id": "seq17", "issues": [ - {"kind": "JunctionLength", "details": {...}}, - {"kind": "VCallMismatch", "details": {...}}, + {"kind": "JunctionLengthMismatch", "details": {...}}, + {"kind": "RecalledAlleleCallMismatch", "details": {...}}, ], } ``` @@ -112,10 +113,15 @@ problem. ## Common issue categories -The validator catalogues divergences into five families. Each family +The validator catalogues divergences into six families, plus two +assessment-level checks (`ProductivityAssessmentMismatch` and +`CodingEndProcessingMismatch`) that sit outside them. Each family groups together the checks that share the same upstream source, so when you see one fire you can localise the suspect code path -quickly. +quickly. The five described below are the ones a user meets; the +sixth covers region and live-call structural invariants +(`MultipleRegionsForSegment`, `MultipleHypothesesInLiveCall`), which +fire only on engine-level corruption. ### Sequence + coordinate consistency @@ -138,7 +144,12 @@ Every counter on the record is re-derived from a canonical source: | `n_v_mutations` / `n_d_mutations` / `n_j_mutations` / `n_np_mutations` | The same SHM events, partitioned by carried segment | | `n_pcr_errors` / `n_quality_errors` / `n_ambiguous_bases` | Each pass's own `BaseChanged` events, counting only writes that changed a base | | `n_indels` / `n_v_indels` / `n_d_indels` / `n_j_indels` | `IndelInserted` + `IndelDeleted` events from the indel pass | -| `end_loss_5_length` / `end_loss_3_length` | Trace addresses on the end-loss pass | +| `end_loss_5_length` / `end_loss_3_length` | Trace addresses on the end-loss pass, summed when more than one pass trims the same end | +| `mutation_rate` | Each mutation stage's realized events over that stage's input molecule length | +| `n_observable_mutations` | The SHM writes still in the molecule and still differing from the germline base recorded there (`NObservableMutationsMismatch`, plus the cross-field `ObservableMutationsExceedCount`) | +| the six `n_*_observable_mutations` | The same, bucketed by IMGT V subregion (`VSubregionObservableExceedsEvents`, `VSubregionObservableSumExceedsTotal`) | +| `np1_n_tdt` / `np1_n_p` / `np1_n_inserted` and the `np2_*` triple | Per-base origin of each NP region, which must partition its length (`NpPartitionMismatch`, `NpPartitionOriginAmbiguous`) | +| `p_v_3_length` / `p_d_5_length` / `p_d_3_length` / `p_j_5_length` | The templated P bytes emitted at each junction side (`PLengthMismatch`) | A `MutationCountSumMismatch` issue means the per-segment counters don't add up to `n_mutations` - almost always a sign that a From 31caa3caae2a5ccfa456773365c21c51d9e542e6 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 5 Nov 2026 21:51:09 +0200 Subject: [PATCH 5/7] docs: a changelog and a migration guide for 2.4.0 The changelog leads with the one fact that matters to anyone holding data: a seed does not reproduce 2.3.1's output any more. DETERMINISM_VERSION moved through 2, 3 and 4 in this cycle, each increment moving what a seed draws, and the entry gives a row per version so a stored dataset can be placed. A dataset with no determinism_version column was drawn by the pre-2.4.0 model. The breaking section also carries the lineage regulation change -- the same n_max now settles roughly three times the population it used to cap -- the expand_clones() deprecation, and the catalogue quarantine that makes 80 of 106 entries refuse to compile without an explicit opt-in. MIGRATING.md gains a 2.3.1 section ahead of the 1.x guide: keep the file rather than the seed, convert n_max by dividing by three at the default lambda_base, and six smaller breakers including pandas no longer arriving with any extra and summary() returning a dict. Every runnable snippet in it was executed. --- CHANGELOG.md | 144 +++++++++++++++++++++++++++++++++++-- MIGRATING.md | 199 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 334 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13e0d412..470e2f41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,60 @@ All notable changes to GenAIRR are documented here. -## Unreleased - -### Changed +## [2.4.0] - 2026-11-05 + +**Read this first if you have stored data.** `DETERMINISM_VERSION` moved from +the unnumbered model that shipped through 2.3.1 to **4**. A given seed does not +produce the 2.3.1 output any more — not approximately, not statistically. Every +exported record and every trace file now carries the number as +`determinism_version`, and `GenAIRR.DETERMINISM_VERSION` reads it without +running anything. A dataset with no such column was drawn by the pre-2.4.0 +model. See [MIGRATING.md](MIGRATING.md) for what to do with data you already +have. + +### Breaking and behaviour-changing + +- **`DETERMINISM_VERSION` is now 4** — three separate increments, each of which + moves what a seed produces: + + | version | what changed | + |---|---| + | 2 | one random substream per pass and per clonal-family role, keyed rather than offset or XOR-salted; a substitution draws a base the site does not already hold | + | 3 | clonal lineage regulation is Beverton-Holt; `n_max` is the population a lineage settles at | + | 4 | affinity selection scores the protein the cell actually expresses | + + Each is described in full below. Version 1 is the unnumbered model that + shipped up to and including 2.3.1. + +- **Clonal lineage regulation obeys a different law, and `n_max` means + something different.** `n_max` is now the live population a lineage + **settles at**, not the pivot of a law that settled lower and not a hard cap. + **The same `n_max` therefore yields about 3x more cells at the default + `lambda_base=1.5`**, with proportional memory and runtime; lower `n_max` to + recover the old population, whose plateau was `n_max * (1 - 1/lambda_base)`. + `lambda_base` is documented as what it is: the Poisson mean for a + founder-baseline cell in an empty germinal center, not the mean a cell draws + against once capacity damping or selection is active. + +- **`Experiment.expand_clones()` is deprecated** and raises a + `DeprecationWarning`. It keeps working, and it is still the only surface that + delivers an exact `n_clones x per_clone` count. Use + `clonal_lineage()` for BCR affinity-maturation trees with a ground-truth tree + per clone, or `clonal_repertoire()` for TCR and flat-BCR abundance + repertoires with clone-size distributions and `duplicate_count`. Neither is a + drop-in replacement for a fixed `per_clone`: a lineage run returns at most + `n_sample` records per clone after genotype collapse, and a repertoire run + collapses identical copies into one record carrying the count. + +- **Most bundled catalogues are quarantined under strict compilation.** A + reproducible reference audit marks every bundled catalogue carrying + unresolved anchor or pseudogene/ORF findings, and strict compilation — the + default — refuses them. 80 of the 106 bundled configs are affected, including + every mouse catalogue and `human_tcrb`; the three human OGRDB cartridges, + `HUMAN_IGH_EXTENDED` and 22 IMGT catalogues still compile strictly. Add + `.allow_curatable_refdata()` to sample from a quarantined catalogue anyway, + or `.curate_refdata(policy)` to drop the offending alleles. The error lists + the findings allele by allele. - **`DETERMINISM_VERSION` is now 4.** Affinity selection scores the protein the cell actually expresses. Seed-moving wherever selection is active; @@ -97,6 +148,58 @@ All notable changes to GenAIRR are documented here. ### Added +- **Two clonal models beside the deprecated star.** + `Experiment.clonal_lineage(...)` grows a real BCR genealogy per clone: a + generation-synchronous birth-death process with per-division S5F + hypermutation, optional affinity selection, live-cell sampling and + genotype collapse. Each clone comes back with an AIRR record per observed + cell -- `clone_id`, `lineage_node_id`, `lineage_parent_id`, + `lineage_generation`, `lineage_abundance`, `lineage_affinity`, + `duplicate_count` -- and with the ground-truth tree itself on + `result.lineage_trees`, exportable as Newick, FASTA or a node table. It is + BCR-only and refuses a TCR locus, because T cells do not hypermutate. + `Experiment.clonal_repertoire(...)` covers what a TCR clone actually is: one + rearrangement proliferated to a heavy-tailed clone size (power-law or + lognormal, with an `unexpanded_fraction` of never-expanded singletons), read + through library prep and collapsed to `duplicate_count`. Neither has a + per-clone size you set directly; see the deprecation note above. + +- **Paired-end read layout and FASTQ export.** + `Experiment.paired_end(r1_length=..., r2_length=..., insert_size=...)` + projects every record as an Illumina-style read pair: `r1_sequence`, + `r2_sequence` (already reverse-complemented), `r1_start` / `r1_end` / + `r2_start` / `r2_end`, `insert_size` and `read_layout`. It is a projection + and mutates nothing. `result.to_fastq(path, quality=...)` writes single-end + FASTQ and `result.to_paired_fastq(r1, r2)` writes two synchronised files + with portable `/1` and `/2` header suffixes; quality strings come from a + pluggable model (`"illumina"` or `"constant"`). A requested insert longer + than the surviving molecule leaves the windows empty rather than clipping, + which `validate_records` reports as `ReadWindowOutOfBounds`. + +- **Each NP region says where its bases came from.** `np1_n_tdt`, `np1_n_p` + and `np1_n_inserted` (and the `np2_*` triple) partition that region's length + into TdT N additions, templated palindromic P bases and bases an indel + inserted. An AIRR `np1` string has always combined all three, so a model + fitted against it as a pure N addition was fitting something else; the + partition makes the question answerable from the record. The validator + checks it through `NpPartitionMismatch` and `NpPartitionOriginAmbiguous`. + +- **The reading frame and every predicate of `productive`.** + `coding_frame_start` / `coding_frame_end` say which span the verdict was + computed over, and `v_anchor_accepted`, `j_anchor_accepted`, `vj_in_frame`, + `stop_codon` and `j_tail_in_frame` are the five predicates it is the + conjunction of, each three-valued, beside `productivity_policy`, + `productivity_scope` and `productivity_reasons`. A record that disagrees with + a tool can now be localised to the predicate instead of the verdict -- and + `j_tail_in_frame` is the one a record can fail on its own, which the other + four cannot show. + +- `n_ambiguous_bases`: the positions `ambiguous_base_calls()` actually masked. + It is not `sequence.count("N")`, which also counts any `N` the reference + allele carries, and it moves neither error counter -- the documentation used + to attribute the masking count to `n_quality_errors`, which + `sequencing_errors()` owns. + - `result.to_genotype_tsv(path)` and `CohortResult.to_genotype_tsv(path)`: a run writes the genotype it was given beside the records it produced, one row per carried allele, with `subject_id`, `locus`, `segment`, `gene`, @@ -140,10 +243,14 @@ All notable changes to GenAIRR are documented here. different versions promise each other nothing at all. The number increments whenever a released change moves what a seed produces, and it travels on the record rather than in a sidecar because a dataset leaves this library as a - file of rows and is then passed on by itself. It is `2`; a dataset with no - such column was produced by the unnumbered model that preceded it, which is - why the numbering does not start at 1. The two changes below are what - separates version 2 from that model. + file of rows and is then passed on by itself. It ships at `4`; a dataset with + no such column was produced by the unnumbered model that preceded the stamp, + which is why the numbering does not start at 1. The table at the top of this + release says what separates each version from the one before it. + + The reproducibility goldens assert the promise in CI on Linux, macOS and + Windows across Python 3.10-3.13. Linux aarch64 ships a wheel but is not + covered by them. - Six per-V-subregion recoverable mutation counters — `n_fwr1_observable_mutations`, `n_cdr1_observable_mutations`, @@ -539,6 +646,22 @@ All notable changes to GenAIRR are documented here. work this way and never did after the change above: each takes a keyed substream, so adjacent clonal seeds share nothing. +- Every code example in the README and on the documentation site was re-run + against this release and the values it prints were re-measured, so the + allele names, counts and field values on the page are the ones the engine + produces at `determinism_version` 4 rather than the ones the pre-stamp model + produced. The reproducibility promise now states what is actually checked + (same seed and same determinism version, on Linux, macOS and Windows across + Python 3.10-3.13), the quarantined-catalogue set is documented with the + `allow_curatable_refdata()` opt-in, and the pages that said the per-region + mutation counters, the cartridge model estimators or the NP origin fields + were deferred now describe them, because they ship. + +- `MIGRATING.md` gained a 2.3.1 to 2.4.0 section; the 1.x to 2.0 guide is + unchanged beneath it. A stale find/replace row that told 1.x users to rename + `expand_clones(n_clones=...)` to `n=...` was removed -- the keyword is and + always was `n_clones`, and the rename it described would raise `TypeError`. + ### Compatibility - Standard `v_call`, `d_call`, and `j_call` now name single generating alleles. @@ -576,6 +699,13 @@ All notable changes to GenAIRR are documented here. - NP length/base estimators reject records with explicitly nonzero P-length provenance. AIRR NP fields combine N and P; fitting them as N-only models while separately fitting P lengths would count P additions twice. +- `pandas` is not part of any extra and never was. `result.to_dataframe()` + needs it; install it yourself. The install docs and the README said it + shipped under `[all]`, which installs numpy, scipy, graphviz, tqdm and + fastmcp. +- The pytest matrix runs on Python 3.10, 3.11, 3.12 and 3.13 across Linux, + macOS and Windows. Python 3.9 remains supported and its wheels are + smoke-tested on all three platforms, but the suite is not run against it. ## [2.3.1] - 2026-07-05 diff --git a/MIGRATING.md b/MIGRATING.md index 66dcd7a1..ec78bd14 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -1,3 +1,197 @@ +# Migrating + +Two migrations live in this file. Start with the one that matches the version +you are coming from. + +- [2.3.1 → 2.4.0](#migrating-from-231-to-240) — stored data and clonal + parameters. +- [1.x → 2.0](#migrating-from-genairr-1x-to-20) — the Experiment DSL rename. + +--- + +# Migrating from 2.3.1 to 2.4.0 + +2.4.0 does not rename anything you call. What it changes is **what a seed +produces**, **what `n_max` means**, and **which bundled catalogues compile +without an opt-in**. Nothing below is a find/replace; each item is a decision +about data or parameters you already have. + +--- + +## 1. Your stored data is from a different sampling model + +`GenAIRR.DETERMINISM_VERSION` is now `4`. Everything GenAIRR released up to and +including 2.3.1 was the unnumbered model that preceded the stamp. A seed that +produced a dataset under 2.3.1 does **not** reproduce it under 2.4.0 — not +approximately, not statistically. Three separate increments moved the draws: +per-pass random substreams and keyed sub-simulation seeds (2), +Beverton-Holt lineage regulation (3), and affinity selection scoring the +protein the cell expresses (4). + +**What to do** + +- **Keep the data, not the seed.** A dataset you still need is a file of rows; + keep the file. Do not plan to regenerate it from the seed. +- **Record the version alongside any dataset you publish.** Every record now + carries a `determinism_version` column and every trace file carries the same + number. Read the installed engine's number without running anything: + + ```python + import GenAIRR as ga + ga.DETERMINISM_VERSION # 4 + ``` + +- **Check the column before scoring against someone else's dataset.** A + dataset with no such column was drawn by the pre-2.4.0 model. Two datasets + with different numbers are not comparable, and a trace file whose number + disagrees with the installed engine now fails replay by name instead of + quietly reproducing something else. +- **Re-pin golden files.** If your test suite asserts simulated bytes for a + fixed seed, regenerate the fixtures once and commit the new values together + with the determinism version you generated them at. + +--- + +## 2. `n_max` delivers about 3x more cells + +`clonal_lineage(n_max=...)` is now the live population a lineage **settles +at**. Under 2.3.1 it was the pivot of a discrete logistic map that settled at +`n_max * (1 - 1/lambda_base)` and a hard cap truncated any generation that +exceeded it — so a requested 1000 delivered about 333 cells at the default +`lambda_base=1.5`, and reaching capacity killed the lineage outright. + +**What to do** + +- If you tuned `n_max` to get a population you were happy with, divide it by + `1 / (1 - 1/lambda_base)` to recover roughly the old plateau. At the default + `lambda_base=1.5` that is `n_max / 3`. +- Budget memory and runtime for the larger population if you keep your old + `n_max`. +- Stop reading `lambda_base` as "mean offspring per cell". It is the Poisson + mean for a founder-baseline cell in an **empty** germinal center. The mean a + cell actually draws against is that damped by the live population and, under + selection, scaled by its fitness relative to the founder. + +```python +# 2.3.1: a settled population of ~333 +exp.clonal_lineage(n_clones=20, n_max=1000, lambda_base=1.5, ...) + +# 2.4.0, same population +exp.clonal_lineage(n_clones=20, n_max=333, lambda_base=1.5, ...) + +# 2.4.0, if you wanted 1000 all along: no change needed +exp.clonal_lineage(n_clones=20, n_max=1000, lambda_base=1.5, ...) +``` + +Affinity numbers from before this release are not comparable either, and +neither are the genealogies selection shaped: under 2.3.1 a cell grown without +mutation scored `3.57e-67` against its own protein instead of `1.0`, and a stop +codon scored as a perfect match. A `target_aa` whose length does not match the +receptor's protein is now refused rather than scored over the overlapping +prefix. + +--- + +## 3. `expand_clones()` is deprecated + +It still works, and it is still the only surface that delivers an exact +`n_clones x per_clone` count. It now raises a `DeprecationWarning`. + +**What to do** + +- **Keeping it:** nothing, unless your test suite turns warnings into errors. + Then filter it: + + ```python + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + result = exp.expand_clones(n_clones=10, per_clone=50).run_records(seed=0) + ``` + +- **Moving off it:** pick by what the clone means in your benchmark. + + | You want | Use | + |---|---| + | A BCR genealogy with ancestral nodes, per-division SHM and a ground-truth tree per clone | `clonal_lineage(n_clones=..., max_generations=..., n_sample=..., rate=...)` | + | TCR or flat-BCR clone **abundance** — membership plus a size | `clonal_repertoire(n_clones=..., size_distribution=..., max_size=...)` | + + Neither is a drop-in replacement for a fixed `per_clone`. A lineage run + returns at most `n_sample` records per clone after identical genotypes are + collapsed; a repertoire run collapses identical copies into one record + carrying `duplicate_count`. + +--- + +## 4. 80 of the 106 bundled catalogues need an explicit opt-in + +Strict compilation — the default — now refuses a bundled catalogue that carries +unresolved anchor or pseudogene/ORF findings. Chains that ran under 2.3.1 can +raise `ValueError` at compile time with the findings listed allele by allele. + +Affected: every mouse catalogue, `human_tcrb`, `rhesus_igk`, and 77 others. +Unaffected: the three human OGRDB cartridges (`human_igh`, `human_igk`, +`human_igl`), `HUMAN_IGH_EXTENDED`, and 22 IMGT catalogues whose anchors all +resolved. + +**What to do** — one of: + +```python +# Sample from the catalogue as it is, acknowledging the findings. +ga.Experiment.on("mouse_igh").allow_curatable_refdata().recombine() + +# Or drop the offending alleles first. +ga.Experiment.on("mouse_igh").curate_refdata("functional_anchors_only").recombine() +``` + +`run_records(..., allow_curatable_refdata=True)` is the per-run equivalent of +the first form. + +--- + +## 5. Counters that now report what you asked for + +`mutate(model="uniform")`, `pcr_amplify` and `sequencing_errors` drew their +replacement base from all four bases, including the one already at the +position, so about a quarter of draws wrote back what was there and were +correctly not counted: `pcr_amplify(count=8)` delivered about six errors. The +replacement is now drawn from the three bases the position does not hold, so +`count=N` delivers `N`. **If you calibrated a corruption rate against the old +behaviour, it is now about 32% higher than you intended** — multiply by 0.757 +to recover the realized rate you had. + +Two further counter changes to know about: + +- `n_ambiguous_bases`, not `n_quality_errors`, is what `ambiguous_base_calls()` + reports on. `n_quality_errors` belongs to `sequencing_errors()`. +- `n_observable_mutations` is the SHM a mutation caller can recover from + `sequence`. **Score a caller against that field, not `n_mutations`**, which + counts mutations on untemplated junction bases and positions later lost to + end loss. + +--- + +## 6. Smaller things that can break a script + +- `result.to_dataframe()` needs `pandas`, which is **not** in any extra. The + docs used to say it shipped under `[all]`; install it yourself. +- `result.validate_records(refdata)` wants the engine-side `RefDataConfig`. + `exp.compile().refdata` gives you one; passing a `DataConfig` raises + `TypeError`. `report.summary()` returns a `dict[str, int]`, not a string. +- `mutate(segment_rates={"V": 0, "D": 0, "J": 0, "NP": 0})` is refused with + `ValueError` instead of silently making the pass a no-op. +- `Experiment.on("x.json")` — any string ending in `.json` — raises `TypeError` + where it raised `ValueError`. Read a portable reference with + `GenAIRR.Reference.read(path)`; a path string is never read implicitly. +- Artifact files GenAIRR writes (portable references, family-replay bundles) + follow your umask instead of being created owner-only `0600`. +- `validate_records` / `validate_families_with_parents` against a reference + other than the one the run used raises `ValueError` instead of misreporting. + A genotype run with novel alleles must be validated against + `compiled.refdata`. + +--- + # Migrating from GenAIRR 1.x to 2.0 GenAIRR 2.0 is a breaking release. The Python `Experiment` DSL was reworked @@ -30,7 +224,8 @@ objects you may have constructed manually still work the same way. | `corrupt_3prime_loss(length=...)` | `primer_trim_3prime(length=...)` | | `with_clonal_structure(n_clones=N, size=K)` | `expand_clones(n_clones=N, per_clone=K)` | -Parameter renames are inside `expand_clones`: `n_clones` → `n`, `size` → `per_clone`. +The parameter rename inside `expand_clones` is `size` → `per_clone`; +`n_clones` keeps its name. --- @@ -59,7 +254,7 @@ The same chain in **v2.0**: result = ( ga.Experiment.on("human_igh") .recombine() - .expand_clones(n=50, per_clone=20) + .expand_clones(n_clones=50, per_clone=20) .mutate(rate=0.05) .primer_trim_5prime(length=(0, 8)) .polymerase_indels(count=(0, 2), insertion_prob=0.5) From 132a62cd871ee2f33aa6304701b8cf6a9cee5b66 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 5 Nov 2026 21:51:18 +0200 Subject: [PATCH 6/7] release: 2.4.0, and the packaged dossier regenerated to record it The version bump alone fails four tests, which is the guard working: the packaged reference dossier records the GenAIRR version that generated it, so a checkout declaring 2.4.0 beside a dossier built by 2.3.1 is a mismatch the suite refuses. The generator refuses too, until the installed build matches what pyproject declares, so the dossier cannot be stamped with a version it was not built by. This would have broken the release rather than the local run: build-wheels.yml patches pyproject.toml from the tag and then runs 'pytest tests/ -x -q', and nothing in that workflow can regenerate a 20 MB artifact. The archive has only existed since 2026-10-27, after the v2.3.1 tag, so no release has passed through this guard before. Of the five pinned digests the generator names, two moved: the manifest hash and the raw archive hash, both of which cover the recorded version. The runtime execution, catalogue and models hashes and the source lock came back byte-identical, and the archive is the same 20110025 bytes, which is the evidence that nothing about the reference build itself changed. The two retargeted pins are declared for the inventory gate, which cannot see a constant's new value by hashing bodies. --- pyproject.toml | 2 +- src/GenAIRR/_reference_sources/bundled.py | 2 +- .../human_igh_ogrdb_10_observed_v1.zip | Bin 20110025 -> 20110025 bytes .../test_published_reference_resources.py | 4 ++-- tools/inventory-declarations.json | 6 ++---- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e263b9fd..2fd81575 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ build-backend = "maturin" [project] name = "GenAIRR" -version = "2.3.1" +version = "2.4.0" description = "Synthetic immune-receptor-sequence simulator for benchmarking alignment models and sequence analysis." readme = "README.md" requires-python = ">=3.9" diff --git a/src/GenAIRR/_reference_sources/bundled.py b/src/GenAIRR/_reference_sources/bundled.py index 5e0e0736..09503e9d 100644 --- a/src/GenAIRR/_reference_sources/bundled.py +++ b/src/GenAIRR/_reference_sources/bundled.py @@ -75,7 +75,7 @@ def __post_init__(self): # coding spans, with source-annotated junction anchors. "human_igh_ogrdb_10_observed_v1": BundledDossier( "GenAIRR.data", "published_references/human_igh_ogrdb_10_observed_v1.zip", - "sha256:9ab6ba63490c06af94098152af388141f8f946bd6eab3c2ef86f8e5ddb512550"), + "sha256:101abcebe884e17ff52c8da6c583ef2740bda19c42e6cac450e4b181a0dc6f9b"), }) diff --git a/src/GenAIRR/data/published_references/human_igh_ogrdb_10_observed_v1.zip b/src/GenAIRR/data/published_references/human_igh_ogrdb_10_observed_v1.zip index 7599eee1e7c25f3cc2b3c3a092635b2a5dd6b6fa..2379aa3a376832ee0c602014f1cd7e144ca7c467 100644 GIT binary patch delta 1721 zcmb`|_j46x6oBEm5PAzO^xh$a?Il=(qS&HXKt&OE@9tWohy?))>cOa>h@#hm#@@vS zUa?`vh#iT7f?`9kH;e`KJ^2^(hv%K0-T8LDop0uB{mQj%>M>t((oC95iL{WG(n?xO z8)+-;q`h>Ij?zg=rL%OAuF_4qOPTbLp3+NtOCRYg{iMGPkbyEt2Fnl`D#K*BjF6Es zN=C~VDVMQQA>+hYVQwlfnYDIIR@hQ*?r82*7{wZQ-_l%RdxcqAm1cfE^vuMwEW`2~ zD~(MjGCVhQd?QV~FpP}ENliNpd^a=PAhR6Dw$sS-ZPyRHxL7&PT-K_}PxGc}dx0P1 z^L7^bNs@)J@0&?Ja-GnyBP&Q_$26SOaedp4JlErX;&@S(nsOBT(Hrq(uA?}pt`g$dXLdtxH?!rquvY^>gA*4CyMapbi%?ta)G2cQ}U;vh`M z!8inm;xJ6XR2+^Ya3rSTC>)LHI0iFtEF#3nBSDG`$KiOKfD>^NPDTw*!Ks*u(=ZFC z;|!dMvoITHV-DtG9?rqJI1lGzJ}$t8xCnv;xEQs#1efA6EW{#Qjw`SjS7HgS!qr%c zYj7Bf5ukj7O#dr7~Kj26FgrD&Xe#H*_ zhDQ93Kkz61!cP2+e-_rbxy8^l(=^wVXj*7mYFcSpYuaeqYT9YqYdUB;YC363HJvqG zG+i~_G~G32njV^-nqHdTnm(Gontq!8ngN=Dnn9YunjxB@nqivZnh~0jno*k3nlYMk z%~(x^W}L>*n7P$8u9a&);C~u;t99Vbk)~fo-O>$KZf4WCt82Z{%1vwbqd zWUP#n@lqj`QYF=rUSVvkC=7BlR_8WX7~2c1DkER#>{p!1ZL2bhYm*>xLqE1{)3uGz zj-0d;TCNq?iD~$O>sz*OT8SO{u9XB~I`kvU@(kPaLL)Go zTsuyZFb)IDvCPoQMj11VP1`lnVUkVE$n$K=kF79tjcgPeaW;&jY;KWfteqG;8Jk5! z>Db`;m@BJs4J(~CS(IyK9g{hx4d03k&kUkC@vV&S`x(azJU+v)+>UiM(;G{tnyFO{ z%^3q`zByx+l@}Jw8C=n@b(#em4qSNfk;!kJRx^KY?SGv?ZdprBrS4S2rj{B^z%Kbs zExVS?E?nAc`GAJj$-7~9)M6s`z@C_dy|6d-$+u44ch;7+7qR6Hb6HHhGTFnrsFsqj}s6eM1&YwBsdW#;bfeGQ*j#Va5`pSCeFYt zoQbn=HqOCpoQrzQ!Cah&^Kk(##5`Pti*X4A^KmH}a2YPg6S+Gz?jg_`!74w@oOM@=V9XH6GPS4}rfcTEpXv8Jb{ zm!`L-kEXAtL{qBir|GX5pc$ws(+tuK)(p`M)s$<7X@+Y?Xhv#AX+~?tXvS*BX~t_R zG?khvO|>SiF;Y!+j+yE@@-ecSici( CQGmq& diff --git a/tests/components/reference/test_published_reference_resources.py b/tests/components/reference/test_published_reference_resources.py index c929a7d1..2fb19157 100644 --- a/tests/components/reference/test_published_reference_resources.py +++ b/tests/components/reference/test_published_reference_resources.py @@ -470,12 +470,12 @@ def test_the_generator_writes_only_a_new_zip_file(tmp_path, capsys): # Pinned from the reviewed build: the typed manifest hash of the packaged dossier (it covers # the GenAIRR version that generated it) and the runtime hashes of its reference. -MANIFEST_PIN = "sha256:9ab6ba63490c06af94098152af388141f8f946bd6eab3c2ef86f8e5ddb512550" +MANIFEST_PIN = "sha256:101abcebe884e17ff52c8da6c583ef2740bda19c42e6cac450e4b181a0dc6f9b" RUNTIME_PINS = {"execution": "sha256:bacd155a623189d5acd7c1720a9decbde4d4c5b6e6f6c5581e6ce263683df6dc", "catalogue": "sha256:fc8d35b4682385b1ef10c1405c7427ca2d7551ee626f4257f7bee4fadafb5266", "models": "sha256:e96b1404c64d36a55c0457163ef3a4aa706e1bc5114914147715cd0f964dffd8"} # The raw sha256 of the packaged archive (it holds the manifest, so it also changes with the version). -ARCHIVE_PIN = "sha256:52df85b32d417c67f882cce18113237eb4889f54962695853dd398bc8a3e33d8" +ARCHIVE_PIN = "sha256:d8d82abcaa21b736cf31fd3eab26d59a2ed59fcdad72738ad4fbf95e731b207a" RECORD_OUTCOMES = {"selected": 230, "excluded": 259, "rejected": 0} PACKAGE, RESOURCE = "GenAIRR.data", f"published_references/{STARTER_ID}.zip" DOSSIER_DIRECTORIES = ["cache", "cache/blobs", "cache/blobs/sha256", "inputs"] diff --git a/tools/inventory-declarations.json b/tools/inventory-declarations.json index d254ea55..5126d74c 100644 --- a/tools/inventory-declarations.json +++ b/tools/inventory-declarations.json @@ -1,9 +1,7 @@ { "revised": { - "VARIANTS": "four clonal variants added: clonal_lineage, clonal_lineage_selected, clonal_power_law and clonal_lognormal. No existing variant changed. They exist to reach code no fixture ran: lineage/poisson.rs:26 (exp, every offspring draw), lineage/affinity.rs:209,221 (exp, every affinity) and lineage/clone_size.rs:33,43,46,52 (ln, sqrt, cos, powf, exp). Rust documents those as platform-dependent in precision, and a clone size is x.round() of a powf, so one ulp is a different integer. Until now no golden grew a lineage or drew a clone size, which is why determinism versions 3 and 4 both passed through this file moving nothing but the version column.", - "FIXTURE_SPECS": "the four clonal fixtures appended, each with n=None because their record count follows from the clone structure rather than a caller's n. The six existing specs are unchanged. The tuple type widens to int | None for that reason.", - "golden_name": "omits the _n{n} segment when n is None, so the clonal fixtures are named seed42_human_igh_clonal_lineage.tsv rather than carrying a meaningless count. The name of every existing fixture is unchanged, which the six untouched committed files confirm.", - "write_reproducibility_tsv": "calls run_records(seed=seed) without n for the clonal variants, which do not accept one, and run_records(n=n, seed=seed) exactly as before for the rest. This is a generator, not an assertion helper: it cannot make a test pass more easily, because what it writes is compared byte for byte against the committed file by test_output_matches_the_committed_golden." + "MANIFEST_PIN": "the packaged reference dossier was regenerated for the 2.4.0 release, so its manifest hash moved from 9ab6ba63 to 101abcebe. The manifest records the GenAIRR version that generated the dossier, which is why a version bump alone retargets this pin; tools/build_published_reference.py prints the new value and names every pin site, and test_the_packaged_dossier_was_generated_by_the_version_this_checkout_declares fails if the two disagree. The same value is pinned once more in src/GenAIRR/_reference_sources/bundled.py's REGISTRY and was updated there in the same change. Nothing about the reference build itself changed: the runtime execution, catalogue and models hashes and the source lock all came back byte-identical, which is the evidence that only the recorded builder version moved.", + "ARCHIVE_PIN": "the raw sha256 of the regenerated archive, 52df85b3 to d8d82abc. The archive holds the manifest, so it changes with the version for the same reason MANIFEST_PIN does -- the comment above this constant already said so before this change. The archive is byte-identical in length (20110025 bytes both times), which is consistent with a change confined to the recorded version string rather than to any packaged member." }, "replaced": {}, "allow": {} From 214d678209e610d5c13ac4e5fbe4ad66f5cb1982 Mon Sep 17 00:00:00 2001 From: thomas Date: Thu, 5 Nov 2026 22:16:41 +0200 Subject: [PATCH 7/7] fix: the extension-path helper imports what its module imports locally duplex_compile_benchmark and duplex_family_benchmark import _engine inside the functions that use it, not at module scope, because they re-enter themselves as worker subprocesses. The helper added in the previous commit was placed at module scope and referenced _engine from there, so ruff's F821 rejected both files and every Python job failed on the lint step. original_end_stress imports _engine at module scope and was unaffected, which is why only two of the three were flagged. The helper now takes the import locally, matching each module's own idiom. Checked by running all three: each returns src/GenAIRR/_engine.abi3.so. 'make validate-release' does not lint, so this could not fail locally; 'ruff check tests/ src/ tools/' is the command CI runs and now passes. --- tools/metadata_biology_audit/duplex_compile_benchmark.py | 2 ++ tools/metadata_biology_audit/duplex_family_benchmark.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tools/metadata_biology_audit/duplex_compile_benchmark.py b/tools/metadata_biology_audit/duplex_compile_benchmark.py index e98de09c..c899816e 100644 --- a/tools/metadata_biology_audit/duplex_compile_benchmark.py +++ b/tools/metadata_biology_audit/duplex_compile_benchmark.py @@ -32,6 +32,8 @@ def _relative_extension_path() -> str: An absolute path identifies the author's filesystem, not the artifact; the sha256 beside this field is what identifies the binary. """ + from GenAIRR import _engine + extension = Path(_engine.__file__).resolve() try: return str(extension.relative_to(ROOT)) diff --git a/tools/metadata_biology_audit/duplex_family_benchmark.py b/tools/metadata_biology_audit/duplex_family_benchmark.py index 9966f3dd..7adee5f3 100644 --- a/tools/metadata_biology_audit/duplex_family_benchmark.py +++ b/tools/metadata_biology_audit/duplex_family_benchmark.py @@ -34,6 +34,8 @@ def _relative_extension_path() -> str: An absolute path identifies the author's filesystem, not the artifact; the sha256 beside this field is what identifies the binary. """ + from GenAIRR import _engine + extension = Path(_engine.__file__).resolve() try: return str(extension.relative_to(ROOT))