From 45fc9cd8a94eb7f154285501cde842ed7374a4bf Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Mon, 12 Oct 2026 17:36:57 +0000 Subject: [PATCH 01/12] feat(blast-jobs): crate scaffold, config, catalogue index, typed advanced table, validation --- .github/workflows/ci.yml | 16 + blast-jobs/.gitignore | 1 + blast-jobs/Cargo.lock | 1349 +++++++++++++++++ blast-jobs/Cargo.toml | 33 + blast-jobs/README.md | 72 + blast-jobs/deny.toml | 9 + blast-jobs/rust-toolchain.toml | 3 + blast-jobs/src/advanced.rs | 734 +++++++++ blast-jobs/src/catalogue.rs | 339 +++++ blast-jobs/src/config.rs | 292 ++++ blast-jobs/src/lib.rs | 8 + blast-jobs/src/main.rs | 25 + blast-jobs/src/types.rs | 294 ++++ blast-jobs/src/validate.rs | 570 +++++++ blast-jobs/tests/fixture/catalogue.json | 11 + .../tests/fixture/db/dmel/dmel-assembly.nsq | 0 .../tests/fixture/db/dmel/dmel-transcript.nsq | 0 .../fixture/db/dmel/dmel-translation.psq | 0 .../fixture/db/dmel_misc/dmel-genbank.nsq | 0 .../fixture/db/dmel_misc/dmel-genpept.psq | 0 .../tests/fixture/db/dsim/dsim-assembly.nsq | 0 21 files changed, 3756 insertions(+) create mode 100644 blast-jobs/.gitignore create mode 100644 blast-jobs/Cargo.lock create mode 100644 blast-jobs/Cargo.toml create mode 100644 blast-jobs/README.md create mode 100644 blast-jobs/deny.toml create mode 100644 blast-jobs/rust-toolchain.toml create mode 100644 blast-jobs/src/advanced.rs create mode 100644 blast-jobs/src/catalogue.rs create mode 100644 blast-jobs/src/config.rs create mode 100644 blast-jobs/src/lib.rs create mode 100644 blast-jobs/src/main.rs create mode 100644 blast-jobs/src/types.rs create mode 100644 blast-jobs/src/validate.rs create mode 100644 blast-jobs/tests/fixture/catalogue.json create mode 100644 blast-jobs/tests/fixture/db/dmel/dmel-assembly.nsq create mode 100644 blast-jobs/tests/fixture/db/dmel/dmel-transcript.nsq create mode 100644 blast-jobs/tests/fixture/db/dmel/dmel-translation.psq create mode 100644 blast-jobs/tests/fixture/db/dmel_misc/dmel-genbank.nsq create mode 100644 blast-jobs/tests/fixture/db/dmel_misc/dmel-genpept.psq create mode 100644 blast-jobs/tests/fixture/db/dsim/dsim-assembly.nsq diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6241437..060e344 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,3 +37,19 @@ jobs: - run: cd mcp && cargo test --tests env: CHADO_URL: postgres://flybase_mcp:flybase_mcp@127.0.0.1:5499/flybase + blast-jobs-unit: + runs-on: ubuntu-latest + defaults: { run: { working-directory: blast-jobs } } + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: { components: "clippy, rustfmt" } + - uses: Swatinem/rust-cache@v2 + with: { workspaces: blast-jobs } + - run: cargo fmt --check + - run: cargo clippy --all-targets -- -D warnings + - uses: EmbarkStudios/cargo-deny-action@v2 + with: + manifest-path: blast-jobs/Cargo.toml + command: check advisories licenses + - run: cargo test diff --git a/blast-jobs/.gitignore b/blast-jobs/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/blast-jobs/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/blast-jobs/Cargo.lock b/blast-jobs/Cargo.lock new file mode 100644 index 0000000..fb75b0b --- /dev/null +++ b/blast-jobs/Cargo.lock @@ -0,0 +1,1349 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "blast-jobs" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "base64 0.22.1", + "rand", + "reqwest", + "serde", + "serde_json", + "subtle", + "tempfile", + "thiserror", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/blast-jobs/Cargo.toml b/blast-jobs/Cargo.toml new file mode 100644 index 0000000..0aa69d3 --- /dev/null +++ b/blast-jobs/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "blast-jobs" +version = "0.1.0" +edition = "2024" +license = "MIT" +description = "Minimal BLAST+ job runner behind the FlyBase MCP's internal BLAST API" +repository = "https://github.com/FlyBase/flybase-ai-tools" +publish = false + +[lib] +name = "blast_jobs" +path = "src/lib.rs" + +[[bin]] +name = "blast-jobs" +path = "src/main.rs" + +[dependencies] +axum = "0.8" +tokio = { version = "1.53", features = ["full"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } +rand = "0.9" +base64 = "0.22" +subtle = "2" +anyhow = "1" +thiserror = "2" + +[dev-dependencies] +reqwest = { version = "0.13", default-features = false, features = ["json"] } +tempfile = "3" diff --git a/blast-jobs/README.md b/blast-jobs/README.md new file mode 100644 index 0000000..664fa3c --- /dev/null +++ b/blast-jobs/README.md @@ -0,0 +1,72 @@ +# blast-jobs + +A minimal BLAST+ job runner behind the FlyBase MCP's internal BLAST API (the contract is in +[`docs/handoff-private-side.md`](../docs/handoff-private-side.md) §7; the MCP's client is +`mcp/src/backends/blast.rs`). It runs NCBI BLAST+ against a directory of BLAST databases described by a +catalogue file. Nothing site-specific is compiled in, so it is reusable by anyone with a BLAST database +directory and a catalogue. + +This crate is a sibling of `mcp/`, not a workspace member: it has its own `Cargo.toml` and `Cargo.lock`. + +## Toolchain + +Stable Rust with `clippy` and `rustfmt`. If cargo is not on your `PATH`: + + export PATH="$HOME/.cargo/bin:$PATH" + +Run every command below from `blast-jobs/`. + + cargo test # all targets + cargo fmt && cargo clippy --all-targets -- -D warnings + cargo deny check advisories licenses + +## Configuration + +One JSON file, passed as the first argument (`blast-jobs `). Unknown keys are refused. + +| Key | Default | Meaning | +|---|---|---| +| `bind` | required | listen address, e.g. `0.0.0.0:8800` | +| `blast_bin_dir` | required | directory holding `blastn blastp blastx tblastn tblastx` | +| `database_root` | required | root the catalogue prefixes are relative to | +| `catalogue` | required | catalogue file (below) | +| `database_release` | required | label returned in every `202` and shown to users | +| `state_root` | required | writable directory for job state | +| `concurrency` | 2 | searches run at once (at least 1) | +| `threads_per_job` | 2 | `-num_threads` per search | +| `max_queued` | 20 | queued jobs before `429 queue_full` | +| `wall_seconds` | 600 | wall-clock limit for blastn and blastp | +| `translated_wall_seconds` | 300 | wall-clock limit for blastx, tblastn and tblastx | +| `retention_seconds` | 86400 | how long a job and its results are kept | +| `max_results_bytes` | 16777216 | cap on the served `results.tsv`; at most 16 MiB, the MCP's limit | +| `default_evalue` | 10 | used when a request has no `evalue` | +| `default_max_hits` | 100 | used when a request has no `max_hits` | +| `max_hsps_per_subject` | 50 | `-max_hsps`, and the ceiling for an `advanced` override | +| `max_residues` | blastn 100000, blastp 100000, blastx 30000, tblastn 30000, tblastx 10000 | query residue cap per program; if given, all five programs must be listed | + +## Catalogue + +Any JSON file with a `databases` array whose entries carry `species`, `category`, `protein` and `prefix` +(other fields are ignored): + +```json +{ "databases": [ + { "species": "dmel", "category": "assembly", "protein": false, "prefix": "dmel/dmel-assembly" } +] } +``` + +A request's `database` is the `category`; its `species` are the `species` codes. `prefix` is relative to +`database_root` and names a BLAST volume (`.nsq` for nucleotide, `.psq` for protein). At +startup every prefix must exist and contain no whitespace or quotes, no `(species, category)` pair may +repeat, and a category must have one molecule type; otherwise the service refuses to start. A search over +several species passes all their prefixes as one `-db` list, so its E-values are computed against the +combined database length. + +## Validation + +Every submit is checked independently of the MCP: program, database and molecule type, species (1 to 20 +lowercase codes, deduplicated), one FASTA record within the program's residue cap and alphabet, `evalue` +in (0, 1000], `max_hits` 1 to 1000, `matrix` (not with blastn), `megablast` (blastn only), `genetic_code` +in the NCBI set (1-6, 9-16, 21-31, 33). The `advanced` string is checked against a typed per-program +allow-list of BLAST+ flags (`src/advanced.rs`, `TABLE`); everything else, including the flags the service +sets itself and anything that could reach the filesystem or network, is refused with `invalid_option`. diff --git a/blast-jobs/deny.toml b/blast-jobs/deny.toml new file mode 100644 index 0000000..1d2fd77 --- /dev/null +++ b/blast-jobs/deny.toml @@ -0,0 +1,9 @@ +[advisories] +version = 2 +ignore = [] + +[licenses] +version = 2 +# CI runs `cargo deny check advisories licenses` for this crate; a new licence on a transitive crate is a +# review decision made by adding it here. +allow = ["MIT", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "Unicode-3.0"] diff --git a/blast-jobs/rust-toolchain.toml b/blast-jobs/rust-toolchain.toml new file mode 100644 index 0000000..d0ead5e --- /dev/null +++ b/blast-jobs/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "stable" +components = ["clippy", "rustfmt"] diff --git a/blast-jobs/src/advanced.rs b/blast-jobs/src/advanced.rs new file mode 100644 index 0000000..f1f8636 --- /dev/null +++ b/blast-jobs/src/advanced.rs @@ -0,0 +1,734 @@ +//! The `advanced` option string (design 3.4): a typed, per-program allow-list of BLAST+ flags. The MCP's +//! own filter only limits length and characters, so this table is the only barrier between a client and +//! the BLAST+ command line. +use crate::types::{Program, Rejection}; +use std::collections::HashSet; + +/// Longest `advanced` string accepted (the MCP's limit). +pub const MAX_LEN: usize = 200; +/// Most flags (a Switch counts as one) in one `advanced` string. +pub const MAX_PAIRS: usize = 8; + +/// How a flag's value is checked. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Kind { + /// Integer matching `^-?[0-9]+$`, within the inclusive bounds. + Int { min: Option, max: Option }, + /// Number matching `^-?[0-9.]+$`, finite, within the inclusive bounds. + Float { min: f64, max: Option }, + /// Exactly one of two literals (`yes`/`no` or `true`/`false`, as BLAST+ spells them for the flag). + Bool { yes: &'static str, no: &'static str }, + /// No value; rendered as a single argv element. + Switch, + /// One of a fixed set of literals. + Enum(&'static [&'static str]), +} + +/// One allowed flag (`name` without the leading `-`) and the programs it applies to. A name may appear +/// in several rows with disjoint program sets when its kind differs per program (`task`, `word_size`). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FlagSpec { + pub name: &'static str, + pub kind: Kind, + pub programs: &'static [Program], +} + +use Program::{Blastn, Blastp, Blastx, Tblastn, Tblastx}; + +const ALL: &[Program] = &[Blastn, Blastp, Blastx, Tblastn, Tblastx]; +const NOT_TBLASTX: &[Program] = &[Blastn, Blastp, Blastx, Tblastn]; +const PROTEIN_SCORED: &[Program] = &[Blastp, Blastx, Tblastn]; +const SEG: &[Program] = &[Blastp, Blastx, Tblastn, Tblastx]; +const NUCLEOTIDE_QUERY: &[Program] = &[Blastn, Blastx, Tblastx]; + +const fn int_min(min: i64) -> Kind { + Kind::Int { + min: Some(min), + max: None, + } +} + +const YES_NO: Kind = Kind::Bool { + yes: "yes", + no: "no", +}; + +/// The allow-list, exactly design 3.4. tblastx has no `task` row: BLAST+ offers it no task choice. +pub const TABLE: &[FlagSpec] = &[ + FlagSpec { + name: "task", + kind: Kind::Enum(&["blastn", "blastn-short", "megablast", "dc-megablast"]), + programs: &[Blastn], + }, + FlagSpec { + name: "task", + kind: Kind::Enum(&["blastp", "blastp-short", "blastp-fast"]), + programs: &[Blastp], + }, + FlagSpec { + name: "task", + kind: Kind::Enum(&["blastx", "blastx-fast"]), + programs: &[Blastx], + }, + FlagSpec { + name: "task", + kind: Kind::Enum(&["tblastn", "tblastn-fast"]), + programs: &[Tblastn], + }, + FlagSpec { + name: "word_size", + kind: int_min(4), + programs: &[Blastn], + }, + FlagSpec { + name: "word_size", + kind: int_min(2), + programs: &[Blastp, Blastx, Tblastn, Tblastx], + }, + FlagSpec { + name: "gapopen", + kind: int_min(0), + programs: NOT_TBLASTX, + }, + FlagSpec { + name: "gapextend", + kind: int_min(0), + programs: NOT_TBLASTX, + }, + FlagSpec { + name: "penalty", + kind: Kind::Int { + min: None, + max: Some(0), + }, + programs: &[Blastn], + }, + FlagSpec { + name: "reward", + kind: int_min(0), + programs: &[Blastn], + }, + FlagSpec { + name: "dust", + kind: YES_NO, + programs: &[Blastn], + }, + FlagSpec { + name: "seg", + kind: YES_NO, + programs: SEG, + }, + FlagSpec { + name: "soft_masking", + kind: Kind::Bool { + yes: "true", + no: "false", + }, + programs: ALL, + }, + FlagSpec { + name: "lcase_masking", + kind: Kind::Switch, + programs: ALL, + }, + FlagSpec { + name: "ungapped", + kind: Kind::Switch, + programs: NOT_TBLASTX, + }, + FlagSpec { + name: "strand", + kind: Kind::Enum(&["both", "plus", "minus"]), + programs: NUCLEOTIDE_QUERY, + }, + FlagSpec { + name: "perc_identity", + kind: Kind::Float { + min: 0.0, + max: Some(100.0), + }, + programs: &[Blastn], + }, + FlagSpec { + name: "qcov_hsp_perc", + kind: Kind::Float { + min: 0.0, + max: Some(100.0), + }, + programs: ALL, + }, + FlagSpec { + name: "culling_limit", + kind: int_min(0), + programs: ALL, + }, + FlagSpec { + name: "max_hsps", + kind: int_min(1), + programs: ALL, + }, + FlagSpec { + name: "threshold", + kind: Kind::Float { + min: 0.0, + max: None, + }, + programs: PROTEIN_SCORED, + }, + FlagSpec { + name: "comp_based_stats", + kind: Kind::Enum(&["0", "1", "2", "3", "D", "F", "T", "t", "f"]), + programs: PROTEIN_SCORED, + }, + FlagSpec { + name: "window_size", + kind: int_min(0), + programs: ALL, + }, +]; + +/// Flags the service sets from request fields or its own config; named in the rejection so the caller +/// knows where the setting lives. +const SERVICE_OWNED: &[&str] = &[ + "evalue", + "max_target_seqs", + "matrix", + "num_threads", + "query_gencode", + "db_gencode", + "outfmt", + "db", + "query", + "out", +]; + +/// The design's deny list (3.4), each with the program it is tested against. Every entry is rejected; +/// Task 1 tests them through `parse` and Task 4 through the API. +pub const DENY_LIST: &[(Program, &str)] = &[ + (Blastn, "-db x"), + (Blastn, "-query x"), + (Blastn, "-out x"), + (Blastn, "-remote"), + (Blastn, "-import_search_strategy x"), + (Blastn, "-export_search_strategy x"), + (Blastn, "-outfmt 5"), + (Blastn, "-num_threads 16"), + (Blastn, "-subject x"), + (Blastn, "-in_pssm x"), + (Blastn, "-gilist x"), + (Blastn, "-seqidlist x"), + (Blastn, "-negative_seqidlist x"), + (Blastn, "-taxids 7227"), + (Blastn, "-entrez_query x"), + (Blastn, "-logfile x"), + (Blastn, "-window_masker_db x"), + (Blastn, "-db_soft_mask x"), + (Blastn, "-db_hard_mask x"), + (Blastn, "-use_index true"), + (Blastn, "-index_name x"), + (Blastn, "-html"), + (Blastn, "-version"), + (Blastn, "-help"), + (Blastn, "--word_size 11"), + (Blastn, "-word_size 11 -word_size 7"), + (Blastn, "-ungapped yes"), + (Blastn, "-penalty 3"), + (Blastn, "-seg yes"), + (Blastp, "-dust yes"), + (Blastn, "-task blastp"), +]; + +fn reject(m: impl std::fmt::Display) -> Rejection { + Rejection::option(format!("advanced: {m}")) +} + +fn allowed_char(c: char) -> bool { + c.is_ascii_alphanumeric() || matches!(c, ' ' | '_' | '.' | '=' | '-') +} + +/// The row for `name` under `program`, or the rejection naming why there is none. +fn lookup(program: Program, name: &str) -> Result<&'static FlagSpec, Rejection> { + let mut known = false; + for spec in TABLE.iter().filter(|s| s.name == name) { + known = true; + if spec.programs.contains(&program) { + return Ok(spec); + } + } + Err(if known { + reject(format_args!("-{name} does not apply to {program}")) + } else if SERVICE_OWNED.contains(&name) { + reject(format_args!( + "-{name} is set by the service; use the request field or leave it to the service" + )) + } else { + reject(format_args!("-{name} is not an allowed option")) + }) +} + +fn check_value(name: &str, kind: Kind, v: &str) -> Result<(), Rejection> { + let numeric = |extra: &[u8]| { + let digits = v.strip_prefix('-').unwrap_or(v); + !digits.is_empty() + && digits + .bytes() + .all(|b| b.is_ascii_digit() || extra.contains(&b)) + }; + let bad = || reject(format_args!("-{name} value {v:?} is not valid")); + match kind { + Kind::Int { min, max } => { + if !numeric(b"") { + return Err(reject(format_args!("-{name} takes an integer, not {v:?}"))); + } + let n: i64 = v.parse().map_err(|_| bad())?; + if min.is_some_and(|m| n < m) || max.is_some_and(|m| n > m) { + return Err(reject(format_args!( + "-{name} {n} is out of range ({})", + range(min.map(|m| m as f64), max.map(|m| m as f64)) + ))); + } + } + Kind::Float { min, max } => { + if !numeric(b".") { + return Err(reject(format_args!("-{name} takes a number, not {v:?}"))); + } + let n: f64 = v.parse().map_err(|_| bad())?; + if !n.is_finite() || n < min || max.is_some_and(|m| n > m) { + return Err(reject(format_args!( + "-{name} {v} is out of range ({})", + range(Some(min), max) + ))); + } + } + Kind::Bool { yes, no } => { + if v != yes && v != no { + return Err(reject(format_args!( + "-{name} takes {yes} or {no}, not {v:?}" + ))); + } + } + Kind::Enum(values) => { + if !values.contains(&v) { + return Err(reject(format_args!( + "-{name} takes one of {}, not {v:?}", + values.join(" ") + ))); + } + } + Kind::Switch => unreachable!("switches take no value"), + } + Ok(()) +} + +fn range(min: Option, max: Option) -> String { + match (min, max) { + (Some(a), Some(b)) => format!("{a} to {b}"), + (Some(a), None) => format!("at least {a}"), + (None, Some(b)) => format!("at most {b}"), + (None, None) => "any".into(), + } +} + +/// Parse `advanced` for `program` into `(flag, value)` pairs in the order given. `flag` is the table +/// name without the leading `-`; `value` is `None` for a Switch. Accepts `-flag value` and +/// `-flag=value`; values of Int/Float flags may be negative (`-penalty -3`), no other value may start +/// with `-`. Rejects (all `invalid_option`): more than 200 characters or a character outside +/// `[A-Za-z0-9 _.=-]`; a token that is not a `-flag`; a `--` token; a flag not in `TABLE` or not for +/// this program (including the service-owned flags); a flag given twice; a Switch with a value; a +/// missing or invalid value; more than 8 flags. +pub fn parse(program: Program, s: &str) -> Result)>, Rejection> { + if s.len() > MAX_LEN { + return Err(reject(format_args!("at most {MAX_LEN} characters"))); + } + if let Some(c) = s.chars().find(|&c| !allowed_char(c)) { + return Err(reject(format_args!( + "character {c:?} is not allowed (letters, digits, space, _ . = -)" + ))); + } + let tokens: Vec<&str> = s.split(' ').filter(|t| !t.is_empty()).collect(); + let mut out: Vec<(String, Option)> = Vec::new(); + let mut seen = HashSet::new(); + let mut i = 0; + while i < tokens.len() { + let tok = tokens[i]; + i += 1; + if tok.starts_with("--") { + return Err(reject(format_args!( + "{tok:?}: options start with a single '-'" + ))); + } + let Some(body) = tok.strip_prefix('-') else { + return Err(reject(format_args!("expected a -flag, found {tok:?}"))); + }; + let (name, inline) = match body.split_once('=') { + Some((n, v)) => (n, Some(v)), + None => (body, None), + }; + if name.is_empty() { + return Err(reject(format_args!("{tok:?} has no flag name"))); + } + let spec = lookup(program, name)?; + if !seen.insert(name) { + return Err(reject(format_args!("-{name} is given more than once"))); + } + if out.len() == MAX_PAIRS { + return Err(reject(format_args!("at most {MAX_PAIRS} options"))); + } + let value = if spec.kind == Kind::Switch { + let trailing = tokens.get(i).filter(|t| !t.starts_with('-')); + if inline.is_some() || trailing.is_some() { + return Err(reject(format_args!("-{name} takes no value"))); + } + None + } else { + let v = match inline { + Some(v) => v, + None => { + // The next token is the value unless it is another flag; a negative number is a + // value only for numeric kinds, and check_value refuses anything else that starts + // with '-'. + let next = tokens.get(i).copied(); + let numeric = matches!(spec.kind, Kind::Int { .. } | Kind::Float { .. }); + match next { + Some(n) if !n.starts_with('-') || numeric => { + i += 1; + n + } + _ => "", + } + } + }; + if v.is_empty() { + return Err(reject(format_args!("-{name} needs a value"))); + } + check_value(name, spec.kind, v)?; + Some(v.to_string()) + }; + out.push((name.to_string(), value)); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::Code; + + fn ok(p: Program, s: &str) -> Vec<(String, Option)> { + parse(p, s).unwrap_or_else(|e| panic!("{p} {s:?} should parse: {e}")) + } + + fn err(p: Program, s: &str) -> String { + match parse(p, s) { + Ok(v) => panic!("{p} {s:?} should be rejected, parsed {v:?}"), + Err(e) => { + assert_eq!(e.code, Code::InvalidOption, "{p} {s:?}"); + assert!(e.message.starts_with("advanced: "), "{}", e.message); + e.message + } + } + } + + fn pair(f: &str, v: Option<&str>) -> (String, Option) { + (f.to_string(), v.map(str::to_string)) + } + + /// One row per allowed flag other than `task`, with the programs the design (3.4) gives it. The + /// program sets are written out from the design, not read from TABLE, so a table edit that drifts + /// from the design fails here. `bad` is None for a Switch. + struct Row { + flag: &'static str, + valid: &'static str, + bad: Option<&'static str>, + programs: &'static [Program], + } + + const ROWS: &[Row] = &[ + Row { + flag: "word_size", + valid: "11", + bad: Some("1"), + programs: ALL, + }, + Row { + flag: "gapopen", + valid: "11", + bad: Some("-1"), + programs: &[Blastn, Blastp, Blastx, Tblastn], + }, + Row { + flag: "gapextend", + valid: "1", + bad: Some("-1"), + programs: &[Blastn, Blastp, Blastx, Tblastn], + }, + Row { + flag: "penalty", + valid: "-3", + bad: Some("3"), + programs: &[Blastn], + }, + Row { + flag: "reward", + valid: "2", + bad: Some("-2"), + programs: &[Blastn], + }, + Row { + flag: "dust", + valid: "no", + bad: Some("true"), + programs: &[Blastn], + }, + Row { + flag: "seg", + valid: "no", + bad: Some("true"), + programs: &[Blastp, Blastx, Tblastn, Tblastx], + }, + Row { + flag: "soft_masking", + valid: "false", + bad: Some("no"), + programs: ALL, + }, + Row { + flag: "lcase_masking", + valid: "", + bad: None, + programs: ALL, + }, + Row { + flag: "ungapped", + valid: "", + bad: None, + programs: &[Blastn, Blastp, Blastx, Tblastn], + }, + Row { + flag: "strand", + valid: "minus", + bad: Some("up"), + programs: &[Blastn, Blastx, Tblastx], + }, + Row { + flag: "perc_identity", + valid: "90.5", + bad: Some("100.1"), + programs: &[Blastn], + }, + Row { + flag: "qcov_hsp_perc", + valid: "50", + bad: Some("101"), + programs: ALL, + }, + Row { + flag: "culling_limit", + valid: "5", + bad: Some("-1"), + programs: ALL, + }, + Row { + flag: "max_hsps", + valid: "10", + bad: Some("0"), + programs: ALL, + }, + Row { + flag: "threshold", + valid: "11.5", + bad: Some("-1"), + programs: &[Blastp, Blastx, Tblastn], + }, + Row { + flag: "comp_based_stats", + valid: "D", + bad: Some("4"), + programs: &[Blastp, Blastx, Tblastn], + }, + Row { + flag: "window_size", + valid: "40", + bad: Some("-1"), + programs: ALL, + }, + ]; + + #[test] + fn rows_cover_every_table_flag() { + for spec in TABLE { + assert!( + spec.name == "task" || ROWS.iter().any(|r| r.flag == spec.name), + "no test row for -{}", + spec.name + ); + } + } + + #[test] + fn each_allowed_flag_valid_wrong_program_bad_value() { + for row in ROWS { + for p in Program::ALL { + let f = row.flag; + if !row.programs.contains(&p) { + let s = if row.bad.is_none() { + format!("-{f}") + } else { + format!("-{f} {}", row.valid) + }; + assert!(err(p, &s).contains("does not apply"), "{p} {s}"); + continue; + } + match row.bad { + None => { + assert_eq!(ok(p, &format!("-{f}")), vec![pair(f, None)]); + assert!(err(p, &format!("-{f} yes")).contains("takes no value")); + assert!(err(p, &format!("-{f}=yes")).contains("takes no value")); + } + Some(bad) => { + let want = vec![pair(f, Some(row.valid))]; + assert_eq!(ok(p, &format!("-{f} {}", row.valid)), want); + assert_eq!(ok(p, &format!("-{f}={}", row.valid)), want); + err(p, &format!("-{f} {bad}")); + assert!(err(p, &format!("-{f}")).contains("needs a value")); + assert!(err(p, &format!("-{f}=")).contains("needs a value")); + } + } + } + } + } + + #[test] + fn task_values_per_program() { + let cases: &[(Program, &[&str], &str)] = &[ + ( + Blastn, + &["blastn", "blastn-short", "megablast", "dc-megablast"], + "blastp", + ), + (Blastp, &["blastp", "blastp-short", "blastp-fast"], "blastn"), + (Blastx, &["blastx", "blastx-fast"], "blastp"), + (Tblastn, &["tblastn", "tblastn-fast"], "tblastx"), + ]; + for (p, values, foreign) in cases { + for v in *values { + assert_eq!(ok(*p, &format!("-task {v}")), vec![pair("task", Some(v))]); + } + assert!(err(*p, &format!("-task {foreign}")).contains("takes one of")); + } + assert!(err(Tblastx, "-task tblastx").contains("does not apply to tblastx")); + } + + #[test] + fn word_size_minimum_differs_by_program() { + assert!(err(Blastn, "-word_size 3").contains("at least 4")); + assert_eq!( + ok(Blastp, "-word_size 3"), + vec![pair("word_size", Some("3"))] + ); + assert!(err(Blastp, "-word_size 1").contains("at least 2")); + } + + #[test] + fn deny_list_is_rejected() { + for (p, s) in DENY_LIST { + err(*p, s); + } + // The program-independent entries are rejected for every program. + for (_, s) in DENY_LIST + .iter() + .filter(|(_, s)| !["-seg yes", "-dust yes", "-task blastp"].contains(s)) + { + for p in Program::ALL { + err(p, s); + } + } + } + + #[test] + fn deny_list_messages_name_the_flag() { + assert!(err(Blastn, "-remote").contains("-remote is not an allowed option")); + assert!(err(Blastn, "-outfmt 5").contains("-outfmt is set by the service")); + assert!(err(Blastn, "-num_threads 16").contains("set by the service")); + assert!(err(Blastn, "-seg yes").contains("-seg does not apply to blastn")); + assert!(err(Blastp, "-dust yes").contains("-dust does not apply to blastp")); + assert!(err(Blastn, "--word_size 11").contains("single '-'")); + assert!(err(Blastn, "-word_size 11 -word_size 7").contains("more than once")); + assert!(err(Blastn, "-ungapped yes").contains("takes no value")); + assert!(err(Blastn, "-penalty 3").contains("at most 0")); + } + + #[test] + fn negative_numeric_values() { + assert_eq!(ok(Blastn, "-penalty -3"), vec![pair("penalty", Some("-3"))]); + assert_eq!(ok(Blastn, "-penalty=-3"), vec![pair("penalty", Some("-3"))]); + assert_eq!(ok(Blastn, "-penalty 0"), vec![pair("penalty", Some("0"))]); + err(Blastn, "-penalty 3"); + // A flag-looking value for a numeric kind is refused, never taken as a separate flag. + assert!(err(Blastn, "-word_size -remote").contains("integer")); + assert!(err(Blastn, "-word_size -3").contains("at least 4")); + // Non-numeric kinds never take a dash value; the dash token is parsed as a (missing) value. + assert!(err(Blastn, "-strand -plus").contains("needs a value")); + assert!(err(Blastn, "-strand=-plus").contains("takes one of")); + } + + #[test] + fn malformed_numbers() { + err(Blastn, "-perc_identity 1.2.3"); + err(Blastn, "-perc_identity ."); + err(Blastn, "-perc_identity -"); + err(Blastn, "-word_size 1.5"); + err(Blastn, "-word_size 99999999999999999999"); + assert_eq!( + ok(Blastn, "-perc_identity 100"), + vec![pair("perc_identity", Some("100"))] + ); + } + + #[test] + fn duplicates_and_double_dash() { + err(Blastn, "-dust no -dust yes"); + err(Blastn, "-dust=no -dust no"); + err(Blastn, "-lcase_masking -lcase_masking"); + err(Blastn, "--remote"); + err(Blastn, "-dust no --"); + } + + #[test] + fn at_most_eight_pairs() { + let eight = "-word_size 11 -gapopen 5 -gapextend 2 -penalty -3 -reward 2 -dust no -soft_masking false -lcase_masking"; + assert_eq!(ok(Blastn, eight).len(), 8); + assert!(err(Blastn, &format!("{eight} -ungapped")).contains("at most 8")); + } + + #[test] + fn length_charset_and_shape() { + assert!(ok(Blastn, "").is_empty()); + assert!(ok(Blastn, " ").is_empty()); + assert_eq!( + ok(Blastn, " -dust no -lcase_masking "), + vec![pair("dust", Some("no")), pair("lcase_masking", None)] + ); + let long = format!("-word_size {}", "1".repeat(MAX_LEN)); + assert!(err(Blastn, &long).contains("at most 200")); + assert!(err(Blastn, "-dust no;rm").contains("not allowed")); + assert!(err(Blastn, "-out /tmp/x").contains("not allowed")); + assert!(err(Blastn, "-dust\tno").contains("not allowed")); + assert!(err(Blastn, "dust no").contains("expected a -flag")); + assert!(err(Blastn, "-dust no extra").contains("expected a -flag")); + assert!(err(Blastn, "-").contains("no flag name")); + assert!(err(Blastn, "-=3").contains("no flag name")); + } + + #[test] + fn order_is_preserved() { + assert_eq!( + ok(Blastp, "-seg no -task blastp-fast -max_hsps 3"), + vec![ + pair("seg", Some("no")), + pair("task", Some("blastp-fast")), + pair("max_hsps", Some("3")) + ] + ); + } +} diff --git a/blast-jobs/src/catalogue.rs b/blast-jobs/src/catalogue.rs new file mode 100644 index 0000000..28af96c --- /dev/null +++ b/blast-jobs/src/catalogue.rs @@ -0,0 +1,339 @@ +//! Database catalogue: maps `(species, category)` to a BLAST volume prefix under the database root +//! (design 3.2, 3.3). Any JSON file with a `databases` array of `{species, category, protein, prefix}` +//! works; extra fields are ignored. +use crate::types::Rejection; +use anyhow::{Context, bail}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap}; +use std::path::{Component, Path, PathBuf}; + +#[derive(Deserialize)] +struct File { + databases: Vec, +} + +#[derive(Deserialize)] +struct Entry { + species: String, + category: String, + protein: bool, + prefix: String, +} + +/// The databases one search runs against: their molecule type and absolute volume prefixes, in the +/// order the species were requested. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct Resolved { + pub protein: bool, + pub prefixes: Vec, +} + +#[derive(Debug)] +pub struct Catalogue { + /// category -> molecule type (one per category; checked at load). + categories: BTreeMap, + /// (species, category) -> absolute prefix. + entries: HashMap<(String, String), PathBuf>, +} + +fn is_code(s: &str) -> bool { + !s.is_empty() + && s.bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_') +} + +impl Catalogue { + /// Load `path` and check every entry against `root`: the prefix is relative, carries no whitespace + /// or quote characters (space separates databases in `-db`), and names an existing `.nsq`/`.psq` + /// volume; no `(species, category)` pair repeats and each category has one molecule type. Any + /// failure refuses to start, so a bad mount shows at boot rather than at the first job. + pub fn load(path: impl AsRef, root: impl AsRef) -> anyhow::Result { + let path = path.as_ref(); + let root = std::path::absolute(root.as_ref()) + .with_context(|| format!("database root {}", root.as_ref().display()))?; + let text = std::fs::read_to_string(path) + .with_context(|| format!("reading catalogue {}", path.display()))?; + let file: File = serde_json::from_str(&text) + .with_context(|| format!("parsing catalogue {}", path.display()))?; + if file.databases.is_empty() { + bail!("catalogue {} lists no databases", path.display()); + } + let mut categories = BTreeMap::new(); + let mut entries = HashMap::new(); + for e in file.databases { + let name = format!("{} {}", e.species, e.category); + if !is_code(&e.species) || !is_code(&e.category) { + bail!("catalogue entry {name:?}: species and category must be [a-z0-9_]+"); + } + if e.prefix.is_empty() + || e.prefix + .chars() + .any(|c| c.is_whitespace() || c == '"' || c == '\'') + { + bail!( + "catalogue entry {name}: prefix {:?} is empty or contains whitespace or quotes", + e.prefix + ); + } + let rel = Path::new(&e.prefix); + if !rel.components().all(|c| matches!(c, Component::Normal(_))) { + bail!( + "catalogue entry {name}: prefix {:?} must be relative to the database root, without '..'", + e.prefix + ); + } + let abs = root.join(rel); + let ext = if e.protein { "psq" } else { "nsq" }; + let volume = PathBuf::from(format!("{}.{ext}", abs.display())); + if !volume.is_file() { + bail!( + "catalogue entry {name}: {} does not exist", + volume.display() + ); + } + match categories.get(&e.category) { + Some(&p) if p != e.protein => bail!( + "catalogue category {} is listed as both protein and nucleotide", + e.category + ), + _ => { + categories.insert(e.category.clone(), e.protein); + } + } + if entries.insert((e.species, e.category), abs).is_some() { + bail!("catalogue entry {name} appears more than once"); + } + } + Ok(Catalogue { + categories, + entries, + }) + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// Molecule type of `category` (`Some(true)` = protein), or `None` when no species has it. + pub fn category_protein(&self, category: &str) -> Option { + self.categories.get(category).copied() + } + + /// The `invalid_database` rejection for a category no species has, listing the available ones. + pub fn unknown_database(&self, category: &str) -> Rejection { + let known: Vec<&str> = self.categories.keys().map(String::as_str).collect(); + Rejection::database(format!( + "database {category:?} is not available; available: {}", + known.join(", ") + )) + } + + /// Resolve every species for `category`. Unknown category: `invalid_database`. Species lacking it: + /// `invalid_species` naming all of them (no silent subset). + pub fn resolve(&self, species: &[String], category: &str) -> Result { + let protein = self + .category_protein(category) + .ok_or_else(|| self.unknown_database(category))?; + let mut prefixes = Vec::with_capacity(species.len()); + let mut missing = Vec::new(); + for s in species { + match self.entries.get(&(s.clone(), category.to_string())) { + Some(p) => prefixes.push(p.clone()), + None => missing.push(s.as_str()), + } + } + if !missing.is_empty() { + return Err(Rejection::species(format!( + "no {category} database for species: {}", + missing.join(", ") + ))); + } + Ok(Resolved { protein, prefixes }) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::types::Code; + use serde_json::json; + + pub(crate) fn fixture_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixture") + } + + pub(crate) fn fixture() -> Catalogue { + let d = fixture_dir(); + Catalogue::load(d.join("catalogue.json"), d.join("db")).unwrap() + } + + fn sv(v: &[&str]) -> Vec { + v.iter().map(|s| s.to_string()).collect() + } + + /// A temp root with the volume files `files` and a catalogue of `entries`. + fn temp( + entries: serde_json::Value, + files: &[&str], + ) -> (tempfile::TempDir, anyhow::Result) { + let dir = tempfile::tempdir().unwrap(); + for f in files { + let p = dir.path().join(f); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, b"").unwrap(); + } + let cat = dir.path().join("catalogue.json"); + std::fs::write( + &cat, + json!({ "version": 1, "databases": entries }).to_string(), + ) + .unwrap(); + let r = Catalogue::load(&cat, dir.path()); + (dir, r) + } + + fn refusal(entries: serde_json::Value, files: &[&str]) -> String { + let (_d, r) = temp(entries, files); + format!("{:#}", r.unwrap_err()) + } + + #[test] + fn fixture_loads_with_hyphenated_prefix() { + let c = fixture(); + assert_eq!(c.len(), 6); + let r = c.resolve(&sv(&["dmel"]), "assembly").unwrap(); + assert!(!r.protein); + assert_eq!( + r.prefixes, + vec![fixture_dir().join("db/dmel/dmel-assembly")] + ); + assert!(r.prefixes[0].is_absolute()); + let r = c.resolve(&sv(&["dmel"]), "translation").unwrap(); + assert!(r.protein); + } + + #[test] + fn two_species_keep_request_order() { + let r = fixture() + .resolve(&sv(&["dsim", "dmel"]), "assembly") + .unwrap(); + assert_eq!( + r.prefixes, + vec![ + fixture_dir().join("db/dsim/dsim-assembly"), + fixture_dir().join("db/dmel/dmel-assembly") + ] + ); + } + + #[test] + fn missing_pair_names_the_species() { + let e = fixture() + .resolve(&sv(&["dmel", "dsim"]), "transcript") + .unwrap_err(); + assert_eq!(e.code, Code::InvalidSpecies); + assert!( + e.message.contains("dsim") && !e.message.contains("dmel"), + "{}", + e.message + ); + } + + #[test] + fn unknown_category_is_invalid_database() { + let e = fixture().resolve(&sv(&["dmel"]), "est").unwrap_err(); + assert_eq!(e.code, Code::InvalidDatabase); + assert!( + e.message.contains("assembly"), + "lists what is available: {}", + e.message + ); + } + + #[test] + fn extra_fields_are_ignored() { + let (_d, r) = temp( + json!([{"species": "dmel", "category": "assembly", "protein": false, + "prefix": "dmel/dmel-assembly", "label": "x", "sequences": 3}]), + &["dmel/dmel-assembly.nsq"], + ); + assert_eq!(r.unwrap().len(), 1); + } + + #[test] + fn refuses_missing_volume() { + let m = refusal( + json!([{"species": "dmel", "category": "assembly", "protein": false, "prefix": "dmel/dmel-assembly"}]), + &[], + ); + assert!(m.contains("does not exist"), "{m}"); + // A nucleotide entry needs .nsq; a .psq alone does not satisfy it. + let m = refusal( + json!([{"species": "dmel", "category": "assembly", "protein": false, "prefix": "dmel/dmel-assembly"}]), + &["dmel/dmel-assembly.psq"], + ); + assert!(m.contains("dmel-assembly.nsq"), "{m}"); + } + + #[test] + fn refuses_whitespace_and_quotes_in_prefix() { + for prefix in [ + "dmel/dmel assembly", + "dmel/dmel\tassembly", + "dmel/\"x", + "dmel/'x", + ] { + let m = refusal( + json!([{"species": "dmel", "category": "assembly", "protein": false, "prefix": prefix}]), + &[], + ); + assert!(m.contains("whitespace or quotes"), "{prefix}: {m}"); + } + } + + #[test] + fn refuses_absolute_or_parent_prefix() { + for prefix in ["/etc/passwd", "../outside", "dmel/../../x"] { + let m = refusal( + json!([{"species": "dmel", "category": "assembly", "protein": false, "prefix": prefix}]), + &[], + ); + assert!(m.contains("relative"), "{prefix}: {m}"); + } + } + + #[test] + fn refuses_duplicate_pair() { + let m = refusal( + json!([ + {"species": "dmel", "category": "assembly", "protein": false, "prefix": "dmel/dmel-assembly"}, + {"species": "dmel", "category": "assembly", "protein": false, "prefix": "dmel/dmel-assembly"} + ]), + &["dmel/dmel-assembly.nsq"], + ); + assert!(m.contains("more than once"), "{m}"); + } + + #[test] + fn refuses_mixed_molecule_category_and_bad_codes() { + let m = refusal( + json!([ + {"species": "dmel", "category": "genbank", "protein": false, "prefix": "a"}, + {"species": "dsim", "category": "genbank", "protein": true, "prefix": "b"} + ]), + &["a.nsq", "b.psq"], + ); + assert!(m.contains("both protein and nucleotide"), "{m}"); + let m = refusal( + json!([{"species": "Dmel", "category": "assembly", "protein": false, "prefix": "a"}]), + &["a.nsq"], + ); + assert!(m.contains("[a-z0-9_]+"), "{m}"); + let m = refusal(json!([]), &[]); + assert!(m.contains("no databases"), "{m}"); + } +} diff --git a/blast-jobs/src/config.rs b/blast-jobs/src/config.rs new file mode 100644 index 0000000..77111d8 --- /dev/null +++ b/blast-jobs/src/config.rs @@ -0,0 +1,292 @@ +//! Service configuration: one JSON file (design 3.2). Nothing site-specific is compiled in. +use crate::types::Program; +use anyhow::{Context, bail}; +use serde::Deserialize; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; + +/// The MCP's hard limit on the `results.tsv` body (`Content-Length`), and so the ceiling for +/// `max_results_bytes`. +pub const MAX_RESULTS_CEILING: u64 = 16 * 1024 * 1024; + +#[derive(Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] +pub struct Config { + pub bind: SocketAddr, + pub blast_bin_dir: PathBuf, + pub database_root: PathBuf, + pub catalogue: PathBuf, + /// Returned verbatim in the `202` and shown to users. + pub database_release: String, + pub state_root: PathBuf, + #[serde(default = "defaults::concurrency")] + pub concurrency: usize, + #[serde(default = "defaults::threads_per_job")] + pub threads_per_job: u32, + #[serde(default = "defaults::max_queued")] + pub max_queued: usize, + #[serde(default = "defaults::wall_seconds")] + pub wall_seconds: u64, + #[serde(default = "defaults::translated_wall_seconds")] + pub translated_wall_seconds: u64, + #[serde(default = "defaults::retention_seconds")] + pub retention_seconds: u64, + #[serde(default = "defaults::max_results_bytes")] + pub max_results_bytes: u64, + #[serde(default = "defaults::default_evalue")] + pub default_evalue: f64, + #[serde(default = "defaults::default_max_hits")] + pub default_max_hits: u32, + #[serde(default = "defaults::max_hsps_per_subject")] + pub max_hsps_per_subject: u32, + /// Residue cap per program; when the key is given it must name all five programs. + #[serde(default = "defaults::max_residues")] + pub max_residues: HashMap, +} + +mod defaults { + use crate::types::Program; + use std::collections::HashMap; + + pub fn concurrency() -> usize { + 2 + } + pub fn threads_per_job() -> u32 { + 2 + } + pub fn max_queued() -> usize { + 20 + } + pub fn wall_seconds() -> u64 { + 600 + } + pub fn translated_wall_seconds() -> u64 { + 300 + } + pub fn retention_seconds() -> u64 { + 86_400 + } + pub fn max_results_bytes() -> u64 { + super::MAX_RESULTS_CEILING + } + pub fn default_evalue() -> f64 { + 10.0 + } + pub fn default_max_hits() -> u32 { + 100 + } + pub fn max_hsps_per_subject() -> u32 { + 50 + } + pub fn max_residues() -> HashMap { + HashMap::from([ + (Program::Blastn, 100_000), + (Program::Blastp, 100_000), + (Program::Blastx, 30_000), + (Program::Tblastn, 30_000), + (Program::Tblastx, 10_000), + ]) + } +} + +impl Config { + /// Read and validate the config file; any problem refuses to start the service. + pub fn load(path: impl AsRef) -> anyhow::Result { + let path = path.as_ref(); + let text = std::fs::read_to_string(path) + .with_context(|| format!("reading config {}", path.display()))?; + let cfg: Config = serde_json::from_str(&text) + .with_context(|| format!("parsing config {}", path.display()))?; + cfg.validate() + .with_context(|| format!("invalid config {}", path.display()))?; + Ok(cfg) + } + + fn validate(&self) -> anyhow::Result<()> { + if self.database_release.trim().is_empty() { + bail!("database_release must not be empty"); + } + if self.concurrency < 1 { + bail!("concurrency must be at least 1"); + } + if self.threads_per_job < 1 { + bail!("threads_per_job must be at least 1"); + } + if self.max_queued < 1 { + bail!("max_queued must be at least 1"); + } + if self.wall_seconds < 1 || self.translated_wall_seconds < 1 { + bail!("wall_seconds and translated_wall_seconds must be at least 1"); + } + if self.retention_seconds < 1 { + bail!("retention_seconds must be at least 1"); + } + if self.max_results_bytes > MAX_RESULTS_CEILING { + bail!( + "max_results_bytes is {}; the MCP reads at most {MAX_RESULTS_CEILING} bytes", + self.max_results_bytes + ); + } + // The served file always starts with the header line, so a smaller cap could not hold it. + if self.max_results_bytes < 1024 { + bail!("max_results_bytes must be at least 1024"); + } + if !(self.default_evalue.is_finite() + && self.default_evalue > 0.0 + && self.default_evalue <= 1000.0) + { + bail!("default_evalue must be in (0, 1000]"); + } + if !(1..=1000).contains(&self.default_max_hits) { + bail!("default_max_hits must be 1 to 1000"); + } + if self.max_hsps_per_subject < 1 { + bail!("max_hsps_per_subject must be at least 1"); + } + for p in Program::ALL { + match self.max_residues.get(&p) { + None => bail!("max_residues has no cap for {p}"), + Some(0) => bail!("max_residues for {p} must be at least 1"), + Some(_) => {} + } + } + Ok(()) + } + + /// Residue cap for `program` (present for every program once `load` succeeded). + pub fn residue_cap(&self, program: Program) -> usize { + self.max_residues[&program] + } + + /// Wall-clock limit for `program`: translated searches get `translated_wall_seconds`. + pub fn wall_seconds_for(&self, program: Program) -> u64 { + if program.translated() { + self.translated_wall_seconds + } else { + self.wall_seconds + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use serde_json::{Value, json}; + + pub(crate) fn sample() -> Value { + json!({ + "bind": "0.0.0.0:8800", + "blast_bin_dir": "/opt/ncbi-blast/bin", + "database_root": "/db", + "catalogue": "/db/catalogue.json", + "database_release": "FB2023_04", + "state_root": "/state", + "concurrency": 2, + "threads_per_job": 2, + "max_queued": 20, + "wall_seconds": 600, + "translated_wall_seconds": 300, + "retention_seconds": 86400, + "max_results_bytes": 16777216, + "default_evalue": 10, + "default_max_hits": 100, + "max_hsps_per_subject": 50, + "max_residues": { "blastn": 100000, "blastp": 100000, "blastx": 30000, "tblastn": 30000, "tblastx": 10000 } + }) + } + + fn load(v: &Value) -> anyhow::Result { + let f = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(f.path(), v.to_string()).unwrap(); + Config::load(f.path()) + } + + fn err(v: &Value) -> String { + format!("{:#}", load(v).unwrap_err()) + } + + #[test] + fn sample_loads() { + let c = load(&sample()).unwrap(); + assert_eq!(c.database_release, "FB2023_04"); + assert_eq!(c.residue_cap(Program::Tblastx), 10_000); + assert_eq!(c.wall_seconds_for(Program::Blastn), 600); + assert_eq!(c.wall_seconds_for(Program::Tblastn), 300); + } + + #[test] + fn defaults_fill_omitted_numbers() { + let mut v = sample(); + let o = v.as_object_mut().unwrap(); + for k in [ + "concurrency", + "max_results_bytes", + "max_residues", + "default_evalue", + ] { + o.remove(k); + } + let c = load(&v).unwrap(); + assert_eq!(c.concurrency, 2); + assert_eq!(c.max_results_bytes, MAX_RESULTS_CEILING); + assert_eq!(c.residue_cap(Program::Blastx), 30_000); + assert_eq!(c.default_evalue, 10.0); + } + + #[test] + fn results_cap_above_16_mib_is_refused() { + let mut v = sample(); + v["max_results_bytes"] = json!(16_777_217); + assert!(err(&v).contains("max_results_bytes")); + v["max_results_bytes"] = json!(16_777_216); + load(&v).unwrap(); + } + + #[test] + fn zero_concurrency_is_refused() { + let mut v = sample(); + v["concurrency"] = json!(0); + assert!(err(&v).contains("concurrency")); + } + + #[test] + fn residue_caps_must_cover_all_five_programs() { + let mut v = sample(); + v["max_residues"].as_object_mut().unwrap().remove("tblastx"); + assert!(err(&v).contains("no cap for tblastx")); + let mut v = sample(); + v["max_residues"]["blastp"] = json!(0); + assert!(err(&v).contains("blastp")); + let mut v = sample(); + v["max_residues"]["psiblast"] = json!(10); + assert!(load(&v).is_err(), "an unknown program key is refused"); + } + + #[test] + fn unknown_key_and_bad_values_are_refused() { + let mut v = sample(); + v["concurency"] = json!(4); + assert!(err(&v).contains("concurency")); + let mut v = sample(); + v["bind"] = json!("not-an-address"); + assert!(load(&v).is_err()); + let mut v = sample(); + v["database_release"] = json!(" "); + assert!(err(&v).contains("database_release")); + let mut v = sample(); + v["default_max_hits"] = json!(5000); + assert!(err(&v).contains("default_max_hits")); + let mut v = sample(); + v["default_evalue"] = json!(0); + assert!(err(&v).contains("default_evalue")); + let mut v = sample(); + v.as_object_mut().unwrap().remove("state_root"); + assert!(err(&v).contains("state_root")); + } + + #[test] + fn missing_file_is_an_error() { + assert!(Config::load("/nonexistent/blast-jobs.json").is_err()); + } +} diff --git a/blast-jobs/src/lib.rs b/blast-jobs/src/lib.rs new file mode 100644 index 0000000..296f96f --- /dev/null +++ b/blast-jobs/src/lib.rs @@ -0,0 +1,8 @@ +//! `blast-jobs`: a minimal BLAST+ job runner implementing the FlyBase MCP's internal BLAST job API +//! (`docs/handoff-private-side.md` §7). Database mapping comes from a catalogue file and every limit +//! from the config file; nothing site-specific is compiled in. +pub mod advanced; +pub mod catalogue; +pub mod config; +pub mod types; +pub mod validate; diff --git a/blast-jobs/src/main.rs b/blast-jobs/src/main.rs new file mode 100644 index 0000000..ba3f079 --- /dev/null +++ b/blast-jobs/src/main.rs @@ -0,0 +1,25 @@ +//! `blast-jobs `: loads and checks the config and the catalogue. The job API and the +//! workers are wired in later tasks; until then this is a configuration check that exits 0 on success. +use anyhow::Context; +use blast_jobs::{catalogue::Catalogue, config::Config}; + +fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .json() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()), + ) + .init(); + let path = std::env::args() + .nth(1) + .context("usage: blast-jobs ")?; + let cfg = Config::load(&path)?; + let catalogue = Catalogue::load(&cfg.catalogue, &cfg.database_root)?; + tracing::info!( + databases = catalogue.len(), + database_release = %cfg.database_release, + concurrency = cfg.concurrency, + "configuration and catalogue are valid" + ); + Ok(()) +} diff --git a/blast-jobs/src/types.rs b/blast-jobs/src/types.rs new file mode 100644 index 0000000..db77ee1 --- /dev/null +++ b/blast-jobs/src/types.rs @@ -0,0 +1,294 @@ +//! Wire types of the internal job API (`docs/handoff-private-side.md` §7) and the program enum. +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fmt; + +/// The five BLAST+ search programs the service runs. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[serde(rename_all = "lowercase")] +pub enum Program { + Blastn, + Blastp, + Blastx, + Tblastn, + Tblastx, +} + +impl Program { + pub const ALL: [Program; 5] = [ + Program::Blastn, + Program::Blastp, + Program::Blastx, + Program::Tblastn, + Program::Tblastx, + ]; + + pub fn parse(s: &str) -> Option { + Self::ALL.into_iter().find(|p| p.as_str() == s) + } + + pub fn as_str(self) -> &'static str { + match self { + Program::Blastn => "blastn", + Program::Blastp => "blastp", + Program::Blastx => "blastx", + Program::Tblastn => "tblastn", + Program::Tblastx => "tblastx", + } + } + + /// The query is protein (blastp, tblastn); the others take nucleotide queries. + pub fn protein_query(self) -> bool { + matches!(self, Program::Blastp | Program::Tblastn) + } + + /// The database is protein (blastp, blastx); the others search nucleotide databases. + pub fn protein_database(self) -> bool { + matches!(self, Program::Blastp | Program::Blastx) + } + + /// Programs that translate the query or the database (they take a genetic code and the shorter + /// `translated_wall_seconds` limit). + pub fn translated(self) -> bool { + matches!(self, Program::Blastx | Program::Tblastn | Program::Tblastx) + } +} + +impl fmt::Display for Program { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// `POST /internal/jobs` body. No `deny_unknown_fields`: a key a future MCP adds must not break submits. +/// Fields are loosely typed so that a wrong value becomes a coded rejection from `validate::check` +/// rather than a deserialisation error. +#[derive(Deserialize, Serialize, Clone, Default)] +pub struct SubmitRequest { + pub program: String, + pub database: String, + #[serde(default)] + pub species: Vec, + pub sequence: String, + #[serde(default)] + pub evalue: Option, + #[serde(default)] + pub megablast: Option, + #[serde(default)] + pub max_hits: Option, + #[serde(default)] + pub matrix: Option, + #[serde(default)] + pub genetic_code: Option, + #[serde(default)] + pub advanced: Option, + #[serde(default)] + pub client: Option, +} + +/// Debug without the query sequence, so a stray `{:?}` never logs residues. +impl fmt::Debug for SubmitRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SubmitRequest") + .field("program", &self.program) + .field("database", &self.database) + .field("species", &self.species) + .field("sequence_len", &self.sequence.len()) + .field("evalue", &self.evalue) + .field("megablast", &self.megablast) + .field("max_hits", &self.max_hits) + .field("matrix", &self.matrix) + .field("genetic_code", &self.genetic_code) + .field("advanced", &self.advanced) + .field("client", &self.client) + .finish() + } +} + +/// Job state as the contract spells it. The service never produces `Cancelled` (there is no cancel +/// route); it is here because the contract lists it. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum State { + Queued, + Running, + Complete, + Failed, + Cancelled, +} + +impl State { + pub fn as_str(self) -> &'static str { + match self { + State::Queued => "queued", + State::Running => "running", + State::Complete => "complete", + State::Failed => "failed", + State::Cancelled => "cancelled", + } + } +} + +/// `202` body of `POST /internal/jobs`. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct SubmitResponse { + pub job_id: String, + pub state: State, + #[serde(skip_serializing_if = "Option::is_none")] + pub queue_position: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub poll_after_seconds: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub database_release: Option, +} + +/// `200` body of `GET /internal/jobs/{id}`. `result_url` and `downloads` stay `None` (ruling 5) and are +/// then omitted from the JSON. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct JobStatus { + pub job_id: String, + pub state: State, + #[serde(skip_serializing_if = "Option::is_none")] + pub queue_position: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub result_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub downloads: Option>, + /// Not part of the wire format: set by the store when `results.tsv` was cut at the size cap, so the + /// results route can add `X-Blast-Jobs-Truncated: true`. + #[serde(skip)] + pub truncated: bool, +} + +/// Error codes of a `400` answer. The first five are the ones the MCP maps to a tool argument; +/// `InvalidRequest` is for a body that is not a valid request object at all. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Code { + InvalidProgram, + InvalidDatabase, + InvalidSpecies, + InvalidSequence, + InvalidOption, + InvalidRequest, +} + +impl Code { + pub fn as_str(self) -> &'static str { + match self { + Code::InvalidProgram => "invalid_program", + Code::InvalidDatabase => "invalid_database", + Code::InvalidSpecies => "invalid_species", + Code::InvalidSequence => "invalid_sequence", + Code::InvalidOption => "invalid_option", + Code::InvalidRequest => "invalid_request", + } + } +} + +/// A `400` rejection; serialises as the contract's `{error, message}`. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{}: {message}", code.as_str())] +pub struct Rejection { + #[serde(rename = "error")] + pub code: Code, + pub message: String, +} + +impl Rejection { + pub fn new(code: Code, message: impl Into) -> Self { + Self { + code, + message: message.into(), + } + } + + pub fn program(m: impl Into) -> Self { + Self::new(Code::InvalidProgram, m) + } + pub fn database(m: impl Into) -> Self { + Self::new(Code::InvalidDatabase, m) + } + pub fn species(m: impl Into) -> Self { + Self::new(Code::InvalidSpecies, m) + } + pub fn sequence(m: impl Into) -> Self { + Self::new(Code::InvalidSequence, m) + } + pub fn option(m: impl Into) -> Self { + Self::new(Code::InvalidOption, m) + } + pub fn request(m: impl Into) -> Self { + Self::new(Code::InvalidRequest, m) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejection_serialises_as_error_and_message() { + let v = + serde_json::to_value(Rejection::option("advanced: -remote is not allowed")).unwrap(); + assert_eq!( + v, + serde_json::json!({"error": "invalid_option", "message": "advanced: -remote is not allowed"}) + ); + } + + #[test] + fn submit_request_accepts_unknown_keys_and_omitted_optionals() { + let r: SubmitRequest = serde_json::from_value(serde_json::json!({ + "program": "blastn", "database": "assembly", "species": ["dmel"], + "sequence": ">query\nACGT\n", "client": "flybase-mcp", "future_key": 1 + })) + .unwrap(); + assert_eq!(r.program, "blastn"); + assert!(r.evalue.is_none() && r.advanced.is_none()); + let dbg = format!("{r:?}"); + assert!(!dbg.contains("ACGT"), "Debug must not print the sequence"); + } + + #[test] + fn status_omits_absent_links_and_truncated() { + let s = JobStatus { + job_id: "abcdefgh".into(), + state: State::Complete, + queue_position: None, + error: None, + result_url: None, + downloads: None, + truncated: true, + }; + assert_eq!( + serde_json::to_value(&s).unwrap(), + serde_json::json!({"job_id": "abcdefgh", "state": "complete"}) + ); + let r = SubmitResponse { + job_id: "abcdefgh".into(), + state: State::Queued, + queue_position: Some(1), + poll_after_seconds: Some(5), + database_release: Some("FB2023_04".into()), + }; + assert_eq!( + serde_json::to_value(&r).unwrap(), + serde_json::json!({"job_id": "abcdefgh", "state": "queued", "queue_position": 1, + "poll_after_seconds": 5, "database_release": "FB2023_04"}) + ); + } + + #[test] + fn program_helpers() { + for p in Program::ALL { + assert_eq!(Program::parse(p.as_str()), Some(p)); + } + assert_eq!(Program::parse("BLASTN"), None); + assert!(Program::Blastx.protein_database() && !Program::Blastx.protein_query()); + assert!(Program::Tblastn.protein_query() && !Program::Tblastn.protein_database()); + assert!(!Program::Blastp.translated() && Program::Tblastx.translated()); + } +} diff --git a/blast-jobs/src/validate.rs b/blast-jobs/src/validate.rs new file mode 100644 index 0000000..7f8683b --- /dev/null +++ b/blast-jobs/src/validate.rs @@ -0,0 +1,570 @@ +//! Service-side validation of a submit (design 3.4), independent of the MCP's own checks. +use crate::advanced; +use crate::catalogue::{Catalogue, Resolved}; +use crate::config::Config; +use crate::types::{Program, Rejection, SubmitRequest}; +use serde::{Deserialize, Serialize}; +use std::fmt; + +/// Most species in one search. +pub const MAX_SPECIES: usize = 20; +pub const MATRICES: &[&str] = &["BLOSUM62", "BLOSUM45", "BLOSUM80", "PAM30", "PAM70"]; +/// Query alphabet for blastn, blastx and tblastx (IUPAC nucleotide codes plus gap). +pub const NUCLEOTIDE: &str = "ACGTUNRYKMSWBDHV-"; +/// Query alphabet for blastp and tblastn: the 20 amino acids plus `BJOUXZ*-`. +pub const PROTEIN: &str = "ACDEFGHIKLMNPQRSTVWYBJOUXZ*-"; +const CLIENT_MAX: usize = 64; + +/// The NCBI genetic codes: 1-6, 9-16, 21-31, 33 (no 7, 8, 17-20, 32). +pub fn genetic_code_known(g: u32) -> bool { + matches!(g, 1..=6 | 9..=16 | 21..=31 | 33) +} + +/// A submit that passed every check, with defaults applied and databases resolved, ready for the +/// store and the command builder. +#[derive(Serialize, Deserialize, Clone, PartialEq)] +pub struct Validated { + pub program: Program, + /// Catalogue category. + pub database: String, + /// Lowercase species codes, deduplicated, in request order. + pub species: Vec, + /// Query residues: uppercase, whitespace and the FASTA header removed. + pub residues: String, + pub evalue: f64, + pub max_hits: u32, + /// `true` only for blastn. + pub megablast: bool, + /// Canonical uppercase matrix name; never set for blastn. + pub matrix: Option, + /// Set only for translated programs (blastx, tblastn, tblastx); ignored and dropped otherwise. + pub genetic_code: Option, + /// Parsed `advanced` pairs (`advanced::parse`): flag without `-`, value `None` for a Switch. + pub advanced: Vec<(String, Option)>, + pub resolved: Resolved, + pub client: Option, +} + +/// Debug without the residues, so a stray `{:?}` never logs a query. +impl fmt::Debug for Validated { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Validated") + .field("program", &self.program) + .field("database", &self.database) + .field("species", &self.species) + .field("residues_len", &self.residues.len()) + .field("evalue", &self.evalue) + .field("max_hits", &self.max_hits) + .field("megablast", &self.megablast) + .field("matrix", &self.matrix) + .field("genetic_code", &self.genetic_code) + .field("advanced", &self.advanced) + .field("resolved", &self.resolved) + .field("client", &self.client) + .finish() + } +} + +/// Validate `req` against the config and catalogue. Checks run in this order: program +/// (`invalid_program`), database category and molecule type (`invalid_database`), species +/// (`invalid_species`), sequence (`invalid_sequence`), then `evalue`, `max_hits`, `matrix`, +/// `megablast`, `genetic_code`, `advanced` and `client` (`invalid_option`, message naming the field). +pub fn check(req: &SubmitRequest, cfg: &Config, cat: &Catalogue) -> Result { + let program = Program::parse(&req.program).ok_or_else(|| { + Rejection::program(format!( + "program {:?} is not one of blastn, blastp, blastx, tblastn, tblastx", + req.program + )) + })?; + + // Category and molecule type are checked before the species, so a wrong database is reported as + // such rather than as every species lacking it. + match cat.category_protein(&req.database) { + None => return Err(cat.unknown_database(&req.database)), + Some(protein) if protein != program.protein_database() => { + let (want, have) = if program.protein_database() { + ("a protein", "nucleotide") + } else { + ("a nucleotide", "protein") + }; + return Err(Rejection::database(format!( + "{program} searches {want} database; {:?} is {have}", + req.database + ))); + } + Some(_) => {} + } + + let species = species(&req.species)?; + let resolved = cat.resolve(&species, &req.database)?; + let residues = sequence(&req.sequence, program, cfg.residue_cap(program))?; + + let evalue = match req.evalue { + None => cfg.default_evalue, + Some(e) if e.is_finite() && e > 0.0 && e <= 1000.0 => e, + Some(e) => { + return Err(Rejection::option(format!( + "evalue {e} must be greater than 0 and at most 1000" + ))); + } + }; + let max_hits = match req.max_hits { + None => cfg.default_max_hits, + Some(n) if (1..=1000).contains(&n) => n, + Some(n) => { + return Err(Rejection::option(format!("max_hits {n} must be 1 to 1000"))); + } + }; + let matrix = match req.matrix.as_deref() { + None => None, + Some(_) if program == Program::Blastn => { + return Err(Rejection::option("matrix does not apply to blastn")); + } + Some(m) => { + let up = m.trim().to_ascii_uppercase(); + if !MATRICES.contains(&up.as_str()) { + return Err(Rejection::option(format!( + "matrix {m:?} is not one of {}", + MATRICES.join(" ") + ))); + } + Some(up) + } + }; + let megablast = req.megablast == Some(true); + if megablast && program != Program::Blastn { + return Err(Rejection::option("megablast applies to blastn only")); + } + let genetic_code = match req.genetic_code { + None => None, + Some(g) if !genetic_code_known(g) => { + return Err(Rejection::option(format!( + "genetic_code {g} is not an NCBI genetic code (1-6, 9-16, 21-31, 33)" + ))); + } + Some(g) => program.translated().then_some(g), + }; + let advanced = match req.advanced.as_deref() { + None => Vec::new(), + Some(a) => advanced::parse(program, a)?, + }; + if megablast + && let Some((_, Some(task))) = advanced.iter().find(|(f, _)| f == "task") + && task != "megablast" + { + return Err(Rejection::option(format!( + "advanced: -task {task} conflicts with megablast: true" + ))); + } + let client = match req.client.as_deref() { + None => None, + Some(c) + if !c.is_empty() + && c.len() <= CLIENT_MAX + && c.bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-')) => + { + Some(c.to_string()) + } + Some(_) => { + return Err(Rejection::option(format!( + "client must be 1 to {CLIENT_MAX} characters of [A-Za-z0-9_.-]" + ))); + } + }; + + Ok(Validated { + program, + database: req.database.clone(), + species, + residues, + evalue, + max_hits, + megablast, + matrix, + genetic_code, + advanced, + resolved, + client, + }) +} + +/// `check` with the argument order the API handler uses. +pub fn validate(cfg: &Config, cat: &Catalogue, req: SubmitRequest) -> Result { + check(&req, cfg, cat) +} + +fn species(raw: &[String]) -> Result, Rejection> { + if raw.is_empty() { + return Err(Rejection::species("give at least one species")); + } + let mut out: Vec = Vec::with_capacity(raw.len()); + for s in raw { + let ok = !s.is_empty() + && s.len() <= 32 + && s.bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'); + if !ok { + return Err(Rejection::species(format!( + "species {s:?} must be a lowercase code such as \"dmel\"" + ))); + } + if !out.contains(s) { + out.push(s.clone()); + } + } + if out.len() > MAX_SPECIES { + return Err(Rejection::species(format!( + "at most {MAX_SPECIES} species per search, got {}", + out.len() + ))); + } + Ok(out) +} + +/// One FASTA record (optional `>` header line) to uppercase residues. +fn sequence(raw: &str, program: Program, cap: usize) -> Result { + let mut lines = raw.lines().skip_while(|l| l.trim().is_empty()).peekable(); + if lines + .peek() + .is_some_and(|l| l.trim_start().starts_with('>')) + { + lines.next(); + } + let mut residues = String::new(); + for line in lines { + if line.trim_start().starts_with('>') { + return Err(Rejection::sequence( + "give exactly one FASTA record; found a second '>' header", + )); + } + residues.extend( + line.chars() + .filter(|c| !c.is_whitespace()) + .map(|c| c.to_ascii_uppercase()), + ); + } + let n = residues.chars().count(); + if n == 0 { + return Err(Rejection::sequence("the query has no residues")); + } + if n > cap { + return Err(Rejection::sequence(format!( + "the query has {n} residues; {program} accepts at most {cap}" + ))); + } + let (alphabet, kind) = if program.protein_query() { + (PROTEIN, "amino-acid") + } else { + (NUCLEOTIDE, "nucleotide") + }; + let mut bad: Vec = Vec::new(); + for c in residues.chars() { + if !alphabet.contains(c) && !bad.contains(&c) { + bad.push(c); + } + } + if !bad.is_empty() { + let shown: Vec = bad.iter().take(10).map(|c| format!("{c:?}")).collect(); + return Err(Rejection::sequence(format!( + "{program} takes a {kind} query; these characters are not {kind} codes: {}", + shown.join(" ") + ))); + } + Ok(residues) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalogue::tests::{fixture, fixture_dir}; + use crate::types::Code; + + fn cfg() -> Config { + serde_json::from_value(crate::config::tests::sample()).unwrap() + } + + fn req(program: &str, database: &str, sequence: &str) -> SubmitRequest { + SubmitRequest { + program: program.into(), + database: database.into(), + species: vec!["dmel".into()], + sequence: sequence.into(), + client: Some("flybase-mcp".into()), + ..Default::default() + } + } + + fn run(r: &SubmitRequest) -> Result { + check(r, &cfg(), &fixture()) + } + + fn rejected(r: &SubmitRequest, code: Code) -> String { + match run(r) { + Ok(v) => panic!("expected {code:?}, got {v:?}"), + Err(e) => { + assert_eq!(e.code, code, "{}", e.message); + e.message + } + } + } + + #[test] + fn happy_path_cleans_and_defaults() { + let v = run(&req("blastn", "assembly", ">query\r\nacgt nn\n\nACGU-\n")).unwrap(); + assert_eq!(v.program, Program::Blastn); + assert_eq!(v.residues, "ACGTNNACGU-"); + assert_eq!(v.evalue, 10.0); + assert_eq!(v.max_hits, 100); + assert!(!v.megablast && v.matrix.is_none() && v.genetic_code.is_none()); + assert!(v.advanced.is_empty()); + assert_eq!(v.client.as_deref(), Some("flybase-mcp")); + assert_eq!( + v.resolved.prefixes, + vec![fixture_dir().join("db/dmel/dmel-assembly")] + ); + assert!(!format!("{v:?}").contains("ACGTNN"), "Debug hides residues"); + // No header line at all is fine too. + assert_eq!( + run(&req("blastn", "assembly", "ACGT")).unwrap().residues, + "ACGT" + ); + } + + #[test] + fn validate_alias_matches_check() { + let r = req("blastp", "translation", "MKV"); + assert_eq!( + validate(&cfg(), &fixture(), r.clone()).unwrap(), + run(&r).unwrap() + ); + } + + #[test] + fn invalid_program() { + for p in ["psiblast", "BLASTN", ""] { + assert!(rejected(&req(p, "assembly", "ACGT"), Code::InvalidProgram).contains("blastn")); + } + } + + #[test] + fn invalid_database() { + let m = rejected(&req("blastn", "est", "ACGT"), Code::InvalidDatabase); + assert!(m.contains("est"), "{m}"); + let m = rejected(&req("blastp", "assembly", "MKV"), Code::InvalidDatabase); + assert!(m.contains("protein database"), "{m}"); + let m = rejected(&req("blastn", "translation", "ACGT"), Code::InvalidDatabase); + assert!(m.contains("nucleotide database"), "{m}"); + rejected(&req("tblastn", "genpept", "MKV"), Code::InvalidDatabase); + rejected(&req("blastx", "genbank", "ACGT"), Code::InvalidDatabase); + } + + #[test] + fn invalid_species() { + let mut r = req("blastn", "assembly", "ACGT"); + r.species = vec![]; + assert!(rejected(&r, Code::InvalidSpecies).contains("at least one")); + r.species = vec!["Dmel".into()]; + assert!(rejected(&r, Code::InvalidSpecies).contains("lowercase")); + r.species = vec!["dmel".into(), "dsim".into()]; + r.database = "transcript".into(); + assert!(rejected(&r, Code::InvalidSpecies).contains("dsim")); + r.database = "assembly".into(); + r.species = (0..21).map(|i| format!("sp{i}")).collect(); + assert!(rejected(&r, Code::InvalidSpecies).contains("at most 20")); + r.species = vec!["dmel".into(), "dyak".into()]; + assert!(rejected(&r, Code::InvalidSpecies).contains("dyak")); + } + + #[test] + fn species_are_deduplicated_in_order() { + let mut r = req("blastn", "assembly", "ACGT"); + r.species = vec!["dsim".into(), "dmel".into(), "dsim".into()]; + let v = run(&r).unwrap(); + assert_eq!(v.species, vec!["dsim", "dmel"]); + assert_eq!(v.resolved.prefixes.len(), 2); + } + + #[test] + fn invalid_sequence() { + for s in ["", " \n", ">query\n", ">query only header"] { + assert!( + rejected(&req("blastn", "assembly", s), Code::InvalidSequence) + .contains("no residues") + ); + } + let m = rejected( + &req("blastn", "assembly", ">a\nACGT\n>b\nACGT\n"), + Code::InvalidSequence, + ); + assert!(m.contains("exactly one FASTA record"), "{m}"); + let m = rejected( + &req("blastn", "assembly", ">q\nMKLEFPQ\n"), + Code::InvalidSequence, + ); + assert!( + m.contains("not nucleotide codes") && m.contains("'E'") && m.contains("'L'"), + "{m}" + ); + assert!(!m.contains("'K'"), "K is an IUPAC nucleotide code: {m}"); + let m = rejected(&req("blastp", "translation", "MKV1"), Code::InvalidSequence); + assert!( + m.contains("not amino-acid codes") && m.contains("'1'"), + "{m}" + ); + rejected(&req("blastn", "assembly", "ACGT*"), Code::InvalidSequence); + rejected(&req("blastn", "assembly", "ACGTé"), Code::InvalidSequence); + } + + #[test] + fn protein_alphabet_extras_are_accepted() { + let v = run(&req("blastp", "translation", "mkvBJOUXZ*-")).unwrap(); + assert_eq!(v.residues, "MKVBJOUXZ*-"); + run(&req("tblastn", "assembly", "MKV*")).unwrap(); + run(&req("blastx", "translation", "ACGTRYKMSWBDHVN")).unwrap(); + } + + #[test] + fn residue_cap_per_program() { + let cases = [ + ("blastn", "assembly", 100_000), + ("blastp", "translation", 100_000), + ("blastx", "translation", 30_000), + ("tblastn", "assembly", 30_000), + ("tblastx", "assembly", 10_000), + ]; + for (p, db, cap) in cases { + run(&req(p, db, &"A".repeat(cap))).unwrap(); + let m = rejected(&req(p, db, &"A".repeat(cap + 1)), Code::InvalidSequence); + assert!( + m.contains(&format!("at most {cap}")) && m.contains(p), + "{m}" + ); + } + // Whitespace and the header do not count toward the cap. + let wrapped = format!(">query\n{}\n", "A\n".repeat(10_000)); + run(&req("tblastx", "assembly", &wrapped)).unwrap(); + } + + #[test] + fn evalue_and_max_hits() { + let mut r = req("blastn", "assembly", "ACGT"); + for e in [0.0, -1.0, 1000.5, f64::NAN, f64::INFINITY] { + r.evalue = Some(e); + assert!(rejected(&r, Code::InvalidOption).contains("evalue")); + } + r.evalue = Some(1000.0); + assert_eq!(run(&r).unwrap().evalue, 1000.0); + r.evalue = Some(1e-30); + assert_eq!(run(&r).unwrap().evalue, 1e-30); + r.evalue = None; + for n in [0, 1001] { + r.max_hits = Some(n); + assert!(rejected(&r, Code::InvalidOption).contains("max_hits")); + } + r.max_hits = Some(1000); + assert_eq!(run(&r).unwrap().max_hits, 1000); + } + + #[test] + fn matrix_rules() { + let mut r = req("blastn", "assembly", "ACGT"); + r.matrix = Some("BLOSUM62".into()); + assert!(rejected(&r, Code::InvalidOption).contains("matrix does not apply to blastn")); + let mut r = req("blastp", "translation", "MKV"); + r.matrix = Some("pam30".into()); + assert_eq!(run(&r).unwrap().matrix.as_deref(), Some("PAM30")); + r.matrix = Some("BLOSUM90".into()); + assert!(rejected(&r, Code::InvalidOption).contains("matrix")); + } + + #[test] + fn megablast_rules() { + let mut r = req("blastp", "translation", "MKV"); + r.megablast = Some(true); + assert!(rejected(&r, Code::InvalidOption).contains("megablast")); + r.megablast = Some(false); + run(&r).unwrap(); + let mut r = req("blastn", "assembly", "ACGT"); + r.megablast = Some(true); + assert!(run(&r).unwrap().megablast); + r.advanced = Some("-task megablast".into()); + run(&r).unwrap(); + r.advanced = Some("-task blastn-short".into()); + assert!(rejected(&r, Code::InvalidOption).contains("conflicts with megablast")); + // megablast:false with -task megablast is an allowed override (Task 3 argv tests). + r.megablast = Some(false); + assert_eq!( + run(&r).unwrap().advanced, + vec![("task".to_string(), Some("blastn-short".to_string()))] + ); + } + + #[test] + fn genetic_code_set() { + let mut r = req("tblastx", "assembly", "ACGT"); + for g in 0..=40u32 { + r.genetic_code = Some(g); + let known = matches!(g, 1..=6 | 9..=16 | 21..=31 | 33); + match run(&r) { + Ok(v) => { + assert!(known, "{g} accepted"); + assert_eq!(v.genetic_code, Some(g)); + } + Err(e) => { + assert!(!known, "{g} rejected"); + assert_eq!(e.code, Code::InvalidOption); + assert!(e.message.contains("genetic_code")); + } + } + } + // Valid but ignored for programs that do not translate. + let mut r = req("blastn", "assembly", "ACGT"); + r.genetic_code = Some(2); + assert_eq!(run(&r).unwrap().genetic_code, None); + let mut r = req("blastp", "translation", "MKV"); + r.genetic_code = Some(2); + assert_eq!(run(&r).unwrap().genetic_code, None); + let mut r = req("blastx", "translation", "ACGT"); + r.genetic_code = Some(2); + assert_eq!(run(&r).unwrap().genetic_code, Some(2)); + } + + #[test] + fn advanced_is_parsed_and_rejections_pass_through() { + let mut r = req("blastn", "assembly", "ACGT"); + r.advanced = Some("-penalty -3 -lcase_masking".into()); + assert_eq!( + run(&r).unwrap().advanced, + vec![ + ("penalty".to_string(), Some("-3".to_string())), + ("lcase_masking".to_string(), None) + ] + ); + r.advanced = Some("-remote".into()); + assert!(rejected(&r, Code::InvalidOption).contains("-remote")); + r.advanced = Some("".into()); + assert!(run(&r).unwrap().advanced.is_empty()); + } + + #[test] + fn client_is_checked() { + let mut r = req("blastn", "assembly", "ACGT"); + r.client = None; + assert_eq!(run(&r).unwrap().client, None); + for c in ["", "has space", &"x".repeat(65), "new\nline"] { + r.client = Some(c.to_string()); + assert!(rejected(&r, Code::InvalidOption).contains("client")); + } + } + + #[test] + fn validated_round_trips_through_json() { + let mut r = req("tblastn", "assembly", "MKV"); + r.genetic_code = Some(11); + r.advanced = Some("-seg no -ungapped".into()); + let v = run(&r).unwrap(); + let back: Validated = serde_json::from_str(&serde_json::to_string(&v).unwrap()).unwrap(); + assert_eq!(back, v); + } +} diff --git a/blast-jobs/tests/fixture/catalogue.json b/blast-jobs/tests/fixture/catalogue.json new file mode 100644 index 0000000..e238dfa --- /dev/null +++ b/blast-jobs/tests/fixture/catalogue.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "databases": [ + { "species": "dmel", "category": "assembly", "label": "D. melanogaster genome assembly", "protein": false, "prefix": "dmel/dmel-assembly", "release": "fixture" }, + { "species": "dmel", "category": "transcript", "label": "D. melanogaster transcripts", "protein": false, "prefix": "dmel/dmel-transcript", "release": "fixture" }, + { "species": "dmel", "category": "translation", "label": "D. melanogaster translations", "protein": true, "prefix": "dmel/dmel-translation", "release": "fixture" }, + { "species": "dmel", "category": "genbank", "label": "D. melanogaster GenBank nucleotide", "protein": false, "prefix": "dmel_misc/dmel-genbank", "release": "fixture" }, + { "species": "dmel", "category": "genpept", "label": "D. melanogaster GenBank protein", "protein": true, "prefix": "dmel_misc/dmel-genpept", "release": "fixture" }, + { "species": "dsim", "category": "assembly", "label": "D. simulans genome assembly", "protein": false, "prefix": "dsim/dsim-assembly", "release": "fixture" } + ] +} diff --git a/blast-jobs/tests/fixture/db/dmel/dmel-assembly.nsq b/blast-jobs/tests/fixture/db/dmel/dmel-assembly.nsq new file mode 100644 index 0000000..e69de29 diff --git a/blast-jobs/tests/fixture/db/dmel/dmel-transcript.nsq b/blast-jobs/tests/fixture/db/dmel/dmel-transcript.nsq new file mode 100644 index 0000000..e69de29 diff --git a/blast-jobs/tests/fixture/db/dmel/dmel-translation.psq b/blast-jobs/tests/fixture/db/dmel/dmel-translation.psq new file mode 100644 index 0000000..e69de29 diff --git a/blast-jobs/tests/fixture/db/dmel_misc/dmel-genbank.nsq b/blast-jobs/tests/fixture/db/dmel_misc/dmel-genbank.nsq new file mode 100644 index 0000000..e69de29 diff --git a/blast-jobs/tests/fixture/db/dmel_misc/dmel-genpept.psq b/blast-jobs/tests/fixture/db/dmel_misc/dmel-genpept.psq new file mode 100644 index 0000000..e69de29 diff --git a/blast-jobs/tests/fixture/db/dsim/dsim-assembly.nsq b/blast-jobs/tests/fixture/db/dsim/dsim-assembly.nsq new file mode 100644 index 0000000..e69de29 From b57e1391ef4a6045747a999c75c065b274173928 Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Mon, 12 Oct 2026 17:43:00 +0000 Subject: [PATCH 02/12] feat(blast-jobs): on-disk job store with restart recovery and retention sweep --- blast-jobs/src/jobid.rs | 99 +++++ blast-jobs/src/lib.rs | 2 + blast-jobs/src/store.rs | 860 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 961 insertions(+) create mode 100644 blast-jobs/src/jobid.rs create mode 100644 blast-jobs/src/store.rs diff --git a/blast-jobs/src/jobid.rs b/blast-jobs/src/jobid.rs new file mode 100644 index 0000000..0f52a14 --- /dev/null +++ b/blast-jobs/src/jobid.rs @@ -0,0 +1,99 @@ +//! Job ids: 16 random bytes as unpadded base64url (22 characters of `[A-Za-z0-9_-]`). `JobId::parse` is +//! the only way a request string becomes a `JobId`, and so the only way a request reaches a path. +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use std::fmt; + +/// Shortest and longest id `parse` accepts (the MCP's own id check uses the same bounds). +pub const MIN_LEN: usize = 8; +pub const MAX_LEN: usize = 64; + +#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct JobId(String); + +impl JobId { + /// A fresh unguessable id from the thread-local CSPRNG. + #[allow(clippy::new_without_default)] + pub fn new() -> JobId { + let bytes: [u8; 16] = rand::random(); + JobId(URL_SAFE_NO_PAD.encode(bytes)) + } + + /// Accepts exactly `[A-Za-z0-9_-]{8,64}`; anything else (including `.`, `/`, `%`, and non-ASCII) + /// is `None`, so a parsed id is always a single safe path component. + pub fn parse(s: &str) -> Option { + let ok = (MIN_LEN..=MAX_LEN).contains(&s.len()) + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + ok.then(|| JobId(s.to_owned())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Display for JobId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl fmt::Debug for JobId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "JobId({})", self.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + #[test] + fn new_ids_have_the_documented_shape_and_parse() { + for _ in 0..200 { + let id = JobId::new(); + assert_eq!(id.as_str().len(), 22, "{id}"); + assert!( + id.as_str() + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'), + "{id}" + ); + assert_eq!(JobId::parse(id.as_str()), Some(id)); + } + } + + #[test] + fn new_ids_are_unique() { + let ids: HashSet = (0..10_000).map(|_| JobId::new().to_string()).collect(); + assert_eq!(ids.len(), 10_000); + } + + #[test] + fn parse_bounds_and_charset() { + assert!(JobId::parse("abcdefgh").is_some()); + assert!(JobId::parse("A-b_9-Z_").is_some()); + assert!(JobId::parse(&"a".repeat(64)).is_some()); + assert!(JobId::parse("abcdefg").is_none(), "7 chars"); + assert!(JobId::parse(&"a".repeat(65)).is_none(), "65 chars"); + assert!(JobId::parse("").is_none()); + for bad in [ + "../../etc/passwd", + "..abcdefgh", + "abcdefgh/..", + "abc/defgh", + "abc\\defgh", + "abcdefgh.", + "abcd%2e%2e", + "abcdefgh\0", + "abcd efgh", + "abcdéfghi", + "abcdefgh\n", + "=abcdefgh", + ] { + assert!(JobId::parse(bad).is_none(), "{bad:?} must be rejected"); + } + } +} diff --git a/blast-jobs/src/lib.rs b/blast-jobs/src/lib.rs index 296f96f..33a5b80 100644 --- a/blast-jobs/src/lib.rs +++ b/blast-jobs/src/lib.rs @@ -4,5 +4,7 @@ pub mod advanced; pub mod catalogue; pub mod config; +pub mod jobid; +pub mod store; pub mod types; pub mod validate; diff --git a/blast-jobs/src/store.rs b/blast-jobs/src/store.rs new file mode 100644 index 0000000..a8af32d --- /dev/null +++ b/blast-jobs/src/store.rs @@ -0,0 +1,860 @@ +//! The job store: an in-memory FIFO plus a map of every known job behind one mutex, a `Notify` that +//! wakes workers, and one directory per job under `state_root/jobs//` (design 3.6). +//! +//! On disk a job directory holds `job.json` (request summary, timestamps, state, error, truncated; +//! rewritten atomically through `job.json.tmp` + rename at every transition), `query.fa`, and what the +//! runner adds (`raw.tsv`, `results.tsv`, `stderr.txt`). The query itself is not persisted in +//! `job.json`: a job that was queued or running when the service stopped is marked `failed` at the next +//! start, never re-run. +//! +//! Blocking: `open`, `create`, `claim_next`, `finish` and `sweep` touch the disk (a few small files and +//! an fsync, never while the mutex is held). `get`, `counts`, `state_writable`, `job_dir` and +//! `notified` are memory-only and safe to call from any async handler. +//! +//! One writer per job: after `claim_next` hands a job out, only that worker calls `finish` for it, so +//! the two `job.json` writes of a job never race. `sweep` only touches terminal jobs. +use crate::jobid::JobId; +use crate::types::{JobStatus, Program, State}; +use crate::validate::Validated; +use anyhow::Context; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, VecDeque}; +use std::ffi::OsStr; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::sync::Notify; + +/// The error a queued or running job gets when the service finds it on disk at startup. +pub const RESTART_MESSAGE: &str = "service restarted before the search finished"; +/// How often `spawn_sweeper` callers should run the retention sweep (design 3.6). +pub const SWEEP_INTERVAL: Duration = Duration::from_secs(300); + +const JOBS_DIR: &str = "jobs"; +const JOB_FILE: &str = "job.json"; +const JOB_TMP: &str = "job.json.tmp"; +const QUERY_FILE: &str = "query.fa"; +const PROBE_FILE: &str = ".write-probe"; + +/// Why `Store::create` refused a job. The API maps `QueueFull` to `429 queue_full` and `Io` to `503`. +#[derive(Debug, thiserror::Error)] +pub enum CreateError { + #[error("the queue is full")] + QueueFull, + #[error("the job directory could not be written: {0}")] + Io(#[from] io::Error), +} + +/// How a running job ended, as the runner reports it to `Store::finish`. +#[derive(Debug, Clone, PartialEq)] +pub enum Outcome { + Complete { + truncated: bool, + }, + /// The user-facing error message (already free of paths). + Failed(String), +} + +/// `job.json`. Never contains the query residues. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +struct Record { + job_id: String, + state: State, + submitted_at_ms: u64, + #[serde(default)] + started_at_ms: Option, + #[serde(default)] + finished_at_ms: Option, + #[serde(default)] + error: Option, + #[serde(default)] + truncated: bool, + program: Program, + database: String, + species: Vec, + query_length: usize, + #[serde(default)] + client: Option, +} + +struct Job { + record: Record, + /// Present while queued; taken by `claim_next`. Jobs reloaded from disk never have one. + request: Option, +} + +#[derive(Default)] +struct Inner { + jobs: HashMap, + fifo: VecDeque, + /// Slots taken by `create` calls that are writing their directory outside the lock. + reserved: usize, +} + +pub struct Store { + jobs_root: PathBuf, + max_queued: usize, + inner: Mutex, + notify: Notify, + writable: AtomicBool, +} + +impl Store { + /// Opens (creating if needed) `state_root/jobs`, records whether it is writable, reloads `complete` + /// and `failed` jobs, and marks `queued` and `running` ones `failed` with `RESTART_MESSAGE`. + /// Entries that are not a job directory with a readable `job.json` naming itself are left alone and + /// logged here, once. An unwritable directory is not an error (health reports it); an existing + /// directory that cannot be listed is. + pub fn open(state_root: &Path, max_queued: usize) -> anyhow::Result> { + let jobs_root = state_root.join(JOBS_DIR); + let writable = match std::fs::create_dir_all(&jobs_root).and_then(|()| probe(&jobs_root)) { + Ok(()) => true, + Err(e) => { + tracing::error!(path = %jobs_root.display(), error = %e, "job directory is not writable"); + false + } + }; + let store = Store { + jobs_root, + max_queued, + inner: Mutex::new(Inner::default()), + notify: Notify::new(), + writable: AtomicBool::new(writable), + }; + + let entries = match std::fs::read_dir(&store.jobs_root) { + Ok(rd) => Some(rd), + Err(e) if e.kind() == io::ErrorKind::NotFound => None, + Err(e) => { + return Err(e).with_context(|| format!("listing {}", store.jobs_root.display())); + } + }; + let (mut loaded, mut recovered) = (0usize, 0usize); + let now = now_ms(SystemTime::now()); + let mut jobs = HashMap::new(); + for entry in entries.into_iter().flatten() { + let entry = entry.with_context(|| format!("listing {}", store.jobs_root.display()))?; + let name = entry.file_name(); + if name == PROBE_FILE { + continue; + } + let Some((id, mut record)) = load_entry(&entry, &name) else { + tracing::warn!(entry = %name.to_string_lossy(), "leaving unknown entry in jobs/ alone"); + continue; + }; + if matches!(record.state, State::Queued | State::Running) { + record.state = State::Failed; + record.error = Some(RESTART_MESSAGE.to_owned()); + record.finished_at_ms = Some(now); + store.persist(&id, &record); + recovered += 1; + } + loaded += 1; + jobs.insert( + id, + Job { + record, + request: None, + }, + ); + } + store.lock().jobs = jobs; + tracing::info!( + loaded, + recovered, + writable, + path = %store.jobs_root.display(), + "job store opened" + ); + Ok(Arc::new(store)) + } + + /// Queues a validated request: writes `query.fa` and `job.json`, pushes it on the FIFO and wakes a + /// worker. Returns the id and the 1-based queue position. `QueueFull` when the FIFO already holds + /// `max_queued` jobs (running ones do not count); `Io` when the directory cannot be written, which + /// also turns `state_writable` false until a later write succeeds. + pub fn create(&self, req: Validated) -> Result<(JobId, u32), CreateError> { + { + let mut inner = self.lock(); + if inner.fifo.len() + inner.reserved >= self.max_queued { + return Err(CreateError::QueueFull); + } + inner.reserved += 1; + } + let id = JobId::new(); + let record = Record { + job_id: id.to_string(), + state: State::Queued, + submitted_at_ms: now_ms(SystemTime::now()), + started_at_ms: None, + finished_at_ms: None, + error: None, + truncated: false, + program: req.program, + database: req.database.clone(), + species: req.species.clone(), + query_length: req.residues.len(), + client: req.client.clone(), + }; + let written = self.write_new_job(&id, &req, &record); + self.writable.store(written.is_ok(), Ordering::Relaxed); + let mut inner = self.lock(); + inner.reserved -= 1; + if let Err(e) = written { + tracing::error!(error = %e, "could not write a new job directory"); + return Err(CreateError::Io(e)); + } + inner.jobs.insert( + id.clone(), + Job { + record, + request: Some(req), + }, + ); + inner.fifo.push_back(id.clone()); + let position = u32::try_from(inner.fifo.len()).unwrap_or(u32::MAX); + drop(inner); + // `notify_one` stores a permit when no worker is waiting yet, so a worker that found the FIFO + // empty just before this push still wakes on its next `notified().await`. + self.notify.notify_one(); + Ok((id, position)) + } + + /// Pops the oldest queued job, marks it `running` (memory and `job.json`) and hands its request to + /// the caller. `None` when the FIFO is empty. Workers loop: `claim_next()`, else + /// `notified().await`, then try again. + pub fn claim_next(&self) -> Option<(JobId, Validated)> { + let (id, req, snapshot) = { + let mut inner = self.lock(); + loop { + let id = inner.fifo.pop_front()?; + let Some(job) = inner.jobs.get_mut(&id) else { + continue; + }; + let Some(req) = job.request.take() else { + continue; + }; + job.record.state = State::Running; + job.record.started_at_ms = Some(now_ms(SystemTime::now())); + break (id, req, job.record.clone()); + } + }; + self.persist(&id, &snapshot); + Some((id, req)) + } + + /// Resolves when `create` has queued a job (or immediately, if one was queued since the last + /// wake-up). Spurious wake-ups are possible; `claim_next` decides. + pub fn notified(&self) -> tokio::sync::futures::Notified<'_> { + self.notify.notified() + } + + /// The contract's status for a known job: `queue_position` 0 while running, the 1-based FIFO + /// position while queued, absent once terminal. `truncated` is set for a cut `results.tsv`. + pub fn get(&self, id: &JobId) -> Option { + let inner = self.lock(); + let job = inner.jobs.get(id)?; + let queue_position = match job.record.state { + State::Running => Some(0), + State::Queued => inner + .fifo + .iter() + .position(|j| j == id) + .map(|p| u32::try_from(p + 1).unwrap_or(u32::MAX)), + _ => None, + }; + Some(JobStatus { + job_id: id.to_string(), + state: job.record.state, + queue_position, + error: job.record.error.clone(), + result_url: None, + downloads: None, + truncated: job.record.truncated, + }) + } + + /// `state_root/jobs/`. Only a parsed `JobId` can name it, so the result never leaves `jobs/`. + pub fn job_dir(&self, id: &JobId) -> PathBuf { + self.jobs_root.join(id.as_str()) + } + + /// Records the end of a job and rewrites its `job.json`. A job already terminal or unknown (swept) + /// is left as it is. + pub fn finish(&self, id: &JobId, outcome: Outcome) { + let snapshot = { + let mut inner = self.lock(); + let Some(job) = inner.jobs.get_mut(id) else { + tracing::warn!(job_id = %id, "finish for an unknown job ignored"); + return; + }; + let was_queued = match job.record.state { + State::Queued => true, + State::Running => false, + _ => { + tracing::warn!(job_id = %id, "finish for a job already finished ignored"); + return; + } + }; + match outcome { + Outcome::Complete { truncated } => { + job.record.state = State::Complete; + job.record.truncated = truncated; + job.record.error = None; + } + Outcome::Failed(message) => { + job.record.state = State::Failed; + job.record.truncated = false; + job.record.error = Some(message); + } + } + job.record.finished_at_ms = Some(now_ms(SystemTime::now())); + job.request = None; + let snapshot = job.record.clone(); + if was_queued { + inner.fifo.retain(|j| j != id); + } + snapshot + }; + self.persist(id, &snapshot); + } + + /// Removes terminal jobs submitted more than `retention` before `now`, from memory and disk. + /// Queued and running jobs are never removed, and directories the store does not know (logged + /// by `open`) are never touched. Returns the number of jobs removed. + pub fn sweep(&self, now: SystemTime, retention: Duration) -> usize { + let cutoff = + now_ms(now).saturating_sub(u64::try_from(retention.as_millis()).unwrap_or(u64::MAX)); + let expired: Vec = { + let mut inner = self.lock(); + let ids: Vec = inner + .jobs + .iter() + .filter(|(_, j)| { + matches!( + j.record.state, + State::Complete | State::Failed | State::Cancelled + ) && j.record.submitted_at_ms < cutoff + }) + .map(|(id, _)| id.clone()) + .collect(); + for id in &ids { + inner.jobs.remove(id); + } + ids + }; + for id in &expired { + match std::fs::remove_dir_all(self.job_dir(id)) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => { + // Still terminal on disk: the next start reloads it and a later sweep retries. + tracing::warn!(job_id = %id, error = %e, "could not remove an expired job directory"); + } + } + } + if !expired.is_empty() { + tracing::info!(removed = expired.len(), "retention sweep"); + } + expired.len() + } + + /// `(queued, running)`. + pub fn counts(&self) -> (usize, usize) { + let inner = self.lock(); + let running = inner + .jobs + .values() + .filter(|j| j.record.state == State::Running) + .count(); + (inner.fifo.len(), running) + } + + /// Whether the last write to the state directory (the startup probe, then every job write) + /// succeeded. + pub fn state_writable(&self) -> bool { + self.writable.load(Ordering::Relaxed) + } + + /// Runs `sweep(now, retention)` every `every` (use `SWEEP_INTERVAL`) on the blocking pool, until + /// the returned task is aborted. The first sweep runs after one interval. + pub fn spawn_sweeper( + self: &Arc, + every: Duration, + retention: Duration, + ) -> tokio::task::JoinHandle<()> { + let store = Arc::clone(self); + tokio::spawn(async move { + let mut tick = tokio::time::interval_at(tokio::time::Instant::now() + every, every); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tick.tick().await; + let s = Arc::clone(&store); + if let Err(e) = + tokio::task::spawn_blocking(move || s.sweep(SystemTime::now(), retention)).await + { + tracing::error!(error = %e, "retention sweep panicked"); + } + } + }) + } + + fn lock(&self) -> MutexGuard<'_, Inner> { + // A panic while holding the lock leaves plain data behind; keep serving. + self.inner.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn write_new_job(&self, id: &JobId, req: &Validated, record: &Record) -> io::Result<()> { + std::fs::create_dir_all(&self.jobs_root)?; + let dir = self.job_dir(id); + // `create_dir`, not `_all`: an existing directory is an error, never reused or removed. + std::fs::create_dir(&dir)?; + let res = (|| { + let mut q = std::fs::File::create(dir.join(QUERY_FILE))?; + q.write_all(b">query\n")?; + q.write_all(req.residues.as_bytes())?; + q.write_all(b"\n")?; + q.sync_all()?; + write_record(&dir, record) + })(); + if res.is_err() { + let _ = std::fs::remove_dir_all(&dir); + } + res + } + + fn persist(&self, id: &JobId, record: &Record) { + let res = write_record(&self.job_dir(id), record); + self.writable.store(res.is_ok(), Ordering::Relaxed); + if let Err(e) = res { + tracing::error!(job_id = %id, error = %e, "could not write job.json"); + } + } +} + +/// Writes `job.json` atomically: the whole record to `job.json.tmp`, fsync, rename over `job.json`. +fn write_record(dir: &Path, record: &Record) -> io::Result<()> { + let tmp = dir.join(JOB_TMP); + let body = serde_json::to_vec_pretty(record).map_err(io::Error::other)?; + let mut f = std::fs::File::create(&tmp)?; + f.write_all(&body)?; + f.sync_all()?; + drop(f); + std::fs::rename(&tmp, dir.join(JOB_FILE)) +} + +fn probe(jobs_root: &Path) -> io::Result<()> { + let p = jobs_root.join(PROBE_FILE); + std::fs::write(&p, b"ok")?; + std::fs::remove_file(&p) +} + +/// A job directory: a real directory (not a symlink) whose name parses as a `JobId` and whose +/// `job.json` parses and names the same id. +fn load_entry(entry: &std::fs::DirEntry, name: &OsStr) -> Option<(JobId, Record)> { + let id = JobId::parse(name.to_str()?)?; + if !entry.file_type().ok()?.is_dir() { + return None; + } + let bytes = std::fs::read(entry.path().join(JOB_FILE)).ok()?; + let record: Record = serde_json::from_slice(&bytes).ok()?; + (record.job_id == id.as_str()).then_some((id, record)) +} + +fn now_ms(t: SystemTime) -> u64 { + t.duration_since(UNIX_EPOCH) + .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalogue::Resolved; + use std::os::unix::fs::PermissionsExt; + + fn validated(residues: &str) -> Validated { + Validated { + program: Program::Blastn, + database: "assembly".into(), + species: vec!["dmel".into()], + residues: residues.into(), + evalue: 10.0, + max_hits: 100, + megablast: false, + matrix: None, + genetic_code: None, + advanced: vec![], + resolved: Resolved { + protein: false, + prefixes: vec![PathBuf::from("/db/dmel/dmel-all-chromosome")], + }, + client: Some("test".into()), + } + } + + fn record_on_disk(store_root: &Path, id: &str, state: State, submitted: SystemTime) { + let dir = store_root.join(JOBS_DIR).join(id); + std::fs::create_dir_all(&dir).unwrap(); + let r = Record { + job_id: id.into(), + state, + submitted_at_ms: now_ms(submitted), + started_at_ms: None, + finished_at_ms: None, + error: None, + truncated: false, + program: Program::Blastp, + database: "translation".into(), + species: vec!["dmel".into()], + query_length: 10, + client: None, + }; + std::fs::write(dir.join(JOB_FILE), serde_json::to_vec(&r).unwrap()).unwrap(); + } + + fn read_record(store: &Store, id: &JobId) -> Record { + serde_json::from_slice(&std::fs::read(store.job_dir(id).join(JOB_FILE)).unwrap()).unwrap() + } + + #[test] + fn create_writes_query_and_job_json() { + let tmp = tempfile::tempdir().unwrap(); + let store = Store::open(tmp.path(), 5).unwrap(); + assert!(store.state_writable()); + let (id, pos) = store.create(validated("ACGTACGT")).unwrap(); + assert_eq!(pos, 1); + let dir = store.job_dir(&id); + assert_eq!(dir, tmp.path().join("jobs").join(id.as_str())); + assert_eq!( + std::fs::read_to_string(dir.join(QUERY_FILE)).unwrap(), + ">query\nACGTACGT\n" + ); + let r = read_record(&store, &id); + assert_eq!(r.state, State::Queued); + assert_eq!(r.job_id, id.as_str()); + assert_eq!(r.query_length, 8); + let raw = std::fs::read_to_string(dir.join(JOB_FILE)).unwrap(); + assert!(!raw.contains("ACGTACGT"), "job.json must not hold residues"); + assert!(!dir.join(JOB_TMP).exists()); + // The probe file is cleaned up. + assert!(!tmp.path().join("jobs").join(PROBE_FILE).exists()); + } + + #[test] + fn fifo_positions_queue_full_and_claim_shifts() { + let tmp = tempfile::tempdir().unwrap(); + let store = Store::open(tmp.path(), 3).unwrap(); + let (a, pa) = store.create(validated("AAAA")).unwrap(); + let (b, pb) = store.create(validated("CCCC")).unwrap(); + let (c, pc) = store.create(validated("GGGG")).unwrap(); + assert_eq!((pa, pb, pc), (1, 2, 3)); + assert!(matches!( + store.create(validated("TTTT")), + Err(CreateError::QueueFull) + )); + assert_eq!(store.counts(), (3, 0)); + for (id, p) in [(&a, 1), (&b, 2), (&c, 3)] { + let s = store.get(id).unwrap(); + assert_eq!((s.state, s.queue_position), (State::Queued, Some(p))); + } + + let (claimed, req) = store.claim_next().unwrap(); + assert_eq!(claimed, a, "FIFO order"); + assert_eq!(req.residues, "AAAA"); + let s = store.get(&a).unwrap(); + assert_eq!((s.state, s.queue_position), (State::Running, Some(0))); + assert_eq!(read_record(&store, &a).state, State::Running); + assert!(read_record(&store, &a).started_at_ms.is_some()); + assert_eq!(store.get(&b).unwrap().queue_position, Some(1)); + assert_eq!(store.get(&c).unwrap().queue_position, Some(2)); + assert_eq!(store.counts(), (2, 1)); + + // A running job frees its queue slot. + let (d, pd) = store.create(validated("TTTT")).unwrap(); + assert_eq!(pd, 3); + assert!(matches!( + store.create(validated("TTTT")), + Err(CreateError::QueueFull) + )); + + assert_eq!(store.claim_next().unwrap().0, b); + assert_eq!(store.claim_next().unwrap().0, c); + assert_eq!(store.claim_next().unwrap().0, d); + assert!(store.claim_next().is_none()); + assert_eq!(store.counts(), (0, 4)); + } + + #[test] + fn finish_records_outcome_and_survives_reopen() { + let tmp = tempfile::tempdir().unwrap(); + let store = Store::open(tmp.path(), 5).unwrap(); + let (ok, _) = store.create(validated("AAAA")).unwrap(); + let (bad, _) = store.create(validated("CCCC")).unwrap(); + store.claim_next().unwrap(); + store.claim_next().unwrap(); + store.finish(&ok, Outcome::Complete { truncated: true }); + store.finish(&bad, Outcome::Failed("BLAST failed (exit 2)".into())); + // A second finish does not overwrite the first. + store.finish(&ok, Outcome::Failed("late".into())); + assert_eq!(store.counts(), (0, 0)); + + let s = store.get(&ok).unwrap(); + assert_eq!( + (s.state, s.queue_position, s.truncated, s.error.as_deref()), + (State::Complete, None, true, None) + ); + for id in [&ok, &bad] { + assert!(!store.job_dir(id).join(JOB_TMP).exists()); + } + drop(store); + + let store = Store::open(tmp.path(), 5).unwrap(); + let s = store.get(&ok).unwrap(); + assert_eq!((s.state, s.truncated), (State::Complete, true)); + let s = store.get(&bad).unwrap(); + assert_eq!(s.state, State::Failed); + assert_eq!(s.error.as_deref(), Some("BLAST failed (exit 2)")); + assert!( + store.claim_next().is_none(), + "terminal jobs are not requeued" + ); + } + + #[test] + fn finish_on_a_queued_job_removes_it_from_the_fifo() { + let tmp = tempfile::tempdir().unwrap(); + let store = Store::open(tmp.path(), 5).unwrap(); + let (a, _) = store.create(validated("AAAA")).unwrap(); + let (b, _) = store.create(validated("CCCC")).unwrap(); + store.finish(&a, Outcome::Failed("x".into())); + assert_eq!(store.get(&b).unwrap().queue_position, Some(1)); + assert_eq!(store.claim_next().unwrap().0, b); + assert!(store.get(&JobId::new()).is_none()); + } + + #[test] + fn restart_fails_queued_and_running_jobs() { + let tmp = tempfile::tempdir().unwrap(); + let store = Store::open(tmp.path(), 5).unwrap(); + let (running, _) = store.create(validated("AAAA")).unwrap(); + let (queued, _) = store.create(validated("CCCC")).unwrap(); + let (done, _) = store.create(validated("GGGG")).unwrap(); + assert_eq!(store.claim_next().unwrap().0, running); + store.finish(&done, Outcome::Complete { truncated: false }); + assert_eq!(read_record(&store, &queued).state, State::Queued); + // The process dies here: `running` is running and `queued` queued on disk. + drop(store); + + let store = Store::open(tmp.path(), 5).unwrap(); + for id in [&running, &queued] { + let s = store.get(id).unwrap(); + assert_eq!(s.state, State::Failed, "{id}"); + assert_eq!(s.error.as_deref(), Some(RESTART_MESSAGE)); + assert_eq!(s.queue_position, None); + let on_disk = read_record(&store, id); + assert_eq!(on_disk.state, State::Failed, "recovery is persisted"); + assert_eq!(on_disk.error.as_deref(), Some(RESTART_MESSAGE)); + assert!(on_disk.finished_at_ms.is_some()); + } + assert_eq!(store.get(&done).unwrap().state, State::Complete); + assert_eq!(store.counts(), (0, 0)); + assert!(store.claim_next().is_none()); + } + + #[test] + fn open_leaves_foreign_entries_alone() { + let tmp = tempfile::tempdir().unwrap(); + let jobs = tmp.path().join("jobs"); + std::fs::create_dir_all(jobs.join("not a job")).unwrap(); + std::fs::create_dir_all(jobs.join("validname_nojson")).unwrap(); + std::fs::create_dir_all(jobs.join("validname_badjson")).unwrap(); + std::fs::write(jobs.join("validname_badjson").join(JOB_FILE), b"{not json").unwrap(); + // job.json naming another id. + record_on_disk( + tmp.path(), + "validname_mismatch", + State::Complete, + SystemTime::now(), + ); + let mut r: Record = serde_json::from_slice( + &std::fs::read(jobs.join("validname_mismatch").join(JOB_FILE)).unwrap(), + ) + .unwrap(); + r.job_id = "someoneelse".into(); + std::fs::write( + jobs.join("validname_mismatch").join(JOB_FILE), + serde_json::to_vec(&r).unwrap(), + ) + .unwrap(); + std::fs::write(jobs.join("stray-file.txt"), b"x").unwrap(); + + let store = Store::open(tmp.path(), 5).unwrap(); + for name in [ + "validname_nojson", + "validname_badjson", + "validname_mismatch", + ] { + assert!(store.get(&JobId::parse(name).unwrap()).is_none(), "{name}"); + } + assert_eq!(store.counts(), (0, 0)); + assert_eq!( + store.sweep( + SystemTime::now() + Duration::from_secs(1 << 30), + Duration::ZERO + ), + 0 + ); + for name in [ + "not a job", + "validname_nojson", + "validname_badjson", + "validname_mismatch", + "stray-file.txt", + ] { + assert!(jobs.join(name).exists(), "{name} must be left alone"); + } + } + + #[test] + fn sweep_removes_only_expired_terminal_jobs() { + let tmp = tempfile::tempdir().unwrap(); + let day = Duration::from_secs(86_400); + let now = SystemTime::now(); + record_on_disk(tmp.path(), "old_complete", State::Complete, now - 2 * day); + record_on_disk(tmp.path(), "old_failed__", State::Failed, now - 2 * day); + record_on_disk(tmp.path(), "young_complete", State::Complete, now - day / 2); + std::fs::create_dir_all(tmp.path().join("jobs").join("foreign-dir")).unwrap(); + let store = Store::open(tmp.path(), 5).unwrap(); + // Fresh jobs: one running, one queued; never swept however old `now` makes them. + let (running, _) = store.create(validated("AAAA")).unwrap(); + store.claim_next().unwrap(); + let (queued, _) = store.create(validated("CCCC")).unwrap(); + + assert_eq!(store.sweep(now, day), 2); + let jobs = tmp.path().join("jobs"); + assert!(!jobs.join("old_complete").exists()); + assert!(!jobs.join("old_failed__").exists()); + assert!(store.get(&JobId::parse("old_complete").unwrap()).is_none()); + assert!(jobs.join("young_complete").exists()); + assert!( + store + .get(&JobId::parse("young_complete").unwrap()) + .is_some() + ); + assert!(jobs.join("foreign-dir").exists()); + + let far = now + 30 * day; + assert_eq!(store.sweep(far, day), 1, "young_complete only"); + assert!(!jobs.join("young_complete").exists()); + assert!(jobs.join("foreign-dir").exists()); + assert!(store.job_dir(&running).exists() && store.job_dir(&queued).exists()); + assert_eq!(store.get(&queued).unwrap().state, State::Queued); + assert_eq!(store.get(&running).unwrap().state, State::Running); + + store.finish(&running, Outcome::Complete { truncated: false }); + assert_eq!(store.sweep(far, day), 1); + assert!(!store.job_dir(&running).exists()); + assert!(store.get(&running).is_none()); + // `finish` after a sweep is ignored rather than resurrecting the directory. + store.finish(&running, Outcome::Failed("late".into())); + assert!(!store.job_dir(&running).exists()); + } + + #[test] + fn read_only_directory_gives_io_and_flips_writable() { + let tmp = tempfile::tempdir().unwrap(); + let jobs = tmp.path().join("jobs"); + std::fs::create_dir_all(&jobs).unwrap(); + std::fs::set_permissions(&jobs, std::fs::Permissions::from_mode(0o555)).unwrap(); + if std::fs::write(jobs.join("x"), b"x").is_ok() { + // Running as root: permissions are not enforced, so this case cannot be produced here. + std::fs::set_permissions(&jobs, std::fs::Permissions::from_mode(0o755)).unwrap(); + eprintln!("skipped: permissions are not enforced for this user"); + return; + } + let store = Store::open(tmp.path(), 5).unwrap(); + assert!(!store.state_writable()); + assert!(matches!( + store.create(validated("AAAA")), + Err(CreateError::Io(_)) + )); + assert!(!store.state_writable()); + assert_eq!(store.counts(), (0, 0), "a failed create takes no slot"); + + std::fs::set_permissions(&jobs, std::fs::Permissions::from_mode(0o755)).unwrap(); + let (id, pos) = store.create(validated("AAAA")).unwrap(); + assert_eq!(pos, 1); + assert!(store.state_writable(), "a successful write clears the flag"); + assert!(store.job_dir(&id).join(JOB_FILE).exists()); + } + + #[test] + fn open_creates_missing_state_root() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("a").join("b"); + let store = Store::open(&root, 5).unwrap(); + assert!(store.state_writable()); + assert!(root.join("jobs").is_dir()); + } + + #[tokio::test] + async fn create_wakes_a_waiting_worker_and_never_loses_a_wakeup() { + let tmp = tempfile::tempdir().unwrap(); + let store = Store::open(tmp.path(), 5).unwrap(); + + // A worker already waiting is woken. + let s = Arc::clone(&store); + let worker = tokio::spawn(async move { + loop { + if let Some((id, _)) = s.claim_next() { + return id; + } + s.notified().await; + } + }); + tokio::task::yield_now().await; + let (id, _) = store.create(validated("AAAA")).unwrap(); + let got = tokio::time::timeout(Duration::from_secs(5), worker) + .await + .expect("worker woke") + .unwrap(); + assert_eq!(got, id); + + // A worker that saw an empty FIFO just before the push still wakes (stored permit). + assert!(store.claim_next().is_none()); + store.create(validated("CCCC")).unwrap(); + tokio::time::timeout(Duration::from_secs(5), store.notified()) + .await + .expect("permit was stored"); + assert!(store.claim_next().is_some()); + } + + #[tokio::test] + async fn sweeper_runs_on_its_interval() { + let tmp = tempfile::tempdir().unwrap(); + record_on_disk( + tmp.path(), + "old_complete", + State::Complete, + SystemTime::now() - Duration::from_secs(10 * 86_400), + ); + let store = Store::open(tmp.path(), 5).unwrap(); + let every = Duration::from_millis(200); + let handle = store.spawn_sweeper(every, Duration::from_secs(86_400)); + let id = JobId::parse("old_complete").unwrap(); + assert!( + store.get(&id).is_some(), + "no sweep before the first interval" + ); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while store.get(&id).is_some() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert!(store.get(&id).is_none(), "swept after one interval"); + assert!(!store.job_dir(&id).exists()); + handle.abort(); + } +} From 877ef1af5727b024a413b47539d04694ec76e1da Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Mon, 12 Oct 2026 17:43:43 +0000 Subject: [PATCH 03/12] fix(blast-jobs): clip echoed request strings in rejections; bound species dedup --- blast-jobs/src/catalogue.rs | 4 +- blast-jobs/src/validate.rs | 85 +++++++++++++++++++++++++++++++------ 2 files changed, 74 insertions(+), 15 deletions(-) diff --git a/blast-jobs/src/catalogue.rs b/blast-jobs/src/catalogue.rs index 28af96c..015b5b0 100644 --- a/blast-jobs/src/catalogue.rs +++ b/blast-jobs/src/catalogue.rs @@ -2,6 +2,7 @@ //! (design 3.2, 3.3). Any JSON file with a `databases` array of `{species, category, protein, prefix}` //! works; extra fields are ignored. use crate::types::Rejection; +use crate::validate::quoted; use anyhow::{Context, bail}; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; @@ -127,7 +128,8 @@ impl Catalogue { pub fn unknown_database(&self, category: &str) -> Rejection { let known: Vec<&str> = self.categories.keys().map(String::as_str).collect(); Rejection::database(format!( - "database {category:?} is not available; available: {}", + "database {} is not available; available: {}", + quoted(category), known.join(", ") )) } diff --git a/blast-jobs/src/validate.rs b/blast-jobs/src/validate.rs index 7f8683b..1572312 100644 --- a/blast-jobs/src/validate.rs +++ b/blast-jobs/src/validate.rs @@ -4,6 +4,7 @@ use crate::catalogue::{Catalogue, Resolved}; use crate::config::Config; use crate::types::{Program, Rejection, SubmitRequest}; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; use std::fmt; /// Most species in one search. @@ -14,6 +15,17 @@ pub const NUCLEOTIDE: &str = "ACGTUNRYKMSWBDHV-"; /// Query alphabet for blastp and tblastn: the 20 amino acids plus `BJOUXZ*-`. pub const PROTEIN: &str = "ACDEFGHIKLMNPQRSTVWYBJOUXZ*-"; const CLIENT_MAX: usize = 64; +/// Longest piece of a caller's string echoed back in a rejection message, in characters. +pub const ECHO_MAX: usize = 40; + +/// `s` quoted for a rejection message, cut to `ECHO_MAX` characters (on a char boundary) with `…` +/// appended when longer, so a huge field never produces a huge error body. +pub fn quoted(s: &str) -> String { + match s.char_indices().nth(ECHO_MAX) { + None => format!("{s:?}"), + Some((cut, _)) => format!("{:?}…", &s[..cut]), + } +} /// The NCBI genetic codes: 1-6, 9-16, 21-31, 33 (no 7, 8, 17-20, 32). pub fn genetic_code_known(g: u32) -> bool { @@ -72,8 +84,8 @@ impl fmt::Debug for Validated { pub fn check(req: &SubmitRequest, cfg: &Config, cat: &Catalogue) -> Result { let program = Program::parse(&req.program).ok_or_else(|| { Rejection::program(format!( - "program {:?} is not one of blastn, blastp, blastx, tblastn, tblastx", - req.program + "program {} is not one of blastn, blastp, blastx, tblastn, tblastx", + quoted(&req.program) )) })?; @@ -88,8 +100,8 @@ pub fn check(req: &SubmitRequest, cfg: &Config, cat: &Catalogue) -> Result {} @@ -124,7 +136,8 @@ pub fn check(req: &SubmitRequest, cfg: &Config, cat: &Catalogue) -> Result Result, Rejection> { if raw.is_empty() { return Err(Rejection::species("give at least one species")); } - let mut out: Vec = Vec::with_capacity(raw.len()); + let mut seen = HashSet::new(); + let mut out: Vec = Vec::new(); for s in raw { let ok = !s.is_empty() && s.len() <= 32 @@ -206,19 +220,20 @@ fn species(raw: &[String]) -> Result, Rejection> { .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_'); if !ok { return Err(Rejection::species(format!( - "species {s:?} must be a lowercase code such as \"dmel\"" + "species {} must be a lowercase code such as \"dmel\"", + quoted(s) ))); } - if !out.contains(s) { + if seen.insert(s.as_str()) { out.push(s.clone()); + // Stop at the first species over the cap, so a huge list costs no more than 21 entries. + if out.len() > MAX_SPECIES { + return Err(Rejection::species(format!( + "at most {MAX_SPECIES} species per search" + ))); + } } } - if out.len() > MAX_SPECIES { - return Err(Rejection::species(format!( - "at most {MAX_SPECIES} species per search, got {}", - out.len() - ))); - } Ok(out) } @@ -376,6 +391,48 @@ mod tests { assert!(rejected(&r, Code::InvalidSpecies).contains("dyak")); } + #[test] + fn echoed_strings_are_clipped() { + let huge = "x".repeat(500_000); + let m = rejected(&req(&huge, "assembly", "ACGT"), Code::InvalidProgram); + assert!(m.len() < 200, "message is {} bytes", m.len()); + assert!(m.contains(&format!("{:?}…", "x".repeat(ECHO_MAX))), "{m}"); + let m = rejected(&req("blastn", &huge, "ACGT"), Code::InvalidDatabase); + assert!(m.len() < 300, "message is {} bytes", m.len()); + let mut r = req("blastn", "assembly", "ACGT"); + r.species = vec![huge.clone()]; + assert!(rejected(&r, Code::InvalidSpecies).len() < 200); + let mut r = req("blastp", "translation", "MKV"); + r.matrix = Some(huge.clone()); + assert!(rejected(&r, Code::InvalidOption).len() < 200); + // Cut on a char boundary for multi-byte input; short strings are untouched. + assert_eq!( + quoted(&"é".repeat(50)), + format!("{:?}…", "é".repeat(ECHO_MAX)) + ); + assert_eq!(quoted("blastq"), "\"blastq\""); + assert_eq!( + quoted(&"y".repeat(ECHO_MAX)), + format!("{:?}", "y".repeat(ECHO_MAX)) + ); + } + + #[test] + fn huge_species_list_is_rejected_fast() { + let mut r = req("blastn", "assembly", "ACGT"); + r.species = (0..100_000).map(|i| format!("sp{i}")).collect(); + let t = std::time::Instant::now(); + assert!(rejected(&r, Code::InvalidSpecies).contains("at most 20")); + assert!( + t.elapsed() < std::time::Duration::from_secs(1), + "{:?}", + t.elapsed() + ); + // Repeats of one code collapse to one species and pass. + r.species = vec!["dmel".to_string(); 100_000]; + assert_eq!(run(&r).unwrap().species, vec!["dmel"]); + } + #[test] fn species_are_deduplicated_in_order() { let mut r = req("blastn", "assembly", "ACGT"); From 0a94f1508219309b0ae41cc87c49e6a46bb889fc Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Mon, 12 Oct 2026 17:48:50 +0000 Subject: [PATCH 04/12] fix(blast-jobs): chain store wake-ups, write job.json first, run store IO on the blocking pool --- blast-jobs/src/store.rs | 348 +++++++++++++++++++++++++++------------- 1 file changed, 238 insertions(+), 110 deletions(-) diff --git a/blast-jobs/src/store.rs b/blast-jobs/src/store.rs index a8af32d..82f7569 100644 --- a/blast-jobs/src/store.rs +++ b/blast-jobs/src/store.rs @@ -7,9 +7,16 @@ //! `job.json`: a job that was queued or running when the service stopped is marked `failed` at the next //! start, never re-run. //! -//! Blocking: `open`, `create`, `claim_next`, `finish` and `sweep` touch the disk (a few small files and -//! an fsync, never while the mutex is held). `get`, `counts`, `state_writable`, `job_dir` and -//! `notified` are memory-only and safe to call from any async handler. +//! Blocking: `create`, `claim_next` and `finish` are async and run their file IO (small files and an +//! fsync) on the blocking pool via `spawn_blocking`, never while the mutex is held. `open` and `sweep` +//! are synchronous (startup, and the sweeper already runs `sweep` on the blocking pool). `get`, +//! `counts`, `state_writable`, `job_dir` and `notified` are memory-only. +//! +//! Cancellation: the in-memory transition of `claim_next` and `finish` happens before their first +//! await, so a dropped future loses at most the `job.json` write. Workers should not cancel +//! `claim_next` (a dropped one would leave a job `running` with nobody to finish it). A dropped +//! `create` gives back its queue slot; a directory it had already written stays `queued` on disk and +//! the next start marks it failed, after which the sweep removes it. //! //! One writer per job: after `claim_next` hands a job out, only that worker calls `finish` for it, so //! the two `job.json` writes of a job never race. `sweep` only touches terminal jobs. @@ -148,7 +155,8 @@ impl Store { record.state = State::Failed; record.error = Some(RESTART_MESSAGE.to_owned()); record.finished_at_ms = Some(now); - store.persist(&id, &record); + let written = write_record(&store.job_dir(&id), &record); + store.note_write(&id, written); recovered += 1; } loaded += 1; @@ -171,18 +179,12 @@ impl Store { Ok(Arc::new(store)) } - /// Queues a validated request: writes `query.fa` and `job.json`, pushes it on the FIFO and wakes a - /// worker. Returns the id and the 1-based queue position. `QueueFull` when the FIFO already holds + /// Queues a validated request: writes `job.json` then `query.fa`, pushes it on the FIFO and wakes + /// a worker. Returns the id and the 1-based queue position. `QueueFull` when the FIFO already holds /// `max_queued` jobs (running ones do not count); `Io` when the directory cannot be written, which /// also turns `state_writable` false until a later write succeeds. - pub fn create(&self, req: Validated) -> Result<(JobId, u32), CreateError> { - { - let mut inner = self.lock(); - if inner.fifo.len() + inner.reserved >= self.max_queued { - return Err(CreateError::QueueFull); - } - inner.reserved += 1; - } + pub async fn create(&self, req: Validated) -> Result<(JobId, u32), CreateError> { + let slot = self.reserve()?; let id = JobId::new(); let record = Record { job_id: id.to_string(), @@ -198,14 +200,20 @@ impl Store { query_length: req.residues.len(), client: req.client.clone(), }; - let written = self.write_new_job(&id, &req, &record); - self.writable.store(written.is_ok(), Ordering::Relaxed); + let (jobs_root, dir) = (self.jobs_root.clone(), self.job_dir(&id)); + let joined = tokio::task::spawn_blocking(move || { + let written = write_new_job(&jobs_root, &dir, &req.residues, &record); + (written, req, record) + }) + .await; + let (req, record) = match joined { + Ok((Ok(()), req, record)) => (req, record), + Ok((Err(e), _, _)) => return Err(self.create_failed(e)), + Err(e) => return Err(self.create_failed(io::Error::other(e))), + }; + self.writable.store(true, Ordering::Relaxed); let mut inner = self.lock(); - inner.reserved -= 1; - if let Err(e) = written { - tracing::error!(error = %e, "could not write a new job directory"); - return Err(CreateError::Io(e)); - } + slot.commit(&mut inner); inner.jobs.insert( id.clone(), Job { @@ -222,11 +230,12 @@ impl Store { Ok((id, position)) } - /// Pops the oldest queued job, marks it `running` (memory and `job.json`) and hands its request to - /// the caller. `None` when the FIFO is empty. Workers loop: `claim_next()`, else - /// `notified().await`, then try again. - pub fn claim_next(&self) -> Option<(JobId, Validated)> { - let (id, req, snapshot) = { + /// Pops the oldest queued job, marks it `running` (memory, then `job.json`) and hands its request + /// to the caller. `None` when the FIFO is empty. Workers loop: `claim_next().await`, else + /// `notified().await`, then try again. When jobs remain after the pop another worker is woken, so + /// two creates that coalesced into one stored permit still reach two idle workers. + pub async fn claim_next(&self) -> Option<(JobId, Validated)> { + let (id, req, snapshot, more) = { let mut inner = self.lock(); loop { let id = inner.fifo.pop_front()?; @@ -238,10 +247,14 @@ impl Store { }; job.record.state = State::Running; job.record.started_at_ms = Some(now_ms(SystemTime::now())); - break (id, req, job.record.clone()); + let snapshot = job.record.clone(); + break (id, req, snapshot, !inner.fifo.is_empty()); } }; - self.persist(&id, &snapshot); + if more { + self.notify.notify_one(); + } + self.persist(&id, snapshot).await; Some((id, req)) } @@ -283,7 +296,7 @@ impl Store { /// Records the end of a job and rewrites its `job.json`. A job already terminal or unknown (swept) /// is left as it is. - pub fn finish(&self, id: &JobId, outcome: Outcome) { + pub async fn finish(&self, id: &JobId, outcome: Outcome) { let snapshot = { let mut inner = self.lock(); let Some(job) = inner.jobs.get_mut(id) else { @@ -318,7 +331,7 @@ impl Store { } snapshot }; - self.persist(id, &snapshot); + self.persist(id, snapshot).await; } /// Removes terminal jobs submitted more than `retention` before `now`, from memory and disk. @@ -406,34 +419,84 @@ impl Store { self.inner.lock().unwrap_or_else(|e| e.into_inner()) } - fn write_new_job(&self, id: &JobId, req: &Validated, record: &Record) -> io::Result<()> { - std::fs::create_dir_all(&self.jobs_root)?; - let dir = self.job_dir(id); - // `create_dir`, not `_all`: an existing directory is an error, never reused or removed. - std::fs::create_dir(&dir)?; - let res = (|| { - let mut q = std::fs::File::create(dir.join(QUERY_FILE))?; - q.write_all(b">query\n")?; - q.write_all(req.residues.as_bytes())?; - q.write_all(b"\n")?; - q.sync_all()?; - write_record(&dir, record) - })(); - if res.is_err() { - let _ = std::fs::remove_dir_all(&dir); + /// Takes a queue slot, or `QueueFull`. The slot is given back when the guard drops uncommitted. + fn reserve(&self) -> Result, CreateError> { + let mut inner = self.lock(); + if inner.fifo.len() + inner.reserved >= self.max_queued { + return Err(CreateError::QueueFull); } - res + inner.reserved += 1; + Ok(Slot { + store: self, + live: true, + }) + } + + fn create_failed(&self, e: io::Error) -> CreateError { + self.writable.store(false, Ordering::Relaxed); + tracing::error!(error = %e, "could not write a new job directory"); + CreateError::Io(e) + } + + async fn persist(&self, id: &JobId, record: Record) { + let dir = self.job_dir(id); + let written = tokio::task::spawn_blocking(move || write_record(&dir, &record)) + .await + .unwrap_or_else(|e| Err(io::Error::other(e))); + self.note_write(id, written); } - fn persist(&self, id: &JobId, record: &Record) { - let res = write_record(&self.job_dir(id), record); - self.writable.store(res.is_ok(), Ordering::Relaxed); - if let Err(e) = res { + fn note_write(&self, id: &JobId, written: io::Result<()>) { + self.writable.store(written.is_ok(), Ordering::Relaxed); + if let Err(e) = written { tracing::error!(job_id = %id, error = %e, "could not write job.json"); } } } +/// A reserved queue slot held by an in-flight `create`. +struct Slot<'a> { + store: &'a Store, + live: bool, +} + +impl Slot<'_> { + /// Turns the reservation into a FIFO entry the caller is about to push (lock already held). + fn commit(mut self, inner: &mut Inner) { + inner.reserved -= 1; + self.live = false; + } +} + +impl Drop for Slot<'_> { + fn drop(&mut self) { + if self.live { + self.store.lock().reserved -= 1; + } + } +} + +/// Creates `dir` and writes `job.json` (queued) before `query.fa`, so a crash between the two leaves a +/// known job that restart recovery fails and the sweep later removes, never a stray directory holding a +/// query. Removes the directory again on any error. +fn write_new_job(jobs_root: &Path, dir: &Path, residues: &str, record: &Record) -> io::Result<()> { + std::fs::create_dir_all(jobs_root)?; + // `create_dir`, not `_all`: an existing directory is an error, never reused or removed. + std::fs::create_dir(dir)?; + let res = (|| { + write_record(dir, record)?; + let mut q = std::fs::File::create(dir.join(QUERY_FILE))?; + q.write_all(b">query\n")?; + q.write_all(residues.as_bytes())?; + q.write_all(b"\n")?; + q.sync_all() + })(); + if res.is_err() { + let _ = std::fs::remove_dir_all(dir); + } + res +} + /// Writes `job.json` atomically: the whole record to `job.json.tmp`, fsync, rename over `job.json`. fn write_record(dir: &Path, record: &Record) -> io::Result<()> { let tmp = dir.join(JOB_TMP); @@ -473,6 +536,7 @@ fn now_ms(t: SystemTime) -> u64 { mod tests { use super::*; use crate::catalogue::Resolved; + use std::future::Future; use std::os::unix::fs::PermissionsExt; fn validated(residues: &str) -> Validated { @@ -519,12 +583,12 @@ mod tests { serde_json::from_slice(&std::fs::read(store.job_dir(id).join(JOB_FILE)).unwrap()).unwrap() } - #[test] - fn create_writes_query_and_job_json() { + #[tokio::test] + async fn create_writes_query_and_job_json() { let tmp = tempfile::tempdir().unwrap(); let store = Store::open(tmp.path(), 5).unwrap(); assert!(store.state_writable()); - let (id, pos) = store.create(validated("ACGTACGT")).unwrap(); + let (id, pos) = store.create(validated("ACGTACGT")).await.unwrap(); assert_eq!(pos, 1); let dir = store.job_dir(&id); assert_eq!(dir, tmp.path().join("jobs").join(id.as_str())); @@ -543,16 +607,16 @@ mod tests { assert!(!tmp.path().join("jobs").join(PROBE_FILE).exists()); } - #[test] - fn fifo_positions_queue_full_and_claim_shifts() { + #[tokio::test] + async fn fifo_positions_queue_full_and_claim_shifts() { let tmp = tempfile::tempdir().unwrap(); let store = Store::open(tmp.path(), 3).unwrap(); - let (a, pa) = store.create(validated("AAAA")).unwrap(); - let (b, pb) = store.create(validated("CCCC")).unwrap(); - let (c, pc) = store.create(validated("GGGG")).unwrap(); + let (a, pa) = store.create(validated("AAAA")).await.unwrap(); + let (b, pb) = store.create(validated("CCCC")).await.unwrap(); + let (c, pc) = store.create(validated("GGGG")).await.unwrap(); assert_eq!((pa, pb, pc), (1, 2, 3)); assert!(matches!( - store.create(validated("TTTT")), + store.create(validated("TTTT")).await, Err(CreateError::QueueFull) )); assert_eq!(store.counts(), (3, 0)); @@ -561,7 +625,7 @@ mod tests { assert_eq!((s.state, s.queue_position), (State::Queued, Some(p))); } - let (claimed, req) = store.claim_next().unwrap(); + let (claimed, req) = store.claim_next().await.unwrap(); assert_eq!(claimed, a, "FIFO order"); assert_eq!(req.residues, "AAAA"); let s = store.get(&a).unwrap(); @@ -573,32 +637,36 @@ mod tests { assert_eq!(store.counts(), (2, 1)); // A running job frees its queue slot. - let (d, pd) = store.create(validated("TTTT")).unwrap(); + let (d, pd) = store.create(validated("TTTT")).await.unwrap(); assert_eq!(pd, 3); assert!(matches!( - store.create(validated("TTTT")), + store.create(validated("TTTT")).await, Err(CreateError::QueueFull) )); - assert_eq!(store.claim_next().unwrap().0, b); - assert_eq!(store.claim_next().unwrap().0, c); - assert_eq!(store.claim_next().unwrap().0, d); - assert!(store.claim_next().is_none()); + assert_eq!(store.claim_next().await.unwrap().0, b); + assert_eq!(store.claim_next().await.unwrap().0, c); + assert_eq!(store.claim_next().await.unwrap().0, d); + assert!(store.claim_next().await.is_none()); assert_eq!(store.counts(), (0, 4)); } - #[test] - fn finish_records_outcome_and_survives_reopen() { + #[tokio::test] + async fn finish_records_outcome_and_survives_reopen() { let tmp = tempfile::tempdir().unwrap(); let store = Store::open(tmp.path(), 5).unwrap(); - let (ok, _) = store.create(validated("AAAA")).unwrap(); - let (bad, _) = store.create(validated("CCCC")).unwrap(); - store.claim_next().unwrap(); - store.claim_next().unwrap(); - store.finish(&ok, Outcome::Complete { truncated: true }); - store.finish(&bad, Outcome::Failed("BLAST failed (exit 2)".into())); + let (ok, _) = store.create(validated("AAAA")).await.unwrap(); + let (bad, _) = store.create(validated("CCCC")).await.unwrap(); + store.claim_next().await.unwrap(); + store.claim_next().await.unwrap(); + store + .finish(&ok, Outcome::Complete { truncated: true }) + .await; + store + .finish(&bad, Outcome::Failed("BLAST failed (exit 2)".into())) + .await; // A second finish does not overwrite the first. - store.finish(&ok, Outcome::Failed("late".into())); + store.finish(&ok, Outcome::Failed("late".into())).await; assert_eq!(store.counts(), (0, 0)); let s = store.get(&ok).unwrap(); @@ -618,32 +686,34 @@ mod tests { assert_eq!(s.state, State::Failed); assert_eq!(s.error.as_deref(), Some("BLAST failed (exit 2)")); assert!( - store.claim_next().is_none(), + store.claim_next().await.is_none(), "terminal jobs are not requeued" ); } - #[test] - fn finish_on_a_queued_job_removes_it_from_the_fifo() { + #[tokio::test] + async fn finish_on_a_queued_job_removes_it_from_the_fifo() { let tmp = tempfile::tempdir().unwrap(); let store = Store::open(tmp.path(), 5).unwrap(); - let (a, _) = store.create(validated("AAAA")).unwrap(); - let (b, _) = store.create(validated("CCCC")).unwrap(); - store.finish(&a, Outcome::Failed("x".into())); + let (a, _) = store.create(validated("AAAA")).await.unwrap(); + let (b, _) = store.create(validated("CCCC")).await.unwrap(); + store.finish(&a, Outcome::Failed("x".into())).await; assert_eq!(store.get(&b).unwrap().queue_position, Some(1)); - assert_eq!(store.claim_next().unwrap().0, b); + assert_eq!(store.claim_next().await.unwrap().0, b); assert!(store.get(&JobId::new()).is_none()); } - #[test] - fn restart_fails_queued_and_running_jobs() { + #[tokio::test] + async fn restart_fails_queued_and_running_jobs() { let tmp = tempfile::tempdir().unwrap(); let store = Store::open(tmp.path(), 5).unwrap(); - let (running, _) = store.create(validated("AAAA")).unwrap(); - let (queued, _) = store.create(validated("CCCC")).unwrap(); - let (done, _) = store.create(validated("GGGG")).unwrap(); - assert_eq!(store.claim_next().unwrap().0, running); - store.finish(&done, Outcome::Complete { truncated: false }); + let (running, _) = store.create(validated("AAAA")).await.unwrap(); + let (queued, _) = store.create(validated("CCCC")).await.unwrap(); + let (done, _) = store.create(validated("GGGG")).await.unwrap(); + assert_eq!(store.claim_next().await.unwrap().0, running); + store + .finish(&done, Outcome::Complete { truncated: false }) + .await; assert_eq!(read_record(&store, &queued).state, State::Queued); // The process dies here: `running` is running and `queued` queued on disk. drop(store); @@ -661,11 +731,11 @@ mod tests { } assert_eq!(store.get(&done).unwrap().state, State::Complete); assert_eq!(store.counts(), (0, 0)); - assert!(store.claim_next().is_none()); + assert!(store.claim_next().await.is_none()); } - #[test] - fn open_leaves_foreign_entries_alone() { + #[tokio::test] + async fn open_leaves_foreign_entries_alone() { let tmp = tempfile::tempdir().unwrap(); let jobs = tmp.path().join("jobs"); std::fs::create_dir_all(jobs.join("not a job")).unwrap(); @@ -718,8 +788,8 @@ mod tests { } } - #[test] - fn sweep_removes_only_expired_terminal_jobs() { + #[tokio::test] + async fn sweep_removes_only_expired_terminal_jobs() { let tmp = tempfile::tempdir().unwrap(); let day = Duration::from_secs(86_400); let now = SystemTime::now(); @@ -729,9 +799,9 @@ mod tests { std::fs::create_dir_all(tmp.path().join("jobs").join("foreign-dir")).unwrap(); let store = Store::open(tmp.path(), 5).unwrap(); // Fresh jobs: one running, one queued; never swept however old `now` makes them. - let (running, _) = store.create(validated("AAAA")).unwrap(); - store.claim_next().unwrap(); - let (queued, _) = store.create(validated("CCCC")).unwrap(); + let (running, _) = store.create(validated("AAAA")).await.unwrap(); + store.claim_next().await.unwrap(); + let (queued, _) = store.create(validated("CCCC")).await.unwrap(); assert_eq!(store.sweep(now, day), 2); let jobs = tmp.path().join("jobs"); @@ -754,17 +824,19 @@ mod tests { assert_eq!(store.get(&queued).unwrap().state, State::Queued); assert_eq!(store.get(&running).unwrap().state, State::Running); - store.finish(&running, Outcome::Complete { truncated: false }); + store + .finish(&running, Outcome::Complete { truncated: false }) + .await; assert_eq!(store.sweep(far, day), 1); assert!(!store.job_dir(&running).exists()); assert!(store.get(&running).is_none()); // `finish` after a sweep is ignored rather than resurrecting the directory. - store.finish(&running, Outcome::Failed("late".into())); + store.finish(&running, Outcome::Failed("late".into())).await; assert!(!store.job_dir(&running).exists()); } - #[test] - fn read_only_directory_gives_io_and_flips_writable() { + #[tokio::test] + async fn read_only_directory_gives_io_and_flips_writable() { let tmp = tempfile::tempdir().unwrap(); let jobs = tmp.path().join("jobs"); std::fs::create_dir_all(&jobs).unwrap(); @@ -778,21 +850,21 @@ mod tests { let store = Store::open(tmp.path(), 5).unwrap(); assert!(!store.state_writable()); assert!(matches!( - store.create(validated("AAAA")), + store.create(validated("AAAA")).await, Err(CreateError::Io(_)) )); assert!(!store.state_writable()); assert_eq!(store.counts(), (0, 0), "a failed create takes no slot"); std::fs::set_permissions(&jobs, std::fs::Permissions::from_mode(0o755)).unwrap(); - let (id, pos) = store.create(validated("AAAA")).unwrap(); + let (id, pos) = store.create(validated("AAAA")).await.unwrap(); assert_eq!(pos, 1); assert!(store.state_writable(), "a successful write clears the flag"); assert!(store.job_dir(&id).join(JOB_FILE).exists()); } - #[test] - fn open_creates_missing_state_root() { + #[tokio::test] + async fn open_creates_missing_state_root() { let tmp = tempfile::tempdir().unwrap(); let root = tmp.path().join("a").join("b"); let store = Store::open(&root, 5).unwrap(); @@ -809,14 +881,14 @@ mod tests { let s = Arc::clone(&store); let worker = tokio::spawn(async move { loop { - if let Some((id, _)) = s.claim_next() { + if let Some((id, _)) = s.claim_next().await { return id; } s.notified().await; } }); tokio::task::yield_now().await; - let (id, _) = store.create(validated("AAAA")).unwrap(); + let (id, _) = store.create(validated("AAAA")).await.unwrap(); let got = tokio::time::timeout(Duration::from_secs(5), worker) .await .expect("worker woke") @@ -824,12 +896,12 @@ mod tests { assert_eq!(got, id); // A worker that saw an empty FIFO just before the push still wakes (stored permit). - assert!(store.claim_next().is_none()); - store.create(validated("CCCC")).unwrap(); + assert!(store.claim_next().await.is_none()); + store.create(validated("CCCC")).await.unwrap(); tokio::time::timeout(Duration::from_secs(5), store.notified()) .await .expect("permit was stored"); - assert!(store.claim_next().is_some()); + assert!(store.claim_next().await.is_some()); } #[tokio::test] @@ -857,4 +929,60 @@ mod tests { assert!(!store.job_dir(&id).exists()); handle.abort(); } + + #[tokio::test] + async fn coalesced_wakeups_still_reach_every_job() { + let tmp = tempfile::tempdir().unwrap(); + let store = Store::open(tmp.path(), 5).unwrap(); + // Two creates with no worker waiting: `Notify` keeps a single permit. + let (a, _) = store.create(validated("AAAA")).await.unwrap(); + let (b, _) = store.create(validated("CCCC")).await.unwrap(); + // Worker one takes the permit and claims `a`; the claim must pass the wake-up on. + tokio::time::timeout(Duration::from_secs(5), store.notified()) + .await + .expect("first permit"); + assert_eq!(store.claim_next().await.unwrap().0, a); + // Worker two, already past its empty-FIFO check, must be woken for `b`. + tokio::time::timeout(Duration::from_secs(5), store.notified()) + .await + .expect("claim_next chains the wake-up while jobs remain"); + assert_eq!(store.claim_next().await.unwrap().0, b); + assert_eq!(store.counts(), (0, 2)); + } + + #[tokio::test] + async fn crash_before_query_fa_leaves_a_recoverable_sweepable_job() { + // What `write_new_job` leaves if the process dies after `job.json` and before `query.fa`. + let tmp = tempfile::tempdir().unwrap(); + let old = SystemTime::now() - Duration::from_secs(2 * 86_400); + record_on_disk(tmp.path(), "crashed_mid_create", State::Queued, old); + let store = Store::open(tmp.path(), 5).unwrap(); + let id = JobId::parse("crashed_mid_create").unwrap(); + let s = store.get(&id).unwrap(); + assert_eq!( + (s.state, s.error.as_deref()), + (State::Failed, Some(RESTART_MESSAGE)) + ); + assert_eq!( + store.sweep(SystemTime::now(), Duration::from_secs(86_400)), + 1 + ); + assert!(!store.job_dir(&id).exists()); + } + + #[tokio::test] + async fn dropped_create_gives_its_slot_back() { + let tmp = tempfile::tempdir().unwrap(); + let store = Store::open(tmp.path(), 1).unwrap(); + { + // Polled once (slot reserved, write handed to the blocking pool), then dropped. + let mut fut = Box::pin(store.create(validated("AAAA"))); + let waker = std::task::Waker::noop(); + let _ = fut + .as_mut() + .poll(&mut std::task::Context::from_waker(waker)); + } + assert_eq!(store.counts(), (0, 0)); + store.create(validated("CCCC")).await.unwrap(); + } } From 7cf0ef652a020d12ecf8077138ca1c142f12c714 Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Mon, 12 Oct 2026 17:52:26 +0000 Subject: [PATCH 05/12] feat(blast-jobs): BLAST+ runner with wall-clock limit and post-exit output cap --- blast-jobs/src/command.rs | 545 +++++++++++++++++++++++++ blast-jobs/src/lib.rs | 2 + blast-jobs/src/runner.rs | 492 ++++++++++++++++++++++ blast-jobs/tests/fake-blast/blastn | 53 +++ blast-jobs/tests/fake-blast/blastp | 53 +++ blast-jobs/tests/fake-blast/blastx | 53 +++ blast-jobs/tests/fake-blast/canned.tsv | 3 + blast-jobs/tests/fake-blast/tblastn | 53 +++ blast-jobs/tests/fake-blast/tblastx | 53 +++ blast-jobs/tests/runner.rs | 412 +++++++++++++++++++ 10 files changed, 1719 insertions(+) create mode 100644 blast-jobs/src/command.rs create mode 100644 blast-jobs/src/runner.rs create mode 100755 blast-jobs/tests/fake-blast/blastn create mode 100755 blast-jobs/tests/fake-blast/blastp create mode 100755 blast-jobs/tests/fake-blast/blastx create mode 100644 blast-jobs/tests/fake-blast/canned.tsv create mode 100755 blast-jobs/tests/fake-blast/tblastn create mode 100755 blast-jobs/tests/fake-blast/tblastx create mode 100644 blast-jobs/tests/runner.rs diff --git a/blast-jobs/src/command.rs b/blast-jobs/src/command.rs new file mode 100644 index 0000000..082ab81 --- /dev/null +++ b/blast-jobs/src/command.rs @@ -0,0 +1,545 @@ +//! The BLAST+ command line for one job (design 3.5). Pure: no IO, no environment. +//! +//! The argv is built from an ordered map `flag -> value` seeded with the service defaults and then +//! updated by the validated `advanced` pairs: a pair whose flag is already in the map replaces the value +//! in place (so `-dust no` replaces the default `-dust yes` and never adds a second `-dust`), any other +//! pair is appended in the order given. A Switch is rendered as one element. Nothing goes through a +//! shell: the `-db` list of prefixes is one argv element with spaces, as BLAST+ expects. +use crate::catalogue::Resolved; +use crate::config::Config; +use crate::types::Program; +use crate::validate::Validated; +use std::ffi::OsString; +use std::path::Path; + +/// The query file the store writes into each job directory. +pub const QUERY_FILE: &str = "query.fa"; +/// The raw BLAST+ output, replaced by `results.tsv` once the search ends. +pub const RAW_FILE: &str = "raw.tsv"; +/// Tabular output with the subject title appended; `runner::HEADER` names these columns. +pub const OUTFMT: &str = "6 std stitle"; + +/// An ordered `flag -> value` map; `None` is a Switch. +#[derive(Default)] +struct Argv(Vec<(String, Option)>); + +impl Argv { + /// Replace the value of `flag` where it already stands, else append it. + fn set(&mut self, flag: &str, value: Option) { + match self.0.iter_mut().find(|(f, _)| f == flag) { + Some(slot) => slot.1 = value, + None => self.0.push((flag.to_string(), value)), + } + } + + fn set_str(&mut self, flag: &str, value: impl Into) { + self.set(flag, Some(value.into())); + } +} + +/// `evalue` as BLAST+ reads it: plain decimal for ordinary values, exponent form for tiny ones so the +/// argument stays short (`1e-300`, not 300 zeros). +fn format_evalue(e: f64) -> String { + if e < 1e-4 { + format!("{e:e}") + } else { + format!("{e}") + } +} + +/// The full argv for `req`, element 0 being the program binary under `cfg.blast_bin_dir`: +/// +/// ```text +/// / -db " [ ...]" -query /query.fa -out /raw.tsv +/// -outfmt "6 std stitle" -evalue -max_target_seqs -max_hsps +/// -num_threads [-task (always for blastn)] -dust yes | -seg yes +/// [-matrix ] [-query_gencode (blastx, tblastx)] [-db_gencode (tblastn, tblastx)] +/// [remaining advanced flags] +/// ``` +/// +/// `-task` is explicit for blastn (BLAST+'s own default there is megablast): `megablast` when the +/// request says so, else `blastn`, unless `advanced` sets it. Other programs get `-task` only from +/// `advanced`. An `advanced` `-max_hsps` is clamped to `max_hsps_per_subject`. +pub fn build(cfg: &Config, req: &Validated, resolved: &Resolved, job_dir: &Path) -> Vec { + let program = req.program; + let mut a = Argv::default(); + + let mut db = OsString::new(); + for (i, prefix) in resolved.prefixes.iter().enumerate() { + if i > 0 { + db.push(" "); + } + db.push(prefix.as_os_str()); + } + a.set("db", Some(db)); + a.set_str("query", job_dir.join(QUERY_FILE)); + a.set_str("out", job_dir.join(RAW_FILE)); + a.set_str("outfmt", OUTFMT); + a.set_str("evalue", format_evalue(req.evalue)); + a.set_str("max_target_seqs", req.max_hits.to_string()); + a.set_str("max_hsps", cfg.max_hsps_per_subject.to_string()); + a.set_str("num_threads", cfg.threads_per_job.to_string()); + if program == Program::Blastn { + a.set_str("task", if req.megablast { "megablast" } else { "blastn" }); + a.set_str("dust", "yes"); + } else { + a.set_str("seg", "yes"); + } + if let Some(m) = &req.matrix { + a.set_str("matrix", m.as_str()); + } + if let Some(g) = req.genetic_code { + if matches!(program, Program::Blastx | Program::Tblastx) { + a.set_str("query_gencode", g.to_string()); + } + if matches!(program, Program::Tblastn | Program::Tblastx) { + a.set_str("db_gencode", g.to_string()); + } + } + + for (flag, value) in &req.advanced { + let value = match (flag.as_str(), value) { + // Validated as an integer >= 1; anything that does not fit a u64 is above the cap anyway. + ("max_hsps", Some(v)) => Some( + v.parse::() + .map_or(cfg.max_hsps_per_subject.into(), |n| { + n.min(cfg.max_hsps_per_subject.into()) + }) + .to_string() + .into(), + ), + (_, v) => v.as_ref().map(OsString::from), + }; + a.set(flag, value); + } + + let mut argv = Vec::with_capacity(1 + 2 * a.0.len()); + argv.push(cfg.blast_bin_dir.join(program.as_str()).into_os_string()); + for (flag, value) in a.0 { + argv.push(OsString::from(format!("-{flag}"))); + if let Some(v) = value { + argv.push(v); + } + } + argv +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::advanced::{Kind, TABLE}; + use crate::catalogue::Catalogue; + use crate::types::SubmitRequest; + use crate::validate; + use std::path::PathBuf; + + const JOB: &str = "/state/jobs/AAAAAAAAAAAAAAAAAAAAAA"; + + fn fixture() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixture") + } + + fn cfg() -> Config { + let mut v = crate::config::tests::sample(); + v["blast_bin_dir"] = serde_json::json!("/opt/ncbi-blast/bin"); + v["threads_per_job"] = serde_json::json!(3); + serde_json::from_value(v).unwrap() + } + + fn catalogue() -> Catalogue { + let f = fixture(); + Catalogue::load(f.join("catalogue.json"), f.join("db")).unwrap() + } + + fn db_for(p: Program) -> &'static str { + if p.protein_database() { + "translation" + } else { + "assembly" + } + } + + fn seq_for(p: Program) -> &'static str { + if p.protein_query() { + ">q\nMKLVQ\n" + } else { + ">q\nACGTACGT\n" + } + } + + fn request(p: Program) -> SubmitRequest { + SubmitRequest { + program: p.as_str().into(), + database: db_for(p).into(), + species: vec!["dmel".into()], + sequence: seq_for(p).into(), + ..Default::default() + } + } + + fn argv_for(req: SubmitRequest) -> Vec { + let c = cfg(); + let v = validate::check(&req, &c, &catalogue()).expect("request should validate"); + build(&c, &v, &v.resolved, Path::new(JOB)) + .into_iter() + .map(|s| s.into_string().unwrap()) + .collect() + } + + fn with_advanced(p: Program, adv: &str) -> Vec { + argv_for(SubmitRequest { + advanced: Some(adv.into()), + ..request(p) + }) + } + + fn prefix(category: &str, species: &str) -> String { + let cat = catalogue(); + cat.resolve(&[species.to_string()], category) + .unwrap() + .prefixes[0] + .display() + .to_string() + } + + /// The value after `flag`, asserting the flag appears exactly once. + fn value_of<'a>(argv: &'a [String], flag: &str) -> Option<&'a str> { + let hits: Vec = (0..argv.len()).filter(|&i| argv[i] == flag).collect(); + assert_eq!(hits.len(), 1, "{flag} must appear exactly once in {argv:?}"); + argv.get(hits[0] + 1).map(String::as_str) + } + + fn count(argv: &[String], flag: &str) -> usize { + argv.iter().filter(|a| *a == flag).count() + } + + fn expected_defaults(p: Program, extra: &[&str]) -> Vec { + let mut v: Vec = vec![ + format!("/opt/ncbi-blast/bin/{p}"), + "-db".into(), + prefix(db_for(p), "dmel"), + "-query".into(), + format!("{JOB}/query.fa"), + "-out".into(), + format!("{JOB}/raw.tsv"), + "-outfmt".into(), + "6 std stitle".into(), + "-evalue".into(), + "10".into(), + "-max_target_seqs".into(), + "100".into(), + "-max_hsps".into(), + "50".into(), + "-num_threads".into(), + "3".into(), + ]; + v.extend(extra.iter().map(|s| s.to_string())); + v + } + + #[test] + fn exact_default_argv_per_program() { + let cases: [(Program, &[&str]); 5] = [ + (Program::Blastn, &["-task", "blastn", "-dust", "yes"]), + (Program::Blastp, &["-seg", "yes"]), + (Program::Blastx, &["-seg", "yes"]), + (Program::Tblastn, &["-seg", "yes"]), + (Program::Tblastx, &["-seg", "yes"]), + ]; + for (p, extra) in cases { + assert_eq!(argv_for(request(p)), expected_defaults(p, extra), "{p}"); + } + } + + #[test] + fn request_fields_land_in_argv() { + let argv = argv_for(SubmitRequest { + evalue: Some(0.001), + max_hits: Some(250), + matrix: Some("pam30".into()), + genetic_code: Some(5), + ..request(Program::Tblastx) + }); + assert_eq!(value_of(&argv, "-evalue"), Some("0.001")); + assert_eq!(value_of(&argv, "-max_target_seqs"), Some("250")); + assert_eq!(value_of(&argv, "-matrix"), Some("PAM30")); + assert_eq!(value_of(&argv, "-query_gencode"), Some("5")); + assert_eq!(value_of(&argv, "-db_gencode"), Some("5")); + let tiny = argv_for(SubmitRequest { + evalue: Some(1e-300), + ..request(Program::Blastn) + }); + assert_eq!(value_of(&tiny, "-evalue"), Some("1e-300")); + } + + #[test] + fn gencode_placement_per_program() { + let gc = |p: Program| { + argv_for(SubmitRequest { + genetic_code: Some(11), + ..request(p) + }) + }; + let bx = gc(Program::Blastx); + assert_eq!(value_of(&bx, "-query_gencode"), Some("11")); + assert_eq!(count(&bx, "-db_gencode"), 0); + let tn = gc(Program::Tblastn); + assert_eq!(value_of(&tn, "-db_gencode"), Some("11")); + assert_eq!(count(&tn, "-query_gencode"), 0); + let tx = gc(Program::Tblastx); + assert_eq!(value_of(&tx, "-query_gencode"), Some("11")); + assert_eq!(value_of(&tx, "-db_gencode"), Some("11")); + // Non-translated programs never get a genetic code (validate drops it). + for p in [Program::Blastn, Program::Blastp] { + let a = gc(p); + assert_eq!(count(&a, "-query_gencode") + count(&a, "-db_gencode"), 0); + } + } + + #[test] + fn two_species_share_one_db_element() { + let argv = argv_for(SubmitRequest { + species: vec!["dmel".into(), "dsim".into()], + ..request(Program::Blastn) + }); + let db = value_of(&argv, "-db").unwrap(); + assert_eq!( + db, + format!( + "{} {}", + prefix("assembly", "dmel"), + prefix("assembly", "dsim") + ) + ); + assert!(db.starts_with('/'), "prefixes are absolute"); + } + + #[test] + fn blastn_task_both_ways_and_overrides() { + let plain = argv_for(request(Program::Blastn)); + assert_eq!(value_of(&plain, "-task"), Some("blastn")); + let mega = argv_for(SubmitRequest { + megablast: Some(true), + ..request(Program::Blastn) + }); + assert_eq!(value_of(&mega, "-task"), Some("megablast")); + let short = with_advanced(Program::Blastn, "-task blastn-short"); + assert_eq!(value_of(&short, "-task"), Some("blastn-short")); + // Overridden in place: same position as the default. + assert_eq!( + short.iter().position(|a| a == "-task"), + plain.iter().position(|a| a == "-task") + ); + // Allowed-and-overridable (3.4): `-task megablast` with megablast false. + let adv_mega = argv_for(SubmitRequest { + megablast: Some(false), + advanced: Some("-task megablast".into()), + ..request(Program::Blastn) + }); + assert_eq!(value_of(&adv_mega, "-task"), Some("megablast")); + let mega_mega = argv_for(SubmitRequest { + megablast: Some(true), + advanced: Some("-task=megablast".into()), + ..request(Program::Blastn) + }); + assert_eq!(value_of(&mega_mega, "-task"), Some("megablast")); + } + + #[test] + fn other_programs_get_task_only_from_advanced() { + for p in [ + Program::Blastp, + Program::Blastx, + Program::Tblastn, + Program::Tblastx, + ] { + assert_eq!(count(&argv_for(request(p)), "-task"), 0, "{p}"); + } + assert_eq!( + value_of( + &with_advanced(Program::Blastp, "-task blastp-short"), + "-task" + ), + Some("blastp-short") + ); + assert_eq!( + value_of( + &with_advanced(Program::Tblastn, "-task tblastn-fast"), + "-task" + ), + Some("tblastn-fast") + ); + } + + #[test] + fn max_hsps_override_and_clamp() { + let low = with_advanced(Program::Blastn, "-max_hsps 7"); + assert_eq!(value_of(&low, "-max_hsps"), Some("7")); + let at = with_advanced(Program::Blastp, "-max_hsps=50"); + assert_eq!(value_of(&at, "-max_hsps"), Some("50")); + let high = with_advanced(Program::Blastn, "-max_hsps 500"); + assert_eq!(value_of(&high, "-max_hsps"), Some("50")); + let huge = with_advanced(Program::Tblastx, "-max_hsps 9223372036854775807"); + assert_eq!(value_of(&huge, "-max_hsps"), Some("50")); + // In place, not appended. + let plain = argv_for(request(Program::Blastn)); + assert_eq!( + high.iter().position(|a| a == "-max_hsps"), + plain.iter().position(|a| a == "-max_hsps") + ); + } + + #[test] + fn masking_defaults_and_their_override() { + let n = argv_for(request(Program::Blastn)); + assert_eq!(value_of(&n, "-dust"), Some("yes")); + assert_eq!(count(&n, "-seg"), 0); + for p in [ + Program::Blastp, + Program::Blastx, + Program::Tblastn, + Program::Tblastx, + ] { + let a = argv_for(request(p)); + assert_eq!(value_of(&a, "-seg"), Some("yes"), "{p}"); + assert_eq!(count(&a, "-dust"), 0, "{p}"); + let off = with_advanced(p, "-seg no"); + assert_eq!(value_of(&off, "-seg"), Some("no"), "{p}"); + assert!(!off.iter().any(|x| x == "yes"), "{p}: no stray yes"); + assert_eq!(off.len(), a.len(), "{p}: replaced, not appended"); + } + let off = with_advanced(Program::Blastn, "-dust=no"); + assert_eq!(value_of(&off, "-dust"), Some("no")); + assert_eq!(off.len(), n.len()); + } + + #[test] + fn switches_are_one_element() { + let a = with_advanced(Program::Blastn, "-lcase_masking -ungapped -penalty -3"); + let i = a.iter().position(|x| x == "-lcase_masking").unwrap(); + assert_eq!(a[i + 1], "-ungapped"); + assert_eq!(a[i + 2], "-penalty"); + assert_eq!(a[i + 3], "-3", "a negative numeric value stays one element"); + assert_eq!(a.len(), i + 4); + assert_eq!(count(&a, "-lcase_masking"), 1); + } + + #[test] + fn advanced_appends_in_given_order_after_defaults() { + let base = argv_for(request(Program::Blastn)); + let a = with_advanced(Program::Blastn, "-word_size 11 -reward 2 -strand=minus"); + assert_eq!(&a[..base.len()], &base[..]); + assert_eq!( + &a[base.len()..], + &["-word_size", "11", "-reward", "2", "-strand", "minus"] + ); + } + + /// A valid value for each kind, chosen from the table's own bounds. + fn sample_value(kind: Kind) -> Option { + match kind { + Kind::Int { min, max } => Some(match (min, max) { + (Some(m), _) => (m + 1).to_string(), + (None, Some(m)) => (m - 2).to_string(), + (None, None) => "1".into(), + }), + Kind::Float { min, .. } => Some(format!("{}", min + 1.5)), + Kind::Bool { no, .. } => Some(no.to_string()), + Kind::Switch => None, + Kind::Enum(values) => Some(values[values.len() - 1].to_string()), + } + } + + /// Every row of the allow-list, on every program it applies to, in both `-f v` and `-f=v` forms: + /// the flag lands exactly once, with its value as the next element (a Switch alone). + #[test] + fn every_allowed_flag_lands_once_in_the_right_form() { + let mut checked = 0; + for spec in TABLE { + for &p in spec.programs { + let value = sample_value(spec.kind); + let forms = match &value { + Some(v) => vec![format!("-{} {v}", spec.name), format!("-{}={v}", spec.name)], + None => vec![format!("-{}", spec.name)], + }; + for adv in forms { + let argv = with_advanced(p, &adv); + let flag = format!("-{}", spec.name); + let got = value_of(&argv, &flag); + match &value { + None => assert!( + got.is_none_or(|next| next.starts_with('-')), + "{p} {adv}: a switch takes no value element, got {argv:?}" + ), + Some(v) if spec.name == "max_hsps" => { + let want = v.parse::().unwrap().min(50).to_string(); + assert_eq!(got, Some(want.as_str()), "{p} {adv}") + } + Some(v) => assert_eq!(got, Some(v.as_str()), "{p} {adv}"), + } + checked += 1; + } + } + } + assert!(checked > 60, "table sweep checked only {checked} cases"); + } + + #[test] + fn hand_written_rows_per_flag() { + use Program::*; + let rows: &[(Program, &str, &str, Option<&str>)] = &[ + (Blastn, "-word_size 7", "-word_size", Some("7")), + (Blastp, "-word_size 3", "-word_size", Some("3")), + (Blastx, "-gapopen 11", "-gapopen", Some("11")), + (Tblastn, "-gapextend 1", "-gapextend", Some("1")), + (Blastn, "-penalty -3", "-penalty", Some("-3")), + (Blastn, "-penalty=0", "-penalty", Some("0")), + (Blastn, "-reward 1", "-reward", Some("1")), + (Blastn, "-dust no", "-dust", Some("no")), + (Tblastx, "-seg no", "-seg", Some("no")), + ( + Blastp, + "-soft_masking false", + "-soft_masking", + Some("false"), + ), + (Tblastx, "-lcase_masking", "-lcase_masking", None), + (Blastx, "-ungapped", "-ungapped", None), + (Tblastx, "-strand plus", "-strand", Some("plus")), + ( + Blastn, + "-perc_identity 97.5", + "-perc_identity", + Some("97.5"), + ), + (Blastp, "-qcov_hsp_perc 80", "-qcov_hsp_perc", Some("80")), + (Tblastn, "-culling_limit 0", "-culling_limit", Some("0")), + (Blastx, "-threshold 11", "-threshold", Some("11")), + ( + Blastp, + "-comp_based_stats F", + "-comp_based_stats", + Some("F"), + ), + (Tblastx, "-window_size 40", "-window_size", Some("40")), + (Blastx, "-task blastx-fast", "-task", Some("blastx-fast")), + ]; + for &(p, adv, flag, want) in rows { + let argv = with_advanced(p, adv); + let got = value_of(&argv, flag); + match want { + Some(w) => assert_eq!(got, Some(w), "{p} {adv}"), + None => assert!(got.is_none_or(|n| n.starts_with('-')), "{p} {adv}"), + } + } + } + + #[test] + fn program_binary_comes_from_config() { + let mut c = cfg(); + c.blast_bin_dir = "/somewhere/else".into(); + let v = validate::check(&request(Program::Tblastn), &c, &catalogue()).unwrap(); + let argv = build(&c, &v, &v.resolved, Path::new(JOB)); + assert_eq!(argv[0], OsString::from("/somewhere/else/tblastn")); + } +} diff --git a/blast-jobs/src/lib.rs b/blast-jobs/src/lib.rs index 33a5b80..69b8e79 100644 --- a/blast-jobs/src/lib.rs +++ b/blast-jobs/src/lib.rs @@ -3,8 +3,10 @@ //! from the config file; nothing site-specific is compiled in. pub mod advanced; pub mod catalogue; +pub mod command; pub mod config; pub mod jobid; +pub mod runner; pub mod store; pub mod types; pub mod validate; diff --git a/blast-jobs/src/runner.rs b/blast-jobs/src/runner.rs new file mode 100644 index 0000000..973e1ae --- /dev/null +++ b/blast-jobs/src/runner.rs @@ -0,0 +1,492 @@ +//! Workers that run queued jobs (design 3.5, 3.6). +//! +//! `concurrency` worker tasks loop on `Store::claim_next`, waiting on the store's `Notify` when the FIFO +//! is empty. Each job runs the argv from `command::build` with `tokio::process::Command`: no shell, a +//! fixed environment, `kill_on_drop`, stderr captured up to 64 KiB into `stderr.txt`. The wall-clock +//! limit (`wall_seconds`, or `translated_wall_seconds` for blastx, tblastn and tblastx) is enforced with +//! `tokio::time::timeout`; on expiry the child gets `start_kill` and is reaped. No process group: BLAST+ +//! is one multithreaded process, and the container's `--init` reaps anything else. +//! +//! After a zero exit, `finalize_results` writes `results.tsv` (the header line, then `raw.tsv` cut once +//! at a row boundary so the whole file stays within `max_results_bytes`) and deletes `raw.tsv`. +use crate::command::{self, RAW_FILE}; +use crate::config::Config; +use crate::jobid::JobId; +use crate::store::{Outcome, Store}; +use crate::validate::Validated; +use std::ffi::OsString; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::os::unix::process::ExitStatusExt; +use std::path::Path; +use std::process::{ExitStatus, Stdio}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; +use tokio::io::AsyncReadExt; +use tokio::process::{ChildStderr, Command}; +use tokio::task::JoinHandle; + +/// The served file's first line: the column names of `-outfmt "6 std stitle"`. +pub const HEADER: &str = "qseqid\tsseqid\tpident\tlength\tmismatch\tgapopen\tqstart\tqend\tsstart\tsend\tevalue\tbitscore\tstitle\n"; +pub const RESULTS_FILE: &str = "results.tsv"; +pub const STDERR_FILE: &str = "stderr.txt"; +/// Most stderr bytes kept per job; the rest is read and discarded so BLAST+ never blocks on the pipe. +pub const STDERR_CAP: usize = 64 * 1024; +/// Longest BLAST+ error line quoted in a user message, in characters. +const DETAIL_MAX: usize = 160; +/// How long to wait for the stderr pipe to close after the child has exited (a grandchild could keep +/// it open); what was captured by then is kept. +const STDERR_GRACE: Duration = Duration::from_secs(2); +/// The environment BLAST+ runs with; nothing is inherited. +const ENV: &[(&str, &str)] = &[ + ("NCBI_DONT_USE_NCBIRC", "1"), + ("NCBI_DONT_USE_LOCAL_CONFIG", "1"), + ("BLAST_USAGE_REPORT", "false"), + ("HOME", "/tmp"), +]; + +pub const OOM_MESSAGE: &str = "search ran out of memory; try a shorter query or fewer species"; + +/// The user-facing message for a search killed at the wall-clock limit. +pub fn timeout_message(seconds: u64) -> String { + format!("search exceeded {seconds} s; try a shorter query or a smaller database") +} + +/// Starts `cfg.concurrency` workers. They run until aborted. +pub fn spawn_workers(cfg: Arc, store: Arc) -> Vec> { + (0..cfg.concurrency) + .map(|n| tokio::spawn(worker(n, cfg.clone(), store.clone()))) + .collect() +} + +async fn worker(n: usize, cfg: Arc, store: Arc) { + loop { + // Register as a waiter before looking at the FIFO, so a `create` between an empty + // `claim_next` and the wait still wakes this worker. A notification taken by a worker that + // then finds a job is forwarded to another waiter when the future is dropped, and the store + // itself wakes another worker when jobs remain after a claim. + let notified = store.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + // Never cancelled: a dropped `claim_next` could leave a job running with nobody to finish it. + let Some((id, req)) = store.claim_next().await else { + notified.await; + continue; + }; + // The job runs in its own task so a panic fails that job, not the worker. + let task = tokio::spawn(run_job(cfg.clone(), store.clone(), id.clone(), req)); + if let Err(e) = task.await { + tracing::error!(worker = n, job_id = %id, error = %e, "job task ended abnormally"); + store + .finish(&id, Outcome::Failed("internal error".into())) + .await; + } + } +} + +/// How the child ended. +enum Exit { + Status(ExitStatus), + TimedOut, +} + +async fn run_job(cfg: Arc, store: Arc, id: JobId, req: Validated) { + let dir = store.job_dir(&id); + let program = req.program; + let wall = cfg.wall_seconds_for(program); + let argv = command::build(&cfg, &req, &req.resolved, &dir); + tracing::info!( + job_id = %id, %program, database = %req.database, species = req.species.len(), + query_length = req.residues.len(), "job started" + ); + tracing::debug!(job_id = %id, ?argv, "argv"); + let started = Instant::now(); + + let outcome = match execute(&argv, &dir, Duration::from_secs(wall)).await { + Err(e) => { + tracing::error!(job_id = %id, error = %e, "could not start BLAST"); + Outcome::Failed("BLAST could not be started".into()) + } + Ok((Exit::TimedOut, _)) => Outcome::Failed(timeout_message(wall)), + Ok((Exit::Status(status), _)) if status.success() => { + let cap = cfg.max_results_bytes; + let d = dir.clone(); + match tokio::task::spawn_blocking(move || finalize_results(&d, cap)).await { + Ok(Ok(truncated)) => Outcome::Complete { truncated }, + Ok(Err(e)) => { + tracing::error!(job_id = %id, error = %e, "could not write results.tsv"); + Outcome::Failed("the results could not be written".into()) + } + Err(e) => { + tracing::error!(job_id = %id, error = %e, "finalize task failed"); + Outcome::Failed("the results could not be written".into()) + } + } + } + Ok((Exit::Status(status), stderr)) => { + let first = first_error_line(&stderr); + tracing::warn!(job_id = %id, ?status, stderr = first.unwrap_or(""), "BLAST failed"); + Outcome::Failed(failure_message(status, first)) + } + }; + if !matches!(outcome, Outcome::Complete { .. }) { + let _ = std::fs::remove_file(dir.join(RAW_FILE)); + } + tracing::info!( + job_id = %id, + state = if matches!(outcome, Outcome::Complete { .. }) { "complete" } else { "failed" }, + elapsed_ms = started.elapsed().as_millis() as u64, + "job finished" + ); + store.finish(&id, outcome).await; +} + +/// Runs `argv` in `dir` under the wall limit; returns how it ended and the captured stderr (also +/// written to `stderr.txt`). `Err` only when the process could not be spawned. +async fn execute(argv: &[OsString], dir: &Path, wall: Duration) -> io::Result<(Exit, Vec)> { + let mut child = Command::new(&argv[0]) + .args(&argv[1..]) + .env_clear() + .envs(ENV.iter().copied()) + .current_dir(dir) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn()?; + let captured = Arc::new(Mutex::new(Vec::new())); + let reader = child + .stderr + .take() + .map(|s| tokio::spawn(collect_stderr(s, captured.clone()))); + + let exit = match tokio::time::timeout(wall, child.wait()).await { + Ok(Ok(status)) => Exit::Status(status), + Ok(Err(e)) => return Err(e), + Err(_) => { + let _ = child.start_kill(); + let _ = child.wait().await; + Exit::TimedOut + } + }; + if let Some(mut r) = reader + && tokio::time::timeout(STDERR_GRACE, &mut r).await.is_err() + { + r.abort(); + } + let stderr = std::mem::take(&mut *captured.lock().unwrap_or_else(|p| p.into_inner())); + if let Err(e) = std::fs::write(dir.join(STDERR_FILE), &stderr) { + tracing::warn!(error = %e, "could not write stderr.txt"); + } + Ok((exit, stderr)) +} + +/// Reads the pipe to its end, keeping the first `STDERR_CAP` bytes. +async fn collect_stderr(mut pipe: ChildStderr, kept: Arc>>) { + let mut buf = [0u8; 8192]; + loop { + match pipe.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => { + let mut k = kept.lock().unwrap_or_else(|p| p.into_inner()); + let room = STDERR_CAP.saturating_sub(k.len()); + k.extend_from_slice(&buf[..n.min(room)]); + } + } + } +} + +/// The first stderr line that reads as an error (`Error`, `error`, `Warning`), else the first non-empty +/// line. +fn first_error_line(stderr: &[u8]) -> Option<&str> { + let text = match std::str::from_utf8(stderr) { + Ok(t) => t, + // Cut at the last valid boundary (the 64 KiB cap can split a character). + Err(e) => std::str::from_utf8(&stderr[..e.valid_up_to()]).unwrap_or(""), + }; + let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty()); + let first = lines.clone().next(); + lines + .find(|l| l.contains("Error") || l.contains("error") || l.contains("Warning")) + .or(first) +} + +/// `line` with every token that starts an absolute path (after `[`, `(` or a quote) replaced by `` and cut to `DETAIL_MAX` characters. +fn scrub(line: &str) -> String { + let words: Vec<&str> = line + .split_whitespace() + .map(|w| { + if w.trim_start_matches(['[', '(', '"', '\'']).starts_with('/') { + "" + } else { + w + } + }) + .collect(); + let joined = words.join(" "); + match joined.char_indices().nth(DETAIL_MAX) { + None => joined, + Some((cut, _)) => format!("{}…", &joined[..cut]), + } +} + +/// The user message for a non-zero exit: fixed text for the known BLAST+ failures and the +/// container's memory kill; otherwise `BLAST failed (exit N)` with the first error line, paths +/// removed and length bounded. +fn failure_message(status: ExitStatus, first: Option<&str>) -> String { + if status.signal() == Some(9) { + return OOM_MESSAGE.into(); + } + if let Some(line) = first { + if line.contains("BLAST Database error") { + return "database unavailable".into(); + } + if line.starts_with("Error") && line.contains("genetic code") { + return "invalid genetic code".into(); + } + if line.contains("no valid query") || line.contains("No valid query") { + return "the query has no valid residues".into(); + } + } + let head = match (status.code(), status.signal()) { + (Some(code), _) => format!("BLAST failed (exit {code})"), + (None, Some(sig)) => format!("BLAST failed (signal {sig})"), + (None, None) => "BLAST failed".to_string(), + }; + match first { + Some(line) => format!("{head}: {}", scrub(line)), + None => head, + } +} + +/// Writes `results.tsv` in `job_dir`: `HEADER`, then `raw.tsv`. When the two together exceed `cap` +/// bytes, `raw.tsv` is cut after its last `\n` that keeps the file within `cap`, and `true` is +/// returned. `raw.tsv` is deleted afterwards. The file is written to `results.tsv.tmp`, synced and +/// renamed, so a reader never sees a partial file. `cap` must hold the header (`Config` enforces +/// at least 1024 bytes). +pub fn finalize_results(job_dir: &Path, cap: u64) -> io::Result { + let raw_path = job_dir.join(RAW_FILE); + let tmp_path = job_dir.join("results.tsv.tmp"); + let mut raw = std::fs::File::open(&raw_path)?; + let raw_len = raw.metadata()?.len(); + let header = HEADER.as_bytes(); + let budget = cap.saturating_sub(header.len() as u64); + let (keep, truncated) = if raw_len <= budget { + (raw_len, false) + } else { + (last_row_end(&mut raw, budget)?, true) + }; + raw.seek(SeekFrom::Start(0))?; + let mut out = std::fs::File::create(&tmp_path)?; + out.write_all(header)?; + let copied = io::copy(&mut (&mut raw).take(keep), &mut out)?; + if copied != keep { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "raw.tsv shrank while being copied", + )); + } + out.sync_all()?; + drop(out); + std::fs::rename(&tmp_path, job_dir.join(RESULTS_FILE))?; + std::fs::remove_file(&raw_path)?; + Ok(truncated) +} + +/// Length of the longest prefix of the first `limit` bytes of `f` that ends in `\n` (0 if none), +/// scanning backwards from `limit` in blocks. +fn last_row_end(f: &mut std::fs::File, limit: u64) -> io::Result { + const BLOCK: u64 = 64 * 1024; + let mut end = limit; + let mut buf = vec![0u8; BLOCK as usize]; + while end > 0 { + let start = end.saturating_sub(BLOCK); + let len = (end - start) as usize; + f.seek(SeekFrom::Start(start))?; + f.read_exact(&mut buf[..len])?; + if let Some(i) = buf[..len].iter().rposition(|&b| b == b'\n') { + return Ok(start + i as u64 + 1); + } + end = start; + } + Ok(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn raw_dir(raw: &[u8]) -> tempfile::TempDir { + let d = tempfile::tempdir().unwrap(); + std::fs::write(d.path().join(RAW_FILE), raw).unwrap(); + d + } + + fn results(d: &tempfile::TempDir) -> Vec { + std::fs::read(d.path().join(RESULTS_FILE)).unwrap() + } + + const H: u64 = HEADER.len() as u64; + + #[test] + fn small_output_is_header_plus_raw() { + let d = raw_dir(b"a\tb\n"); + assert!(!finalize_results(d.path(), 1024).unwrap()); + assert_eq!(results(&d), [HEADER.as_bytes(), b"a\tb\n"].concat()); + assert!(!d.path().join(RAW_FILE).exists(), "raw.tsv deleted"); + assert!(!d.path().join("results.tsv.tmp").exists()); + } + + #[test] + fn empty_output_is_header_only() { + let d = raw_dir(b""); + assert!(!finalize_results(d.path(), 1024).unwrap()); + assert_eq!(results(&d), HEADER.as_bytes()); + } + + #[test] + fn exactly_at_cap_is_not_truncated() { + let row = b"0123456789\n"; + let raw: Vec = row.repeat(20); + let cap = H + raw.len() as u64; + let d = raw_dir(&raw); + assert!(!finalize_results(d.path(), cap).unwrap()); + assert_eq!(results(&d).len() as u64, cap); + } + + #[test] + fn one_byte_over_cuts_the_last_row() { + let row = b"0123456789\n"; + let raw: Vec = row.repeat(20); + let cap = H + raw.len() as u64 - 1; + let d = raw_dir(&raw); + assert!(finalize_results(d.path(), cap).unwrap()); + let r = results(&d); + assert_eq!(r, [HEADER.as_bytes(), &row.repeat(19)].concat()); + assert!(r.len() as u64 <= cap); + } + + #[test] + fn cut_lands_on_a_row_boundary_for_every_cap() { + // Rows of varying length; the cut must keep whole rows only and be the longest such prefix. + let mut raw = Vec::new(); + for i in 0..300 { + raw.extend(std::iter::repeat_n(b'x', i % 37 + 1)); + raw.push(b'\n'); + } + for cap in (H..H + raw.len() as u64 + 5).step_by(97) { + let d = raw_dir(&raw); + let truncated = finalize_results(d.path(), cap).unwrap(); + let r = results(&d); + assert!(r.len() as u64 <= cap, "cap {cap}"); + assert!(r.starts_with(HEADER.as_bytes())); + let body = &r[HEADER.len()..]; + assert!(raw.starts_with(body)); + assert!(body.is_empty() || body.ends_with(b"\n")); + assert_eq!(truncated, body.len() < raw.len(), "cap {cap}"); + // Longest: the next row would not have fit. + if truncated { + let next_end = + raw[body.len()..].iter().position(|&b| b == b'\n').unwrap() + body.len() + 1; + assert!(H + next_end as u64 > cap, "cap {cap}: cut too early"); + } + } + } + + #[test] + fn row_longer_than_a_scan_block_is_found() { + // One 200 KiB row then a short one: the last newline within budget is 200 KiB back. + let mut raw = vec![b'y'; 200 * 1024]; + raw.push(b'\n'); + raw.extend(vec![b'z'; 300 * 1024]); + raw.push(b'\n'); + let cap = H + 400 * 1024; + let d = raw_dir(&raw); + assert!(finalize_results(d.path(), cap).unwrap()); + assert_eq!(results(&d).len(), HEADER.len() + 200 * 1024 + 1); + } + + #[test] + fn no_newline_within_budget_leaves_header_only() { + let d = raw_dir(&vec![b'q'; 4096]); + assert!(finalize_results(d.path(), 1024).unwrap()); + assert_eq!(results(&d), HEADER.as_bytes()); + } + + #[test] + fn missing_raw_is_an_error() { + let d = tempfile::tempdir().unwrap(); + assert!(finalize_results(d.path(), 1024).is_err()); + assert!(!d.path().join(RESULTS_FILE).exists()); + } + + #[test] + fn header_matches_outfmt_columns() { + let cols: Vec<&str> = HEADER.trim_end().split('\t').collect(); + assert_eq!(cols.len(), 13); + assert_eq!(cols.first(), Some(&"qseqid")); + assert_eq!(cols.last(), Some(&"stitle")); + assert!(HEADER.ends_with('\n') && !HEADER.ends_with("\n\n")); + } + + fn status(raw: i32) -> ExitStatus { + ExitStatus::from_raw(raw) + } + + #[test] + fn failure_messages() { + // Raw wait statuses: exit code N is N << 8; signal S is S. + assert_eq!(failure_message(status(9), None), OOM_MESSAGE); + assert_eq!( + failure_message( + status(2 << 8), + Some( + "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" + ) + ), + "database unavailable" + ); + assert_eq!( + failure_message(status(1 << 8), Some("Error: Invalid genetic code 99")), + "invalid genetic code" + ); + assert_eq!( + failure_message( + status(1 << 8), + Some( + "Warning: [blastn] Query_1 query: Could not calculate ungapped Karlin-Altschul parameters due to an invalid query sequence or its translation. Please verify the query sequence(s) and/or filtering options; no valid query" + ) + ), + "the query has no valid residues" + ); + let m = failure_message( + status(3 << 8), + Some("BLAST query/options error: Invalid value for word_size in /etc/x/y.conf"), + ); + assert_eq!( + m, + "BLAST failed (exit 3): BLAST query/options error: Invalid value for word_size in " + ); + assert_eq!( + failure_message(status(3 << 8), None), + "BLAST failed (exit 3)" + ); + assert_eq!( + failure_message(status(15), None), + "BLAST failed (signal 15)" + ); + let long = "Error: ".to_string() + &"e".repeat(5000); + assert!(failure_message(status(1 << 8), Some(&long)).chars().count() < DETAIL_MAX + 40); + } + + #[test] + fn first_error_line_prefers_errors() { + let s = b"\n some banner\nWarning: thing\nError: bad\n"; + assert_eq!(first_error_line(s), Some("Warning: thing")); + assert_eq!(first_error_line(b"\n\nplain\n"), Some("plain")); + assert_eq!(first_error_line(b""), None); + // A cut multi-byte character at the end is tolerated. + let mut cut = b"Error: caf".to_vec(); + cut.push(0xC3); + assert_eq!(first_error_line(&cut), Some("Error: caf")); + } +} diff --git a/blast-jobs/tests/fake-blast/blastn b/blast-jobs/tests/fake-blast/blastn new file mode 100755 index 0000000..58a632e --- /dev/null +++ b/blast-jobs/tests/fake-blast/blastn @@ -0,0 +1,53 @@ +#!/bin/sh +# Fake BLAST+ program for the blast-jobs tests (the same script is committed as blastn, blastp, blastx, +# tblastn and tblastx, mode 100755). It records its argv, one element per line, to argv.txt in the job +# directory, then writes canned.tsv to the -out path. +# +# The runner gives BLAST+ a fixed environment, so the test controls come from +# /fake-blast.env (sourced when present; state_root is three levels above -out): +# FAKE_BLAST_SLEEP= sleep before doing anything else +# FAKE_BLAST_FAIL=db BLAST database error on stderr, exit 2 +# FAKE_BLAST_FAIL=oom die by SIGKILL, as under the container's memory cap +# FAKE_BLAST_FAIL=noisy 200 KiB of error lines on stderr, exit 4 +# FAKE_BLAST_FAIL= one options error line on stderr, exit 3 +# FAKE_BLAST_BIG=1 write 20 MiB of rows instead of canned.tsv +PATH=/usr/bin:/bin +export PATH +here=$(dirname "$0") +out= +prev= +for a in "$@"; do + if [ "$prev" = "-out" ]; then out=$a; fi + prev=$a +done +if [ -z "$out" ]; then + echo "fake-blast: no -out argument" >&2 + exit 64 +fi +job=$(dirname "$out") +ctl=$(dirname "$(dirname "$job")")/fake-blast.env +if [ -f "$ctl" ]; then . "$ctl"; fi +printf '%s\n' "$0" "$@" > "$job/argv.txt" +if [ -n "$FAKE_BLAST_SLEEP" ]; then + # Detached from the stderr pipe, so a killed parent does not leave the pipe open. + sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 +fi +case "$FAKE_BLAST_FAIL" in + "") ;; + db) + echo "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" >&2 + exit 2 ;; + oom) + kill -9 $$ ;; + noisy) + yes "Error: noisy failure reading /some/private/path/file.nsq" | head -c 204800 >&2 + exit 4 ;; + *) + echo "BLAST query/options error: fake failure for /opt/private/thing" >&2 + exit 3 ;; +esac +if [ -n "$FAKE_BLAST_BIG" ]; then + yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" +else + cat "$here/canned.tsv" > "$out" +fi diff --git a/blast-jobs/tests/fake-blast/blastp b/blast-jobs/tests/fake-blast/blastp new file mode 100755 index 0000000..58a632e --- /dev/null +++ b/blast-jobs/tests/fake-blast/blastp @@ -0,0 +1,53 @@ +#!/bin/sh +# Fake BLAST+ program for the blast-jobs tests (the same script is committed as blastn, blastp, blastx, +# tblastn and tblastx, mode 100755). It records its argv, one element per line, to argv.txt in the job +# directory, then writes canned.tsv to the -out path. +# +# The runner gives BLAST+ a fixed environment, so the test controls come from +# /fake-blast.env (sourced when present; state_root is three levels above -out): +# FAKE_BLAST_SLEEP= sleep before doing anything else +# FAKE_BLAST_FAIL=db BLAST database error on stderr, exit 2 +# FAKE_BLAST_FAIL=oom die by SIGKILL, as under the container's memory cap +# FAKE_BLAST_FAIL=noisy 200 KiB of error lines on stderr, exit 4 +# FAKE_BLAST_FAIL= one options error line on stderr, exit 3 +# FAKE_BLAST_BIG=1 write 20 MiB of rows instead of canned.tsv +PATH=/usr/bin:/bin +export PATH +here=$(dirname "$0") +out= +prev= +for a in "$@"; do + if [ "$prev" = "-out" ]; then out=$a; fi + prev=$a +done +if [ -z "$out" ]; then + echo "fake-blast: no -out argument" >&2 + exit 64 +fi +job=$(dirname "$out") +ctl=$(dirname "$(dirname "$job")")/fake-blast.env +if [ -f "$ctl" ]; then . "$ctl"; fi +printf '%s\n' "$0" "$@" > "$job/argv.txt" +if [ -n "$FAKE_BLAST_SLEEP" ]; then + # Detached from the stderr pipe, so a killed parent does not leave the pipe open. + sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 +fi +case "$FAKE_BLAST_FAIL" in + "") ;; + db) + echo "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" >&2 + exit 2 ;; + oom) + kill -9 $$ ;; + noisy) + yes "Error: noisy failure reading /some/private/path/file.nsq" | head -c 204800 >&2 + exit 4 ;; + *) + echo "BLAST query/options error: fake failure for /opt/private/thing" >&2 + exit 3 ;; +esac +if [ -n "$FAKE_BLAST_BIG" ]; then + yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" +else + cat "$here/canned.tsv" > "$out" +fi diff --git a/blast-jobs/tests/fake-blast/blastx b/blast-jobs/tests/fake-blast/blastx new file mode 100755 index 0000000..58a632e --- /dev/null +++ b/blast-jobs/tests/fake-blast/blastx @@ -0,0 +1,53 @@ +#!/bin/sh +# Fake BLAST+ program for the blast-jobs tests (the same script is committed as blastn, blastp, blastx, +# tblastn and tblastx, mode 100755). It records its argv, one element per line, to argv.txt in the job +# directory, then writes canned.tsv to the -out path. +# +# The runner gives BLAST+ a fixed environment, so the test controls come from +# /fake-blast.env (sourced when present; state_root is three levels above -out): +# FAKE_BLAST_SLEEP= sleep before doing anything else +# FAKE_BLAST_FAIL=db BLAST database error on stderr, exit 2 +# FAKE_BLAST_FAIL=oom die by SIGKILL, as under the container's memory cap +# FAKE_BLAST_FAIL=noisy 200 KiB of error lines on stderr, exit 4 +# FAKE_BLAST_FAIL= one options error line on stderr, exit 3 +# FAKE_BLAST_BIG=1 write 20 MiB of rows instead of canned.tsv +PATH=/usr/bin:/bin +export PATH +here=$(dirname "$0") +out= +prev= +for a in "$@"; do + if [ "$prev" = "-out" ]; then out=$a; fi + prev=$a +done +if [ -z "$out" ]; then + echo "fake-blast: no -out argument" >&2 + exit 64 +fi +job=$(dirname "$out") +ctl=$(dirname "$(dirname "$job")")/fake-blast.env +if [ -f "$ctl" ]; then . "$ctl"; fi +printf '%s\n' "$0" "$@" > "$job/argv.txt" +if [ -n "$FAKE_BLAST_SLEEP" ]; then + # Detached from the stderr pipe, so a killed parent does not leave the pipe open. + sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 +fi +case "$FAKE_BLAST_FAIL" in + "") ;; + db) + echo "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" >&2 + exit 2 ;; + oom) + kill -9 $$ ;; + noisy) + yes "Error: noisy failure reading /some/private/path/file.nsq" | head -c 204800 >&2 + exit 4 ;; + *) + echo "BLAST query/options error: fake failure for /opt/private/thing" >&2 + exit 3 ;; +esac +if [ -n "$FAKE_BLAST_BIG" ]; then + yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" +else + cat "$here/canned.tsv" > "$out" +fi diff --git a/blast-jobs/tests/fake-blast/canned.tsv b/blast-jobs/tests/fake-blast/canned.tsv new file mode 100644 index 0000000..88e4f09 --- /dev/null +++ b/blast-jobs/tests/fake-blast/canned.tsv @@ -0,0 +1,3 @@ +query FBtr0070000 100.000 120 0 0 1 120 301 420 2.1e-60 222 Notch-RA type=mRNA; loc=X:3134870..3172221; ID=FBtr0070000; name=N-RA +query FBtr0070001 98.333 120 2 0 1 120 301 420 4.4e-57 211 Notch-RB type=mRNA; loc=X:3134870..3172221; ID=FBtr0070001; name=N-RB +query FBtr0080000 85.000 60 9 0 30 89 12 71 1.0e-08 55.4 dx-RA type=mRNA; loc=X:2111111..2122222; ID=FBtr0080000; name=dx-RA diff --git a/blast-jobs/tests/fake-blast/tblastn b/blast-jobs/tests/fake-blast/tblastn new file mode 100755 index 0000000..58a632e --- /dev/null +++ b/blast-jobs/tests/fake-blast/tblastn @@ -0,0 +1,53 @@ +#!/bin/sh +# Fake BLAST+ program for the blast-jobs tests (the same script is committed as blastn, blastp, blastx, +# tblastn and tblastx, mode 100755). It records its argv, one element per line, to argv.txt in the job +# directory, then writes canned.tsv to the -out path. +# +# The runner gives BLAST+ a fixed environment, so the test controls come from +# /fake-blast.env (sourced when present; state_root is three levels above -out): +# FAKE_BLAST_SLEEP= sleep before doing anything else +# FAKE_BLAST_FAIL=db BLAST database error on stderr, exit 2 +# FAKE_BLAST_FAIL=oom die by SIGKILL, as under the container's memory cap +# FAKE_BLAST_FAIL=noisy 200 KiB of error lines on stderr, exit 4 +# FAKE_BLAST_FAIL= one options error line on stderr, exit 3 +# FAKE_BLAST_BIG=1 write 20 MiB of rows instead of canned.tsv +PATH=/usr/bin:/bin +export PATH +here=$(dirname "$0") +out= +prev= +for a in "$@"; do + if [ "$prev" = "-out" ]; then out=$a; fi + prev=$a +done +if [ -z "$out" ]; then + echo "fake-blast: no -out argument" >&2 + exit 64 +fi +job=$(dirname "$out") +ctl=$(dirname "$(dirname "$job")")/fake-blast.env +if [ -f "$ctl" ]; then . "$ctl"; fi +printf '%s\n' "$0" "$@" > "$job/argv.txt" +if [ -n "$FAKE_BLAST_SLEEP" ]; then + # Detached from the stderr pipe, so a killed parent does not leave the pipe open. + sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 +fi +case "$FAKE_BLAST_FAIL" in + "") ;; + db) + echo "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" >&2 + exit 2 ;; + oom) + kill -9 $$ ;; + noisy) + yes "Error: noisy failure reading /some/private/path/file.nsq" | head -c 204800 >&2 + exit 4 ;; + *) + echo "BLAST query/options error: fake failure for /opt/private/thing" >&2 + exit 3 ;; +esac +if [ -n "$FAKE_BLAST_BIG" ]; then + yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" +else + cat "$here/canned.tsv" > "$out" +fi diff --git a/blast-jobs/tests/fake-blast/tblastx b/blast-jobs/tests/fake-blast/tblastx new file mode 100755 index 0000000..58a632e --- /dev/null +++ b/blast-jobs/tests/fake-blast/tblastx @@ -0,0 +1,53 @@ +#!/bin/sh +# Fake BLAST+ program for the blast-jobs tests (the same script is committed as blastn, blastp, blastx, +# tblastn and tblastx, mode 100755). It records its argv, one element per line, to argv.txt in the job +# directory, then writes canned.tsv to the -out path. +# +# The runner gives BLAST+ a fixed environment, so the test controls come from +# /fake-blast.env (sourced when present; state_root is three levels above -out): +# FAKE_BLAST_SLEEP= sleep before doing anything else +# FAKE_BLAST_FAIL=db BLAST database error on stderr, exit 2 +# FAKE_BLAST_FAIL=oom die by SIGKILL, as under the container's memory cap +# FAKE_BLAST_FAIL=noisy 200 KiB of error lines on stderr, exit 4 +# FAKE_BLAST_FAIL= one options error line on stderr, exit 3 +# FAKE_BLAST_BIG=1 write 20 MiB of rows instead of canned.tsv +PATH=/usr/bin:/bin +export PATH +here=$(dirname "$0") +out= +prev= +for a in "$@"; do + if [ "$prev" = "-out" ]; then out=$a; fi + prev=$a +done +if [ -z "$out" ]; then + echo "fake-blast: no -out argument" >&2 + exit 64 +fi +job=$(dirname "$out") +ctl=$(dirname "$(dirname "$job")")/fake-blast.env +if [ -f "$ctl" ]; then . "$ctl"; fi +printf '%s\n' "$0" "$@" > "$job/argv.txt" +if [ -n "$FAKE_BLAST_SLEEP" ]; then + # Detached from the stderr pipe, so a killed parent does not leave the pipe open. + sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 +fi +case "$FAKE_BLAST_FAIL" in + "") ;; + db) + echo "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" >&2 + exit 2 ;; + oom) + kill -9 $$ ;; + noisy) + yes "Error: noisy failure reading /some/private/path/file.nsq" | head -c 204800 >&2 + exit 4 ;; + *) + echo "BLAST query/options error: fake failure for /opt/private/thing" >&2 + exit 3 ;; +esac +if [ -n "$FAKE_BLAST_BIG" ]; then + yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" +else + cat "$here/canned.tsv" > "$out" +fi diff --git a/blast-jobs/tests/runner.rs b/blast-jobs/tests/runner.rs new file mode 100644 index 0000000..a812742 --- /dev/null +++ b/blast-jobs/tests/runner.rs @@ -0,0 +1,412 @@ +//! Runner tests against the fake BLAST+ scripts in `tests/fake-blast/` (design 3.12). Each test gets +//! its own state root; the fake reads its controls from `/fake-blast.env` because the +//! runner gives BLAST+ a fixed environment. +use blast_jobs::catalogue::Catalogue; +use blast_jobs::command; +use blast_jobs::config::Config; +use blast_jobs::jobid::JobId; +use blast_jobs::runner::{self, HEADER, OOM_MESSAGE, STDERR_CAP}; +use blast_jobs::store::Store; +use blast_jobs::types::{JobStatus, Program, State, SubmitRequest}; +use blast_jobs::validate; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::task::JoinHandle; + +const PROGRAMS: [&str; 5] = ["blastn", "blastp", "blastx", "tblastn", "tblastx"]; + +fn manifest() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +struct Env { + _tmp: tempfile::TempDir, + cfg: Arc, + cat: Catalogue, + store: Arc, + workers: Vec>, +} + +impl Drop for Env { + fn drop(&mut self) { + for w in &self.workers { + w.abort(); + } + } +} + +/// A store and running workers over the fixture catalogue and the fake BLAST binaries. `overrides` +/// are merged into the config; `controls` is written to `fake-blast.env`. +fn setup(overrides: Value, controls: &str) -> Env { + let tmp = tempfile::tempdir().unwrap(); + let state = tmp.path().join("state"); + std::fs::create_dir_all(&state).unwrap(); + std::fs::write(state.join("fake-blast.env"), controls).unwrap(); + let fixture = manifest().join("tests/fixture"); + let mut v = json!({ + "bind": "127.0.0.1:0", + "blast_bin_dir": manifest().join("tests/fake-blast"), + "database_root": fixture.join("db"), + "catalogue": fixture.join("catalogue.json"), + "database_release": "FB2023_04", + "state_root": state, + }); + for (k, val) in overrides.as_object().unwrap() { + v[k] = val.clone(); + } + let path = tmp.path().join("config.json"); + std::fs::write(&path, v.to_string()).unwrap(); + let cfg = Arc::new(Config::load(&path).unwrap()); + let cat = Catalogue::load(&cfg.catalogue, &cfg.database_root).unwrap(); + let store = Store::open(&cfg.state_root, cfg.max_queued).unwrap(); + let workers = runner::spawn_workers(cfg.clone(), store.clone()); + assert_eq!(workers.len(), cfg.concurrency); + Env { + _tmp: tmp, + cfg, + cat, + store, + workers, + } +} + +fn request(program: &str) -> SubmitRequest { + let p = Program::parse(program).unwrap(); + SubmitRequest { + program: program.into(), + database: if p.protein_database() { + "translation" + } else { + "assembly" + } + .into(), + species: vec!["dmel".into()], + sequence: if p.protein_query() { + ">q\nMKLVQWERTY\n" + } else { + ">q\nACGTACGTACGT\n" + } + .into(), + ..Default::default() + } +} + +impl Env { + async fn submit(&self, req: SubmitRequest) -> JobId { + let v = validate::check(&req, &self.cfg, &self.cat).unwrap(); + self.store.create(v).await.unwrap().0 + } + + fn status(&self, id: &JobId) -> JobStatus { + self.store.get(id).unwrap() + } + + async fn wait_terminal(&self, id: &JobId, within: Duration) -> JobStatus { + let end = Instant::now() + within; + loop { + let s = self.status(id); + if matches!(s.state, State::Complete | State::Failed) { + return s; + } + assert!( + Instant::now() < end, + "job still {:?} after {within:?}", + s.state + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + fn dir(&self, id: &JobId) -> PathBuf { + self.store.job_dir(id) + } +} + +fn canned() -> Vec { + std::fs::read(manifest().join("tests/fake-blast/canned.tsv")).unwrap() +} + +#[test] +fn fake_blast_scripts_are_executable() { + use std::os::unix::fs::PermissionsExt; + let dir = manifest().join("tests/fake-blast"); + for p in PROGRAMS { + let path = dir.join(p); + let mode = std::fs::metadata(&path) + .unwrap_or_else(|e| panic!("{} is missing: {e}", path.display())) + .permissions() + .mode(); + assert_eq!( + mode & 0o111, + 0o111, + "{} must be executable (git update-index --chmod=+x); mode is {mode:o}", + path.display() + ); + } + // In a git checkout, the committed mode must be 100755 too, or a fresh clone loses the bit. + let out = std::process::Command::new("git") + .arg("-C") + .arg(&dir) + .args(["ls-files", "-s", "--"]) + .args(PROGRAMS) + .output(); + match out { + Ok(o) if o.status.success() && !o.stdout.is_empty() => { + let listing = String::from_utf8_lossy(&o.stdout); + for line in listing.lines() { + assert!( + line.starts_with("100755 "), + "committed without the executable bit: {line}" + ); + } + } + _ => eprintln!("fake-blast git mode check skipped: not in a git checkout with them staged"), + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn success_writes_header_then_rows_and_argv_matches_the_builder() { + let env = setup(json!({}), ""); + for program in PROGRAMS { + let mut req = request(program); + if program == "blastn" { + // Two species: the `-db` value must arrive as one argv element with a space. + req.species = vec!["dmel".into(), "dsim".into()]; + req.advanced = Some("-dust no -lcase_masking -penalty -3".into()); + } + let validated = validate::check(&req, &env.cfg, &env.cat).unwrap(); + let id = env.submit(req).await; + let s = env.wait_terminal(&id, Duration::from_secs(10)).await; + assert_eq!(s.state, State::Complete, "{program}: {:?}", s.error); + assert!(!s.truncated); + let dir = env.dir(&id); + let body = std::fs::read(dir.join("results.tsv")).unwrap(); + assert_eq!(body, [HEADER.as_bytes(), &canned()].concat(), "{program}"); + assert!(!dir.join("raw.tsv").exists(), "{program}: raw.tsv deleted"); + assert!(dir.join("stderr.txt").exists()); + + let want: Vec = command::build(&env.cfg, &validated, &validated.resolved, &dir) + .into_iter() + .map(|s| s.into_string().unwrap()) + .collect(); + let got: Vec = std::fs::read_to_string(dir.join("argv.txt")) + .unwrap() + .lines() + .map(String::from) + .collect(); + assert_eq!(got, want, "{program}: argv as the fake BLAST received it"); + assert!(got[0].ends_with(&format!("/{program}"))); + let query = &got[got.iter().position(|a| a == "-query").unwrap() + 1]; + assert!( + Path::new(query).is_file(), + "{program}: -query names the file the store wrote" + ); + if program == "blastn" { + let db = &got[got.iter().position(|a| a == "-db").unwrap() + 1]; + assert_eq!(db.split(' ').count(), 2, "one -db element, two prefixes"); + assert!(got.windows(2).any(|w| w == ["-dust", "no"])); + assert!(got.windows(2).any(|w| w == ["-penalty", "-3"])); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn database_error_becomes_a_fixed_message() { + let env = setup(json!({}), "FAKE_BLAST_FAIL=db\n"); + let id = env.submit(request("blastn")).await; + let s = env.wait_terminal(&id, Duration::from_secs(10)).await; + assert_eq!(s.state, State::Failed); + assert_eq!(s.error.as_deref(), Some("database unavailable")); + let dir = env.dir(&id); + assert!(!dir.join("results.tsv").exists()); + assert!(!dir.join("raw.tsv").exists()); + let stderr = std::fs::read_to_string(dir.join("stderr.txt")).unwrap(); + assert!( + stderr.contains("BLAST Database error"), + "stderr kept server-side" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn other_failure_names_the_exit_and_strips_paths() { + let env = setup(json!({}), "FAKE_BLAST_FAIL=other\n"); + let id = env.submit(request("blastp")).await; + let s = env.wait_terminal(&id, Duration::from_secs(10)).await; + assert_eq!(s.state, State::Failed); + assert_eq!( + s.error.as_deref(), + Some("BLAST failed (exit 3): BLAST query/options error: fake failure for ") + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn sigkill_not_from_us_is_out_of_memory() { + let env = setup(json!({}), "FAKE_BLAST_FAIL=oom\n"); + let id = env.submit(request("tblastn")).await; + let s = env.wait_terminal(&id, Duration::from_secs(10)).await; + assert_eq!(s.state, State::Failed); + assert_eq!(s.error.as_deref(), Some(OOM_MESSAGE)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stderr_is_capped_and_the_message_bounded() { + let env = setup(json!({}), "FAKE_BLAST_FAIL=noisy\n"); + let id = env.submit(request("blastx")).await; + let s = env.wait_terminal(&id, Duration::from_secs(10)).await; + assert_eq!(s.state, State::Failed); + let msg = s.error.unwrap(); + assert!( + msg.starts_with("BLAST failed (exit 4): Error: noisy failure"), + "{msg}" + ); + assert!(!msg.contains("/some"), "paths never reach the user: {msg}"); + assert!(msg.len() < 300, "{} bytes", msg.len()); + let kept = std::fs::metadata(env.dir(&id).join("stderr.txt")) + .unwrap() + .len(); + assert_eq!( + kept, STDERR_CAP as u64, + "first 64 KiB kept, the rest drained" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn wall_limit_kills_and_fails_the_job() { + let env = setup(json!({"wall_seconds": 1}), "FAKE_BLAST_SLEEP=5\n"); + let started = Instant::now(); + let id = env.submit(request("blastn")).await; + let s = env.wait_terminal(&id, Duration::from_secs(10)).await; + let took = started.elapsed(); + assert_eq!(s.state, State::Failed); + assert_eq!( + s.error.as_deref(), + Some("search exceeded 1 s; try a shorter query or a smaller database") + ); + assert!( + took < Duration::from_secs(4), + "killed at the limit, took {took:?}" + ); + assert!(!env.dir(&id).join("results.tsv").exists()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn translated_programs_use_the_translated_wall_limit() { + let env = setup( + json!({"wall_seconds": 30, "translated_wall_seconds": 1}), + "FAKE_BLAST_SLEEP=3\n", + ); + let tx = env.submit(request("tblastx")).await; + let n = env.submit(request("blastn")).await; + let s = env.wait_terminal(&tx, Duration::from_secs(10)).await; + assert_eq!(s.state, State::Failed); + assert_eq!( + s.error.as_deref(), + Some("search exceeded 1 s; try a shorter query or a smaller database") + ); + let s = env.wait_terminal(&n, Duration::from_secs(10)).await; + assert_eq!( + s.state, + State::Complete, + "blastn runs under wall_seconds: {:?}", + s.error + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn big_output_is_cut_at_a_row_boundary_within_the_cap() { + const CAP: u64 = 1024 * 1024; + let env = setup(json!({"max_results_bytes": CAP}), "FAKE_BLAST_BIG=1\n"); + let id = env.submit(request("blastn")).await; + let s = env.wait_terminal(&id, Duration::from_secs(30)).await; + assert_eq!(s.state, State::Complete, "{:?}", s.error); + assert!(s.truncated, "the store records the cut"); + let dir = env.dir(&id); + let body = std::fs::read(dir.join("results.tsv")).unwrap(); + assert!(body.len() as u64 <= CAP, "{} bytes", body.len()); + assert!(body.starts_with(HEADER.as_bytes())); + assert!(body.ends_with(b"\n")); + let row = canned().split(|&b| b == b'\n').next().unwrap().len() + 1; + assert!( + body.len() as u64 + row as u64 > CAP, + "longest cut: one more row would pass the cap" + ); + for line in body[HEADER.len()..] + .split(|&b| b == b'\n') + .filter(|l| !l.is_empty()) + { + assert_eq!(line.split(|&b| b == b'\t').count(), 13, "whole rows only"); + } + assert!(!dir.join("raw.tsv").exists(), "raw.tsv deleted"); +} + +async fn wait_counts(store: &Store, want: (usize, usize), within: Duration) { + let end = Instant::now() + within; + while store.counts() != want { + assert!( + Instant::now() < end, + "counts {:?}, expected {want:?}", + store.counts() + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +/// `concurrency` jobs run at once and the next waits in the queue until a slot frees. +async fn runs_n_at_a_time(n: usize) { + let env = setup(json!({"concurrency": n}), "FAKE_BLAST_SLEEP=2\n"); + // Let the workers go idle first, so the jobs reach them through the store's wake-up. + tokio::time::sleep(Duration::from_millis(100)).await; + let mut ids = Vec::new(); + for _ in 0..=n { + ids.push(env.submit(request("blastn")).await); + } + wait_counts(&env.store, (1, n), Duration::from_millis(1500)).await; + // Hold for a while: no extra job starts while the slots are busy. + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!(env.store.counts(), (1, n)); + let last = &ids[n]; + assert_eq!(env.status(last).state, State::Queued); + assert_eq!(env.status(last).queue_position, Some(1)); + for id in &ids[..n] { + assert_eq!(env.status(id).state, State::Running); + assert_eq!(env.status(id).queue_position, Some(0)); + } + // The queued job starts once a slot frees, and everything completes. + wait_counts(&env.store, (0, 1), Duration::from_secs(5)).await; + for id in &ids { + let s = env.wait_terminal(id, Duration::from_secs(10)).await; + assert_eq!(s.state, State::Complete, "{:?}", s.error); + } + assert_eq!(env.store.counts(), (0, 0)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrency_two_runs_two_and_queues_a_third() { + runs_n_at_a_time(2).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrency_three_runs_three_and_queues_a_fourth() { + runs_n_at_a_time(3).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrency_one_runs_jobs_in_submission_order() { + let env = setup(json!({"concurrency": 1}), ""); + let mut ids = Vec::new(); + for _ in 0..4 { + ids.push(env.submit(request("blastp")).await); + } + let mut finished = Vec::new(); + for id in &ids { + env.wait_terminal(id, Duration::from_secs(10)).await; + let m = std::fs::metadata(env.dir(id).join("results.tsv")) + .unwrap() + .modified() + .unwrap(); + finished.push(m); + } + assert!(finished.windows(2).all(|w| w[0] <= w[1]), "FIFO order"); +} From 307adb31d07fc3046c6076504769e01e856846d1 Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Mon, 12 Oct 2026 17:59:46 +0000 Subject: [PATCH 06/12] feat(blast-jobs): internal job API with bearer auth and health --- blast-jobs/src/auth.rs | 132 +++++++++ blast-jobs/src/http.rs | 394 +++++++++++++++++++++++++++ blast-jobs/src/lib.rs | 2 + blast-jobs/src/main.rs | 96 ++++++- blast-jobs/tests/api.rs | 590 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 1205 insertions(+), 9 deletions(-) create mode 100644 blast-jobs/src/auth.rs create mode 100644 blast-jobs/src/http.rs create mode 100644 blast-jobs/tests/api.rs diff --git a/blast-jobs/src/auth.rs b/blast-jobs/src/auth.rs new file mode 100644 index 0000000..652b171 --- /dev/null +++ b/blast-jobs/src/auth.rs @@ -0,0 +1,132 @@ +//! Bearer authentication for the job routes (design 3.2). The token comes from `BLAST_JOBS_TOKEN`; an +//! empty or unset token refuses to start unless `BLAST_JOBS_ALLOW_NO_AUTH=1` (local runs). The token is +//! compared in constant time and never logged. +use crate::http::AppState; +use axum::extract::{Request, State}; +use axum::http::{StatusCode, header}; +use axum::middleware::Next; +use axum::response::{IntoResponse, Response}; +use std::fmt; +use std::sync::Arc; +use subtle::ConstantTimeEq; + +pub const TOKEN_ENV: &str = "BLAST_JOBS_TOKEN"; +pub const ALLOW_NO_AUTH_ENV: &str = "BLAST_JOBS_ALLOW_NO_AUTH"; + +#[derive(Clone)] +pub enum Auth { + /// Every job route needs `Authorization: Bearer `. + Token(String), + /// No authentication (only with `BLAST_JOBS_ALLOW_NO_AUTH=1`). + Disabled, +} + +/// Never prints the token. +impl fmt::Debug for Auth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Auth::Token(_) => f.write_str("Auth::Token()"), + Auth::Disabled => f.write_str("Auth::Disabled"), + } + } +} + +impl Auth { + /// Reads `BLAST_JOBS_TOKEN` and `BLAST_JOBS_ALLOW_NO_AUTH`. + pub fn from_env() -> anyhow::Result { + Self::from_values( + std::env::var(TOKEN_ENV).ok(), + std::env::var(ALLOW_NO_AUTH_ENV).ok(), + ) + } + + /// `from_env` without the environment: a non-empty token wins; otherwise `allow_no_auth` must be + /// exactly `"1"`. + pub fn from_values( + token: Option, + allow_no_auth: Option, + ) -> anyhow::Result { + match token { + Some(t) if !t.trim().is_empty() => Ok(Auth::Token(t)), + _ if allow_no_auth.as_deref() == Some("1") => Ok(Auth::Disabled), + _ => anyhow::bail!( + "{TOKEN_ENV} is empty or unset; set it, or set {ALLOW_NO_AUTH_ENV}=1 for a local run" + ), + } + } + + /// Whether an `Authorization` header value carries the token. Constant time in the token bytes. + pub fn accepts(&self, authorization: Option<&[u8]>) -> bool { + match self { + Auth::Disabled => true, + Auth::Token(t) => authorization + .and_then(|h| h.strip_prefix(b"Bearer ")) + .is_some_and(|given| bool::from(given.ct_eq(t.as_bytes()))), + } + } +} + +/// Middleware for the job routes: `401 {"error":"unauthorized"}` unless the bearer token matches. +pub async fn require_bearer( + State(state): State>, + req: Request, + next: Next, +) -> Response { + let given = req + .headers() + .get(header::AUTHORIZATION) + .map(|v| v.as_bytes()); + if state.auth().accepts(given) { + next.run(req).await + } else { + tracing::warn!(path = %req.uri().path(), "request with a missing or wrong bearer token"); + ( + StatusCode::UNAUTHORIZED, + axum::Json(serde_json::json!({"error": "unauthorized"})), + ) + .into_response() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn token_required_unless_explicitly_disabled() { + assert!(matches!( + Auth::from_values(Some("s3cret".into()), None).unwrap(), + Auth::Token(_) + )); + assert!(Auth::from_values(None, None).is_err()); + assert!(Auth::from_values(Some(" ".into()), None).is_err()); + assert!(Auth::from_values(None, Some("true".into())).is_err()); + assert!(matches!( + Auth::from_values(None, Some("1".into())).unwrap(), + Auth::Disabled + )); + assert!(matches!( + Auth::from_values(Some("".into()), Some("1".into())).unwrap(), + Auth::Disabled + )); + } + + #[test] + fn accepts_only_the_exact_bearer_token() { + let a = Auth::Token("s3cret".into()); + assert!(a.accepts(Some(b"Bearer s3cret"))); + assert!(!a.accepts(None)); + assert!(!a.accepts(Some(b"Bearer s3cre"))); + assert!(!a.accepts(Some(b"Bearer s3cretx"))); + assert!(!a.accepts(Some(b"bearer s3cret"))); + assert!(!a.accepts(Some(b"Basic s3cret"))); + assert!(!a.accepts(Some(b"s3cret"))); + assert!(Auth::Disabled.accepts(None)); + } + + #[test] + fn debug_redacts_the_token() { + let s = format!("{:?}", Auth::Token("s3cret".into())); + assert!(!s.contains("s3cret"), "{s}"); + } +} diff --git a/blast-jobs/src/http.rs b/blast-jobs/src/http.rs new file mode 100644 index 0000000..d774f24 --- /dev/null +++ b/blast-jobs/src/http.rs @@ -0,0 +1,394 @@ +//! The internal job API (`docs/handoff-private-side.md` §7, design 3.6 to 3.8). +//! +//! - `POST /internal/jobs` -> `202 SubmitResponse`; `400 {error, message}` for a rejected request +//! (`invalid_request` when the body is not a request object at all, `413` for a body over +//! `MAX_BODY_BYTES`); `429 {error:"queue_full", retry_after_seconds}`; `503` when the job cannot be +//! stored. +//! - `GET /internal/jobs/{id}` -> `200 JobStatus` (no `result_url`/`downloads`, ruling 5) or +//! `404 {error:"unknown_job"}`. +//! - `GET /internal/jobs/{id}/results.tsv` -> `200 text/tab-separated-values` with `Content-Length` +//! and `X-Blast-Jobs-Truncated: true` when the file was cut at the cap; `409 {error:"not_complete", +//! state}` before completion; `404`. +//! - `GET /internal/health[?liveness=1]` -> `200` or `503`, unauthenticated. +//! +//! The three job routes sit behind the bearer middleware (`auth::require_bearer`). Job ids pass +//! `JobId::parse` before any lookup, so a malformed id is a `404` and never reaches a path. +use crate::auth::{self, Auth}; +use crate::catalogue::Catalogue; +use crate::config::Config; +use crate::jobid::JobId; +use crate::runner::RESULTS_FILE; +use crate::store::{CreateError, Store}; +use crate::types::{Program, State, SubmitRequest, SubmitResponse}; +use crate::validate; +use axum::extract::rejection::JsonRejection; +use axum::extract::{DefaultBodyLimit, Path, Query, State as AxState}; +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use axum::routing::{get, post}; +use axum::{Json, Router, middleware}; +use serde::Deserialize; +use serde_json::json; +use std::path::Path as FsPath; +use std::sync::Arc; +use std::time::Instant; +use tokio::task::JoinHandle; + +/// Largest accepted request body. A maximal query (100,000 residues) is about 100 KB of JSON. +pub const MAX_BODY_BYTES: usize = 1024 * 1024; +/// `retry_after_seconds` of a `429`, the MCP's default as well. +pub const RETRY_AFTER_SECONDS: u32 = 30; +/// Response header set on `results.tsv` when the file was cut at `max_results_bytes` (ruling 3). +pub const TRUNCATED_HEADER: &str = "x-blast-jobs-truncated"; +/// `Content-Type` of `results.tsv`. +pub const TSV_CONTENT_TYPE: &str = "text/tab-separated-values; charset=utf-8"; +/// `poll_after_seconds` when the job should start within the next free slot, and otherwise. +pub const POLL_SOON: u32 = 5; +pub const POLL_LATER: u32 = 10; + +/// What every handler shares. +pub struct AppState { + cfg: Arc, + catalogue: Arc, + store: Arc, + auth: Auth, + /// The runner's worker tasks; health is degraded once any of them has ended. + workers: Vec>, +} + +impl AppState { + pub fn new( + cfg: Arc, + catalogue: Arc, + store: Arc, + auth: Auth, + workers: Vec>, + ) -> Arc { + Arc::new(AppState { + cfg, + catalogue, + store, + auth, + workers, + }) + } + + pub fn auth(&self) -> &Auth { + &self.auth + } + + pub fn store(&self) -> &Arc { + &self.store + } +} + +/// The four routes. Health is outside the bearer layer. +pub fn router(state: Arc) -> Router { + let jobs = Router::new() + .route("/internal/jobs", post(submit)) + .route("/internal/jobs/{id}", get(status)) + .route("/internal/jobs/{id}/results.tsv", get(results)) + .route_layer(middleware::from_fn_with_state( + state.clone(), + auth::require_bearer, + )); + Router::new() + .merge(jobs) + .route("/internal/health", get(health)) + .fallback(|| async { error(StatusCode::NOT_FOUND, json!({"error": "not_found"})) }) + .layer(DefaultBodyLimit::max(MAX_BODY_BYTES)) + .with_state(state) +} + +fn error(status: StatusCode, body: serde_json::Value) -> Response { + (status, Json(body)).into_response() +} + +/// Longest extractor message passed back in an `invalid_request`. +const MESSAGE_MAX: usize = 200; + +/// `s` cut on a char boundary to at most `max` characters, with `…` when cut. +fn clip(s: &str, max: usize) -> String { + match s.char_indices().nth(max) { + Some((i, _)) => format!("{}…", &s[..i]), + None => s.to_string(), + } +} + +fn unknown_job() -> Response { + error(StatusCode::NOT_FOUND, json!({"error": "unknown_job"})) +} + +/// `poll_after_seconds` (design 3.6): 5 when the job is within the slots free right now, else 10. +pub fn poll_after(queue_position: u32, running: usize, concurrency: usize) -> u32 { + let idle = concurrency.saturating_sub(running); + if (queue_position as usize) <= idle { + POLL_SOON + } else { + POLL_LATER + } +} + +async fn submit( + AxState(st): AxState>, + body: Result, JsonRejection>, +) -> Response { + let started = Instant::now(); + let req = match body { + Ok(Json(r)) => r, + Err(rej) => { + let status = if rej.status() == StatusCode::PAYLOAD_TOO_LARGE { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + }; + let message = if status == StatusCode::PAYLOAD_TOO_LARGE { + format!("the request body is larger than {MAX_BODY_BYTES} bytes") + } else { + // serde_json's message names the field and position; a value it quotes is clipped. + clip(&rej.body_text(), MESSAGE_MAX) + }; + tracing::info!( + outcome = "invalid_request", + status = status.as_u16(), + duration_ms = started.elapsed().as_millis() as u64, + "submit rejected" + ); + return error( + status, + json!({"error": "invalid_request", "message": message}), + ); + } + }; + let client = req.client.clone(); + let v = match validate::check(&req, &st.cfg, &st.catalogue) { + Ok(v) => v, + Err(rej) => { + tracing::info!( + outcome = rej.code.as_str(), + program = %validate::quoted(&req.program), + database = %validate::quoted(&req.database), + species = req.species.len(), + client = %validate::quoted(client.as_deref().unwrap_or("")), + duration_ms = started.elapsed().as_millis() as u64, + "submit rejected" + ); + return (StatusCode::BAD_REQUEST, Json(rej)).into_response(); + } + }; + let (program, database, species) = (v.program, v.database.clone(), v.species.len()); + match st.store.create(v).await { + Ok((id, position)) => { + let (_, running) = st.store.counts(); + let poll = poll_after(position, running, st.cfg.concurrency); + tracing::info!( + job_id = %id, %program, %database, species, + client = client.as_deref().unwrap_or(""), + queue_position = position, + duration_ms = started.elapsed().as_millis() as u64, + outcome = "queued", + "job submitted" + ); + let body = SubmitResponse { + job_id: id.to_string(), + state: State::Queued, + queue_position: Some(position), + poll_after_seconds: Some(poll), + database_release: Some(st.cfg.database_release.clone()), + }; + (StatusCode::ACCEPTED, Json(body)).into_response() + } + Err(CreateError::QueueFull) => { + tracing::warn!( + %program, %database, species, + client = client.as_deref().unwrap_or(""), + duration_ms = started.elapsed().as_millis() as u64, + outcome = "queue_full", + "submit refused" + ); + let mut r = error( + StatusCode::TOO_MANY_REQUESTS, + json!({"error": "queue_full", "retry_after_seconds": RETRY_AFTER_SECONDS}), + ); + r.headers_mut() + .insert(header::RETRY_AFTER, HeaderValue::from(RETRY_AFTER_SECONDS)); + r + } + Err(CreateError::Io(e)) => { + tracing::error!( + %program, %database, error = %e, outcome = "unavailable", + duration_ms = started.elapsed().as_millis() as u64, + "submit failed: the job could not be stored" + ); + error( + StatusCode::SERVICE_UNAVAILABLE, + json!({"error": "unavailable", "message": "the job could not be stored; try again later"}), + ) + } + } +} + +async fn status(AxState(st): AxState>, Path(id): Path) -> Response { + let Some(id) = JobId::parse(&id) else { + return unknown_job(); + }; + match st.store.get(&id) { + Some(s) => Json(s).into_response(), + None => unknown_job(), + } +} + +async fn results(AxState(st): AxState>, Path(id): Path) -> Response { + let started = Instant::now(); + let Some(id) = JobId::parse(&id) else { + return unknown_job(); + }; + let Some(s) = st.store.get(&id) else { + return unknown_job(); + }; + if s.state != State::Complete { + return error( + StatusCode::CONFLICT, + json!({"error": "not_complete", "state": s.state}), + ); + } + let path = st.store.job_dir(&id).join(RESULTS_FILE); + let body = match tokio::fs::read(&path).await { + Ok(b) => b, + // Swept between the lookup and the read. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return unknown_job(), + Err(e) => { + tracing::error!(job_id = %id, error = %e, "could not read results.tsv"); + return error( + StatusCode::SERVICE_UNAVAILABLE, + json!({"error": "unavailable", "message": "the results could not be read; try again later"}), + ); + } + }; + if body.len() as u64 > st.cfg.max_results_bytes { + // The runner never writes more than the cap; refuse rather than send the MCP an over-limit body. + tracing::error!(job_id = %id, bytes = body.len(), "results.tsv is larger than max_results_bytes"); + return error( + StatusCode::SERVICE_UNAVAILABLE, + json!({"error": "unavailable", "message": "the results file is larger than the service limit"}), + ); + } + tracing::info!( + job_id = %id, bytes = body.len(), truncated = s.truncated, + duration_ms = started.elapsed().as_millis() as u64, + "results served" + ); + let mut r = ( + [( + header::CONTENT_TYPE, + HeaderValue::from_static(TSV_CONTENT_TYPE), + )], + body, + ) + .into_response(); + if s.truncated { + r.headers_mut() + .insert(TRUNCATED_HEADER, HeaderValue::from_static("true")); + } + r +} + +#[derive(Deserialize)] +struct HealthQuery { + liveness: Option, +} + +/// Every BLAST+ program binary under `dir` is a regular file with an execute bit. Used by health and +/// by `blast-jobs check`. +pub fn probe_binaries(dir: &FsPath) -> Result<(), String> { + use std::os::unix::fs::PermissionsExt; + for p in Program::ALL { + let path = dir.join(p.as_str()); + match std::fs::metadata(&path) { + Ok(m) if m.is_file() && m.permissions().mode() & 0o111 != 0 => {} + Ok(_) => return Err(format!("{p} is not an executable file")), + Err(_) => return Err(format!("{p} is missing")), + } + } + Ok(()) +} + +/// `200 {status:"ok", queued, running, database_release}` when jobs can be accepted, else +/// `503 {status:"degraded", reason}` (design 3.8). `?liveness=1` skips the queue check, so a busy +/// service is not restarted by Docker's health check. Never touches BLAST. +async fn health(AxState(st): AxState>, Query(q): Query) -> Response { + let liveness = q + .liveness + .as_deref() + .is_some_and(|v| v == "1" || v == "true"); + let (queued, running) = st.store.counts(); + let reason = if st.workers.iter().any(|w| w.is_finished()) { + Some("a job worker has stopped".to_string()) + } else if let Err(e) = probe_binaries(&st.cfg.blast_bin_dir) { + Some(format!("BLAST binaries unavailable: {e}")) + } else if !st.store.state_writable() { + Some("the state directory is not writable".to_string()) + } else if !liveness && queued >= st.cfg.max_queued { + Some("the queue is full".to_string()) + } else { + None + }; + match reason { + None => Json(json!({ + "status": "ok", + "queued": queued, + "running": running, + "database_release": st.cfg.database_release, + })) + .into_response(), + Some(reason) => error( + StatusCode::SERVICE_UNAVAILABLE, + json!({"status": "degraded", "reason": reason, "queued": queued, "running": running}), + ), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clip_bounds_on_char_boundaries() { + assert_eq!(clip("abc", 5), "abc"); + assert_eq!(clip("abcdef", 3), "abc…"); + assert_eq!(clip("ééééé", 2), "éé…"); + } + + #[test] + fn poll_after_uses_free_slots() { + // Two slots, none busy: positions 1 and 2 start now. + assert_eq!(poll_after(1, 0, 2), POLL_SOON); + assert_eq!(poll_after(2, 0, 2), POLL_SOON); + assert_eq!(poll_after(3, 0, 2), POLL_LATER); + // One busy: only position 1 starts now. + assert_eq!(poll_after(1, 1, 2), POLL_SOON); + assert_eq!(poll_after(2, 1, 2), POLL_LATER); + // All busy (or more running than slots after a race). + assert_eq!(poll_after(1, 2, 2), POLL_LATER); + assert_eq!(poll_after(1, 5, 2), POLL_LATER); + } + + #[test] + fn probe_reports_missing_and_non_executable() { + use std::os::unix::fs::PermissionsExt; + let d = tempfile::tempdir().unwrap(); + assert_eq!(probe_binaries(d.path()).unwrap_err(), "blastn is missing"); + for p in Program::ALL { + let f = d.path().join(p.as_str()); + std::fs::write(&f, "#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + probe_binaries(d.path()).unwrap(); + let f = d.path().join("tblastx"); + std::fs::set_permissions(&f, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert_eq!( + probe_binaries(d.path()).unwrap_err(), + "tblastx is not an executable file" + ); + } +} diff --git a/blast-jobs/src/lib.rs b/blast-jobs/src/lib.rs index 69b8e79..837551f 100644 --- a/blast-jobs/src/lib.rs +++ b/blast-jobs/src/lib.rs @@ -2,9 +2,11 @@ //! (`docs/handoff-private-side.md` §7). Database mapping comes from a catalogue file and every limit //! from the config file; nothing site-specific is compiled in. pub mod advanced; +pub mod auth; pub mod catalogue; pub mod command; pub mod config; +pub mod http; pub mod jobid; pub mod runner; pub mod store; diff --git a/blast-jobs/src/main.rs b/blast-jobs/src/main.rs index ba3f079..692d5fc 100644 --- a/blast-jobs/src/main.rs +++ b/blast-jobs/src/main.rs @@ -1,25 +1,103 @@ -//! `blast-jobs `: loads and checks the config and the catalogue. The job API and the -//! workers are wired in later tasks; until then this is a configuration check that exits 0 on success. -use anyhow::Context; -use blast_jobs::{catalogue::Catalogue, config::Config}; +//! `blast-jobs serve `: runs the internal job API, the workers and the retention sweep. +//! `blast-jobs check `: loads the config and the catalogue, probes the BLAST+ binaries, +//! and exits 0 when all is well. +//! +//! The bearer token comes from `BLAST_JOBS_TOKEN` (never logged). Logs are JSON lines on stdout; +//! `RUST_LOG` sets the filter (default `info`). SIGTERM or Ctrl-C stops accepting connections and +//! lets in-flight requests finish; jobs still queued or running are marked failed at the next start. +use anyhow::{Context, bail}; +use blast_jobs::auth::Auth; +use blast_jobs::catalogue::Catalogue; +use blast_jobs::config::Config; +use blast_jobs::http::{self, AppState}; +use blast_jobs::runner; +use blast_jobs::store::{SWEEP_INTERVAL, Store}; +use std::sync::Arc; +use std::time::Duration; + +const USAGE: &str = "usage: blast-jobs serve | blast-jobs check "; fn main() -> anyhow::Result<()> { tracing_subscriber::fmt() .json() + .with_writer(std::io::stdout) .with_env_filter( tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()), ) .init(); - let path = std::env::args() - .nth(1) - .context("usage: blast-jobs ")?; - let cfg = Config::load(&path)?; + let args: Vec = std::env::args().skip(1).collect(); + let (cmd, path) = match args.as_slice() { + [cmd, path] => (cmd.as_str(), path.as_str()), + _ => bail!(USAGE), + }; + let cfg = Config::load(path)?; let catalogue = Catalogue::load(&cfg.catalogue, &cfg.database_root)?; + match cmd { + "check" => { + http::probe_binaries(&cfg.blast_bin_dir).map_err(|e| { + anyhow::anyhow!("BLAST binaries in {}: {e}", cfg.blast_bin_dir.display()) + })?; + tracing::info!( + databases = catalogue.len(), + database_release = %cfg.database_release, + "configuration, catalogue and BLAST binaries are valid" + ); + Ok(()) + } + "serve" => { + let auth = Auth::from_env()?; + tokio::runtime::Runtime::new()?.block_on(serve(cfg, catalogue, auth)) + } + _ => bail!(USAGE), + } +} + +async fn serve(cfg: Config, catalogue: Catalogue, auth: Auth) -> anyhow::Result<()> { + if let Err(e) = http::probe_binaries(&cfg.blast_bin_dir) { + // Not fatal: health reports it, so the container stays up and says why. + tracing::error!(dir = %cfg.blast_bin_dir.display(), error = %e, "BLAST binaries unavailable"); + } + let cfg = Arc::new(cfg); + let store = Store::open(&cfg.state_root, cfg.max_queued)?; + let workers = runner::spawn_workers(cfg.clone(), store.clone()); + let _sweeper = store.spawn_sweeper(SWEEP_INTERVAL, Duration::from_secs(cfg.retention_seconds)); + let listener = tokio::net::TcpListener::bind(cfg.bind) + .await + .with_context(|| format!("binding {}", cfg.bind))?; tracing::info!( + bind = %cfg.bind, databases = catalogue.len(), database_release = %cfg.database_release, concurrency = cfg.concurrency, - "configuration and catalogue are valid" + auth = matches!(auth, Auth::Token(_)), + "blast-jobs listening" ); + let state = AppState::new(cfg.clone(), Arc::new(catalogue), store, auth, workers); + axum::serve(listener, http::router(state)) + .with_graceful_shutdown(shutdown()) + .await?; + tracing::info!("blast-jobs stopped"); Ok(()) } + +async fn shutdown() { + let ctrl_c = async { + let _ = tokio::signal::ctrl_c().await; + }; + let term = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut s) => { + s.recv().await; + } + Err(e) => { + tracing::error!(error = %e, "cannot listen for SIGTERM"); + std::future::pending::<()>().await; + } + } + }; + tokio::select! { + () = ctrl_c => {}, + () = term => {}, + } + tracing::info!("shutdown signal received"); +} diff --git a/blast-jobs/tests/api.rs b/blast-jobs/tests/api.rs new file mode 100644 index 0000000..e5b1c3a --- /dev/null +++ b/blast-jobs/tests/api.rs @@ -0,0 +1,590 @@ +//! HTTP API tests (design 3.12, Task 4) against the fake BLAST+ scripts in `tests/fake-blast/`. Each +//! test starts its own router on an ephemeral port over a fresh state root; the fake reads its controls +//! from `/fake-blast.env`. +use blast_jobs::advanced::DENY_LIST; +use blast_jobs::auth::Auth; +use blast_jobs::catalogue::Catalogue; +use blast_jobs::config::Config; +use blast_jobs::http::{self, AppState, MAX_BODY_BYTES}; +use blast_jobs::runner::{self, HEADER}; +use blast_jobs::store::Store; +use blast_jobs::types::Program; +use reqwest::StatusCode; +use serde_json::{Value, json}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::task::JoinHandle; + +const TOKEN: &str = "test-token-0123456789"; +const VALID_UNKNOWN_ID: &str = "AAAAAAAAAAAAAAAAAAAAAA"; + +fn manifest() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +struct Server { + tmp: tempfile::TempDir, + base: String, + http: reqwest::Client, + tasks: Vec>, +} + +impl Drop for Server { + fn drop(&mut self) { + for t in &self.tasks { + t.abort(); + } + } +} + +/// Starts the API over the fixture catalogue and the fake BLAST binaries. `overrides` are merged into +/// the config; `controls` is written to `fake-blast.env`. +async fn start(overrides: Value, controls: &str) -> Server { + start_with(overrides, controls, None).await +} + +/// As `start`; `workers` replaces the runner's workers (for the stopped-worker health case). +async fn start_with( + overrides: Value, + controls: &str, + workers: Option>>, +) -> Server { + let tmp = tempfile::tempdir().unwrap(); + let state = tmp.path().join("state"); + std::fs::create_dir_all(&state).unwrap(); + std::fs::write(state.join("fake-blast.env"), controls).unwrap(); + let fixture = manifest().join("tests/fixture"); + let mut v = json!({ + "bind": "127.0.0.1:0", + "blast_bin_dir": manifest().join("tests/fake-blast"), + "database_root": fixture.join("db"), + "catalogue": fixture.join("catalogue.json"), + "database_release": "FB2023_04", + "state_root": state, + }); + for (k, val) in overrides.as_object().unwrap() { + v[k] = val.clone(); + } + let path = tmp.path().join("config.json"); + std::fs::write(&path, v.to_string()).unwrap(); + let cfg = Arc::new(Config::load(&path).unwrap()); + let cat = Arc::new(Catalogue::load(&cfg.catalogue, &cfg.database_root).unwrap()); + let store = Store::open(&cfg.state_root, cfg.max_queued).unwrap(); + let mut tasks = Vec::new(); + // Worker tasks end with the test's runtime; the state keeps their handles for health. + let workers = workers.unwrap_or_else(|| runner::spawn_workers(cfg.clone(), store.clone())); + let state = AppState::new(cfg.clone(), cat, store, Auth::Token(TOKEN.into()), workers); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tasks.push(tokio::spawn(async move { + axum::serve(listener, http::router(state)).await.unwrap(); + })); + Server { + tmp, + base: format!("http://{addr}"), + http: reqwest::Client::new(), + tasks, + } +} + +fn request(program: &str) -> Value { + let p = Program::parse(program).unwrap(); + json!({ + "program": program, + "database": if p.protein_database() { "translation" } else { "assembly" }, + "species": ["dmel"], + "sequence": if p.protein_query() { ">query\nMKLVQWERTY\n" } else { ">query\nACGTACGTACGT\n" }, + "client": "flybase-mcp", + }) +} + +impl Server { + fn url(&self, path: &str) -> String { + format!("{}{path}", self.base) + } + + async fn post(&self, body: &Value) -> reqwest::Response { + self.http + .post(self.url("/internal/jobs")) + .bearer_auth(TOKEN) + .json(body) + .send() + .await + .unwrap() + } + + async fn get(&self, path: &str) -> reqwest::Response { + self.http + .get(self.url(path)) + .bearer_auth(TOKEN) + .send() + .await + .unwrap() + } + + /// Submits and returns the `202` body. + async fn submit(&self, body: &Value) -> Value { + let r = self.post(body).await; + assert_eq!(r.status(), StatusCode::ACCEPTED, "{:?}", r.text().await); + r.json().await.unwrap() + } + + async fn status(&self, id: &str) -> Value { + let r = self.get(&format!("/internal/jobs/{id}")).await; + assert_eq!(r.status(), StatusCode::OK); + r.json().await.unwrap() + } + + /// Polls until `state` matches `pred`, returning the last status. + async fn wait_for(&self, id: &str, within: Duration, pred: impl Fn(&str) -> bool) -> Value { + let end = Instant::now() + within; + loop { + let s = self.status(id).await; + if pred(s["state"].as_str().unwrap()) { + return s; + } + assert!(Instant::now() < end, "job {id} still {s} after {within:?}"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + async fn wait_terminal(&self, id: &str) -> Value { + self.wait_for(id, Duration::from_secs(20), |s| { + matches!(s, "complete" | "failed") + }) + .await + } + + async fn health(&self, query: &str) -> (StatusCode, Value) { + let r = self + .http + .get(self.url(&format!("/internal/health{query}"))) + .send() + .await + .unwrap(); + (r.status(), r.json().await.unwrap()) + } +} + +fn canned_rows() -> String { + std::fs::read_to_string(manifest().join("tests/fake-blast/canned.tsv")).unwrap() +} + +#[tokio::test] +async fn happy_path_submit_status_results() { + let s = start(json!({"concurrency": 1}), "FAKE_BLAST_SLEEP=1\n").await; + let a = s.submit(&request("blastn")).await; + let keys: Vec<&String> = a.as_object().unwrap().keys().collect(); + assert_eq!( + keys, + [ + "database_release", + "job_id", + "poll_after_seconds", + "queue_position", + "state" + ] + ); + assert_eq!(a["state"], "queued"); + assert_eq!(a["database_release"], "FB2023_04"); + assert_eq!(a["queue_position"], 1); + assert_eq!(a["poll_after_seconds"], 5, "an idle slot is free"); + let id = a["job_id"].as_str().unwrap().to_string(); + assert_eq!(id.len(), 22); + assert!( + id.bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + ); + + // Running: position 0. A second job queued behind it: position 1, poll later. + let running = s + .wait_for(&id, Duration::from_secs(5), |st| st == "running") + .await; + assert_eq!(running["queue_position"], 0); + let b = s.submit(&request("blastn")).await; + assert_eq!(b["queue_position"], 1); + assert_eq!(b["poll_after_seconds"], 10, "the only slot is busy"); + let bid = b["job_id"].as_str().unwrap().to_string(); + let queued = s.status(&bid).await; + assert_eq!(queued["state"], "queued"); + assert_eq!(queued["queue_position"], 1); + + let done = s.wait_terminal(&id).await; + assert_eq!( + done, + json!({"job_id": id, "state": "complete"}), + "no result_url, downloads, error or queue_position once complete" + ); + + let r = s.get(&format!("/internal/jobs/{id}/results.tsv")).await; + assert_eq!(r.status(), StatusCode::OK); + assert_eq!( + r.headers()["content-type"], + "text/tab-separated-values; charset=utf-8" + ); + assert!(r.headers().get("x-blast-jobs-truncated").is_none()); + let len: usize = r.headers()["content-length"] + .to_str() + .unwrap() + .parse() + .unwrap(); + let body = r.text().await.unwrap(); + assert_eq!(len, body.len()); + assert_eq!(body, format!("{HEADER}{}", canned_rows())); + let rows: Vec<&str> = body.lines().skip(1).collect(); + assert_eq!(rows.len(), 3); + assert!(rows.iter().all(|r| r.split('\t').count() == 13)); + s.wait_terminal(&bid).await; +} + +#[tokio::test] +async fn job_routes_need_the_bearer_token_and_health_does_not() { + let s = start(json!({}), "").await; + let paths = [ + format!("/internal/jobs/{VALID_UNKNOWN_ID}"), + format!("/internal/jobs/{VALID_UNKNOWN_ID}/results.tsv"), + ]; + let auths: [Option; 5] = [ + None, + Some("Bearer wrong".into()), + Some(format!("Bearer {TOKEN}x")), + Some(format!("Basic {TOKEN}")), + Some(TOKEN.into()), + ]; + for auth in &auths { + let mut post = s + .http + .post(s.url("/internal/jobs")) + .json(&request("blastn")); + if let Some(a) = auth { + post = post.header("authorization", a); + } + let r = post.send().await.unwrap(); + assert_eq!(r.status(), StatusCode::UNAUTHORIZED, "POST with {auth:?}"); + assert_eq!( + r.json::().await.unwrap(), + json!({"error": "unauthorized"}) + ); + for p in &paths { + let mut get = s.http.get(s.url(p)); + if let Some(a) = auth { + get = get.header("authorization", a); + } + let r = get.send().await.unwrap(); + assert_eq!( + r.status(), + StatusCode::UNAUTHORIZED, + "GET {p} with {auth:?}" + ); + } + } + let (code, body) = s.health("").await; + assert_eq!(code, StatusCode::OK); + assert_eq!(body["status"], "ok"); + assert_eq!(body["database_release"], "FB2023_04"); + assert_eq!(body["queued"], 0); + assert_eq!(body["running"], 0); +} + +#[tokio::test] +async fn unknown_and_malformed_ids_are_404() { + let s = start(json!({}), "").await; + for id in [ + VALID_UNKNOWN_ID, + "short", + "has.dot123", + "x%2F..%2F..%2Fetc", + &"a".repeat(65), + ] { + for suffix in ["", "/results.tsv"] { + let r = s.get(&format!("/internal/jobs/{id}{suffix}")).await; + assert_eq!(r.status(), StatusCode::NOT_FOUND, "{id}{suffix}"); + assert_eq!( + r.json::().await.unwrap(), + json!({"error": "unknown_job"}), + "{id}{suffix}" + ); + } + } +} + +#[tokio::test] +async fn results_before_completion_are_409() { + let s = start(json!({"concurrency": 1}), "FAKE_BLAST_SLEEP=3\n").await; + let a = s.submit(&request("blastn")).await; + let b = s.submit(&request("blastn")).await; + let (aid, bid) = (a["job_id"].as_str().unwrap(), b["job_id"].as_str().unwrap()); + s.wait_for(aid, Duration::from_secs(5), |st| st == "running") + .await; + for (id, state) in [(aid, "running"), (bid, "queued")] { + let r = s.get(&format!("/internal/jobs/{id}/results.tsv")).await; + assert_eq!(r.status(), StatusCode::CONFLICT); + assert_eq!( + r.json::().await.unwrap(), + json!({"error": "not_complete", "state": state}) + ); + } +} + +#[tokio::test] +async fn failed_job_reports_error_and_409() { + let s = start(json!({}), "FAKE_BLAST_FAIL=db\n").await; + let a = s.submit(&request("blastn")).await; + let id = a["job_id"].as_str().unwrap(); + let st = s.wait_terminal(id).await; + assert_eq!( + st, + json!({"job_id": id, "state": "failed", "error": "database unavailable"}) + ); + let r = s.get(&format!("/internal/jobs/{id}/results.tsv")).await; + assert_eq!(r.status(), StatusCode::CONFLICT); + assert_eq!( + r.json::().await.unwrap(), + json!({"error": "not_complete", "state": "failed"}) + ); +} + +#[tokio::test] +async fn full_queue_is_429_and_fails_health_but_not_liveness() { + let s = start( + json!({"concurrency": 1, "max_queued": 2}), + "FAKE_BLAST_SLEEP=5\n", + ) + .await; + let first = s.submit(&request("blastn")).await; + s.wait_for( + first["job_id"].as_str().unwrap(), + Duration::from_secs(5), + |st| st == "running", + ) + .await; + let q1 = s.submit(&request("blastn")).await; + assert_eq!(q1["queue_position"], 1); + let (code, _) = s.health("").await; + assert_eq!(code, StatusCode::OK, "one free queue slot left"); + let q2 = s.submit(&request("blastn")).await; + assert_eq!(q2["queue_position"], 2); + assert_eq!(q2["poll_after_seconds"], 10); + + let r = s.post(&request("blastn")).await; + assert_eq!(r.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!(r.headers()["retry-after"], "30"); + assert_eq!( + r.json::().await.unwrap(), + json!({"error": "queue_full", "retry_after_seconds": 30}) + ); + + let (code, body) = s.health("").await; + assert_eq!(code, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(body["status"], "degraded"); + assert_eq!(body["reason"], "the queue is full"); + let (code, body) = s.health("?liveness=1").await; + assert_eq!(code, StatusCode::OK, "liveness ignores the queue: {body}"); + assert_eq!(body["queued"], 2); + assert_eq!(body["running"], 1); +} + +#[tokio::test] +async fn health_is_503_when_the_blast_binaries_are_missing() { + let s = start(json!({"blast_bin_dir": "/nonexistent/blast/bin"}), "").await; + for q in ["", "?liveness=1"] { + let (code, body) = s.health(q).await; + assert_eq!(code, StatusCode::SERVICE_UNAVAILABLE, "{q}"); + assert_eq!(body["status"], "degraded"); + assert_eq!( + body["reason"], + "BLAST binaries unavailable: blastn is missing" + ); + } +} + +#[tokio::test] +async fn health_is_503_when_a_worker_has_stopped() { + let dead = tokio::spawn(async {}); + while !dead.is_finished() { + tokio::task::yield_now().await; + } + let s = start_with(json!({}), "", Some(vec![dead])).await; + let (code, body) = s.health("?liveness=1").await; + assert_eq!(code, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(body["reason"], "a job worker has stopped"); +} + +#[tokio::test] +async fn malformed_bodies_are_invalid_request() { + let s = start(json!({}), "").await; + let send = |body: &'static str, ct: Option<&'static str>| { + let mut rb = s + .http + .post(s.url("/internal/jobs")) + .bearer_auth(TOKEN) + .body(body); + if let Some(ct) = ct { + rb = rb.header("content-type", ct); + } + rb.send() + }; + for (body, ct) in [ + ("{not json", Some("application/json")), + ("[1, 2]", Some("application/json")), + (r#"{"program": "blastn"}"#, Some("application/json")), + ( + r#"{"program": 5, "database": "assembly", "sequence": "ACGT"}"#, + Some("application/json"), + ), + ( + r#"{"program": "blastn", "database": "assembly", "sequence": "ACGT"}"#, + None, + ), + ] { + let r = send(body, ct).await.unwrap(); + assert_eq!(r.status(), StatusCode::BAD_REQUEST, "{body} {ct:?}"); + let v: Value = r.json().await.unwrap(); + assert_eq!(v["error"], "invalid_request", "{body}"); + assert!(v["message"].as_str().is_some_and(|m| !m.is_empty()), "{v}"); + assert_eq!( + v.as_object().unwrap().len(), + 2, + "exactly error and message: {v}" + ); + } +} + +#[tokio::test] +async fn unknown_extra_key_is_accepted() { + let s = start(json!({}), "").await; + let mut body = request("blastn"); + body["future_key"] = json!({"nested": [1, 2, 3]}); + let a = s.submit(&body).await; + let st = s.wait_terminal(a["job_id"].as_str().unwrap()).await; + assert_eq!(st["state"], "complete"); +} + +#[tokio::test] +async fn validation_rejections_carry_the_mcp_codes() { + let s = start(json!({}), "").await; + let mut cases: Vec<(Value, &str)> = Vec::new(); + let mut b = request("blastn"); + b["program"] = json!("psiblast"); + cases.push((b, "invalid_program")); + let mut b = request("blastn"); + b["database"] = json!("nosuchdb"); + cases.push((b, "invalid_database")); + let mut b = request("blastn"); + b["database"] = json!("translation"); + cases.push((b, "invalid_database")); + let mut b = request("blastn"); + b["species"] = json!(["dsim"]); + b["database"] = json!("transcript"); + cases.push((b, "invalid_species")); + let mut b = request("blastn"); + b["sequence"] = json!(">query\nMKLVQWEFF\n"); + cases.push((b, "invalid_sequence")); + let mut b = request("blastn"); + b["evalue"] = json!(0); + cases.push((b, "invalid_option")); + for (body, code) in cases { + let r = s.post(&body).await; + assert_eq!(r.status(), StatusCode::BAD_REQUEST, "{body}"); + let v: Value = r.json().await.unwrap(); + assert_eq!(v["error"], code, "{body} -> {v}"); + assert!(v["message"].as_str().is_some_and(|m| !m.is_empty())); + assert_eq!(v.as_object().unwrap().len(), 2, "{v}"); + } +} + +#[tokio::test] +async fn every_deny_list_entry_is_invalid_option() { + let s = start(json!({}), "").await; + assert!(!DENY_LIST.is_empty()); + for (program, adv) in DENY_LIST { + let mut body = request(program.as_str()); + body["advanced"] = json!(adv); + let r = s.post(&body).await; + assert_eq!(r.status(), StatusCode::BAD_REQUEST, "{program} {adv}"); + let v: Value = r.json().await.unwrap(); + assert_eq!(v["error"], "invalid_option", "{program} {adv} -> {v}"); + } + let (_, h) = s.health("").await; + assert_eq!(h["queued"], 0, "no rejected request was queued"); +} + +#[tokio::test] +async fn big_output_is_capped_and_flagged_by_header() { + // Default cap: 16 MiB, the MCP's Content-Length limit. The fake writes 20 MiB of rows. + let s = start(json!({}), "FAKE_BLAST_BIG=1\n").await; + let a = s.submit(&request("blastn")).await; + let id = a["job_id"].as_str().unwrap(); + let st = s.wait_terminal(id).await; + assert_eq!(st["state"], "complete"); + assert!( + st.get("truncated").is_none(), + "truncation is never in the JSON" + ); + let r = s.get(&format!("/internal/jobs/{id}/results.tsv")).await; + assert_eq!(r.status(), StatusCode::OK); + assert_eq!( + r.headers() + .get("X-Blast-Jobs-Truncated") + .expect("X-Blast-Jobs-Truncated header present") + .to_str() + .unwrap(), + "true" + ); + let len: u64 = r.headers()["content-length"] + .to_str() + .unwrap() + .parse() + .unwrap(); + assert!(len <= 16_777_216, "{len}"); + let body = r.bytes().await.unwrap(); + assert_eq!(body.len() as u64, len); + assert!(body.starts_with(HEADER.as_bytes())); + assert!(body.ends_with(b"\n")); + assert!( + len > 16_777_216 - 4096, + "cut near the cap, at a row boundary" + ); + + // A normal job on the same server carries no such header. + std::fs::write(s.tmp.path().join("state/fake-blast.env"), "").unwrap(); + let b = s.submit(&request("blastn")).await; + let bid = b["job_id"].as_str().unwrap(); + s.wait_terminal(bid).await; + let r = s.get(&format!("/internal/jobs/{bid}/results.tsv")).await; + assert_eq!(r.status(), StatusCode::OK); + assert!(r.headers().get("X-Blast-Jobs-Truncated").is_none()); +} + +#[tokio::test] +async fn body_over_the_limit_is_413_invalid_request() { + let s = start(json!({}), "").await; + let mut body = request("blastn"); + body["sequence"] = json!(format!(">query\n{}\n", "A".repeat(MAX_BODY_BYTES))); + let r = s.post(&body).await; + assert_eq!(r.status(), StatusCode::PAYLOAD_TOO_LARGE); + let v: Value = r.json().await.unwrap(); + assert_eq!(v["error"], "invalid_request"); + + // Just under the limit the body is read and validated (here: over the residue cap). + let mut body = request("blastn"); + body["sequence"] = json!(format!(">query\n{}\n", "A".repeat(200_000))); + let r = s.post(&body).await; + assert_eq!(r.status(), StatusCode::BAD_REQUEST); + assert_eq!( + r.json::().await.unwrap()["error"], + "invalid_sequence" + ); +} + +#[tokio::test] +async fn unknown_route_is_json_404() { + let s = start(json!({}), "").await; + let r = s.get("/internal/nothing").await; + assert_eq!(r.status(), StatusCode::NOT_FOUND); + assert_eq!( + r.json::().await.unwrap(), + json!({"error": "not_found"}) + ); +} From 7419239f4e7a0f94fb0aa6fc41b07f1523c96ac7 Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Mon, 12 Oct 2026 18:02:12 +0000 Subject: [PATCH 07/12] fix(blast-jobs): absolute config paths, service-owned flags never rendered, runner cleanups --- blast-jobs/src/command.rs | 87 ++++++++++++++++++++++++++++- blast-jobs/src/config.rs | 39 ++++++++++++- blast-jobs/src/runner.rs | 46 +++++++++++---- blast-jobs/tests/fake-blast/blastn | 36 +++++++----- blast-jobs/tests/fake-blast/blastp | 36 +++++++----- blast-jobs/tests/fake-blast/blastx | 36 +++++++----- blast-jobs/tests/fake-blast/tblastn | 36 +++++++----- blast-jobs/tests/fake-blast/tblastx | 36 +++++++----- blast-jobs/tests/runner.rs | 30 +++++++++- 9 files changed, 299 insertions(+), 83 deletions(-) diff --git a/blast-jobs/src/command.rs b/blast-jobs/src/command.rs index 082ab81..e20c3b9 100644 --- a/blast-jobs/src/command.rs +++ b/blast-jobs/src/command.rs @@ -5,6 +5,7 @@ //! in place (so `-dust no` replaces the default `-dust yes` and never adds a second `-dust`), any other //! pair is appended in the order given. A Switch is rendered as one element. Nothing goes through a //! shell: the `-db` list of prefixes is one argv element with spaces, as BLAST+ expects. +use crate::advanced; use crate::catalogue::Resolved; use crate::config::Config; use crate::types::Program; @@ -19,6 +20,31 @@ pub const RAW_FILE: &str = "raw.tsv"; /// Tabular output with the subject title appended; `runner::HEADER` names these columns. pub const OUTFMT: &str = "6 std stitle"; +/// Flags whose value only the service sets: paths, output format, resources and the request-field +/// flags. An `advanced` pair with one of these names is never rendered, even if it reached `build` +/// some way other than `advanced::parse` (which already rejects them). +pub const SERVICE_OWNED: &[&str] = &[ + "db", + "query", + "out", + "outfmt", + "evalue", + "max_target_seqs", + "num_threads", + "matrix", + "query_gencode", + "db_gencode", +]; + +/// Whether an `advanced` pair may reach the argv: not service-owned, and in `advanced::TABLE` for +/// `program`. Defence in depth behind `advanced::parse`. +fn allowed(program: Program, flag: &str) -> bool { + !SERVICE_OWNED.contains(&flag) + && advanced::TABLE + .iter() + .any(|spec| spec.name == flag && spec.programs.contains(&program)) +} + /// An ordered `flag -> value` map; `None` is a Switch. #[derive(Default)] struct Argv(Vec<(String, Option)>); @@ -59,7 +85,12 @@ fn format_evalue(e: f64) -> String { /// /// `-task` is explicit for blastn (BLAST+'s own default there is megablast): `megablast` when the /// request says so, else `blastn`, unless `advanced` sets it. Other programs get `-task` only from -/// `advanced`. An `advanced` `-max_hsps` is clamped to `max_hsps_per_subject`. +/// `advanced`. An `advanced` `-max_hsps` is clamped to `max_hsps_per_subject`. A pair that is +/// service-owned or not in the allow-list for the program is dropped (and logged), never rendered. +/// +/// Disk use before truncation is bounded only by this argv: at most 1000 subjects +/// (`-max_target_seqs`) x `max_hsps_per_subject` HSPs (50) rows. Recheck the state volume before raising +/// either cap. pub fn build(cfg: &Config, req: &Validated, resolved: &Resolved, job_dir: &Path) -> Vec { let program = req.program; let mut a = Argv::default(); @@ -98,6 +129,10 @@ pub fn build(cfg: &Config, req: &Validated, resolved: &Resolved, job_dir: &Path) } for (flag, value) in &req.advanced { + if !allowed(program, flag) { + tracing::error!(%program, flag = %flag, "advanced flag not allowed here; dropped from argv"); + continue; + } let value = match (flag.as_str(), value) { // Validated as an integer >= 1; anything that does not fit a u64 is above the cap anyway. ("max_hsps", Some(v)) => Some( @@ -534,6 +569,56 @@ mod tests { } } + /// A `Validated` built by hand (bypassing `advanced::parse`) cannot move the service's paths, + /// output format or resources, nor add a flag outside the allow-list. + #[test] + fn service_owned_and_unlisted_pairs_never_reach_argv() { + let c = cfg(); + let mut v = validate::check(&request(Program::Blastn), &c, &catalogue()).unwrap(); + let clean = build(&c, &v, &v.resolved, Path::new(JOB)); + let pair = |f: &str, x: Option<&str>| (f.to_string(), x.map(String::from)); + v.advanced = vec![ + pair("db", Some("/etc/passwd")), + pair("query", Some("/etc/shadow")), + pair("out", Some("/tmp/evil")), + pair("outfmt", Some("5")), + pair("num_threads", Some("64")), + pair("evalue", Some("1000")), + pair("remote", None), + pair("seg", Some("no")), // seg is not a blastn flag + pair("dust", Some("no")), + ]; + let argv: Vec = build(&c, &v, &v.resolved, Path::new(JOB)) + .into_iter() + .map(|s| s.into_string().unwrap()) + .collect(); + assert_eq!( + value_of(&argv, "-db"), + Some(prefix("assembly", "dmel").as_str()) + ); + assert_eq!( + value_of(&argv, "-query"), + Some(format!("{JOB}/query.fa").as_str()) + ); + assert_eq!( + value_of(&argv, "-out"), + Some(format!("{JOB}/raw.tsv").as_str()) + ); + assert_eq!(value_of(&argv, "-outfmt"), Some("6 std stitle")); + assert_eq!(value_of(&argv, "-num_threads"), Some("3")); + assert_eq!(value_of(&argv, "-evalue"), Some("10")); + assert_eq!(count(&argv, "-remote") + count(&argv, "-seg"), 0); + // The one allowed pair still applies, in place. + assert_eq!(value_of(&argv, "-dust"), Some("no")); + assert_eq!(argv.len(), clean.len()); + for f in SERVICE_OWNED { + assert!( + !advanced::TABLE.iter().any(|s| s.name == *f), + "{f} is service-owned and must not be in the allow-list" + ); + } + } + #[test] fn program_binary_comes_from_config() { let mut c = cfg(); diff --git a/blast-jobs/src/config.rs b/blast-jobs/src/config.rs index 77111d8..b4ef89e 100644 --- a/blast-jobs/src/config.rs +++ b/blast-jobs/src/config.rs @@ -96,13 +96,31 @@ impl Config { let path = path.as_ref(); let text = std::fs::read_to_string(path) .with_context(|| format!("reading config {}", path.display()))?; - let cfg: Config = serde_json::from_str(&text) + let mut cfg: Config = serde_json::from_str(&text) .with_context(|| format!("parsing config {}", path.display()))?; + cfg.make_paths_absolute() + .with_context(|| format!("invalid config {}", path.display()))?; cfg.validate() .with_context(|| format!("invalid config {}", path.display()))?; Ok(cfg) } + /// Relative paths are resolved against the process working directory (as `Catalogue::load` does + /// for the database root), never the config file's directory. The runner starts BLAST+ with the + /// job directory as its working directory, so a relative program or job path would otherwise + /// resolve against that directory and every spawn would fail. + fn make_paths_absolute(&mut self) -> anyhow::Result<()> { + for (key, p) in [ + ("blast_bin_dir", &mut self.blast_bin_dir), + ("database_root", &mut self.database_root), + ("catalogue", &mut self.catalogue), + ("state_root", &mut self.state_root), + ] { + *p = std::path::absolute(&*p).with_context(|| format!("{key} {}", p.display()))?; + } + Ok(()) + } + fn validate(&self) -> anyhow::Result<()> { if self.database_release.trim().is_empty() { bail!("database_release must not be empty"); @@ -285,6 +303,25 @@ pub(crate) mod tests { assert!(err(&v).contains("state_root")); } + #[test] + fn relative_paths_become_absolute_against_the_working_directory() { + let mut v = sample(); + v["blast_bin_dir"] = json!("./blast/bin"); + v["state_root"] = json!("state"); + v["catalogue"] = json!("db/catalogue.json"); + v["database_root"] = json!("db"); + let c = load(&v).unwrap(); + let cwd = std::env::current_dir().unwrap(); + assert_eq!(c.blast_bin_dir, cwd.join("blast/bin")); + assert_eq!(c.state_root, cwd.join("state")); + assert_eq!(c.catalogue, cwd.join("db/catalogue.json")); + assert_eq!(c.database_root, cwd.join("db")); + // Absolute paths are kept as written. + let c = load(&sample()).unwrap(); + assert_eq!(c.blast_bin_dir, PathBuf::from("/opt/ncbi-blast/bin")); + assert_eq!(c.state_root, PathBuf::from("/state")); + } + #[test] fn missing_file_is_an_error() { assert!(Config::load("/nonexistent/blast-jobs.json").is_err()); diff --git a/blast-jobs/src/runner.rs b/blast-jobs/src/runner.rs index 973e1ae..513a69c 100644 --- a/blast-jobs/src/runner.rs +++ b/blast-jobs/src/runner.rs @@ -76,6 +76,7 @@ async fn worker(n: usize, cfg: Arc, store: Arc) { let task = tokio::spawn(run_job(cfg.clone(), store.clone(), id.clone(), req)); if let Err(e) = task.await { tracing::error!(worker = n, job_id = %id, error = %e, "job task ended abnormally"); + let _ = std::fs::remove_file(store.job_dir(&id).join(RAW_FILE)); store .finish(&id, Outcome::Failed("internal error".into())) .await; @@ -87,6 +88,8 @@ async fn worker(n: usize, cfg: Arc, store: Arc) { enum Exit { Status(ExitStatus), TimedOut, + /// `wait` itself failed; the child is killed on drop. + WaitFailed(io::Error), } async fn run_job(cfg: Arc, store: Arc, id: JobId, req: Validated) { @@ -107,6 +110,10 @@ async fn run_job(cfg: Arc, store: Arc, id: JobId, req: Validated) Outcome::Failed("BLAST could not be started".into()) } Ok((Exit::TimedOut, _)) => Outcome::Failed(timeout_message(wall)), + Ok((Exit::WaitFailed(e), _)) => { + tracing::error!(job_id = %id, error = %e, "waiting for BLAST failed"); + Outcome::Failed("BLAST failed".into()) + } Ok((Exit::Status(status), _)) if status.success() => { let cap = cfg.max_results_bytes; let d = dir.clone(); @@ -141,7 +148,8 @@ async fn run_job(cfg: Arc, store: Arc, id: JobId, req: Validated) } /// Runs `argv` in `dir` under the wall limit; returns how it ended and the captured stderr (also -/// written to `stderr.txt`). `Err` only when the process could not be spawned. +/// written to `stderr.txt`). `Err` only when the process could not be spawned. The stderr reader is +/// aborted if the pipe is still open `STDERR_GRACE` after the child ended. async fn execute(argv: &[OsString], dir: &Path, wall: Duration) -> io::Result<(Exit, Vec)> { let mut child = Command::new(&argv[0]) .args(&argv[1..]) @@ -161,7 +169,10 @@ async fn execute(argv: &[OsString], dir: &Path, wall: Duration) -> io::Result<(E let exit = match tokio::time::timeout(wall, child.wait()).await { Ok(Ok(status)) => Exit::Status(status), - Ok(Err(e)) => return Err(e), + Ok(Err(e)) => { + let _ = child.start_kill(); + Exit::WaitFailed(e) + } Err(_) => { let _ = child.start_kill(); let _ = child.wait().await; @@ -195,19 +206,21 @@ async fn collect_stderr(mut pipe: ChildStderr, kept: Arc>>) { } } -/// The first stderr line that reads as an error (`Error`, `error`, `Warning`), else the first non-empty -/// line. +/// The stderr line that best explains a failure: the first that starts with `Error` or `BLAST`, else +/// the first containing `error`/`Error`, else the first `Warning`, else the first non-empty line. A +/// benign early warning (BLAST+ warns when `max_target_seqs < 5`) never hides a later error. fn first_error_line(stderr: &[u8]) -> Option<&str> { let text = match std::str::from_utf8(stderr) { Ok(t) => t, // Cut at the last valid boundary (the 64 KiB cap can split a character). Err(e) => std::str::from_utf8(&stderr[..e.valid_up_to()]).unwrap_or(""), }; - let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty()); - let first = lines.clone().next(); - lines - .find(|l| l.contains("Error") || l.contains("error") || l.contains("Warning")) - .or(first) + let lines = || text.lines().map(str::trim).filter(|l| !l.is_empty()); + lines() + .find(|l| l.starts_with("Error") || l.starts_with("BLAST")) + .or_else(|| lines().find(|l| l.contains("Error") || l.contains("error"))) + .or_else(|| lines().find(|l| l.contains("Warning"))) + .or_else(|| lines().next()) } /// `line` with every token that starts an absolute path (after `[`, `(` or a quote) replaced by `` and cut to `DETAIL_MAX` characters. @@ -481,7 +494,20 @@ mod tests { #[test] fn first_error_line_prefers_errors() { let s = b"\n some banner\nWarning: thing\nError: bad\n"; - assert_eq!(first_error_line(s), Some("Warning: thing")); + assert_eq!(first_error_line(s), Some("Error: bad")); + let s = b"Warning: [blastn] Examining 5 or more matches is recommended\nBLAST query/options error: bad word_size\n"; + assert_eq!( + first_error_line(s), + Some("BLAST query/options error: bad word_size") + ); + assert_eq!( + first_error_line(b"banner\nsomething error-ish\nWarning: w\n"), + Some("something error-ish") + ); + assert_eq!( + first_error_line(b"banner\nWarning: w\n"), + Some("Warning: w") + ); assert_eq!(first_error_line(b"\n\nplain\n"), Some("plain")); assert_eq!(first_error_line(b""), None); // A cut multi-byte character at the end is tolerated. diff --git a/blast-jobs/tests/fake-blast/blastn b/blast-jobs/tests/fake-blast/blastn index 58a632e..57f0fce 100755 --- a/blast-jobs/tests/fake-blast/blastn +++ b/blast-jobs/tests/fake-blast/blastn @@ -1,11 +1,12 @@ #!/bin/sh # Fake BLAST+ program for the blast-jobs tests (the same script is committed as blastn, blastp, blastx, # tblastn and tblastx, mode 100755). It records its argv, one element per line, to argv.txt in the job -# directory, then writes canned.tsv to the -out path. +# directory and its pid to pid.txt, then writes canned.tsv to the -out path. # # The runner gives BLAST+ a fixed environment, so the test controls come from # /fake-blast.env (sourced when present; state_root is three levels above -out): -# FAKE_BLAST_SLEEP= sleep before doing anything else +# FAKE_BLAST_SLEEP= sleep after writing the output (as the last step, by exec, so a kill +# leaves no orphan) and before exiting # FAKE_BLAST_FAIL=db BLAST database error on stderr, exit 2 # FAKE_BLAST_FAIL=oom die by SIGKILL, as under the container's memory cap # FAKE_BLAST_FAIL=noisy 200 KiB of error lines on stderr, exit 4 @@ -28,26 +29,33 @@ job=$(dirname "$out") ctl=$(dirname "$(dirname "$job")")/fake-blast.env if [ -f "$ctl" ]; then . "$ctl"; fi printf '%s\n' "$0" "$@" > "$job/argv.txt" -if [ -n "$FAKE_BLAST_SLEEP" ]; then - # Detached from the stderr pipe, so a killed parent does not leave the pipe open. - sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 -fi +echo $$ > "$job/pid.txt" +rc=0 case "$FAKE_BLAST_FAIL" in - "") ;; + "") + if [ -n "$FAKE_BLAST_BIG" ]; then + yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" + else + cat "$here/canned.tsv" > "$out" + fi ;; db) echo "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" >&2 - exit 2 ;; + rc=2 ;; oom) kill -9 $$ ;; noisy) yes "Error: noisy failure reading /some/private/path/file.nsq" | head -c 204800 >&2 - exit 4 ;; + rc=4 ;; *) echo "BLAST query/options error: fake failure for /opt/private/thing" >&2 - exit 3 ;; + rc=3 ;; esac -if [ -n "$FAKE_BLAST_BIG" ]; then - yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" -else - cat "$here/canned.tsv" > "$out" +if [ -n "$FAKE_BLAST_SLEEP" ]; then + if [ "$rc" -eq 0 ]; then + # The shell becomes the sleep, so a kill at the wall limit ends it and nothing is orphaned. + # Its stdio leaves the stderr pipe, so the pipe closes with it either way. + exec sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 + fi + sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 fi +exit "$rc" diff --git a/blast-jobs/tests/fake-blast/blastp b/blast-jobs/tests/fake-blast/blastp index 58a632e..57f0fce 100755 --- a/blast-jobs/tests/fake-blast/blastp +++ b/blast-jobs/tests/fake-blast/blastp @@ -1,11 +1,12 @@ #!/bin/sh # Fake BLAST+ program for the blast-jobs tests (the same script is committed as blastn, blastp, blastx, # tblastn and tblastx, mode 100755). It records its argv, one element per line, to argv.txt in the job -# directory, then writes canned.tsv to the -out path. +# directory and its pid to pid.txt, then writes canned.tsv to the -out path. # # The runner gives BLAST+ a fixed environment, so the test controls come from # /fake-blast.env (sourced when present; state_root is three levels above -out): -# FAKE_BLAST_SLEEP= sleep before doing anything else +# FAKE_BLAST_SLEEP= sleep after writing the output (as the last step, by exec, so a kill +# leaves no orphan) and before exiting # FAKE_BLAST_FAIL=db BLAST database error on stderr, exit 2 # FAKE_BLAST_FAIL=oom die by SIGKILL, as under the container's memory cap # FAKE_BLAST_FAIL=noisy 200 KiB of error lines on stderr, exit 4 @@ -28,26 +29,33 @@ job=$(dirname "$out") ctl=$(dirname "$(dirname "$job")")/fake-blast.env if [ -f "$ctl" ]; then . "$ctl"; fi printf '%s\n' "$0" "$@" > "$job/argv.txt" -if [ -n "$FAKE_BLAST_SLEEP" ]; then - # Detached from the stderr pipe, so a killed parent does not leave the pipe open. - sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 -fi +echo $$ > "$job/pid.txt" +rc=0 case "$FAKE_BLAST_FAIL" in - "") ;; + "") + if [ -n "$FAKE_BLAST_BIG" ]; then + yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" + else + cat "$here/canned.tsv" > "$out" + fi ;; db) echo "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" >&2 - exit 2 ;; + rc=2 ;; oom) kill -9 $$ ;; noisy) yes "Error: noisy failure reading /some/private/path/file.nsq" | head -c 204800 >&2 - exit 4 ;; + rc=4 ;; *) echo "BLAST query/options error: fake failure for /opt/private/thing" >&2 - exit 3 ;; + rc=3 ;; esac -if [ -n "$FAKE_BLAST_BIG" ]; then - yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" -else - cat "$here/canned.tsv" > "$out" +if [ -n "$FAKE_BLAST_SLEEP" ]; then + if [ "$rc" -eq 0 ]; then + # The shell becomes the sleep, so a kill at the wall limit ends it and nothing is orphaned. + # Its stdio leaves the stderr pipe, so the pipe closes with it either way. + exec sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 + fi + sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 fi +exit "$rc" diff --git a/blast-jobs/tests/fake-blast/blastx b/blast-jobs/tests/fake-blast/blastx index 58a632e..57f0fce 100755 --- a/blast-jobs/tests/fake-blast/blastx +++ b/blast-jobs/tests/fake-blast/blastx @@ -1,11 +1,12 @@ #!/bin/sh # Fake BLAST+ program for the blast-jobs tests (the same script is committed as blastn, blastp, blastx, # tblastn and tblastx, mode 100755). It records its argv, one element per line, to argv.txt in the job -# directory, then writes canned.tsv to the -out path. +# directory and its pid to pid.txt, then writes canned.tsv to the -out path. # # The runner gives BLAST+ a fixed environment, so the test controls come from # /fake-blast.env (sourced when present; state_root is three levels above -out): -# FAKE_BLAST_SLEEP= sleep before doing anything else +# FAKE_BLAST_SLEEP= sleep after writing the output (as the last step, by exec, so a kill +# leaves no orphan) and before exiting # FAKE_BLAST_FAIL=db BLAST database error on stderr, exit 2 # FAKE_BLAST_FAIL=oom die by SIGKILL, as under the container's memory cap # FAKE_BLAST_FAIL=noisy 200 KiB of error lines on stderr, exit 4 @@ -28,26 +29,33 @@ job=$(dirname "$out") ctl=$(dirname "$(dirname "$job")")/fake-blast.env if [ -f "$ctl" ]; then . "$ctl"; fi printf '%s\n' "$0" "$@" > "$job/argv.txt" -if [ -n "$FAKE_BLAST_SLEEP" ]; then - # Detached from the stderr pipe, so a killed parent does not leave the pipe open. - sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 -fi +echo $$ > "$job/pid.txt" +rc=0 case "$FAKE_BLAST_FAIL" in - "") ;; + "") + if [ -n "$FAKE_BLAST_BIG" ]; then + yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" + else + cat "$here/canned.tsv" > "$out" + fi ;; db) echo "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" >&2 - exit 2 ;; + rc=2 ;; oom) kill -9 $$ ;; noisy) yes "Error: noisy failure reading /some/private/path/file.nsq" | head -c 204800 >&2 - exit 4 ;; + rc=4 ;; *) echo "BLAST query/options error: fake failure for /opt/private/thing" >&2 - exit 3 ;; + rc=3 ;; esac -if [ -n "$FAKE_BLAST_BIG" ]; then - yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" -else - cat "$here/canned.tsv" > "$out" +if [ -n "$FAKE_BLAST_SLEEP" ]; then + if [ "$rc" -eq 0 ]; then + # The shell becomes the sleep, so a kill at the wall limit ends it and nothing is orphaned. + # Its stdio leaves the stderr pipe, so the pipe closes with it either way. + exec sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 + fi + sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 fi +exit "$rc" diff --git a/blast-jobs/tests/fake-blast/tblastn b/blast-jobs/tests/fake-blast/tblastn index 58a632e..57f0fce 100755 --- a/blast-jobs/tests/fake-blast/tblastn +++ b/blast-jobs/tests/fake-blast/tblastn @@ -1,11 +1,12 @@ #!/bin/sh # Fake BLAST+ program for the blast-jobs tests (the same script is committed as blastn, blastp, blastx, # tblastn and tblastx, mode 100755). It records its argv, one element per line, to argv.txt in the job -# directory, then writes canned.tsv to the -out path. +# directory and its pid to pid.txt, then writes canned.tsv to the -out path. # # The runner gives BLAST+ a fixed environment, so the test controls come from # /fake-blast.env (sourced when present; state_root is three levels above -out): -# FAKE_BLAST_SLEEP= sleep before doing anything else +# FAKE_BLAST_SLEEP= sleep after writing the output (as the last step, by exec, so a kill +# leaves no orphan) and before exiting # FAKE_BLAST_FAIL=db BLAST database error on stderr, exit 2 # FAKE_BLAST_FAIL=oom die by SIGKILL, as under the container's memory cap # FAKE_BLAST_FAIL=noisy 200 KiB of error lines on stderr, exit 4 @@ -28,26 +29,33 @@ job=$(dirname "$out") ctl=$(dirname "$(dirname "$job")")/fake-blast.env if [ -f "$ctl" ]; then . "$ctl"; fi printf '%s\n' "$0" "$@" > "$job/argv.txt" -if [ -n "$FAKE_BLAST_SLEEP" ]; then - # Detached from the stderr pipe, so a killed parent does not leave the pipe open. - sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 -fi +echo $$ > "$job/pid.txt" +rc=0 case "$FAKE_BLAST_FAIL" in - "") ;; + "") + if [ -n "$FAKE_BLAST_BIG" ]; then + yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" + else + cat "$here/canned.tsv" > "$out" + fi ;; db) echo "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" >&2 - exit 2 ;; + rc=2 ;; oom) kill -9 $$ ;; noisy) yes "Error: noisy failure reading /some/private/path/file.nsq" | head -c 204800 >&2 - exit 4 ;; + rc=4 ;; *) echo "BLAST query/options error: fake failure for /opt/private/thing" >&2 - exit 3 ;; + rc=3 ;; esac -if [ -n "$FAKE_BLAST_BIG" ]; then - yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" -else - cat "$here/canned.tsv" > "$out" +if [ -n "$FAKE_BLAST_SLEEP" ]; then + if [ "$rc" -eq 0 ]; then + # The shell becomes the sleep, so a kill at the wall limit ends it and nothing is orphaned. + # Its stdio leaves the stderr pipe, so the pipe closes with it either way. + exec sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 + fi + sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 fi +exit "$rc" diff --git a/blast-jobs/tests/fake-blast/tblastx b/blast-jobs/tests/fake-blast/tblastx index 58a632e..57f0fce 100755 --- a/blast-jobs/tests/fake-blast/tblastx +++ b/blast-jobs/tests/fake-blast/tblastx @@ -1,11 +1,12 @@ #!/bin/sh # Fake BLAST+ program for the blast-jobs tests (the same script is committed as blastn, blastp, blastx, # tblastn and tblastx, mode 100755). It records its argv, one element per line, to argv.txt in the job -# directory, then writes canned.tsv to the -out path. +# directory and its pid to pid.txt, then writes canned.tsv to the -out path. # # The runner gives BLAST+ a fixed environment, so the test controls come from # /fake-blast.env (sourced when present; state_root is three levels above -out): -# FAKE_BLAST_SLEEP= sleep before doing anything else +# FAKE_BLAST_SLEEP= sleep after writing the output (as the last step, by exec, so a kill +# leaves no orphan) and before exiting # FAKE_BLAST_FAIL=db BLAST database error on stderr, exit 2 # FAKE_BLAST_FAIL=oom die by SIGKILL, as under the container's memory cap # FAKE_BLAST_FAIL=noisy 200 KiB of error lines on stderr, exit 4 @@ -28,26 +29,33 @@ job=$(dirname "$out") ctl=$(dirname "$(dirname "$job")")/fake-blast.env if [ -f "$ctl" ]; then . "$ctl"; fi printf '%s\n' "$0" "$@" > "$job/argv.txt" -if [ -n "$FAKE_BLAST_SLEEP" ]; then - # Detached from the stderr pipe, so a killed parent does not leave the pipe open. - sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 -fi +echo $$ > "$job/pid.txt" +rc=0 case "$FAKE_BLAST_FAIL" in - "") ;; + "") + if [ -n "$FAKE_BLAST_BIG" ]; then + yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" + else + cat "$here/canned.tsv" > "$out" + fi ;; db) echo "BLAST Database error: No alias or index file found for nucleotide database [/db/dmel/x] in search path [/tmp::]" >&2 - exit 2 ;; + rc=2 ;; oom) kill -9 $$ ;; noisy) yes "Error: noisy failure reading /some/private/path/file.nsq" | head -c 204800 >&2 - exit 4 ;; + rc=4 ;; *) echo "BLAST query/options error: fake failure for /opt/private/thing" >&2 - exit 3 ;; + rc=3 ;; esac -if [ -n "$FAKE_BLAST_BIG" ]; then - yes "$(head -n 1 "$here/canned.tsv")" | head -c 20971520 > "$out" -else - cat "$here/canned.tsv" > "$out" +if [ -n "$FAKE_BLAST_SLEEP" ]; then + if [ "$rc" -eq 0 ]; then + # The shell becomes the sleep, so a kill at the wall limit ends it and nothing is orphaned. + # Its stdio leaves the stderr pipe, so the pipe closes with it either way. + exec sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 + fi + sleep "$FAKE_BLAST_SLEEP" /dev/null 2>&1 fi +exit "$rc" diff --git a/blast-jobs/tests/runner.rs b/blast-jobs/tests/runner.rs index a812742..41c1f8e 100644 --- a/blast-jobs/tests/runner.rs +++ b/blast-jobs/tests/runner.rs @@ -288,7 +288,35 @@ async fn wall_limit_kills_and_fails_the_job() { took < Duration::from_secs(4), "killed at the limit, took {took:?}" ); - assert!(!env.dir(&id).join("results.tsv").exists()); + let dir = env.dir(&id); + assert!(!dir.join("results.tsv").exists()); + assert!(!dir.join("raw.tsv").exists(), "partial output removed"); + // Nothing the job started is still running: every process BLAST+ (or the fake) started has the + // job directory as its working directory, so scan /proc for one. + let left: Vec = std::fs::read_dir("/proc") + .unwrap() + .flatten() + .filter(|e| { + e.file_name() + .to_string_lossy() + .bytes() + .all(|b| b.is_ascii_digit()) + }) + .filter(|e| std::fs::read_link(e.path().join("cwd")).is_ok_and(|c| c == dir)) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect(); + assert!(left.is_empty(), "processes left in the job dir: {left:?}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn relative_bin_dir_in_the_config_still_runs() { + // cargo runs integration tests from the crate root, where this relative path names the fakes. + assert!(Path::new("tests/fake-blast/blastn").is_file()); + let env = setup(json!({"blast_bin_dir": "tests/fake-blast"}), ""); + assert!(env.cfg.blast_bin_dir.is_absolute()); + let id = env.submit(request("blastn")).await; + let s = env.wait_terminal(&id, Duration::from_secs(10)).await; + assert_eq!(s.state, State::Complete, "{:?}", s.error); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] From f6ad188e230241f4ca290a9325c27500bc790ec8 Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Mon, 12 Oct 2026 18:10:57 +0000 Subject: [PATCH 08/12] feat(blast-jobs): container image, real BLAST+ integration test, docs Dockerfile: rust:1.90-bookworm --locked build; NCBI BLAST+ 2.17.0 x64 tarball from the versioned HTTPS path pinned by SHA-256 3888112d...a411b; bookworm-slim runtime with curl and libgomp1, the seven binaries in /opt/ncbi-blast/bin, -version checks at build time, uid 10002, HEALTHCHECK on /internal/health?liveness=1. tests/real_blast.rs (skips unless BLAST_JOBS_REAL_BLAST names a BLAST+ bin dir): builds the Notch fixture with the archive's makeblastdb flags (-parse_seqids) and checks a known hit for blastn, blastp, blastx, tblastn and tblastx, -task blastn-short via advanced, and the dust/seg defaults. tests/image-smoke.sh: blast-jobs check plus one real search and the health check inside the image. CI: blast-jobs-integration and blast-jobs-image, on master pushes and blast-jobs/ changes only. Docs: docs/blast-jobs.md, deploy samples, handoff section 7 points at blast-jobs. --- .github/workflows/ci.yml | 53 ++++ README.md | 9 + blast-jobs/.dockerignore | 9 + blast-jobs/Dockerfile | 44 ++++ blast-jobs/README.md | 24 +- blast-jobs/deploy/config.sample.json | 19 ++ blast-jobs/deploy/run.sample.sh | 22 ++ blast-jobs/fixture/README.md | 18 ++ blast-jobs/fixture/build-db.sh | 12 + blast-jobs/fixture/catalogue.json | 7 + blast-jobs/fixture/notch-protein.fa | 62 +++++ blast-jobs/fixture/notch.fa | 174 +++++++++++++ blast-jobs/tests/image-smoke.sh | 74 ++++++ blast-jobs/tests/real_blast.rs | 368 +++++++++++++++++++++++++++ docs/blast-jobs.md | 202 +++++++++++++++ docs/handoff-private-side.md | 7 +- 16 files changed, 1102 insertions(+), 2 deletions(-) create mode 100644 blast-jobs/.dockerignore create mode 100644 blast-jobs/Dockerfile create mode 100644 blast-jobs/deploy/config.sample.json create mode 100755 blast-jobs/deploy/run.sample.sh create mode 100644 blast-jobs/fixture/README.md create mode 100755 blast-jobs/fixture/build-db.sh create mode 100644 blast-jobs/fixture/catalogue.json create mode 100644 blast-jobs/fixture/notch-protein.fa create mode 100644 blast-jobs/fixture/notch.fa create mode 100755 blast-jobs/tests/image-smoke.sh create mode 100644 blast-jobs/tests/real_blast.rs create mode 100644 docs/blast-jobs.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 060e344..c4bfeb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,3 +53,56 @@ jobs: manifest-path: blast-jobs/Cargo.toml command: check advisories licenses - run: cargo test + blast-jobs-changes: + runs-on: ubuntu-latest + permissions: { contents: read, pull-requests: read } + outputs: + blast_jobs: ${{ steps.filter.outputs.blast_jobs }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + blast_jobs: + - 'blast-jobs/**' + - '.github/workflows/ci.yml' + # The two jobs below download BLAST+ from NCBI, so they run on master pushes and on changes to blast-jobs/ + # only; a slow NCBI must not redden unrelated PRs (design 3.12). + blast-jobs-integration: + needs: blast-jobs-changes + if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || needs.blast-jobs-changes.outputs.blast_jobs == 'true' + runs-on: ubuntu-latest + defaults: { run: { working-directory: blast-jobs } } + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: { workspaces: blast-jobs } + # The tarball URL and SHA-256 are pinned once, in the Dockerfile. + - id: pin + run: | + echo "url=$(sed -n 's/^ARG BLAST_URL=//p' Dockerfile)" >> "$GITHUB_OUTPUT" + echo "sha=$(sed -n 's/^ARG BLAST_SHA256=//p' Dockerfile)" >> "$GITHUB_OUTPUT" + - uses: actions/cache@v4 + id: blast-cache + with: + path: ~/ncbi-blast.tar.gz + key: ncbi-blast-${{ steps.pin.outputs.sha }} + - if: steps.blast-cache.outputs.cache-hit != 'true' + run: curl -fsSL --retry 3 -o ~/ncbi-blast.tar.gz "${{ steps.pin.outputs.url }}" + - run: | + echo "${{ steps.pin.outputs.sha }} $HOME/ncbi-blast.tar.gz" | sha256sum -c - + mkdir -p ~/ncbi-blast && tar -xzf ~/ncbi-blast.tar.gz -C ~/ncbi-blast --strip-components 1 + ~/ncbi-blast/bin/blastn -version + - run: BLAST_JOBS_REAL_BLAST="$HOME/ncbi-blast/bin" cargo test --test real_blast + blast-jobs-image: + needs: blast-jobs-changes + if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || needs.blast-jobs-changes.outputs.blast_jobs == 'true' + runs-on: ubuntu-latest + defaults: { run: { working-directory: blast-jobs } } + steps: + - uses: actions/checkout@v4 + - run: docker build -t blast-jobs:ci . + - run: docker image ls blast-jobs:ci + - run: tests/image-smoke.sh blast-jobs:ci diff --git a/README.md b/README.md index 128ac89..6c90562 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ Tools for using FlyBase data from AI assistants. - `mcp/`: the hosted FlyBase MCP server (Rust). Endpoint: `https://mcp.flybase.org/mcp`. - `plugins/flybase/`: a Claude Code plugin that registers the MCP server. Skills will live here later. +- `blast-jobs/`: the BLAST job service behind the MCP's BLAST tools (Rust, NCBI BLAST+ 2.17.0). The server's tools and their arguments are listed in [docs/tools.md](docs/tools.md). @@ -19,6 +20,14 @@ features in a range, cytology, release 5 to 6 coordinates), controlled vocabular gene groups, disease models, stocks, interactions), RNA-seq expression, and BLAST. Each is described in [docs/tools.md](docs/tools.md). +## BLAST + +The MCP's BLAST tools are clients of a private job API. `blast-jobs/` implements it. The service is a +small Rust process that runs NCBI BLAST+ 2.17.0 against a directory of BLAST databases described by a +catalogue file, in its own container with the databases mounted read-only. Nothing in it is specific to +FlyBase, so anyone with BLAST databases and a catalogue can run it. Configuration, the image and the run +shape are in [docs/blast-jobs.md](docs/blast-jobs.md). + ## Develop See `docs/deployment.md` for running the server and `mcp/README.md` for the crate. diff --git a/blast-jobs/.dockerignore b/blast-jobs/.dockerignore new file mode 100644 index 0000000..566b50d --- /dev/null +++ b/blast-jobs/.dockerignore @@ -0,0 +1,9 @@ +target +tests +fixture +deploy +bench +rust-toolchain.toml +README.md +Dockerfile +.dockerignore diff --git a/blast-jobs/Dockerfile b/blast-jobs/Dockerfile new file mode 100644 index 0000000..5ddb414 --- /dev/null +++ b/blast-jobs/Dockerfile @@ -0,0 +1,44 @@ +# blast-jobs: the service binary plus NCBI BLAST+ 2.17.0 from NCBI's official tarball (design 3.9, ruling 6). +# rust-toolchain.toml is excluded by .dockerignore: the image pins its own toolchain here, and rustup would +# otherwise try to install the file's channel at build time. +FROM rust:1.90-bookworm AS build +WORKDIR /src +COPY Cargo.toml Cargo.lock ./ +COPY src ./src +RUN cargo build --release --locked --bin blast-jobs + +# Versioned path (never LATEST), pinned by SHA-256. NCBI publishes only an MD5 next to the tarball +# (bdec166721de3b55f90a3badc83538e8); the SHA-256 below was computed from a download that matched it. +FROM debian:bookworm-slim AS blast +ARG BLAST_URL=https://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.17.0/ncbi-blast-2.17.0+-x64-linux.tar.gz +ARG BLAST_SHA256=3888112d8207831aa47371d93583c601f058f88b5db22dc782438b039a3a411b +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && rm -rf /var/lib/apt/lists/* +RUN curl -fsSL --retry 3 -o /tmp/blast.tar.gz "$BLAST_URL" \ + && echo "$BLAST_SHA256 /tmp/blast.tar.gz" | sha256sum -c - \ + && mkdir -p /opt/ncbi-blast/bin \ + && tar -xzf /tmp/blast.tar.gz -C /tmp \ + && cd /tmp/ncbi-blast-2.17.0+/bin \ + && cp blastn blastp blastx tblastn tblastx blastdbcmd makeblastdb /opt/ncbi-blast/bin/ \ + && cp /tmp/ncbi-blast-2.17.0+/LICENSE /opt/ncbi-blast/LICENSE \ + && rm -rf /tmp/blast.tar.gz /tmp/ncbi-blast-2.17.0+ + +FROM debian:bookworm-slim +# curl for the HEALTHCHECK; libgomp1 is the one shared library BLAST+ needs beyond the base image. +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl libgomp1 && rm -rf /var/lib/apt/lists/* +COPY --from=blast /opt/ncbi-blast /opt/ncbi-blast +ENV PATH=/opt/ncbi-blast/bin:$PATH NCBI_DONT_USE_NCBIRC=1 NCBI_DONT_USE_LOCAL_CONFIG=1 BLAST_USAGE_REPORT=false +# A missing shared library or a wrong build fails here, not at the first job. +RUN for b in blastn blastp blastx tblastn tblastx blastdbcmd makeblastdb; do \ + "$b" -version | grep -q '2\.17\.0+' || { echo "$b is not BLAST+ 2.17.0" >&2; exit 1; }; \ + done +RUN useradd --system --uid 10002 --no-create-home blastjobs +COPY --from=build /src/target/release/blast-jobs /usr/local/bin/blast-jobs +USER blastjobs +EXPOSE 8800 +ENV RUST_LOG=info +# Liveness only (design 3.8): a full queue answers 503 on the plain health route but must not restart +# the container. The port is the config's `bind`; the sample uses 8800. +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s \ + CMD curl -fsS 'http://127.0.0.1:8800/internal/health?liveness=1' > /dev/null || exit 1 +ENTRYPOINT ["blast-jobs"] +CMD ["serve", "/etc/blast-jobs/config.json"] diff --git a/blast-jobs/README.md b/blast-jobs/README.md index 664fa3c..832cfbd 100644 --- a/blast-jobs/README.md +++ b/blast-jobs/README.md @@ -20,9 +20,31 @@ Run every command below from `blast-jobs/`. cargo fmt && cargo clippy --all-targets -- -D warnings cargo deny check advisories licenses +`tests/real_blast.rs` runs only when `BLAST_JOBS_REAL_BLAST` names a BLAST+ bin directory; otherwise each +test prints `skipped: ...` and passes. It builds two tiny databases from `fixture/` (Notch N-RA and N-PA +from the FB2023_04 archive, see `fixture/README.md`) and searches them through the API with all five +programs: + + BLAST_JOBS_REAL_BLAST=/path/to/ncbi-blast-2.17.0+/bin cargo test --test real_blast + +## Running + + blast-jobs check # validate config, catalogue, volumes and binaries; exit 0 or 1 + blast-jobs serve # run the API (needs BLAST_JOBS_TOKEN, or BLAST_JOBS_ALLOW_NO_AUTH=1) + +The image bundles NCBI BLAST+ 2.17.0 (pinned tarball) and runs `serve /etc/blast-jobs/config.json` as +uid 10002: + + docker build -t blast-jobs:dev . + tests/image-smoke.sh blast-jobs:dev # check + one real search inside the image + +`deploy/config.sample.json` and `deploy/run.sample.sh` are the config and `docker run` shape used next to +the MCP. [`docs/blast-jobs.md`](../docs/blast-jobs.md) covers the API, the masking defaults, limits, token +pairing with the MCP and rollback. + ## Configuration -One JSON file, passed as the first argument (`blast-jobs `). Unknown keys are refused. +One JSON file, passed after the subcommand (`blast-jobs serve `). Unknown keys are refused. | Key | Default | Meaning | |---|---|---| diff --git a/blast-jobs/deploy/config.sample.json b/blast-jobs/deploy/config.sample.json new file mode 100644 index 0000000..3391f42 --- /dev/null +++ b/blast-jobs/deploy/config.sample.json @@ -0,0 +1,19 @@ +{ + "bind": "0.0.0.0:8800", + "blast_bin_dir": "/opt/ncbi-blast/bin", + "database_root": "/db", + "catalogue": "/db/catalogue.json", + "database_release": "FB2023_04", + "state_root": "/state", + "concurrency": 2, + "threads_per_job": 2, + "max_queued": 20, + "wall_seconds": 600, + "translated_wall_seconds": 300, + "retention_seconds": 86400, + "max_results_bytes": 16777216, + "default_evalue": 10, + "default_max_hits": 100, + "max_hsps_per_subject": 50, + "max_residues": { "blastn": 100000, "blastp": 100000, "blastx": 30000, "tblastn": 30000, "tblastx": 10000 } +} diff --git a/blast-jobs/deploy/run.sample.sh b/blast-jobs/deploy/run.sample.sh new file mode 100755 index 0000000..8080d9c --- /dev/null +++ b/blast-jobs/deploy/run.sample.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Sample run script for blast-jobs next to the FlyBase MCP. Every variable below is required: +# IMAGE the image tag, e.g. blast-jobs: +# NETWORK the private Docker network the MCP container is on +# DATABASE_DIR the BLAST database directory (catalogue.json and the volumes); mounted read-only +# STATE_DIR writable job state, owned by uid 10002 +# CONFIG config.sample.json from this directory, edited for the host +# SECRETS_ENV an env file holding only BLAST_JOBS_TOKEN=; the MCP's BLAST_SERVICE_TOKEN is the +# same value +# The MCP reaches the service as http://blast-jobs:8800; no port is published. For host-side checks add +# `-p 127.0.0.1::8800` (loopback only). +set -eu +: "${IMAGE:?}" "${NETWORK:?}" "${DATABASE_DIR:?}" "${STATE_DIR:?}" "${CONFIG:?}" "${SECRETS_ENV:?}" +docker rm -f blast-jobs 2>/dev/null || true +exec docker run -d --name blast-jobs --network "$NETWORK" --init \ + --cpus 4 --cpu-shares 512 --memory 4g --pids-limit 256 --read-only --tmpfs /tmp:size=256m \ + --restart unless-stopped --log-opt max-size=20m --log-opt max-file=3 \ + --mount "type=bind,src=$DATABASE_DIR,dst=/db,readonly" \ + --mount "type=bind,src=$STATE_DIR,dst=/state" \ + --mount "type=bind,src=$CONFIG,dst=/etc/blast-jobs/config.json,readonly" \ + --env-file "$SECRETS_ENV" \ + "$IMAGE" diff --git a/blast-jobs/fixture/README.md b/blast-jobs/fixture/README.md new file mode 100644 index 0000000..45829d2 --- /dev/null +++ b/blast-jobs/fixture/README.md @@ -0,0 +1,18 @@ +# Real-BLAST fixture + +Two tiny BLAST databases for `tests/real_blast.rs` and the CI image check. The sequences and deflines are +copied verbatim (trailing spaces trimmed) from the archived FB2023_04 BLAST set the service serves in +production, read with `blastdbcmd -outfmt %f`: + +| File | Entries | +|---|---| +| `notch.fa` (nucleotide, category `transcript`) | `fbdb.dmel.transcript.5460` N-RA (FBtr0070507, 10,150 bp); `fbdb.dmel.transcript.1` snoRNA:M-RA (FBtr0070292) | +| `notch-protein.fa` (protein, category `translation`) | `fbdb.dmel.translation.529` N-PA (FBpp0070483, 2,703 aa); `fbdb.dmel.translation.1` Nep3-PA (FBpp0070000) | + +The archive was built with `-parse_seqids`, so `sseqid` is the local id (`fbdb.dmel.transcript.5460`) and +the FlyBase id is in `stitle` (`... ID=FBtr0070507; name=N-RA; ...`). `build-db.sh` uses the same flags: + + fixture/build-db.sh fixture + +`catalogue.json` maps `dmel transcript` and `dmel translation` to the built volumes, in the same layout as +the archive (`dmel/dmel-transcript`, `dmel/dmel-translation`). diff --git a/blast-jobs/fixture/build-db.sh b/blast-jobs/fixture/build-db.sh new file mode 100755 index 0000000..644297a --- /dev/null +++ b/blast-jobs/fixture/build-db.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Builds the fixture BLAST databases the way the FB2023_04 archive was built (-parse_seqids, v5 volumes), +# laid out as catalogue.json expects: /dmel/dmel-transcript.* and /dmel/dmel-translation.*. +# usage: build-db.sh +set -eu +bin=$1 fixture=$2 out=$3 +mkdir -p "$out/dmel" +cp "$fixture/catalogue.json" "$out/catalogue.json" +"$bin/makeblastdb" -in "$fixture/notch.fa" -dbtype nucl -parse_seqids -blastdb_version 5 \ + -title "Notch fixture transcripts" -out "$out/dmel/dmel-transcript" >/dev/null +"$bin/makeblastdb" -in "$fixture/notch-protein.fa" -dbtype prot -parse_seqids -blastdb_version 5 \ + -title "Notch fixture translations" -out "$out/dmel/dmel-translation" >/dev/null diff --git a/blast-jobs/fixture/catalogue.json b/blast-jobs/fixture/catalogue.json new file mode 100644 index 0000000..109a622 --- /dev/null +++ b/blast-jobs/fixture/catalogue.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "databases": [ + { "species": "dmel", "category": "transcript", "label": "Notch fixture transcripts", "protein": false, "prefix": "dmel/dmel-transcript" }, + { "species": "dmel", "category": "translation", "label": "Notch fixture translations", "protein": true, "prefix": "dmel/dmel-translation" } + ] +} diff --git a/blast-jobs/fixture/notch-protein.fa b/blast-jobs/fixture/notch-protein.fa new file mode 100644 index 0000000..5f4a84f --- /dev/null +++ b/blast-jobs/fixture/notch-protein.fa @@ -0,0 +1,62 @@ +>fbdb.dmel.translation.529 fb_original=646d656c3a4642707030303730343833; type=polypeptide; loc=X:join(3135668..3135827,3147975..3148065,3155927..3156189,3156260..3156464,3159103..3159233,3163479..3169627,3169722..3170031,3170104..3170559,3170635..3170981); ID=FBpp0070483; name=N-PA; parent=FBgn0004647,FBtr0070507; dbxref=FlyBase:FBpp0070483,FlyBase_Annotation_IDs:CG3936-PA,GB_protein:AAF45848.2,REFSEQ:NP_476859,GB_protein:AAF45848,UniProt/Swiss-Prot:P07207,FlyMine:FBpp0070483,modMine:FBpp0070483; MD5=eb325d68765649f3755c5a72e1db32ce; length=2703; release=r6.53; species=Dmel; +MQSQRSRRRSRAPNTWICFWINKMHAVASLPASLPLLLLTLAFANLPNTVRGTDTALVAA +SCTSVGCQNGGTCVTQLNGKTYCACDSHYVGDYCEHRNPCNSMRCQNGGTCQVTFRNGRP +GISCKCPLGFDESLCEIAVPNACDHVTCLNGGTCQLKTLEEYTCACANGYTGERCETKNL +CASSPCRNGATCTALAGSSSFTCSCPPGFTGDTCSYDIEECQSNPCKYGGTCVNTHGSYQ +CMCPTGYTGKDCDTKYKPCSPSPCQNGGICRSNGLSYECKCPKGFEGKNCEQNYDDCLGH +LCQNGGTCIDGISDYTCRCPPNFTGRFCQDDVDECAQRDHPVCQNGATCTNTHGSYSCIC +VNGWAGLDCSNNTDDCKQAACFYGATCIDGVGSFYCQCTKGKTGLLCHLDDACTSNPCHA +DAICDTSPINGSYACSCATGYKGVDCSEDIDECDQGSPCEHNGICVNTPGSYRCNCSQGF +TGPRCETNINECESHPCQNEGSCLDDPGTFRCVCMPGFTGTQCEIDIDECQSNPCLNDGT +CHDKINGFKCSCALGFTGARCQINIDDCQSQPCRNRGICHDSIAGYSCECPPGYTGTSCE +ININDCDSNPCHRGKCIDDVNSFKCLCDPGYTGYICQKQINECESNPCQFDGHCQDRVGS +YYCQCQAGTSGKNCEVNVNECHSNPCNNGATCIDGINSYKCQCVPGFTGQHCEKNVDECI +SSPCANNGVCIDQVNGYKCECPRGFYDAHCLSDVDECASNPCVNEGRCEDGINEFICHCP +PGYTGKRCELDIDECSSNPCQHGGTCYDKLNAFSCQCMPGYTGQKCETNIDDCVTNPCGN +GGTCIDKVNGYKCVCKVPFTGRDCESKMDPCASNRCKNEAKCTPSSNFLDFSCTCKLGYT +GRYCDEDIDECSLSSPCRNGASCLNVPGSYRCLCTKGYEGRDCAINTDDCASFPCQNGGT +CLDGIGDYSCLCVDGFDGKHCETDINECLSQPCQNGATCSQYVNSYTCTCPLGFSGINCQ +TNDEDCTESSCLNGGSCIDGINGYNCSCLAGYSGANCQYKLNKCDSNPCLNGATCHEQNN +EYTCHCPSGFTGKQCSEYVDWCGQSPCENGATCSQMKHQFSCKCSAGWTGKLCDVQTISC +QDAADRKGLSLRQLCNNGTCKDYGNSHVCYCSQGYAGSYCQKEIDECQSQPCQNGGTCRD +LIGAYECQCRQGFQGQNCELNIDDCAPNPCQNGGTCHDRVMNFSCSCPPGTMGIICEINK +DDCKPGACHNNGSCIDRVGGFECVCQPGFVGARCEGDINECLSNPCSNAGTLDCVQLVNN +YHCNCRPGHMGRHCEHKVDFCAQSPCQNGGNCNIRQSGHHCICNNGFYGKNCELSGQDCD +SNPCRVGNCVVADEGFGYRCECPRGTLGEHCEIDTLDECSPNPCAQGAACEDLLGDYECL +CPSKWKGKRCDIYDANYPGWNGGSGSGNDRYAADLEQQRAMCDKRGCTEKQGNGICDSDC +NTYACNFDGNDCSLGINPWANCTANECWNKFKNGKCNEECNNAACHYDGHDCERKLKSCD +SLFDAYCQKHYGDGFCDYGCNNAECSWDGLDCENKTQSPVLAEGAMSVVMLMNVEAFREI +QAQFLRNMSHMLRTTVRLKKDALGHDIIINWKDNVRVPEIEDTDFARKNKILYTQQVHQT +GIQIYLEIDNRKCTECFTHAVEAAEFLAATAAKHQLRNDFQIHSVRGIKNPGDEDNGEPP +ANVKYVITGIILVIIALAFFGMVLSTQRKRAHGVTWFPEGFRAPAAVMSRRRRDPHGQEM +RNLNKQVAMQSQGVGQPGAHWSDDESDMPLPKRQRSDPVSGVGLGNNGGYASDHTMVSEY +EEADQRVWSQAHLDVVDVRAIMTPPAHQDGGKHDVDARGPCGLTPLMIAAVRGGGLDTGE +DIENNEDSTAQVISDLLAQGAELNATMDKTGETSLHLAARFARADAAKRLLDAGADANCQ +DNTGRTPLHAAVAADAMGVFQILLRNRATNLNARMHDGTTPLILAARLAIEGMVEDLITA +DADINAADNSGKTALHWAAAVNNTEAVNILLMHHANRDAQDDKDETPLFLAAREGSYEAC +KALLDNFANREITDHMDRLPRDVASERLHHDIVRLLDEHVPRSPQMLSMTPQAMIGSPPP +GQQQPQLITQPTVISAGNGGNNGNGNASGKQSNQTAKQKAAKKAKLIEGSPDNGLDATGS +LRRKASSKKTSAASKKAANLNGLNPGQLTGGVSGVPGVPPTNSAAQAAAAAAAAVAAMSH +ELEGSPVGVGMGGNLPSPYDTSSMYSNAMAAPLANGNPNTGAKQPPSYEDCIKNAQSMQS +LQGNGLDMIKLDNYAYSMGSPFQQELLNGQGLGMNGNGQRNGVGPGVLPGGLCGMGGLSG +AGNGNSHEQGLSPPYSNQSPPHSVQSSLALSPHAYLGSPSPAKSRPSLPTSPTHIQAMRH +ATQQKQFGGSNLNSLLGGANGGGVVGGGGGGGGGVGQGPQNSPVSLGIISPTGSDMGIML +APPQSSKNSAIMQTISPQQQQQQQQQQQQQHQQQQQQQQQQQQQQQQQLGGLEFGSAGLD +LNGFCGSPDSFHSGQMNPPSIQSSMSGSSPSTNMLSPSSQHNQQAFYQYLTPSSQHSGGH +TPQHLVQTLDSYPTPSPESPGHWSSSSPRSNSDWSEGVQSPAANNLYISGGHQANKGSEA +IYI +>fbdb.dmel.translation.1 fb_original=646d656c3a4642707030303730303030; type=polypeptide; loc=X:join(19963955..19964071,19964782..19964944,19965006..19965126,19965197..19965511,19965577..19966071,19966183..19967012,19967081..19967223,19967284..19967460); ID=FBpp0070000; name=Nep3-PA; parent=FBgn0031081,FBtr0070000; dbxref=FlyBase:FBpp0070000,FlyBase_Annotation_IDs:CG9565-PA,GB_protein:AAF45370.2,REFSEQ:NP_523417,GB_protein:AAF45370,UniProt/Swiss-Prot:Q9W5Y0,FlyMine:FBpp0070000,modMine:FBpp0070000; MD5=19dfee3c4d8ec74f121a5b5f7f7682e1; length=786; release=r6.53; species=Dmel; +MTRYKQTEFTEDDSSSIGGIQLNEATGHTGMQIRYHTARATWNWRSRNKTEKWLLITTFV +MAITIFTLLIVLFTDGGSSDATKHVLHVQPHQKDCPSGNELPCLNKHCIFASSEILKSID +VTVDPCDDFYGYSCNQWIKNNPIPEGKSTWGTFGKLEQMNQLIIRNVLEKPAKSFKSDAE +RKAKVYYESCLDADEHMEKLGAKPMNDLLLQIGGWNVTKSGYNVANWTMGHTLKILHNKY +NFNCLFGWAIGEDDKNSSRHVIQIDQGGLTLPTADYYNNKTDNHRKVLNEYIEYMTKVCV +LLGANESDARAQMIGVINFEKKLANITIPLEDRRNEEAMYHPMQLRQLSKLAPFLNWTDH +FDNAMQMVGRRVTDDEVVVVYAPDFLKNLSDIILKMEQTEEGKITLNNYLVWQAVRTLTS +CLSKPFRDAYKGVRKALMGSDGGEEIWRYCVSDTNNVVGFAVGAIFVRQAFHGESKPAAE +QMIAEIREAFKMNLQNLTWVDKQTREKAIEKANQISDMIGFPDYILDPVELDKKYAELNI +TPNAYFENNIQVAIYNLKSNLKRLDQPVNKTNWGMTPQTVNAYYTPTKNQIVFPAGILQT +PFFDINNPKSLNFGAMGVVMGHELTHAFDDQGREYDKFGNINRWWDSKSIERFNEKSECI +ARQYSGYKMNGRTLNGKQTLGENIADNGGLKAAYHAYQRTKSDRDVDILKLPGLNLTHSQ +LFFVSFAQVWCSSTTDETNLLQMEKDPHSPSQFRVIGTLSNMKEFAEVFQCKPGKRMNPT +EKCEVW diff --git a/blast-jobs/fixture/notch.fa b/blast-jobs/fixture/notch.fa new file mode 100644 index 0000000..f7afdc8 --- /dev/null +++ b/blast-jobs/fixture/notch.fa @@ -0,0 +1,174 @@ +>fbdb.dmel.transcript.5460 fb_original=646d656c3a4642747230303730353037; type=mRNA; loc=X:join(3134870..3135827,3147975..3148065,3155927..3156189,3156260..3156464,3159103..3159233,3163479..3169627,3169722..3170031,3170104..3170559,3170635..3172221); ID=FBtr0070507; name=N-RA; dbxref=FlyBase:FBtr0070507,FlyBase_Annotation_IDs:CG3936-RA,REFSEQ:NM_057511,REFSEQ:NM_057511; MD5=b292093e76da25a55978c83e59f4f2ca; length=10150; parent=FBgn0004647; release=r6.53; species=Dmel; +ACAGATCGCTTTTTTCCAGTGGACGAAACGGTTGTGAAAGCGGACGAGCGTTAGGCAGAC +GAACCTGGAAAGCGCAGAGCACAGTTCTCAACATTTATTTTTTTTTTGAATGTGTGTGCA +ACAACGCACGTAAAATCGCGCTGCCAACAGGATATACAAACAAATCAATTACACAGCAAG +CAAATGCAATGAAATGAAAAGGATGGCCCCAGCGGGAAAGCCGTTCAGCAAGAGCAAGGA +GTGCCTGTCGCAGGGATAGCAACGAGAGAGCGACACAGAGAGCGAGAGAGAGAGAGGGAG +AGAAACAAGGATTTTCGAAAAGTGTATCTACCTCGAGTCGCGCGTGTGTGAGAGTGAGAC +GAAAGCCGAGTGCAAGAAGCGCAAGAGGAAGAGAGCAATACGCAAGCGTGCGGCGTCGGT +TTGAATTTGAATTTGTGGCTCGATCCTCGCGAAGAGAAAAGCAAGCAAAAGATACACGAA +AAGCGTTCTTTTTTTGCCACTTTTTTTTTATGTTTCAAAACGGAAAATGTCGCCGTCGTC +GGGAGAGTGCCTCCTCTTAGTTTATCAAATAAAGCTTTAAAGTCTGAAGTTCAAAACTAT +AAAAAACAAAAAAAAAAACAAAAAAACAACAAACGCGCTAAAACAAAAAATTGCTTAAAC +TACTGAGACTAGCATACTAAACCTAAACTCGCAGTTAAACATATCCTCAAAGAGAGACCA +AAGAGGAGCGCTAAAGAAACCGAGAAGCGAGTAGAGGTGAAAAACTGGAAAAGTGGTTAA +CTGGAAAACTGGATTACAATGCAATCGCAGCGCAGCCGACGTCGCAGTCGCGCACCAAAC +ACTTGGATTTGTTTTTGGATTAACAAAATGCACGCCGTTGCGTCGCTGCCGGCGTCGCTG +CCTCTGCTGCTGTTGACGCTGGCGTTTGCGAATTTGCCAAACACCGTTCGCGGAACTGAT +ACCGCGTTGGTGGCCGCTTCCTGCACAAGTGTCGGTTGCCAGAATGGCGGCACATGCGTT +ACACAACTCAATGGCAAAACCTACTGCGCATGCGATTCGCACTATGTCGGCGACTACTGT +GAACACCGCAATCCGTGCAATTCGATGCGTTGCCAAAATGGTGGCACCTGTCAGGTGACC +TTTCGCAACGGCCGTCCGGGCATCTCGTGCAAGTGTCCTTTGGGCTTCGACGAGTCCCTG +TGCGAAATTGCGGTACCGAATGCCTGCGATCATGTGACCTGCCTCAATGGAGGCACCTGT +CAGCTAAAGACACTGGAGGAGTACACGTGTGCGTGTGCCAATGGCTATACAGGTGAACGA +TGCGAGACGAAGAATCTGTGCGCAAGTTCTCCCTGCCGGAATGGAGCCACCTGCACCGCT +TTGGCCGGAAGCAGCAGCTTCACCTGCTCCTGTCCGCCGGGCTTCACCGGTGACACCTGT +TCCTATGACATCGAGGAGTGCCAGTCGAATCCGTGCAAATACGGCGGCACATGTGTCAAC +ACCCACGGATCTTACCAGTGCATGTGTCCCACGGGCTACACGGGAAAGGATTGCGACACC +AAGTACAAGCCCTGCTCGCCATCGCCGTGCCAGAATGGCGGAATTTGCCGGTCGAACGGT +CTGTCCTACGAGTGCAAGTGCCCCAAAGGTTTCGAGGGCAAGAACTGCGAACAGAACTAC +GACGATTGCCTGGGACATCTTTGCCAGAACGGAGGAACCTGCATCGATGGCATCTCGGAC +TACACCTGCCGCTGTCCACCGAACTTCACGGGCAGATTCTGCCAGGACGATGTGGACGAG +TGCGCCCAACGGGATCATCCGGTGTGCCAGAACGGAGCCACCTGTACAAACACTCACGGA +TCGTACAGCTGCATCTGCGTGAATGGCTGGGCGGGCTTGGACTGCAGCAACAACACGGAT +GACTGCAAACAGGCGGCCTGTTTCTACGGAGCCACGTGCATCGATGGCGTTGGCAGCTTC +TATTGCCAGTGTACGAAGGGCAAGACGGGTCTGTTGTGCCACCTGGACGATGCCTGTACC +TCGAATCCCTGCCATGCGGACGCCATTTGTGACACGAGTCCGATAAACGGCTCCTATGCC +TGTTCCTGTGCCACCGGCTACAAGGGCGTGGATTGTTCAGAGGACATAGATGAATGCGAT +CAGGGATCACCGTGCGAACACAATGGCATATGCGTGAACACACCGGGCAGCTACAGATGC +AATTGCTCCCAGGGCTTTACGGGTCCGCGCTGCGAGACGAACATCAATGAGTGCGAATCG +CATCCATGCCAGAACGAGGGATCTTGCCTGGATGATCCGGGAACGTTCCGATGCGTCTGT +ATGCCAGGATTCACGGGCACACAGTGCGAAATCGACATTGACGAATGCCAATCGAATCCC +TGCTTGAACGATGGAACTTGCCACGACAAGATCAATGGCTTTAAATGCAGTTGTGCCCTG +GGCTTTACGGGCGCACGATGTCAAATCAATATAGACGACTGTCAGTCGCAACCATGCCGG +AATCGTGGCATCTGTCACGACTCTATAGCGGGCTACAGCTGCGAATGTCCGCCGGGATAT +ACGGGCACCAGTTGCGAGATCAATATCAACGATTGCGATTCGAATCCCTGCCATCGGGGC +AAGTGCATCGACGATGTGAACAGTTTCAAGTGCCTCTGTGATCCCGGCTACACGGGCTAC +ATATGCCAGAAGCAGATCAACGAATGCGAATCGAATCCGTGCCAGTTCGATGGCCATTGC +CAGGATCGTGTGGGCAGCTACTATTGTCAGTGCCAGGCGGGAACCAGTGGCAAGAATTGC +GAGGTCAATGTGAACGAATGCCACAGTAATCCCTGCAACAATGGTGCCACCTGCATCGAT +GGCATCAATTCGTACAAGTGCCAGTGTGTGCCCGGTTTTACGGGTCAGCACTGCGAGAAG +AACGTCGACGAGTGCATTAGCAGTCCGTGTGCCAACAATGGTGTCTGCATAGATCAGGTG +AATGGCTACAAGTGCGAGTGTCCGCGTGGATTCTACGATGCCCATTGCCTGAGCGATGTG +GACGAGTGCGCATCGAATCCGTGCGTGAACGAGGGACGTTGCGAGGATGGAATCAATGAG +TTCATCTGCCACTGTCCGCCAGGATACACGGGCAAGCGTTGTGAGCTGGACATAGACGAG +TGCAGCAGCAATCCCTGCCAGCATGGTGGCACCTGTTATGATAAGCTCAACGCCTTCAGT +TGCCAATGCATGCCGGGCTATACGGGCCAAAAGTGCGAAACGAATATTGACGACTGTGTG +ACCAATCCGTGCGGAAATGGAGGCACCTGCATTGACAAGGTCAACGGCTACAAGTGCGTC +TGCAAAGTGCCCTTCACGGGCAGGGATTGTGAAAGCAAAATGGATCCCTGCGCCAGCAAT +CGTTGCAAAAACGAGGCGAAATGCACGCCAAGCTCGAATTTCCTGGACTTCTCCTGCACC +TGCAAACTGGGCTATACGGGTCGCTATTGCGATGAGGACATAGACGAATGCTCACTGAGT +TCACCCTGTCGGAATGGAGCTAGCTGTTTGAATGTTCCCGGATCGTATAGGTGCCTCTGC +ACCAAGGGCTATGAGGGCAGAGATTGCGCCATCAATACGGACGATTGTGCCTCGTTTCCG +TGCCAGAACGGTGGAACCTGTCTGGATGGGATCGGTGACTATAGCTGCCTGTGCGTGGAT +GGCTTCGATGGCAAGCACTGCGAGACGGACATCAATGAGTGCTTGAGTCAGCCGTGCCAG +AATGGAGCCACGTGTAGTCAGTATGTGAATAGCTATACCTGCACCTGTCCACTCGGATTC +TCCGGCATCAATTGTCAGACGAACGATGAGGATTGCACGGAGAGCTCGTGCTTAAACGGT +GGCAGTTGCATCGATGGCATCAATGGCTACAACTGTAGCTGTTTGGCTGGATATTCCGGT +GCCAATTGTCAGTACAAGTTGAATAAGTGCGATTCGAATCCCTGTTTGAACGGAGCCACC +TGCCACGAGCAAAACAACGAGTACACATGCCATTGTCCCAGTGGCTTTACGGGCAAACAG +TGTTCCGAATATGTGGATTGGTGTGGTCAATCGCCGTGCGAGAATGGAGCGACCTGTAGC +CAGATGAAGCATCAGTTTAGCTGCAAATGCTCGGCAGGATGGACGGGTAAGCTGTGCGAT +GTTCAGACGATTTCGTGTCAGGATGCAGCAGATCGGAAGGGTCTGAGCCTGCGGCAGTTG +TGCAACAATGGCACGTGCAAGGATTATGGAAATAGTCATGTGTGCTACTGCTCCCAGGGA +TACGCGGGTAGCTATTGCCAAAAGGAGATCGACGAGTGCCAATCGCAACCGTGCCAGAAT +GGTGGAACGTGCCGAGATCTCATCGGAGCCTACGAATGCCAGTGTCGCCAGGGATTTCAG +GGTCAGAATTGCGAACTGAACATCGATGACTGTGCACCGAATCCCTGCCAGAATGGCGGC +ACTTGCCACGACCGTGTGATGAACTTTAGCTGCAGCTGTCCACCGGGAACGATGGGCATC +ATATGCGAGATCAACAAGGATGATTGCAAACCGGGAGCATGCCACAACAACGGCAGCTGC +ATCGATCGCGTTGGAGGCTTCGAGTGCGTCTGTCAGCCCGGTTTTGTGGGTGCCCGCTGC +GAGGGCGACATCAACGAGTGCCTAAGCAATCCCTGTTCGAATGCCGGCACACTGGACTGC +GTCCAGTTGGTCAATAACTATCACTGCAACTGTCGACCTGGACATATGGGTCGGCACTGT +GAGCACAAGGTGGACTTTTGTGCCCAGAGTCCGTGCCAGAATGGTGGCAATTGTAATATC +CGACAGAGTGGCCACCACTGCATCTGTAATAATGGATTCTACGGTAAGAACTGTGAGCTG +AGTGGTCAGGATTGTGATTCGAATCCCTGCCGTGTGGGTAACTGTGTTGTCGCGGACGAG +GGATTCGGCTATAGATGCGAGTGTCCAAGGGGCACACTGGGTGAGCACTGCGAGATTGAT +ACGCTAGACGAATGTTCGCCGAATCCCTGTGCCCAGGGAGCTGCCTGCGAGGATTTGCTC +GGTGATTACGAATGCCTCTGTCCGAGCAAATGGAAAGGCAAACGCTGCGATATCTATGAT +GCCAACTATCCGGGATGGAATGGCGGTTCGGGATCCGGAAATGATCGCTATGCCGCTGAT +TTGGAGCAACAACGTGCCATGTGCGATAAGCGAGGTTGCACCGAAAAGCAGGGCAACGGC +ATTTGCGATTCCGATTGCAATACGTATGCCTGCAACTTTGATGGAAACGATTGCTCACTG +GGCATCAATCCGTGGGCCAATTGCACGGCCAACGAGTGCTGGAACAAATTCAAGAACGGC +AAATGCAACGAGGAGTGCAACAATGCTGCCTGCCACTACGATGGTCACGATTGCGAACGG +AAATTGAAGAGTTGCGATAGCCTTTTCGATGCCTACTGCCAGAAGCATTACGGCGATGGT +TTCTGCGACTATGGATGCAATAATGCGGAATGCAGTTGGGACGGACTCGACTGTGAGAAC +AAGACCCAGTCACCGGTTTTGGCCGAGGGTGCCATGTCCGTGGTGATGCTAATGAATGTG +GAGGCATTCCGCGAGATTCAGGCTCAGTTCCTTAGGAATATGAGCCACATGCTGCGGACA +ACGGTGCGGTTGAAGAAGGACGCTCTTGGTCATGACATCATTATCAATTGGAAGGACAAT +GTGCGCGTGCCGGAGATCGAGGATACGGACTTTGCGAGAAAAAACAAGATTCTGTACACC +CAACAGGTCCATCAGACGGGCATTCAAATCTATTTGGAGATTGACAATCGCAAATGCACC +GAATGCTTTACGCACGCCGTTGAGGCAGCCGAATTTCTGGCTGCCACGGCGGCCAAACAT +CAGCTGCGGAACGATTTTCAGATACACAGTGTTAGGGGCATCAAGAATCCCGGCGATGAG +GACAATGGTGAGCCGCCGGCGAATGTGAAATACGTAATTACTGGCATTATACTGGTCATC +ATTGCATTGGCCTTCTTTGGCATGGTCTTGAGTACGCAAAGAAAGCGGGCACATGGCGTC +ACCTGGTTCCCGGAGGGATTCCGTGCACCGGCGGCGGTTATGTCGCGACGTCGACGTGAT +CCACACGGCCAAGAGATGCGGAATCTCAACAAGCAGGTGGCCATGCAATCGCAGGGTGTT +GGCCAGCCAGGTGCTCATTGGTCCGACGATGAATCGGATATGCCGTTGCCCAAGCGACAG +CGCAGTGATCCCGTTTCCGGTGTCGGTCTGGGCAACAATGGTGGCTATGCCAGTGATCAT +ACAATGGTCAGCGAATATGAGGAGGCTGATCAGCGGGTCTGGTCGCAGGCCCATCTGGAT +GTGGTCGATGTGCGGGCCATAATGACACCGCCGGCGCATCAGGATGGTGGCAAGCACGAT +GTGGATGCACGTGGACCTTGCGGACTGACTCCGCTAATGATTGCTGCCGTACGAGGCGGT +GGCCTGGACACTGGCGAGGATATCGAGAACAATGAGGACAGTACTGCCCAGGTGATATCG +GATCTATTGGCACAGGGAGCCGAGTTAAATGCCACAATGGATAAAACGGGCGAGACATCA +TTGCATTTGGCAGCACGTTTTGCTCGAGCAGATGCCGCCAAGAGATTATTGGACGCCGGT +GCGGATGCCAATTGTCAGGATAACACGGGCAGGACACCGCTGCACGCTGCTGTGGCCGCC +GATGCCATGGGTGTATTTCAGATATTGCTTAGGAATAGGGCTACCAATCTAAATGCTAGA +ATGCATGACGGCACAACGCCACTAATACTGGCCGCTCGTCTAGCCATCGAGGGCATGGTG +GAGGATTTGATCACAGCCGATGCGGATATTAATGCAGCGGATAATTCCGGCAAGACCGCT +CTCCATTGGGCAGCGGCGGTTAACAATACCGAAGCGGTGAACATTCTGCTAATGCATCAT +GCCAATCGCGATGCCCAGGACGATAAGGATGAGACACCGCTGTTTCTGGCCGCACGCGAG +GGTAGCTATGAGGCGTGCAAGGCGCTGTTGGATAATTTTGCCAATCGCGAGATTACCGAT +CACATGGATCGATTGCCACGCGATGTGGCCAGCGAGCGACTGCATCACGATATTGTCAGG +CTGCTGGACGAGCATGTGCCGCGATCTCCGCAAATGCTGAGCATGACACCGCAGGCAATG +ATCGGATCCCCGCCGCCGGGTCAACAGCAGCCGCAGTTGATCACACAGCCGACGGTCATT +TCCGCCGGAAATGGTGGCAACAATGGCAATGGCAATGCCAGCGGAAAGCAGAGCAACCAA +ACGGCCAAACAGAAGGCGGCCAAGAAGGCCAAACTAATCGAGGGCAGTCCGGACAATGGG +CTGGATGCCACGGGCAGTTTGCGACGCAAGGCAAGTTCAAAAAAGACAAGTGCGGCCTCA +AAAAAGGCCGCCAACTTGAATGGCCTAAATCCGGGCCAGCTGACGGGGGGAGTGTCGGGC +GTGCCGGGTGTACCGCCAACGAATTCAGCGGCACAAGCAGCTGCAGCGGCAGCGGCAGCC +GTGGCGGCCATGTCCCACGAACTGGAGGGTTCGCCGGTGGGCGTGGGCATGGGCGGGAAT +CTGCCCAGTCCGTACGACACCAGTTCGATGTACTCGAATGCGATGGCCGCACCGCTGGCG +AACGGCAATCCGAACACGGGCGCCAAGCAGCCGCCGAGCTATGAGGATTGCATCAAGAAT +GCGCAATCGATGCAATCGCTGCAGGGCAATGGTCTGGATATGATCAAGCTGGATAATTAT +GCCTACTCGATGGGCTCGCCATTTCAGCAGGAGCTGCTCAATGGCCAAGGACTCGGGATG +AATGGTAATGGCCAGCGAAATGGAGTCGGTCCCGGCGTTCTGCCCGGCGGCCTATGCGGA +ATGGGCGGTCTGAGTGGAGCCGGCAATGGAAATAGCCACGAACAGGGACTGAGTCCGCCG +TACTCGAATCAGTCGCCGCCGCATTCGGTGCAGAGCAGCCTGGCGCTATCGCCCCACGCC +TACTTGGGCTCGCCATCGCCGGCCAAGTCGCGACCCAGTCTACCCACCTCGCCCACCCAC +ATTCAGGCGATGAGGCATGCCACACAGCAGAAGCAATTCGGTGGCAGTAATCTGAATAGT +TTGCTGGGCGGTGCCAATGGCGGGGGCGTGGTCGGCGGAGGAGGAGGTGGTGGTGGTGGT +GTTGGCCAGGGGCCACAAAACTCACCCGTAAGCTTGGGAATCATCTCGCCGACGGGCAGC +GATATGGGCATCATGCTCGCCCCGCCCCAATCCTCGAAGAATAGTGCAATAATGCAAACG +ATATCACCCCAGCAACAGCAGCAGCAGCAGCAACAGCAACAGCAGCAACATCAGCAGCAG +CAACAGCAGCAGCAACAGCAGCAGCAGCAACAGCAGCAGCAACTCGGAGGCCTGGAGTTC +GGTTCAGCGGGCTTGGACCTGAATGGATTTTGTGGATCTCCGGACTCATTTCACTCGGGT +CAAATGAATCCGCCCTCGATACAAAGTTCAATGTCCGGCTCATCGCCGTCGACCAATATG +TTGTCGCCGTCGTCGCAGCACAATCAGCAGGCCTTCTACCAGTACCTAACGCCTTCGAGC +CAACATTCCGGCGGCCATACGCCGCAGCATTTGGTCCAAACGTTGGATAGCTATCCGACG +CCCTCGCCGGAGTCACCTGGCCACTGGTCCTCCTCGTCGCCGCGATCGAATTCCGATTGG +AGCGAGGGCGTTCAGTCGCCGGCGGCAAATAATCTTTACATATCCGGTGGCCATCAGGCT +AACAAGGGTTCCGAGGCCATCTACATTTGACCGTGATCTGGATGATTTGGTATGTTGAAT +ATCGCGCAAGGATAATTGGATGGCTTGCCAAGGATAACAGCAAAAGGGGACTCTTAAAGA +GTCCGCGGCTAAGTTCGATCTAAAATATGCTATATATTGTAATTCTCTAATCTCTACGGA +TTATGTATTTTTTCCAACGACTCTTTTTTTTTTTTTTTGAGTGTGTCTATAAGTTTTAGA +TGTTAGTGCGAAATACGCAATTCTAGCGGAAGAAGCGTCATAAATTGTAGTAACTTAAAT +TTTATTTAATGCTTGTTGTACAGTGCGCGCCGATCGCATATATATATATATATATAGATA +TTGATATAGATATATGTATATCTACTCATAGATATACCAGTTAAGAGCTCGTCTTGTGGC +GCATTTTATTTTATTGTCGTAACGTCATCGTAGCTTCCAGTAAAAAAAAAAACAAAAAAC +AAAAGAATCAACATAAAAAGATGAGCAACTAAACTTAGCGCAAATTCTGATTGGATCGAT +TAAACGTTTGTGGGACAATTTGTAGTTGTAAGTGACCATTATATCACTAGGCCATAAGAC +TACGCTAAGTTTTTTTTATCGAATTCTACATTATTAATTATGGCCCCACATGCTATATAT +AAATACATAAATTTCGTAGTTCTGTAGGTTTTGAAACTGAAGTGCTTATATAAACAAAAT +TGTTAGACCGTACACCGCATCGAACCTTTTTTTTTACACCAAGTTTTTTTTTTTTATTAT +TATTATTATTATTATGACTATTATGTATTATTACATATTGACTAAGCTAAACTGTAAAAA +GATTCTCATAACTGTTTGATTTAAGTTGACACATACAAAATACAAAAGGATTTGCAAAAA +AAATTACACAAAAATCACCAATGGAAACGTATAAGTCTAATTGAAAGAAAGAAAGAAAAG +AAGAAAAAAAAGGAAAAAACCACGAAAAAGACAGTAACCAGCCAAGTTTACTATATATAT +AAGGACACTCGACGATCTACGATCTAAGTTGATGAAAAGATATTTTCAAAAAATATGCAA +CAGAAAGAAAAAAAACGCACACACTTAGAGAAAACTGCAAAATACGAATTTTCTCGTTCG +ATTTGCATTAATTTTAATTTGAACGGTATTGCTTTCTTTCGTCTATATTTTGTGTGTAAG +TTTTTAATGTGTAAATTGTAAAATTATTTGTTTTCCTAACTTAAACAGCCAGACAGCAAA +ACGAAACTGC +>fbdb.dmel.transcript.1 fb_original=646d656c3a4642747230303730323932; type=snoRNA; loc=X:complement(1482492..1482590); ID=FBtr0070292; name=snoRNA:M-RA; dbxref=FlyBase:FBtr0070292,FlyBase_Annotation_IDs:CR32807-RA,REFSEQ:NR_002093,RNAcentral:URS00002398EB_7227; MD5=81314d41b5db15222ab4c0daca9e3a65; length=99; parent=FBgn0044508; release=r6.53; species=Dmel; +AATTCAATGATTTCAACTTCACTGCTGACCAGCCACGCCTCAACGCCTCGGAGTGTTTGC +CCCCAGTCTCTGATAGATACACCATATTCTAATACACAC diff --git a/blast-jobs/tests/image-smoke.sh b/blast-jobs/tests/image-smoke.sh new file mode 100755 index 0000000..29cfaff --- /dev/null +++ b/blast-jobs/tests/image-smoke.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# Smoke test for the built image (CI job blast-jobs-image; also run by hand). Builds the fixture +# databases with the image's own makeblastdb, runs `blast-jobs check`, then serves and runs one real +# blastn search end to end inside the container, and waits for the HEALTHCHECK to report healthy. +# usage: tests/image-smoke.sh (run from blast-jobs/) +set -eu +image=$1 +here=$(cd "$(dirname "$0")/.." && pwd) +work=$(mktemp -d) +name=blast-jobs-smoke-$$ +cleanup() { + docker rm -f "$name" >/dev/null 2>&1 || true + # Job files belong to the image's uid 10002, so remove them as that user. + docker run --rm -v "$work/state:/state" --entrypoint rm "$image" -rf /state/jobs >/dev/null 2>&1 || true + rm -rf "$work" +} +trap cleanup EXIT +mkdir -p "$work/db" "$work/state" +chmod 0777 "$work/state" # the image runs as uid 10002 + +docker run --rm --user "$(id -u):$(id -g)" --entrypoint sh \ + -v "$here/fixture:/fixture:ro" -v "$work/db:/db" \ + "$image" /fixture/build-db.sh /opt/ncbi-blast/bin /fixture /db + +cat > "$work/config.json" </dev/null +api() { docker exec "$name" curl -fsS -H 'authorization: Bearer smoke-token' "$@"; } +for _ in $(seq 1 50); do + docker exec "$name" curl -fsS http://127.0.0.1:8800/internal/health >/dev/null 2>&1 && break + sleep 0.2 +done +docker exec "$name" curl -fsS http://127.0.0.1:8800/internal/health; echo + +query=$(awk '/^>/{n++; next} n==1' "$here/fixture/notch.fa" | tr -d '\n' | cut -c2001-2400) +submit=$(api -H 'content-type: application/json' \ + -d "{\"program\":\"blastn\",\"database\":\"transcript\",\"species\":[\"dmel\"],\"sequence\":\">query\\n$query\\n\",\"client\":\"smoke\"}" \ + http://127.0.0.1:8800/internal/jobs) +echo "submit: $submit" +id=$(echo "$submit" | sed -n 's/.*"job_id":"\([A-Za-z0-9_-]*\)".*/\1/p') +[ -n "$id" ] || { echo "no job_id" >&2; exit 1; } +state= +for _ in $(seq 1 100); do + state=$(api "http://127.0.0.1:8800/internal/jobs/$id" | sed -n 's/.*"state":"\([a-z]*\)".*/\1/p') + [ "$state" = complete ] || [ "$state" = failed ] && break + sleep 0.2 +done +[ "$state" = complete ] || { echo "job ended $state" >&2; api "http://127.0.0.1:8800/internal/jobs/$id" >&2; exit 1; } +api "http://127.0.0.1:8800/internal/jobs/$id/results.tsv" > "$work/results.tsv" +head -c 200 "$work/results.tsv"; echo +head -n 1 "$work/results.tsv" | grep -q '^qseqid sseqid pident' || { echo "bad header" >&2; exit 1; } +sed -n 2p "$work/results.tsv" | grep -q '^query fbdb\.dmel\.transcript\.5460 100\.000 400 .*ID=FBtr0070507;' \ + || { echo "expected N-RA as the top hit" >&2; exit 1; } +echo "search: ok" + +for _ in $(seq 1 60); do + health=$(docker inspect -f '{{.State.Health.Status}}' "$name") + [ "$health" = healthy ] && break + [ "$health" = unhealthy ] && { echo "HEALTHCHECK unhealthy" >&2; exit 1; } + sleep 1 +done +[ "$health" = healthy ] || { echo "HEALTHCHECK still $health" >&2; exit 1; } +echo "healthcheck: $health" diff --git a/blast-jobs/tests/real_blast.rs b/blast-jobs/tests/real_blast.rs new file mode 100644 index 0000000..a93c5a4 --- /dev/null +++ b/blast-jobs/tests/real_blast.rs @@ -0,0 +1,368 @@ +//! End-to-end tests against real NCBI BLAST+ (design 3.12, Task 5). Skipped unless +//! `BLAST_JOBS_REAL_BLAST` is set. Its value is the directory holding the BLAST+ binaries (`blastn` ... +//! `makeblastdb`); any other non-empty value means "take them from `PATH`". +//! +//! Each test builds the fixture databases (`fixture/build-db.sh`, the archive's `makeblastdb` flags) in a +//! temp directory, starts the API over them on an ephemeral port, and drives submit, poll and results. +//! Queries are slices of the fixture's own sequences, so every program has a known hit: N-RA +//! (`FBtr0070507`, `fbdb.dmel.transcript.5460`) and N-PA (`FBpp0070483`, `fbdb.dmel.translation.529`). +//! +//! BLAST_JOBS_REAL_BLAST=/path/to/ncbi-blast-2.17.0+/bin cargo test --test real_blast +use blast_jobs::auth::Auth; +use blast_jobs::catalogue::Catalogue; +use blast_jobs::config::Config; +use blast_jobs::http::{self, AppState}; +use blast_jobs::runner::{self, HEADER}; +use blast_jobs::store::Store; +use reqwest::StatusCode; +use serde_json::{Value, json}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::task::JoinHandle; + +const ENV: &str = "BLAST_JOBS_REAL_BLAST"; +const TOKEN: &str = "real-blast-test-token"; +const NOTCH_TRANSCRIPT: &str = "fbdb.dmel.transcript.5460"; +const NOTCH_PROTEIN: &str = "fbdb.dmel.translation.529"; + +fn manifest() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +/// The BLAST+ bin directory, or `None` (with a printed reason) when the test should skip. +fn blast_bin() -> Option { + let v = match std::env::var(ENV) { + Ok(v) if !v.trim().is_empty() => v, + _ => { + println!("skipped: {ENV} is unset (set it to a BLAST+ bin directory to run)"); + return None; + } + }; + let dir = PathBuf::from(v.trim()); + if dir.join("makeblastdb").is_file() { + return Some(dir); + } + let path = std::env::var_os("PATH").unwrap_or_default(); + let found = std::env::split_paths(&path).find(|d| d.join("makeblastdb").is_file()); + // Set but unusable is a configuration error, not a skip. + Some(found.unwrap_or_else(|| panic!("{ENV}={v}: no makeblastdb there or on PATH"))) +} + +/// Reads one FASTA record's residues by its id from a fixture file. +fn fixture_seq(file: &str, id: &str) -> String { + let text = std::fs::read_to_string(manifest().join("fixture").join(file)).unwrap(); + let mut out = String::new(); + let mut inside = false; + for line in text.lines() { + if let Some(h) = line.strip_prefix('>') { + inside = h.split_whitespace().next() == Some(id); + } else if inside { + out.push_str(line.trim()); + } + } + assert!(!out.is_empty(), "{id} not in fixture/{file}"); + out +} + +fn notch_rna(range: std::ops::Range) -> String { + fixture_seq("notch.fa", NOTCH_TRANSCRIPT)[range].to_string() +} + +fn notch_protein(range: std::ops::Range) -> String { + fixture_seq("notch-protein.fa", NOTCH_PROTEIN)[range].to_string() +} + +struct Server { + _tmp: tempfile::TempDir, + base: String, + http: reqwest::Client, + tasks: Vec>, +} + +impl Drop for Server { + fn drop(&mut self) { + for t in &self.tasks { + t.abort(); + } + } +} + +/// One parsed `results.tsv` row. +#[derive(Debug)] +struct Row { + sseqid: String, + pident: f64, + length: u64, + bitscore: f64, + stitle: String, +} + +impl Server { + async fn start(bin: &Path) -> Server { + let tmp = tempfile::tempdir().unwrap(); + let db = tmp.path().join("db"); + let state = tmp.path().join("state"); + std::fs::create_dir_all(&state).unwrap(); + let fixture = manifest().join("fixture"); + let out = Command::new("sh") + .arg(fixture.join("build-db.sh")) + .args([bin, &fixture, &db]) + .output() + .unwrap(); + assert!( + out.status.success(), + "build-db.sh failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + let cfg = json!({ + "bind": "127.0.0.1:0", + "blast_bin_dir": bin, + "database_root": db, + "catalogue": db.join("catalogue.json"), + "database_release": "FB2023_04", + "state_root": state, + "concurrency": 2, + "threads_per_job": 1, + "wall_seconds": 60, + "translated_wall_seconds": 60, + }); + let path = tmp.path().join("config.json"); + std::fs::write(&path, cfg.to_string()).unwrap(); + let cfg = Arc::new(Config::load(&path).unwrap()); + let cat = Arc::new(Catalogue::load(&cfg.catalogue, &cfg.database_root).unwrap()); + http::probe_binaries(&cfg.blast_bin_dir).unwrap(); + let store = Store::open(&cfg.state_root, cfg.max_queued).unwrap(); + let workers = runner::spawn_workers(cfg.clone(), store.clone()); + let app = AppState::new(cfg, cat, store, Auth::Token(TOKEN.into()), workers); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + axum::serve(listener, http::router(app)).await.unwrap(); + }); + Server { + _tmp: tmp, + base: format!("http://{addr}"), + http: reqwest::Client::new(), + tasks: vec![task], + } + } + + /// Submits, polls to completion, and returns the parsed rows. Asserts the `202`, the final + /// `complete` state, the TSV content type and the header line. + async fn search(&self, req: Value) -> Vec { + let r = self + .http + .post(format!("{}/internal/jobs", self.base)) + .bearer_auth(TOKEN) + .json(&req) + .send() + .await + .unwrap(); + assert_eq!(r.status(), StatusCode::ACCEPTED, "{:?}", r.text().await); + let a: Value = r.json().await.unwrap(); + assert_eq!(a["database_release"], "FB2023_04"); + let id = a["job_id"].as_str().unwrap().to_string(); + + let end = Instant::now() + Duration::from_secs(120); + let status = loop { + let s: Value = self + .http + .get(format!("{}/internal/jobs/{id}", self.base)) + .bearer_auth(TOKEN) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + if matches!(s["state"].as_str(), Some("complete" | "failed")) { + break s; + } + assert!(Instant::now() < end, "job {id} still {s}"); + tokio::time::sleep(Duration::from_millis(50)).await; + }; + assert_eq!(status["state"], "complete", "{req} -> {status}"); + + let r = self + .http + .get(format!("{}/internal/jobs/{id}/results.tsv", self.base)) + .bearer_auth(TOKEN) + .send() + .await + .unwrap(); + assert_eq!(r.status(), StatusCode::OK); + let ct = r.headers()["content-type"].to_str().unwrap().to_string(); + assert!(ct.starts_with("text/tab-separated-values"), "{ct}"); + assert!(r.headers().get("x-blast-jobs-truncated").is_none()); + let body = r.text().await.unwrap(); + let (header, rows) = body.split_at(HEADER.len()); + assert_eq!( + header, HEADER, + "results.tsv must start with the header line" + ); + assert_eq!( + HEADER, + "qseqid\tsseqid\tpident\tlength\tmismatch\tgapopen\tqstart\tqend\tsstart\tsend\tevalue\tbitscore\tstitle\n" + ); + rows.lines() + .map(|l| { + let f: Vec<&str> = l.split('\t').collect(); + assert_eq!(f.len(), 13, "row {l:?}"); + assert_eq!(f[0], "query"); + Row { + sseqid: f[1].to_string(), + pident: f[2].parse().unwrap(), + length: f[3].parse().unwrap(), + bitscore: f[11].parse().unwrap(), + stitle: f[12].to_string(), + } + }) + .collect() + } +} + +fn req(program: &str, database: &str, residues: &str) -> Value { + json!({ + "program": program, + "database": database, + "species": ["dmel"], + "sequence": format!(">query\n{residues}\n"), + "client": "flybase-mcp", + }) +} + +/// Asserts the best row is the expected subject, carrying the FlyBase id in its title. +fn assert_top_hit(rows: &[Row], sseqid: &str, flybase_id: &str) { + let top = rows + .first() + .unwrap_or_else(|| panic!("no hits; expected {sseqid}")); + assert_eq!(top.sseqid, sseqid, "{rows:?}"); + assert!( + top.stitle.contains(&format!("ID={flybase_id};")), + "stitle {:?} lacks {flybase_id}", + top.stitle + ); +} + +macro_rules! real { + () => { + match blast_bin() { + Some(b) => Server::start(&b).await, + None => return, + } + }; +} + +#[tokio::test] +async fn blastn_finds_notch_transcript() { + let s = real!(); + let rows = s + .search(req("blastn", "transcript", ¬ch_rna(2000..2400))) + .await; + assert_top_hit(&rows, NOTCH_TRANSCRIPT, "FBtr0070507"); + assert_eq!(rows[0].length, 400); + assert_eq!(rows[0].pident, 100.0); +} + +#[tokio::test] +async fn blastn_short_task_via_advanced() { + let s = real!(); + let q = notch_rna(3000..3022); + let mut short = req("blastn", "transcript", &q); + short["advanced"] = json!("-task blastn-short"); + let short = s.search(short).await; + let plain = s.search(req("blastn", "transcript", &q)).await; + assert_top_hit(&short, NOTCH_TRANSCRIPT, "FBtr0070507"); + assert_top_hit(&plain, NOTCH_TRANSCRIPT, "FBtr0070507"); + assert_eq!(short[0].length, 22); + // blastn-short scores a match +1/-3 with word size 7; blastn +2/-3 with word size 11. The same + // 22-residue exact match therefore scores differently, which shows the task reached BLAST. + assert!( + short[0].bitscore > plain[0].bitscore, + "blastn-short {} vs blastn {}", + short[0].bitscore, + plain[0].bitscore + ); + assert!( + short.len() > plain.len(), + "word size 7 finds more short HSPs" + ); +} + +#[tokio::test] +async fn blastp_finds_notch_protein() { + let s = real!(); + let rows = s + .search(req("blastp", "translation", ¬ch_protein(100..300))) + .await; + assert_top_hit(&rows, NOTCH_PROTEIN, "FBpp0070483"); + assert_eq!(rows[0].length, 200); + assert_eq!(rows[0].pident, 100.0); +} + +#[tokio::test] +async fn tblastn_finds_notch_transcript() { + let s = real!(); + let rows = s + .search(req("tblastn", "transcript", ¬ch_protein(100..300))) + .await; + assert_top_hit(&rows, NOTCH_TRANSCRIPT, "FBtr0070507"); + assert_eq!(rows[0].pident, 100.0); +} + +#[tokio::test] +async fn blastx_finds_notch_protein() { + let s = real!(); + // Inside the N-RA coding sequence (the CDS starts at transcript position 799). + let rows = s + .search(req("blastx", "translation", ¬ch_rna(1500..2100))) + .await; + assert_top_hit(&rows, NOTCH_PROTEIN, "FBpp0070483"); + assert_eq!(rows[0].length, 200); + assert_eq!(rows[0].pident, 100.0); +} + +#[tokio::test] +async fn tblastx_finds_notch_transcript() { + let s = real!(); + let rows = s + .search(req("tblastx", "transcript", ¬ch_rna(4000..4300))) + .await; + assert_top_hit(&rows, NOTCH_TRANSCRIPT, "FBtr0070507"); +} + +/// The Notch opa repeat (CAG/CAA) is low complexity: the default `-dust yes` masks it, so nothing +/// is found, and `-dust no` finds it in full. +#[tokio::test] +async fn blastn_masks_low_complexity_by_default() { + let s = real!(); + let q = notch_rna(8400..8510); + let masked = s.search(req("blastn", "transcript", &q)).await; + assert!( + masked.is_empty(), + "dust should mask the opa repeat: {masked:?}" + ); + let mut open = req("blastn", "transcript", &q); + open["advanced"] = json!("-dust no"); + let open = s.search(open).await; + assert_top_hit(&open, NOTCH_TRANSCRIPT, "FBtr0070507"); + assert_eq!(open[0].length, 110); +} + +/// The N-PA polyglutamine stretch: the default `-seg yes` leaves only the unmasked tail to align, +/// and `-seg no` aligns the whole 40-residue query. +#[tokio::test] +async fn blastp_masks_low_complexity_by_default() { + let s = real!(); + let q = notch_protein(2540..2580); + let masked = s.search(req("blastp", "translation", &q)).await; + let longest = masked.iter().map(|r| r.length).max().unwrap_or(0); + assert!(longest < 40, "seg should mask the polyQ run: {masked:?}"); + let mut open = req("blastp", "translation", &q); + open["advanced"] = json!("-seg no"); + let open = s.search(open).await; + assert_top_hit(&open, NOTCH_PROTEIN, "FBpp0070483"); + assert_eq!(open[0].length, 40); +} diff --git a/docs/blast-jobs.md b/docs/blast-jobs.md new file mode 100644 index 0000000..4f46c90 --- /dev/null +++ b/docs/blast-jobs.md @@ -0,0 +1,202 @@ +# blast-jobs: the BLAST service behind the MCP + +`blast-jobs/` is a small Rust service that implements the MCP's internal BLAST job API +([handoff §7](handoff-private-side.md#7-blast-internal-job-api-ticket-for-the-blast-service)). It runs +NCBI BLAST+ 2.17.0 directly against a directory of BLAST databases described by a catalogue file. It keeps +jobs in memory with one directory per job on disk, and it has no database. It serves no public result +pages. The MCP's three BLAST tools (`blast_submit`, `blast_status`, `blast_results`) are its only intended +client. Nothing FlyBase-specific is compiled in: paths, the release label, limits and the species set +all come from the config file and the catalogue. + +## API + +| Route | Answers | +|---|---| +| `POST /internal/jobs` | `202 {job_id, state: "queued", queue_position, poll_after_seconds, database_release}`; `400 {error, message}` where `error` is `invalid_program`, `invalid_database`, `invalid_species`, `invalid_sequence`, `invalid_option`, or `invalid_request` for a malformed body; `413` over 1 MiB; `429 {error: "queue_full", retry_after_seconds: 30}` with `Retry-After`; `503` when the state directory cannot be written | +| `GET /internal/jobs/{id}` | `200 {job_id, state, queue_position?, error?}`, where `state` is `queued`, `running`, `complete` or `failed`; `404 {error: "unknown_job"}` | +| `GET /internal/jobs/{id}/results.tsv` | `200 text/tab-separated-values`, a header line and then BLAST's `-outfmt "6 std stitle"` rows; `X-Blast-Jobs-Truncated: true` when the 16 MiB cap cut the rows (at a row boundary); `409 {error: "not_complete", state}`; `404` | +| `GET /internal/health` | `200 {status: "ok", queued, running, database_release}` or `503 {status: "degraded", reason}`; `?liveness=1` skips the queue-full check | + +Every route except health needs `Authorization: Bearer `; the token is compared in +constant time. `result_url` and `downloads` are never sent, and `cancelled` is never produced. Job ids +are 22 random URL-safe characters. Jobs and results are kept for `retention_seconds` after submission. + +`sseqid` is the BLAST database's own sequence id. The FB2023_04 archive was built with `-parse_seqids`, so +`sseqid` is a local id such as `fbdb.dmel.transcript.5460`. The FlyBase id is in `stitle` +(`... ID=FBtr0070507; name=N-RA; ...`), and the MCP takes `flybase_id` from there. + +## Configuration + +One JSON file, `/etc/blast-jobs/config.json` in the image. A sample is in +[`blast-jobs/deploy/config.sample.json`](../blast-jobs/deploy/config.sample.json). Unknown keys are refused. + +| Key | Default | Meaning | +|---|---|---| +| `bind` | required | listen address; `0.0.0.0:8800` in the container (the image's health check probes port 8800) | +| `blast_bin_dir` | required | directory holding `blastn blastp blastx tblastn tblastx`; `/opt/ncbi-blast/bin` in the image | +| `database_root` | required | root that catalogue prefixes are relative to | +| `catalogue` | required | the catalogue file (below) | +| `database_release` | required | returned in every `202` and shown to users; `FB2023_04` for FlyBase | +| `state_root` | required | writable job state (`jobs//`) | +| `concurrency` | 2 | searches run at once | +| `threads_per_job` | 2 | `-num_threads` per search | +| `max_queued` | 20 | queued jobs before `429 queue_full` | +| `wall_seconds` | 600 | wall-clock limit for blastn and blastp | +| `translated_wall_seconds` | 300 | wall-clock limit for blastx, tblastn and tblastx | +| `retention_seconds` | 86400 | job and result lifetime (the MCP's token lifetime) | +| `max_results_bytes` | 16777216 | cap on the served `results.tsv`; at most 16 MiB, the MCP's limit | +| `default_evalue` | 10 | used when a request has no `evalue` | +| `default_max_hits` | 100 | `-max_target_seqs` when a request has no `max_hits` | +| `max_hsps_per_subject` | 50 | `-max_hsps`, and the ceiling for an `advanced` override | +| `max_residues` | see below | query residue cap per program; if given, all five programs must be listed | + +The token comes from the environment, never the config: `BLAST_JOBS_TOKEN`. The service refuses to start +without it, unless `BLAST_JOBS_ALLOW_NO_AUTH=1` is set (for local runs only). + +`blast-jobs check ` loads the config and the catalogue, checks every database volume and the +BLAST binaries, and exits 0 or 1 with the reason. `blast-jobs serve ` runs the service; it is +the image's default command. + +## Catalogue + +Any JSON file with a `databases` array. Each entry carries `species`, `category`, `protein` and `prefix`; +other fields are ignored, so the archive's own `catalogue.json` works as is. + +```json +{ "databases": [ + { "species": "dmel", "category": "transcript", "protein": false, "prefix": "dmel/dmel-transcript" } +] } +``` + +A request's `database` is a `category` and its `species` are `species` codes. The service accepts every +species in the catalogue; the MCP's species list is a client-side restriction. `protein` must match the +program: blastp and blastx search protein databases, and the others search nucleotide ones. Every +requested species must have the category, otherwise the request is `invalid_species` naming the missing +ones. `prefix` is relative to `database_root` and names a BLAST volume (`.nsq` or `.psq`). +At startup every prefix must exist and contain no whitespace or quotes, and no `(species, category)` pair +may repeat. Otherwise the service refuses to start. + +**Multi-species E-values.** A search over several species passes all their prefixes as one `-db` list. Its +E-values are computed against the combined database length, so they are not comparable with a +one-species search. + +## Searches + +Each job runs `/` with argv only (no shell), an empty environment plus +`NCBI_DONT_USE_NCBIRC=1 NCBI_DONT_USE_LOCAL_CONFIG=1 BLAST_USAGE_REPORT=false HOME=/tmp`, and these flags: +`-db`, `-query`, `-out`, `-outfmt "6 std stitle"`, `-evalue`, `-max_target_seqs`, `-max_hsps`, +`-num_threads`, the `-task` default, the masking default, `-matrix`, and the genetic code for translated +programs. `advanced` options then override these defaults; they are never appended twice. + +- **Task.** blastn always gets an explicit `-task`: `megablast` when `megablast: true`, otherwise `blastn`. + BLAST+'s own default for `blastn` is megablast. The other programs use BLAST+'s default task unless + `advanced` sets one. +- **Masking defaults.** `-dust yes` for blastn and `-seg yes` for blastp, blastx, tblastn and tblastx. This + matches the FlyBase site's "low-complexity filter on" default (BLAST+ itself defaults blastp to + `-seg no`). `advanced` can turn either off (`-dust no`, `-seg no`). +- **Residue caps and wall limits.** + + | Program | Query residues | Wall limit | + |---|---|---| + | blastn | 100,000 | `wall_seconds` (600 s) | + | blastp | 100,000 | `wall_seconds` (600 s) | + | blastx | 30,000 | `translated_wall_seconds` (300 s) | + | tblastn | 30,000 | `translated_wall_seconds` (300 s) | + | tblastx | 10,000 | `translated_wall_seconds` (300 s) | + + A search that runs past its limit is killed and fails with "search exceeded N s; try a shorter query or + a smaller database". A search killed by the container's memory cap fails with "search ran out of + memory; try a shorter query or fewer species". Failure messages carry no paths. +- **Request fields.** `evalue` is in (0, 1000]. `max_hits` is 1 to 1000. `matrix` is one of `BLOSUM62 + BLOSUM45 BLOSUM80 PAM30 PAM70` and is refused for blastn. `megablast` is blastn only. `genetic_code` is in + the NCBI set 1-6, 9-16, 21-31 and 33, and is used by blastx, tblastn and tblastx. The sequence is one + FASTA record in the program's alphabet. + +### `advanced` + +At most 200 characters of `[A-Za-z0-9 _.=-]`, written as `-flag value` or `-flag=value` pairs, with at most +8 pairs. Each flag must be in this table for the program: + +| Flag | Value | Programs | +|---|---|---| +| `task` | blastn: `blastn blastn-short megablast dc-megablast`; blastp: `blastp blastp-short blastp-fast`; blastx: `blastx blastx-fast`; tblastn: `tblastn tblastn-fast` | blastn blastp blastx tblastn | +| `word_size` | integer >= 4 (blastn), >= 2 (others) | all | +| `gapopen`, `gapextend` | integer >= 0 | all but tblastx | +| `penalty` | integer <= 0 | blastn | +| `reward` | integer >= 0 | blastn | +| `dust` | `yes` / `no` | blastn | +| `seg` | `yes` / `no` | blastp blastx tblastn tblastx | +| `soft_masking` | `true` / `false` | all | +| `lcase_masking` | switch, no value | all | +| `ungapped` | switch, no value | all but tblastx | +| `strand` | `both plus minus` | blastn blastx tblastx | +| `perc_identity` | number 0 to 100 | blastn | +| `qcov_hsp_perc` | number 0 to 100 | all | +| `culling_limit` | integer >= 0 | all | +| `max_hsps` | integer >= 1, clamped to `max_hsps_per_subject` | all | +| `threshold` | number >= 0 | blastp blastx tblastn | +| `comp_based_stats` | `0 1 2 3 D F T t f` | blastp blastx tblastn | +| `window_size` | integer >= 0 | all | + +Anything else is `invalid_option`. That covers a flag outside the table or not valid for the program, a +repeated flag, a `--` token, a value starting with `-` (except a negative number for a numeric flag, as in +`-penalty -3`), and a value for a switch. The flags the service owns are refused too (`evalue +max_target_seqs matrix num_threads query_gencode db_gencode outfmt db query out`), as is anything that +reaches files or the network (`-remote`, `-subject`, `-import_search_strategy`, `-seqidlist`, ...). With +`megablast: true`, a `-task` other than `megablast` is refused. + +## Image + +`blast-jobs/Dockerfile` builds the binary with `--locked` on `rust:1.90-bookworm`. It downloads +`ncbi-blast-2.17.0+-x64-linux.tar.gz` from NCBI's versioned HTTPS path and checks it against a pinned +SHA-256. The runtime stage is `debian:bookworm-slim` with `ca-certificates curl libgomp1` and the seven +binaries `blastn blastp blastx tblastn tblastx blastdbcmd makeblastdb` in `/opt/ncbi-blast/bin`. The build +runs each binary's `-version`, so a missing library fails the build. The image runs as uid 10002, is +about 330 MB, and has a `HEALTHCHECK` on `/internal/health?liveness=1`, so a busy service is not +restarted. + + docker build -t blast-jobs: blast-jobs/ + blast-jobs/tests/image-smoke.sh blast-jobs: # check + one real search inside the image + +## Running it next to the MCP + +[`blast-jobs/deploy/run.sample.sh`](../blast-jobs/deploy/run.sample.sh) has the full command. Its shape: + +``` +docker run -d --name blast-jobs --network --init \ + --cpus 4 --cpu-shares 512 --memory 4g --pids-limit 256 --read-only --tmpfs /tmp:size=256m \ + --restart unless-stopped --log-opt max-size=20m --log-opt max-file=3 \ + --mount type=bind,src=,dst=/db,readonly \ + --mount type=bind,src=,dst=/state \ + --mount type=bind,src=,dst=/etc/blast-jobs/config.json,readonly \ + --env-file \ + blast-jobs: +``` + +- `--init` reaps a BLAST process the service killed on timeout. `--cpus` should be at least + `concurrency x threads_per_job`. `--cpu-shares 512` lets other services on the host win when it is busy. +- No port is published. The MCP reaches the service by container name on the shared network. For + host-side checks, add `-p 127.0.0.1::8800`. +- **Token pairing.** The MCP's `BLAST_SERVICE_TOKEN` must equal the service's `BLAST_JOBS_TOKEN`. Set + `BLAST_BASE_URL=http://blast-jobs:8800` and `BLAST_SERVICE_TOKEN=` in the MCP's + environment and recreate the MCP container. Its `/health` then shows `blast: ok`. +- **`unreachable` versus a full queue.** The MCP's `/health` probes `/internal/health` without + `?liveness=1`, so when the queue holds `max_queued` jobs it reports `blast: unreachable` even though the + service is up. Submits get `429 queue_full` at that point, and the MCP tells callers to retry later. +- **Rollback.** Remove `BLAST_BASE_URL` from the MCP's environment and recreate the MCP container. Its + `/health` then shows `blast: unconfigured`, and the BLAST tools answer `backend_unavailable`. + +## Tests + +From `blast-jobs/`: + + cargo test # unit, API and fake-BLAST tests + BLAST_JOBS_REAL_BLAST= cargo test --test real_blast + +`tests/real_blast.rs` skips itself, printing why, unless `BLAST_JOBS_REAL_BLAST` is set. When it runs, it +builds two tiny databases from `blast-jobs/fixture/` (N-RA and N-PA plus a decoy each, copied from the +FB2023_04 archive with the archive's `makeblastdb` flags). It runs blastn, blastp, blastx, tblastn and +tblastx through the API and checks the known Notch hit for each. It also checks that `-task blastn-short` +changes the scoring, and that the masking defaults hide Notch's low-complexity opa and polyglutamine +repeats until `advanced` turns them off. CI runs this test with the pinned NCBI tarball. It also builds +the image and runs `tests/image-smoke.sh` against it. diff --git a/docs/handoff-private-side.md b/docs/handoff-private-side.md index 65fcfe3..84e9d66 100644 --- a/docs/handoff-private-side.md +++ b/docs/handoff-private-side.md @@ -144,7 +144,12 @@ example `FB2023_04 (Dmel r6.53)`) is shown to users, so keep it accurate. The `e also shown to users verbatim, so it must carry no paths or hostnames. The service never sees the MCP's per-job client tokens. -- [ ] Routes implemented; the base URL and token recorded with the deployment secrets. +`blast-jobs/` in this repository implements these routes ([docs/blast-jobs.md](blast-jobs.md)): NCBI BLAST+ +2.17.0 in its own container, run against a read-only BLAST database directory. It omits `result_url` and +`downloads` and never produces `cancelled`. Its `BLAST_JOBS_TOKEN` is the MCP's `BLAST_SERVICE_TOKEN`. + +- [x] Routes implemented (`blast-jobs/`, reference implementation). +- [ ] Service deployed next to the MCP; the base URL and token recorded with the deployment secrets. - [ ] `BLAST_BASE_URL` and `BLAST_SERVICE_TOKEN` added to the compose service; `/health` shows `blast: ok`. ## 8. Monitoring and secrets From 683e1474a2a3b28de82b12e174f8d9534df8b524 Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Mon, 12 Oct 2026 18:16:44 +0000 Subject: [PATCH 09/12] fix(blast-jobs): native path-filtered workflow for NCBI jobs; real BLAST test pins 2.17.0 Move blast-jobs-integration and blast-jobs-image to .github/workflows/blast-jobs.yml with on.paths triggers and contents: read, dropping dorny/paths-filter. Pass the tarball pin through env. The real-BLAST test no longer falls back to PATH: every binary comes from BLAST_JOBS_REAL_BLAST and must report 2.17.0+, or the test fails. --- .github/workflows/blast-jobs.yml | 48 +++++++++++++++++++++++++++++ .github/workflows/ci.yml | 53 -------------------------------- blast-jobs/tests/real_blast.rs | 33 +++++++++++++++----- 3 files changed, 73 insertions(+), 61 deletions(-) create mode 100644 .github/workflows/blast-jobs.yml diff --git a/.github/workflows/blast-jobs.yml b/.github/workflows/blast-jobs.yml new file mode 100644 index 0000000..442b6a8 --- /dev/null +++ b/.github/workflows/blast-jobs.yml @@ -0,0 +1,48 @@ +# The blast-jobs jobs that download BLAST+ from NCBI. They live apart from ci.yml so a native path filter +# keeps a slow NCBI from reddening unrelated changes (design 3.12). The fast unit job stays in ci.yml. +name: blast-jobs +on: + push: + branches: [master] + paths: ['blast-jobs/**', '.github/workflows/blast-jobs.yml'] + pull_request: + paths: ['blast-jobs/**', '.github/workflows/blast-jobs.yml'] +permissions: + contents: read +jobs: + blast-jobs-integration: + runs-on: ubuntu-latest + defaults: { run: { working-directory: blast-jobs } } + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: { workspaces: blast-jobs } + # The tarball URL and SHA-256 are pinned once, in the Dockerfile. + - id: pin + run: | + echo "url=$(sed -n 's/^ARG BLAST_URL=//p' Dockerfile)" >> "$GITHUB_OUTPUT" + echo "sha=$(sed -n 's/^ARG BLAST_SHA256=//p' Dockerfile)" >> "$GITHUB_OUTPUT" + - uses: actions/cache@v4 + id: blast-cache + with: + path: ~/ncbi-blast.tar.gz + key: ncbi-blast-${{ steps.pin.outputs.sha }} + - if: steps.blast-cache.outputs.cache-hit != 'true' + env: { BLAST_URL: "${{ steps.pin.outputs.url }}" } + run: curl -fsSL --retry 3 -o ~/ncbi-blast.tar.gz "$BLAST_URL" + - env: { BLAST_SHA256: "${{ steps.pin.outputs.sha }}" } + run: | + test -n "$BLAST_SHA256" + echo "$BLAST_SHA256 $HOME/ncbi-blast.tar.gz" | sha256sum -c - + mkdir -p ~/ncbi-blast && tar -xzf ~/ncbi-blast.tar.gz -C ~/ncbi-blast --strip-components 1 + ~/ncbi-blast/bin/blastn -version | grep -F '2.17.0+' + - run: BLAST_JOBS_REAL_BLAST="$HOME/ncbi-blast/bin" cargo test --test real_blast + blast-jobs-image: + runs-on: ubuntu-latest + defaults: { run: { working-directory: blast-jobs } } + steps: + - uses: actions/checkout@v4 + - run: docker build -t blast-jobs:ci . + - run: docker image ls blast-jobs:ci + - run: tests/image-smoke.sh blast-jobs:ci diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4bfeb8..060e344 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,56 +53,3 @@ jobs: manifest-path: blast-jobs/Cargo.toml command: check advisories licenses - run: cargo test - blast-jobs-changes: - runs-on: ubuntu-latest - permissions: { contents: read, pull-requests: read } - outputs: - blast_jobs: ${{ steps.filter.outputs.blast_jobs }} - steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - blast_jobs: - - 'blast-jobs/**' - - '.github/workflows/ci.yml' - # The two jobs below download BLAST+ from NCBI, so they run on master pushes and on changes to blast-jobs/ - # only; a slow NCBI must not redden unrelated PRs (design 3.12). - blast-jobs-integration: - needs: blast-jobs-changes - if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || needs.blast-jobs-changes.outputs.blast_jobs == 'true' - runs-on: ubuntu-latest - defaults: { run: { working-directory: blast-jobs } } - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - with: { workspaces: blast-jobs } - # The tarball URL and SHA-256 are pinned once, in the Dockerfile. - - id: pin - run: | - echo "url=$(sed -n 's/^ARG BLAST_URL=//p' Dockerfile)" >> "$GITHUB_OUTPUT" - echo "sha=$(sed -n 's/^ARG BLAST_SHA256=//p' Dockerfile)" >> "$GITHUB_OUTPUT" - - uses: actions/cache@v4 - id: blast-cache - with: - path: ~/ncbi-blast.tar.gz - key: ncbi-blast-${{ steps.pin.outputs.sha }} - - if: steps.blast-cache.outputs.cache-hit != 'true' - run: curl -fsSL --retry 3 -o ~/ncbi-blast.tar.gz "${{ steps.pin.outputs.url }}" - - run: | - echo "${{ steps.pin.outputs.sha }} $HOME/ncbi-blast.tar.gz" | sha256sum -c - - mkdir -p ~/ncbi-blast && tar -xzf ~/ncbi-blast.tar.gz -C ~/ncbi-blast --strip-components 1 - ~/ncbi-blast/bin/blastn -version - - run: BLAST_JOBS_REAL_BLAST="$HOME/ncbi-blast/bin" cargo test --test real_blast - blast-jobs-image: - needs: blast-jobs-changes - if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || needs.blast-jobs-changes.outputs.blast_jobs == 'true' - runs-on: ubuntu-latest - defaults: { run: { working-directory: blast-jobs } } - steps: - - uses: actions/checkout@v4 - - run: docker build -t blast-jobs:ci . - - run: docker image ls blast-jobs:ci - - run: tests/image-smoke.sh blast-jobs:ci diff --git a/blast-jobs/tests/real_blast.rs b/blast-jobs/tests/real_blast.rs index a93c5a4..74213bd 100644 --- a/blast-jobs/tests/real_blast.rs +++ b/blast-jobs/tests/real_blast.rs @@ -1,6 +1,7 @@ //! End-to-end tests against real NCBI BLAST+ (design 3.12, Task 5). Skipped unless -//! `BLAST_JOBS_REAL_BLAST` is set. Its value is the directory holding the BLAST+ binaries (`blastn` ... -//! `makeblastdb`); any other non-empty value means "take them from `PATH`". +//! `BLAST_JOBS_REAL_BLAST` is set. Its value must be the directory holding the BLAST+ 2.17.0 binaries +//! (`blastn blastp blastx tblastn tblastx makeblastdb`). Every binary comes from that directory, never from +//! `PATH`, and each must report `2.17.0+`; a set but wrong value fails the tests instead of skipping them. //! //! Each test builds the fixture databases (`fixture/build-db.sh`, the archive's `makeblastdb` flags) in a //! temp directory, starts the API over them on an ephemeral port, and drives submit, poll and results. @@ -26,6 +27,16 @@ const ENV: &str = "BLAST_JOBS_REAL_BLAST"; const TOKEN: &str = "real-blast-test-token"; const NOTCH_TRANSCRIPT: &str = "fbdb.dmel.transcript.5460"; const NOTCH_PROTEIN: &str = "fbdb.dmel.translation.529"; +/// The pinned BLAST+ release (ruling 6); the suite proves behaviour on exactly this version. +const VERSION: &str = "2.17.0+"; +const BINARIES: [&str; 6] = [ + "blastn", + "blastp", + "blastx", + "tblastn", + "tblastx", + "makeblastdb", +]; fn manifest() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -41,13 +52,19 @@ fn blast_bin() -> Option { } }; let dir = PathBuf::from(v.trim()); - if dir.join("makeblastdb").is_file() { - return Some(dir); - } - let path = std::env::var_os("PATH").unwrap_or_default(); - let found = std::env::split_paths(&path).find(|d| d.join("makeblastdb").is_file()); // Set but unusable is a configuration error, not a skip. - Some(found.unwrap_or_else(|| panic!("{ENV}={v}: no makeblastdb there or on PATH"))) + for b in BINARIES { + let out = Command::new(dir.join(b)) + .arg("-version") + .output() + .unwrap_or_else(|e| panic!("{ENV}={v}: cannot run {b} from that directory: {e}")); + let text = String::from_utf8_lossy(&out.stdout); + assert!( + out.status.success() && text.contains(VERSION), + "{ENV}={v}: {b} -version does not report {VERSION}: {text}" + ); + } + Some(dir) } /// Reads one FASTA record's residues by its id from a fixture file. From 853406c7251a716c7e024921aa98947a0352e507 Mon Sep 17 00:00:00 2001 From: Christopher Tabone Date: Mon, 12 Oct 2026 18:17:25 +0000 Subject: [PATCH 10/12] chore(blast-jobs): benchmark harness --- blast-jobs/bench/README.md | 143 +++ blast-jobs/bench/queries/cdna_2kb.fa | 30 + blast-jobs/bench/queries/exon_500bp.fa | 9 + blast-jobs/bench/queries/genomic_100kb.fa | 1430 +++++++++++++++++++++ blast-jobs/bench/queries/genomic_10kb.fa | 144 +++ blast-jobs/bench/queries/genomic_30kb.fa | 430 +++++++ blast-jobs/bench/queries/manifest.tsv | 14 + blast-jobs/bench/queries/protein_376aa.fa | 7 + blast-jobs/bench/run.sh | 376 ++++++ 9 files changed, 2583 insertions(+) create mode 100644 blast-jobs/bench/README.md create mode 100644 blast-jobs/bench/queries/cdna_2kb.fa create mode 100644 blast-jobs/bench/queries/exon_500bp.fa create mode 100644 blast-jobs/bench/queries/genomic_100kb.fa create mode 100644 blast-jobs/bench/queries/genomic_10kb.fa create mode 100644 blast-jobs/bench/queries/genomic_30kb.fa create mode 100644 blast-jobs/bench/queries/manifest.tsv create mode 100644 blast-jobs/bench/queries/protein_376aa.fa create mode 100755 blast-jobs/bench/run.sh diff --git a/blast-jobs/bench/README.md b/blast-jobs/bench/README.md new file mode 100644 index 0000000..02d8ac6 --- /dev/null +++ b/blast-jobs/bench/README.md @@ -0,0 +1,143 @@ +# blast-jobs benchmark harness + +`run.sh` loads a running `blast-jobs` service through its internal job API and records how long every +search takes. It sizes `concurrency`, `threads_per_job`, the container's CPU limit and the two wall limits +before a deployment. It is plain bash, `curl` and `jq` (bash 4.3+, curl 7.55+, jq 1.6+). It holds no +hostnames or secrets: the service URL and label are arguments, and the bearer token comes from the +environment. + +## Usage + + export BLAST_JOBS_TOKEN=... # the service's bearer token + bench/run.sh