diff --git a/.ci-repro-marker b/.ci-repro-marker new file mode 100644 index 00000000000..d0ff51d0577 --- /dev/null +++ b/.ci-repro-marker @@ -0,0 +1,9 @@ +reproducibility demo marker +run 2 +run 3 +run 4 +run 5 +repro iteration A +repro iteration B +repro iteration C +repro iteration D diff --git a/.github/actions/setup-nativelink-cloud/action.yaml b/.github/actions/setup-nativelink-cloud/action.yaml index 5165e4dd3f1..f259a5d15a0 100644 --- a/.github/actions/setup-nativelink-cloud/action.yaml +++ b/.github/actions/setup-nativelink-cloud/action.yaml @@ -77,7 +77,7 @@ inputs: outputs: enabled: - description: "'true' when a user.bazelrc was written, else 'false'." + description: "'true' when remote cache or execution is enabled, else 'false'." value: ${{ steps.configure.outputs.enabled }} runs: diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml index fd60d6eb788..376fea249fb 100644 --- a/.github/workflows/native-bazel.yaml +++ b/.github/workflows/native-bazel.yaml @@ -49,18 +49,16 @@ jobs: - name: Free disk space uses: ./.github/actions/free-disk - - name: Setup Bazel - uses: >- # v0.19.0 - bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 - with: - bazelisk-cache: true - repository-cache: true - disk-cache: ${{ github.workflow }}-${{ matrix.os }}-bazel-${{ matrix.bazel_version }} - - name: Setup NativeLink Cloud + id: setup-nativelink-cloud uses: ./.github/actions/setup-nativelink-cloud with: - ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + # Default to 'cache' when the repo variable is unset so forks and + # untrusted PRs still get the committed anonymous read-only cache + # (cred_kind=public: cache_read only, --remote_upload_local_results + # =false, no BES). An explicit 'off' still disables; trusted lanes + # with secrets are unaffected. + ci-mode: ${{ vars.NATIVELINK_CI_MODE || 'cache' }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} api-key: >- @@ -72,6 +70,16 @@ jobs: # non-Linux runners, so macOS lanes fall back to cache. exec: ${{ runner.os == 'Linux' && 'on' || 'off' }} + - name: Setup Bazel + uses: >- # v0.19.0 + bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 + with: + bazelisk-cache: true + repository-cache: true + # Only enable when we don't have remote cache, otherwise it just gets very slow + # and enormous + disk-cache: ${{ steps.setup-nativelink-cloud.outputs.enabled == 'false'}} + - name: Show Bazel version run: bazel --version @@ -119,18 +127,16 @@ jobs: with: compose-file: src/bin/docker-compose.store-tester.yaml - - name: Setup Bazel - uses: >- # v0.19.0 - bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 - with: - bazelisk-cache: true - repository-cache: true - disk-cache: ${{ github.workflow }}-ubuntu-24.04 - - name: Setup NativeLink Cloud + id: setup-nativelink-cloud uses: ./.github/actions/setup-nativelink-cloud with: - ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + # Default to 'cache' when the repo variable is unset so forks and + # untrusted PRs still get the committed anonymous read-only cache + # (cred_kind=public: cache_read only, --remote_upload_local_results + # =false, no BES). An explicit 'off' still disables; trusted lanes + # with secrets are unaffected. + ci-mode: ${{ vars.NATIVELINK_CI_MODE || 'cache' }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} api-key: >- @@ -142,6 +148,16 @@ jobs: # non-Linux runners, so macOS lanes fall back to cache. exec: ${{ runner.os == 'Linux' && 'on' || 'off' }} + - name: Setup Bazel + uses: >- # v0.19.0 + bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 + with: + bazelisk-cache: true + repository-cache: true + # Only enable when we don't have remote cache, otherwise it just gets very slow + # and enormous + disk-cache: ${{ steps.setup-nativelink-cloud.outputs.enabled == 'false'}} + - name: Run Store tester with sentinel run: | bazel run //:redis_store_tester \ diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a400a0769..c5853fcf257 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,53 @@ All notable changes to this project will be documented in this file. +## [1.7.3](https://github.com/TraceMachina/nativelink/compare/v1.7.2..v1.7.3) - 2026-10-01 + + + +### โ›ฐ๏ธ Features + +- Support queueing-service through an opt-in gRPC event sink ([#2811](https://github.com/TraceMachina/nativelink/issues/2811)) - ([181a1d3](https://github.com/TraceMachina/nativelink/commit/181a1d34a116aafd93f7b735f224a7e31f314b54)) +- bound the input fetch with max_download_timeout and let a kill reach it ([#2846](https://github.com/TraceMachina/nativelink/issues/2846)) - ([82240c6](https://github.com/TraceMachina/nativelink/commit/82240c640970e8341f6358404456845aad17df47)) +- read capacity and free memory from the worker's own cgroup directory ([#2845](https://github.com/TraceMachina/nativelink/issues/2845)) - ([b4a5468](https://github.com/TraceMachina/nativelink/commit/b4a5468248f4699fcd1ad928e3ea65167e92af86)) +- Historical resource scheduler: hints by digest keys, size classes with a cold-start policy, and where each reservation came from ([#2832](https://github.com/TraceMachina/nativelink/issues/2832)) - ([59cbad2](https://github.com/TraceMachina/nativelink/commit/59cbad26b2a793e06d5e436b3e984f3d1c2df74a)) +- Persistent workers: a configurable pool with an idle sweeper, the action environment, namespaces, measured usage, and a wait at the cap ([#2830](https://github.com/TraceMachina/nativelink/issues/2830)) - ([035dd63](https://github.com/TraceMachina/nativelink/commit/035dd631650a30334db2f5d062dbde8ff3cde301)) +- memory escalation with retry budgets by cause, the whole worker as the last step ([#2829](https://github.com/TraceMachina/nativelink/issues/2829)) - ([3c52ee1](https://github.com/TraceMachina/nativelink/commit/3c52ee1a7026014deef13f92d9cef1e6b474136b)) +- a disk soft limit, and peak_disk_kb from the files an action writes ([#2825](https://github.com/TraceMachina/nativelink/issues/2825)) - ([7bd3c54](https://github.com/TraceMachina/nativelink/commit/7bd3c5424d00bd29f6e3e1516d60105196f439e2)) +- each action gets a private /tmp, its own tmp directory bound over /tmp ([#2824](https://github.com/TraceMachina/nativelink/issues/2824)) - ([c79e3bb](https://github.com/TraceMachina/nativelink/commit/c79e3bbe69e4ecac3e87f9503f47083bffd07fd6)) +- Worker reaps the zombies actions leave behind, and an action's stragglers no longer hold its output open ([#2823](https://github.com/TraceMachina/nativelink/issues/2823)) - ([4fe0a27](https://github.com/TraceMachina/nativelink/commit/4fe0a27ba9e44e02aedf2d197e6f2189c6e543b2)) +- SIGTERM grace before the kill, output kept, a TMPDIR per action, and resource usage with an outcome ([#2822](https://github.com/TraceMachina/nativelink/issues/2822)) - ([9a95e29](https://github.com/TraceMachina/nativelink/commit/9a95e29153f9cbd64208355f211293b97c6d6106)) +- Worker readiness follows scheduler registration ([#2821](https://github.com/TraceMachina/nativelink/issues/2821)) - ([de7a540](https://github.com/TraceMachina/nativelink/commit/de7a540222ca697da3fec83cbb97ba7bcfab039c)) +- Worker advertises CPU and memory from its own cgroup limits ([#2820](https://github.com/TraceMachina/nativelink/issues/2820)) - ([d86ff4f](https://github.com/TraceMachina/nativelink/commit/d86ff4f38ca6e40e09235adfff13cbb036eef9cc)) +- soft memory enforcement against the reservation, and a disk guard before inputs are fetched ([#2812](https://github.com/TraceMachina/nativelink/issues/2812)) - ([fcdccc8](https://github.com/TraceMachina/nativelink/commit/fcdccc81fd4e29e824f8eed733f72988f3b81a18)) +- bound output capture, upload fan-out, precondition scripts and orphaned action directories ([#2810](https://github.com/TraceMachina/nativelink/issues/2810)) - ([f10c7b7](https://github.com/TraceMachina/nativelink/commit/f10c7b70c10a450cefb0062922b33e0840eec8de)) +- kill the whole process group when an action times out or is cancelled ([#2809](https://github.com/TraceMachina/nativelink/issues/2809)) - ([e6aa38a](https://github.com/TraceMachina/nativelink/commit/e6aa38a6b490c281a9df03cf205c951e1a1dbedc)) +- Dispatch is acknowledged or declined by the worker ([#2808](https://github.com/TraceMachina/nativelink/issues/2808)) - ([a21d82c](https://github.com/TraceMachina/nativelink/commit/a21d82c8bdd4225e7f2287ff8018ba243384157b)) +- GoingAway with a drain flag, reconnect backoff with jitter, and a shutdown guard that waits for the drain ([#2807](https://github.com/TraceMachina/nativelink/issues/2807)) - ([2bfa020](https://github.com/TraceMachina/nativelink/commit/2bfa0207f8910d2af4d1b3d2df564a39c1730235)) +- Matching pass: room that opens mid-pass goes to the oldest waiting action, one store read per queued action ([#2806](https://github.com/TraceMachina/nativelink/issues/2806)) - ([2762510](https://github.com/TraceMachina/nativelink/commit/276251061994325162b96a79e12dca3b4dce9e20)) +- Redis scheduler store: RESP3 cursor pages, the retention default, an unreadable row skipped, a wait for the new index ([#2803](https://github.com/TraceMachina/nativelink/issues/2803)) - ([5ef7a83](https://github.com/TraceMachina/nativelink/commit/5ef7a8315d472f92dbb785eb82dfe8bd85968e06)) +- least_loaded and best_fit allocation strategies, typed properties cached per pass ([#2805](https://github.com/TraceMachina/nativelink/issues/2805)) - ([a7cc44d](https://github.com/TraceMachina/nativelink/commit/a7cc44da142e0a8a136176213fa48f2e71464145)) +- Admin API: list connected workers and queued demand ([#2804](https://github.com/TraceMachina/nativelink/issues/2804)) - ([02a64f2](https://github.com/TraceMachina/nativelink/commit/02a64f2b4c69ba5ae157c8c88f7bf6aad8aaa730)) +- Serve the scheduler queue in order, and let workers report their load on the keepalive ([#2796](https://github.com/TraceMachina/nativelink/issues/2796)) - ([4c06bb8](https://github.com/TraceMachina/nativelink/commit/4c06bb81ec4702d830d8aba7b6f39f225e016aa6)) +- Unsatisfiable actions: the clock follows the fleet's shapes, and a timeout for a fleet with no worker at all ([#2799](https://github.com/TraceMachina/nativelink/issues/2799)) - ([31be880](https://github.com/TraceMachina/nativelink/commit/31be88082f7c376d8b8648b1017d10975215c66a)) +- Scheduler worker state: budget released at completion, pause until the next keepalive, retry cap fails with FailedPrecondition ([#2798](https://github.com/TraceMachina/nativelink/issues/2798)) - ([71bb05a](https://github.com/TraceMachina/nativelink/commit/71bb05a0eedb89b69a4d8b5b5b562bea261644cc)) +- Worker liveness: any message counts, a 10s default, a dead keepalive reconnects, SIGINT drains ([#2797](https://github.com/TraceMachina/nativelink/issues/2797)) - ([2f54cc3](https://github.com/TraceMachina/nativelink/commit/2f54cc3bcb9548b83eedc447bfa02faca8120374)) + +### ๐Ÿ› Bug Fixes + +- a stalled scheduler connection reconnects instead of leaving the worker evicted and idle ([#2856](https://github.com/TraceMachina/nativelink/issues/2856)) - ([389cf73](https://github.com/TraceMachina/nativelink/commit/389cf73574e82ebee101beacea4453424908b06d)) +- a fast tier that ends short on a digest resumes from the slow store, and a missing input names its digest so the client re-uploads it ([#2855](https://github.com/TraceMachina/nativelink/issues/2855)) - ([bd1bb34](https://github.com/TraceMachina/nativelink/commit/bd1bb3402e03fd6b377ec7166dd3ba6f0c1b4416)) +- Give a live worker a grace period to report its own timeout ([#2826](https://github.com/TraceMachina/nativelink/issues/2826)) - ([5b4e555](https://github.com/TraceMachina/nativelink/commit/5b4e555070fe5e1b3e8709d11d03c1fc6812023b)) +- a read that stops short of the digest's size is an error, not EOF ([#2844](https://github.com/TraceMachina/nativelink/issues/2844)) - ([fd6530e](https://github.com/TraceMachina/nativelink/commit/fd6530ea797355506dc4f2ca6e730c398f8900cc)) +- build the cleanup mark's guard after the lock is released ([#2843](https://github.com/TraceMachina/nativelink/issues/2843)) - ([9a03932](https://github.com/TraceMachina/nativelink/commit/9a03932c0fd5b38150a865770c3222737b306836)) +- Scheduler skips and counts a queue entry it cannot read ([#2795](https://github.com/TraceMachina/nativelink/issues/2795)) - ([4001f52](https://github.com/TraceMachina/nativelink/commit/4001f5256eccfbfa1f09b849a7f03e4092c4f9bc)) +- Sorted index reads return the whole set, not ten rows ([#2794](https://github.com/TraceMachina/nativelink/issues/2794)) - ([7b2c02d](https://github.com/TraceMachina/nativelink/commit/7b2c02db4993bffb6073d408d7955d34e7882e1f)) + +### ๐Ÿงช Testing & CI + +- Give the oversized-upload store tests the 30 s deadline the third already has ([#2818](https://github.com/TraceMachina/nativelink/issues/2818)) - ([3ff63f1](https://github.com/TraceMachina/nativelink/commit/3ff63f17f52372c7ee7459d4f5f7690713fa6e5c)) +- Give trivy three vulnerability database mirrors to fall through ([#2801](https://github.com/TraceMachina/nativelink/issues/2801)) - ([a9c6801](https://github.com/TraceMachina/nativelink/commit/a9c6801f4e374016734811eb35231e14303778c8)) + ## [1.7.2](https://github.com/TraceMachina/nativelink/compare/v1.7.1..v1.7.2) - 2026-09-25 ### Major changes diff --git a/Cargo.lock b/Cargo.lock index ec5790ff881..89618fc6192 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2926,7 +2926,7 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "nativelink" -version = "1.7.2" +version = "1.7.3" dependencies = [ "async-lock", "axum", @@ -2957,7 +2957,7 @@ dependencies = [ [[package]] name = "nativelink-config" -version = "1.7.2" +version = "1.7.3" dependencies = [ "byte-unit", "humantime", @@ -2976,7 +2976,7 @@ dependencies = [ [[package]] name = "nativelink-error" -version = "1.7.2" +version = "1.7.3" dependencies = [ "base64 0.22.1", "mongodb", @@ -3000,7 +3000,7 @@ dependencies = [ [[package]] name = "nativelink-macro" -version = "1.7.2" +version = "1.7.3" dependencies = [ "proc-macro2", "quote", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "nativelink-metric" -version = "1.7.2" +version = "1.7.3" dependencies = [ "async-lock", "nativelink-metric-macro-derive", @@ -3020,7 +3020,7 @@ dependencies = [ [[package]] name = "nativelink-metric-macro-derive" -version = "1.7.2" +version = "1.7.3" dependencies = [ "proc-macro2", "quote", @@ -3029,7 +3029,7 @@ dependencies = [ [[package]] name = "nativelink-proto" -version = "1.7.2" +version = "1.7.3" dependencies = [ "derive_more 2.1.0", "prost", @@ -3041,7 +3041,7 @@ dependencies = [ [[package]] name = "nativelink-redis-tester" -version = "1.7.2" +version = "1.7.3" dependencies = [ "either", "nativelink-util", @@ -3054,7 +3054,7 @@ dependencies = [ [[package]] name = "nativelink-scheduler" -version = "1.7.2" +version = "1.7.3" dependencies = [ "async-lock", "async-trait", @@ -3091,7 +3091,7 @@ dependencies = [ [[package]] name = "nativelink-service" -version = "1.7.2" +version = "1.7.3" dependencies = [ "async-lock", "async-trait", @@ -3135,7 +3135,7 @@ dependencies = [ [[package]] name = "nativelink-store" -version = "1.7.2" +version = "1.7.3" dependencies = [ "async-lock", "async-trait", @@ -3214,7 +3214,7 @@ dependencies = [ [[package]] name = "nativelink-util" -version = "1.7.2" +version = "1.7.3" dependencies = [ "anyhow", "async-trait", @@ -3277,7 +3277,7 @@ dependencies = [ [[package]] name = "nativelink-worker" -version = "1.7.2" +version = "1.7.3" dependencies = [ "async-lock", "async-trait", @@ -4525,9 +4525,9 @@ dependencies = [ [[package]] name = "serial_test" -version = "3.5.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699f4197115b8a7e7ff19c9a315a4bd6fffec26cc4626ef45ecaea389e081c6d" +checksum = "a6df5ed973ad8d834e09f824f9e9f449af6b9a3745f78dec7cc752770bd3bf11" dependencies = [ "futures-executor", "futures-util", @@ -4538,13 +4538,13 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "3.5.0" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" +checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1643d711a50..b3c9f51bfb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ resolver = "2" edition = "2024" name = "nativelink" rust-version = "1.97.1" -version = "1.7.2" +version = "1.7.3" [profile.release] lto = true diff --git a/MODULE.bazel b/MODULE.bazel index d05045b0bf7..04f49a90337 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,6 +1,6 @@ module( name = "nativelink", - version = "1.7.2", + version = "1.7.3", compatibility_level = 0, ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 257355b8d7b..9c74a820ace 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1145,8 +1145,8 @@ "serde_urlencoded_0.7.1": "{\"dependencies\":[{\"name\":\"form_urlencoded\",\"req\":\"^1\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"name\":\"serde\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{}}", "serde_with_3.21.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"default_features\":false,\"name\":\"bs58\",\"optional\":true,\"req\":\"^0.5.1\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"chrono_0_4\",\"optional\":true,\"package\":\"chrono\",\"req\":\"^0.4.20\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"expect-test\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.6\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_14\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.14.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_15\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.15.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_16\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.16.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_17\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.17.0\"},{\"default_features\":false,\"name\":\"hex\",\"optional\":true,\"req\":\"^0.4.3\"},{\"default_features\":false,\"features\":[\"serde-1\"],\"name\":\"indexmap_1\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^1.8\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap_2\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"features\":[\"resolve-file\"],\"kind\":\"dev\",\"name\":\"jsonschema\",\"req\":\"^0.33.0\"},{\"kind\":\"dev\",\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.12.1\"},{\"kind\":\"dev\",\"name\":\"rmp-serde\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"ron\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"default_features\":false,\"name\":\"schemars_0_8\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^0.8.16\"},{\"kind\":\"dev\",\"name\":\"schemars_0_8\",\"package\":\"schemars\",\"req\":\"^0.8.16\"},{\"default_features\":false,\"name\":\"schemars_0_9\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"schemars_0_9\",\"package\":\"schemars\",\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"schemars_1\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"schemars_1\",\"package\":\"schemars\",\"req\":\"^1.0.2\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde-xml-rs\",\"req\":\"^0.8.1\"},{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"^1.0.225\"},{\"default_features\":false,\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.145\"},{\"features\":[\"preserve_order\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.124\"},{\"name\":\"serde_with_macros\",\"optional\":true,\"req\":\"=3.21.0\"},{\"default_features\":false,\"name\":\"smallvec_1\",\"optional\":true,\"package\":\"smallvec\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"time_0_3\",\"optional\":true,\"package\":\"time\",\"req\":\"~0.3.47\"},{\"kind\":\"dev\",\"name\":\"yaml_serde\",\"req\":\"^0.10.3\"}],\"features\":{\"alloc\":[\"serde_core/alloc\",\"base64?/alloc\",\"bs58?/alloc\",\"chrono_0_4?/alloc\",\"hex?/alloc\",\"serde_json?/alloc\",\"time_0_3?/alloc\"],\"base58\":[\"dep:bs58\",\"alloc\"],\"base64\":[\"dep:base64\",\"alloc\"],\"chrono\":[\"chrono_0_4\"],\"chrono_0_4\":[\"dep:chrono_0_4\"],\"default\":[\"std\",\"macros\"],\"guide\":[\"dep:document-features\",\"macros\",\"std\"],\"hashbrown_0_14\":[\"dep:hashbrown_0_14\",\"alloc\"],\"hashbrown_0_15\":[\"dep:hashbrown_0_15\",\"alloc\"],\"hashbrown_0_16\":[\"dep:hashbrown_0_16\",\"alloc\"],\"hashbrown_0_17\":[\"dep:hashbrown_0_17\",\"alloc\"],\"hex\":[\"dep:hex\",\"alloc\"],\"indexmap\":[\"indexmap_1\"],\"indexmap_1\":[\"dep:indexmap_1\",\"alloc\"],\"indexmap_2\":[\"dep:indexmap_2\",\"alloc\"],\"json\":[\"dep:serde_json\",\"alloc\"],\"macros\":[\"dep:serde_with_macros\"],\"schemars_0_8\":[\"dep:schemars_0_8\",\"std\",\"serde_with_macros?/schemars_0_8\",\"dep:serde_json\"],\"schemars_0_9\":[\"dep:schemars_0_9\",\"alloc\",\"serde_with_macros?/schemars_0_9\",\"dep:serde_json\"],\"schemars_1\":[\"dep:schemars_1\",\"alloc\",\"serde_with_macros?/schemars_1\",\"dep:serde_json\"],\"smallvec_1\":[\"dep:smallvec_1\"],\"std\":[\"alloc\",\"bs58?/std\",\"serde_core/std\",\"chrono_0_4?/clock\",\"chrono_0_4?/std\",\"indexmap_1?/std\",\"indexmap_2?/std\",\"time_0_3?/serde-well-known\",\"time_0_3?/std\",\"schemars_0_9?/std\",\"schemars_1?/std\"],\"time_0_3\":[\"dep:time_0_3\"]}}", "serde_with_macros_3.21.0": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.23.0\"},{\"kind\":\"dev\",\"name\":\"expect-test\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.12.1\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.25\"},{\"features\":[\"extra-traits\",\"full\",\"parsing\"],\"name\":\"syn\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.111\"}],\"features\":{\"schemars_0_8\":[],\"schemars_0_9\":[],\"schemars_1\":[]}}", - "serial_test_3.5.0": "{\"dependencies\":[{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"env_logger\",\"optional\":true,\"req\":\">=0.6.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"fslock\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-executor\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"use_std\"],\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\">=0.4\"},{\"name\":\"log\",\"optional\":true,\"req\":\">=0.4.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"once_cell\",\"req\":\"^1.19\"},{\"default_features\":false,\"name\":\"parking_lot\",\"req\":\"^0.12\"},{\"name\":\"serial_test_derive\",\"req\":\"~3.5.0\"}],\"features\":{\"async\":[\"dep:futures-executor\",\"dep:futures-util\",\"serial_test_derive/async\"],\"default\":[\"logging\",\"async\"],\"docsrs\":[\"dep:document-features\"],\"file_locks\":[\"dep:fslock\"],\"logging\":[\"dep:log\"],\"test_logging\":[\"logging\",\"dep:env_logger\",\"serial_test_derive/test_logging\"]}}", - "serial_test_derive_3.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\">=0.6.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"full\",\"printing\",\"parsing\",\"clone-impls\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{\"async\":[],\"default\":[],\"file_locks\":[],\"test_logging\":[]}}", + "serial_test_4.0.1": "{\"dependencies\":[{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"env_logger\",\"optional\":true,\"req\":\">=0.6.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"fslock\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-executor\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"use_std\"],\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\">=0.4\"},{\"name\":\"log\",\"optional\":true,\"req\":\">=0.4.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"once_cell\",\"req\":\"^1.19\"},{\"default_features\":false,\"name\":\"parking_lot\",\"req\":\"^0.12\"},{\"name\":\"serial_test_derive\",\"req\":\"~4.0.1\"}],\"features\":{\"async\":[\"dep:futures-executor\",\"dep:futures-util\",\"serial_test_derive/async\"],\"default\":[\"logging\",\"async\"],\"docsrs\":[\"dep:document-features\"],\"file_locks\":[\"dep:fslock\"],\"logging\":[\"dep:log\"],\"test_logging\":[\"logging\",\"dep:env_logger\",\"serial_test_derive/test_logging\"]}}", + "serial_test_derive_4.0.1": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\">=0.6.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"full\",\"printing\",\"parsing\",\"clone-impls\"],\"name\":\"syn\",\"req\":\"^3\"}],\"features\":{\"async\":[],\"default\":[],\"file_locks\":[],\"test_logging\":[]}}", "sha1_0.10.6": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"cpufeatures\",\"req\":\"^0.2\",\"target\":\"cfg(any(target_arch = \\\"aarch64\\\", target_arch = \\\"x86\\\", target_arch = \\\"x86_64\\\"))\"},{\"name\":\"digest\",\"req\":\"^0.10.7\"},{\"features\":[\"dev\"],\"kind\":\"dev\",\"name\":\"digest\",\"req\":\"^0.10.7\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.2.2\"},{\"name\":\"sha1-asm\",\"optional\":true,\"req\":\"^0.5\",\"target\":\"cfg(any(target_arch = \\\"aarch64\\\", target_arch = \\\"x86\\\", target_arch = \\\"x86_64\\\"))\"}],\"features\":{\"asm\":[\"sha1-asm\"],\"compress\":[],\"default\":[\"std\"],\"force-soft\":[],\"loongarch64_asm\":[],\"oid\":[\"digest/oid\"],\"std\":[\"digest/std\"]}}", "sha1_0.11.0": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"cpufeatures\",\"req\":\"^0.3\",\"target\":\"cfg(any(target_arch = \\\"aarch64\\\", target_arch = \\\"x86\\\", target_arch = \\\"x86_64\\\"))\"},{\"name\":\"digest\",\"req\":\"^0.11\"},{\"features\":[\"dev\"],\"kind\":\"dev\",\"name\":\"digest\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^1\"}],\"features\":{\"alloc\":[\"digest/alloc\"],\"default\":[\"alloc\",\"oid\"],\"oid\":[\"digest/oid\"],\"zeroize\":[\"digest/zeroize\"]}}", "sha1_smol_1.0.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"openssl\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.4\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[],\"std\":[\"alloc\"]}}", diff --git a/cliff.toml b/cliff.toml index 716467aaf87..65d460d3982 100644 --- a/cliff.toml +++ b/cliff.toml @@ -72,6 +72,40 @@ commit_parsers = [ # Custom rules for specific commits. + { message = "Dispatch is acknowledged or declined by the worker", group = "โ›ฐ๏ธ Features" }, + { message = "Give a live worker a grace period to report its own timeout", group = "๐Ÿ› Bug Fixes" }, + { message = "Give the oversized-upload store tests the 30 s deadline", group = "๐Ÿงช Testing & CI" }, + { message = "Give trivy three vulnerability database mirrors", group = "๐Ÿงช Testing & CI" }, + { message = "GoingAway with a drain flag, reconnect backoff with jitter", group = "โ›ฐ๏ธ Features" }, + { message = "RESP3 cursor pages, the retention default", group = "โ›ฐ๏ธ Features" }, + { message = "SIGTERM grace before the kill, output kept", group = "โ›ฐ๏ธ Features" }, + { message = "Scheduler skips and counts a queue entry it cannot read", group = "๐Ÿ› Bug Fixes" }, + { message = "Serve the scheduler queue in order", group = "โ›ฐ๏ธ Features" }, + { message = "Sorted index reads return the whole set", group = "๐Ÿ› Bug Fixes" }, + { message = "Support queueing-service through an opt-in gRPC event sink", group = "โ›ฐ๏ธ Features" }, + { message = "Worker advertises CPU and memory from its own cgroup limits", group = "โ›ฐ๏ธ Features" }, + { message = "Worker readiness follows scheduler registration", group = "โ›ฐ๏ธ Features" }, + { message = "a configurable pool with an idle sweeper", group = "โ›ฐ๏ธ Features" }, + { message = "a disk soft limit, and peak_disk_kb", group = "โ›ฐ๏ธ Features" }, + { message = "a fast tier that ends short on a digest resumes from the slow store", group = "๐Ÿ› Bug Fixes" }, + { message = "a read that stops short of the digest's size is an error", group = "๐Ÿ› Bug Fixes" }, + { message = "a stalled scheduler connection reconnects instead of leaving the worker evicted", group = "๐Ÿ› Bug Fixes" }, + { message = "any message counts, a 10s default", group = "โ›ฐ๏ธ Features" }, + { message = "bound output capture, upload fan-out, precondition scripts", group = "โ›ฐ๏ธ Features" }, + { message = "bound the input fetch with max_download_timeout", group = "โ›ฐ๏ธ Features" }, + { message = "budget released at completion, pause until the next keepalive", group = "โ›ฐ๏ธ Features" }, + { message = "build the cleanup mark's guard after the lock is released", group = "๐Ÿ› Bug Fixes" }, + { message = "each action gets a private /tmp", group = "โ›ฐ๏ธ Features" }, + { message = "hints by digest keys, size classes with a cold-start policy", group = "โ›ฐ๏ธ Features" }, + { message = "kill the whole process group when an action times out", group = "โ›ฐ๏ธ Features" }, + { message = "list connected workers and queued demand", group = "โ›ฐ๏ธ Features" }, + { message = "memory escalation with retry budgets by cause", group = "โ›ฐ๏ธ Features" }, + { message = "read capacity and free memory from the worker's own cgroup directory", group = "โ›ฐ๏ธ Features" }, + { message = "reaps the zombies actions leave behind", group = "โ›ฐ๏ธ Features" }, + { message = "room that opens mid-pass goes to the oldest waiting action", group = "โ›ฐ๏ธ Features" }, + { message = "soft memory enforcement against the reservation", group = "โ›ฐ๏ธ Features" }, + { message = "the clock follows the fleet's shapes", group = "โ›ฐ๏ธ Features" }, + { message = "Account for block size in filesystem store for eviction purposes", group = "๐Ÿ› Bug Fixes" }, { message = "Add ONTAP S3 Store with existence cache", group = "โ›ฐ๏ธ Features" }, { message = "Add OSFamily property in basic_cas.json", group = "๐Ÿ“š Documentation" }, diff --git a/llms-full.txt b/llms-full.txt index d2abea0b8e7..3c49ff85bca 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -11416,6 +11416,23 @@ restart. forward selected headers from the incoming request. Use it to front an existing cache. +Development builds also support `grpc.store_type: "event_sink"` for forwarding +BEP and origin events to a dedicated receiver. This mode accepts string keys +with exact payload sizes and sends uncompressed `ByteStream.Write` requests. +The resource hash identifies the event key, while the size describes the event +bytes. Configure it only as an event publisher's store: reads, existence checks, +and digest-key writes are rejected. A normal CAS endpoint cannot receive these +writes. The receiver must acknowledge the full payload size only after its +durable append completes; NativeLink rejects a different committed size, and +its BEP response waits for that acknowledgement. A lost acknowledgement can +cause a retry after the receiver has already persisted an event. The receiver +must use the stable event key to detect and ignore duplicate events, including +retries that start a new upload; the upload UUID alone is not an event identity. +Origin publishing retains its existing bounded retry policy. This transport +provides neither exactly-once processing nor a durable publisher recovery ledger. + + + `noop` takes nothing at all and discards everything written to it, reporting every read as a miss. It is how you deliberately turn a half of something off. diff --git a/nativelink-config/Cargo.toml b/nativelink-config/Cargo.toml index d3d52c8e2a3..ff5a5765ae9 100644 --- a/nativelink-config/Cargo.toml +++ b/nativelink-config/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-config" -version = "1.7.2" +version = "1.7.3" [dependencies] nativelink-error = { path = "../nativelink-error" } diff --git a/nativelink-config/src/cas_server.rs b/nativelink-config/src/cas_server.rs index 3e52a8d1741..6fddb93bf70 100644 --- a/nativelink-config/src/cas_server.rs +++ b/nativelink-config/src/cas_server.rs @@ -877,10 +877,35 @@ pub struct EndpointConfig { #[serde(deserialize_with = "convert_string_with_shellexpand")] pub uri: String, - /// Timeout in seconds that a request should take. + /// Timeout in seconds for connecting to the endpoint and for a request + /// to be answered. The worker's messages to the scheduler (keepalives, + /// acknowledgements, results) ride on the one `ConnectWorker` stream + /// opened at registration, which this does not bound; the `GoingAway` + /// sent at shutdown is bounded by it, so a stalled connection cannot + /// hold a pod past its grace period. A keepalive goes out every half of + /// it. /// Default: 5 seconds pub timeout: Option, + /// TCP keepalive interval (seconds), probing the socket under the + /// scheduler connection so a dead peer is noticed at the OS level. + /// If not set or 0, defaults to 30 seconds. + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + pub tcp_keepalive_s: u64, + + /// HTTP/2 keepalive interval (seconds): PING frames on the scheduler + /// connection, so one that has gone dead underneath fails instead of + /// hanging every message on it. + /// If not set or 0, defaults to 30 seconds. + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + pub http2_keepalive_interval_s: u64, + + /// HTTP/2 keepalive timeout (seconds): a PING unanswered for this long + /// ends the connection, and the worker reconnects. + /// If not set or 0, defaults to 20 seconds. + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + pub http2_keepalive_timeout_s: u64, + /// The TLS configuration to use to connect to the endpoint. pub tls_config: Option, } diff --git a/nativelink-error/Cargo.toml b/nativelink-error/Cargo.toml index 0a97dfedfb3..fca255e5870 100644 --- a/nativelink-error/Cargo.toml +++ b/nativelink-error/Cargo.toml @@ -8,7 +8,7 @@ autoexamples = false autotests = true edition = "2024" name = "nativelink-error" -version = "1.7.2" +version = "1.7.3" [dependencies] nativelink-metric = { path = "../nativelink-metric" } diff --git a/nativelink-macro/Cargo.toml b/nativelink-macro/Cargo.toml index 8954214929e..7684456403f 100644 --- a/nativelink-macro/Cargo.toml +++ b/nativelink-macro/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-macro" -version = "1.7.2" +version = "1.7.3" [lib] proc-macro = true diff --git a/nativelink-metric/Cargo.toml b/nativelink-metric/Cargo.toml index e14e11f0151..1542fe48dd3 100644 --- a/nativelink-metric/Cargo.toml +++ b/nativelink-metric/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-metric" -version = "1.7.2" +version = "1.7.3" [dependencies] nativelink-metric-macro-derive = { path = "nativelink-metric-macro-derive" } diff --git a/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml b/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml index 9cf8b22a8ac..a5ef6436211 100644 --- a/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml +++ b/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2024" name = "nativelink-metric-macro-derive" -version = "1.7.2" +version = "1.7.3" [lib] proc-macro = true diff --git a/nativelink-proto/Cargo.toml b/nativelink-proto/Cargo.toml index e1b95545f10..7f421f771fd 100644 --- a/nativelink-proto/Cargo.toml +++ b/nativelink-proto/Cargo.toml @@ -2,7 +2,7 @@ [package] edition = "2024" name = "nativelink-proto" -version = "1.7.2" +version = "1.7.3" [lib] doctest = false # because some of the generated protos have things that look like doctests but break diff --git a/nativelink-redis-tester/Cargo.toml b/nativelink-redis-tester/Cargo.toml index 79d0ab2401e..7969bca12a9 100644 --- a/nativelink-redis-tester/Cargo.toml +++ b/nativelink-redis-tester/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-redis-tester" -version = "1.7.2" +version = "1.7.3" [dependencies] nativelink-util = { path = "../nativelink-util" } diff --git a/nativelink-scheduler/Cargo.toml b/nativelink-scheduler/Cargo.toml index b6747197275..ff752b77941 100644 --- a/nativelink-scheduler/Cargo.toml +++ b/nativelink-scheduler/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-scheduler" -version = "1.7.2" +version = "1.7.3" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-scheduler/src/simple_scheduler_state_manager.rs b/nativelink-scheduler/src/simple_scheduler_state_manager.rs index 664aa1eba4c..79489efc9fe 100644 --- a/nativelink-scheduler/src/simple_scheduler_state_manager.rs +++ b/nativelink-scheduler/src/simple_scheduler_state_manager.rs @@ -46,6 +46,34 @@ use super::awaited_action_db::{ }; use crate::worker_registry::{ORPHANED_ACTION_TIMEOUT, SharedWorkerRegistry, WorkerLiveness}; +/// Why the scheduler times out an executing action. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TimeoutCause { + /// The action's `Action.timeout` passed, plus the grace a live worker + /// has to report its own result. + ActionTimeout { timeout: Duration, grace: Duration }, + /// The worker sent no update on the action for this long. + NoWorkerUpdate(Duration), +} + +impl core::fmt::Display for TimeoutCause { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::ActionTimeout { timeout, grace } => write!( + f, + "exceeded its Action.timeout of {} seconds, and its worker had not reported within {} seconds after that", + timeout.as_secs_f32(), + grace.as_secs_f32(), + ), + Self::NoWorkerUpdate(ceiling) => write!( + f, + "timed out after {} seconds with no update from its worker", + ceiling.as_secs_f32(), + ), + } + } +} + /// Maximum number of times an update to the database /// can fail before giving up. const MAX_UPDATE_RETRIES: usize = 5; @@ -312,25 +340,23 @@ where .err_tip(|| format!("Failed to upgrade weak reference to SimpleSchedulerStateManager in MatchingEngineActionStateResult::changed at attempt: {timeout_attempts}"))?; // Check if worker is alive via registry before timing out. - let should_timeout = simple_scheduler_state_manager - .should_timeout_operation(&awaited_action) - .await; - - if !should_timeout { + let Some(cause) = simple_scheduler_state_manager + .timeout_cause(&awaited_action) + .await + else { // Worker is alive, continue waiting for updates trace!( operation_id = %awaited_action.operation_id(), "Operation timeout check passed, worker is alive" ); continue; - } + }; warn!( ?awaited_action, - "OperationId {} / {} timed out after {} seconds issuing a retry", + "OperationId {} / {} {cause}, issuing a retry", awaited_action.operation_id(), awaited_action.state().client_operation_id, - self.no_event_action_timeout.as_secs_f32(), ); simple_scheduler_state_manager @@ -680,23 +706,41 @@ where } pub async fn should_timeout_operation(&self, awaited_action: &AwaitedAction) -> bool { + self.timeout_cause(awaited_action).await.is_some() + } + + /// Why the scheduler should time out an executing action, or `None` if + /// it should not. + pub async fn timeout_cause(&self, awaited_action: &AwaitedAction) -> Option { if !matches!(awaited_action.state().stage, ActionStage::Executing) { - return false; + return None; } let now = (self.now_fn)().now(); // Honor the per-action `Action.timeout` from the RBE protocol as a - // backend wall-clock deadline. Without this, the only enforcement is - // the Bazel client's --test_timeout, which surfaces as TIMEOUT/NO - // STATUS instead of a backend signal pointing at the worker. + // backend wall-clock deadline, in every liveness case below. The + // worker enforces the same timeout, but its clock starts when the + // command starts, after the inputs are fetched, which the scheduler + // cannot observe. Enforcing it at `Action.timeout` from assignment + // would pre-empt every live worker's own DEADLINE_EXCEEDED result, so + // allow `no_event_action_timeout` more, measured from the later of the + // assignment and the worker's last update: a live worker then reports + // first, and a wedged one (hung input fetch, unkillable child, broken + // external timeout, orphan) is still ended here. let action_timeout = awaited_action.action_info().timeout; if action_timeout > Duration::ZERO { - let executing_started_at = awaited_action.state().last_transition_timestamp; - if let Ok(elapsed) = now.duration_since(executing_started_at) - && elapsed > action_timeout + let anchor = awaited_action + .state() + .last_transition_timestamp + .max(awaited_action.last_worker_updated_timestamp()); + if let Ok(elapsed) = now.duration_since(anchor) + && elapsed > action_timeout.saturating_add(self.no_event_action_timeout) { - return true; + return Some(TimeoutCause::ActionTimeout { + timeout: action_timeout, + grace: self.no_event_action_timeout, + }); } } @@ -711,17 +755,23 @@ where _ => WorkerLiveness::Stale, }; + let last_update = awaited_action.last_worker_updated_timestamp(); + let silent_for = |ceiling: Duration| { + now.duration_since(last_update) + .is_ok_and(|elapsed| elapsed > ceiling) + .then_some(TimeoutCause::NoWorkerUpdate(ceiling)) + }; + match liveness { - // Ours and heartbeating: only the stuck-but-alive ceiling applies, - // and disabling that means no ceiling on a live worker. + // Ours and heartbeating: besides Action.timeout above, only the + // stuck-but-alive ceiling applies, and disabling that leaves an + // action with no Action.timeout no ceiling on a live worker. WorkerLiveness::Alive => { if self.max_executing_timeout > Duration::ZERO { - let last_update = awaited_action.last_worker_updated_timestamp(); - if let Ok(elapsed) = now.duration_since(last_update) { - return elapsed > self.max_executing_timeout; - } + silent_for(self.max_executing_timeout) + } else { + None } - false } // Usually a peer's healthy worker, so worker_timeout_s must not @@ -729,27 +779,15 @@ where // max_action_executing_timeout_s defaults to disabled, so fall // back to a ceiling rather than never timing out. WorkerLiveness::Unknown => { - let ceiling = if self.max_executing_timeout > Duration::ZERO { - self.max_executing_timeout + if self.max_executing_timeout > Duration::ZERO { + silent_for(self.max_executing_timeout) } else { - ORPHANED_ACTION_TIMEOUT - }; - let last_update = awaited_action.last_worker_updated_timestamp(); - match now.duration_since(last_update) { - Ok(elapsed) => elapsed > ceiling, - Err(_) => false, + silent_for(ORPHANED_ACTION_TIMEOUT) } } // Registered here and gone quiet: ours, and it looks dead. - WorkerLiveness::Stale => { - let worker_should_update_before = awaited_action - .last_worker_updated_timestamp() - .checked_add(self.no_event_action_timeout) - .unwrap_or(now); - - worker_should_update_before < now - } + WorkerLiveness::Stale => silent_for(self.no_event_action_timeout), } } @@ -929,28 +967,25 @@ where // Re-check under the lock against freshly loaded state, and delegate // rather than re-deriving the rule: the two copies had drifted. - if !self.should_timeout_operation(&awaited_action).await { + let Some(cause) = self.timeout_cause(&awaited_action).await else { trace!( %operation_id, worker_id = ?awaited_action.worker_id(), "Operation no longer needs timing out, skipping" ); return Ok(()); - } + }; warn!( %operation_id, worker_id = ?awaited_action.worker_id(), + %cause, "Timing out operation" ); self.assign_operation( operation_id, - Err(make_err!( - Code::DeadlineExceeded, - "Operation timed out after {} seconds", - self.no_event_action_timeout.as_secs_f32(), - )), + Err(make_err!(Code::DeadlineExceeded, "Operation {cause}")), ) .await } diff --git a/nativelink-scheduler/tests/simple_scheduler_state_manager_test.rs b/nativelink-scheduler/tests/simple_scheduler_state_manager_test.rs index eb96633b6cd..371487a668b 100644 --- a/nativelink-scheduler/tests/simple_scheduler_state_manager_test.rs +++ b/nativelink-scheduler/tests/simple_scheduler_state_manager_test.rs @@ -19,7 +19,9 @@ use nativelink_scheduler::awaited_action_db::{ }; use nativelink_scheduler::default_scheduler_factory::memory_awaited_action_db_factory; use nativelink_scheduler::simple_scheduler::SimpleScheduler; -use nativelink_scheduler::simple_scheduler_state_manager::SimpleSchedulerStateManager; +use nativelink_scheduler::simple_scheduler_state_manager::{ + SimpleSchedulerStateManager, TimeoutCause, +}; use nativelink_scheduler::worker::Worker; use nativelink_scheduler::worker_registry::WorkerRegistry; use nativelink_scheduler::worker_scheduler::WorkerScheduler; @@ -262,6 +264,231 @@ async fn does_not_time_out_a_live_worker_mid_action() -> Result<(), Error> { Ok(()) } +/// `Action.timeout` for the deadline tests below. The grace is +/// `WORKER_TIMEOUT`, the state managers' `no_event_action_timeout`. +const ACTION_TIMEOUT: Duration = Duration::from_mins(1); + +/// An action with `ACTION_TIMEOUT`, assigned to `worker_id` and executing +/// since `started`. +fn action_with_timeout(worker_id: &WorkerId, started: SystemTime) -> AwaitedAction { + let mut info = action_info(started); + info.timeout = ACTION_TIMEOUT; + let action_digest = info.digest(); + let operation_id = OperationId::default(); + let mut action = AwaitedAction::new(operation_id.clone(), Arc::new(info), started); + action.worker_set_state( + Arc::new(ActionState { + stage: ActionStage::Executing, + client_operation_id: operation_id, + action_digest, + last_transition_timestamp: started, + }), + started, + ); + action.set_worker_id(Some(worker_id.clone()), started); + action +} + +const fn action_timeout_cause() -> TimeoutCause { + TimeoutCause::ActionTimeout { + timeout: ACTION_TIMEOUT, + grace: WORKER_TIMEOUT, + } +} + +/// Just short of `Action.timeout` plus the grace. +const WITHIN_GRACE: Duration = + Duration::from_secs(ACTION_TIMEOUT.as_secs() + WORKER_TIMEOUT.as_secs() - 1); + +/// Just past `Action.timeout` plus the grace. +const PAST_GRACE: Duration = + Duration::from_secs(ACTION_TIMEOUT.as_secs() + WORKER_TIMEOUT.as_secs() + 1); + +/// A heartbeating worker enforces `Action.timeout` itself, from when the +/// command starts, and reports a completed `DEADLINE_EXCEEDED` result. The +/// scheduler's clock starts at assignment, earlier, so ending the action at +/// `Action.timeout` would requeue every action that runs to its timeout onto +/// the worker still running it, and the worker's own result would then evict +/// it. The grace lets the live worker report first. +#[nativelink_test] +async fn gives_a_live_worker_the_grace_to_report_its_own_timeout() -> Result<(), Error> { + MockClock::set_time(Duration::from_secs(NOW_TIME)); + let worker_id = WorkerId::from(String::from("enforces-its-own-timeout")); + let action = action_with_timeout(&worker_id, make_system_time(0)); + + let registry = Arc::new(WorkerRegistry::new()); + registry + .update_worker_heartbeat(&worker_id, make_system_time(WITHIN_GRACE.as_secs())) + .await; + let state_mgr = state_manager_no_executing_ceiling(registry); + + MockClock::advance(WITHIN_GRACE); + assert!( + !state_mgr.should_timeout_operation(&action).await, + "a live worker's action past Action.timeout but within the grace is the worker's to end" + ); + Ok(()) +} + +/// A heartbeating worker that never reports (a hung input fetch, an +/// unkillable child, a broken `timeout_handled_externally` wrapper) is timed +/// out by `Action.timeout` once the grace has passed, even with +/// `max_action_executing_timeout_s` disabled. +#[nativelink_test] +async fn times_out_a_live_but_wedged_worker_past_the_grace() -> Result<(), Error> { + MockClock::set_time(Duration::from_secs(NOW_TIME)); + let worker_id = WorkerId::from(String::from("heartbeats-but-never-reports")); + let action = action_with_timeout(&worker_id, make_system_time(0)); + + let registry = Arc::new(WorkerRegistry::new()); + registry + .update_worker_heartbeat(&worker_id, make_system_time(PAST_GRACE.as_secs())) + .await; + let state_mgr = state_manager_no_executing_ceiling(registry); + + MockClock::advance(PAST_GRACE); + let cause = state_mgr.timeout_cause(&action).await; + assert_eq!( + cause, + Some(action_timeout_cause()), + "a live worker's action past Action.timeout plus the grace must time out" + ); + assert!( + cause + .unwrap() + .to_string() + .contains("Action.timeout of 60 seconds"), + "the client's error must name the deadline that fired" + ); + Ok(()) +} + +/// The grace runs from the worker's last update when that is later than the +/// assignment, so it only has to cover the time since the worker last showed +/// signs of life. +#[nativelink_test] +async fn measures_the_grace_from_the_workers_last_update() -> Result<(), Error> { + MockClock::set_time(Duration::from_secs(NOW_TIME)); + let worker_id = WorkerId::from(String::from("updated-after-assignment")); + let mut action = action_with_timeout(&worker_id, make_system_time(0)); + let updated = Duration::from_secs(30); + let state = action.state().clone(); + action.worker_set_state(state, make_system_time(updated.as_secs())); + + let registry = Arc::new(WorkerRegistry::new()); + registry + .update_worker_heartbeat( + &worker_id, + make_system_time((PAST_GRACE + updated).as_secs()), + ) + .await; + let state_mgr = state_manager_no_executing_ceiling(registry); + + MockClock::advance(PAST_GRACE); + assert!( + !state_mgr.should_timeout_operation(&action).await, + "the grace must run from the last worker update, not the assignment" + ); + + MockClock::advance(updated); + assert_eq!( + state_mgr.timeout_cause(&action).await, + Some(action_timeout_cause()), + "past Action.timeout plus the grace from the last update, the action must time out" + ); + Ok(()) +} + +/// A peer instance's live worker enforces `Action.timeout` itself too, so it +/// gets the same grace here. +#[nativelink_test] +async fn gives_a_peer_instances_worker_the_grace() -> Result<(), Error> { + MockClock::set_time(Duration::from_secs(NOW_TIME)); + // Deliberately not registered here: it belongs to another instance. + let action = action_with_timeout( + &WorkerId::from(String::from("owned-by-another-scheduler")), + make_system_time(0), + ); + let state_mgr = state_manager_no_executing_ceiling(Arc::new(WorkerRegistry::new())); + + MockClock::advance(WITHIN_GRACE); + assert!( + !state_mgr.should_timeout_operation(&action).await, + "a peer's worker's action within the grace is the worker's to end" + ); + Ok(()) +} + +/// An orphan nobody reaps, or a peer's worker that never reports, is timed +/// out by `Action.timeout` past the grace rather than waiting for the orphan +/// ceiling. +#[nativelink_test] +async fn times_out_an_orphan_on_action_timeout_past_the_grace() -> Result<(), Error> { + MockClock::set_time(Duration::from_secs(NOW_TIME)); + let action = action_with_timeout( + &WorkerId::from(String::from("owner-was-scaled-down")), + make_system_time(0), + ); + let state_mgr = state_manager_no_executing_ceiling(Arc::new(WorkerRegistry::new())); + + MockClock::advance(PAST_GRACE); + assert_eq!( + state_mgr.timeout_cause(&action).await, + Some(action_timeout_cause()), + "an orphan past Action.timeout plus the grace must time out, not wait an hour" + ); + Ok(()) +} + +/// Registry heartbeats refresh only on keepalives, so a worker whose +/// keepalives lag reads Stale while its action is still running and still +/// updating. `Action.timeout` must not pre-empt it there either. +#[nativelink_test] +async fn gives_a_stale_worker_with_a_recent_update_the_grace() -> Result<(), Error> { + MockClock::set_time(Duration::from_secs(NOW_TIME)); + let worker_id = WorkerId::from(String::from("keepalives-lag")); + let assigned = WORKER_TIMEOUT + Duration::from_secs(30); + let action = action_with_timeout(&worker_id, make_system_time(assigned.as_secs())); + + let registry = Arc::new(WorkerRegistry::new()); + registry + .register_worker(&worker_id, make_system_time(0)) + .await; + let state_mgr = state_manager_no_executing_ceiling(registry); + + // Past Action.timeout, and less than worker_timeout_s since the action's + // last update, so only Action.timeout could fire. + MockClock::advance(assigned + ACTION_TIMEOUT + Duration::from_secs(1)); + assert!( + !state_mgr.should_timeout_operation(&action).await, + "a stale worker's recently updated action must get the grace too" + ); + Ok(()) +} + +/// A Stale worker's action past `Action.timeout` plus the grace is timed out +/// for that reason, which is what the client is told. +#[nativelink_test] +async fn times_out_a_stale_worker_on_action_timeout_past_the_grace() -> Result<(), Error> { + MockClock::set_time(Duration::from_secs(NOW_TIME)); + let worker_id = WorkerId::from(String::from("connected-to-us")); + let action = action_with_timeout(&worker_id, make_system_time(0)); + + let registry = Arc::new(WorkerRegistry::new()); + registry + .register_worker(&worker_id, make_system_time(0)) + .await; + let state_mgr = state_manager_no_executing_ceiling(registry); + + MockClock::advance(PAST_GRACE); + assert_eq!( + state_mgr.timeout_cause(&action).await, + Some(action_timeout_cause()), + "a stale worker's action past Action.timeout plus the grace must name Action.timeout" + ); + Ok(()) +} + /// A heartbeating worker stuck on one action past the executing ceiling must /// still be timed out, which is what `max_action_executing_timeout_s` is for. #[nativelink_test] diff --git a/nativelink-scheduler/tests/simple_scheduler_test.rs b/nativelink-scheduler/tests/simple_scheduler_test.rs index 351474edae6..2df93f1625e 100644 --- a/nativelink-scheduler/tests/simple_scheduler_test.rs +++ b/nativelink-scheduler/tests/simple_scheduler_test.rs @@ -2571,7 +2571,9 @@ async fn worker_disconnect_loop_caps_at_max_worker_loss_retries_test() -> Result #[nativelink_test] async fn action_timeout_is_enforced_backend_side_test() -> Result<(), Error> { use nativelink_scheduler::awaited_action_db::AwaitedAction; - use nativelink_scheduler::simple_scheduler_state_manager::SimpleSchedulerStateManager; + use nativelink_scheduler::simple_scheduler_state_manager::{ + SimpleSchedulerStateManager, TimeoutCause, + }; // Anchor MockClock so MockInstantWrapped::now() == make_system_time(0). MockClock::set_time(Duration::from_secs(NOW_TIME)); @@ -2614,12 +2616,25 @@ async fn action_timeout_is_enforced_backend_side_test() -> Result<(), Error> { "Should not time out before Action.timeout elapses", ); - // Advance past the 2s per-action deadline. + // Past the 2s per-action deadline, but within the grace a live worker + // has to report its own DEADLINE_EXCEEDED result. MockClock::advance(Duration::from_secs(5)); assert!( - state_mgr.should_timeout_operation(&awaited_action).await, - "Scheduler must mark Executing action timed out once Action.timeout has elapsed", + !state_mgr.should_timeout_operation(&awaited_action).await, + "Should not time out within the grace after Action.timeout", + ); + + // Past Action.timeout plus the grace (no_event_action_timeout). + MockClock::advance(Duration::from_mins(1)); + + assert_eq!( + state_mgr.timeout_cause(&awaited_action).await, + Some(TimeoutCause::ActionTimeout { + timeout: Duration::from_secs(2), + grace: Duration::from_mins(1), + }), + "Scheduler must time out an Executing action once Action.timeout plus the grace has elapsed", ); Ok(()) diff --git a/nativelink-service/Cargo.toml b/nativelink-service/Cargo.toml index e7b5f799ebb..9910222b285 100644 --- a/nativelink-service/Cargo.toml +++ b/nativelink-service/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-service" -version = "1.7.2" +version = "1.7.3" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-service/src/worker_api_server.rs b/nativelink-service/src/worker_api_server.rs index 86239a37256..68b84dde2d4 100644 --- a/nativelink-service/src/worker_api_server.rs +++ b/nativelink-service/src/worker_api_server.rs @@ -303,19 +303,38 @@ impl WorkerConnection { }; background_spawn!("worker_api", async move { - let mut had_going_away = false; - while let Some(maybe_update) = connection.next().await { - let update = match maybe_update.map(|u| u.update) { - Ok(Some(update)) => update, - Ok(None) => { - tracing::warn!(worker_id=?instance.worker_id, "Empty update"); - continue; - } - Err(err) => { - tracing::warn!(worker_id=?instance.worker_id, ?err, "Error from worker"); + // The stream is read by a task of its own that only hands each + // message on; they are processed here, in order. Processing one + // inline in the reader (a result's store write, say) stopped the + // reads for its duration, and a worker that filled the stream's + // flow-control window in the meantime had nothing it was allowed + // to put on the wire: its keepalives waited behind the stall + // until this scheduler evicted it. The queue is unbounded on + // purpose: a worker's messages are bounded by its in-flight + // actions and its keepalive interval, and backpressure here is + // exactly the stall. + let (tx, mut rx) = mpsc::unbounded_channel(); + let reader_worker_id = instance.worker_id.clone(); + background_spawn!("worker_api_reader", async move { + while let Some(maybe_update) = connection.next().await { + let update = match maybe_update.map(|u| u.update) { + Ok(Some(update)) => update, + Ok(None) => { + tracing::warn!(worker_id=?reader_worker_id, "Empty update"); + continue; + } + Err(err) => { + tracing::warn!(worker_id=?reader_worker_id, ?err, "Error from worker"); + break; + } + }; + if tx.send(update).is_err() { break; } - }; + } + }); + let mut had_going_away = false; + while let Some(update) = rx.recv().await { let result = match update { Update::ConnectWorkerRequest(_connect_worker_request) => Err(make_err!( Code::Internal, diff --git a/nativelink-service/tests/worker_api_server_test.rs b/nativelink-service/tests/worker_api_server_test.rs index 4b63d264380..f50c7ac28d9 100644 --- a/nativelink-service/tests/worker_api_server_test.rs +++ b/nativelink-service/tests/worker_api_server_test.rs @@ -1176,3 +1176,70 @@ pub async fn a_full_channel_defers_the_kill_instead_of_evicting_test() ); Ok(()) } + +/// A message the scheduler is slow to process must not stop it reading the +/// worker's stream: the keepalives behind it still come off the wire (here, +/// a one-slot channel the test could not fill otherwise), and are taken in, +/// in order, once the slow one is done. Reading inline, the stream's +/// flow-control window filled behind a slow result and the worker's +/// keepalives never left it, so this scheduler evicted a worker that was +/// only waiting for it. +#[nativelink_test] +pub async fn a_slow_result_does_not_stop_the_stream_reads_test() +-> Result<(), Box> { + let mut test_context = setup_api_server(BASE_WORKER_TIMEOUT_S, Box::new(static_now_fn)).await?; + let operation_id = dispatch(&test_context, 1).await?; + let update_for_worker = test_context + .connection_worker_stream + .next() + .await + .expect("Worker stream ended early")? + .update + .expect("Expected update field to be populated"); + let update_for_worker::Update::StartAction(start_execute) = update_for_worker else { + panic!("Expected StartAction message"); + }; + assert_eq!(operation_id.to_string(), start_execute.operation_id); + + // The result's store write is not answered yet: processing it blocks. + test_context + .worker_stream + .send(Update::ExecuteResult(ExecuteResult { + instance_name: "instance_name".to_string(), + operation_id: operation_id.to_string(), + result: Some(execute_result::Result::InternalError( + make_err!(tonic::Code::Internal, "the action failed").into(), + )), + resource_usage: None, + })) + .await?; + + // The keepalives behind it are still read off the stream. + for free_memory_kb in 1..=8u64 { + tokio::time::timeout( + Duration::from_secs(5), + test_context + .worker_stream + .send(Update::KeepAliveRequest(KeepAliveRequest { + load: Some(WorkerLoad { free_memory_kb }), + })), + ) + .await + .map_err(|_| { + make_err!( + tonic::Code::DeadlineExceeded, + "the stream read stalled behind the result" + ) + })??; + } + + // Once the result is through, the keepalives are taken in after it. + let (operation_id_seen, worker_id, _) = test_context + .state_manager + .expect_update_operation(Ok(())) + .await; + assert_eq!(operation_id_seen, operation_id); + assert_eq!(worker_id, test_context.worker_id); + summary_after_report(&test_context, 8).await; + Ok(()) +} diff --git a/nativelink-store/Cargo.toml b/nativelink-store/Cargo.toml index 5cfee9301f2..6699ea8df8f 100644 --- a/nativelink-store/Cargo.toml +++ b/nativelink-store/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-store" -version = "1.7.2" +version = "1.7.3" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-store/src/fast_slow_store.rs b/nativelink-store/src/fast_slow_store.rs index 299d432552f..746858c5593 100644 --- a/nativelink-store/src/fast_slow_store.rs +++ b/nativelink-store/src/fast_slow_store.rs @@ -276,6 +276,19 @@ impl FastSlowStore { offset: u64, length: Option, ) -> Result<(), Error> { + // A `NotFound` for a digest key carries the digest as typed + // context, so the worker can report it as the missing input and the + // client re-uploads exactly that blob; see `ErrorContext`. + fn name_missing_digest(err: Error, key: &StoreKey<'_>) -> Error { + if let StoreKey::Digest(d) = key { + err.with_context(ErrorContext::MissingDigest { + hash: d.packed_hash().to_string(), + size: d.size_bytes().try_into().unwrap_or(i64::MAX), + }) + } else { + err + } + } let reader_stream_size = if self .slow_store .inner_store(Some(key.borrow())) @@ -294,20 +307,15 @@ impl FastSlowStore { .await .err_tip(|| "Failed to run has() on slow store")? .ok_or_else(|| { - let err = make_err!( - Code::NotFound, - "Object {} not found in either fast or slow store. \ - If using multiple workers, ensure all workers share the same CAS storage path.", - key.as_str() - ); - if let StoreKey::Digest(d) = key.borrow() { - err.with_context(ErrorContext::MissingDigest { - hash: d.packed_hash().to_string(), - size: d.size_bytes().try_into().unwrap_or(i64::MAX), - }) - } else { - err - } + name_missing_digest( + make_err!( + Code::NotFound, + "Object {} not found in either fast or slow store. \ + If using multiple workers, ensure all workers share the same CAS storage path.", + key.as_str() + ), + &key, + ) })? ) }; @@ -387,7 +395,12 @@ impl FastSlowStore { ) } Err(err) => match slow_res { - Err(slow_err) if slow_err.code == Code::NotFound => Err(slow_err), + // The slow store lost the blob under the read (an evicting + // tier behind a gRPC store, say): the same missing input as + // a blob that was never there. + Err(slow_err) if slow_err.code == Code::NotFound => { + Err(name_missing_digest(slow_err, &key)) + } _ => fast_res.merge(slow_res).merge(Err(err)), }, } @@ -811,9 +824,16 @@ impl StoreDriver for FastSlowStore { ) -> Result<(), Error> { // `has()` can report a stale map entry whose file is gone, so // get_part may still return NotFound; fall through to the slow - // store unless we have already streamed bytes to the caller. + // store. With bytes already streamed to the caller that is a resume: + // a digest names an immutable blob, so what the fast tier sent is a + // correct prefix and the slow store serves the rest from there (a + // Redis tier evicting the key under a read is the live case). Any + // other key could hold different content in each tier, so a short + // read there stays an error. // One existence check, reused: this is the hot read path and `has()` // on a filesystem fast store is a syscall. + let mut offset = offset; + let mut length = length; let in_fast_store = self.fast_store.has(key.borrow()).await?.is_some(); if !in_fast_store { record_store_tier_read("fast", "miss"); @@ -846,6 +866,28 @@ impl StoreDriver for FastSlowStore { warn!(%key, ?e, "Stale fast-store map entry; falling through to slow store"); // fall through to populate path } + Err(e) if e.code == Code::NotFound && matches!(key, StoreKey::Digest(_)) => { + let sent = writer.get_bytes_written() - bytes_before; + if let Some(wanted) = length { + let rest = wanted.saturating_sub(sent); + if rest == 0 { + return Ok(()); + } + length = Some(rest); + } + offset += sent; + self.metrics + .fast_store_short_reads_resumed + .fetch_add(1, Ordering::Acquire); + record_store_tier_read("fast", "short"); + warn!( + %key, + sent, + ?e, + "Fast store ended short; resuming from the slow store" + ); + // fall through to populate path, from the next byte + } Err(e) => return Err(e), } } @@ -1009,6 +1051,10 @@ struct FastSlowStoreMetrics { huge_blob_dedup_bypasses: AtomicU64, #[metric(help = "Stale fast-store map entries that fell through to the slow store")] fast_store_stale_map_falls_through: AtomicU64, + #[metric( + help = "Reads of a digest the fast store ended short, resumed from the slow store at the byte it stopped" + )] + fast_store_short_reads_resumed: AtomicU64, } /// Maximum time a follower will wait on the leader-populator before diff --git a/nativelink-store/src/filesystem_store.rs b/nativelink-store/src/filesystem_store.rs index 51459ca2acd..dfb7e0d099e 100644 --- a/nativelink-store/src/filesystem_store.rs +++ b/nativelink-store/src/filesystem_store.rs @@ -1518,7 +1518,7 @@ impl FilesystemStore { fn get_and_update_generation(&self) -> Result { self.next_generation - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |generation| { + .try_update(Ordering::Relaxed, Ordering::Relaxed, |generation| { generation.checked_add(1) }) .map(Generation::new) diff --git a/nativelink-store/tests/fast_slow_store_test.rs b/nativelink-store/tests/fast_slow_store_test.rs index d234f2ac804..e3f6c11f82a 100644 --- a/nativelink-store/tests/fast_slow_store_test.rs +++ b/nativelink-store/tests/fast_slow_store_test.rs @@ -23,7 +23,7 @@ use futures::future::join_all; use nativelink_config::stores::{ FastSlowSpec, FilesystemSpec, MemorySpec, NoopSpec, StoreDirection, StoreSpec, }; -use nativelink_error::{Code, Error, ResultExt, make_err}; +use nativelink_error::{Code, Error, ErrorContext, ResultExt, make_err}; use nativelink_macro::nativelink_test; use nativelink_metric::MetricsComponent; use nativelink_store::fast_slow_store::FastSlowStore; @@ -2019,9 +2019,12 @@ async fn get_part_falls_through_to_slow_on_stale_fast_map_entry() -> Result<(), Ok(()) } -/// `NotFound` after partial bytes must propagate, not retry (would corrupt). +/// `NotFound` after partial bytes of a digest is a resume: the bytes the +/// fast store sent are a correct prefix of an immutable blob, so the slow +/// store serves the rest from where the fast one stopped and the caller +/// sees the whole blob. #[nativelink_test] -async fn get_part_propagates_not_found_after_partial_fast_read() -> Result<(), Error> { +async fn get_part_resumes_from_slow_after_partial_fast_read() -> Result<(), Error> { let original_data = make_random_data(MEGABYTE_SZ); let digest = DigestInfo::try_new(VALID_HASH, original_data.len()).unwrap(); let (fast_slow_store, slow_store, fast_get_part_calls) = @@ -2031,19 +2034,156 @@ async fn get_part_propagates_not_found_after_partial_fast_read() -> Result<(), E .update_oneshot(digest, original_data.clone().into()) .await?; - let result = fast_slow_store.get_part_unchunked(digest, 0, None).await; + let served = fast_slow_store + .get_part_unchunked(digest, 0, None) + .await + .err_tip(|| "a short fast read of a digest must resume from the slow store")?; - let err = result.expect_err("partial fast read then NotFound must not fall through"); - assert_eq!( - err.code, - Code::NotFound, - "original NotFound must propagate, got: {err:?}" - ); + // The stale fast store sends sixteen zero bytes and stops; the slow + // store has to have served everything after them. + assert_eq!(served.len(), original_data.len()); + assert_eq!(&served[16..], &original_data[16..]); assert_eq!( fast_get_part_calls.load(Ordering::Acquire), 1, "fast store get_part must be attempted exactly once" ); + // A range read resumes within the range. + let served = fast_slow_store + .get_part_unchunked(digest, 100, Some(1000)) + .await?; + assert_eq!(served.len(), 1000); + assert_eq!(&served[16..], &original_data[116..1100]); + + Ok(()) +} + +/// A key that is not a digest could hold different content in each tier, +/// so a short read of one stays an error rather than splicing the tiers. +#[nativelink_test] +async fn get_part_propagates_not_found_after_partial_fast_read_of_a_plain_key() -> Result<(), Error> +{ + let original_data = make_random_data(MEGABYTE_SZ); + let key = StoreKey::new_str("some-action-cache-entry"); + let (fast_slow_store, slow_store, fast_get_part_calls) = + make_stores_with_stale_fast(original_data.len() as u64, 16); + + slow_store + .update_oneshot(key.borrow(), original_data.clone().into()) + .await?; + + let err = fast_slow_store + .get_part_unchunked(key.borrow(), 0, None) + .await + .expect_err("a short fast read of a plain key must not resume"); + assert_eq!(err.code, Code::NotFound, "{err:?}"); + assert_eq!(fast_get_part_calls.load(Ordering::Acquire), 1); + + Ok(()) +} + +/// A slow store that sends part of a blob and then loses it, the way an +/// evicting tier behind a gRPC store does. +#[derive(MetricsComponent)] +struct PartialThenGoneStore { + bytes_before_error: usize, +} + +#[async_trait] +impl StoreDriver for PartialThenGoneStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + + async fn has_with_results( + self: Pin<&Self>, + keys: &[StoreKey<'_>], + results: &mut [Option], + ) -> Result<(), Error> { + for (key, result) in keys.iter().zip(results.iter_mut()) { + *result = match key { + StoreKey::Digest(d) => Some(d.size_bytes()), + StoreKey::Str(_) => None, + }; + } + Ok(()) + } + + async fn update( + self: Pin<&Self>, + _key: StoreKey<'_>, + mut reader: DropCloserReadHalf, + _size_info: UploadSizeInfo, + ) -> Result { + reader.drain().await + } + + async fn get_part( + self: Pin<&Self>, + _key: StoreKey<'_>, + writer: &mut DropCloserWriteHalf, + _offset: u64, + _length: Option, + ) -> Result<(), Error> { + writer + .send(Bytes::from(vec![1u8; self.bytes_before_error])) + .await?; + Err(make_err!(Code::NotFound, "the key is gone under the read")) + } + + fn inner_store(&self, _digest: Option) -> &'_ dyn StoreDriver { + self + } + + fn as_any(&self) -> &(dyn core::any::Any + Sync + Send + 'static) { + self + } + + fn as_any_arc(self: Arc) -> Arc { + self + } + + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { + Ok(()) + } +} + +default_health_status_indicator!(PartialThenGoneStore); + +/// A `NotFound` the slow store raises under the read names the digest, the +/// way one for a blob that was never there does, so the worker reports the +/// missing input and the client re-uploads exactly that blob. +#[nativelink_test] +async fn slow_store_not_found_under_the_read_names_the_digest() -> Result<(), Error> { + let digest = DigestInfo::try_new(VALID_HASH, 4096).unwrap(); + let fast_slow_store = Store::new(FastSlowStore::new( + &FastSlowSpec { + fast: StoreSpec::Memory(MemorySpec::default()), + slow: StoreSpec::Memory(MemorySpec::default()), + fast_direction: StoreDirection::default(), + slow_direction: StoreDirection::default(), + bypass_dedup_threshold_bytes: 0, + }, + Store::new(MemoryStore::new(&MemorySpec::default())), + Store::new(Arc::new(PartialThenGoneStore { + bytes_before_error: 1024, + })), + )); + + let err = fast_slow_store + .get_part_unchunked(digest, 0, None) + .await + .expect_err("the slow store lost the blob under the read"); + assert_eq!(err.code, Code::NotFound, "{err:?}"); + assert_eq!( + err.context, + ErrorContext::MissingDigest { + hash: digest.packed_hash().to_string(), + size: 4096, + }, + "{err:?}" + ); + Ok(()) } diff --git a/nativelink-test/fuzz/Cargo.lock b/nativelink-test/fuzz/Cargo.lock index f1b80903e7b..b5da392017e 100644 --- a/nativelink-test/fuzz/Cargo.lock +++ b/nativelink-test/fuzz/Cargo.lock @@ -1022,7 +1022,7 @@ dependencies = [ [[package]] name = "nativelink-config" -version = "1.7.2" +version = "1.7.3" dependencies = [ "byte-unit", "humantime", @@ -1037,7 +1037,7 @@ dependencies = [ [[package]] name = "nativelink-error" -version = "1.7.2" +version = "1.7.3" dependencies = [ "base64", "mongodb", @@ -1069,7 +1069,7 @@ dependencies = [ [[package]] name = "nativelink-metric" -version = "1.7.2" +version = "1.7.3" dependencies = [ "async-lock", "nativelink-metric-macro-derive", @@ -1080,7 +1080,7 @@ dependencies = [ [[package]] name = "nativelink-metric-macro-derive" -version = "1.7.2" +version = "1.7.3" dependencies = [ "proc-macro2", "quote", @@ -1089,7 +1089,7 @@ dependencies = [ [[package]] name = "nativelink-proto" -version = "1.7.2" +version = "1.7.3" dependencies = [ "derive_more", "prost", diff --git a/nativelink-util/Cargo.toml b/nativelink-util/Cargo.toml index af810191aef..90a34b86a71 100644 --- a/nativelink-util/Cargo.toml +++ b/nativelink-util/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-util" -version = "1.7.2" +version = "1.7.3" [dependencies] nativelink-config = { path = "../nativelink-config" } @@ -105,7 +105,7 @@ opentelemetry-proto = { version = "0.32.0", default-features = false, features = "gen-tonic", "metrics", ] } -serial_test = { version = "3.2.0", features = [ +serial_test = { version = "4.0.0", features = [ "async", ], default-features = false } diff --git a/nativelink-worker/Cargo.toml b/nativelink-worker/Cargo.toml index 58184fc0791..01cef3d9e9f 100644 --- a/nativelink-worker/Cargo.toml +++ b/nativelink-worker/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-worker" -version = "1.7.2" +version = "1.7.3" [features] nix = [] @@ -74,7 +74,7 @@ pretty_assertions = { version = "1.4.1", features = [ "std", ], default-features = false } prost-types = { version = "0.14.4", default-features = false } -serial_test = { version = "3.2.0", features = [ +serial_test = { version = "4.0.0", features = [ "async", ], default-features = false } tempfile = { version = "3.15.0", default-features = false } diff --git a/nativelink-worker/src/local_worker.rs b/nativelink-worker/src/local_worker.rs index 0f93d6b86b3..79d1be494c1 100644 --- a/nativelink-worker/src/local_worker.rs +++ b/nativelink-worker/src/local_worker.rs @@ -110,6 +110,49 @@ fn reconnect_delay(attempt: u32) -> Duration { /// `cas_server.rs` must also be updated. const DEFAULT_ENDPOINT_TIMEOUT_S: f32 = 5.; +/// Keepalive defaults for the scheduler connection, the same as a store's +/// gRPC endpoint (`tls_utils::endpoint`). +const DEFAULT_TCP_KEEPALIVE: Duration = Duration::from_secs(30); +const DEFAULT_HTTP2_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(30); +const DEFAULT_HTTP2_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(20); + +/// A configured number of seconds, or the default when unset (0). +const fn seconds_or(configured: u64, default: Duration) -> Duration { + if configured > 0 { + Duration::from_secs(configured) + } else { + default + } +} + +/// Sends `GoingAway` within `deadline`. Logged rather than fatal: the stream +/// close that follows is what removes the worker, this only says why. +async fn send_going_away( + grpc_client: &mut T, + drain: bool, + deadline: Duration, +) -> Result<(), Error> { + match time::timeout(deadline, grpc_client.going_away(GoingAwayRequest { drain })).await { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => { + error!(?e, drain, "Failed to send GoingAwayRequest"); + Err(e.append("Failed to send GoingAwayRequest")) + } + Err(_) => { + error!( + drain, + deadline_s = deadline.as_secs_f32(), + "GoingAwayRequest did not complete; the scheduler connection is stalled" + ); + Err(make_err!( + Code::DeadlineExceeded, + "GoingAwayRequest did not complete within {}s", + deadline.as_secs_f32() + )) + } + } +} + /// Default maximum amount of time a task is allowed to run for. /// If this value gets modified the documentation in `cas_server.rs` must also be updated. const DEFAULT_MAX_ACTION_TIMEOUT: Duration = Duration::from_mins(20); @@ -139,7 +182,7 @@ struct LocalWorkerImpl<'a, T: WorkerApiClientTrait + 'static, U: RunningActionsM // always be zero if there are no actions running and no actions being waited // on by the scheduler. actions_in_transit: Arc, - accepted_action: AtomicBool, + accepted_action: Arc, /// The scheduler said it understands `ExecuteAccepted` and /// `ExecuteDeclined`. Without it the worker runs whatever it is sent, /// as every earlier release did. @@ -294,7 +337,7 @@ impl<'a, T: WorkerApiClientTrait + 'static, U: RunningActionsManager> LocalWorke // always be zero if there are no actions running and no actions being waited // on by the scheduler. actions_in_transit: Arc::new(AtomicU64::new(0)), - accepted_action: AtomicBool::new(false), + accepted_action: Arc::new(AtomicBool::new(false)), metrics, } } @@ -319,13 +362,47 @@ impl<'a, T: WorkerApiClientTrait + 'static, U: RunningActionsManager> LocalWorke /// Tells the scheduler the action is refused; only meaningful when it /// understands the message. - async fn decline(&self, operation_id: String, refusal: Refusal) -> Result<(), Error> { + /// The one message bounded by the endpoint timeout: `GoingAway` at + /// shutdown, so a stalled connection cannot hold a pod past its + /// termination grace. Everything else the worker sends rides on the + /// `ConnectWorker` stream unbounded, and the scheduler's liveness + /// timeout is the clock that evicts a worker it cannot hear. + fn going_away_deadline(&self) -> Duration { + Duration::from_secs_f32( + self.config + .worker_api_endpoint + .timeout + .unwrap_or(DEFAULT_ENDPOINT_TIMEOUT_S), + ) + } + + /// A message to the scheduler the run loop does not wait for inline: + /// it rides in the loop's future set, so the loop keeps reading its + /// stream while the message waits its turn on the one-slot channel + /// feeding the `ConnectWorker` stream. No deadline on it: a message + /// waits behind whatever the scheduler is still processing, and a + /// scheduler that stops hearing this worker closes the stream, which + /// the loop now sees. + fn scheduler_send( + &self, + what: &'static str, + send: impl Future> + Send + 'static, + ) -> BoxFuture<'static, Result<(), Error>> { + async move { send.await.err_tip(|| format!("Could not send {what}")) }.boxed() + } + + fn decline( + &self, + operation_id: String, + refusal: Refusal, + ) -> BoxFuture<'static, Result<(), Error>> { self.metrics.actions_declined.inc(); - self.grpc_client - .clone() - .execute_declined(refusal.into_declined(operation_id)) - .await - .err_tip(|| "Could not send ExecuteDeclined") + let mut grpc_client = self.grpc_client.clone(); + self.scheduler_send("ExecuteDeclined", async move { + grpc_client + .execute_declined(refusal.into_declined(operation_id)) + .await + }) } /// Starts a background spawn/thread that will send a message to the server every `timeout / 2`. @@ -412,11 +489,11 @@ impl<'a, T: WorkerApiClientTrait + 'static, U: RunningActionsManager> LocalWorke { // The action, CAS/AC uploads, cleanup, and execution_response // acknowledgment have all completed. This container is spent. - if let Err(err) = self - .grpc_client - .clone() - .going_away(GoingAwayRequest { drain: false }) - .await + // Bounded: a stalled connection must not keep a spent + // container from exiting. + let mut grpc_client = self.grpc_client.clone(); + if let Err(err) = + send_going_away(&mut grpc_client, false, self.going_away_deadline()).await { warn!(?err, "Could not unregister completed single-use worker"); } @@ -457,47 +534,62 @@ impl<'a, T: WorkerApiClientTrait + 'static, U: RunningActionsManager> LocalWorke if shutting_down || (self.config.single_use && self.accepted_action.load(Ordering::Acquire)) { if self.dispatch_ack { - self.decline(start_execute.operation_id, Refusal::ShuttingDown).await?; + futures.push(self.decline(start_execute.operation_id, Refusal::ShuttingDown)); } else if let Some(instance_name) = start_execute.execute_request.map(|request| request.instance_name) { - self.grpc_client.clone().execution_response( - ExecuteResult{ - instance_name, - operation_id: start_execute.operation_id, - result: Some(execute_result::Result::InternalError(make_err!(Code::ResourceExhausted, "Worker shutting down").into())), - resource_usage: None, - } - ).await?; + let mut grpc_client = self.grpc_client.clone(); + let operation_id = start_execute.operation_id; + futures.push(self.scheduler_send("ExecutionResponse", async move { + grpc_client.execution_response( + ExecuteResult{ + instance_name, + operation_id, + result: Some(execute_result::Result::InternalError(make_err!(Code::ResourceExhausted, "Worker shutting down").into())), + resource_usage: None, + } + ).await + })); } continue; } // Admission, then the acknowledgement: what the // scheduler charged on the send is confirmed or - // handed back before anything runs. A scheduler - // that does not speak the acknowledgement gets - // the old behaviour, run whatever arrives. - if self.dispatch_ack { + // handed back, and a declined action never runs. + // A scheduler that does not speak the + // acknowledgement gets the old behaviour, run + // whatever arrives. Neither message is awaited + // here: this loop is the only reader of the + // scheduler's stream, and a send that hangs on a + // stalled connection would keep it from ever + // seeing that stream close (the worker then sat + // evicted and idle until the provisioner retired + // it). The action starts once its acknowledgement + // has gone out, so the scheduler hears the + // acknowledgement before anything the run sends, + // and one that fails leaves the action unstarted. + let acknowledgement = if self.dispatch_ack { if let Some(refusal) = self.admission( &start_execute, actions_in_flight.load(Ordering::Acquire), ) { - self.decline(start_execute.operation_id, refusal).await?; + futures.push(self.decline(start_execute.operation_id, refusal)); continue; } - self.grpc_client - .clone() - .execute_accepted(ExecuteAccepted { - operation_id: start_execute.operation_id.clone(), - }) - .await - .err_tip(|| "Could not send ExecuteAccepted")?; - } - // Admitted: a single-use worker is spent from - // here. A decline above must not spend it, or - // every dispatch it turns away costs a pod. - if self.config.single_use { - self.accepted_action.store(true, Ordering::Release); - } + let mut grpc_client = self.grpc_client.clone(); + let operation_id = start_execute.operation_id.clone(); + Some(self.scheduler_send("ExecuteAccepted", async move { + grpc_client + .execute_accepted(ExecuteAccepted { operation_id }) + .await + })) + } else { + None + }; + // Admitted: a single-use worker is spent by this + // action once it is acknowledged. A decline above + // must not spend it, or every dispatch it turns + // away costs a pod. + let spent_mark = self.config.single_use.then(|| self.accepted_action.clone()); self.metrics.start_actions_received.inc(); @@ -636,12 +728,19 @@ impl<'a, T: WorkerApiClientTrait + 'static, U: RunningActionsManager> LocalWorke ?e, "Missing CAS inputs during prepare_action, returning FAILED_PRECONDITION" ); + // The context names the digest; the + // execute response turns it into the + // PreconditionFailure detail Bazel + // re-uploads and retries on. let action_result = ActionResult { - error: Some(make_err!( - Code::FailedPrecondition, - "{}", - e.message_string() - )), + error: Some( + make_err!( + Code::FailedPrecondition, + "{}", + e.message_string() + ) + .with_context(e.context.clone()), + ), ..ActionResult::default() }; let action_stage = ActionStage::Completed(action_result); @@ -667,46 +766,69 @@ impl<'a, T: WorkerApiClientTrait + 'static, U: RunningActionsManager> LocalWorke let add_future_channel = add_future_channel.clone(); - info_span!( - "worker_start_action_ctx", - operation_id = operation_id_to_log, - digest_function = %digest_hasher.to_string(), - ).in_scope(|| { - let _guard = Context::current_with_value(digest_hasher) - .attach(); - - let actions_in_flight = actions_in_flight.clone(); - let actions_notify = actions_notify.clone(); - let actions_in_flight_fail = actions_in_flight.clone(); - let actions_notify_fail = actions_notify.clone(); - actions_in_flight.fetch_add(1, Ordering::Release); - - futures.push( - spawn!("worker_start_action", start_action_fut).map(move |res| { - let res = res.err_tip(|| "Failed to launch spawn")?; - if let Err(err) = &res { - error!(?err, "Error executing action"); - } - add_future_channel - .send(make_publish_future(res).then(move |res| { - actions_in_flight.fetch_sub(1, Ordering::Release); - actions_notify.notify_one(); - core::future::ready(res) - }).boxed()) - .map_err(|err| - Error::from_std_err(Code::Internal, &err).append("LocalWorker could not send future") - )?; - Ok(()) - }) - .or_else(move |err| { - // If the make_publish_future is not run we still need to notify. - actions_in_flight_fail.fetch_sub(1, Ordering::Release); - actions_notify_fail.notify_one(); - core::future::ready(Err(err)) - }) - .boxed() + // Boxed so the launch closure captures one + // erased type, not the whole action future, and + // the compiler's Send check does not overflow. + let start_action_fut: BoxFuture<'static, Result> = + start_action_fut.boxed(); + let launch = { + let span = info_span!( + "worker_start_action_ctx", + operation_id = operation_id_to_log, + digest_function = %digest_hasher.to_string(), ); - }); + move || { + span.in_scope(|| { + let _guard = Context::current_with_value(digest_hasher) + .attach(); + spawn!("worker_start_action", start_action_fut) + }) + } + }; + + let actions_in_flight = actions_in_flight.clone(); + let actions_notify = actions_notify.clone(); + let actions_in_flight_fail = actions_in_flight.clone(); + let actions_notify_fail = actions_notify.clone(); + // Counted from the dispatch, so the next one is + // admitted against this slot while the + // acknowledgement is still on its way. + actions_in_flight.fetch_add(1, Ordering::Release); + + futures.push( + async move { + if let Some(acknowledgement) = acknowledgement { + acknowledgement.await?; + } + if let Some(accepted_action) = spent_mark { + accepted_action.store(true, Ordering::Release); + } + launch().await.err_tip(|| "Failed to launch spawn") + } + .map(move |res| { + let res = res?; + if let Err(err) = &res { + error!(?err, "Error executing action"); + } + add_future_channel + .send(make_publish_future(res).then(move |res| { + actions_in_flight.fetch_sub(1, Ordering::Release); + actions_notify.notify_one(); + core::future::ready(res) + }).boxed()) + .map_err(|err| + Error::from_std_err(Code::Internal, &err).append("LocalWorker could not send future") + )?; + Ok(()) + }) + .or_else(move |err| { + // If the make_publish_future is not run we still need to notify. + actions_in_flight_fail.fetch_sub(1, Ordering::Release); + actions_notify_fail.notify_one(); + core::future::ready(Err(err)) + }) + .boxed() + ); } } }, @@ -723,21 +845,28 @@ impl<'a, T: WorkerApiClientTrait + 'static, U: RunningActionsManager> LocalWorke let actions_in_flight = actions_in_flight.clone(); let actions_notify = actions_notify.clone(); let drain_on_shutdown = self.config.drain_on_shutdown; + // The GoingAway messages below are bounded: a stalled + // connection must not hold the pod past its grace + // period. One that fails or misses is logged and the + // shutdown goes on, the in-flight actions still drained; + // the scheduler removes this worker when the stream + // closes either way. + let going_away_deadline = self.going_away_deadline(); let drain_deadline = if self.config.max_action_timeout_s == 0 { DEFAULT_MAX_ACTION_TIMEOUT } else { Duration::from_secs(self.config.max_action_timeout_s as u64) }; let shutdown_future = async move { - if drain_on_shutdown { - // Say so first, so nothing new is dispatched here - // while the running actions finish; the scheduler - // removes this worker when the stream closes. - if let Err(e) = grpc_client.going_away(GoingAwayRequest { drain: true }).await { - error!("Failed to send GoingAwayRequest: {e}",); - return Err(e); - } - } + // Said first on a drain, so nothing new is dispatched + // here while the running actions finish; the + // scheduler removes this worker when the stream + // closes. + let mut going_away_result = if drain_on_shutdown { + send_going_away(&mut grpc_client, true, going_away_deadline).await + } else { + Ok(()) + }; // Wait for in-flight operations to be fully completed, // for as long as one action is allowed to run. let wait = async { @@ -754,14 +883,11 @@ impl<'a, T: WorkerApiClientTrait + 'static, U: RunningActionsManager> LocalWorke if !drain_on_shutdown { // Sending this message immediately evicts all jobs from // this worker, of which there should be none. - if let Err(e) = grpc_client.going_away(GoingAwayRequest { drain: false }).await { - error!("Failed to send GoingAwayRequest: {e}",); - return Err(e); - } + going_away_result = send_going_away(&mut grpc_client, false, going_away_deadline).await; } // Allow shutdown to occur now. drop(shutdown_guard); - Ok::<(), Error>(()) + going_away_result }; futures.push(shutdown_future.boxed()); shutting_down = true; @@ -1163,7 +1289,23 @@ pub async fn new_local_worker( .append("Invalid URI for worker endpoint") })? .connect_timeout(timeout_duration) - .timeout(timeout_duration); + .timeout(timeout_duration) + // Probes under and between the messages, so a + // connection that has gone dead underneath fails + // here instead of hanging every message on it. + .tcp_keepalive(Some(seconds_or( + config.worker_api_endpoint.tcp_keepalive_s, + DEFAULT_TCP_KEEPALIVE, + ))) + .http2_keep_alive_interval(seconds_or( + config.worker_api_endpoint.http2_keepalive_interval_s, + DEFAULT_HTTP2_KEEPALIVE_INTERVAL, + )) + .keep_alive_timeout(seconds_or( + config.worker_api_endpoint.http2_keepalive_timeout_s, + DEFAULT_HTTP2_KEEPALIVE_TIMEOUT, + )) + .keep_alive_while_idle(true); let transport = endpoint.connect().await.map_err(|e| { Error::from_std_err(Code::Internal, &e).append(format!( diff --git a/nativelink-worker/tests/local_worker_test.rs b/nativelink-worker/tests/local_worker_test.rs index f9f4dc23fc1..8e071845f55 100644 --- a/nativelink-worker/tests/local_worker_test.rs +++ b/nativelink-worker/tests/local_worker_test.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use core::sync::atomic::Ordering; use core::time::Duration; use std::collections::HashMap; #[cfg(target_family = "unix")] @@ -34,7 +35,7 @@ use nativelink_config::cas_server::{EndpointConfig, LocalWorkerConfig, WorkerPro use nativelink_config::stores::{ FastSlowSpec, FilesystemSpec, MemorySpec, StoreDirection, StoreSpec, }; -use nativelink_error::{Code, Error, make_err, make_input_err}; +use nativelink_error::{Code, Error, ErrorContext, make_err, make_input_err}; use nativelink_macro::nativelink_test; use nativelink_proto::build::bazel::remote::execution::v2::Platform; use nativelink_proto::build::bazel::remote::execution::v2::platform::Property; @@ -541,17 +542,14 @@ async fn start_action_lifecycle_test(single_use: bool) -> Result<(), Error> { assert_eq!(stored_result, action_result.clone()); assert_eq!(digest_hasher, DigestHasherFunc::Sha256); assert_eq!( - test_context - .client - .going_away_count - .load(std::sync::atomic::Ordering::Relaxed), + test_context.client.going_away_count.load(Ordering::Relaxed), 0 ); assert_eq!( test_context .client .execution_complete_count - .load(std::sync::atomic::Ordering::Relaxed), + .load(Ordering::Relaxed), u64::from(!single_use) ); @@ -576,7 +574,7 @@ async fn start_action_lifecycle_test(single_use: bool) -> Result<(), Error> { tokio::time::timeout(Duration::from_secs(5), test_context.finish()) .await .map_err(|_| make_input_err!("Single-use worker did not exit"))??; - assert_eq!(going_away.load(std::sync::atomic::Ordering::Relaxed), 1); + assert_eq!(going_away.load(Ordering::Relaxed), 1); } Ok(()) @@ -1001,7 +999,11 @@ async fn cas_not_found_returns_failed_precondition_test() -> Result<(), Error> { // input: NotFound tagged with the missing-input tip, whatever the store // itself said. let missing_input = make_err!(Code::NotFound, "Hash 0123456789abcdef not found") - .append(MISSING_INPUT_ERROR_TIP); + .append(MISSING_INPUT_ERROR_TIP) + .with_context(ErrorContext::MissingDigest { + hash: "0123456789abcdef".to_string(), + size: 42, + }); running_action .expect_prepare_action(Err(missing_input.clone())) .await?; @@ -1013,12 +1015,17 @@ async fn cas_not_found_returns_failed_precondition_test() -> Result<(), Error> { // NOT an InternalError. This allows Bazel to re-upload the missing artifacts. let execution_response = test_context.client.expect_execution_response(Ok(())).await; + // The digest rides along as context, which the execute response turns + // into the PreconditionFailure detail Bazel re-uploads on. let expected_action_result = ActionResult { - error: Some(make_err!( - Code::FailedPrecondition, - "{}", - missing_input.message_string() - )), + error: Some( + make_err!( + Code::FailedPrecondition, + "{}", + missing_input.message_string() + ) + .with_context(missing_input.context.clone()), + ), ..ActionResult::default() }; assert_eq!( @@ -1213,6 +1220,300 @@ async fn keep_alive_fail_logs() -> Result<(), Error> { )) } +/// A keepalive that never completes does not end the connection by itself: +/// a send waits behind whatever the scheduler is still processing, and the +/// scheduler's liveness timeout is the clock. When the scheduler closes the +/// stream, the loop sees it, the running actions are killed, and the worker +/// registers again. +#[nativelink_test] +async fn a_hung_keepalive_waits_for_the_scheduler_to_close_the_stream() -> Result<(), Error> { + let local_worker_config = LocalWorkerConfig { + platform_properties: HashMap::new(), + worker_api_endpoint: EndpointConfig { + timeout: Some(0.2), + ..Default::default() + }, + ..Default::default() + }; + + let mut test_context = setup_local_worker_with_config(local_worker_config).await; + test_context + .client + .keep_alive_hangs + .store(true, Ordering::Release); + let streaming_response = test_context.maybe_streaming_response.take().unwrap(); + test_context + .client + .expect_connect_worker(Ok(streaming_response)) + .await; + let tx_stream = test_context.maybe_tx_stream.take().unwrap(); + tx_stream + .send(Frame::data( + encode_stream_proto(&UpdateForWorker { + update: Some(Update::ConnectionResult(ConnectionResult { + worker_id: "foobar".to_string(), + dispatch_ack: false, + memory_property: String::new(), + })), + }) + .unwrap(), + )) + .await + .map_err(|e| make_input_err!("Could not send : {:?}", e))?; + + // The first keepalive is held and stays held: no deadline on it ends the + // connection. + while test_context + .client + .keep_alives_hanging + .load(Ordering::Acquire) + == 0 + { + sleep(Duration::from_millis(10)).await; + } + assert!( + tokio::time::timeout( + Duration::from_millis(500), + test_context.actions_manager.expect_kill_all() + ) + .await + .is_err(), + "a hung keepalive must not end the connection" + ); + + // The scheduler closes the stream (it evicted the worker): the loop sees + // it, the actions are killed, and the worker comes back to register. + drop(tx_stream); + tokio::time::timeout(Duration::from_secs(10), async { + test_context.actions_manager.expect_kill_all().await; + let (tx_stream2, streaming_response2) = setup_grpc_stream(); + test_context + .client + .expect_connect_worker(Ok(streaming_response2)) + .await; + drop(tx_stream2); + }) + .await + .map_err(|_| make_err!(Code::DeadlineExceeded, "the worker did not reconnect"))?; + assert!(logs_contain( + "Worker disconnected from scheduler, reconnecting" + )); + Ok(()) +} + +/// The run loop is the only reader of the scheduler's stream, so no call to +/// the scheduler is awaited inside it: a dispatch acknowledgement that hangs +/// leaves the loop reading, and a kill that arrives next is still acted on. +#[nativelink_test] +async fn a_hung_acknowledgement_does_not_block_the_loop() -> Result<(), Error> { + let mut test_context = setup_local_worker_with_config(LocalWorkerConfig { + max_inflight_tasks: 2, + worker_api_endpoint: EndpointConfig { + timeout: Some(10000.), + ..Default::default() + }, + ..Default::default() + }) + .await; + let streaming_response = test_context.maybe_streaming_response.take().unwrap(); + test_context + .client + .expect_connect_worker(Ok(streaming_response)) + .await; + + let worker_id = "foobar".to_string(); + let tx_stream = test_context.maybe_tx_stream.take().unwrap(); + tx_stream + .send(Frame::data( + encode_stream_proto(&UpdateForWorker { + update: Some(Update::ConnectionResult(ConnectionResult { + worker_id: worker_id.clone(), + dispatch_ack: true, + memory_property: String::new(), + })), + }) + .unwrap(), + )) + .await + .map_err(|e| make_input_err!("Could not send : {:?}", e))?; + + let action_info = ActionInfo { + command_digest: DigestInfo::new([1u8; 32], 10), + input_root_digest: DigestInfo::new([2u8; 32], 10), + timeout: Duration::from_secs(1), + platform_properties: HashMap::new(), + priority: 0, + load_timestamp: SystemTime::UNIX_EPOCH, + insert_timestamp: SystemTime::UNIX_EPOCH, + unique_qualifier: ActionUniqueQualifier::Uncacheable(ActionUniqueKey { + execution_scope: None, + instance_name: INSTANCE_NAME.to_string(), + digest_function: DigestHasherFunc::Blake3, + digest: DigestInfo::new([3u8; 32], 10), + }), + }; + // The dispatch's acknowledgement is never answered: the mock holds it. + tx_stream + .send(Frame::data( + encode_stream_proto(&UpdateForWorker { + update: Some(Update::StartAction(StartExecute { + request_metadata: None, + execute_request: Some((&action_info).into()), + operation_id: "first".to_string(), + queued_timestamp: None, + platform: Some(Platform::default()), + worker_id: worker_id.clone(), + })), + }) + .unwrap(), + )) + .await + .map_err(|e| make_input_err!("Could not send : {:?}", e))?; + + // A kill for it arrives while the acknowledgement hangs; the loop reads + // it and passes it on. + tx_stream + .send(Frame::data( + encode_stream_proto(&UpdateForWorker { + update: Some(Update::KillOperationRequest(KillOperationRequest { + operation_id: "first".to_string(), + })), + }) + .unwrap(), + )) + .await + .map_err(|e| make_input_err!("Could not send : {:?}", e))?; + let killed = tokio::time::timeout( + Duration::from_secs(10), + test_context.actions_manager.expect_kill_operation(), + ) + .await + .map_err(|_| make_err!(Code::DeadlineExceeded, "the loop never read the kill"))?; + assert_eq!(killed, OperationId::from("first")); + // The action waits on its acknowledgement; it has not started. + assert!( + tokio::time::timeout( + Duration::from_millis(300), + test_context + .actions_manager + .expect_create_and_add_action_no_reply() + ) + .await + .is_err(), + "the action must not start before its acknowledgement is sent" + ); + Ok(()) +} + +/// An acknowledgement that fails leaves the action unstarted and a +/// single-use worker unspent: the loop ends on the error, the worker +/// registers again, and the next dispatch runs. +#[nativelink_test] +async fn a_failed_acknowledgement_leaves_a_single_use_worker_unspent() -> Result<(), Error> { + let mut test_context = setup_local_worker_with_config(LocalWorkerConfig { + single_use: true, + max_inflight_tasks: 1, + worker_api_endpoint: EndpointConfig { + timeout: Some(10000.), + ..Default::default() + }, + ..Default::default() + }) + .await; + let streaming_response = test_context.maybe_streaming_response.take().unwrap(); + test_context + .client + .expect_connect_worker(Ok(streaming_response)) + .await; + + let worker_id = "foobar".to_string(); + let connection_result = UpdateForWorker { + update: Some(Update::ConnectionResult(ConnectionResult { + worker_id: worker_id.clone(), + dispatch_ack: true, + memory_property: String::new(), + })), + }; + let tx_stream = test_context.maybe_tx_stream.take().unwrap(); + tx_stream + .send(Frame::data( + encode_stream_proto(&connection_result).unwrap(), + )) + .await + .map_err(|e| make_input_err!("Could not send : {:?}", e))?; + + let action_info = ActionInfo { + command_digest: DigestInfo::new([1u8; 32], 10), + input_root_digest: DigestInfo::new([2u8; 32], 10), + timeout: Duration::from_secs(1), + platform_properties: HashMap::new(), + priority: 0, + load_timestamp: SystemTime::UNIX_EPOCH, + insert_timestamp: SystemTime::UNIX_EPOCH, + unique_qualifier: ActionUniqueQualifier::Uncacheable(ActionUniqueKey { + execution_scope: None, + instance_name: INSTANCE_NAME.to_string(), + digest_function: DigestHasherFunc::Blake3, + digest: DigestInfo::new([3u8; 32], 10), + }), + }; + let start = |operation_id: &str| UpdateForWorker { + update: Some(Update::StartAction(StartExecute { + request_metadata: None, + execute_request: Some((&action_info).into()), + operation_id: operation_id.to_string(), + queued_timestamp: None, + platform: Some(Platform::default()), + worker_id: worker_id.clone(), + })), + }; + + // The acknowledgement of the first dispatch is lost. + tx_stream + .send(Frame::data(encode_stream_proto(&start("first")).unwrap())) + .await + .map_err(|e| make_input_err!("Could not send : {:?}", e))?; + let accepted = test_context + .client + .expect_execute_accepted(Err(make_err!(Code::Unavailable, "acknowledgement lost"))) + .await; + assert_eq!(accepted.operation_id, "first"); + + // The loop ends on it and the worker, not spent, registers again. + let tx_stream2 = tokio::time::timeout(Duration::from_secs(10), async { + test_context.actions_manager.expect_kill_all().await; + let (tx_stream2, streaming_response2) = setup_grpc_stream(); + test_context + .client + .expect_connect_worker(Ok(streaming_response2)) + .await; + tx_stream2 + }) + .await + .map_err(|_| make_err!(Code::DeadlineExceeded, "the worker did not reconnect"))?; + tx_stream2 + .send(Frame::data( + encode_stream_proto(&connection_result).unwrap(), + )) + .await + .map_err(|e| make_input_err!("Could not send : {:?}", e))?; + + // The next dispatch is acknowledged and is the first action to start. + tx_stream2 + .send(Frame::data(encode_stream_proto(&start("second")).unwrap())) + .await + .map_err(|e| make_input_err!("Could not send : {:?}", e))?; + let accepted = test_context.client.expect_execute_accepted(Ok(())).await; + assert_eq!(accepted.operation_id, "second"); + let (_, started) = test_context + .actions_manager + .expect_create_and_add_action(Ok(Arc::new(MockRunningAction::new()))) + .await; + assert_eq!(started.operation_id, "second"); + assert!(logs_contain("Could not send ExecuteAccepted")); + Ok(()) +} + /// Regression test: a disconnect from the scheduler while an action is still /// "in transit" (`StartAction` received, inputs still downloading) must lead to /// kill-all + reconnect like any other disconnect. It used to strand the diff --git a/nativelink-worker/tests/utils/local_worker_test_utils.rs b/nativelink-worker/tests/utils/local_worker_test_utils.rs index e143a314628..1f7c10f80f2 100644 --- a/nativelink-worker/tests/utils/local_worker_test_utils.rs +++ b/nativelink-worker/tests/utils/local_worker_test_utils.rs @@ -79,6 +79,10 @@ pub(crate) struct MockWorkerApiClient { rx_resp: Arc>>, tx_resp: mpsc::UnboundedSender, keep_alives_count: u8, + /// A keepalive that never completes: a send waiting its turn on a + /// stalled connection. `keep_alives_hanging` counts the ones held. + pub keep_alive_hangs: Arc, + pub keep_alives_hanging: Arc, pub going_away_count: Arc, pub going_away_drain: Arc, pub execution_complete_count: Arc, @@ -94,6 +98,8 @@ impl MockWorkerApiClient { rx_resp: Arc::new(Mutex::new(rx_resp)), tx_resp, keep_alives_count: 0, + keep_alive_hangs: Arc::new(std::sync::atomic::AtomicBool::new(false)), + keep_alives_hanging: Arc::new(AtomicU64::new(0)), going_away_count: Arc::new(AtomicU64::new(0)), going_away_drain: Arc::new(std::sync::atomic::AtomicBool::new(false)), execution_complete_count: Arc::new(AtomicU64::new(0)), @@ -212,6 +218,10 @@ impl WorkerApiClientTrait for MockWorkerApiClient { async fn keep_alive(&mut self, _request: KeepAliveRequest) -> Result<(), Error> { debug!("Got KeepAlive"); + if self.keep_alive_hangs.load(Ordering::Acquire) { + self.keep_alives_hanging.fetch_add(1, Ordering::AcqRel); + core::future::pending::<()>().await; + } if self.keep_alives_count == 0 { self.keep_alives_count += 1; Ok(()) diff --git a/src/bin/nativelink.rs b/src/bin/nativelink.rs index 55022b1ee1f..bc37f9146d4 100644 --- a/src/bin/nativelink.rs +++ b/src/bin/nativelink.rs @@ -12,6 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +// The worker's action future is a long chain of combinators; proving it +// Send for the spawn walks the whole chain, and the default limit of 128 is +// a few steps short of it on the sanitizer toolchain. +#![recursion_limit = "256"] + use core::net::SocketAddr; use core::time::Duration; use std::collections::{HashMap, HashSet}; diff --git a/web/apps/docs/content/docs/reference/nativelink-config/index.mdx b/web/apps/docs/content/docs/reference/nativelink-config/index.mdx index 999e3e05510..a5b358d4ee2 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/index.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/index.mdx @@ -5,14 +5,14 @@ full: true --- {/* AUTOGENERATED โ€” do not edit by hand. - Source: nativelink-config @ v1.7.2 (daee0a8d) + Source: nativelink-config @ v1.7.3 (521838cd) Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} -This is the canonical NativeLink configuration reference for **v1.7.2**. +This is the canonical NativeLink configuration reference for **v1.7.3**. It is autogenerated from the Rust config crate -([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.7.2/nativelink-config/src)) via the `build-schema` binary, so +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.7.3/nativelink-config/src)) via the `build-schema` binary, so it can never drift from what the binary actually deserializes. ## Top-level fields @@ -918,23 +918,34 @@ Configuration for `ExperimentalMongoDB` store. | `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | โ€” | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | | `max_action_timeout_s` | integer (uint) | โ€” | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | | `max_upload_timeout_s` | integer (uint) | โ€” | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_download_timeout_s` | integer (uint) | โ€” | 10 minutes | Maximum time allowed for fetching an action's inputs into the worker's store and its directory before the command runs. A fetch that hangs (a store that never answers) otherwise holds the action's slot for good while the worker looks healthy; past this the action fails with `DeadlineExceeded` and may be retried by the scheduler, and a warning names the action every minute before that. Value in seconds. | | `max_cleanup_wait_s` | integer (uint) | โ€” | 30 seconds | Maximum time to wait for action directory cleanup before timing out. Value in seconds. | | `max_cleanup_backoff_ms` | integer (uint) | โ€” | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. Value in milliseconds. | | `max_inflight_tasks` | integer (uint64) | โ€” | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | | `single_use` | boolean | โ€” | false | Accept one action, upload its results, and exit the worker process. The worker advertises one execution slot and never accepts a second action, including after a scheduler disconnect. A launcher must create a fresh container and fresh writable volumes for its replacement. Use one local worker per process, with no scheduler or services other than health checks. Reusing the container or its writable volumes does not provide isolation from the previous action. | | `experimental_buck2_file_capture` | [Buck2FileCaptureConfig](#buck2filecaptureconfig) | โ€” | โ€” | Capture full execution-container files for Buck2 builds only. Disabled when absent. Requires `single_use`; read the capture configuration's isolation requirements before enabling it for a worker pool. | | `timeout_handled_externally` | boolean | โ€” | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | +| `drain_on_shutdown` | boolean | โ€” | false | On shutdown, tell the scheduler at once that this worker is draining so it stops dispatching here, then finish the running actions (up to `max_action_timeout_s`) and exit. Off, the worker finishes first and only then announces itself, while the scheduler keeps offering it work it has to refuse. Needs a scheduler that understands the drain flag (v1.7.3+); an older one removes the worker and requeues its actions as soon as the message arrives. | +| `resource_enforcement` | [ResourceEnforcementConfig](#resourceenforcementconfig) | โ€” | โ€” | Kill an action that grows past its memory reservation before the pod's cgroup limit does. On cgroup v2 the container cgroup carries `memory.oom.group`, so the kernel's kill takes the worker and every action on it; this one takes only the offender and fails it with `FailedPrecondition`. Linux only; accepted and ignored elsewhere. Actions run through a persistent worker are not covered, since that process outlives the action and serves others. With either axis on, the worker reads an action's platform properties from the scheduler's `StartExecute` rather than the client's request, so a reservation the scheduler placed the action by is the one enforced and exported to the environment. | | `entrypoint` | string | โ€” | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | | `experimental_precondition_script` | string | โ€” | โ€” | An optional script to run before every action is processed on the worker. The value should be the full path to the script to execute and will pause all actions on the worker if it returns an exit code other than 0. If not set, then the worker will never pause and will continue to accept jobs according to the scheduler configuration. This is useful, for example, if the worker should not take any more actions until there is enough resource available on the machine to handle them. | +| `precondition_timeout_ms` | integer (uint64) | โ€” | 30000 | Milliseconds the precondition script may run before it counts as failed and the action is refused as backpressure. A script that hangs used to hold the action forever. | +| `max_captured_output_bytes` | integer (uint64) | โ€” | 0 | Bytes of an action's stdout or stderr kept in memory. Past this the output spills to a file under the action directory and is uploaded from disk, so an action that prints gigabytes does not grow the worker by that much. 0 keeps everything in memory. | +| `orphan_sweep_interval_s` | integer (uint64) | โ€” | 0 | Seconds between sweeps of `work_directory` for action directories no running action owns, left by a cleanup that failed. 0 disables the sweep; the directory is still purged at startup. | +| `kill_grace_ms` | integer (uint64) | โ€” | 5000 | Milliseconds between the SIGTERM a timed-out or cancelled action receives and the SIGKILL that follows if it is still running, so a tool that handles SIGTERM can write its own cleanup. The memory reservation kill skips the grace and sends SIGKILL at once. Linux only; elsewhere the kill is immediate. 0 takes the default. | +| `set_tmpdir` | boolean | โ€” | true | Give every action its own `TMPDIR` under its action directory, removed with it, so concurrent actions writing the same file name under `$TMPDIR` no longer collide. A `TMPDIR` in the action's own environment still wins. Code that hard-codes `/tmp` is unaffected. | +| `persistent_workers` | [PersistentWorkersConfig](#persistentworkersconfig) | โ€” | โ€” | Bazel persistent workers: how an action whose platform properties carry `supports-workers=1` is run and how the pool of worker processes is sized. Unset takes every default below. | | `cas_fast_slow_store` | string | Yes | โ€” | Underlying CAS store that the worker will use to download CAS artifacts. This store must be a `FastSlowStore`. The `fast` store must be a `FileSystemStore` because it will use hardlinks when building out the files instead of copying the files. The slow store must eventually resolve to the same store the scheduler/client uses to send job requests. | | `upload_action_result` | [UploadActionResultConfig](#uploadactionresultconfig) | โ€” | โ€” | Configuration for uploading action results. | | `work_directory` | string | Yes | โ€” | The directory work jobs will be executed from. This directory will be fully managed by the worker service and will be purged on startup. This directory and the directory referenced in `local_filesystem_store_ref`'s `stores::FilesystemStore::content_path` must be on the same filesystem. Hardlinks will be used when placing files that are accessible to the jobs that are sourced from `local_filesystem_store_ref`'s `content_path`. | | `platform_properties` | map of string to [WorkerProperty](#workerproperty) | Yes | โ€” | Properties of this worker. This configuration will be sent to the scheduler and used to tell the scheduler to restrict what should be executed on this worker. | +| `capacity` | [CapacityConfig](#capacityconfig) | โ€” | โ€” | Derive the CPU and memory properties from the worker's own cgroup limits instead of the values above; see `CapacityConfig`. When the cgroup cannot be read (not Linux, cgroup v1, no permission) the values above stand and a warning says so; if they carry neither property the worker refuses to start rather than register unable to take any action that asks for CPU or memory. | | `additional_environment` | map of string to [EnvironmentSource](#environmentsource) | โ€” | โ€” | An optional mapping of environment names to set for the execution as well as those specified in the action itself. If set, will set each key as an environment variable before executing the job with the value of the environment variable being the value of the property of the action being executed of that name or the fixed value. | | `directory_cache` | [DirectoryCacheConfig](#directorycacheconfig) | โ€” | โ€” | Optional directory cache configuration for improving performance by caching reconstructed input directories and using hardlinks instead of rebuilding them from CAS for every action. | | `experimental_active_input_leases` | boolean | โ€” | false (eviction behavior is unchanged) | Optional and experimental: lease every digest in an active action's input Merkle closure in the worker's locally eviction-managed CAS tiers (the `cas_fast_slow_store`'s filesystem-backed stores) until the action has finished cleanup. This prevents `Lost inputs no longer available remotely` failures caused by local fast-tier eviction while inputs are being materialized under cache pressure. | | `use_namespaces` | boolean | โ€” | False | Whether to use namespaces to isolate the execution. This is only available on Linux. It is highly recommended as it avoids a number of issues with zombie processes and also provides additional hermeticity. If explicitly set to true and it is not supported the worker will exit with an error. | | `use_mount_namespace` | boolean | โ€” | False | Whether to use a mount namespace to isolate the worker root. This is only available on Linux and when `use_namespaces` is true. It is highly recommended provides additional hermeticity. If explicitly set to true and it is not supported or `use_namespaces` is not set to true the worker will exit with an error. | +| `isolate_tmp` | boolean | โ€” | True when `use_mount_namespace` is true and `/tmp` exists, | Whether to give each action a private `/tmp` inside its mount namespace: the action's own `tmp` directory (the one `TMPDIR` points at) is bound over `/tmp`, so tools that hardcode `/tmp` write there too. Concurrent actions no longer collide on predictable paths under `/tmp`, nothing leaks between actions through it, and what an action writes there is removed with the action. | ## SimpleSpec @@ -944,12 +955,17 @@ Configuration for `ExperimentalMongoDB` store. | `retain_completed_for_s` | integer (uint32) | โ€” | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | | `client_action_timeout_s` | integer (uint64) | โ€” | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | | `enable_active_action_count_metric` | boolean | โ€” | false | Periodically count the actions in each stage and report them as the `execution.active.count` metric. | -| `worker_timeout_s` | integer (uint64) | โ€” | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `worker_timeout_s` | integer (uint64) | โ€” | 10 seconds (four default keepalive intervals) | Remove workers from pool once the worker has not responded in this amount of time in seconds. Any message from the worker counts, not only keepalives. Eviction requeues everything the worker held, so keep this well above the longest pause a loaded worker can see. | | `max_action_executing_timeout_s` | integer (uint64) | โ€” | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | | `unacknowledged_kill_timeout_s` | integer (uint64) | โ€” | 60 seconds | Evict a worker that has not reported back on an operation it was told to kill within this many seconds. A healthy worker acknowledges a kill in moments; one that cannot is wedged, and its keepalives would otherwise keep `worker_timeout_s` from ever firing while the dead operation holds its slot. Eviction requeues the worker's other operations. | +| `dispatch_ack_timeout_s` | integer (uint64) | โ€” | unset (off) | Requeue a dispatched action the worker has not acknowledged within this many seconds, untried and without counting an attempt, and pause the worker. The clock is the worker's own messages: a dispatch is dated by the worker's last message before it, and the check runs on each later message, so keep this above a few keepalive intervals. Workers from v1.7.3 acknowledge every dispatch; older workers never do, so leave this unset while any are connected or their every action would be requeued. | | `unsatisfiable_action_timeout_s` | integer (uint64) | โ€” | 0 (never fail) | Fail a queued action once no connected worker has been able to run it for this many seconds, instead of leaving it queued forever. An action counts as impossible to run when no connected worker could take it even when idle: an `exact` value no worker has, a required key no worker declares, or a `minimum` larger than any worker's total. An action that is only waiting for a busy worker is never failed by this setting, and neither is any action while no workers are connected at all. | -| `max_job_retries` | integer (uint) | โ€” | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | +| `no_worker_action_timeout_s` | integer (uint64) | โ€” | 0 (never fail) | Seconds a queued action waits while no worker at all is connected to this scheduler, nor, on a shared backend, to any peer, before it fails with `FailedPrecondition`. `unsatisfiable_action_timeout_s` only runs against workers that are there, so a pool that scales from zero is never failed on sight; but a provisioner that will not create a pod for a shape it cannot serve leaves such an action waiting for a fleet that never comes, and this is the bound on that. Keep it above the longest time a pool needs to bring up its first worker. | +| `max_job_retries` | integer (uint) | โ€” | 3 | If a job returns an internal error or times out this many times the scheduler completes it with `FailedPrecondition` carrying the last error, a code clients do not retry. This is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. A lost worker and a memory escalation are not the action's failures and have their own budgets: `max_worker_loss_retries` and `memory_escalation.max_steps`. | +| `max_worker_loss_retries` | integer (uint) | โ€” | 10 | How many times an action whose worker was lost (disconnected, timed out, evicted, or OOM-killed as a whole) is queued again before the scheduler completes it with `FailedPrecondition`. A lost worker is usually not the action's fault, so these do not spend `max_job_retries`; the cap only stops an action that takes a worker down every time it runs. | | `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | โ€” | `"least_recently_used"` | The strategy used to assign workers jobs. | +| `live_memory_veto` | string | โ€” | unset (off) | The name of a `minimum` platform property, usually `memory_kb`, that the scheduler compares against the free memory each worker reports with its keepalive: a worker reporting less than the action asks for is skipped for that action, whatever the admission ledger says. The ledger only subtracts what actions declare; this catches the ones that declared too little. Workers older than v1.7.3 report nothing and are never vetoed. | +| `memory_escalation` | [MemoryEscalationSpec](#memoryescalationspec) | โ€” | unset (a memory kill fails the action) | When a worker reports an action killed for memory (`KILLED_MEMORY`, by its own reservation enforcement or by the kernel), requeue the action with a larger reservation instead of failing it. Escalations have their own budget (`max_steps`) and do not spend `max_job_retries`. The reservation grows to the largest memory any connected worker advertises (or `max_kb`); the last step reserves that worker whole, memory and CPU, so the action runs alone, and only a kill there fails the action: nothing in the fleet could run it. | | `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | โ€” | memory | The storage backend to use for the scheduler. | | `worker_match_logging_interval_s` | integer (int64) | โ€” | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | | `fallback_match_interval_s` | integer (int64) | โ€” | `5` | Every N seconds, run a worker matching pass even if no task or worker change notification arrived. This is a safety net for missed notifications and for scheduler backends with eventually consistent searches (for example Redis), where an operation that was re-queued may not be visible to the search triggered by its own notification. Without this, such an operation can stay queued until an unrelated event triggers another matching pass. Defaults to 5s. Zero or any negative value disables it. | @@ -988,8 +1004,14 @@ build at the main scheduler directly though. | --- | --- | --- | --- | --- | | `hints_file` | string | Yes | โ€” | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | | `refresh_interval_s` | integer (uint64) | โ€” | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | -| `cpu_property_name` | string | โ€” | `cpu_count` | Platform property name used for CPU minimum values. | -| `memory_property_name` | string | โ€” | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | +| `cpu_property_name` | string | โ€” | `cpu_count` | Platform property name used for CPU minimum values. The nested scheduler must declare it as a `minimum` property (as it must the memory and disk names): an undeclared property is matched as an exact string, and an action carrying a number would then match no worker. | +| `memory_property_name` | string | โ€” | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. Declared as `minimum` on the nested scheduler, like the CPU name. | +| `disk_property_name` | string | โ€” | `disk_kb` | Platform property name used for disk minimum values, expressed in KiB. Declared as `minimum` on the nested scheduler, like the CPU name. | +| `classes` | array of [SizeClassSpec](#sizeclassspec) | โ€” | โ€” | The fleet's size ladder. A hint may name a `class` instead of numbers, and the cold-start policy below picks one for actions no hint matches. Empty means raw numbers only. | +| `default_class` | string | โ€” | โ€” | The class an untagged action gets when no hint and no rule below matches. Takes precedence over `cold_start` when both are set. | +| `class_by_mnemonic` | map of string to string | โ€” | โ€” | Class by Bazel `action_mnemonic` for untagged actions; a known heavy mnemonic (`TestRunner`, `Link`) starts in the right class. | +| `class_by_timeout` | array of [TimeoutClassSpec](#timeoutclassspec) | โ€” | โ€” | Class by the action's timeout, the largest `min_timeout_s` at or below the timeout wins. Checked before `default_class`, after the mnemonic rule. | +| `cold_start` | [ColdStartSpec](#coldstartspec) | โ€” | โ€” | Reservation given to an action that no hint matches and the client left untagged. Only a property that is absent is filled; a value the client sent is kept. Without this an untagged action costs the scheduler's ledger nothing, so any number of them can land on one worker. | | `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | โ€” | The nested scheduler to use after applying resource hints. | ## ListenerConfig @@ -1153,6 +1175,7 @@ Configuration for an individual shard of the store. | --- | --- | | `"cas"` | The store is content addressable storage. | | `"ac"` | The store is an action cache. | +| `"event_sink"` | A write-only event sink using `ByteStream.Write`. Only string keys and exact upload sizes are accepted. The resource hash identifies the key, not the payload; the size describes the payload. The upstream must be an event receiver, not a CAS. Reads and existence checks are unsupported. Use this for BEP/origin event forwarding, never for cache data. | ## GrpcReadBatchingConfig @@ -1181,7 +1204,10 @@ Generic config for an endpoint and associated configs. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `uri` | string | Yes | โ€” | URI of the endpoint. | -| `timeout` | number (float) | โ€” | 5 seconds | Timeout in seconds that a request should take. | +| `timeout` | number (float) | โ€” | 5 seconds | Timeout in seconds for connecting to the endpoint and for a request to be answered. The worker's messages to the scheduler (keepalives, acknowledgements, results) ride on the one `ConnectWorker` stream opened at registration, which this does not bound; the `GoingAway` sent at shutdown is bounded by it, so a stalled connection cannot hold a pod past its grace period. A keepalive goes out every half of it. | +| `tcp_keepalive_s` | integer (uint64) | โ€” | 30 seconds | TCP keepalive interval (seconds), probing the socket under the scheduler connection so a dead peer is noticed at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | โ€” | 30 seconds | HTTP/2 keepalive interval (seconds): PING frames on the scheduler connection, so one that has gone dead underneath fails instead of hanging every message on it. | +| `http2_keepalive_timeout_s` | integer (uint64) | โ€” | 20 seconds | HTTP/2 keepalive timeout (seconds): a PING unanswered for this long ends the connection, and the worker reconnects. | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | โ€” | โ€” | The TLS configuration to use to connect to the endpoint. | ## Buck2FileCaptureConfig @@ -1209,6 +1235,30 @@ archive is partial. See `nativelink-worker/README.md` for the control protocol. | `internal_paths` | array of string | โ€” | โ€” | Internal CAS/cache paths excluded from browsing. Materialized action inputs hard-linked from these caches remain visible in the action tree. | | `finalize_timeout_s` | integer (uint) | โ€” | 120 seconds (zero selects this default) | Maximum final archive time before action cleanup proceeds. The helper records a partial capture on timeout; build success/failure is unchanged. | +## ResourceEnforcementConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `memory` | [MemoryEnforcement](#memoryenforcement) | โ€” | soft | What to do with an action that exceeds its memory reservation. | +| `disk` | [DiskEnforcement](#diskenforcement) | โ€” | `"none"` | Whether to refuse an action whose disk reservation exceeds the free space under the work directory, less what the actions already admitted reserved, before its inputs are fetched. The refusal is `ResourceExhausted`, which the scheduler requeues without counting an attempt and holds off this worker until its next keepalive. | +| `disk_property_name` | string | โ€” | `disk_kb` | Platform property carrying the action's disk reservation in KiB. | +| `memory_property_name` | string | โ€” | `memory_kb` | Platform property carrying the action's memory reservation in KiB, as the scheduler sends it. An action without the property is not enforced. | +| `memory_headroom_percent` | integer (uint64) | โ€” | 20 | Percent above the reservation an action may reach before it is killed. | +| `disk_headroom_percent` | integer (uint64) | โ€” | 20 | Percent above its disk reservation an action's own files may reach before it is killed, with `disk: soft`. | + +## PersistentWorkersConfig + +The pool of persistent worker processes (Bazel `supports-workers`). + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `enabled` | boolean | โ€” | true | Run an action whose platform properties carry `supports-workers=1` in a pooled worker process that outlives it and serves the next action with the same executable, startup arguments and environment. Off, every such action runs as a one-shot process like any other. | +| `max_workers_per_key` | integer (uint) | โ€” | 4 | Most worker processes kept per key (executable, startup arguments, environment, protocol). An action whose key is at the cap and has no idle process waits `acquire_timeout_s` for one to come back, then runs one-shot. 0 takes the default. | +| `idle_timeout_s` | integer (uint64) | โ€” | 300 | Seconds an idle worker process is kept before the sweeper shuts it down. 0 takes the default. | +| `max_requests_per_worker` | integer (uint64) | โ€” | 200 | Requests a worker process serves before it is retired and replaced, which bounds what a long-lived process accumulates. 0 takes the default. | +| `shutdown_grace_ms` | integer (uint64) | โ€” | 5000 | Milliseconds a worker process being shut down gets to exit on its own (its stdin is closed) before SIGKILL. 0 takes the default. | +| `acquire_timeout_s` | integer (uint64) | โ€” | 30 | Seconds an action waits for a worker process to come back when its key is at `max_workers_per_key` with none idle, before it runs one-shot instead. 0 takes the default. | + ## UploadActionResultConfig | Field | Type | Required | Default | Description | @@ -1239,6 +1289,27 @@ A dynamic configuration. The string will be executed as a command **Type:** string +## CapacityConfig + +Advertise CPU and memory from what the worker can actually see. The +worker reads `cpu.max` and `memory.max` from its own cgroup v2 directory +(found through `/proc/self/cgroup`, so a privileged container that sees +the host's tree still finds its own) up to the nearest limited ancestor, +the pod's limits on Kubernetes; with no limit at any level the configured +properties stand. It takes off what it needs for itself, divides +the memory by the enforcement headroom, and sets the two properties to +the result at registration. What the scheduler packs against is then +derived from the one number that is enforced, instead of typed in twice. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cpu_property_name` | string | โ€” | `cpu_count` | Property that carries CPU, on the scale `cpu_unit` names. | +| `memory_property_name` | string | โ€” | `memory_kb` | Property that carries memory, in KiB. | +| `cpu_unit` | [CpuUnit](#cpuunit) | โ€” | cores | The scale the CPU property is advertised on. Whole cores by default, the scale every other example in this configuration uses; a fleet that advertises and requests thousandths of a core sets `millicores`. Getting this wrong is a thousandfold error in how many actions the scheduler packs onto the worker. | +| `overhead_cpu_millicores` | integer (uint64) | โ€” | 1000 | CPU the worker keeps for itself, in thousandths of a core whatever `cpu_unit` says. | +| `overhead_memory_kb` | integer (uint64) | โ€” | 2097152 (2 GiB) | Memory the worker keeps for itself (the process, its directory cache, output buffers), in KiB. | +| `memory_headroom_percent` | integer (uint64) | โ€” | unset | Percent the advertised memory is reduced by, so that actions at their enforcement ceiling (reservation plus headroom) still fit the limit. Unset, it follows `resource_enforcement.memory_headroom_percent` while memory enforcement is on, and is 0 otherwise, so the two cannot drift apart by being typed twice. | + ## EnvironmentSource One of: @@ -1283,6 +1354,19 @@ workers are able to run the task. | --- | --- | | `"least_recently_used"` | Prefer workers that have been least recently used to run a job. | | `"most_recently_used"` | Prefer workers that have been most recently used to run a job. | +| `"least_loaded"` | Prefer the worker running the fewest actions, least recently used among equals. A worker that just joined has nothing running, so it takes the next action instead of waiting for every older worker to be used once more, and a burst spreads across the fleet instead of filling one worker to its concurrency cap. | +| `"best_fit"` | Prefer the worker the action fits most tightly: among the workers that can take it, the one whose remaining `minimum` properties (memory, CPU) would be smallest afterwards, as a share of what it advertises. Big actions are kept for the workers with big room and small ones fill the gaps, so a fleet of mixed sizes wastes the least. A `priority` property on the action that a worker carries with the same value prefers that worker before the fit is judged, which is what such a property is for (a zone, a cache locality). Ties go to the fewest running actions, then least recently used. | + +## MemoryEscalationSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `property` | string | โ€” | `memory_kb` | The `minimum` platform property carrying the memory reservation. | +| `percent` | integer (uint64) | โ€” | 200 | Each kill scales the reservation by this factor, in hundredths: 200 doubles it, 150 adds half. | +| `max_kb` | integer (uint64) | โ€” | 0 | Never reserve more than this many KiB; 0 takes the largest memory any connected worker advertises. | +| `ladder_kb` | array of integer (uint64) | โ€” | empty (scale by `percent`) | The memory values of the fleet's size classes, ascending, in KiB. When set, a kill steps the reservation to the next class above it instead of scaling by `percent`; past the top class the last step reserves the largest worker whole. The chart fills this from the same classes the `historical_resource` scheduler uses. | +| `cpu_property` | string | โ€” | `cpu_count` | The `minimum` platform property carrying the CPU reservation. The last escalation reserves the largest worker's whole memory and, through this property, its whole CPU, so nothing shares the worker with the action. Empty leaves CPU alone. | +| `max_steps` | integer (uint64) | โ€” | 8 | The most escalations one action gets before the scheduler completes it with `FailedPrecondition`; 0 lets the ceiling alone bound them. | ## ExperimentalSimpleSchedulerBackend @@ -1337,6 +1421,38 @@ If a property is found, then replace it with another one. **Type:** [PlatformPropertyReplacement](#platformpropertyreplacement) +## SizeClassSpec + +A named point on the fleet's size ladder. List them ascending; each +should dominate the one before on every dimension it names. A zero +leaves that dimension alone. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | โ€” | Free text, referenced by hints and by the cold-start policy. | +| `cpu_count` | integer (uint64) | โ€” | `0` | Value for `cpu_property_name`, on the scale the workers advertise. | +| `memory_kb` | integer (uint64) | โ€” | `0` | Value for `memory_property_name`, in KiB. | +| `disk_kb` | integer (uint64) | โ€” | `0` | Value for `disk_property_name`, in KiB. | + +## TimeoutClassSpec + +A class for actions whose timeout is at least `min_timeout_s`. Bazel's +test sizes arrive as timeouts (short 60 s, moderate 300 s, long 900 s, +eternal 3600 s), so this maps them to classes without a CAS fetch. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `min_timeout_s` | integer (uint64) | Yes | โ€” | | +| `class` | string | Yes | โ€” | | + +## ColdStartSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cpu_count` | integer (uint64) | โ€” | `0` | Value for `cpu_property_name`, on the scale the workers advertise. 0 leaves the property alone. | +| `memory_kb` | integer (uint64) | โ€” | `0` | Value for `memory_property_name`, in KiB. 0 leaves the property alone. | +| `disk_kb` | integer (uint64) | โ€” | `0` | Value for `disk_property_name`, in KiB. 0 leaves the property alone. | + ## HttpListener | Field | Type | Required | Default | Description | @@ -1431,6 +1547,7 @@ If a property is found, then replace it with another one. | --- | --- | --- | --- | --- | | `path` | string | โ€” | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | | `timeout_seconds` | integer (uint64) | โ€” | 5s | Timeout on health checks. | +| `readiness_path` | string | โ€” | "/ready" | Path of the readiness check, the stricter sibling of `path`: it answers 503 while any component is still initializing, where `path` answers 200. A worker's registration with its scheduler is such a component, so a Kubernetes readiness probe on this path turns Ready only once the worker can take work. Must differ from `path`; the same path for both is refused at startup. | ## ErrorCode @@ -1471,6 +1588,21 @@ The possible error codes that might occur on an upstream request. | `key_file` | string | โ€” | โ€” | Path to the private key file for client authentication. | | `use_native_roots` | boolean | โ€” | false | If set the client will use the native roots for TLS connections. | +## MemoryEnforcement + +| Value | Description | +| --- | --- | +| `"none"` | Measure only. | +| `"soft"` | Kill the action's process group once two consecutive samples exceed the reservation plus headroom, or one sample exceeds twice it. | + +## DiskEnforcement + +| Value | Description | +| --- | --- | +| `"none"` | Fetch inputs regardless of free space. | +| `"guard"` | Refuse the action when the free space under the work directory is below its reservation. | +| `"soft"` | `guard`, and kill an action whose files under its own directory grow past its reservation plus `disk_headroom_percent`. Sampled every ten seconds by walking the action directory, counting only files the action itself wrote (inputs are hard links into the CAS and are not counted). | + ## UploadCacheResultsStrategy | Value | Description | @@ -1480,6 +1612,15 @@ The possible error codes that might occur on an upstream request. | `"everything"` | Upload all action results that complete. | | `"failures_only"` | Only upload action results that fail. | +## CpuUnit + +The scale a CPU property is advertised on. + +| Value | Description | +| --- | --- | +| `"cores"` | Whole cores, rounded down: what `cpu_count: { query_cmd: "nproc" }` advertises and what an action asking for `cpu_count=1` means. | +| `"millicores"` | Thousandths of a core, for a fleet whose actions and workers already speak that scale. | + ## ExperimentalRedisSchedulerBackend | Field | Type | Required | Default | Description | @@ -1576,6 +1717,6 @@ specified. If anything here disagrees with the binary, the source wins: -- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.7.2/nativelink-config/src/stores.rs) -- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.7.2/nativelink-config/src/cas_server.rs) -- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.7.2/nativelink-config/src/schedulers.rs) +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.7.3/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.7.3/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.7.3/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.7.2.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.7.2.mdx new file mode 100644 index 00000000000..999e3e05510 --- /dev/null +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.7.2.mdx @@ -0,0 +1,1581 @@ +--- +title: Configuration reference +description: Every knob in the NativeLink JSON5 configuration โ€” types, defaults, and links to source, autogenerated from the Rust config crate. +full: true +--- + +{/* AUTOGENERATED โ€” do not edit by hand. + Source: nativelink-config @ v1.7.2 (daee0a8d) + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} + + + +This is the canonical NativeLink configuration reference for **v1.7.2**. +It is autogenerated from the Rust config crate +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.7.2/nativelink-config/src)) via the `build-schema` binary, so +it can never drift from what the binary actually deserializes. + +## Top-level fields + +The root object (`CasConfig`) accepts the following fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `stores` | array of [NamedStoreConfig](#namedstoreconfig) | Yes | List of stores available to use in this config. The keys can be used in other configs when needing to reference a store. | +| `workers` | array of [WorkerConfig](#workerconfig) | โ€” | Worker configurations used to execute jobs. | +| `schedulers` | array of [NamedSchedulerConfig](#namedschedulerconfig) | โ€” | List of schedulers available to use in this config. The keys can be used in other configs when needing to reference a scheduler. | +| `servers` | array of [ServerConfig](#serverconfig) | Yes | Servers to setup for this process. | +| `experimental_origin_events` | [OriginEventsSpec](#origineventsspec) | โ€” | Experimental - Origin events configuration. This is the service that will collect and publish nativelink events to a store for processing by an external service. | +| `global` | [GlobalConfig](#globalconfig) | โ€” | Any global configurations that apply to all modules live here. | + +## Configuration types + +Every type reachable from the root configuration, in reading order. + +## NamedStoreConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | โ€” | | + +Plus exactly one of the following variants (the key selects the variant): + +### `cache_metrics` + +Cache metrics store wraps another store and emits low-cardinality +OpenTelemetry cache operation metrics for the wrapped store. + +This wrapper is opt-in. Stores that are not explicitly wrapped by +`cache_metrics` are constructed exactly as they are without this +wrapper and do not pay its hot-path timing or recording cost. + +**Example JSON5 config:** +```json5 +"cache_metrics": { + "cache_type": "cas", + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas" + } + } +} +``` + +**Type:** [CacheMetricsSpec](#cachemetricsspec) + +### `memory` + +Memory store will store all data in a hash map in memory. + +**Example JSON5 config:** +```json5 +"memory": { + "eviction_policy": { + "max_bytes": "10mb", + } +} +``` + +**Type:** [MemorySpec](#memoryspec) + +### `experimental_cloud_object_store` + +A generic blob store that will store files on the cloud +provider. This configuration will never delete files, so you are +responsible for purging old files in other ways. +It supports the following backends: + +1. **Amazon S3:** + S3 store will use Amazon's S3 service as a backend to store + the files. This configuration can be used to share files + across multiple instances. Uses system certificates for TLS + verification via `rustls-platform-verifier`. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "aws", + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +2. **Google Cloud Storage:** + GCS store uses Google's GCS service as a backend to store + the files. This configuration can be used to share files + across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "gcs", + "bucket": "test-bucket", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +3. **Azure Blob Store:** + Azure Blob store will use Microsoft's Azure Blob service as a + backend to store the files. This configuration can be used to + share files across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "azure", + "account_name": "cloudshell1393657559", + "container": "simple-test-container", + "key_prefix": "folder/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + to store files. This store is specifically configured for ONTAP's S3 requirements + including custom TLS configuration, credentials management, and proper vserver + configuration. + + This store uses AWS environment variables for credentials: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_DEFAULT_REGION` + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "ontap", + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "root_certificates": "/path/to/certs.pem", // Optional + "key_prefix": "test-prefix/", // Optional + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +5. **Cloudflare R2:** + R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + S3 API, so this is a thin wrapper that derives the account-scoped + endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + Storage API. The path-style endpoint is derived from your Object + Storage `namespace` and `region` as + `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + Authenticate with a Customer Secret Key (Access Key/Secret Key pair + created under User Settings -> Customer secret keys in the OCI + console); the secret cannot be retrieved after generation, so read + it from an env var via shellexpand. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "oci_access_key_id", + "secret_access_key": "oci_secret_access_key", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + ``` + +**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) + +### `ontap_s3_existence_cache` + +ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store +to optimize repeated existence checks. It maintains an in-memory cache of object +digests and periodically syncs this cache to disk for persistence. + +The cache helps reduce latency for repeated calls to check object existence, +while still ensuring eventual consistency with the underlying ONTAP S3 store. + +Example JSON5 config: +```json5 +"ontap_s3_existence_cache": { + "index_path": "/path/to/cache/index.json", + "sync_interval_seconds": 300, + "backend": { + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "key_prefix": "test-prefix/" + } +} +``` + +**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) + +### `verify` + +Verify store is used to apply verifications to an underlying +store implementation. It is strongly encouraged to validate +as much data as you can before accepting data from a client, +failing to do so may cause the data in the store to be +populated with invalid data causing all kinds of problems. + +The suggested configuration is to have the CAS validate the +hash and size and the AC validate nothing. + +**Example JSON5 config:** +```json5 +"verify": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb" + } + }, + }, + "verify_size": true, + "verify_hash": true +} +``` + +**Type:** [VerifySpec](#verifyspec) + +### `completeness_checking` + +Completeness checking store verifies if the +output files & folders exist in the CAS before forwarding +the request to the underlying store. +Note: This store should only be used on AC stores. + +**Example JSON5 config:** +```json5 +"completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } +} +``` + +**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) + +### `compression` + +A compression store that will compress the data inbound and +outbound. There will be a non-trivial cost to compress and +decompress the data, but in many cases if the final store is +a store that requires network transport and/or storage space +is a concern it is often faster and more efficient to use this +store before those stores. + +**Example JSON5 config:** +```json5 +"compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + "max_bytes": "2gb", + } + } + } +} +``` + +**Type:** [CompressionSpec](#compressionspec) + +### `dedup` + +A dedup store will take the inputs and run a rolling hash +algorithm on them to slice the input into smaller parts then +run a sha256 algorithm on the slice and if the object doesn't +already exist, upload the slice to the `content_store` using +a new digest of just the slice. Once all parts exist, an +Action-Cache-like digest will be built and uploaded to the +`index_store` which will contain a reference to each +chunk/digest of the uploaded file. Downloading a request will +first grab the index from the `index_store`, and forward the +download content of each chunk as if it were one file. + +This store is exceptionally good when the following conditions +are met: +* Content is mostly the same (inserts, updates, deletes are ok) +* Content is not compressed or encrypted +* Uploading or downloading from `content_store` is the bottleneck. + +Note: This store pairs well when used with `CompressionSpec` as +the `content_store`, but never put `DedupSpec` as the backend of +`CompressionSpec` as it will negate all the gains. + +Note: When running `.has()` on this store, it will only check +to see if the entry exists in the `index_store` and not check +if the individual chunks exist in the `content_store`. + +**Example JSON5 config:** +```json5 +"dedup": { + "index_store": { + "memory": { + "eviction_policy": { + "max_bytes": "1GB", + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": "500MB", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + "max_bytes": "2gb" + } + } + } + } + } + } + } +} +``` + +**Type:** [DedupSpec](#dedupspec) + +### `existence_cache` + +Existence store will wrap around another store and cache calls +to has so that subsequent `has_with_results` calls will be +faster. This is useful for cases when you have a store that +is slow to respond to has calls. +Note: This store should only be used on CAS stores. + +**Example JSON5 config:** +```json5 +"existence_cache": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + // Note this is the existence store policy, not the backend policy + "eviction_policy": { + "max_seconds": 100, + } +} +``` + +**Type:** [ExistenceCacheSpec](#existencecachespec) + +### `fast_slow` + +`FastSlow` store will first try to fetch the data from the `fast` +store and then if it does not exist try the `slow` store. +When the object does exist in the `slow` store, it will copy +the data to the `fast` store while returning the data. +This store should be thought of as a store that "buffers" +the data to the `fast` store. +On uploads it will mirror data to both `fast` and `slow` stores. + +WARNING: If you need data to always exist in the `slow` store +for something like remote execution, be careful because this +store will never check to see if the objects exist in the +`slow` store if it exists in the `fast` store (i.e. it assumes +that if an object exists in the `fast` store it will exist in +the `slow` store). + +***Example JSON5 config:*** +```json5 +"fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + } +} +``` + +**Type:** [FastSlowSpec](#fastslowspec) + +### `shard` + +Shards the data to multiple stores. This is useful for cases +when you want to distribute the load across multiple stores. +The digest hash is used to determine which store to send the +data to. + +**Example JSON5 config:** +```json5 +"shard": { + "stores": [ + { + "store": { + "memory": { + "eviction_policy": { + "max_bytes": "10mb" + }, + }, + }, + "weight": 1 + }] +} +``` + +**Type:** [ShardSpec](#shardspec) + +### `filesystem` + +Stores the data on the filesystem. This store is designed for +local persistent storage. Restarts of this program should restore +the previous state, meaning anything uploaded will be persistent +as long as the filesystem integrity holds. + +**Example JSON5 config:** +```json5 +"filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": "10gb", + } +} +``` + +**Type:** [FilesystemSpec](#filesystemspec) + +### `ref_store` + +Store used to reference a store in the root store manager. +This is useful for cases when you want to share a store in different +nested stores. Example, you may want to share the same memory store +used for the action cache, but use a `FastSlowSpec` and have the fast +store also share the memory store for efficiency. + +**Example JSON5 config:** +```json5 +"ref_store": { + "name": "FS_CONTENT_STORE" +} +``` + +**Type:** [RefSpec](#refspec) + +### `size_partitioning` + +Uses the size field of the digest to separate which store to send the +data. This is useful for cases when you'd like to put small objects +in one store and large objects in another store. This should only be +used if the size field is the real size of the content, in other +words, don't use on AC (Action Cache) stores. Any store where you can +safely use `VerifySpec.verify_size = true`, this store should be safe +to use (i.e. CAS stores). + +**Example JSON5 config:** +```json5 +"size_partitioning": { + "size": "128mib", + "lower_store": { + "memory": { + "eviction_policy": { + "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" + } + } + }, + "upper_store": { + /// This store discards data larger than 128mib. + "noop": {} + } +} +``` + +**Type:** [SizePartitioningSpec](#sizepartitioningspec) + +### `grpc` + +This store will pass-through calls to another gRPC store. This store +is not designed to be used as a sub-store of another store, but it +does satisfy the interface and will likely work. + +One major GOTCHA is that some stores use a special function on this +store to get the size of the underlying object, which is only reliable +when this store is serving the a CAS store, not an AC store. If using +this store directly without being a child of any store there are no +side effects and is the most efficient way to use it. + +**Example JSON5 config:** +```json5 +"grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "connections_per_endpoint": "5", + "rpc_timeout_s": "5m", + "store_type": "ac", + // Static headers attached to every outgoing request to the upstream + // remote cache. Useful for fixed service-account credentials. + "headers": { + "authorization": "Bearer my-static-token" + }, + // Header names to copy from the inbound client request and forward to + // the upstream remote cache. Use this to pass through dynamic + // credentials such as a JWT sent by the build client. + "forward_headers": ["authorization", "x-custom-token"] +} +``` + +**Type:** [GrpcSpec](#grpcspec) + +### `redis_store` + +Stores data in any stores compatible with Redis APIs. + +Pairs well with `SizePartitioning` and/or `FastSlow` stores. +Ideal for accepting small object sizes as most Redis store +services have a max file upload of between 256Mb-512Mb. + +If you are using Redis together with any stores above it +e.g. existence cache, you will need to configure `notify-keyspace-events` +to `KA` as per [https://redis.io/docs/latest/develop/pubsub/keyspace-notifications/#configuration](https://redis.io/docs/latest/develop/pubsub/keyspace-notifications/#configuration) +in order for us to get eviction events. Failing to do so will get you +log messages complaining about it, as well as errors like [https://github.com/TraceMachina/nativelink/issues/2436](https://github.com/TraceMachina/nativelink/issues/2436) + +**Example JSON5 config:** +```json5 +"redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", + ], + "max_client_permits": 1000, +} +``` + +**Type:** [RedisSpec](#redisspec) + +### `noop` + +Noop store is a store that sends streams into the void and all data +retrieval will return 404 (`NotFound`). This can be useful for cases +where you may need to partition your data and part of your data needs +to be discarded. + +**Example JSON5 config:** +```json5 +"noop": {} +``` + +**Type:** [NoopSpec](#noopspec) + +### `experimental_mongo` + +Experimental `MongoDB` store implementation. + +This store uses `MongoDB` as a backend for storing data. It supports +both CAS (Content Addressable Storage) and scheduler data with +optional change streams for real-time updates. + +**Example JSON5 config:** +```json5 +"experimental_mongo": { + "connection_string": "mongodb://localhost:27017", + "database": "nativelink", + "cas_collection": "cas", + "key_prefix": "cas:", + "read_chunk_size": 65536, + "max_concurrent_uploads": 10, + "enable_change_streams": false, + "max_requests": "100" +} +``` + +**Type:** [ExperimentalMongoSpec](#experimentalmongospec) + +## WorkerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `local` + +A worker type that executes jobs locally on this machine. + +**Type:** [LocalWorkerConfig](#localworkerconfig) + +## NamedSchedulerConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | โ€” | | + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## ServerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | โ€” | {index of server in config} | Name of the server. This is used to help identify the service for telemetry and logs. | +| `listener` | [ListenerConfig](#listenerconfig) | Yes | โ€” | Configuration | +| `services` | [ServicesConfig](#servicesconfig) | โ€” | โ€” | Services to attach to server. | +| `experimental_identity_header` | [IdentityHeaderSpec](#identityheaderspec) | โ€” | {see `IdentityHeaderSpec`} | The config related to identifying the client. | + +## OriginEventsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `publisher` | [OriginEventsPublisherSpec](#origineventspublisherspec) | Yes | โ€” | The publisher configuration for origin events. | +| `max_event_queue_size` | integer (uint) | โ€” | 65536 (zero defaults to this) | The maximum number of events to queue before applying back pressure. IMPORTANT: Backpressure causes all clients to slow down significantly. Zero is default. | + +## GlobalConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_open_files` | integer (uint) | Yes | 24576 (= 24 * 1024) | Maximum number of open files that can be opened at one time. This value is not strictly enforced, it is a best effort. Some internal libraries open files or read metadata from a files which do not obey this limit, however the vast majority of cases will have this limit be honored. This value must be larger than `ulimit -n` to have any effect. Any network open file descriptors is not counted in this limit, but is counted in the kernel limit. It is a good idea to set a very large `ulimit -n`. Note: This value must be greater than 10. | +| `default_digest_hash_function` | [ConfigDigestHashFunction](#configdigesthashfunction) | โ€” | `ConfigDigestHashFunction::sha256` | Default hash function to use while uploading blobs to the CAS when not set by client. | +| `default_digest_size_health_check` | integer (uint) | โ€” | 1024*1024 (1MiB) | Default digest size to use for health check when running diagnostics checks. Health checks are expected to use this size for filling a buffer that is used for creation of digest. | + +## CacheMetricsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cache_type` | string | Yes | โ€” | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | Store to wrap with cache operation metrics. | + +## MemorySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | โ€” | โ€” | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## ExperimentalCloudObjectSpec + +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details + +## OntapS3ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_path` | string | Yes | โ€” | | +| `sync_interval_seconds` | integer (uint32) | Yes | โ€” | | +| `backend` | [ExperimentalOntapS3Spec](#experimentalontaps3spec) | Yes | โ€” | | + +## VerifySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `verify_size` | boolean | โ€” | `false` | If set the store will verify the size of the data before accepting an upload of data. | +| `verify_hash` | boolean | โ€” | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | + +## CompletenessCheckingSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | + +## CompressionSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | โ€” | The compression algorithm to use. | + +## DedupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `min_size` | integer (uint32) | โ€” | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | โ€” | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | โ€” | 512k | Maximum size a chunk is allowed to be. | +| `max_concurrent_fetch_per_get` | integer (uint32) | โ€” | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | + +## ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | โ€” | โ€” | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## FastSlowSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast_direction` | [StoreDirection](#storedirection) | โ€” | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow_direction` | [StoreDirection](#storedirection) | โ€” | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | โ€” | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold โ€” 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | + +## ShardSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `stores` | array of [ShardConfig](#shardconfig) | Yes | โ€” | Stores to shard the data to. | + +## FilesystemSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `content_path` | string | Yes | โ€” | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service boot this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | โ€” | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e. move without copy). All files in this folder will be deleted on every startup. | +| `read_buffer_size` | integer (uint32) | โ€” | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | โ€” | โ€” | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | +| `block_size` | integer (uint64) | โ€” | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | โ€” | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | +| `evict_page_cache` | boolean | โ€” | false | When true, advise the kernel to drop the page cache for each blob after it is written or read (`posix_fadvise` with `POSIX_FADV_DONTNEED`). On real filesystems this takes a globally serialized, all-CPU kernel path that stalls on many-core hosts, and it evicts the page-cache tier that fast-disk deployments rely on. Leave off unless you specifically want to keep this store's I/O out of the page cache. | + +## RefSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | โ€” | Name of the store under the root "stores" config object. | + +## SizePartitioningSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `size` | integer (uint64) | Yes | โ€” | Size to partition the data on. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | Store to send data when object is >= (less than eq) size. | + +## GrpcSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | โ€” | `""` | Instance name for gRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | โ€” | The endpoint of the grpc connection. | +| `store_type` | [StoreType](#storetype) | Yes | โ€” | The type of the upstream store, this ensures that the correct server calls are made. | +| `retry` | [Retry](#retry) | โ€” | โ€” | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | โ€” | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | โ€” | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | +| `rpc_timeout_s` | integer (uint64) | โ€” | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | +| `use_legacy_resource_names` | boolean | โ€” | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | +| `headers` | map of string to string | โ€” | โ€” | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | +| `forward_headers` | array of string | โ€” | โ€” | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | +| `experimental_read_batching` | [GrpcReadBatchingConfig](#grpcreadbatchingconfig) | โ€” | unset (disabled) | Optional and experimental: coalesce small-blob reads into `BatchReadBlobs` RPCs instead of issuing one `ByteStream` `Read` stream per blob. Each `ByteStream` read carries a fixed per-RPC cost, so batching many small reads into a single `BatchReadBlobs` request can dramatically reduce read latency for small blobs. | +| `experimental_remote_cache_compression` | boolean | โ€” | inherited from the process-wide zstd wire-compression intent | Compress this store's own blob transfers on the wire with REAPI `compressed-blobs/zstd`. Uploads and full-blob downloads of blobs at or above 64 KiB are zstd-compressed; smaller blobs and ranged reads keep the identity path. | + +## RedisSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `addresses` | array of string | Yes | โ€” | The hostname or IP address of the Redis server. Ex: `["redis://username:password@redis-server-url:6380/99"]` 99 Represents database ID, 6380 represents the port. | +| `response_timeout_s` | integer (uint64) | โ€” | 10 | DEPRECATED: use `command_timeout_ms` The response timeout for the Redis connection in seconds. | +| `connection_timeout_s` | integer (uint64) | โ€” | 10 | DEPRECATED: use `connection_timeout_ms` | +| `experimental_pub_sub_channel` | string | โ€” | (Empty String / No Channel) | An optional and experimental Redis channel to publish write events to. | +| `key_prefix` | string | โ€” | (Empty String / No Prefix) | An optional prefix to prepend to all keys in this store. | +| `mode` | [RedisMode](#redismode) | โ€” | standard, | Set the mode Redis is operating in. | +| `broadcast_channel_capacity` | integer (uint) | โ€” | `0` | Deprecated as redis-rs doesn't use it | +| `command_timeout_ms` | integer (uint64) | โ€” | 10000 (10 seconds) | The amount of time in milliseconds until the Redis store considers the command to be timed out. This will trigger a retry of the command and potentially a reconnection to the Redis server. | +| `connection_timeout_ms` | integer (uint64) | โ€” | 3000 (3 seconds) | The amount of time in milliseconds until the Redis store considers the connection to unresponsive. This will trigger a reconnection to the Redis server. | +| `health_check_timeout_ms` | integer (uint64) | โ€” | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | +| `read_chunk_size` | integer (uint) | โ€” | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | +| `connection_pool_size` | integer (uint) | โ€” | 3 | The number of connections to keep open to the Redis servers. | +| `key_ttl_s` | integer (uint64) | โ€” | 0 (keys never expire) | Expire keys written by this store after this many seconds. | +| `max_chunk_uploads_per_update` | integer (uint) | โ€” | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it's a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `scan_count` | integer (uint) | โ€” | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | +| `retry` | [Retry](#retry) | โ€” | โ€” | Retry configuration to use when a network request fails. | +| `max_client_permits` | integer (uint) | โ€” | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | +| `max_count_per_cursor` | integer (uint64) | โ€” | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | + +## NoopSpec + +_No fields._ + +## ExperimentalMongoSpec + +Configuration for `ExperimentalMongoDB` store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `connection_string` | string | Yes | โ€” | `ExperimentalMongoDB` connection string. Example: <mongodb://localhost:27017> or <mongodb+srv://cluster.mongodb.net> | +| `database` | string | โ€” | "nativelink" | The database name to use. | +| `cas_collection` | string | โ€” | "cas" | The collection name for CAS data. | +| `scheduler_collection` | string | โ€” | "scheduler" | The collection name for scheduler data. | +| `key_prefix` | string | โ€” | "" | Prefix to prepend to all keys stored in `MongoDB`. | +| `read_chunk_size` | integer (uint) | โ€” | 65536 (64KB) | The maximum amount of data to read from `MongoDB` in a single chunk (in bytes). | +| `max_concurrent_uploads` | integer (uint) | โ€” | 10 | Deprecated, unused Maximum number of concurrent uploads allowed. | +| `connection_timeout_ms` | integer (uint64) | โ€” | 3000 | Connection timeout in milliseconds. | +| `command_timeout_ms` | integer (uint64) | โ€” | 10000 | Command timeout in milliseconds. | +| `enable_change_streams` | boolean | โ€” | false | Enable `MongoDB` change streams for real-time updates. Required for scheduler subscriptions. | +| `write_concern_w` | string | โ€” | โ€” | Write concern 'w' parameter. Can be a number (e.g., 1) or string (e.g., "majority"). | +| `write_concern_j` | boolean | โ€” | โ€” | Write concern 'j' parameter (journal acknowledgment). | +| `write_concern_timeout_ms` | integer (uint32) | โ€” | โ€” | Write concern timeout in milliseconds. | +| `max_requests` | integer (uint) | โ€” | Unlimited | Limits the number of requests at any one time | + +## LocalWorkerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | โ€” | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e. "{name}{uuidv6}"). | +| `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | โ€” | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | +| `max_action_timeout_s` | integer (uint) | โ€” | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout_s` | integer (uint) | โ€” | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_cleanup_wait_s` | integer (uint) | โ€” | 30 seconds | Maximum time to wait for action directory cleanup before timing out. Value in seconds. | +| `max_cleanup_backoff_ms` | integer (uint) | โ€” | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. Value in milliseconds. | +| `max_inflight_tasks` | integer (uint64) | โ€” | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | +| `single_use` | boolean | โ€” | false | Accept one action, upload its results, and exit the worker process. The worker advertises one execution slot and never accepts a second action, including after a scheduler disconnect. A launcher must create a fresh container and fresh writable volumes for its replacement. Use one local worker per process, with no scheduler or services other than health checks. Reusing the container or its writable volumes does not provide isolation from the previous action. | +| `experimental_buck2_file_capture` | [Buck2FileCaptureConfig](#buck2filecaptureconfig) | โ€” | โ€” | Capture full execution-container files for Buck2 builds only. Disabled when absent. Requires `single_use`; read the capture configuration's isolation requirements before enabling it for a worker pool. | +| `timeout_handled_externally` | boolean | โ€” | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | +| `entrypoint` | string | โ€” | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | +| `experimental_precondition_script` | string | โ€” | โ€” | An optional script to run before every action is processed on the worker. The value should be the full path to the script to execute and will pause all actions on the worker if it returns an exit code other than 0. If not set, then the worker will never pause and will continue to accept jobs according to the scheduler configuration. This is useful, for example, if the worker should not take any more actions until there is enough resource available on the machine to handle them. | +| `cas_fast_slow_store` | string | Yes | โ€” | Underlying CAS store that the worker will use to download CAS artifacts. This store must be a `FastSlowStore`. The `fast` store must be a `FileSystemStore` because it will use hardlinks when building out the files instead of copying the files. The slow store must eventually resolve to the same store the scheduler/client uses to send job requests. | +| `upload_action_result` | [UploadActionResultConfig](#uploadactionresultconfig) | โ€” | โ€” | Configuration for uploading action results. | +| `work_directory` | string | Yes | โ€” | The directory work jobs will be executed from. This directory will be fully managed by the worker service and will be purged on startup. This directory and the directory referenced in `local_filesystem_store_ref`'s `stores::FilesystemStore::content_path` must be on the same filesystem. Hardlinks will be used when placing files that are accessible to the jobs that are sourced from `local_filesystem_store_ref`'s `content_path`. | +| `platform_properties` | map of string to [WorkerProperty](#workerproperty) | Yes | โ€” | Properties of this worker. This configuration will be sent to the scheduler and used to tell the scheduler to restrict what should be executed on this worker. | +| `additional_environment` | map of string to [EnvironmentSource](#environmentsource) | โ€” | โ€” | An optional mapping of environment names to set for the execution as well as those specified in the action itself. If set, will set each key as an environment variable before executing the job with the value of the environment variable being the value of the property of the action being executed of that name or the fixed value. | +| `directory_cache` | [DirectoryCacheConfig](#directorycacheconfig) | โ€” | โ€” | Optional directory cache configuration for improving performance by caching reconstructed input directories and using hardlinks instead of rebuilding them from CAS for every action. | +| `experimental_active_input_leases` | boolean | โ€” | false (eviction behavior is unchanged) | Optional and experimental: lease every digest in an active action's input Merkle closure in the worker's locally eviction-managed CAS tiers (the `cas_fast_slow_store`'s filesystem-backed stores) until the action has finished cleanup. This prevents `Lost inputs no longer available remotely` failures caused by local fast-tier eviction while inputs are being materialized under cache pressure. | +| `use_namespaces` | boolean | โ€” | False | Whether to use namespaces to isolate the execution. This is only available on Linux. It is highly recommended as it avoids a number of issues with zombie processes and also provides additional hermeticity. If explicitly set to true and it is not supported the worker will exit with an error. | +| `use_mount_namespace` | boolean | โ€” | False | Whether to use a mount namespace to isolate the worker root. This is only available on Linux and when `use_namespaces` is true. It is highly recommended provides additional hermeticity. If explicitly set to true and it is not supported or `use_namespaces` is not set to true the worker will exit with an error. | + +## SimpleSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `supported_platform_properties` | map of string to [PropertyType](#propertytype) | โ€” | โ€” | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | +| `retain_completed_for_s` | integer (uint32) | โ€” | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | โ€” | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | +| `enable_active_action_count_metric` | boolean | โ€” | false | Periodically count the actions in each stage and report them as the `execution.active.count` metric. | +| `worker_timeout_s` | integer (uint64) | โ€” | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `max_action_executing_timeout_s` | integer (uint64) | โ€” | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | +| `unacknowledged_kill_timeout_s` | integer (uint64) | โ€” | 60 seconds | Evict a worker that has not reported back on an operation it was told to kill within this many seconds. A healthy worker acknowledges a kill in moments; one that cannot is wedged, and its keepalives would otherwise keep `worker_timeout_s` from ever firing while the dead operation holds its slot. Eviction requeues the worker's other operations. | +| `unsatisfiable_action_timeout_s` | integer (uint64) | โ€” | 0 (never fail) | Fail a queued action once no connected worker has been able to run it for this many seconds, instead of leaving it queued forever. An action counts as impossible to run when no connected worker could take it even when idle: an `exact` value no worker has, a required key no worker declares, or a `minimum` larger than any worker's total. An action that is only waiting for a busy worker is never failed by this setting, and neither is any action while no workers are connected at all. | +| `max_job_retries` | integer (uint) | โ€” | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | +| `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | โ€” | `"least_recently_used"` | The strategy used to assign workers jobs. | +| `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | โ€” | memory | The storage backend to use for the scheduler. | +| `worker_match_logging_interval_s` | integer (int64) | โ€” | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | +| `fallback_match_interval_s` | integer (int64) | โ€” | `5` | Every N seconds, run a worker matching pass even if no task or worker change notification arrived. This is a safety net for missed notifications and for scheduler backends with eventually consistent searches (for example Redis), where an operation that was re-queued may not be visible to the search triggered by its own notification. Without this, such an operation can stay queued until an unrelated event triggers another matching pass. Defaults to 5s. Zero or any negative value disables it. | + +## SchedulerGrpcSpec + +A scheduler that forwards requests to an upstream scheduler. This +is useful to use when doing some kind of local action cache or CAS away from +the main cluster of workers. In general, it's more efficient to point the +build at the main scheduler directly though. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | โ€” | The upstream scheduler to forward requests to. | +| `retry` | [Retry](#retry) | โ€” | โ€” | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | โ€” | unlimited | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | โ€” | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | + +## CacheLookupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | Yes | โ€” | The reference to the action cache store used to return cached actions from rather than running them again. To prevent unintended issues, this store should probably be a `CompletenessCheckingSpec`. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | โ€” | The nested scheduler to use if cache lookup fails. | + +## PropertyModifierSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `modifications` | array of [PropertyModification](#propertymodification) | Yes | โ€” | A list of modifications to perform to incoming actions for the nested scheduler. These are performed in order and blindly, so removing a property that doesn't exist is fine and overwriting an existing property is also fine. If adding properties that do not exist in the nested scheduler is not supported and will likely cause unexpected behaviour. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | โ€” | The nested scheduler to use after modifying the properties. | + +## HistoricalResourceSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `hints_file` | string | Yes | โ€” | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | +| `refresh_interval_s` | integer (uint64) | โ€” | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | +| `cpu_property_name` | string | โ€” | `cpu_count` | Platform property name used for CPU minimum values. | +| `memory_property_name` | string | โ€” | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | โ€” | The nested scheduler to use after applying resource hints. | + +## ListenerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `http` + +Listener for HTTP/HTTPS/HTTP2 sockets. + +**Type:** [HttpListener](#httplistener) + +## ServicesConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cas` | array of [CasServiceConfig](#casserviceconfig) | โ€” | โ€” | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | +| `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | โ€” | โ€” | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | โ€” | โ€” | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel gRPC service. | +| `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | โ€” | โ€” | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | +| `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | โ€” | โ€” | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | +| `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | โ€” | โ€” | These two are collectively the Remote Asset protocol, but it's defined as two separate services | +| `push` | array of [PushServiceConfig](#pushserviceconfig) | โ€” | โ€” | | +| `worker_api` | [WorkerApiConfig](#workerapiconfig) | โ€” | โ€” | This is the service used for workers to connect and communicate through. NOTE: This service should be served on a different, non-public port. In other words, `worker_api` configuration should not have any other services that are served on the same port. Doing so is a security risk, as workers have a different permission set than a client that makes the remote execution/cache requests. | +| `experimental_bep` | [BepConfig](#bepconfig) | โ€” | โ€” | Experimental - Build Event Protocol (BEP) configuration. This is the service that will consume build events from the client and publish them to a store for processing by an external service. | +| `admin` | [AdminConfig](#adminconfig) | โ€” | โ€” | This is the service for any administrative tasks. It provides a REST API endpoint for administrative purposes. | +| `health` | [HealthConfig](#healthconfig) | โ€” | โ€” | This is the service for health status check. | + +## IdentityHeaderSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `header_name` | string | โ€” | "x-identity" | The name of the header to look for the identity in. | +| `required` | boolean | โ€” | `false` | If the header is required to be set or fail the request. | + +## OriginEventsPublisherSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | โ€” | The store to publish nativelink events to. The store name referenced in the `stores` map in the main config. | + +## ConfigDigestHashFunction + +| Value | Description | +| --- | --- | +| `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | +| `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | + +## EvictionPolicy + +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | โ€” | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | โ€” | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | โ€” | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | โ€” | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | + +## Retry + +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, +} +``` +will result in: + +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | + +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | โ€” | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | โ€” | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | โ€” | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | โ€” | โ€” | A list of error codes to retry on, if this isn't set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | + +## ExperimentalOntapS3Spec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | โ€” | | +| `vserver_name` | string | Yes | โ€” | | +| `bucket` | string | Yes | โ€” | | +| `root_certificates` | string | โ€” | โ€” | | +| `key_prefix` | string | โ€” | โ€” | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | โ€” | โ€” | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | โ€” | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | โ€” | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | โ€” | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | โ€” | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | โ€” | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | + +## CompressionAlgorithm + +Plus exactly one of the following variants (the key selects the variant): + +### `lz4` + +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. + +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) + +**Type:** [Lz4Config](#lz4config) + +## StoreDirection + +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | + +## ShardConfig + +Configuration for an individual shard of the store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | โ€” | Store to shard the data to. | +| `weight` | integer (uint32) | โ€” | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | + +## GrpcEndpoint + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | โ€” | The endpoint address (i.e. `grpc(s)://example.com:443`). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | โ€” | โ€” | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | โ€” | โ€” | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | โ€” | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | โ€” | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | โ€” | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | โ€” | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | + +## StoreType + +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | + +## GrpcReadBatchingConfig + +Configuration for experimental small-blob read coalescing in a gRPC +store. See [`GrpcSpec::experimental_read_batching`]. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_blob_size_bytes` | integer (uint64) | โ€” | 131072 (128 KiB) | Only blobs at or below this size (in bytes) are eligible for batching. Larger blobs always use the `ByteStream` `Read` path. | +| `max_batch_bytes` | integer (uint64) | โ€” | 3145728 (3 MiB) | Maximum total payload bytes packed into a single `BatchReadBlobs` request. This should leave headroom under the 4 MiB default gRPC message limit for protobuf framing overhead. | +| `dispatch_slots` | integer (uint) | โ€” | 4 | Maximum number of concurrent `BatchReadBlobs` RPCs dispatched by the coalescer. Must be greater than zero. | +| `max_queued_bytes` | integer (uint64) | โ€” | 33554432 (32 MiB) | Bound on the number of payload bytes waiting in the coalescer queue. When exceeded, new read requests bypass batching and fall back to the regular `ByteStream` `Read` path instead of blocking. | + +## RedisMode + +| Value | Description | +| --- | --- | +| `"cluster"` | Use Redis Cluster. | +| `"sentinel"` | Use Redis Sentinel. | +| `"standard"` | Use a standalone Redis server. | + +## EndpointConfig + +Generic config for an endpoint and associated configs. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `uri` | string | Yes | โ€” | URI of the endpoint. | +| `timeout` | number (float) | โ€” | 5 seconds | Timeout in seconds that a request should take. | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | โ€” | โ€” | The TLS configuration to use to connect to the endpoint. | + +## Buck2FileCaptureConfig + +Opt-in file capture for the actual container executing a Buck2 action. +Requires a fresh single-use container with no outer execution wrapper or +separate action mount namespace. For nested runtimes, run `NativeLink` and the +capture helper inside the execution container. Other build tools never start +capture. The helper uploads an archive before `NativeLink` cleans up the action. + +The operator supplies the helper executable and its authenticated ingest +service. `NativeLink` sends one JSON record on stdin and expects `ready\n` +on stdout within 30 seconds. Closing stdin requests final capture; the helper +must exit only after durable acknowledgement, or exit unsuccessfully if the +archive is partial. See `nativelink-worker/README.md` for the control protocol. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `executable` | string | Yes | โ€” | Absolute path to the compatible `buck2-file-capture` executable. | +| `gateway` | string | Yes | โ€” | Authenticated ingest endpoint understood by the capture helper. | +| `token_file` | string | Yes | โ€” | Private token file. The helper excludes its path and hard-link aliases. The token is never included in the action environment or command line. | +| `state_directory` | string | Yes | โ€” | Private directory for the helper's bounded on-disk metadata index. Mount a fresh empty volume here for each single-use container. | +| `container_id` | string | Yes | โ€” | Identity of the actual executing container, normally the pod UID. | +| `protected_paths` | array of string | โ€” | โ€” | Credential files/directories and platform configuration excluded from browsing. Their inode aliases and private secret mounts are also denied. | +| `internal_paths` | array of string | โ€” | โ€” | Internal CAS/cache paths excluded from browsing. Materialized action inputs hard-linked from these caches remain visible in the action tree. | +| `finalize_timeout_s` | integer (uint) | โ€” | 120 seconds (zero selects this default) | Maximum final archive time before action cleanup proceeds. The helper records a partial capture on timeout; build success/failure is unchanged. | + +## UploadActionResultConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | โ€” | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | โ€” | `SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `historical_results_store` | string | โ€” | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | โ€” | `FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `success_message_template` | string | โ€” | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | +| `failure_message_template` | string | โ€” | "" (no message) | Same as `success_message_template` but for failure case. | + +## WorkerProperty + +Plus exactly one of the following variants (the key selects the variant): + +### `values` + +List of static values. +Note: Generally there should only ever be 1 value, but if the platform +property key is `PropertyType::Priority` it may have more than one value. + +**Type:** array of string + +### `query_cmd` + +A dynamic configuration. The string will be executed as a command +(not shell) and will be split by "\n" (new line character). + +**Type:** string + +## EnvironmentSource + +One of: + +- `property`: string โ€” The name of the platform property in the action to get the value from. +- `value`: string โ€” The raw value to set. +- `"from_environment"` โ€” Take the value from the local environment corresponding to the name key +- `"timeout_millis"` โ€” The max amount of time in milliseconds the command is allowed to run (requested by the client). +- `"side_channel_file"` โ€” A special file path will be provided that can be used to communicate with the parent process about out-of-band information. This file will be read after the command has finished executing. Based on the contents of the file, the behavior of the result may be modified. +- `"action_directory"` โ€” A "root" directory for the action. This directory can be used to store temporary files that are not needed after the action has completed. This directory will be purged after the action has completed. + +## DirectoryCacheConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_entries` | integer (uint) | โ€” | 1000 | Maximum number of cached directories. | +| `max_size_bytes` | integer (uint64) | โ€” | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | +| `cache_root` | string | โ€” | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | +| `experimental_subtree_caching` | boolean | โ€” | false (only root directories are cached; existing behavior) | Optional and experimental: additionally cache every subdirectory by its own `Directory` digest, not just the root directory. REAPI Merkle nodes are content-addressed, so a subtree that is byte-identical between two different roots has the same digest and can be materialized with a single hardlink pass instead of being rebuilt from the CAS. This makes the common "one input file changed out of thousands" case reuse every unchanged subtree. | +| `max_concurrent_fetches` | integer (uint) | โ€” | 64 | Maximum number of concurrent slow-store fetches across ALL directory constructions of this cache. This bound protects backing stores from RPC storms: per-level construction concurrency compounds multiplicatively across tree levels and concurrent actions. | +| `experimental_get_tree_prefetch` | boolean | โ€” | false | On a directory-cache miss, prefetch every `Directory` proto of the tree with one logical `GetTree` traversal instead of fetching protos level by level (which costs one round trip per tree DEPTH). The [REAPI request contract] permits servers to impose their own limit even when no page size is specified, and the [REAPI response contract] requires clients to continue with the returned page token. A paginated traversal may therefore use multiple RPCs. Only takes effect when the slow tier is a `grpc` store; any prefetch failure falls back to the per-level path. Worthwhile when worker-to-CAS latency is non-trivial and trees are deep; measured 5-34x on the proto phase at 5-25ms RTT. Note the serving CAS pays the tree walk against its own backend, so its directory protos should be served from a fast tier. | + +## PropertyType + +When the scheduler matches tasks to workers that are capable of running +the task, this value will be used to determine how the property is treated. + +One of: + +- string +- `"minimum"` โ€” Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. +- `"exact"` โ€” Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. +- `"priority"` โ€” Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. + +## WorkerAllocationStrategy + +When a worker is being searched for to run a job, this will be used +on how to choose which worker should run the job when multiple +workers are able to run the task. + +| Value | Description | +| --- | --- | +| `"least_recently_used"` | Prefer workers that have been least recently used to run a job. | +| `"most_recently_used"` | Prefer workers that have been most recently used to run a job. | + +## ExperimentalSimpleSchedulerBackend + +One of: + +- `"memory"` โ€” Use an in-memory store for the scheduler. +- `redis`: [ExperimentalRedisSchedulerBackend](#experimentalredisschedulerbackend) โ€” Use a Redis store for the scheduler. + +## SchedulerSpec + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## PropertyModification + +Plus exactly one of the following variants (the key selects the variant): + +### `add` + +Add a property to the action properties. + +**Type:** [PlatformPropertyAddition](#platformpropertyaddition) + +### `remove` + +Remove a named property from the action. + +**Type:** string + +### `replace` + +If a property is found, then replace it with another one. + +**Type:** [PlatformPropertyReplacement](#platformpropertyreplacement) + +## HttpListener + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `socket_address` | string | Yes | โ€” | Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen to all IPs. | +| `freebind` | boolean | โ€” | false | Allow binding `socket_address` before it is assigned locally. | +| `compression` | [HttpCompressionConfig](#httpcompressionconfig) | โ€” | โ€” | Data transport compression configuration to use for this service. | +| `advanced_http` | [HttpServerConfig](#httpserverconfig) | โ€” | โ€” | Advanced HTTP server configuration. | +| `max_decoding_message_size` | integer (uint) | โ€” | 4 MiB | Maximum number of bytes to decode on each grpc stream chunk. | +| `tls` | [TlsConfig](#tlsconfig) | โ€” | โ€” | TLS configuration for this server. If not set, the server will not use TLS. | + +## CasServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | โ€” | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | โ€” | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `experimental_chunking` | [CasChunkingConfig](#caschunkingconfig) | โ€” | not set โ€” chunking RPCs are rejected, nothing is advertised, | Optional and experimental: enables the REAPI `SplitBlob`/`SpliceBlob` RPCs used by content-defined chunking clients (e.g. Bazel's `--experimental_remote_cache_chunking`). When set, the capabilities service advertises blob split/splice support and `FastCDC` 2020 parameters for this instance. When `cas_store` is a grpc store the RPCs are forwarded to the backend (which must support chunking with matching parameters); otherwise they are served locally. | + +## ActionCacheServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | โ€” | `""` | Used when the config references `instance_name` in the protocol. | +| `ac_store` | string | Yes | โ€” | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | โ€” | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## CapabilitiesServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | โ€” | `""` | Used when the config references `instance_name` in the protocol. | +| `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | โ€” | โ€” | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | +| `remote_cache_compression` | boolean | โ€” | โ€” | Whether this instance supports Bazel remote cache compression. When enabled, the capabilities service advertises zstd wire compression and the ByteStream/CAS services accept REAPI compressed-blobs/zstd data. | + +## ExecutionServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | โ€” | `""` | Used when the config references `instance_name` in the protocol. | +| `experimental_buck2_invocation_isolation` | boolean | โ€” | `false` | Keep identical in-flight Buck2 actions from different invocations on separate workers. Completed action-cache results remain shared. Pair this with single-use workers when capturing execution-container files. Other build tools and requests without Buck2 invocation metadata retain the default deduplication behavior. All schedulers serving the instance must use a runtime supporting this execution scope before enabling it. | +| `cas_store` | string | Yes | โ€” | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | +| `scheduler` | string | Yes | โ€” | The scheduler name referenced in the `schedulers` map in the main config. | + +## ByteStreamServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | โ€” | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | โ€” | Name of the store in the "stores" configuration. | +| `max_bytes_per_stream` | integer (uint) | โ€” | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | +| `persist_stream_on_disconnect_timeout_s` | integer (uint) | โ€” | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | + +## FetchServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | โ€” | `""` | Used when the config references `instance_name` in the protocol. | +| `fetch_store` | string | Yes | โ€” | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## PushServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | โ€” | `""` | Used when the config references `instance_name` in the protocol. | +| `push_store` | string | Yes | โ€” | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | โ€” | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## WorkerApiConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | โ€” | The scheduler name referenced in the `schedulers` map in the main config. | +| `disable_kill_revoked_operations` | boolean | โ€” | false (the sweep runs) | Disable the periodic sweep that tells workers to kill operations the scheduler no longer has executing on them (for example because every client disconnected before the action finished). With the sweep disabled such orphaned actions run to completion and still warm the action cache, which can be preferable for deployments with long or expensive actions whose clients merely retry. | +| `kill_revoked_operations_interval_s` | integer (uint64) | โ€” | 5 (0 uses the default) | How often, in seconds, the scheduler checks for operations that are still running on a worker but are no longer executing according to the state manager, and tells the worker to kill them. Each pass costs one state-manager lookup per running operation, so store-backed deployments with many concurrent actions may want a longer interval. | + +## BepConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | โ€” | The store to publish build events to. The store name referenced in the `stores` map in the main config. | + +## AdminConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | โ€” | "/admin" | Path to register the admin API. If path is "/admin", and your domain is "example.com", you can reach the endpoint with: [http://example.com/admin](http://example.com/admin). | + +## HealthConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | โ€” | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | +| `timeout_seconds` | integer (uint64) | โ€” | 5s | Timeout on health checks. | + +## ErrorCode + +The possible error codes that might occur on an upstream request. + +| Value | Description | +| --- | --- | +| `"Cancelled"` | | +| `"Unknown"` | | +| `"InvalidArgument"` | | +| `"DeadlineExceeded"` | | +| `"NotFound"` | | +| `"AlreadyExists"` | | +| `"PermissionDenied"` | | +| `"ResourceExhausted"` | | +| `"FailedPrecondition"` | | +| `"Aborted"` | | +| `"OutOfRange"` | | +| `"Unimplemented"` | | +| `"Internal"` | | +| `"Unavailable"` | | +| `"DataLoss"` | | +| `"Unauthenticated"` | | + +## Lz4Config + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `block_size` | integer (uint32) | โ€” | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | +| `max_decode_block_size` | integer (uint32) | โ€” | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt to deserialize. | + +## ClientTlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ca_file` | string | โ€” | โ€” | Path to the certificate authority to use to validate the remote. | +| `cert_file` | string | โ€” | โ€” | Path to the certificate file for client authentication. | +| `key_file` | string | โ€” | โ€” | Path to the private key file for client authentication. | +| `use_native_roots` | boolean | โ€” | false | If set the client will use the native roots for TLS connections. | + +## UploadCacheResultsStrategy + +| Value | Description | +| --- | --- | +| `"success_only"` | Only upload action results with an exit code of 0. | +| `"never"` | Don't upload any action results. | +| `"everything"` | Upload all action results that complete. | +| `"failures_only"` | Only upload action results that fail. | + +## ExperimentalRedisSchedulerBackend + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `redis_store` | string | Yes | โ€” | A reference to the Redis store to use for the scheduler. Note: This MUST resolve to a `RedisSpec`. | + +## PlatformPropertyAddition + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | โ€” | The name of the property to add. | +| `value` | string | Yes | โ€” | The value to assign to the property. | + +## PlatformPropertyReplacement + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | โ€” | The name of the property to replace. | +| `value` | string | โ€” | โ€” | The value to match against, if unset then any instance matches. | +| `new_name` | string | Yes | โ€” | The new name of the property. | +| `new_value` | string | โ€” | โ€” | The value to assign to the property, if unset will remain the same. | + +## HttpCompressionConfig + +Note: Compressing data in the cloud rarely has a benefit, since most +cloud providers have very high bandwidth backplanes. However, for +clients not inside the data center, it might be a good idea to +compress data to and from the cloud. This will however come at a high +CPU and performance cost. If you are making remote execution share the +same CAS/AC servers as client's remote cache, you can create multiple +services with different compression settings that are served on +different ports. Then configure the non-cloud clients to use one port +and cloud-clients to use another. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `send_compression_algorithm` | [HttpCompressionAlgorithm](#httpcompressionalgorithm) | โ€” | `HttpCompressionAlgorithm::None` | The compression algorithm that the server will use when sending responses to clients. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | +| `accepted_compression_algorithms` | array of [HttpCompressionAlgorithm](#httpcompressionalgorithm) | Yes | {no supported compression} | The compression algorithm that the server will accept from clients. The server will broadcast the supported compression algorithms to clients and the client will choose which compression algorithm to use. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | + +## HttpServerConfig + +Advanced HTTP configuration. These generally should not be set. +For documentation on these settings, see the hyper documentation: +See: [hyper HTTP server docs](https://docs.rs/hyper/latest/hyper/server/conn/struct.Http.html) + +Note: All of these default to the default values from hyper unless otherwise +specified. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `http2_keep_alive_interval` | integer (uint32) | โ€” | โ€” | Interval to send keep-alive pings via HTTP2. Note: This is in seconds. | +| `experimental_http2_max_pending_accept_reset_streams` | integer (uint32) | โ€” | โ€” | | +| `experimental_http2_initial_stream_window_size` | integer (uint32) | โ€” | โ€” | | +| `experimental_http2_initial_connection_window_size` | integer (uint32) | โ€” | โ€” | | +| `experimental_http2_adaptive_window` | boolean | โ€” | โ€” | | +| `experimental_http2_max_frame_size` | integer (uint32) | โ€” | โ€” | | +| `experimental_http2_max_concurrent_streams` | integer (uint32) | โ€” | โ€” | | +| `experimental_http2_keep_alive_timeout_s` | integer (uint32) | โ€” | โ€” | | +| `experimental_http2_max_send_buf_size` | integer (uint32) | โ€” | โ€” | | +| `experimental_http2_enable_connect_protocol` | boolean | โ€” | โ€” | | +| `experimental_http2_max_header_list_size` | integer (uint32) | โ€” | โ€” | | + +## TlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cert_file` | string | Yes | โ€” | Path to the certificate file. | +| `key_file` | string | Yes | โ€” | Path to the private key file. | +| `client_ca_file` | string | โ€” | โ€” | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | +| `client_crl_file` | string | โ€” | โ€” | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | + +## CasChunkingConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | string | โ€” | โ€” | The store name referenced in the `stores` map in the main config used to persist blob-to-chunks layouts. Keys are the digests of the original blobs and values are serialized chunk layouts (which do not hash to those digests), so this store MUST NOT perform content digest verification and MUST NOT be the same store as `cas_store` โ€” writing layouts into the CAS would overwrite blob content. Using the same store name as `cas_store` is rejected at startup. | +| `avg_chunk_size_bytes` | integer (uint64) | โ€” | 524288 (512 KiB) | The average chunk size in bytes advertised to clients through the `FastCDC` 2020 capability parameters and used for server-side chunking in `SplitBlob`. Clients derive the minimum and maximum chunk sizes from this value (avg / 4 and avg * 4). The value must be between 1 KiB and 1 MiB. | +| `max_chunk_count` | integer (uint64) | โ€” | 50000 | Maximum number of chunks accepted in a `SpliceBlob` request or produced by on-demand chunking in `SplitBlob`. Blobs that would produce more chunks are served without chunking (`SplitBlob` returns `NOT_FOUND` and clients fall back to a regular download). This bounds the size of stored chunk layouts and of `SplitBlobResponse` messages (roughly 80-140 bytes per chunk). At the default average chunk size the default cap supports blobs up to ~25 GiB; note that values above ~50000 may produce responses that exceed default gRPC message size limits on clients. | + +## CapabilitiesRemoteExecutionConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | โ€” | Scheduler used to configure the capabilities of remote execution. | + +## HttpCompressionAlgorithm + +| Value | Description | +| --- | --- | +| `"none"` | No compression. | +| `"gzip"` | Zlib compression. | + +## Reading the source + +If anything here disagrees with the binary, the source wins: + +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.7.2/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.7.2/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.7.2/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/lib/config-versions.ts b/web/apps/docs/lib/config-versions.ts index 5aa4afe5e0a..0c2d603ac66 100644 --- a/web/apps/docs/lib/config-versions.ts +++ b/web/apps/docs/lib/config-versions.ts @@ -28,13 +28,21 @@ export const CONFIG_VERSIONS: ConfigVersion[] = [ "isDev": true }, { - "version": "v1.7.2", - "label": "v1.7.2 (latest)", + "version": "v1.7.3", + "label": "v1.7.3 (latest)", "href": "/reference/nativelink-config", - "ref": "v1.7.2", + "ref": "v1.7.3", "isLatest": true, "isDev": false }, + { + "version": "v1.7.2", + "label": "v1.7.2", + "href": "/reference/nativelink-config/v1.7.2", + "ref": "v1.7.2", + "isLatest": false, + "isDev": false + }, { "version": "v1.7.1", "label": "v1.7.1", diff --git a/web/apps/docs/package.json b/web/apps/docs/package.json index a1264654ab4..f4fc5db3c28 100644 --- a/web/apps/docs/package.json +++ b/web/apps/docs/package.json @@ -22,7 +22,7 @@ "@types/react": "^19.0.7", "@types/react-dom": "^19.0.3", "tailwindcss": "^4.0.0", - "typescript": "^6.0.0" + "typescript": "^7.0.0" }, "private": true, "scripts": { diff --git a/web/apps/web/package.json b/web/apps/web/package.json index 01bc210ddfb..f936e69eef6 100644 --- a/web/apps/web/package.json +++ b/web/apps/web/package.json @@ -23,7 +23,7 @@ "@types/react": "^19.0.7", "@types/react-dom": "^19.0.3", "tailwindcss": "^4.0.0", - "typescript": "^6.0.0" + "typescript": "^7.0.0" }, "private": true, "scripts": { diff --git a/web/bun.lock b/web/bun.lock index e39ffe92fb7..1854dd49983 100644 --- a/web/bun.lock +++ b/web/bun.lock @@ -1,13 +1,12 @@ { "lockfileVersion": 1, - "configVersion": 0, "workspaces": { "": { "name": "nativelink-web-next", "devDependencies": { "@biomejs/biome": "1.9.4", "turbo": "^2.3.0", - "typescript": "^6.0.0", + "typescript": "^7.0.0", }, }, "apps/docs": { @@ -34,7 +33,7 @@ "@types/react": "^19.0.7", "@types/react-dom": "^19.0.3", "tailwindcss": "^4.0.0", - "typescript": "^6.0.0", + "typescript": "^7.0.0", }, }, "apps/web": { @@ -62,7 +61,7 @@ "@types/react": "^19.0.7", "@types/react-dom": "^19.0.3", "tailwindcss": "^4.0.0", - "typescript": "^6.0.0", + "typescript": "^7.0.0", }, }, "packages/config": { @@ -87,7 +86,7 @@ "@nativelink/config": "workspace:*", "@types/react": "^19.0.0", "@types/react-dom": "^19.0.0", - "typescript": "^6.0.0", + "typescript": "^7.0.0", }, "peerDependencies": { "react": "^19.0.0", @@ -519,6 +518,46 @@ "@types/unist": ["@types/unist@3.0.3", "", {}, "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="], + "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], + + "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], + + "@typescript/typescript-darwin-x64": ["@typescript/typescript-darwin-x64@7.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA=="], + + "@typescript/typescript-freebsd-arm64": ["@typescript/typescript-freebsd-arm64@7.0.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ=="], + + "@typescript/typescript-freebsd-x64": ["@typescript/typescript-freebsd-x64@7.0.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw=="], + + "@typescript/typescript-linux-arm": ["@typescript/typescript-linux-arm@7.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ=="], + + "@typescript/typescript-linux-arm64": ["@typescript/typescript-linux-arm64@7.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ=="], + + "@typescript/typescript-linux-loong64": ["@typescript/typescript-linux-loong64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ=="], + + "@typescript/typescript-linux-mips64el": ["@typescript/typescript-linux-mips64el@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA=="], + + "@typescript/typescript-linux-ppc64": ["@typescript/typescript-linux-ppc64@7.0.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA=="], + + "@typescript/typescript-linux-riscv64": ["@typescript/typescript-linux-riscv64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ=="], + + "@typescript/typescript-linux-s390x": ["@typescript/typescript-linux-s390x@7.0.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw=="], + + "@typescript/typescript-linux-x64": ["@typescript/typescript-linux-x64@7.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A=="], + + "@typescript/typescript-netbsd-arm64": ["@typescript/typescript-netbsd-arm64@7.0.2", "", { "os": "none", "cpu": "arm64" }, "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA=="], + + "@typescript/typescript-netbsd-x64": ["@typescript/typescript-netbsd-x64@7.0.2", "", { "os": "none", "cpu": "x64" }, "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA=="], + + "@typescript/typescript-openbsd-arm64": ["@typescript/typescript-openbsd-arm64@7.0.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ=="], + + "@typescript/typescript-openbsd-x64": ["@typescript/typescript-openbsd-x64@7.0.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg=="], + + "@typescript/typescript-sunos-x64": ["@typescript/typescript-sunos-x64@7.0.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g=="], + + "@typescript/typescript-win32-arm64": ["@typescript/typescript-win32-arm64@7.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ=="], + + "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], + "@ungap/structured-clone": ["@ungap/structured-clone@1.3.1", "", {}, "sha512-mUFwbeTqrVgDQxFveS+df2yfap6iuP20NAKAsBt5jDEoOTDew+zwLAOilHCeQJOVSvmgCX4ogqIrA0mnyr08yQ=="], "@upsetjs/venn.js": ["@upsetjs/venn.js@2.0.0", "", { "optionalDependencies": { "d3-selection": "^3.0.0", "d3-transition": "^3.0.1" } }, "sha512-WbBhLrooyePuQ1VZxrJjtLvTc4NVfpOyKx0sKqioq9bX1C1m7Jgykkn8gLrtwumBioXIqam8DLxp88Adbue6Hw=="], @@ -1041,7 +1080,7 @@ "turbo": ["turbo@2.9.15", "", { "optionalDependencies": { "@turbo/darwin-64": "2.9.15", "@turbo/darwin-arm64": "2.9.15", "@turbo/linux-64": "2.9.15", "@turbo/linux-arm64": "2.9.15", "@turbo/windows-64": "2.9.15", "@turbo/windows-arm64": "2.9.15" }, "bin": { "turbo": "bin/turbo" } }, "sha512-VpKvD9Z0Hu/xrGUAYX1wnhfpqv835wIwGqeKfulvBPTOcDap0E3nFwyzCAVV85fB1sBcBDEfTP+7FSW7GzwWSQ=="], - "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], + "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], diff --git a/web/package.json b/web/package.json index 1d424ec262c..54187e1118e 100644 --- a/web/package.json +++ b/web/package.json @@ -4,7 +4,7 @@ "devDependencies": { "@biomejs/biome": "1.9.4", "turbo": "^2.3.0", - "typescript": "^6.0.0" + "typescript": "^7.0.0" }, "engines": { "node": ">=20" diff --git a/web/packages/ui/package.json b/web/packages/ui/package.json index e827deef4d3..c6b6990e202 100644 --- a/web/packages/ui/package.json +++ b/web/packages/ui/package.json @@ -12,7 +12,7 @@ "@nativelink/config": "workspace:*", "@types/react": "^19.0.0", "@types/react-dom": "^19.0.0", - "typescript": "^6.0.0" + "typescript": "^7.0.0" }, "exports": { ".": "./src/index.ts",