From b280bb33d90aebfe5ba61d8dee3bc5a8235f6b86 Mon Sep 17 00:00:00 2001
From: Aman Kumar <aman@tracemachina.com>
Date: Wed, 14 Oct 2026 19:13:18 -0400
Subject: [PATCH 1/3] Bound object store reads with an adaptive permit pool,
 Azure first

---
 nativelink-config/src/stores.rs               |  38 +++
 nativelink-store/src/azure_blob_store.rs      |  87 +++++-
 .../tests/azure_blob_store_test.rs            | 101 ++++++-
 nativelink-util/BUILD.bazel                   |   2 +
 nativelink-util/src/adaptive_permits.rs       | 211 ++++++++++++++
 nativelink-util/src/lib.rs                    |   1 +
 nativelink-util/src/metrics.rs                |  31 ++
 .../tests/adaptive_permits_test.rs            | 168 +++++++++++
 .../docs/content/docs/reference/metrics.mdx   | 266 ++++++++++--------
 9 files changed, 770 insertions(+), 135 deletions(-)
 create mode 100644 nativelink-util/src/adaptive_permits.rs
 create mode 100644 nativelink-util/tests/adaptive_permits_test.rs

diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs
index bd05c9383f1..7af41a99103 100644
--- a/nativelink-config/src/stores.rs
+++ b/nativelink-config/src/stores.rs
@@ -1433,6 +1433,41 @@ pub struct CommonObjectSpec {
     /// Default: 120.0
     #[serde(default = "default_throttle_retry_budget_s")]
     pub throttle_retry_budget_s: f32,
+
+    /// Most reads (`GET` and `HEAD`) this process keeps in flight against
+    /// the object store at once. Further reads wait in the process for a
+    /// permit instead of being sent and refused, which turns an account
+    /// over its limit into queueing here rather than wasted requests and
+    /// retries in lockstep. 0 (the default) leaves reads unbounded, as
+    /// before.
+    ///
+    /// Default: 0
+    #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")]
+    pub max_concurrent_reads: usize,
+
+    /// With `max_concurrent_reads` set, let the bound move: a throttled
+    /// response (503 `ServerBusy` or 429) halves it, not below
+    /// `min_concurrent_reads`, and a run of successful reads raises it by
+    /// one back toward `max_concurrent_reads`. Many processes behind one
+    /// account then converge on a share of its limit. Off, the bound is
+    /// fixed at `max_concurrent_reads`.
+    ///
+    /// Default: false
+    #[serde(default, deserialize_with = "convert_boolean_with_shellexpand")]
+    pub adaptive_read_concurrency: bool,
+
+    /// Floor for the adaptive read bound.
+    ///
+    /// Default: 4
+    #[serde(
+        default = "default_min_concurrent_reads",
+        deserialize_with = "convert_numeric_with_shellexpand"
+    )]
+    pub min_concurrent_reads: usize,
+}
+
+const fn default_min_concurrent_reads() -> usize {
+    4
 }
 
 const fn default_throttle_retry_delay_s() -> f32 {
@@ -1460,6 +1495,9 @@ impl Default for CommonObjectSpec {
             throttle_retry_delay_s: default_throttle_retry_delay_s(),
             throttle_retry_max_delay_s: default_throttle_retry_max_delay_s(),
             throttle_retry_budget_s: default_throttle_retry_budget_s(),
+            max_concurrent_reads: 0,
+            adaptive_read_concurrency: false,
+            min_concurrent_reads: default_min_concurrent_reads(),
         }
     }
 }
diff --git a/nativelink-store/src/azure_blob_store.rs b/nativelink-store/src/azure_blob_store.rs
index 0de1fd508b9..c6eb918408a 100644
--- a/nativelink-store/src/azure_blob_store.rs
+++ b/nativelink-store/src/azure_blob_store.rs
@@ -35,6 +35,7 @@ use futures::{FutureExt, StreamExt, TryStreamExt};
 use nativelink_config::stores::ExperimentalAzureSpec;
 use nativelink_error::{Code, Error, ResultExt, make_err};
 use nativelink_metric::MetricsComponent;
+use nativelink_util::adaptive_permits::{AdaptivePermit, AdaptivePermits};
 use nativelink_util::buf_channel::{
     DropCloserReadHalf, DropCloserWriteHalf, make_buf_channel_pair,
 };
@@ -74,6 +75,9 @@ const DEFAULT_MAX_CONCURRENT_UPLOADS: usize = 10;
 // Default public Azure Blob Storage endpoint suffix.
 const DEFAULT_BLOB_ENDPOINT_SUFFIX: &str = "blob.core.windows.net";
 
+/// Successful reads in a row before the adaptive read limit grows by one.
+const READ_LIMIT_GROW_AFTER: usize = 16;
+
 #[derive(MetricsComponent)]
 pub struct AzureBlobStore<NowFn> {
     client: Arc<BlobContainerClient>,
@@ -91,6 +95,9 @@ pub struct AzureBlobStore<NowFn> {
     max_concurrent_uploads: usize,
     throttle_retry_delay: Duration,
     throttle_retry_max_delay: Duration,
+    /// Bounds `GET` and `HEAD` requests in flight; `None` leaves them
+    /// unbounded. See `CommonObjectSpec::max_concurrent_reads`.
+    read_permits: Option<Arc<AdaptivePermits>>,
 }
 
 impl<NowFn> core::fmt::Debug for AzureBlobStore<NowFn> {
@@ -204,6 +211,18 @@ where
                 spec.common.throttle_retry_delay_s
             ));
         }
+        let read_permits = (spec.common.max_concurrent_reads > 0).then(|| {
+            let max = spec.common.max_concurrent_reads;
+            let (min, grow_after) = if spec.common.adaptive_read_concurrency {
+                (
+                    spec.common.min_concurrent_reads.min(max),
+                    READ_LIMIT_GROW_AFTER,
+                )
+            } else {
+                (max, 0)
+            };
+            AdaptivePermits::new("azure", max, min, max, grow_after)
+        });
         let throttle_retry_budget =
             Duration::try_from_secs_f32(spec.common.throttle_retry_budget_s).map_err(|e| {
                 make_err!(
@@ -239,9 +258,39 @@ where
                 .unwrap_or(DEFAULT_MAX_CONCURRENT_UPLOADS),
             throttle_retry_delay,
             throttle_retry_max_delay,
+            read_permits,
         }))
     }
 
+    /// The read limit the permit pool stands at, or `None` with reads
+    /// unbounded.
+    pub fn read_limit(&self) -> Option<usize> {
+        self.read_permits.as_ref().map(|pool| pool.limit())
+    }
+
+    /// A read permit when reads are bounded. Held for the request, body
+    /// included, so what is in flight is what the limit says.
+    async fn read_permit(&self) -> Option<AdaptivePermit> {
+        match &self.read_permits {
+            Some(pool) => Some(pool.acquire().await),
+            None => None,
+        }
+    }
+
+    /// A read was answered (found or not): one toward growing the limit.
+    fn note_read_ok(&self) {
+        if let Some(pool) = &self.read_permits {
+            pool.record_success();
+        }
+    }
+
+    /// A read was throttled: the limit is cut.
+    fn note_read_throttled(&self) {
+        if let Some(pool) = &self.read_permits {
+            pool.record_throttle();
+        }
+    }
+
     fn make_blob_path(&self, key: &StoreKey<'_>) -> String {
         format!("{}{}", self.blob_prefix, key.as_str())
     }
@@ -317,6 +366,7 @@ where
                 let blob_path = blob_path.clone();
                 let client = Arc::clone(&self.client);
                 async move {
+                    let _read_permit = self.read_permit().await;
                     let _permit = match fs::get_permit().await {
                         Ok(permit) => permit,
                         Err(e) => {
@@ -334,6 +384,7 @@ where
 
                     match result {
                         Ok(props) => {
+                            self.note_read_ok();
                             if self.consider_expired_after_s > 0
                                 && let Some(last_modified) = props.last_modified().ok().flatten()
                             {
@@ -368,19 +419,21 @@ where
                                         state,
                                     ));
                                 }
+                                self.note_read_ok();
                                 Some((RetryResult::Ok(None), state))
                             } else {
-                                Some((
-                                    self.retry_later(
-                                        &err,
-                                        "has",
-                                        make_err!(
-                                            Code::Unavailable,
-                                            "Failed to get blob properties: {err:?}"
-                                        ),
+                                let retry = self.retry_later(
+                                    &err,
+                                    "has",
+                                    make_err!(
+                                        Code::Unavailable,
+                                        "Failed to get blob properties: {err:?}"
                                     ),
-                                    state,
-                                ))
+                                );
+                                if matches!(retry, RetryResult::RetryAfter(..)) {
+                                    self.note_read_throttled();
+                                }
+                                Some((retry, state))
                             }
                         }
                     }
@@ -780,6 +833,9 @@ where
                 let client = Arc::clone(&self.client);
                 let blob_path = blob_path.clone();
                 async move {
+                    // Held through the body: what is in flight is what the
+                    // read limit says, not only what has been started.
+                    let _read_permit = self.read_permit().await;
                     let _permit = match fs::get_permit().await {
                         Ok(permit) => permit,
                         Err(e) => {
@@ -874,11 +930,18 @@ where
                     .await;
 
                     match result {
-                        Ok(()) => Some((RetryResult::Ok(()), writer)),
+                        Ok(()) => {
+                            self.note_read_ok();
+                            Some((RetryResult::Ok(()), writer))
+                        }
                         Err((e, _)) if e.code == Code::NotFound => {
+                            self.note_read_ok();
                             Some((RetryResult::Err(e), writer))
                         }
-                        Err((e, Some(wait))) => Some((RetryResult::RetryAfter(e, wait), writer)),
+                        Err((e, Some(wait))) => {
+                            self.note_read_throttled();
+                            Some((RetryResult::RetryAfter(e, wait), writer))
+                        }
                         Err((e, None)) => Some((RetryResult::Retry(e), writer)),
                     }
                 }
diff --git a/nativelink-store/tests/azure_blob_store_test.rs b/nativelink-store/tests/azure_blob_store_test.rs
index 685786b329a..2791c8b3274 100644
--- a/nativelink-store/tests/azure_blob_store_test.rs
+++ b/nativelink-store/tests/azure_blob_store_test.rs
@@ -200,6 +200,20 @@ fn create_test_store(
     mock: Arc<MockTransport>,
     retry: Option<Retry>,
     consider_expired_after_s: Option<u32>,
+) -> Result<TestStore, Error> {
+    create_test_store_with_common(
+        mock,
+        CommonObjectSpec {
+            consider_expired_after_s: consider_expired_after_s.unwrap_or(0),
+            retry: retry.unwrap_or_default(),
+            ..Default::default()
+        },
+    )
+}
+
+fn create_test_store_with_common(
+    mock: Arc<MockTransport>,
+    common: CommonObjectSpec,
 ) -> Result<TestStore, Error> {
     let mut options = BlobContainerClientOptions::default();
     // NativeLink's own `Retrier` wraps each operation, so disable the SDK's retry
@@ -217,11 +231,7 @@ fn create_test_store(
     let spec = ExperimentalAzureSpec {
         account_name: TEST_ACCOUNT.to_string(),
         container: TEST_CONTAINER.to_string(),
-        common: CommonObjectSpec {
-            consider_expired_after_s: consider_expired_after_s.unwrap_or(0),
-            retry: retry.unwrap_or_default(),
-            ..Default::default()
-        },
+        common,
         ..Default::default()
     };
 
@@ -847,3 +857,84 @@ async fn the_requested_range_starts_at_the_offset_plus_what_was_sent() -> Result
     }
     Ok(())
 }
+
+/// With adaptive read concurrency on, each throttled response halves the
+/// read limit (the two here are a retry wait apart, so they are two
+/// events), and a run of answered reads grows it back one at a time.
+#[nativelink_test(start_paused = true)]
+async fn throttled_reads_halve_the_read_limit_and_successes_regrow_it() -> Result<(), Error> {
+    const VALUE: &str = "test_content";
+
+    let mock = MockTransport::new();
+    mock.push_download(error_response(StatusCode::ServiceUnavailable));
+    mock.push_download(error_response(StatusCode::ServiceUnavailable));
+    mock.push_download(download_ok(StatusCode::Ok, VALUE.as_bytes().to_vec()));
+
+    let store = create_test_store_with_common(
+        Arc::clone(&mock),
+        CommonObjectSpec {
+            retry: Retry {
+                max_retries: 0,
+                delay: 0.0,
+                jitter: 0.0,
+                ..Default::default()
+            },
+            max_concurrent_reads: 16,
+            adaptive_read_concurrency: true,
+            min_concurrent_reads: 2,
+            ..Default::default()
+        },
+    )?;
+    assert_eq!(store.read_limit(), Some(16));
+
+    let result = store
+        .get_part_unchunked(create_test_digest()?, 0, None)
+        .await?;
+    assert_eq!(result, VALUE.as_bytes());
+    assert_eq!(mock.request_count(), 3);
+    assert_eq!(
+        store.read_limit(),
+        Some(4),
+        "two throttles, two halvings; the download that got through is one toward regrowth"
+    );
+
+    // Sixteen answered reads in a row (the download above was the first)
+    // add one permit.
+    for _ in 0..15 {
+        mock.push_properties(properties_ok(512));
+        store.has(create_test_digest()?).await?;
+    }
+    assert_eq!(store.read_limit(), Some(5));
+    Ok(())
+}
+
+/// Without `adaptive_read_concurrency` the bound is fixed at
+/// `max_concurrent_reads` whatever the store answers.
+#[nativelink_test(start_paused = true)]
+async fn a_fixed_read_limit_does_not_move_on_a_throttle() -> Result<(), Error> {
+    const VALUE: &str = "test_content";
+
+    let mock = MockTransport::new();
+    mock.push_download(error_response(StatusCode::ServiceUnavailable));
+    mock.push_download(download_ok(StatusCode::Ok, VALUE.as_bytes().to_vec()));
+
+    let store = create_test_store_with_common(
+        Arc::clone(&mock),
+        CommonObjectSpec {
+            retry: Retry {
+                max_retries: 0,
+                delay: 0.0,
+                jitter: 0.0,
+                ..Default::default()
+            },
+            max_concurrent_reads: 8,
+            ..Default::default()
+        },
+    )?;
+
+    store
+        .get_part_unchunked(create_test_digest()?, 0, None)
+        .await?;
+    assert_eq!(store.read_limit(), Some(8));
+    Ok(())
+}
diff --git a/nativelink-util/BUILD.bazel b/nativelink-util/BUILD.bazel
index 43f61050daf..bc008ef5ea0 100644
--- a/nativelink-util/BUILD.bazel
+++ b/nativelink-util/BUILD.bazel
@@ -14,6 +14,7 @@ rust_library(
     name = "nativelink-util",
     srcs = [
         "src/action_messages.rs",
+        "src/adaptive_permits.rs",
         "src/buf_channel.rs",
         "src/channel_body_for_tests.rs",
         "src/chunked_stream.rs",
@@ -100,6 +101,7 @@ rust_test_suite(
     timeout = "short",
     srcs = [
         "tests/action_messages_test.rs",
+        "tests/adaptive_permits_test.rs",
         "tests/buf_channel_test.rs",
         "tests/cache_size_metrics_test.rs",
         "tests/channel_body_for_tests_test.rs",
diff --git a/nativelink-util/src/adaptive_permits.rs b/nativelink-util/src/adaptive_permits.rs
new file mode 100644
index 00000000000..1f3473df2d9
--- /dev/null
+++ b/nativelink-util/src/adaptive_permits.rs
@@ -0,0 +1,211 @@
+// Copyright 2026 The NativeLink Authors. All rights reserved.
+//
+// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    See LICENSE file for details
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use core::sync::atomic::{AtomicUsize, Ordering};
+use core::time::Duration;
+use std::sync::Arc;
+
+use parking_lot::Mutex;
+use tokio::sync::{OwnedSemaphorePermit, Semaphore};
+use tokio::time::Instant;
+use tracing::{debug, info};
+
+use crate::metrics::{record_object_store_read_limit, record_object_store_read_wait};
+
+/// A throttle within this long of the last cut is the same event seen by
+/// another request in flight, not a second signal.
+const CUT_WINDOW: Duration = Duration::from_millis(200);
+
+/// A pool of permits in front of a shared resource whose capacity is not
+/// known, sized the way TCP sizes its window: a rejection halves the limit,
+/// a run of successes raises it by one. Requests wait for a permit inside
+/// the process instead of being sent and refused, and many processes
+/// behind one limit converge on a share of it.
+///
+/// Built for reads against a cloud object store that answers 503 or 429
+/// when an account is over its limit; the store calls [`record_throttle`]
+/// on such a response and [`record_success`] on a completed request.
+///
+/// [`record_throttle`]: Self::record_throttle
+/// [`record_success`]: Self::record_success
+#[derive(Debug)]
+pub struct AdaptivePermits {
+    semaphore: Arc<Semaphore>,
+    /// The current limit, between `min` and `max`.
+    limit: AtomicUsize,
+    min: usize,
+    max: usize,
+    /// Permits a cut could not take back because they were out; each is
+    /// forgotten as it is released instead of returned.
+    owed: AtomicUsize,
+    /// Successes since the limit last moved.
+    streak: AtomicUsize,
+    /// Successes in a row needed to add a permit. Zero fixes the limit.
+    grow_after: usize,
+    last_cut: Mutex<Option<Instant>>,
+    /// Who the pool belongs to, the `store.object.provider` label.
+    name: &'static str,
+}
+
+/// One permit from an [`AdaptivePermits`] pool, held for the length of a
+/// request.
+#[derive(Debug)]
+pub struct AdaptivePermit {
+    pool: Arc<AdaptivePermits>,
+    inner: Option<OwnedSemaphorePermit>,
+}
+
+impl AdaptivePermits {
+    /// A pool that starts at `initial` permits and moves between `min` and
+    /// `max` as [`record_throttle`] and [`record_success`] are called; with
+    /// `grow_after` zero the limit never grows and only falls. `initial` is
+    /// clamped into `[min, max]`.
+    ///
+    /// [`record_throttle`]: Self::record_throttle
+    /// [`record_success`]: Self::record_success
+    pub fn new(
+        name: &'static str,
+        initial: usize,
+        min: usize,
+        max: usize,
+        grow_after: usize,
+    ) -> Arc<Self> {
+        let min = min.max(1);
+        let max = max.max(min);
+        let initial = initial.clamp(min, max);
+        record_object_store_read_limit(name, initial);
+        Arc::new(Self {
+            semaphore: Arc::new(Semaphore::new(initial)),
+            limit: AtomicUsize::new(initial),
+            min,
+            max,
+            owed: AtomicUsize::new(0),
+            streak: AtomicUsize::new(0),
+            grow_after,
+            last_cut: Mutex::new(None),
+            name,
+        })
+    }
+
+    /// Waits for a permit. The permit is returned to the pool when dropped,
+    /// unless the limit was cut while it was out.
+    pub async fn acquire(self: &Arc<Self>) -> AdaptivePermit {
+        if self.semaphore.available_permits() == 0 {
+            record_object_store_read_wait(self.name);
+        }
+        let inner = Arc::clone(&self.semaphore)
+            .acquire_owned()
+            .await
+            .expect("AdaptivePermits semaphore is never closed");
+        AdaptivePermit {
+            pool: Arc::clone(self),
+            inner: Some(inner),
+        }
+    }
+
+    /// The limit as it stands.
+    pub fn limit(&self) -> usize {
+        self.limit.load(Ordering::Acquire)
+    }
+
+    /// Permits that can be taken right now.
+    pub fn available(&self) -> usize {
+        self.semaphore.available_permits()
+    }
+
+    /// A request went through. After `grow_after` of these in a row the
+    /// limit rises by one, up to `max`.
+    pub fn record_success(&self) {
+        if self.grow_after == 0 {
+            return;
+        }
+        let streak = self.streak.fetch_add(1, Ordering::AcqRel) + 1;
+        if streak < self.grow_after {
+            return;
+        }
+        self.streak.store(0, Ordering::Release);
+        let _guard = self.last_cut.lock();
+        let limit = self.limit.load(Ordering::Acquire);
+        if limit >= self.max {
+            return;
+        }
+        // A permit still owed from a cut is cancelled instead of a new one
+        // being added, so the pool never holds more than `limit`.
+        if self
+            .owed
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |owed| {
+                owed.checked_sub(1)
+            })
+            .is_err()
+        {
+            self.semaphore.add_permits(1);
+        }
+        self.limit.store(limit + 1, Ordering::Release);
+        record_object_store_read_limit(self.name, limit + 1);
+        debug!(pool = self.name, limit = limit + 1, "Read limit raised");
+    }
+
+    /// The resource refused a request for being busy. The limit halves, not
+    /// below `min`; permits that are out are forgotten as they come back.
+    /// A second throttle within `CUT_WINDOW` of the first is the same event
+    /// and does not cut again.
+    pub fn record_throttle(&self) {
+        let mut last_cut = self.last_cut.lock();
+        let now = Instant::now();
+        if last_cut.is_some_and(|at| now.duration_since(at) < CUT_WINDOW) {
+            return;
+        }
+        *last_cut = Some(now);
+        self.streak.store(0, Ordering::Release);
+        let limit = self.limit.load(Ordering::Acquire);
+        let target = (limit / 2).max(self.min);
+        if target >= limit {
+            return;
+        }
+        let cut = limit - target;
+        // Take back what is idle now; the rest is owed and leaves with the
+        // permits that hold it.
+        let idle = self.semaphore.available_permits().min(cut);
+        self.semaphore.forget_permits(idle);
+        self.owed.fetch_add(cut - idle, Ordering::AcqRel);
+        self.limit.store(target, Ordering::Release);
+        record_object_store_read_limit(self.name, target);
+        info!(
+            pool = self.name,
+            from = limit,
+            to = target,
+            "Read limit halved after a throttled response"
+        );
+    }
+}
+
+impl Drop for AdaptivePermit {
+    fn drop(&mut self) {
+        let Some(permit) = self.inner.take() else {
+            return;
+        };
+        if self
+            .pool
+            .owed
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |owed| {
+                owed.checked_sub(1)
+            })
+            .is_ok()
+        {
+            permit.forget();
+        } else {
+            drop(permit);
+        }
+    }
+}
diff --git a/nativelink-util/src/lib.rs b/nativelink-util/src/lib.rs
index d082f3c3a5f..0974adbe66e 100644
--- a/nativelink-util/src/lib.rs
+++ b/nativelink-util/src/lib.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 pub mod action_messages;
+pub mod adaptive_permits;
 pub mod buf_channel;
 pub mod channel_body_for_tests;
 pub mod chunked_stream;
diff --git a/nativelink-util/src/metrics.rs b/nativelink-util/src/metrics.rs
index cd6405b1576..ad315397483 100644
--- a/nativelink-util/src/metrics.rs
+++ b/nativelink-util/src/metrics.rs
@@ -1478,6 +1478,18 @@ pub static STORE_METRICS: LazyLock<StoreMetrics> = LazyLock::new(|| {
             .with_description("Object store responses that asked the client to slow down (HTTP 503 ServerBusy or 429), by provider and operation")
             .with_unit("{response}")
             .build(),
+
+        object_read_limit: meter
+            .u64_gauge("store.object.read_limit")
+            .with_description("Reads this process keeps in flight against an object store at most, as the adaptive permit pool stands: halved on a throttled response, raised by one after a run of successes")
+            .with_unit("{request}")
+            .build(),
+
+        object_read_waits: meter
+            .u64_counter("store.object.read_waits")
+            .with_description("Object store reads that found every read permit taken and waited for one, by provider")
+            .with_unit("{request}")
+            .build(),
     }
 });
 
@@ -1486,6 +1498,25 @@ pub static STORE_METRICS: LazyLock<StoreMetrics> = LazyLock::new(|| {
 pub struct StoreMetrics {
     /// Throttled responses, by provider and operation.
     pub object_throttles: metrics::Counter<u64>,
+    /// The adaptive read limit as it stands, by provider.
+    pub object_read_limit: metrics::Gauge<u64>,
+    /// Reads that waited for a read permit, by provider.
+    pub object_read_waits: metrics::Counter<u64>,
+}
+
+/// Records the read limit an object store's permit pool settled on.
+pub fn record_object_store_read_limit(provider: &'static str, limit: usize) {
+    STORE_METRICS.object_read_limit.record(
+        u64::try_from(limit).unwrap_or(u64::MAX),
+        &[KeyValue::new(STORE_OBJECT_PROVIDER, provider)],
+    );
+}
+
+/// Records a read that found no permit free and waited for one.
+pub fn record_object_store_read_wait(provider: &'static str) {
+    STORE_METRICS
+        .object_read_waits
+        .add(1, &[KeyValue::new(STORE_OBJECT_PROVIDER, provider)]);
 }
 
 /// Records a response from an object store that asked the client to slow down.
diff --git a/nativelink-util/tests/adaptive_permits_test.rs b/nativelink-util/tests/adaptive_permits_test.rs
new file mode 100644
index 00000000000..6cec2c70ba5
--- /dev/null
+++ b/nativelink-util/tests/adaptive_permits_test.rs
@@ -0,0 +1,168 @@
+// Copyright 2026 The NativeLink Authors. All rights reserved.
+//
+// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    See LICENSE file for details
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use core::time::Duration;
+
+use nativelink_error::Error;
+use nativelink_macro::nativelink_test;
+use nativelink_util::adaptive_permits::AdaptivePermits;
+use pretty_assertions::assert_eq;
+
+/// A throttle halves the limit; the floor holds.
+#[nativelink_test(start_paused = true)]
+async fn a_throttle_halves_the_limit_down_to_the_floor() -> Result<(), Error> {
+    let pool = AdaptivePermits::new("test", 64, 4, 256, 10);
+    assert_eq!(pool.limit(), 64);
+    pool.record_throttle();
+    assert_eq!(pool.limit(), 32);
+    assert_eq!(pool.available(), 32, "idle permits are taken back at once");
+    for _ in 0..10 {
+        tokio::time::sleep(Duration::from_millis(250)).await;
+        pool.record_throttle();
+    }
+    assert_eq!(pool.limit(), 4, "never below the floor");
+    assert_eq!(pool.available(), 4);
+    Ok(())
+}
+
+/// Two throttles within one round trip are one event.
+#[nativelink_test]
+async fn throttles_in_the_same_window_cut_once() -> Result<(), Error> {
+    let pool = AdaptivePermits::new("test", 64, 4, 256, 10);
+    pool.record_throttle();
+    pool.record_throttle();
+    pool.record_throttle();
+    assert_eq!(pool.limit(), 32);
+    Ok(())
+}
+
+/// A run of successes adds one permit; the ceiling holds.
+#[nativelink_test]
+async fn a_streak_of_successes_grows_the_limit_to_the_ceiling() -> Result<(), Error> {
+    let pool = AdaptivePermits::new("test", 8, 4, 10, 3);
+    for _ in 0..2 {
+        pool.record_success();
+    }
+    assert_eq!(pool.limit(), 8, "two of three is not a streak");
+    pool.record_success();
+    assert_eq!(pool.limit(), 9);
+    assert_eq!(pool.available(), 9);
+    for _ in 0..30 {
+        pool.record_success();
+    }
+    assert_eq!(pool.limit(), 10, "never above the ceiling");
+    assert_eq!(pool.available(), 10);
+    Ok(())
+}
+
+/// A throttle resets the streak, so growth starts over from the cut.
+#[nativelink_test]
+async fn a_throttle_resets_the_streak() -> Result<(), Error> {
+    let pool = AdaptivePermits::new("test", 8, 4, 16, 3);
+    pool.record_success();
+    pool.record_success();
+    pool.record_throttle();
+    assert_eq!(pool.limit(), 4);
+    pool.record_success();
+    assert_eq!(pool.limit(), 4, "the two before the cut do not count");
+    pool.record_success();
+    pool.record_success();
+    assert_eq!(pool.limit(), 5);
+    Ok(())
+}
+
+/// A cut while permits are out takes back what is idle and forgets the
+/// rest as they are released, so the pool never exceeds the new limit.
+#[nativelink_test]
+async fn a_cut_with_permits_out_is_settled_on_release() -> Result<(), Error> {
+    let pool = AdaptivePermits::new("test", 8, 2, 8, 0);
+    let held: Vec<_> = {
+        let mut held = Vec::new();
+        for _ in 0..6 {
+            held.push(pool.acquire().await);
+        }
+        held
+    };
+    assert_eq!(pool.available(), 2);
+    pool.record_throttle();
+    assert_eq!(pool.limit(), 4);
+    assert_eq!(pool.available(), 0, "the two idle permits were taken back");
+    drop(held);
+    assert_eq!(
+        pool.available(),
+        4,
+        "two of the six released permits were forgotten, four returned"
+    );
+    Ok(())
+}
+
+/// Growth while permits are still owed cancels an owed permit instead of
+/// adding one, so the pool size tracks the limit exactly.
+#[nativelink_test]
+async fn growth_while_owed_cancels_the_debt_first() -> Result<(), Error> {
+    let pool = AdaptivePermits::new("test", 8, 2, 8, 1);
+    let held: Vec<_> = {
+        let mut held = Vec::new();
+        for _ in 0..8 {
+            held.push(pool.acquire().await);
+        }
+        held
+    };
+    pool.record_throttle();
+    assert_eq!(pool.limit(), 4, "all four cut permits are owed");
+    pool.record_success();
+    assert_eq!(pool.limit(), 5);
+    assert_eq!(pool.available(), 0, "the growth cancelled one owed permit");
+    drop(held);
+    assert_eq!(pool.available(), 5, "three forgotten, five returned");
+    Ok(())
+}
+
+/// With `grow_after` zero the limit only ever falls.
+#[nativelink_test]
+async fn a_fixed_pool_never_grows() -> Result<(), Error> {
+    let pool = AdaptivePermits::new("test", 8, 4, 8, 0);
+    for _ in 0..100 {
+        pool.record_success();
+    }
+    assert_eq!(pool.limit(), 8);
+    pool.record_throttle();
+    assert_eq!(pool.limit(), 4);
+    for _ in 0..100 {
+        pool.record_success();
+    }
+    assert_eq!(pool.limit(), 4);
+    Ok(())
+}
+
+/// Acquire waits for a release when the pool is empty.
+#[nativelink_test]
+async fn acquire_waits_for_a_free_permit() -> Result<(), Error> {
+    let pool = AdaptivePermits::new("test", 1, 1, 1, 0);
+    let first = pool.acquire().await;
+    let waiter = tokio::spawn({
+        let pool = pool.clone();
+        async move {
+            let _second = pool.acquire().await;
+        }
+    });
+    tokio::time::sleep(Duration::from_millis(50)).await;
+    assert!(!waiter.is_finished(), "no permit to take yet");
+    drop(first);
+    tokio::time::timeout(Duration::from_secs(5), waiter)
+        .await
+        .expect("the waiter got the released permit")
+        .expect("the waiter task did not panic");
+    Ok(())
+}
diff --git a/web/apps/docs/content/docs/reference/metrics.mdx b/web/apps/docs/content/docs/reference/metrics.mdx
index bdde26a82f8..f9ba11f8258 100644
--- a/web/apps/docs/content/docs/reference/metrics.mdx
+++ b/web/apps/docs/content/docs/reference/metrics.mdx
@@ -5,12 +5,12 @@ full: true
 ---
 
 {/* AUTOGENERATED. Do not edit by hand.
-   Source: nativelink-util/src/metrics.rs @ v1.7.6 (ddde488d)
+   Source: nativelink-util/src/metrics.rs @ v1.7.7 (274164f0e)
    Regenerate from web/: bun --filter @nativelink/docs gen:metrics-reference */}
 
-NativeLink **v1.7.6** declares 50 OpenTelemetry instruments. This page is
+NativeLink **v1.7.7** declares 52 OpenTelemetry instruments. This page is
 generated from the declarations in
-[`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs)
+[`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs)
 and from every `.add()` and `.record()` call site across the workspace, so the
 **Emitted** column is not a claim, it is the result of looking.
 
@@ -67,6 +67,8 @@ wrong.
 | [`scheduler.sweep.failures`](#scheduler-sweep-failures) | `nativelink_scheduler_sweep_failures` (unreferenced) | Counter | Conditionally | scheduler process only |
 | [`scheduler.hints.resolutions`](#scheduler-hints-resolutions) | `nativelink_scheduler_hints_resolutions` (unreferenced) | Counter | Conditionally | scheduler process only |
 | [`store.object.throttles`](#store-object-throttles) | `nativelink_store_object_throttles` (unreferenced) | Counter | Yes | whenever the code path runs |
+| [`store.object.read_limit`](#store-object-read-limit) | `nativelink_store_object_read_limit` (unreferenced) | Gauge | Yes | whenever the code path runs |
+| [`store.object.read_waits`](#store-object-read-waits) | `nativelink_store_object_read_waits` (unreferenced) | Counter | Yes | whenever the code path runs |
 | [`store.tier.operations`](#store-tier-operations) | `nativelink_store_tier_operations` (unreferenced) | Counter | Conditionally | only through a `fast_slow` store |
 | [`store.tier.io`](#store-tier-io) | `nativelink_store_tier_io` (unreferenced) | Counter | Conditionally | only through a `fast_slow` store |
 | [`health.checks`](#health-checks) | `nativelink_health_checks` (unreferenced) | Counter | Yes | whenever the code path runs |
@@ -82,7 +84,7 @@ binary. The names in the middle column above are what the shipped OpenTelemetry
 collector produces after two transformations:
 
 1. The Prometheus exporter in
-   [`otel-collector-config.yaml`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/deployment-examples/metrics/otel-collector-config.yaml)
+   [`otel-collector-config.yaml`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/deployment-examples/metrics/otel-collector-config.yaml)
    is configured with `namespace: nativelink`, which prefixes every series with
    `nativelink_`.
 2. The exporter rewrites the OTLP name into Prometheus form: dots become
@@ -109,7 +111,7 @@ Every instrument declared in `metrics.rs` has at least one call site.
 
 Global cache metrics instruments.
 
-Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:449`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L449), backed by the `CacheMetrics` struct.
+Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:449`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L449), backed by the `CacheMetrics` struct.
 
 Every cache instrument is emitted from one place: the `cache_metrics` store wrapper. A store that is not wrapped emits nothing, no matter how the collector is configured.
 
@@ -124,8 +126,8 @@ Histogram of cache operation durations in milliseconds
 | Unit | `ms` |
 | Prometheus series | `nativelink_cache_operation_duration` |
 | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec |
-| Declared at | [`nativelink-util/src/metrics.rs:453`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L453) |
-| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:73`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L73) |
+| Declared at | [`nativelink-util/src/metrics.rs:453`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L453) |
+| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:73`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L73) |
 
 **Histogram buckets:** 19 explicit boundaries, from
 `0.001` to `5000.0` `ms`. Anything past the last boundary lands in the
@@ -143,8 +145,8 @@ Counter of cache operations by type and result
 | Unit | — |
 | Prometheus series | `nativelink_cache_operations_total` |
 | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec |
-| Declared at | [`nativelink-util/src/metrics.rs:487`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L487) |
-| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:127`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L127), [`nativelink-store/src/cache_metrics_store.rs:132`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L132), [`nativelink-store/src/cache_metrics_store.rs:143`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L143), [`nativelink-store/src/cache_metrics_store.rs:168`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L168), [`nativelink-store/src/cache_metrics_store.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L174), [`nativelink-store/src/cache_metrics_store.rs:200`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L200), [`nativelink-store/src/cache_metrics_store.rs:206`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L206), [`nativelink-store/src/cache_metrics_store.rs:219`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L219), [`nativelink-store/src/cache_metrics_store.rs:225`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L225), [`nativelink-store/src/cache_metrics_store.rs:246`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L246), [`nativelink-store/src/cache_metrics_store.rs:254`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L254), [`nativelink-store/src/cache_metrics_store.rs:260`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L260) |
+| Declared at | [`nativelink-util/src/metrics.rs:487`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L487) |
+| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:127`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L127), [`nativelink-store/src/cache_metrics_store.rs:132`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L132), [`nativelink-store/src/cache_metrics_store.rs:143`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L143), [`nativelink-store/src/cache_metrics_store.rs:168`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L168), [`nativelink-store/src/cache_metrics_store.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L174), [`nativelink-store/src/cache_metrics_store.rs:200`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L200), [`nativelink-store/src/cache_metrics_store.rs:206`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L206), [`nativelink-store/src/cache_metrics_store.rs:219`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L219), [`nativelink-store/src/cache_metrics_store.rs:225`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L225), [`nativelink-store/src/cache_metrics_store.rs:246`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L246), [`nativelink-store/src/cache_metrics_store.rs:254`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L254), [`nativelink-store/src/cache_metrics_store.rs:260`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L260) |
 
 ### `cache.io`
 
@@ -157,8 +159,8 @@ Counter of bytes read/written during cache operations
 | Unit | `By` |
 | Prometheus series | `nativelink_cache_io_total` |
 | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec |
-| Declared at | [`nativelink-util/src/metrics.rs:492`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L492) |
-| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:80`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L80), [`nativelink-store/src/cache_metrics_store.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L248) |
+| Declared at | [`nativelink-util/src/metrics.rs:492`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L492) |
+| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:80`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L80), [`nativelink-store/src/cache_metrics_store.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L248) |
 
 ### `cache.size`
 
@@ -171,8 +173,8 @@ Current total size of all cached data in bytes
 | Unit | `By` |
 | Prometheus series | `nativelink_cache_size` |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:498`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L498) |
-| Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) |
+| Declared at | [`nativelink-util/src/metrics.rs:498`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L498) |
+| Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/evicting_map.rs#L468) |
 
 ### `cache.entries`
 
@@ -185,8 +187,8 @@ Current number of entries in cache
 | Unit | `{entry}` |
 | Prometheus series | `nativelink_cache_entries` |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:504`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L504) |
-| Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) |
+| Declared at | [`nativelink-util/src/metrics.rs:504`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L504) |
+| Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/evicting_map.rs#L468) |
 
 ### `cache.item.size`
 
@@ -199,14 +201,14 @@ Histogram of individual cache entry sizes in bytes
 | Unit | `By` |
 | Prometheus series | `nativelink_cache_item_size` |
 | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec |
-| Declared at | [`nativelink-util/src/metrics.rs:510`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L510) |
-| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:83`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L83) |
+| Declared at | [`nativelink-util/src/metrics.rs:510`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L510) |
+| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:83`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs#L83) |
 
 ## Execution metrics
 
 Global remote execution metrics instruments.
 
-Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:550`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L550), backed by the `ExecutionMetrics` struct.
+Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:550`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L550), backed by the `ExecutionMetrics` struct.
 
 Execution instruments are emitted from the scheduler process as actions change state, plus the per-action CPU time and peak memory a worker reports back when it finishes. Worker processes emit no execution metrics of their own.
 
@@ -221,8 +223,8 @@ Histogram of stage durations in seconds
 | Unit | `s` |
 | Prometheus series | `nativelink_execution_stage_duration` |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:554`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L554) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) |
+| Declared at | [`nativelink-util/src/metrics.rs:554`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L554) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) |
 
 **Histogram buckets:** 15 explicit boundaries, from
 `0.001` to `3600.0` `s`. Anything past the last boundary lands in the
@@ -240,8 +242,8 @@ Histogram of total execution durations in seconds
 | Unit | `s` |
 | Prometheus series | `nativelink_execution_total_duration` |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:579`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L579) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) |
+| Declared at | [`nativelink-util/src/metrics.rs:579`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L579) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) |
 
 **Histogram buckets:** 13 explicit boundaries, from
 `0.01` to `7200.0` `s`. Anything past the last boundary lands in the
@@ -259,8 +261,8 @@ Histogram of queue wait times in seconds
 | Unit | `s` |
 | Prometheus series | `nativelink_execution_queue_time` |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:604`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L604) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) |
+| Declared at | [`nativelink-util/src/metrics.rs:604`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L604) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) |
 
 **Histogram buckets:** 12 explicit boundaries, from
 `0.001` to `600.0` `s`. Anything past the last boundary lands in the
@@ -278,8 +280,8 @@ Current number of actions in each stage
 | Unit | `{action}` |
 | Prometheus series | `nativelink_execution_active_count` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:624`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L624) |
-| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:425`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L425), [`nativelink-scheduler/src/memory_awaited_action_db.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L672), [`nativelink-scheduler/src/memory_awaited_action_db.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L679), [`nativelink-scheduler/src/memory_awaited_action_db.rs:824`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L824), [`nativelink-scheduler/src/store_awaited_action_db.rs:716`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/store_awaited_action_db.rs#L716) |
+| Declared at | [`nativelink-util/src/metrics.rs:624`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L624) |
+| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:425`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/memory_awaited_action_db.rs#L425), [`nativelink-scheduler/src/memory_awaited_action_db.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/memory_awaited_action_db.rs#L672), [`nativelink-scheduler/src/memory_awaited_action_db.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/memory_awaited_action_db.rs#L679), [`nativelink-scheduler/src/memory_awaited_action_db.rs:824`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/memory_awaited_action_db.rs#L824), [`nativelink-scheduler/src/store_awaited_action_db.rs:716`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/store_awaited_action_db.rs#L716) |
 
 ### `execution.completed.count`
 
@@ -292,8 +294,8 @@ Total number of completed executions
 | Unit | `{action}` |
 | Prometheus series | `nativelink_execution_completed_count_total` |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:630`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L630) |
-| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:163`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L163) |
+| Declared at | [`nativelink-util/src/metrics.rs:630`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L630) |
+| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:163`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L163) |
 
 ### `execution.stage.transitions`
 
@@ -306,8 +308,8 @@ Number of stage transitions
 | Unit | `{transition}` |
 | Prometheus series | `nativelink_execution_stage_transitions_total` |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:636`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L636) |
-| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:142`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L142) |
+| Declared at | [`nativelink-util/src/metrics.rs:636`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L636) |
+| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:142`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L142) |
 
 ### `execution.cpu.time`
 
@@ -320,8 +322,8 @@ CPU time an action used, as sampled by the worker.
 | Unit | `s` |
 | Prometheus series | `nativelink_execution_cpu_time` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:642`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L642) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2480`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2480) |
+| Declared at | [`nativelink-util/src/metrics.rs:642`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L642) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2480`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L2480) |
 
 **Histogram buckets:** 12 explicit boundaries, from
 `0.1` to `7200.0` `s`. Anything past the last boundary lands in the
@@ -339,8 +341,8 @@ Peak memory an action used, as sampled by the worker.
 | Unit | `By` |
 | Prometheus series | `nativelink_execution_peak_memory` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:651`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L651) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2472`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2472) |
+| Declared at | [`nativelink-util/src/metrics.rs:651`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L651) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2472`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L2472) |
 
 **Histogram buckets:** 8 explicit boundaries, from
 `1_048_576.0` to `68_719_476_736.0` `By`. Anything past the last boundary lands in the
@@ -358,8 +360,8 @@ Histogram of output sizes in bytes
 | Unit | `By` |
 | Prometheus series | `nativelink_execution_output_size` |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:667`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L667) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) |
+| Declared at | [`nativelink-util/src/metrics.rs:667`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L667) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) |
 
 **Histogram buckets:** 8 explicit boundaries, from
 `1_024.0` to `10_737_418_240.0` `By`. Anything past the last boundary lands in the
@@ -377,14 +379,14 @@ Counter for execution retries
 | Unit | `{retry}` |
 | Prometheus series | `nativelink_execution_retry_count_total` |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:683`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L683) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:175`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L175) |
+| Declared at | [`nativelink-util/src/metrics.rs:683`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L683) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:175`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L175) |
 
 ## Worker fleet metrics
 
 Global worker fleet metrics instruments.
 
-Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:892`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L892), backed by the `WorkerMetrics` struct.
+Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:892`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L892), backed by the `WorkerMetrics` struct.
 
 Worker fleet instruments are emitted from the scheduler process as workers connect, heartbeat, pause, drain and leave. They describe the pool as this scheduler instance sees it.
 
@@ -399,8 +401,8 @@ Workers currently connected.
 | Unit | `{worker}` |
 | Prometheus series | `nativelink_worker_connected_count` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:896`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L896) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:581`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L581), [`nativelink-scheduler/src/api_worker_scheduler.rs:2019`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2019) |
+| Declared at | [`nativelink-util/src/metrics.rs:896`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L896) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:581`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L581), [`nativelink-scheduler/src/api_worker_scheduler.rs:2019`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L2019) |
 
 ### `worker.connections`
 
@@ -413,8 +415,8 @@ Workers that have joined, cumulative.
 | Unit | `{worker}` |
 | Prometheus series | `nativelink_worker_connections` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:902`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L902) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:581`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L581) |
+| Declared at | [`nativelink-util/src/metrics.rs:902`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L902) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:581`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L581) |
 
 ### `worker.disconnections`
 
@@ -427,8 +429,8 @@ Workers that have left, cumulative, by reason.
 | Unit | `{worker}` |
 | Prometheus series | `nativelink_worker_disconnections` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:908`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L908) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2019`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2019) |
+| Declared at | [`nativelink-util/src/metrics.rs:908`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L908) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2019`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L2019) |
 
 ### `worker.input.relinks`
 
@@ -441,8 +443,8 @@ Input hardlinks made again after their source was evicted mid-staging, by kind.
 | Unit | — |
 | Prometheus series | `nativelink_worker_input_relinks` (not referenced by any shipped rule or dashboard) |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:914`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L914) |
-| Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1038), [`nativelink-worker/src/running_actions_manager.rs:1172`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1172), [`nativelink-worker/src/running_actions_manager.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1190) |
+| Declared at | [`nativelink-util/src/metrics.rs:914`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L914) |
+| Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-worker/src/running_actions_manager.rs#L1038), [`nativelink-worker/src/running_actions_manager.rs:1172`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-worker/src/running_actions_manager.rs#L1172), [`nativelink-worker/src/running_actions_manager.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-worker/src/running_actions_manager.rs#L1190) |
 
 ### `worker.prepare.retries`
 
@@ -455,8 +457,8 @@ Input fetches tried again on the worker after a transient store failure: `retrie
 | Unit | — |
 | Prometheus series | `nativelink_worker_prepare_retries` (not referenced by any shipped rule or dashboard) |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L919) |
-| Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1014`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1014) |
+| Declared at | [`nativelink-util/src/metrics.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L919) |
+| Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1014`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-worker/src/running_actions_manager.rs#L1014) |
 
 ### `scheduler.dispatch.requeues`
 
@@ -469,8 +471,8 @@ Dispatches requeued untried, by reason.
 | Unit | — |
 | Prometheus series | `nativelink_scheduler_dispatch_requeues` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:924`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L924) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1680`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1680), [`nativelink-scheduler/src/api_worker_scheduler.rs:1909`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1909), [`nativelink-scheduler/src/api_worker_scheduler.rs:2251`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2251), [`nativelink-scheduler/src/api_worker_scheduler.rs:2568`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2568), [`nativelink-scheduler/src/api_worker_scheduler.rs:2599`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2599), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:680`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L680) |
+| Declared at | [`nativelink-util/src/metrics.rs:924`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L924) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1680`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L1680), [`nativelink-scheduler/src/api_worker_scheduler.rs:1909`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L1909), [`nativelink-scheduler/src/api_worker_scheduler.rs:2251`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L2251), [`nativelink-scheduler/src/api_worker_scheduler.rs:2568`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L2568), [`nativelink-scheduler/src/api_worker_scheduler.rs:2599`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L2599), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:680`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L680) |
 
 ### `worker.keepalives`
 
@@ -483,8 +485,8 @@ Keepalives received, cumulative.
 | Unit | `{keepalive}` |
 | Prometheus series | `nativelink_worker_keepalives` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:929`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L929) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:539`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L539) |
+| Declared at | [`nativelink-util/src/metrics.rs:929`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L929) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:539`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L539) |
 
 ### `worker.keepalive.gap`
 
@@ -497,8 +499,8 @@ Seconds between consecutive messages from a worker.
 | Unit | `s` |
 | Prometheus series | `nativelink_worker_keepalive_gap` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L935) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:473`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L473) |
+| Declared at | [`nativelink-util/src/metrics.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L935) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:473`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L473) |
 
 **Histogram buckets:** 12 explicit boundaries, from
 `1.0` to `60.0` `s`. Anything past the last boundary lands in the
@@ -516,8 +518,8 @@ Connected workers currently paused or draining.
 | Unit | `{worker}` |
 | Prometheus series | `nativelink_worker_state_count` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:946`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L946) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:522`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L522), [`nativelink-scheduler/src/api_worker_scheduler.rs:575`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L575), [`nativelink-scheduler/src/api_worker_scheduler.rs:578`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L578), [`nativelink-scheduler/src/api_worker_scheduler.rs:696`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L696), [`nativelink-scheduler/src/api_worker_scheduler.rs:1641`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1641), [`nativelink-scheduler/src/api_worker_scheduler.rs:1683`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1683), [`nativelink-scheduler/src/api_worker_scheduler.rs:1835`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1835) |
+| Declared at | [`nativelink-util/src/metrics.rs:946`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L946) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:522`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L522), [`nativelink-scheduler/src/api_worker_scheduler.rs:575`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L575), [`nativelink-scheduler/src/api_worker_scheduler.rs:578`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L578), [`nativelink-scheduler/src/api_worker_scheduler.rs:696`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L696), [`nativelink-scheduler/src/api_worker_scheduler.rs:1641`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L1641), [`nativelink-scheduler/src/api_worker_scheduler.rs:1683`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L1683), [`nativelink-scheduler/src/api_worker_scheduler.rs:1835`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L1835) |
 
 ## gRPC server metrics
 
@@ -526,7 +528,7 @@ Global gRPC serving metrics.
 One duration histogram covers rate, errors and latency: the count gives
 rate, the status attribute separates errors, and the buckets give latency.
 
-Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1070`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1070), backed by the `RpcMetrics` struct.
+Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1070`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1070), backed by the `RpcMetrics` struct.
 
 Emitted by the `OtlpLayer` tower middleware on every gRPC server NativeLink starts, so a CAS, a scheduler and a worker API endpoint each report their own.
 
@@ -541,8 +543,8 @@ Duration of inbound gRPC calls, by service, method and status.
 | Unit | `s` |
 | Prometheus series | `nativelink_rpc_server_duration` (not referenced by any shipped rule or dashboard) |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:1074`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1074) |
-| Emitted from | [`nativelink-util/src/telemetry.rs:371`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/telemetry.rs#L371) |
+| Declared at | [`nativelink-util/src/metrics.rs:1074`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1074) |
+| Emitted from | [`nativelink-util/src/telemetry.rs:371`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/telemetry.rs#L371) |
 
 **Histogram buckets:** 15 explicit boundaries, from
 `0.001` to `300.0` `s`. Anything past the last boundary lands in the
@@ -553,7 +555,7 @@ measurement.
 
 Global scheduler metrics.
 
-Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1123`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1123), backed by the `SchedulerOtlpMetrics` struct.
+Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1123`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1123), backed by the `SchedulerOtlpMetrics` struct.
 
 Emitted from the scheduler process once per matching pass, whether or not the pass assigned anything.
 
@@ -569,8 +571,8 @@ while the queue is non-empty means matching is the bottleneck.
 | Unit | `s` |
 | Prometheus series | `nativelink_scheduler_matching_duration` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1127`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1127) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:478`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L478) |
+| Declared at | [`nativelink-util/src/metrics.rs:1127`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1127) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:478`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L478) |
 
 **Histogram buckets:** 13 explicit boundaries, from
 `0.0001` to `10.0` `s`. Anything past the last boundary lands in the
@@ -588,8 +590,8 @@ Matching passes, by result.
 | Unit | `{pass}` |
 | Prometheus series | `nativelink_scheduler_matching_passes` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1136`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1136) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:478`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L478) |
+| Declared at | [`nativelink-util/src/metrics.rs:1136`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1136) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:478`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L478) |
 
 ### `scheduler.unsatisfiable.queued`
 
@@ -604,8 +606,8 @@ instance, judged against that instance's own workers.
 | Unit | `{action}` |
 | Prometheus series | `nativelink_scheduler_unsatisfiable_queued` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1142`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1142) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1073`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1073) |
+| Declared at | [`nativelink-util/src/metrics.rs:1142`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1142) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1073`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L1073) |
 
 ### `scheduler.unsatisfiable.failed`
 
@@ -619,8 +621,8 @@ the properties that could not be satisfied.
 | Unit | `{action}` |
 | Prometheus series | `nativelink_scheduler_unsatisfiable_failed` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1150`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1150) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:580`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L580) |
+| Declared at | [`nativelink-util/src/metrics.rs:1150`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1150) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:580`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L580) |
 
 ### `scheduler.queue.depth`
 
@@ -633,8 +635,8 @@ Queued actions the last matching pass read.
 | Unit | `{action}` |
 | Prometheus series | `nativelink_scheduler_queue_depth` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1156`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1156) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1074`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1074) |
+| Declared at | [`nativelink-util/src/metrics.rs:1156`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1156) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1074`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L1074) |
 
 ### `scheduler.queue.age`
 
@@ -647,8 +649,8 @@ Queued actions the last matching pass read, by age bucket.
 | Unit | `{action}` |
 | Prometheus series | `nativelink_scheduler_queue_age` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1162`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1162) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1075`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1075) |
+| Declared at | [`nativelink-util/src/metrics.rs:1162`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1162) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1075`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L1075) |
 
 ### `scheduler.queue.oldest_age`
 
@@ -661,8 +663,8 @@ Age of the oldest action the last matching pass read, in seconds.
 | Unit | `s` |
 | Prometheus series | `nativelink_scheduler_queue_oldest_age` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1170`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1170) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1075`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1075) |
+| Declared at | [`nativelink-util/src/metrics.rs:1170`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1170) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1075`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L1075) |
 
 ### `scheduler.executing.age`
 
@@ -675,8 +677,8 @@ Executing actions the last executing sweep read, by age bucket.
 | Unit | `{action}` |
 | Prometheus series | `nativelink_scheduler_executing_age` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1178`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1178) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1349`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1349) |
+| Declared at | [`nativelink-util/src/metrics.rs:1178`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1178) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1349`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L1349) |
 
 ### `scheduler.executing.oldest_age`
 
@@ -690,8 +692,8 @@ executing, in seconds.
 | Unit | `s` |
 | Prometheus series | `nativelink_scheduler_executing_oldest_age` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1186`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1186) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1349`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1349) |
+| Declared at | [`nativelink-util/src/metrics.rs:1186`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1186) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1349`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L1349) |
 
 ### `scheduler.queue.retired`
 
@@ -704,8 +706,8 @@ Queued actions retired for having no client.
 | Unit | `{action}` |
 | Prometheus series | `nativelink_scheduler_queue_retired` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1194`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1194) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:784`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L784) |
+| Declared at | [`nativelink-util/src/metrics.rs:1194`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1194) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:784`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L784) |
 
 ### `scheduler.matching.parked_dispatched`
 
@@ -719,8 +721,8 @@ opened before the pass ended, ahead of the newer actions it was reading.
 | Unit | `{action}` |
 | Prometheus series | `nativelink_scheduler_matching_parked_dispatched` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1200`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1200) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1050`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1050) |
+| Declared at | [`nativelink-util/src/metrics.rs:1200`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1200) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1050`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L1050) |
 
 ### `scheduler.worker.holds`
 
@@ -734,8 +736,8 @@ began or ended the hold.
 | Unit | `{hold}` |
 | Prometheus series | `nativelink_scheduler_worker_holds` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1208`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1208) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:719`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L719), [`nativelink-scheduler/src/api_worker_scheduler.rs:932`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L932) |
+| Declared at | [`nativelink-util/src/metrics.rs:1208`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1208) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:719`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L719), [`nativelink-scheduler/src/api_worker_scheduler.rs:932`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L932) |
 
 ### `scheduler.worker.held`
 
@@ -748,8 +750,8 @@ Workers held for such an action right now.
 | Unit | `{worker}` |
 | Prometheus series | `nativelink_scheduler_worker_held` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1216`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1216) |
-| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:720`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L720), [`nativelink-scheduler/src/api_worker_scheduler.rs:933`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L933) |
+| Declared at | [`nativelink-util/src/metrics.rs:1216`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1216) |
+| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:720`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L720), [`nativelink-scheduler/src/api_worker_scheduler.rs:933`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/api_worker_scheduler.rs#L933) |
 
 ### `scheduler.awaited_action.orphans`
 
@@ -762,8 +764,8 @@ Listed operations whose record was gone on read, by site.
 | Unit | `{operation}` |
 | Prometheus series | `nativelink_scheduler_awaited_action_orphans` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1222`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1222) |
-| Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:620`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L620), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:615`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L615), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:732`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L732), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1663`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1663), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1686`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1686) |
+| Declared at | [`nativelink-util/src/metrics.rs:1222`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1222) |
+| Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:620`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L620), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:615`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L615), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:732`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L732), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1663`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1663), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1686`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1686) |
 
 ### `scheduler.sweep.failures`
 
@@ -776,8 +778,8 @@ Abandoned-queue sweeps that failed before finishing.
 | Unit | `{sweep}` |
 | Prometheus series | `nativelink_scheduler_sweep_failures` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1230`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1230) |
-| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1301`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1301), [`nativelink-scheduler/src/simple_scheduler.rs:1360`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1360) |
+| Declared at | [`nativelink-util/src/metrics.rs:1230`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1230) |
+| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1301`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L1301), [`nativelink-scheduler/src/simple_scheduler.rs:1360`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler.rs#L1360) |
 
 ### `scheduler.hints.resolutions`
 
@@ -790,14 +792,14 @@ Actions sized by the historical resource scheduler, by source.
 | Unit | `{action}` |
 | Prometheus series | `nativelink_scheduler_hints_resolutions` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. scheduler process only |
-| Declared at | [`nativelink-util/src/metrics.rs:1236`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1236) |
-| Emitted from | [`nativelink-scheduler/src/historical_resource_scheduler.rs:643`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L643), [`nativelink-scheduler/src/historical_resource_scheduler.rs:647`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L647), [`nativelink-scheduler/src/historical_resource_scheduler.rs:650`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L650) |
+| Declared at | [`nativelink-util/src/metrics.rs:1236`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1236) |
+| Emitted from | [`nativelink-scheduler/src/historical_resource_scheduler.rs:643`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/historical_resource_scheduler.rs#L643), [`nativelink-scheduler/src/historical_resource_scheduler.rs:647`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/historical_resource_scheduler.rs#L647), [`nativelink-scheduler/src/historical_resource_scheduler.rs:650`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/historical_resource_scheduler.rs#L650) |
 
 ## STORE_METRICS
 
 Global object-store metrics.
 
-Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1472`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1472), backed by the `StoreMetrics` struct.
+Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1472`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1472), backed by the `StoreMetrics` struct.
 
 ### `store.object.throttles`
 
@@ -810,14 +812,42 @@ Throttled responses, by provider and operation.
 | Unit | `{response}` |
 | Prometheus series | `nativelink_store_object_throttles` (not referenced by any shipped rule or dashboard) |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:1476`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1476) |
-| Emitted from | [`nativelink-store/src/azure_blob_store.rs:269`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/azure_blob_store.rs#L269) |
+| Declared at | [`nativelink-util/src/metrics.rs:1476`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1476) |
+| Emitted from | [`nativelink-store/src/azure_blob_store.rs:315`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/azure_blob_store.rs#L315) |
+
+### `store.object.read_limit`
+
+The adaptive read limit as it stands, by provider.
+
+| Property | Value |
+| --- | --- |
+| Instrument type | `Gauge<u64>` |
+| Field | `object_read_limit` |
+| Unit | `{request}` |
+| Prometheus series | `nativelink_store_object_read_limit` (not referenced by any shipped rule or dashboard) |
+| Emitted | Yes. whenever the code path runs |
+| Declared at | [`nativelink-util/src/metrics.rs:1482`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1482) |
+| Emitted from | [`nativelink-util/src/adaptive_permits.rs:87`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/adaptive_permits.rs#L87), [`nativelink-util/src/adaptive_permits.rs:155`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/adaptive_permits.rs#L155), [`nativelink-util/src/adaptive_permits.rs:183`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/adaptive_permits.rs#L183) |
+
+### `store.object.read_waits`
+
+Reads that waited for a read permit, by provider.
+
+| Property | Value |
+| --- | --- |
+| Instrument type | `Counter<u64>` |
+| Field | `object_read_waits` |
+| Unit | `{request}` |
+| Prometheus series | `nativelink_store_object_read_waits` (not referenced by any shipped rule or dashboard) |
+| Emitted | Yes. whenever the code path runs |
+| Declared at | [`nativelink-util/src/metrics.rs:1488`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1488) |
+| Emitted from | [`nativelink-util/src/adaptive_permits.rs:105`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/adaptive_permits.rs#L105) |
 
 ## Store tier metrics
 
 Global tiered-store metrics.
 
-Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1503`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1503), backed by the `StoreTierMetrics` struct.
+Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1534`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1534), backed by the `StoreTierMetrics` struct.
 
 Emitted from the `fast_slow` store wrapper only. A deployment without a `fast_slow` store emits none of these.
 
@@ -832,8 +862,8 @@ Reads served per tier, by result.
 | Unit | `{operation}` |
 | Prometheus series | `nativelink_store_tier_operations` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. only through a `fast_slow` store |
-| Declared at | [`nativelink-util/src/metrics.rs:1507`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1507) |
-| Emitted from | [`nativelink-store/src/fast_slow_store.rs:348`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L348), [`nativelink-store/src/fast_slow_store.rs:839`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L839), [`nativelink-store/src/fast_slow_store.rs:855`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L855), [`nativelink-store/src/fast_slow_store.rs:865`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L865), [`nativelink-store/src/fast_slow_store.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L882), [`nativelink-store/src/fast_slow_store.rs:912`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L912), [`nativelink-store/src/fast_slow_store.rs:934`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L934) |
+| Declared at | [`nativelink-util/src/metrics.rs:1538`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1538) |
+| Emitted from | [`nativelink-store/src/fast_slow_store.rs:348`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L348), [`nativelink-store/src/fast_slow_store.rs:839`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L839), [`nativelink-store/src/fast_slow_store.rs:855`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L855), [`nativelink-store/src/fast_slow_store.rs:865`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L865), [`nativelink-store/src/fast_slow_store.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L882), [`nativelink-store/src/fast_slow_store.rs:912`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L912), [`nativelink-store/src/fast_slow_store.rs:934`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L934) |
 
 ### `store.tier.io`
 
@@ -846,14 +876,14 @@ Bytes moved per tier and direction.
 | Unit | `By` |
 | Prometheus series | `nativelink_store_tier_io` (not referenced by any shipped rule or dashboard) |
 | Emitted | Conditionally. only through a `fast_slow` store |
-| Declared at | [`nativelink-util/src/metrics.rs:1513`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1513) |
-| Emitted from | [`nativelink-store/src/fast_slow_store.rs:357`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L357), [`nativelink-store/src/fast_slow_store.rs:856`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L856), [`nativelink-store/src/fast_slow_store.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L913), [`nativelink-store/src/fast_slow_store.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L935) |
+| Declared at | [`nativelink-util/src/metrics.rs:1544`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1544) |
+| Emitted from | [`nativelink-store/src/fast_slow_store.rs:357`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L357), [`nativelink-store/src/fast_slow_store.rs:856`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L856), [`nativelink-store/src/fast_slow_store.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L913), [`nativelink-store/src/fast_slow_store.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/fast_slow_store.rs#L935) |
 
 ## Health check metrics
 
 Global health-check metrics.
 
-Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1554`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1554), backed by the `HealthMetrics` struct.
+Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1585`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1585), backed by the `HealthMetrics` struct.
 
 Emitted each time the health check endpoint runs, one observation per registered health indicator.
 
@@ -868,14 +898,14 @@ Health check results, by namespace and status.
 | Unit | `{check}` |
 | Prometheus series | `nativelink_health_checks` (not referenced by any shipped rule or dashboard) |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:1558`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1558) |
-| Emitted from | [`nativelink-util/src/health_utils.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/health_utils.rs#L248) |
+| Declared at | [`nativelink-util/src/metrics.rs:1589`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1589) |
+| Emitted from | [`nativelink-util/src/health_utils.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/health_utils.rs#L248) |
 
 ## Connection pool metrics
 
 Global connection-pool metrics.
 
-Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1585`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1585), backed by the `ConnectionMetrics` struct.
+Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1616`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1616), backed by the `ConnectionMetrics` struct.
 
 Emitted by the gRPC connection manager and the Redis store as they hand out and re-establish connections. A deployment that uses neither emits none of these.
 
@@ -890,8 +920,8 @@ Free slots at the moment a connection was taken.
 | Unit | `{connection}` |
 | Prometheus series | `nativelink_connection_pool_available` (not referenced by any shipped rule or dashboard) |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:1589`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1589) |
-| Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:342`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L342), [`nativelink-util/src/connection_manager.rs:352`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L352) |
+| Declared at | [`nativelink-util/src/metrics.rs:1620`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1620) |
+| Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:342`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/connection_manager.rs#L342), [`nativelink-util/src/connection_manager.rs:352`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/connection_manager.rs#L352) |
 
 **Histogram buckets:** 11 explicit boundaries, from
 `0.0` to `512.0` `{connection}`. Anything past the last boundary lands in the
@@ -909,8 +939,8 @@ Acquisitions, by pool and result.
 | Unit | `{acquisition}` |
 | Prometheus series | `nativelink_connection_pool_acquisitions` (not referenced by any shipped rule or dashboard) |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:1598`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1598) |
-| Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:342`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L342), [`nativelink-util/src/connection_manager.rs:352`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L352) |
+| Declared at | [`nativelink-util/src/metrics.rs:1629`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1629) |
+| Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:342`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/connection_manager.rs#L342), [`nativelink-util/src/connection_manager.rs:352`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/connection_manager.rs#L352) |
 
 ### `connection.reconnects`
 
@@ -923,14 +953,14 @@ Reconnects, by pool.
 | Unit | `{reconnect}` |
 | Prometheus series | `nativelink_connection_reconnects` (not referenced by any shipped rule or dashboard) |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:1604`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1604) |
-| Emitted from | [`nativelink-store/src/redis_store.rs:319`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L319) |
+| Declared at | [`nativelink-util/src/metrics.rs:1635`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1635) |
+| Emitted from | [`nativelink-store/src/redis_store.rs:319`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/redis_store.rs#L319) |
 
 ## RETRY_METRICS
 
 Global metrics for operations run under a [`crate::retry::Retrier`].
 
-Declared as `RETRY_METRICS` in [`nativelink-util/src/metrics.rs:1685`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1685), backed by the `RetryMetrics` struct.
+Declared as `RETRY_METRICS` in [`nativelink-util/src/metrics.rs:1716`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1716), backed by the `RetryMetrics` struct.
 
 ### `retry.outcomes`
 
@@ -943,8 +973,8 @@ Retry events, by client and outcome.
 | Unit | `{event}` |
 | Prometheus series | `nativelink_retry_outcomes` (not referenced by any shipped rule or dashboard) |
 | Emitted | Yes. whenever the code path runs |
-| Declared at | [`nativelink-util/src/metrics.rs:1689`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1689) |
-| Emitted from | [`nativelink-util/src/retry.rs:171`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/retry.rs#L171) |
+| Declared at | [`nativelink-util/src/metrics.rs:1720`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs#L1720) |
+| Emitted from | [`nativelink-util/src/retry.rs:171`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/retry.rs#L171) |
 
 ## Attributes
 
@@ -1067,10 +1097,10 @@ underscores.
 
 If anything here disagrees with the binary, the source wins:
 
-- [`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs): every declaration
-- [`nativelink-util/src/telemetry.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/telemetry.rs): the OTLP exporter and its interval
-- [`nativelink-store/src/cache_metrics_store.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs): every cache call site
-- [`nativelink-scheduler/src/simple_scheduler_state_manager.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs): every execution call site
+- [`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/metrics.rs): every declaration
+- [`nativelink-util/src/telemetry.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-util/src/telemetry.rs): the OTLP exporter and its interval
+- [`nativelink-store/src/cache_metrics_store.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-store/src/cache_metrics_store.rs): every cache call site
+- [`nativelink-scheduler/src/simple_scheduler_state_manager.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.7/nativelink-scheduler/src/simple_scheduler_state_manager.rs): every execution call site
 
 ## Where to go next
 

From 6971cfb458cf359305e0a858bc5b2493443970fc Mon Sep 17 00:00:00 2001
From: Aman Kumar <aman@tracemachina.com>
Date: Wed, 14 Oct 2026 20:24:30 -0400
Subject: [PATCH 2/3] Settle owed permits with a compare-exchange loop

---
 nativelink-util/src/adaptive_permits.rs | 36 ++++++++++++++-----------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/nativelink-util/src/adaptive_permits.rs b/nativelink-util/src/adaptive_permits.rs
index 1f3473df2d9..0d82fefe76c 100644
--- a/nativelink-util/src/adaptive_permits.rs
+++ b/nativelink-util/src/adaptive_permits.rs
@@ -114,6 +114,25 @@ impl AdaptivePermits {
         }
     }
 
+    /// Settles one owed permit if any is owed. True when it did.
+    fn take_owed(&self) -> bool {
+        let mut owed = self.owed.load(Ordering::Acquire);
+        loop {
+            if owed == 0 {
+                return false;
+            }
+            match self.owed.compare_exchange_weak(
+                owed,
+                owed - 1,
+                Ordering::AcqRel,
+                Ordering::Acquire,
+            ) {
+                Ok(_) => return true,
+                Err(current) => owed = current,
+            }
+        }
+    }
+
     /// The limit as it stands.
     pub fn limit(&self) -> usize {
         self.limit.load(Ordering::Acquire)
@@ -142,13 +161,7 @@ impl AdaptivePermits {
         }
         // A permit still owed from a cut is cancelled instead of a new one
         // being added, so the pool never holds more than `limit`.
-        if self
-            .owed
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |owed| {
-                owed.checked_sub(1)
-            })
-            .is_err()
-        {
+        if !self.take_owed() {
             self.semaphore.add_permits(1);
         }
         self.limit.store(limit + 1, Ordering::Release);
@@ -195,14 +208,7 @@ impl Drop for AdaptivePermit {
         let Some(permit) = self.inner.take() else {
             return;
         };
-        if self
-            .pool
-            .owed
-            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |owed| {
-                owed.checked_sub(1)
-            })
-            .is_ok()
-        {
+        if self.pool.take_owed() {
             permit.forget();
         } else {
             drop(permit);

From 0058b343ea7cb785a290fcabfd69171ca237cb41 Mon Sep 17 00:00:00 2001
From: Aman Kumar <aman@tracemachina.com>
Date: Fri, 16 Oct 2026 10:38:19 -0400
Subject: [PATCH 3/3] Owe what a cut could not forget and keep existence checks
 out of the download pool

---
 nativelink-config/src/stores.rs               | 21 +++++----
 nativelink-store/src/azure_blob_store.rs      | 31 ++++++-------
 .../tests/azure_blob_store_test.rs            | 46 ++++++++++++++++++-
 nativelink-util/src/adaptive_permits.rs       |  8 ++--
 4 files changed, 75 insertions(+), 31 deletions(-)

diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs
index 7af41a99103..4f6a8bce03c 100644
--- a/nativelink-config/src/stores.rs
+++ b/nativelink-config/src/stores.rs
@@ -1434,21 +1434,24 @@ pub struct CommonObjectSpec {
     #[serde(default = "default_throttle_retry_budget_s")]
     pub throttle_retry_budget_s: f32,
 
-    /// Most reads (`GET` and `HEAD`) this process keeps in flight against
-    /// the object store at once. Further reads wait in the process for a
-    /// permit instead of being sent and refused, which turns an account
-    /// over its limit into queueing here rather than wasted requests and
-    /// retries in lockstep. 0 (the default) leaves reads unbounded, as
-    /// before.
+    /// Most downloads (`GET`) this process keeps in flight against the
+    /// object store at once, each counted until its body has been sent on.
+    /// Further downloads wait in the process for a permit instead of being
+    /// sent and refused, which turns an account over its limit into
+    /// queueing here rather than wasted requests and retries in lockstep.
+    /// Existence checks (`HEAD`) are not counted: they move no bytes, so
+    /// they never wait behind a body going to a slow client, and a
+    /// throttled `HEAD` does not move the limit. 0 (the default) leaves
+    /// downloads unbounded, as before.
     ///
     /// Default: 0
     #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")]
     pub max_concurrent_reads: usize,
 
     /// With `max_concurrent_reads` set, let the bound move: a throttled
-    /// response (503 `ServerBusy` or 429) halves it, not below
-    /// `min_concurrent_reads`, and a run of successful reads raises it by
-    /// one back toward `max_concurrent_reads`. Many processes behind one
+    /// download (503 `ServerBusy` or 429) halves it, not below
+    /// `min_concurrent_reads`, and a run of answered downloads raises it
+    /// by one back toward `max_concurrent_reads`. Many processes behind one
     /// account then converge on a share of its limit. Off, the bound is
     /// fixed at `max_concurrent_reads`.
     ///
diff --git a/nativelink-store/src/azure_blob_store.rs b/nativelink-store/src/azure_blob_store.rs
index c6eb918408a..61f8eef314b 100644
--- a/nativelink-store/src/azure_blob_store.rs
+++ b/nativelink-store/src/azure_blob_store.rs
@@ -268,8 +268,11 @@ where
         self.read_permits.as_ref().map(|pool| pool.limit())
     }
 
-    /// A read permit when reads are bounded. Held for the request, body
-    /// included, so what is in flight is what the limit says.
+    /// A download permit when reads are bounded. Held for the request,
+    /// body included, so what is in flight is what the limit says. Only
+    /// `get_part` takes one: a `HEAD` moves no bytes and must not wait
+    /// behind a body still being sent to a slow client, and a throttle
+    /// on either side says nothing about the other.
     async fn read_permit(&self) -> Option<AdaptivePermit> {
         match &self.read_permits {
             Some(pool) => Some(pool.acquire().await),
@@ -366,7 +369,6 @@ where
                 let blob_path = blob_path.clone();
                 let client = Arc::clone(&self.client);
                 async move {
-                    let _read_permit = self.read_permit().await;
                     let _permit = match fs::get_permit().await {
                         Ok(permit) => permit,
                         Err(e) => {
@@ -384,7 +386,6 @@ where
 
                     match result {
                         Ok(props) => {
-                            self.note_read_ok();
                             if self.consider_expired_after_s > 0
                                 && let Some(last_modified) = props.last_modified().ok().flatten()
                             {
@@ -419,21 +420,19 @@ where
                                         state,
                                     ));
                                 }
-                                self.note_read_ok();
                                 Some((RetryResult::Ok(None), state))
                             } else {
-                                let retry = self.retry_later(
-                                    &err,
-                                    "has",
-                                    make_err!(
-                                        Code::Unavailable,
-                                        "Failed to get blob properties: {err:?}"
+                                Some((
+                                    self.retry_later(
+                                        &err,
+                                        "has",
+                                        make_err!(
+                                            Code::Unavailable,
+                                            "Failed to get blob properties: {err:?}"
+                                        ),
                                     ),
-                                );
-                                if matches!(retry, RetryResult::RetryAfter(..)) {
-                                    self.note_read_throttled();
-                                }
-                                Some((retry, state))
+                                    state,
+                                ))
                             }
                         }
                     }
diff --git a/nativelink-store/tests/azure_blob_store_test.rs b/nativelink-store/tests/azure_blob_store_test.rs
index 2791c8b3274..3e04378262e 100644
--- a/nativelink-store/tests/azure_blob_store_test.rs
+++ b/nativelink-store/tests/azure_blob_store_test.rs
@@ -898,13 +898,55 @@ async fn throttled_reads_halve_the_read_limit_and_successes_regrow_it() -> Resul
         "two throttles, two halvings; the download that got through is one toward regrowth"
     );
 
-    // Sixteen answered reads in a row (the download above was the first)
+    // Sixteen answered downloads in a row (the one above was the first)
     // add one permit.
     for _ in 0..15 {
+        mock.push_download(download_ok(StatusCode::Ok, VALUE.as_bytes().to_vec()));
+        store
+            .get_part_unchunked(create_test_digest()?, 0, None)
+            .await?;
+    }
+    assert_eq!(store.read_limit(), Some(5));
+    Ok(())
+}
+
+/// Existence checks are outside the download pool: a throttled `HEAD`
+/// does not cut the limit and an answered one does not grow it.
+#[nativelink_test(start_paused = true)]
+async fn existence_checks_do_not_move_the_read_limit() -> Result<(), Error> {
+    let mock = MockTransport::new();
+    mock.push_properties(error_response(StatusCode::ServiceUnavailable));
+    mock.push_properties(properties_ok(512));
+
+    let store = create_test_store_with_common(
+        Arc::clone(&mock),
+        CommonObjectSpec {
+            retry: Retry {
+                max_retries: 0,
+                delay: 0.0,
+                jitter: 0.0,
+                ..Default::default()
+            },
+            max_concurrent_reads: 16,
+            adaptive_read_concurrency: true,
+            min_concurrent_reads: 2,
+            ..Default::default()
+        },
+    )?;
+
+    assert_eq!(store.has(create_test_digest()?).await?, Some(512));
+    assert_eq!(mock.request_count(), 2, "the throttled HEAD was retried");
+    assert_eq!(
+        store.read_limit(),
+        Some(16),
+        "a throttled HEAD cuts nothing"
+    );
+
+    for _ in 0..16 {
         mock.push_properties(properties_ok(512));
         store.has(create_test_digest()?).await?;
     }
-    assert_eq!(store.read_limit(), Some(5));
+    assert_eq!(store.read_limit(), Some(16), "answered HEADs grow nothing");
     Ok(())
 }
 
diff --git a/nativelink-util/src/adaptive_permits.rs b/nativelink-util/src/adaptive_permits.rs
index 0d82fefe76c..4c1d7d38b0f 100644
--- a/nativelink-util/src/adaptive_permits.rs
+++ b/nativelink-util/src/adaptive_permits.rs
@@ -188,10 +188,10 @@ impl AdaptivePermits {
         }
         let cut = limit - target;
         // Take back what is idle now; the rest is owed and leaves with the
-        // permits that hold it.
-        let idle = self.semaphore.available_permits().min(cut);
-        self.semaphore.forget_permits(idle);
-        self.owed.fetch_add(cut - idle, Ordering::AcqRel);
+        // permits that hold it. `acquire` does not take this lock, so the
+        // count forgotten is what counts, not what looked idle a moment ago.
+        let taken = self.semaphore.forget_permits(cut);
+        self.owed.fetch_add(cut - taken, Ordering::AcqRel);
         self.limit.store(target, Ordering::Release);
         record_object_store_read_limit(self.name, target);
         info!(