From 03355510bb92ac24e35f0ec53f5bebf59543cac8 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 13 Oct 2026 12:01:01 -0400 Subject: [PATCH 1/3] Hold a worker for a queued action whose reservation only fits an idle one --- nativelink-config/src/schedulers.rs | 65 +++ nativelink-scheduler/BUILD.bazel | 1 + .../src/api_worker_scheduler.rs | 264 ++++++++++- nativelink-scheduler/src/simple_scheduler.rs | 39 +- .../tests/reservation_hold_test.rs | 416 ++++++++++++++++++ .../worker_running_operations_report_test.rs | 1 + nativelink-util/src/metrics.rs | 36 ++ .../docs/explanations/scheduler-internals.mdx | 19 + .../docs/content/docs/reference/metrics.mdx | 190 ++++---- 9 files changed, 940 insertions(+), 91 deletions(-) create mode 100644 nativelink-scheduler/tests/reservation_hold_test.rs diff --git a/nativelink-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs index bebe74a0ee8..5fb52fd9f94 100644 --- a/nativelink-config/src/schedulers.rs +++ b/nativelink-config/src/schedulers.rs @@ -345,6 +345,22 @@ pub struct SimpleSpec { #[serde(default)] pub memory_escalation: Option, + /// Hold a worker for a queued action that keeps finding no room. An + /// action whose reservation only fits an idle worker starves under + /// load: every pass tries it first and finds each worker partly busy, + /// then fills the room that opens with newer, smaller actions, so no + /// worker ever drains for it. With this set, once such an action has + /// waited `hold_after_s` while some connected worker would fit it + /// idle, the scheduler picks one such worker and offers it nothing + /// else until the action is placed, leaves the queue, or `max_hold_s` + /// passes. An action no connected worker would fit even idle gets no + /// hold. With the matcher in every CAS replica a hold is local to the + /// replica the worker is connected to, which is the only one that can + /// dispatch to it. + /// Default: unset (off) + #[serde(default)] + pub reservation_hold: Option, + /// The storage backend to use for the scheduler. /// Default: memory pub experimental_backend: Option, @@ -571,6 +587,55 @@ const fn default_memory_escalation_percent() -> u64 { 200 } +#[derive(Deserialize, Serialize, Debug, Clone, Copy)] +#[serde(deny_unknown_fields)] +#[cfg_attr(feature = "dev-schema", derive(JsonSchema))] +pub struct ReservationHoldSpec { + /// Seconds a queued action has to have waited, finding no room on + /// every pass, before a worker is held for it. + /// Default: 30 + #[serde( + default = "default_reservation_hold_after_s", + deserialize_with = "convert_duration_with_shellexpand" + )] + pub hold_after_s: u64, + + /// Seconds a hold lasts at most. A hold that long without the action + /// fitting means the worker never reported room for it even drained; + /// the hold is given up so the worker is not idle for good, and the + /// action is not held for again until as long has passed. + /// Default: 300 + #[serde( + default = "default_reservation_max_hold_s", + deserialize_with = "convert_duration_with_shellexpand" + )] + pub max_hold_s: u64, + + /// Most workers held at once on this scheduler. 0 is one tenth of the + /// workers connected to it, and at least one. + /// Default: 0 + #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + pub max_held_workers: usize, +} + +impl Default for ReservationHoldSpec { + fn default() -> Self { + Self { + hold_after_s: default_reservation_hold_after_s(), + max_hold_s: default_reservation_max_hold_s(), + max_held_workers: 0, + } + } +} + +const fn default_reservation_hold_after_s() -> u64 { + 30 +} + +const fn default_reservation_max_hold_s() -> u64 { + 300 +} + #[derive(Deserialize, Serialize, Debug, Default, Clone, Copy)] #[serde(deny_unknown_fields)] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] diff --git a/nativelink-scheduler/BUILD.bazel b/nativelink-scheduler/BUILD.bazel index 334a8d97511..e17cff43c66 100644 --- a/nativelink-scheduler/BUILD.bazel +++ b/nativelink-scheduler/BUILD.bazel @@ -86,6 +86,7 @@ rust_test_suite( "tests/property_modifier_scheduler_test.rs", "tests/queue_order_test.rs", "tests/redis_store_awaited_action_db_test.rs", + "tests/reservation_hold_test.rs", "tests/simple_scheduler_state_manager_test.rs", "tests/simple_scheduler_test.rs", "tests/store_awaited_action_db_test.rs", diff --git a/nativelink-scheduler/src/api_worker_scheduler.rs b/nativelink-scheduler/src/api_worker_scheduler.rs index eb28ab2b331..830c3187495 100644 --- a/nativelink-scheduler/src/api_worker_scheduler.rs +++ b/nativelink-scheduler/src/api_worker_scheduler.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use core::cmp::Reverse; use core::ops::{Deref, DerefMut}; use core::sync::atomic::{AtomicU64, Ordering}; use core::time::Duration; @@ -22,7 +23,9 @@ use std::time::{Instant, SystemTime, UNIX_EPOCH}; use async_lock::Mutex; use futures::{StreamExt, future}; use lru::LruCache; -use nativelink_config::schedulers::{MemoryEscalationSpec, WorkerAllocationStrategy}; +use nativelink_config::schedulers::{ + MemoryEscalationSpec, ReservationHoldSpec, WorkerAllocationStrategy, +}; use nativelink_error::{Code, Error, ResultExt, make_err, make_input_err}; use nativelink_metric::{ MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, @@ -34,11 +37,12 @@ use nativelink_proto::com::github::trace_machina::nativelink::events::{ use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::{ ActionResourceUsage, ResourceOutcome, WorkerLoad, }; -use nativelink_util::action_messages::{ActionStage, OperationId, WorkerId}; +use nativelink_util::action_messages::{ActionStage, ActionUniqueQualifier, OperationId, WorkerId}; use nativelink_util::metrics::{ WorkerDisconnectReason, record_dispatch_requeue, record_execution_cpu_time, - record_execution_peak_memory, record_worker_connected, record_worker_disconnected, - record_worker_keepalive, record_worker_keepalive_gap, record_worker_state, + record_execution_peak_memory, record_held_workers, record_worker_connected, + record_worker_disconnected, record_worker_hold, record_worker_keepalive, + record_worker_keepalive_gap, record_worker_state, }; use nativelink_util::operation_state_manager::{ Decline, Escalation, UpdateOperationType, WorkerStateManager, @@ -176,6 +180,19 @@ impl MetricsComponent for Workers { } } +/// A worker kept for one queued action that found no room anywhere. +#[derive(Debug, Clone)] +struct Hold { + /// The action the worker is kept for. + action: ActionUniqueQualifier, + /// When the hold is given up. + until: SystemTime, + /// The last matching pass that read the action, by `pass_generation`. + /// A pass that read the whole queue without meeting it means the + /// action is gone, and so is the reason to hold. + seen_in: u64, +} + /// A collection of workers that are available to run tasks. #[derive(MetricsComponent)] struct ApiWorkerSchedulerImpl { @@ -194,6 +211,19 @@ struct ApiWorkerSchedulerImpl { /// How a memory kill is turned into a retry with a larger reservation; /// unset means the kill fails the action. memory_escalation: Option, + /// How a queued action that keeps finding no room gets a worker held + /// for it; unset means never. + reservation_hold: Option, + /// Workers held for one queued action each, by worker, and the same + /// holds by action. A held worker is offered nothing but its action. + holds: HashMap, + holds_by_action: HashMap, + /// Actions whose hold expired, and until when they are not held for + /// again, so one that never fits does not keep a worker idle in turns. + hold_cooldowns: HashMap, + /// Counts matching passes, so a hold can record the last pass that + /// met its action; see `Hold::seen_in`. + pass_generation: u64, /// A channel to notify the matching engine that the worker pool has changed. worker_change_notify: Arc, /// Bumped with every notification, under the same lock as placement, so @@ -471,6 +501,7 @@ impl ApiWorkerSchedulerImpl { /// Note: The caller is responsible for any rescheduling of any tasks that might be /// running. fn remove_worker(&mut self, worker_id: &WorkerId) -> Option { + self.end_hold(worker_id, "released"); // Remove from capability index self.capability_index.remove_worker(worker_id); // Pop before recomputing the shapes, or the departing worker still @@ -571,10 +602,163 @@ impl ApiWorkerSchedulerImpl { record_worker_state("draining", is_draining); } worker.is_draining = is_draining; + if is_draining { + // A draining worker takes nothing, its holder included. + self.end_hold(worker_id, "released"); + } self.capacity_changed(); Ok(()) } + /// Ends the hold on `worker_id`, if any, for `outcome`. The worker is + /// room the fleet did not have a moment ago, so the matcher is woken. + fn end_hold(&mut self, worker_id: &WorkerId, outcome: &'static str) { + let Some(hold) = self.holds.remove(worker_id) else { + return; + }; + self.holds_by_action.remove(&hold.action); + info!(?worker_id, action = ?hold.action, outcome, "Worker hold ended"); + record_worker_hold(outcome); + record_held_workers(self.holds.len()); + self.capacity_changed(); + } + + /// Ends the hold for `action`, if any, for `outcome`. + fn end_hold_for_action(&mut self, action: &ActionUniqueQualifier, outcome: &'static str) { + if let Some(worker_id) = self.holds_by_action.get(action).cloned() { + self.end_hold(&worker_id, outcome); + } + } + + /// Holds a worker for `action`, which has waited long enough finding + /// no room: of the workers neither draining nor held that would fit + /// it idle, the one with the fewest actions running and then the most + /// memory left. Nothing is held when holds are off, the cap is + /// reached, the action's last hold expired too recently, or no + /// connected worker would fit it even idle, which is not a matter of + /// waiting. Returns whether the action holds a worker now. + fn hold_worker_for( + &mut self, + action: &ActionUniqueQualifier, + platform_properties: &PlatformProperties, + now: SystemTime, + ) -> bool { + let Some(spec) = &self.reservation_hold else { + return false; + }; + if self.holds_by_action.contains_key(action) { + return true; + } + if self + .hold_cooldowns + .get(action) + .is_some_and(|until| now < *until) + { + return false; + } + let cap = if spec.max_held_workers == 0 { + (self.workers.len() / 10).max(1) + } else { + spec.max_held_workers + }; + if self.holds.len() >= cap { + return false; + } + let memory_property = self.live_memory_veto.as_deref(); + let memory_left = |worker: &Worker| match memory_property + .and_then(|property| worker.platform_properties.properties.get(property)) + { + Some(PlatformPropertyValue::Minimum(kb)) => *kb, + _ => 0, + }; + let holds = &self.holds; + let chosen = self + .workers + .iter() + .filter(|(worker_id, worker)| !worker.is_draining && !holds.contains_key(*worker_id)) + .filter(|(_, worker)| { + platform_properties.is_satisfied_by(&worker.total_platform_properties, false) + }) + .min_by_key(|(_, worker)| { + ( + worker.running_action_infos.len(), + Reverse(memory_left(worker)), + ) + }) + .map(|(worker_id, _)| worker_id.clone()); + let Some(worker_id) = chosen else { + return false; + }; + info!( + ?worker_id, + ?action, + max_hold_s = spec.max_hold_s, + "Holding a worker for a queued action that keeps finding no room" + ); + self.holds.insert( + worker_id.clone(), + Hold { + action: action.clone(), + until: now + Duration::from_secs(spec.max_hold_s), + seen_in: self.pass_generation, + }, + ); + self.holds_by_action.insert(action.clone(), worker_id); + record_worker_hold("held"); + record_held_workers(self.holds.len()); + true + } + + /// A matching pass begins; returns its generation, which the holds it + /// meets record and `end_pass_holds` compares against. + const fn begin_pass_holds(&mut self) -> u64 { + self.pass_generation += 1; + self.pass_generation + } + + /// Ends the holds that no pass since `pass` began has met, whose + /// action is gone, and the ones past their time, whose action is then + /// not held for again for as long. Judged by generation rather than a + /// flag so that two passes in flight at once (a test entry point + /// beside the matcher's own loop) cannot clear each other's marks. + fn end_pass_holds(&mut self, pass: u64, now: SystemTime) { + let ended: Vec<(WorkerId, &'static str)> = self + .holds + .iter() + .filter_map(|(worker_id, hold)| { + if hold.seen_in < pass { + Some((worker_id.clone(), "released")) + } else if hold.until <= now { + Some((worker_id.clone(), "expired")) + } else { + None + } + }) + .collect(); + for (worker_id, outcome) in ended { + if outcome == "expired" + && let Some(hold) = self.holds.get(&worker_id) + && let Some(spec) = &self.reservation_hold + { + self.hold_cooldowns.insert( + hold.action.clone(), + now + Duration::from_secs(spec.max_hold_s), + ); + } + self.end_hold(&worker_id, outcome); + } + self.hold_cooldowns.retain(|_, until| now < *until); + } + + /// Marks the hold for `action`, if any, as met by the pass under way. + fn note_held_action_seen(&mut self, action: &ActionUniqueQualifier) { + if let Some(worker_id) = self.holds_by_action.get(action) + && let Some(hold) = self.holds.get_mut(worker_id) + { + hold.seen_in = self.pass_generation; + } + } + /// Whether peer-absence can be trusted to fail an action right now — a /// shared census must be known, warm and fresh; a non-shared db is trusted /// as soon as it is known. See the free `peer_census_trusted`. @@ -671,6 +855,7 @@ impl ApiWorkerSchedulerImpl { platform_properties: &PlatformProperties, candidates: &HashSet, full_worker_logging: bool, + asking: Option<&ActionUniqueQualifier>, ) -> Option { // Check function for availability AND dynamic Minimum property verification. // The index only does presence checks for Minimum properties since their @@ -690,6 +875,16 @@ impl ApiWorkerSchedulerImpl { return false; } + // A held worker takes nothing but the action it is held for. + if let Some(hold) = self.holds.get(*worker_id) + && asking != Some(&hold.action) + { + if full_worker_logging { + info!("Worker {worker_id} is held for another action"); + } + return false; + } + if let (Some(needed_kb), Some(declined_kb)) = (needed_kb, w.idle_declined_kb) && needed_kb >= declined_kb { @@ -782,7 +977,11 @@ impl ApiWorkerSchedulerImpl { platform_properties: &PlatformProperties, full_worker_logging: bool, now: SystemTime, + asking: Option<&ActionUniqueQualifier>, ) -> MatchOutcome { + if let Some(action) = asking { + self.note_held_action_seen(action); + } if self.workers.is_empty() { if full_worker_logging { info!("No workers available to match!"); @@ -798,9 +997,12 @@ impl ApiWorkerSchedulerImpl { let candidates = self .capability_index .find_matching_workers(platform_properties, full_worker_logging); - if let Some(worker_id) = - self.find_available_worker(platform_properties, &candidates, full_worker_logging) - { + if let Some(worker_id) = self.find_available_worker( + platform_properties, + &candidates, + full_worker_logging, + asking, + ) { return MatchOutcome::Matched(worker_id); } Some(candidates) @@ -1245,6 +1447,9 @@ impl ApiWorkerSchedulerImpl { .await; return Result::<(), _>::Err(err).merge(evicted).merge(requeued); } + // The action ran, here or elsewhere; a worker held for it is + // free for everything again. + self.end_hold_for_action(&action_info.inner.unique_qualifier, "placed"); Ok(()) } else { warn!( @@ -1569,6 +1774,7 @@ impl ApiWorkerScheduler { allocation_strategy: WorkerAllocationStrategy, live_memory_veto: Option, memory_escalation: Option, + reservation_hold: Option, worker_change_notify: Arc, worker_timeout_s: u64, unacknowledged_kill_timeout_s: u64, @@ -1591,6 +1797,11 @@ impl ApiWorkerScheduler { allocation_strategy, live_memory_veto, memory_escalation, + reservation_hold, + holds: HashMap::new(), + holds_by_action: HashMap::new(), + hold_cooldowns: HashMap::new(), + pass_generation: 0, worker_change_notify, capacity_generation: capacity_generation.clone(), worker_registry: worker_registry.clone(), @@ -1769,12 +1980,40 @@ impl ApiWorkerScheduler { platform_properties: &PlatformProperties, full_worker_logging: bool, now: SystemTime, + asking: Option<&ActionUniqueQualifier>, ) -> MatchOutcome { - self.find_worker_for_action_observed(platform_properties, full_worker_logging, now) + self.find_worker_for_action_observed(platform_properties, full_worker_logging, now, asking) .await .0 } + /// Holds a worker for `action`, a queued action that has waited long + /// enough finding no room; see `ReservationHoldSpec`. Returns whether + /// the action holds a worker now. + pub async fn hold_worker_for( + &self, + action: &ActionUniqueQualifier, + platform_properties: &PlatformProperties, + now: SystemTime, + ) -> bool { + self.inner + .lock() + .await + .hold_worker_for(action, platform_properties, now) + } + + /// A matching pass is about to read the queue; returns the pass, for + /// `end_pass_holds`. + pub async fn begin_pass_holds(&self) -> u64 { + self.inner.lock().await.begin_pass_holds() + } + + /// The pass read the whole queue: release the holds no pass since it + /// began has met, and the ones past their time. + pub async fn end_pass_holds(&self, pass: u64, now: SystemTime) { + self.inner.lock().await.end_pass_holds(pass, now); + } + /// `find_worker_for_action`, also returning the capacity generation read /// under the same lock as the placement. A pass compares it with the one /// it saw last: a change means a worker gained room since, and an older @@ -1785,6 +2024,7 @@ impl ApiWorkerScheduler { platform_properties: &PlatformProperties, full_worker_logging: bool, now: SystemTime, + asking: Option<&ActionUniqueQualifier>, ) -> (MatchOutcome, u64) { let start = Instant::now(); self.metrics @@ -1793,8 +2033,12 @@ impl ApiWorkerScheduler { let mut inner = self.inner.lock().await; let worker_count = inner.workers.len() as u64; - let result = - inner.inner_find_worker_for_action(platform_properties, full_worker_logging, now); + let result = inner.inner_find_worker_for_action( + platform_properties, + full_worker_logging, + now, + asking, + ); let generation = inner.capacity_generation.load(Ordering::Relaxed); // Track workers iterated (worst case is all workers) diff --git a/nativelink-scheduler/src/simple_scheduler.rs b/nativelink-scheduler/src/simple_scheduler.rs index 4cb5cccd305..3e85f4ed419 100644 --- a/nativelink-scheduler/src/simple_scheduler.rs +++ b/nativelink-scheduler/src/simple_scheduler.rs @@ -235,6 +235,10 @@ pub struct SimpleScheduler { /// burst drains in a few passes. See `unsatisfiable_fail_cap`. last_unsatisfiable_backlog: AtomicUsize, + /// How long a queued action waits, finding no room on every pass, + /// before a worker is held for it; `None` when holds are off. + hold_after: Option, + /// The clock the state manager uses. now_fn: Box SystemTime + Send + Sync>, } @@ -801,6 +805,7 @@ impl SimpleScheduler { &loaded.action_info.platform_properties, full_worker_logging, now, + Some(&loaded.action_info.inner.unique_qualifier), ) .await else { @@ -883,6 +888,7 @@ impl SimpleScheduler { let maybe_origin_event_tx = self.maybe_origin_event_tx.as_ref(); let mut parked = Parked::default(); let mut placement = Placement::default(); + let hold_pass = workers.begin_pass_holds().await; let mut capacity_seen = workers.capacity_generation(); let mut queued_seen: u64 = 0; // How long each action read has waited since its client submitted @@ -920,6 +926,7 @@ impl SimpleScheduler { &loaded.action_info.platform_properties, full_worker_logging, unsatisfiable_pass.now, + Some(&loaded.action_info.inner.unique_qualifier), ) .await; if generation == capacity_seen || parked.is_empty() { @@ -958,8 +965,26 @@ impl SimpleScheduler { .map(|_| ()), ); } - // A worker could run it once one has room. - MatchOutcome::WaitingForCapacity => parked.park(loaded), + // A worker could run it once one has room. One that has + // waited long enough for that gets a worker held for it, + // if one would fit it idle (`reservation_hold`). + MatchOutcome::WaitingForCapacity => { + if let Some(hold_after) = self.hold_after + && unsatisfiable_pass + .now + .duration_since(loaded.action_info.inner.insert_timestamp) + .is_ok_and(|waited| waited >= hold_after) + { + workers + .hold_worker_for( + &loaded.action_info.inner.unique_qualifier, + &loaded.action_info.platform_properties, + unsatisfiable_pass.now, + ) + .await; + } + parked.park(loaded); + } // Nothing to wait for; a worker connecting starts a new pass, // unless the no-worker clock is running. MatchOutcome::NoWorkersConnected => { @@ -1010,6 +1035,11 @@ impl SimpleScheduler { .await, ); } + // The whole queue was read: a held action the pass did not meet + // is gone, and a hold past its time is given up. + workers + .end_pass_holds(hold_pass, unsatisfiable_pass.now) + .await; record_parked_dispatched(placement.dispatched); if placement.attempted > 0 || parked.cap_dropped > 0 { debug!( @@ -1221,6 +1251,7 @@ impl SimpleScheduler { spec.allocation_strategy, spec.live_memory_veto.clone(), spec.memory_escalation.clone(), + spec.reservation_hold, worker_change_notify.clone(), worker_timeout_s, unacknowledged_kill_timeout_s, @@ -1567,6 +1598,10 @@ impl SimpleScheduler { no_worker_tracker: no_worker_action_timeout .map(|timeout| Mutex::new(UnsatisfiableTracker::new(Some(timeout)))), last_unsatisfiable_backlog: AtomicUsize::new(0), + hold_after: spec + .reservation_hold + .as_ref() + .map(|hold| Duration::from_secs(hold.hold_after_s)), now_fn: scheduler_now_fn, } }); diff --git a/nativelink-scheduler/tests/reservation_hold_test.rs b/nativelink-scheduler/tests/reservation_hold_test.rs new file mode 100644 index 00000000000..c6a7224e8e0 --- /dev/null +++ b/nativelink-scheduler/tests/reservation_hold_test.rs @@ -0,0 +1,416 @@ +// Copyright 2026 The NativeLink Authors. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::time::Duration; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::UNIX_EPOCH; + +use mock_instant::thread_local::MockClock; +use nativelink_config::schedulers::{PropertyType, ReservationHoldSpec, SimpleSpec}; +use nativelink_error::{Error, ResultExt}; +use nativelink_macro::nativelink_test; +use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::{ + UpdateForWorker, update_for_worker, +}; +use nativelink_scheduler::default_scheduler_factory::memory_awaited_action_db_factory; +use nativelink_scheduler::simple_scheduler::SimpleScheduler; +use nativelink_scheduler::worker::Worker; +use nativelink_scheduler::worker_scheduler::WorkerScheduler; +use nativelink_util::action_messages::{ActionResult, ActionStage, OperationId, WorkerId}; +use nativelink_util::common::DigestInfo; +use nativelink_util::instant_wrapper::MockInstantWrapped; +use nativelink_util::operation_state_manager::{ + ActionStateResult, ClientStateManager, UpdateOperationType, +}; +use nativelink_util::platform_properties::{PlatformProperties, PlatformPropertyValue}; +use pretty_assertions::assert_eq; +use tokio::sync::{Notify, mpsc}; +use utils::scheduler_utils::make_base_action_info; + +mod utils { + pub(crate) mod scheduler_utils; +} + +const NOW_TIME: u64 = 10000; +const HOLD_AFTER_S: u64 = 30; +const MAX_HOLD_S: u64 = 300; +/// Each worker advertises this much memory and runs up to four actions. +const WORKER_KB: u64 = 100_000; +const SMALL_KB: u64 = 10_000; +/// Fits a worker only when nothing else runs on it. +const LARGE_KB: u64 = 95_000; + +/// The client listeners of the queued actions, kept so that dropping one +/// does not drop its action. +type Listeners = Vec>; + +fn make_scheduler(max_held_workers: usize) -> Arc { + MockClock::set_time(Duration::from_secs(NOW_TIME)); + let spec = SimpleSpec { + supported_platform_properties: Some(HashMap::from([( + "memory_kb".to_string(), + PropertyType::Minimum, + )])), + reservation_hold: Some(ReservationHoldSpec { + hold_after_s: HOLD_AFTER_S, + max_hold_s: MAX_HOLD_S, + max_held_workers, + }), + // The clock jumps past a hold here, and no client polls meanwhile; + // neither the client timeout nor the backend's retention window + // (below) may retire a queued action over the jump. + client_action_timeout_s: 100_000, + ..SimpleSpec::default() + }; + let task_change_notify = Arc::new(Notify::new()); + let (scheduler, _worker_scheduler) = SimpleScheduler::new_with_callback( + &spec, + memory_awaited_action_db_factory(100_000, &task_change_notify, MockInstantWrapped::default), + || async move {}, + task_change_notify, + MockInstantWrapped::default, + None, + ); + scheduler +} + +async fn add_worker( + scheduler: &SimpleScheduler, + name: &str, +) -> Result, Error> { + let (tx, mut rx) = mpsc::channel(64); + let worker = Worker::new( + WorkerId(name.to_string()), + PlatformProperties::new(HashMap::from([( + "memory_kb".to_string(), + PlatformPropertyValue::Minimum(WORKER_KB), + )])), + tx, + NOW_TIME, + /* max_inflight_tasks */ 4, + ); + scheduler + .add_worker(worker) + .await + .err_tip(|| "Failed to add worker")?; + tokio::task::yield_now().await; + let first = rx.try_recv().expect("worker got no connection result"); + assert!(matches!( + first.update, + Some(update_for_worker::Update::ConnectionResult(_)) + )); + Ok(rx) +} + +async fn add_action( + scheduler: &SimpleScheduler, + listeners: &mut Listeners, + digest_byte: u8, + kb: u64, +) -> Result<(), Error> { + let mut action_info = make_base_action_info( + UNIX_EPOCH + MockClock::time(), + DigestInfo::new([digest_byte; 32], 512), + ); + Arc::make_mut(&mut action_info).platform_properties = + HashMap::from([("memory_kb".to_string(), kb.to_string())]); + let listener = scheduler + .add_action(OperationId::default(), action_info) + .await?; + listeners.push(listener); + tokio::task::yield_now().await; + Ok(()) +} + +/// The dispatches a worker received since last asked: operation id and the +/// memory each reserved. +fn dispatches(rx: &mut mpsc::Receiver) -> Vec<(String, u64)> { + let mut seen = Vec::new(); + while let Ok(msg) = rx.try_recv() { + if let Some(update_for_worker::Update::StartAction(start)) = msg.update { + let memory_kb = start + .platform + .as_ref() + .and_then(|platform| { + platform + .properties + .iter() + .find(|p| p.name == "memory_kb") + .and_then(|p| p.value.parse().ok()) + }) + .expect("the dispatch carries memory_kb"); + seen.push((start.operation_id, memory_kb)); + } + } + seen +} + +async fn complete( + scheduler: &SimpleScheduler, + worker: &str, + operation_id: &str, +) -> Result<(), Error> { + scheduler + .update_action( + &WorkerId(worker.to_string()), + &OperationId::from(operation_id), + UpdateOperationType::UpdateWithActionStage(ActionStage::Completed( + ActionResult::default(), + )), + ) + .await?; + tokio::task::yield_now().await; + Ok(()) +} + +/// Lets the clock run and runs a pass, as the matcher would. +async fn pass_after(scheduler: &SimpleScheduler, seconds: u64) -> Result<(), Error> { + MockClock::advance(Duration::from_secs(seconds)); + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + Ok(()) +} + +/// Two workers each running one small action, and a large action that fits +/// neither beside it. Returns the small actions' operation ids on a and b. +async fn two_busy_workers_and_a_large_action( + scheduler: &SimpleScheduler, + listeners: &mut Listeners, + a: &mut mpsc::Receiver, + b: &mut mpsc::Receiver, +) -> Result<(String, String), Error> { + add_action(scheduler, listeners, 1, SMALL_KB).await?; + add_action(scheduler, listeners, 2, SMALL_KB).await?; + let (on_a, on_b) = (dispatches(a), dispatches(b)); + assert_eq!( + (on_a.len(), on_b.len()), + (1, 1), + "one small action per worker" + ); + add_action(scheduler, listeners, 3, LARGE_KB).await?; + assert_eq!( + (dispatches(a).len(), dispatches(b).len()), + (0, 0), + "the large action fits neither worker beside a small one" + ); + Ok((on_a[0].0.clone(), on_b[0].0.clone())) +} + +/// Once a worker is held, two small actions both land on the other one. +/// Returns the names of the held and the free worker. +async fn held_and_free( + scheduler: &SimpleScheduler, + listeners: &mut Listeners, + a: &mut mpsc::Receiver, + b: &mut mpsc::Receiver, +) -> Result<(&'static str, &'static str), Error> { + add_action(scheduler, listeners, 4, SMALL_KB).await?; + add_action(scheduler, listeners, 5, SMALL_KB).await?; + let (on_a, on_b) = (dispatches(a).len(), dispatches(b).len()); + match (on_a, on_b) { + (0, 2) => Ok(("a", "b")), + (2, 0) => Ok(("b", "a")), + other => panic!("one worker should be held and take nothing, got {other:?}"), + } +} + +/// A large action waits behind small ones on every worker. Once it has +/// waited `hold_after_s`, one worker is held: new small actions all go to +/// the other, and when the held worker's own action finishes the large one +/// runs there. The hold then ends and the worker takes small actions again. +#[nativelink_test] +async fn a_starving_reservation_gets_a_worker_held_then_runs_there() -> Result<(), Error> { + let scheduler = make_scheduler(0); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + let (small_on_a, small_on_b) = + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + + // Not long enough yet: nothing is held, a small action goes anywhere. + pass_after(&scheduler, HOLD_AFTER_S - 1).await?; + add_action(&scheduler, &mut listeners, 9, SMALL_KB).await?; + let (on_a, on_b) = (dispatches(&mut a), dispatches(&mut b)); + assert_eq!(on_a.len() + on_b.len(), 1, "nothing is held yet"); + // Put the fleet back to one small action per worker. + let (third_worker, third) = if on_a.is_empty() { + ("b", on_b[0].0.clone()) + } else { + ("a", on_a[0].0.clone()) + }; + complete(&scheduler, third_worker, &third).await?; + scheduler.do_try_match_for_test().await?; + assert_eq!( + (dispatches(&mut a).len(), dispatches(&mut b).len()), + (0, 0), + "the room that opened is still too small for the large action" + ); + + pass_after(&scheduler, 1).await?; + let (held, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let (held_rx, free_rx) = if held == "a" { + (&mut a, &mut b) + } else { + (&mut b, &mut a) + }; + let small_on_held = if held == "a" { small_on_a } else { small_on_b }; + + // The held worker's own action finishes: the large action runs there. + complete(&scheduler, held, &small_on_held).await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + let on_held = dispatches(held_rx); + assert_eq!( + on_held.iter().map(|(_, kb)| *kb).collect::>(), + vec![LARGE_KB], + "the held worker runs the action it was held for" + ); + assert_eq!(dispatches(free_rx).len(), 0); + + // The hold is over. Once the large action is done, the worker is idle + // and the free one has a slot left, so two small actions both run only + // if the formerly held worker takes one. + complete(&scheduler, held, &on_held[0].0).await?; + add_action(&scheduler, &mut listeners, 6, SMALL_KB).await?; + add_action(&scheduler, &mut listeners, 7, SMALL_KB).await?; + assert_eq!( + dispatches(held_rx).len() + dispatches(free_rx).len(), + 2, + "the worker is offered everything again once its action ran" + ); + Ok(()) +} + +/// A hold that never fits is given up after `max_hold_s`, and the action is +/// not held for again straight away: the worker takes other work. +#[nativelink_test] +async fn a_hold_expires_and_the_worker_takes_other_work() -> Result<(), Error> { + let scheduler = make_scheduler(0); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + let (held, free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let (held_rx, free_rx) = if held == "a" { + (&mut a, &mut b) + } else { + (&mut b, &mut a) + }; + + // Fill the free worker (three running) to its four slots. + add_action(&scheduler, &mut listeners, 6, SMALL_KB).await?; + assert_eq!( + (dispatches(held_rx).len(), dispatches(free_rx).len()), + (0, 1) + ); + // Now a small action has nowhere to go: the free worker is full and + // the held worker takes nothing else. + add_action(&scheduler, &mut listeners, 7, SMALL_KB).await?; + assert_eq!( + (dispatches(held_rx).len(), dispatches(free_rx).len()), + (0, 0) + ); + + // The hold runs out; the pass that notices wakes the matcher, and the + // next pass places the small action on the worker, which is not held + // for the large action again while its cooldown lasts. + pass_after(&scheduler, MAX_HOLD_S).await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!( + dispatches(held_rx) + .iter() + .map(|(_, kb)| *kb) + .collect::>(), + vec![SMALL_KB], + "the expired hold's worker takes the small action, not the large one" + ); + assert_eq!(dispatches(free_rx).len(), 0, "{free} was full"); + + // Cooldown over, and the large action still waits: a worker is held + // for it again, and a small action finds no room on it. + pass_after(&scheduler, MAX_HOLD_S).await?; + add_action(&scheduler, &mut listeners, 8, SMALL_KB).await?; + assert_eq!( + dispatches(held_rx).len() + dispatches(free_rx).len(), + 0, + "one worker full, the other held again" + ); + Ok(()) +} + +/// The large action finds room elsewhere first: an idle worker joins and +/// takes it at once, the hold ends as placed, and the held worker takes +/// small actions again. +#[nativelink_test] +async fn a_hold_ends_when_its_action_runs_elsewhere() -> Result<(), Error> { + let scheduler = make_scheduler(0); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + let (held, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let (held_rx, free_rx) = if held == "a" { + (&mut a, &mut b) + } else { + (&mut b, &mut a) + }; + + let mut c = add_worker(&scheduler, "c").await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!( + dispatches(&mut c) + .iter() + .map(|(_, kb)| *kb) + .collect::>(), + vec![LARGE_KB], + "an idle worker that fits the action takes it at once" + ); + assert_eq!(dispatches(held_rx).len(), 0); + + // The hold ended as placed. The new worker has no memory left beside + // the large action, the free worker one slot, and the formerly held + // worker three: five small actions run four deep only if it takes + // part again, and one deep if it were still held. + for digest in 20..25 { + add_action(&scheduler, &mut listeners, digest, SMALL_KB).await?; + } + let total = dispatches(held_rx).len() + dispatches(free_rx).len() + dispatches(&mut c).len(); + assert_eq!( + total, 4, + "the formerly held worker takes small actions again" + ); + Ok(()) +} + +/// With the cap at one hold, a second starving action waits unheld. +#[nativelink_test] +async fn at_most_the_cap_is_held() -> Result<(), Error> { + let scheduler = make_scheduler(1); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + add_action(&scheduler, &mut listeners, 10, LARGE_KB).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + // One worker held, the other still takes small actions. + add_action(&scheduler, &mut listeners, 4, SMALL_KB).await?; + add_action(&scheduler, &mut listeners, 5, SMALL_KB).await?; + assert_eq!(dispatches(&mut a).len() + dispatches(&mut b).len(), 2); + Ok(()) +} diff --git a/nativelink-scheduler/tests/worker_running_operations_report_test.rs b/nativelink-scheduler/tests/worker_running_operations_report_test.rs index 32c2f129fc1..d6eb1aeff8d 100644 --- a/nativelink-scheduler/tests/worker_running_operations_report_test.rs +++ b/nativelink-scheduler/tests/worker_running_operations_report_test.rs @@ -109,6 +109,7 @@ async fn dispatched() -> Result { WorkerAllocationStrategy::default(), None, None, + None, Arc::new(Notify::new()), WORKER_TIMEOUT_S, 60, diff --git a/nativelink-util/src/metrics.rs b/nativelink-util/src/metrics.rs index f44b024cd3c..e6f4fe09dd7 100644 --- a/nativelink-util/src/metrics.rs +++ b/nativelink-util/src/metrics.rs @@ -46,6 +46,7 @@ pub const RPC_STATUS_CODE: &str = "rpc.grpc.status_code"; pub const SCHEDULER_MATCH_RESULT: &str = "scheduler.match.result"; /// The platform properties no worker could satisfy, comma separated. pub const SCHEDULER_UNSATISFIABLE_PROPERTIES: &str = "scheduler.unsatisfiable.properties"; +pub const SCHEDULER_HOLD_OUTCOME: &str = "scheduler.hold.outcome"; // Metric attribute keys for tiered stores. pub const STORE_TIER: &str = "store.tier"; @@ -1194,6 +1195,20 @@ pub static SCHEDULER_METRICS: LazyLock = LazyLock::new(|| .with_unit("{action}") .build(), + worker_holds: meter + .u64_counter("scheduler.worker.holds") + .with_description( + "Workers held for a queued action that kept finding no room, by outcome: held when the hold began, placed when the action ran, released when the action or the worker left, expired when max_hold_s passed first", + ) + .with_unit("{hold}") + .build(), + + held_workers: meter + .u64_gauge("scheduler.worker.held") + .with_description("Workers this scheduler holds for a queued action right now") + .with_unit("{worker}") + .build(), + awaited_action_orphans: meter .u64_counter("scheduler.awaited_action.orphans") .with_description( @@ -1252,6 +1267,11 @@ pub struct SchedulerOtlpMetrics { /// Actions a pass parked for want of room and dispatched when room /// opened before the pass ended, ahead of the newer actions it was reading. pub parked_dispatched: metrics::Counter, + /// Workers held for a queued action that kept finding no room, by what + /// began or ended the hold. + pub worker_holds: metrics::Counter, + /// Workers held for such an action right now. + pub held_workers: metrics::Gauge, /// Listed operations whose record was gone on read, by site. pub awaited_action_orphans: metrics::Counter, /// Abandoned-queue sweeps that failed before finishing. @@ -1388,6 +1408,22 @@ pub fn record_parked_dispatched(count: u64) { } } +/// Records a worker hold beginning (`held`) or ending: `placed` when the +/// action ran, `released` when the action left the queue or the worker +/// left, `expired` when `max_hold_s` passed first. +pub fn record_worker_hold(outcome: &'static str) { + SCHEDULER_METRICS + .worker_holds + .add(1, &[KeyValue::new(SCHEDULER_HOLD_OUTCOME, outcome)]); +} + +/// Records how many workers are held right now. +pub fn record_held_workers(count: usize) { + SCHEDULER_METRICS + .held_workers + .record(u64::try_from(count).unwrap_or(u64::MAX), &[]); +} + /// Records an operation the queue listed whose record was gone when read; /// `site` is where it was met (`matching`, `sweep`). pub fn record_awaited_action_orphan(site: &'static str) { diff --git a/web/apps/docs/content/docs/explanations/scheduler-internals.mdx b/web/apps/docs/content/docs/explanations/scheduler-internals.mdx index a87f651c88d..495262b753d 100644 --- a/web/apps/docs/content/docs/explanations/scheduler-internals.mdx +++ b/web/apps/docs/content/docs/explanations/scheduler-internals.mdx @@ -186,6 +186,25 @@ the parked actions a worker was actually told to run that way; a non-zero rate is normal on a busy fleet and means the head of the queue is being served. +Parking does not help an action whose reservation only fits a worker with +nothing else on it. Every pass tries it first and finds each worker partly +busy, then fills the room that opens with newer, smaller actions, so no +worker ever drains for it while the fleet reads as free. **`reservation_hold`** +is for that action: once it has waited `hold_after_s` finding no room while +some connected worker would fit it idle, the pass holds one such worker +(fewest actions running, then most memory left) and offers it nothing else; +as its actions finish the held action is placed there the moment the ledger +and the live veto allow. The hold ends when the action runs, anywhere, when +it leaves the queue, when the worker leaves or drains, or after `max_hold_s`, +whichever is first; an expired hold is not repeated for that action until as +long again has passed, so an action that never fits does not keep workers +idle in turns. At most `max_held_workers` are held at once (a tenth of the +connected workers by default), and a hold is local to the scheduler the +worker is connected to. `scheduler.worker.holds` counts holds by outcome and +`scheduler.worker.held` is how many workers are held right now. An action no +connected worker would fit even idle gets no hold; that is the unsatisfiable +path, or a sizing class larger than the workers less their overhead. + When a worker reports an action killed for memory, by its own reservation enforcement or by the kernel near the limit, **`memory_escalation`** requeues the action with the reservation multiplied by `percent` (default 200) instead diff --git a/web/apps/docs/content/docs/reference/metrics.mdx b/web/apps/docs/content/docs/reference/metrics.mdx index 4b5b19ea27c..b175e58aece 100644 --- a/web/apps/docs/content/docs/reference/metrics.mdx +++ b/web/apps/docs/content/docs/reference/metrics.mdx @@ -5,10 +5,10 @@ full: true --- {/* AUTOGENERATED. Do not edit by hand. - Source: nativelink-util/src/metrics.rs @ v1.7.6 (d0e6aa8e3) + Source: nativelink-util/src/metrics.rs @ v1.7.6 (1911c9c30) Regenerate from web/: bun --filter @nativelink/docs gen:metrics-reference */} -NativeLink **v1.7.6** declares 47 OpenTelemetry instruments. This page is +NativeLink **v1.7.6** declares 49 OpenTelemetry instruments. This page is generated from the declarations in [`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs) and from every `.add()` and `.record()` call site across the workspace, so the @@ -61,6 +61,8 @@ wrong. | [`scheduler.executing.oldest_age`](#scheduler-executing-oldest-age) | `nativelink_scheduler_executing_oldest_age` (unreferenced) | Gauge | Conditionally | scheduler process only | | [`scheduler.queue.retired`](#scheduler-queue-retired) | `nativelink_scheduler_queue_retired` (unreferenced) | Counter | Conditionally | scheduler process only | | [`scheduler.matching.parked_dispatched`](#scheduler-matching-parked-dispatched) | `nativelink_scheduler_matching_parked_dispatched` (unreferenced) | Counter | Conditionally | scheduler process only | +| [`scheduler.worker.holds`](#scheduler-worker-holds) | `nativelink_scheduler_worker_holds` (unreferenced) | Counter | Conditionally | scheduler process only | +| [`scheduler.worker.held`](#scheduler-worker-held) | `nativelink_scheduler_worker_held` (unreferenced) | Gauge | Conditionally | scheduler process only | | [`scheduler.awaited_action.orphans`](#scheduler-awaited-action-orphans) | `nativelink_scheduler_awaited_action_orphans` (unreferenced) | Counter | Conditionally | scheduler process only | | [`scheduler.sweep.failures`](#scheduler-sweep-failures) | `nativelink_scheduler_sweep_failures` (unreferenced) | Counter | Conditionally | scheduler process only | | [`scheduler.hints.resolutions`](#scheduler-hints-resolutions) | `nativelink_scheduler_hints_resolutions` (unreferenced) | Counter | Conditionally | scheduler process only | @@ -106,7 +108,7 @@ Every instrument declared in `metrics.rs` has at least one call site. Global cache metrics instruments. -Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:438`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L438), backed by the `CacheMetrics` struct. +Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:439`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L439), backed by the `CacheMetrics` struct. Every cache instrument is emitted from one place: the `cache_metrics` store wrapper. A store that is not wrapped emits nothing, no matter how the collector is configured. @@ -121,7 +123,7 @@ Histogram of cache operation durations in milliseconds | Unit | `ms` | | Prometheus series | `nativelink_cache_operation_duration` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:442`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L442) | +| Declared at | [`nativelink-util/src/metrics.rs:443`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L443) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:73`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L73) | **Histogram buckets:** 19 explicit boundaries, from @@ -140,7 +142,7 @@ Counter of cache operations by type and result | Unit | — | | Prometheus series | `nativelink_cache_operations_total` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:476`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L476) | +| Declared at | [`nativelink-util/src/metrics.rs:477`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L477) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:127`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L127), [`nativelink-store/src/cache_metrics_store.rs:132`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L132), [`nativelink-store/src/cache_metrics_store.rs:143`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L143), [`nativelink-store/src/cache_metrics_store.rs:168`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L168), [`nativelink-store/src/cache_metrics_store.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L174), [`nativelink-store/src/cache_metrics_store.rs:200`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L200), [`nativelink-store/src/cache_metrics_store.rs:206`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L206), [`nativelink-store/src/cache_metrics_store.rs:219`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L219), [`nativelink-store/src/cache_metrics_store.rs:225`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L225), [`nativelink-store/src/cache_metrics_store.rs:246`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L246), [`nativelink-store/src/cache_metrics_store.rs:254`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L254), [`nativelink-store/src/cache_metrics_store.rs:260`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L260) | ### `cache.io` @@ -154,7 +156,7 @@ Counter of bytes read/written during cache operations | Unit | `By` | | Prometheus series | `nativelink_cache_io_total` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:481`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L481) | +| Declared at | [`nativelink-util/src/metrics.rs:482`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L482) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:80`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L80), [`nativelink-store/src/cache_metrics_store.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L248) | ### `cache.size` @@ -168,7 +170,7 @@ Current total size of all cached data in bytes | Unit | `By` | | Prometheus series | `nativelink_cache_size` | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:487`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L487) | +| Declared at | [`nativelink-util/src/metrics.rs:488`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L488) | | Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) | ### `cache.entries` @@ -182,7 +184,7 @@ Current number of entries in cache | Unit | `{entry}` | | Prometheus series | `nativelink_cache_entries` | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:493`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L493) | +| Declared at | [`nativelink-util/src/metrics.rs:494`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L494) | | Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) | ### `cache.item.size` @@ -196,14 +198,14 @@ Histogram of individual cache entry sizes in bytes | Unit | `By` | | Prometheus series | `nativelink_cache_item_size` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:499`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L499) | +| Declared at | [`nativelink-util/src/metrics.rs:500`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L500) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:83`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L83) | ## Execution metrics Global remote execution metrics instruments. -Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:539`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L539), backed by the `ExecutionMetrics` struct. +Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:540`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L540), backed by the `ExecutionMetrics` struct. Execution instruments are emitted from the scheduler process as actions change state, plus the per-action CPU time and peak memory a worker reports back when it finishes. Worker processes emit no execution metrics of their own. @@ -218,7 +220,7 @@ Histogram of stage durations in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_stage_duration` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:543`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L543) | +| Declared at | [`nativelink-util/src/metrics.rs:544`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L544) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 15 explicit boundaries, from @@ -237,7 +239,7 @@ Histogram of total execution durations in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_total_duration` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:568`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L568) | +| Declared at | [`nativelink-util/src/metrics.rs:569`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L569) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 13 explicit boundaries, from @@ -256,7 +258,7 @@ Histogram of queue wait times in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_queue_time` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:593`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L593) | +| Declared at | [`nativelink-util/src/metrics.rs:594`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L594) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 12 explicit boundaries, from @@ -275,7 +277,7 @@ Current number of actions in each stage | Unit | `{action}` | | Prometheus series | `nativelink_execution_active_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:613`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L613) | +| Declared at | [`nativelink-util/src/metrics.rs:614`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L614) | | Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:425`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L425), [`nativelink-scheduler/src/memory_awaited_action_db.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L672), [`nativelink-scheduler/src/memory_awaited_action_db.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L679), [`nativelink-scheduler/src/memory_awaited_action_db.rs:824`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L824), [`nativelink-scheduler/src/store_awaited_action_db.rs:716`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/store_awaited_action_db.rs#L716) | ### `execution.completed.count` @@ -289,7 +291,7 @@ Total number of completed executions | Unit | `{action}` | | Prometheus series | `nativelink_execution_completed_count_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:619`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L619) | +| Declared at | [`nativelink-util/src/metrics.rs:620`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L620) | | Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:152`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L152), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:162`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L162) | ### `execution.stage.transitions` @@ -303,7 +305,7 @@ Number of stage transitions | Unit | `{transition}` | | Prometheus series | `nativelink_execution_stage_transitions_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:625`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L625) | +| Declared at | [`nativelink-util/src/metrics.rs:626`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L626) | | Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:141`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L141) | ### `execution.cpu.time` @@ -317,8 +319,8 @@ CPU time an action used, as sampled by the worker. | Unit | `s` | | Prometheus series | `nativelink_execution_cpu_time` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:631`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L631) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1911`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1911) | +| Declared at | [`nativelink-util/src/metrics.rs:632`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L632) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2155`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2155) | **Histogram buckets:** 12 explicit boundaries, from `0.1` to `7200.0` `s`. Anything past the last boundary lands in the @@ -336,8 +338,8 @@ Peak memory an action used, as sampled by the worker. | Unit | `By` | | Prometheus series | `nativelink_execution_peak_memory` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:640`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L640) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1903`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1903) | +| Declared at | [`nativelink-util/src/metrics.rs:641`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L641) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2147`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2147) | **Histogram buckets:** 8 explicit boundaries, from `1_048_576.0` to `68_719_476_736.0` `By`. Anything past the last boundary lands in the @@ -355,7 +357,7 @@ Histogram of output sizes in bytes | Unit | `By` | | Prometheus series | `nativelink_execution_output_size` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:656`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L656) | +| Declared at | [`nativelink-util/src/metrics.rs:657`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L657) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | **Histogram buckets:** 8 explicit boundaries, from @@ -374,14 +376,14 @@ Counter for execution retries | Unit | `{retry}` | | Prometheus series | `nativelink_execution_retry_count_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L672) | +| Declared at | [`nativelink-util/src/metrics.rs:673`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L673) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L174) | ## Worker fleet metrics Global worker fleet metrics instruments. -Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:881`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L881), backed by the `WorkerMetrics` struct. +Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L882), backed by the `WorkerMetrics` struct. Worker fleet instruments are emitted from the scheduler process as workers connect, heartbeat, pause, drain and leave. They describe the pool as this scheduler instance sees it. @@ -396,8 +398,8 @@ Workers currently connected. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_connected_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:885`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L885) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:464`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L464), [`nativelink-scheduler/src/api_worker_scheduler.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1491) | +| Declared at | [`nativelink-util/src/metrics.rs:886`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L886) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:494`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L494), [`nativelink-scheduler/src/api_worker_scheduler.rs:1696`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1696) | ### `worker.connections` @@ -410,8 +412,8 @@ Workers that have joined, cumulative. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_connections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:891`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L891) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:464`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L464) | +| Declared at | [`nativelink-util/src/metrics.rs:892`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L892) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:494`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L494) | ### `worker.disconnections` @@ -424,8 +426,8 @@ Workers that have left, cumulative, by reason. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_disconnections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:897`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L897) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1491) | +| Declared at | [`nativelink-util/src/metrics.rs:898`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L898) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1696`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1696) | ### `worker.input.relinks` @@ -438,7 +440,7 @@ Input hardlinks made again after their source was evicted mid-staging, by kind. | Unit | — | | Prometheus series | `nativelink_worker_input_relinks` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:903`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L903) | +| Declared at | [`nativelink-util/src/metrics.rs:904`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L904) | | Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1038), [`nativelink-worker/src/running_actions_manager.rs:1172`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1172), [`nativelink-worker/src/running_actions_manager.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1190) | ### `worker.prepare.retries` @@ -452,7 +454,7 @@ Input fetches tried again on the worker after a transient store failure: `retrie | Unit | — | | Prometheus series | `nativelink_worker_prepare_retries` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:908`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L908) | +| Declared at | [`nativelink-util/src/metrics.rs:909`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L909) | | Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1014`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1014) | ### `scheduler.dispatch.requeues` @@ -466,8 +468,8 @@ Dispatches requeued untried, by reason. | Unit | — | | Prometheus series | `nativelink_scheduler_dispatch_requeues` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L913) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1187`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1187), [`nativelink-scheduler/src/api_worker_scheduler.rs:1381`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1381), [`nativelink-scheduler/src/api_worker_scheduler.rs:1701`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1701), [`nativelink-scheduler/src/api_worker_scheduler.rs:1999`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1999), [`nativelink-scheduler/src/api_worker_scheduler.rs:2030`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2030), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L679) | +| Declared at | [`nativelink-util/src/metrics.rs:914`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L914) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1389`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1389), [`nativelink-scheduler/src/api_worker_scheduler.rs:1586`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1586), [`nativelink-scheduler/src/api_worker_scheduler.rs:1912`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1912), [`nativelink-scheduler/src/api_worker_scheduler.rs:2243`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2243), [`nativelink-scheduler/src/api_worker_scheduler.rs:2274`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2274), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L679) | ### `worker.keepalives` @@ -480,8 +482,8 @@ Keepalives received, cumulative. | Unit | `{keepalive}` | | Prometheus series | `nativelink_worker_keepalives` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:918`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L918) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:422`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L422) | +| Declared at | [`nativelink-util/src/metrics.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L919) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:452`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L452) | ### `worker.keepalive.gap` @@ -494,8 +496,8 @@ Seconds between consecutive messages from a worker. | Unit | `s` | | Prometheus series | `nativelink_worker_keepalive_gap` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:924`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L924) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:356`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L356) | +| Declared at | [`nativelink-util/src/metrics.rs:925`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L925) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:386`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L386) | **Histogram buckets:** 12 explicit boundaries, from `1.0` to `60.0` `s`. Anything past the last boundary lands in the @@ -513,8 +515,8 @@ Connected workers currently paused or draining. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_state_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L935) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:405`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L405), [`nativelink-scheduler/src/api_worker_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L458), [`nativelink-scheduler/src/api_worker_scheduler.rs:461`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L461), [`nativelink-scheduler/src/api_worker_scheduler.rs:571`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L571), [`nativelink-scheduler/src/api_worker_scheduler.rs:1148`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1148), [`nativelink-scheduler/src/api_worker_scheduler.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1190), [`nativelink-scheduler/src/api_worker_scheduler.rs:1326`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1326) | +| Declared at | [`nativelink-util/src/metrics.rs:936`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L936) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:435`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L435), [`nativelink-scheduler/src/api_worker_scheduler.rs:488`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L488), [`nativelink-scheduler/src/api_worker_scheduler.rs:491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L491), [`nativelink-scheduler/src/api_worker_scheduler.rs:602`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L602), [`nativelink-scheduler/src/api_worker_scheduler.rs:1350`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1350), [`nativelink-scheduler/src/api_worker_scheduler.rs:1392`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1392), [`nativelink-scheduler/src/api_worker_scheduler.rs:1531`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1531) | ## gRPC server metrics @@ -523,7 +525,7 @@ Global gRPC serving metrics. One duration histogram covers rate, errors and latency: the count gives rate, the status attribute separates errors, and the buckets give latency. -Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1059`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1059), backed by the `RpcMetrics` struct. +Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1060`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1060), backed by the `RpcMetrics` struct. Emitted by the `OtlpLayer` tower middleware on every gRPC server NativeLink starts, so a CAS, a scheduler and a worker API endpoint each report their own. @@ -538,7 +540,7 @@ Duration of inbound gRPC calls, by service, method and status. | Unit | `s` | | Prometheus series | `nativelink_rpc_server_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1063`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1063) | +| Declared at | [`nativelink-util/src/metrics.rs:1064`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1064) | | Emitted from | [`nativelink-util/src/telemetry.rs:371`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/telemetry.rs#L371) | **Histogram buckets:** 15 explicit boundaries, from @@ -550,7 +552,7 @@ measurement. Global scheduler metrics. -Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1112`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1112), backed by the `SchedulerOtlpMetrics` struct. +Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1113`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1113), backed by the `SchedulerOtlpMetrics` struct. Emitted from the scheduler process once per matching pass, whether or not the pass assigned anything. @@ -566,8 +568,8 @@ while the queue is non-empty means matching is the bottleneck. | Unit | `s` | | Prometheus series | `nativelink_scheduler_matching_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1116`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1116) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L458) | +| Declared at | [`nativelink-util/src/metrics.rs:1117`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1117) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:462`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L462) | **Histogram buckets:** 13 explicit boundaries, from `0.0001` to `10.0` `s`. Anything past the last boundary lands in the @@ -585,8 +587,8 @@ Matching passes, by result. | Unit | `{pass}` | | Prometheus series | `nativelink_scheduler_matching_passes` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1125`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1125) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L458) | +| Declared at | [`nativelink-util/src/metrics.rs:1126`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1126) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:462`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L462) | ### `scheduler.unsatisfiable.queued` @@ -601,8 +603,8 @@ instance, judged against that instance's own workers. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_unsatisfiable_queued` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1131`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1131) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1036`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1036) | +| Declared at | [`nativelink-util/src/metrics.rs:1132`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1132) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1066`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1066) | ### `scheduler.unsatisfiable.failed` @@ -616,8 +618,8 @@ the properties that could not be satisfied. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_unsatisfiable_failed` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1139`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1139) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:560`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L560) | +| Declared at | [`nativelink-util/src/metrics.rs:1140`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1140) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:564`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L564) | ### `scheduler.queue.depth` @@ -630,8 +632,8 @@ Queued actions the last matching pass read. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_depth` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1145`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1145) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1037`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1037) | +| Declared at | [`nativelink-util/src/metrics.rs:1146`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1146) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1067`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1067) | ### `scheduler.queue.age` @@ -644,8 +646,8 @@ Queued actions the last matching pass read, by age bucket. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1151`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1151) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1038) | +| Declared at | [`nativelink-util/src/metrics.rs:1152`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1152) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1068`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1068) | ### `scheduler.queue.oldest_age` @@ -658,8 +660,8 @@ Age of the oldest action the last matching pass read, in seconds. | Unit | `s` | | Prometheus series | `nativelink_scheduler_queue_oldest_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1159`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1159) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1038) | +| Declared at | [`nativelink-util/src/metrics.rs:1160`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1160) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1068`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1068) | ### `scheduler.executing.age` @@ -672,8 +674,8 @@ Executing actions the last executing sweep read, by age bucket. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_executing_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1167`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1167) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1307`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1307) | +| Declared at | [`nativelink-util/src/metrics.rs:1168`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1168) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1338`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1338) | ### `scheduler.executing.oldest_age` @@ -687,8 +689,8 @@ executing, in seconds. | Unit | `s` | | Prometheus series | `nativelink_scheduler_executing_oldest_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1175`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1175) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1307`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1307) | +| Declared at | [`nativelink-util/src/metrics.rs:1176`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1176) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1338`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1338) | ### `scheduler.queue.retired` @@ -701,7 +703,7 @@ Queued actions retired for having no client. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_retired` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1183`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1183) | +| Declared at | [`nativelink-util/src/metrics.rs:1184`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1184) | | Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:783`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L783) | ### `scheduler.matching.parked_dispatched` @@ -716,8 +718,37 @@ opened before the pass ended, ahead of the newer actions it was reading. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_matching_parked_dispatched` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1189`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1189) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1013`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1013) | +| Declared at | [`nativelink-util/src/metrics.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1190) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1043`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1043) | + +### `scheduler.worker.holds` + +Workers held for a queued action that kept finding no room, by what +began or ended the hold. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `worker_holds` | +| Unit | `{hold}` | +| Prometheus series | `nativelink_scheduler_worker_holds` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:1198`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1198) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:621`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L621), [`nativelink-scheduler/src/api_worker_scheduler.rs:707`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L707) | + +### `scheduler.worker.held` + +Workers held for such an action right now. + +| Property | Value | +| --- | --- | +| Instrument type | `Gauge` | +| Field | `held_workers` | +| Unit | `{worker}` | +| Prometheus series | `nativelink_scheduler_worker_held` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:1206`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1206) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:622`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L622), [`nativelink-scheduler/src/api_worker_scheduler.rs:708`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L708) | ### `scheduler.awaited_action.orphans` @@ -730,8 +761,8 @@ Listed operations whose record was gone on read, by site. | Unit | `{operation}` | | Prometheus series | `nativelink_scheduler_awaited_action_orphans` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1197`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1197) | -| Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:600`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L600), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:614`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L614), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:731`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L731), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1662`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1662), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1685`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1685) | +| Declared at | [`nativelink-util/src/metrics.rs:1212`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1212) | +| Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:604`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L604), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:614`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L614), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:731`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L731), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1662`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1662), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1685`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1685) | ### `scheduler.sweep.failures` @@ -744,8 +775,8 @@ Abandoned-queue sweeps that failed before finishing. | Unit | `{sweep}` | | Prometheus series | `nativelink_scheduler_sweep_failures` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1205`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1205) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1259`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1259), [`nativelink-scheduler/src/simple_scheduler.rs:1318`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1318) | +| Declared at | [`nativelink-util/src/metrics.rs:1220`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1220) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1290`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1290), [`nativelink-scheduler/src/simple_scheduler.rs:1349`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1349) | ### `scheduler.hints.resolutions` @@ -758,14 +789,14 @@ Actions sized by the historical resource scheduler, by source. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_hints_resolutions` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1211`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1211) | +| Declared at | [`nativelink-util/src/metrics.rs:1226`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1226) | | Emitted from | [`nativelink-scheduler/src/historical_resource_scheduler.rs:643`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L643), [`nativelink-scheduler/src/historical_resource_scheduler.rs:647`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L647), [`nativelink-scheduler/src/historical_resource_scheduler.rs:650`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L650) | ## STORE_METRICS Global object-store metrics. -Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1427`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1427), backed by the `StoreMetrics` struct. +Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1463`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1463), backed by the `StoreMetrics` struct. ### `store.object.throttles` @@ -778,14 +809,14 @@ Throttled responses, by provider and operation. | Unit | `{response}` | | Prometheus series | `nativelink_store_object_throttles` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1431`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1431) | +| Declared at | [`nativelink-util/src/metrics.rs:1467`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1467) | | Emitted from | [`nativelink-store/src/azure_blob_store.rs:268`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/azure_blob_store.rs#L268) | ## Store tier metrics Global tiered-store metrics. -Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1455`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1455), backed by the `StoreTierMetrics` struct. +Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1491), backed by the `StoreTierMetrics` struct. Emitted from the `fast_slow` store wrapper only. A deployment without a `fast_slow` store emits none of these. @@ -800,7 +831,7 @@ Reads served per tier, by result. | Unit | `{operation}` | | Prometheus series | `nativelink_store_tier_operations` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1459`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1459) | +| Declared at | [`nativelink-util/src/metrics.rs:1495`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1495) | | Emitted from | [`nativelink-store/src/fast_slow_store.rs:348`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L348), [`nativelink-store/src/fast_slow_store.rs:839`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L839), [`nativelink-store/src/fast_slow_store.rs:855`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L855), [`nativelink-store/src/fast_slow_store.rs:865`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L865), [`nativelink-store/src/fast_slow_store.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L882), [`nativelink-store/src/fast_slow_store.rs:912`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L912), [`nativelink-store/src/fast_slow_store.rs:934`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L934) | ### `store.tier.io` @@ -814,14 +845,14 @@ Bytes moved per tier and direction. | Unit | `By` | | Prometheus series | `nativelink_store_tier_io` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1465`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1465) | +| Declared at | [`nativelink-util/src/metrics.rs:1501`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1501) | | Emitted from | [`nativelink-store/src/fast_slow_store.rs:357`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L357), [`nativelink-store/src/fast_slow_store.rs:856`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L856), [`nativelink-store/src/fast_slow_store.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L913), [`nativelink-store/src/fast_slow_store.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L935) | ## Health check metrics Global health-check metrics. -Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1506`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1506), backed by the `HealthMetrics` struct. +Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1542`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1542), backed by the `HealthMetrics` struct. Emitted each time the health check endpoint runs, one observation per registered health indicator. @@ -836,14 +867,14 @@ Health check results, by namespace and status. | Unit | `{check}` | | Prometheus series | `nativelink_health_checks` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1510`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1510) | +| Declared at | [`nativelink-util/src/metrics.rs:1546`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1546) | | Emitted from | [`nativelink-util/src/health_utils.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/health_utils.rs#L248) | ## Connection pool metrics Global connection-pool metrics. -Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1537`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1537), backed by the `ConnectionMetrics` struct. +Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1573`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1573), backed by the `ConnectionMetrics` struct. Emitted by the gRPC connection manager and the Redis store as they hand out and re-establish connections. A deployment that uses neither emits none of these. @@ -858,7 +889,7 @@ Free slots at the moment a connection was taken. | Unit | `{connection}` | | Prometheus series | `nativelink_connection_pool_available` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1541`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1541) | +| Declared at | [`nativelink-util/src/metrics.rs:1577`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1577) | | Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L351) | **Histogram buckets:** 11 explicit boundaries, from @@ -877,7 +908,7 @@ Acquisitions, by pool and result. | Unit | `{acquisition}` | | Prometheus series | `nativelink_connection_pool_acquisitions` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1550`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1550) | +| Declared at | [`nativelink-util/src/metrics.rs:1586`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1586) | | Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L351) | ### `connection.reconnects` @@ -891,7 +922,7 @@ Reconnects, by pool. | Unit | `{reconnect}` | | Prometheus series | `nativelink_connection_reconnects` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1556`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1556) | +| Declared at | [`nativelink-util/src/metrics.rs:1592`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1592) | | Emitted from | [`nativelink-store/src/redis_store.rs:319`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L319) | ## Attributes @@ -943,6 +974,7 @@ underscores. | --- | --- | --- | --- | | `scheduler.match.result` | `scheduler_match_result` | `SCHEDULER_MATCH_RESULT` | open (free-form string) | | `scheduler.unsatisfiable.properties` | `scheduler_unsatisfiable_properties` | `SCHEDULER_UNSATISFIABLE_PROPERTIES` | open (free-form string) | +| `scheduler.hold.outcome` | `scheduler_hold_outcome` | `SCHEDULER_HOLD_OUTCOME` | open (free-form string) | ### Store tier attributes From d7005bb9f1ab8709c79f0667c2a614b0662f1ceb Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 13 Oct 2026 14:10:40 -0400 Subject: [PATCH 2/3] Pass the new scheduler arguments in the worker API test --- nativelink-service/tests/worker_api_server_test.rs | 4 ++++ web/apps/docs/content/docs/reference/metrics.mdx | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/nativelink-service/tests/worker_api_server_test.rs b/nativelink-service/tests/worker_api_server_test.rs index bc9fcffb335..f7f07711a60 100644 --- a/nativelink-service/tests/worker_api_server_test.rs +++ b/nativelink-service/tests/worker_api_server_test.rs @@ -193,6 +193,7 @@ async fn setup_api_server_with( WorkerAllocationStrategy::default(), None, None, + None, tasks_or_worker_change_notify, worker_timeout, 60, // unacknowledged_kill_timeout_s @@ -525,6 +526,7 @@ pub async fn going_away_with_drain_stops_dispatch_and_keeps_the_worker_until_the &PlatformProperties::new(HashMap::new()), false, make_system_time(BASE_NOW_S), + None, ) .await; assert!( @@ -955,6 +957,7 @@ pub async fn workers_only_allow_max_tasks() -> Result<(), Box Result<(), Box Date: Tue, 13 Oct 2026 16:48:43 -0400 Subject: [PATCH 3/3] Keep a hold through the dispatch, trigger it on need, and take turns across replicas --- nativelink-config/src/schedulers.rs | 33 +- .../src/api_worker_scheduler.rs | 469 ++++++++++-- .../src/awaited_action_db/mod.rs | 20 +- .../src/default_scheduler_factory.rs | 9 + nativelink-scheduler/src/simple_scheduler.rs | 70 +- .../src/simple_scheduler_state_manager.rs | 5 +- .../src/store_awaited_action_db.rs | 9 +- .../tests/fleet_capabilities_test.rs | 32 +- .../tests/reservation_hold_test.rs | 676 +++++++++++++++--- .../worker_running_operations_report_test.rs | 1 + .../tests/worker_api_server_test.rs | 1 + nativelink-util/src/metrics.rs | 10 +- .../docs/explanations/scheduler-internals.mdx | 31 +- .../docs/content/docs/reference/metrics.mdx | 86 +-- 14 files changed, 1161 insertions(+), 291 deletions(-) diff --git a/nativelink-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs index 5fb52fd9f94..ad00f642c80 100644 --- a/nativelink-config/src/schedulers.rs +++ b/nativelink-config/src/schedulers.rs @@ -350,13 +350,17 @@ pub struct SimpleSpec { /// load: every pass tries it first and finds each worker partly busy, /// then fills the room that opens with newer, smaller actions, so no /// worker ever drains for it. With this set, once such an action has - /// waited `hold_after_s` while some connected worker would fit it - /// idle, the scheduler picks one such worker and offers it nothing - /// else until the action is placed, leaves the queue, or `max_hold_s` + /// waited `hold_after_s` and fits no busy worker until more than one + /// of that worker's actions finish (one finishing would make room for + /// an action that is only waiting its turn), the scheduler picks a + /// worker that would fit it idle and offers it nothing else until a + /// worker takes the action, it leaves the queue, or `max_hold_s` /// passes. An action no connected worker would fit even idle gets no - /// hold. With the matcher in every CAS replica a hold is local to the - /// replica the worker is connected to, which is the only one that can - /// dispatch to it. + /// hold. Each scheduler replica holds only workers connected to it; + /// replicas sharing a queue take turns per action, by a hash of the + /// action over the replicas whose workers would fit it as the fleet + /// census last read them, so one worker is held for it fleet-wide. + /// While a replica cannot read the census it holds on its own. /// Default: unset (off) #[serde(default)] pub reservation_hold: Option, @@ -592,7 +596,7 @@ const fn default_memory_escalation_percent() -> u64 { #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] pub struct ReservationHoldSpec { /// Seconds a queued action has to have waited, finding no room on - /// every pass, before a worker is held for it. + /// every pass, before a worker is held for it. Must be above zero. /// Default: 30 #[serde( default = "default_reservation_hold_after_s", @@ -600,10 +604,13 @@ pub struct ReservationHoldSpec { )] pub hold_after_s: u64, - /// Seconds a hold lasts at most. A hold that long without the action - /// fitting means the worker never reported room for it even drained; - /// the hold is given up so the worker is not idle for good, and the - /// action is not held for again until as long has passed. + /// Seconds a hold lasts at most, capped at a week. A hold that long + /// without the action fitting means the worker never reported room for + /// it even drained; the hold is given up so the worker is not idle for + /// good, and the action is not held for again until as long has + /// passed, twice as long for each such expiry in a row on a worker + /// that was idle (up to sixteen times), since then nothing kept the + /// action off the worker but the action itself. Must be above zero. /// Default: 300 #[serde( default = "default_reservation_max_hold_s", @@ -611,8 +618,8 @@ pub struct ReservationHoldSpec { )] pub max_hold_s: u64, - /// Most workers held at once on this scheduler. 0 is one tenth of the - /// workers connected to it, and at least one. + /// Most workers held at once on this scheduler replica. 0 is one tenth + /// of the workers connected to it, and at least one. /// Default: 0 #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub max_held_workers: usize, diff --git a/nativelink-scheduler/src/api_worker_scheduler.rs b/nativelink-scheduler/src/api_worker_scheduler.rs index 830c3187495..64d7439e449 100644 --- a/nativelink-scheduler/src/api_worker_scheduler.rs +++ b/nativelink-scheduler/src/api_worker_scheduler.rs @@ -13,9 +13,11 @@ // limitations under the License. use core::cmp::Reverse; +use core::hash::{Hash, Hasher}; use core::ops::{Deref, DerefMut}; use core::sync::atomic::{AtomicU64, Ordering}; use core::time::Duration; +use std::collections::hash_map::DefaultHasher; use std::collections::{BTreeSet, HashMap, HashSet}; use std::sync::Arc; use std::time::{Instant, SystemTime, UNIX_EPOCH}; @@ -137,6 +139,7 @@ pub struct SchedulerMetrics { pub dispatches_unacknowledged: AtomicU64, } +use crate::awaited_action_db::PeerScheduler; use crate::match_outcome::{ MatchOutcome, PropertyShape, UnsatisfiableReason, explain_unsatisfiable, }; @@ -191,6 +194,82 @@ struct Hold { /// A pass that read the whole queue without meeting it means the /// action is gone, and so is the reason to hold. seen_in: u64, + /// Whether the action is on its way to a worker, which took it off the + /// queue: the passes meanwhile do not meet it, and the hold waits for + /// the worker to take it or send it back. + dispatched: bool, +} + +/// An action whose hold expired: it is not held for again until `until`. +#[derive(Debug, Clone)] +struct HoldCooldown { + until: SystemTime, + /// Expiries in a row on a worker that was idle, each doubling the + /// cooldown, up to `MAX_IDLE_EXPIRY_DOUBLINGS`. + idle_expiries: u32, + /// The last matching pass that read the action; see `Hold::seen_in`. + seen_in: u64, +} + +/// The longest a hold lasts whatever `max_hold_s` says, so that the time +/// it ends at can always be computed. +const HOLD_CEILING_S: u64 = 7 * 24 * 60 * 60; + +/// How many times in a row an expiry on an idle worker doubles the wait +/// before the action is held for again: sixteen times `max_hold_s` at most. +const MAX_IDLE_EXPIRY_DOUBLINGS: u32 = 4; + +fn hold_duration(spec: ReservationHoldSpec) -> Duration { + Duration::from_secs(spec.max_hold_s.min(HOLD_CEILING_S)) +} + +/// A queued action a matching pass asks a worker for: which action, so a +/// worker held for it recognises it, and since when it has waited, for +/// whether to hold a worker for it. +#[derive(Debug, Clone, Copy)] +pub struct QueuedAction<'a> { + pub unique_qualifier: &'a ActionUniqueQualifier, + pub insert_timestamp: SystemTime, +} + +/// Which of `owners`, sorted, distinct and not empty, holds a worker for +/// `action`: the same one on every scheduler that sees the same owners, +/// by a hash of the action. Replicas run one binary, so the hash agrees. +#[must_use] +pub fn hold_owner<'a>(action: &ActionUniqueQualifier, owners: &[&'a str]) -> &'a str { + let mut hasher = DefaultHasher::new(); + action.hash(&mut hasher); + let count = u64::try_from(owners.len()).unwrap_or(u64::MAX); + let index = usize::try_from(hasher.finish() % count).unwrap_or(0); + owners[index] +} + +/// Whether `platform_properties` would fit `worker` once any one of its +/// running actions finished: every `Minimum` it asks is within what the +/// worker has left plus the largest reservation of that property among +/// the actions running on it. +fn fits_after_one_finishes(platform_properties: &PlatformProperties, worker: &Worker) -> bool { + platform_properties.properties.iter().all(|(name, value)| { + let PlatformPropertyValue::Minimum(needed) = value else { + return true; + }; + let left = match worker.platform_properties.properties.get(name) { + Some(PlatformPropertyValue::Minimum(left)) => *left, + _ => 0, + }; + let largest_running = worker + .running_action_infos + .values() + .filter_map(|pending| { + match pending.action_info.platform_properties.properties.get(name) { + Some(PlatformPropertyValue::Minimum(reserved)) => Some(*reserved), + _ => None, + } + }) + .max() + .unwrap_or(0); + *needed <= left.saturating_add(largest_running) + }) } /// A collection of workers that are available to run tasks. @@ -220,7 +299,7 @@ struct ApiWorkerSchedulerImpl { holds_by_action: HashMap, /// Actions whose hold expired, and until when they are not held for /// again, so one that never fits does not keep a worker idle in turns. - hold_cooldowns: HashMap, + hold_cooldowns: HashMap, /// Counts matching passes, so a hold can record the last pass that /// met its action; see `Hold::seen_in`. pass_generation: u64, @@ -258,6 +337,14 @@ struct ApiWorkerSchedulerImpl { /// judged unsatisfiable, since a peer might have a worker for it. peer_fleet: Option>, + /// The peers as the census last read them, sorted by id, for deciding + /// which scheduler holds a worker for an action; empty while the census + /// is unknown. + peer_schedulers: Vec, + + /// The id this scheduler publishes to its peers under. + scheduler_id: String, + /// Whether this scheduler shares its state with peers. A non-shared db is /// the whole fleet, so peer-absence is authoritative at once; a shared one /// applies the census warm-up / staleness checks below before it trusts @@ -549,19 +636,26 @@ impl ApiWorkerSchedulerImpl { .collect() } - fn set_peer_fleet(&mut self, peer_fleet: Option>, now: SystemTime) { + fn set_peer_fleet(&mut self, peers: Option>, now: SystemTime) { // Peers list their workers in no particular order and may share // shapes, so sort and dedup before comparing, or an unchanged fleet // would look changed on every exchange and empty `static_verdicts`. - let peer_fleet = peer_fleet.map(|peer_fleet| { - let mut keyed: Vec<_> = peer_fleet - .into_iter() - .map(|totals| (PropertyShape::from(&totals), totals)) + let peer_fleet = peers.as_ref().map(|peers| { + let mut keyed: Vec<_> = peers + .iter() + .flat_map(|peer| peer.workers.iter()) + .map(|totals| (PropertyShape::from(totals), totals.clone())) .collect(); keyed.sort_unstable_by(|a, b| a.0.cmp(&b.0)); keyed.dedup_by(|a, b| a.0 == b.0); keyed.into_iter().map(|(_, totals)| totals).collect() }); + // Which scheduler holds a worker for which action goes by the + // peers' ids as well as their shapes, so the peers are kept as read + // even when the union of shapes did not move. + let mut peer_schedulers = peers.unwrap_or_default(); + peer_schedulers.sort_unstable_by(|a, b| a.scheduler_id.cmp(&b.scheduler_id)); + self.peer_schedulers = peer_schedulers; // Freshness bookkeeping: update the warm-up / // staleness clocks BEFORE the unchanged-shapes early return, so a // successful exchange that happens to return the same shapes still @@ -612,11 +706,15 @@ impl ApiWorkerSchedulerImpl { /// Ends the hold on `worker_id`, if any, for `outcome`. The worker is /// room the fleet did not have a moment ago, so the matcher is woken. + /// An action placed starts over with its cooldown. fn end_hold(&mut self, worker_id: &WorkerId, outcome: &'static str) { let Some(hold) = self.holds.remove(worker_id) else { return; }; self.holds_by_action.remove(&hold.action); + if outcome == "placed" { + self.hold_cooldowns.remove(&hold.action); + } info!(?worker_id, action = ?hold.action, outcome, "Worker hold ended"); record_worker_hold(outcome); record_held_workers(self.holds.len()); @@ -630,31 +728,46 @@ impl ApiWorkerSchedulerImpl { } } - /// Holds a worker for `action`, which has waited long enough finding - /// no room: of the workers neither draining nor held that would fit - /// it idle, the one with the fewest actions running and then the most - /// memory left. Nothing is held when holds are off, the cap is - /// reached, the action's last hold expired too recently, or no - /// connected worker would fit it even idle, which is not a matter of - /// waiting. Returns whether the action holds a worker now. - fn hold_worker_for( + /// Records that the held action was sent to a worker (`true`), or came + /// back from one untried (`false`); see `Hold::dispatched`. + fn note_hold_dispatched(&mut self, action: &ActionUniqueQualifier, dispatched: bool) { + if let Some(worker_id) = self.holds_by_action.get(action) + && let Some(hold) = self.holds.get_mut(worker_id) + { + hold.dispatched = dispatched; + } + } + + /// A queued action found no room. Once it has waited `hold_after_s`, + /// if it fits no busy worker until more than one of that worker's + /// actions finish, and it is this scheduler's turn to hold for it, a + /// worker is held for it here, under the lock the match ran under. + fn consider_hold( &mut self, - action: &ActionUniqueQualifier, + asking: Option>, platform_properties: &PlatformProperties, now: SystemTime, - ) -> bool { - let Some(spec) = &self.reservation_hold else { - return false; + full_worker_logging: bool, + ) { + let (Some(spec), Some(asking)) = (self.reservation_hold, asking) else { + return; }; + let action = asking.unique_qualifier; if self.holds_by_action.contains_key(action) { - return true; + return; + } + let waited = now + .duration_since(asking.insert_timestamp) + .unwrap_or_default(); + if waited < Duration::from_secs(spec.hold_after_s) { + return; } if self .hold_cooldowns .get(action) - .is_some_and(|until| now < *until) + .is_some_and(|cooldown| now < cooldown.until) { - return false; + return; } let cap = if spec.max_held_workers == 0 { (self.workers.len() / 10).max(1) @@ -662,14 +775,116 @@ impl ApiWorkerSchedulerImpl { spec.max_held_workers }; if self.holds.len() >= cap { + return; + } + if !self.only_an_idle_worker_fits(platform_properties) { + if full_worker_logging { + info!("No hold: one action finishing on some worker would make room for this one"); + } + return; + } + if !self.owns_hold(action, platform_properties) { + if full_worker_logging { + info!("No hold here: a peer scheduler holds a worker for this action"); + } + return; + } + self.hold_worker_for(action, platform_properties, now, spec); + } + + /// Whether `platform_properties` fits no matching worker that has an + /// action running until more than one of its actions finish: on each + /// such worker some `Minimum` the action asks exceeds what the worker + /// has left plus the most any one of its running actions reserved. An + /// action one completion would make room for is only waiting its turn, + /// which the pass order gives it; a hold for it would idle slots for + /// nothing. False too when no matching worker is busy: the idle ones + /// refuse it for a reason a hold does not change. + fn only_an_idle_worker_fits(&self, platform_properties: &PlatformProperties) -> bool { + let mut any_busy = false; + for (_, worker) in self.workers.iter() { + if worker.is_draining + || worker.running_action_infos.is_empty() + || !platform_properties.is_satisfied_by(&worker.total_platform_properties, false) + { + continue; + } + any_busy = true; + if fits_after_one_finishes(platform_properties, worker) { + return false; + } + } + any_busy + } + + /// Whether `action`, which found no room, is this scheduler's to hold a + /// worker for. Every replica reads the same queue in the same order, so + /// left to themselves they would each drain a worker of their own for + /// the same action. The schedulers with a worker that would fit it idle + /// (this one, and the peers as the census last read them) take turns + /// by a hash of the action; while the census cannot be read, this one + /// holds. + fn owns_hold( + &self, + action: &ActionUniqueQualifier, + platform_properties: &PlatformProperties, + ) -> bool { + let fits_here = self.workers.iter().any(|(_, worker)| { + platform_properties.is_satisfied_by(&worker.total_platform_properties, false) + }); + if !fits_here { return false; } + if self.peer_fleet.is_none() { + return true; + } + let mut owners: Vec<&str> = self + .peer_schedulers + .iter() + .filter(|peer| { + peer.workers + .iter() + .any(|totals| platform_properties.is_satisfied_by(totals, false)) + }) + .map(|peer| peer.scheduler_id.as_str()) + .collect(); + if owners.is_empty() { + return true; + } + owners.push(self.scheduler_id.as_str()); + owners.sort_unstable(); + owners.dedup(); + hold_owner(action, &owners) == self.scheduler_id + } + + /// Holds a worker for `action`: of the workers neither draining nor + /// held that would fit it idle and have not declined that much while + /// idle, the one with the fewest actions running and then the most + /// memory free, as last reported or else as the ledger has it. Nothing + /// is held when no connected worker would fit the action even idle, + /// which is not a matter of waiting. + fn hold_worker_for( + &mut self, + action: &ActionUniqueQualifier, + platform_properties: &PlatformProperties, + now: SystemTime, + spec: ReservationHoldSpec, + ) { + let needed_kb = self.live_memory_veto_kb(platform_properties); let memory_property = self.live_memory_veto.as_deref(); - let memory_left = |worker: &Worker| match memory_property - .and_then(|property| worker.platform_properties.properties.get(property)) - { - Some(PlatformPropertyValue::Minimum(kb)) => *kb, - _ => 0, + let memory_free = |worker: &Worker| { + worker + .last_load + .map(|load| load.free_memory_kb) + .or_else(|| { + match memory_property + .and_then(|property| worker.platform_properties.properties.get(property)) + { + Some(PlatformPropertyValue::Minimum(kb)) => Some(*kb), + _ => None, + } + }) + .unwrap_or(0) }; let holds = &self.holds; let chosen = self @@ -679,34 +894,43 @@ impl ApiWorkerSchedulerImpl { .filter(|(_, worker)| { platform_properties.is_satisfied_by(&worker.total_platform_properties, false) }) + // A worker that declined this much while idle is not offered + // it again until it reports that much free; a hold on it would + // wait for the worker, not for its actions. + .filter(|(_, worker)| { + !matches!( + (needed_kb, worker.idle_declined_kb), + (Some(needed_kb), Some(declined_kb)) if needed_kb >= declined_kb + ) + }) .min_by_key(|(_, worker)| { ( worker.running_action_infos.len(), - Reverse(memory_left(worker)), + Reverse(memory_free(worker)), ) }) .map(|(worker_id, _)| worker_id.clone()); let Some(worker_id) = chosen else { - return false; + return; }; info!( ?worker_id, ?action, max_hold_s = spec.max_hold_s, - "Holding a worker for a queued action that keeps finding no room" + "Holding a worker for a queued action that fits only an idle one" ); self.holds.insert( worker_id.clone(), Hold { action: action.clone(), - until: now + Duration::from_secs(spec.max_hold_s), + until: now.checked_add(hold_duration(spec)).unwrap_or(now), seen_in: self.pass_generation, + dispatched: false, }, ); self.holds_by_action.insert(action.clone(), worker_id); record_worker_hold("held"); record_held_workers(self.holds.len()); - true } /// A matching pass begins; returns its generation, which the holds it @@ -716,20 +940,25 @@ impl ApiWorkerSchedulerImpl { self.pass_generation } - /// Ends the holds that no pass since `pass` began has met, whose - /// action is gone, and the ones past their time, whose action is then - /// not held for again for as long. Judged by generation rather than a - /// flag so that two passes in flight at once (a test entry point - /// beside the matcher's own loop) cannot clear each other's marks. + /// Ends the holds past their time, whose action is then not held for + /// again for as long (twice as long for each such expiry in a row on a + /// worker that was idle, up to a bound: nothing kept the action off it + /// but the action itself), and the ones no pass since `pass` began has + /// met, whose action is gone. A hold whose action is in flight to a + /// worker is not met by passes either and waits for the worker to + /// answer. Judged by generation rather than a flag so that two passes + /// in flight at once (a test entry point beside the matcher's own loop) + /// cannot clear each other's marks. A cooldown no pass has met since is + /// dropped the same way. fn end_pass_holds(&mut self, pass: u64, now: SystemTime) { let ended: Vec<(WorkerId, &'static str)> = self .holds .iter() .filter_map(|(worker_id, hold)| { - if hold.seen_in < pass { - Some((worker_id.clone(), "released")) - } else if hold.until <= now { + if hold.until <= now { Some((worker_id.clone(), "expired")) + } else if hold.seen_in < pass && !hold.dispatched { + Some((worker_id.clone(), "released")) } else { None } @@ -737,26 +966,46 @@ impl ApiWorkerSchedulerImpl { .collect(); for (worker_id, outcome) in ended { if outcome == "expired" + && let Some(spec) = self.reservation_hold && let Some(hold) = self.holds.get(&worker_id) - && let Some(spec) = &self.reservation_hold { - self.hold_cooldowns.insert( - hold.action.clone(), - now + Duration::from_secs(spec.max_hold_s), - ); + let idle = self + .workers + .peek(&worker_id) + .is_some_and(|worker| worker.running_action_infos.is_empty()); + let cooldown = + self.hold_cooldowns + .entry(hold.action.clone()) + .or_insert(HoldCooldown { + until: now, + idle_expiries: 0, + seen_in: pass, + }); + if idle { + cooldown.idle_expiries = + (cooldown.idle_expiries + 1).min(MAX_IDLE_EXPIRY_DOUBLINGS); + } + cooldown.until = now + .checked_add(hold_duration(spec) * (1 << cooldown.idle_expiries)) + .unwrap_or(now); } self.end_hold(&worker_id, outcome); } - self.hold_cooldowns.retain(|_, until| now < *until); + self.hold_cooldowns + .retain(|_, cooldown| cooldown.seen_in >= pass); } - /// Marks the hold for `action`, if any, as met by the pass under way. + /// Marks the hold and the cooldown for `action`, if any, as met by the + /// pass under way. fn note_held_action_seen(&mut self, action: &ActionUniqueQualifier) { if let Some(worker_id) = self.holds_by_action.get(action) && let Some(hold) = self.holds.get_mut(worker_id) { hold.seen_in = self.pass_generation; } + if let Some(cooldown) = self.hold_cooldowns.get_mut(action) { + cooldown.seen_in = self.pass_generation; + } } /// Whether peer-absence can be trusted to fail an action right now — a @@ -973,6 +1222,26 @@ impl ApiWorkerSchedulerImpl { } fn inner_find_worker_for_action( + &mut self, + platform_properties: &PlatformProperties, + full_worker_logging: bool, + now: SystemTime, + asking: Option>, + ) -> MatchOutcome { + let outcome = self.match_worker_for_action( + platform_properties, + full_worker_logging, + now, + asking.map(|action| action.unique_qualifier), + ); + if matches!(outcome, MatchOutcome::WaitingForCapacity) { + self.consider_hold(asking, platform_properties, now, full_worker_logging); + } + outcome + } + + /// The match itself; `inner_find_worker_for_action` adds the hold. + fn match_worker_for_action( &mut self, platform_properties: &PlatformProperties, full_worker_logging: bool, @@ -1248,6 +1517,28 @@ impl ApiWorkerSchedulerImpl { operation_id: &OperationId, update: UpdateOperationType, ) -> Result<(), Error> { + // Any word from the worker about an action it was sent, other than + // sending it back, is the worker taking it: a hold for it ends as + // placed. A decline or a disconnect requeues the action, and its + // hold waits for it in the queue again. + let held_for = self + .workers + .peek(worker_id) + .and_then(|worker| worker.running_action_infos.get(operation_id)) + .map(|pending| pending.action_info.inner.unique_qualifier.clone()) + .filter(|action| self.holds_by_action.contains_key(action)); + if let Some(action) = held_for { + if matches!( + update, + UpdateOperationType::UpdateWithDecline(_) + | UpdateOperationType::UpdateWithDisconnect + ) { + self.note_hold_dispatched(&action, false); + } else { + self.end_hold_for_action(&action, "placed"); + } + } + let worker = self.workers.get_mut(worker_id).err_tip(|| { format!("Worker {worker_id} does not exist in SimpleScheduler::update_action") })?; @@ -1447,9 +1738,10 @@ impl ApiWorkerSchedulerImpl { .await; return Result::<(), _>::Err(err).merge(evicted).merge(requeued); } - // The action ran, here or elsewhere; a worker held for it is - // free for everything again. - self.end_hold_for_action(&action_info.inner.unique_qualifier, "placed"); + // A worker held for the action keeps its hold until the worker + // the action went to takes it; a decline sends the action back + // to the queue, and the hold waits for it there again. + self.note_hold_dispatched(&action_info.inner.unique_qualifier, true); Ok(()) } else { warn!( @@ -1479,6 +1771,13 @@ impl ApiWorkerSchedulerImpl { return; }; if worker.mark_accepted(operation_id) { + let action = worker + .running_action_infos + .get(operation_id) + .map(|pending| pending.action_info.inner.unique_qualifier.clone()); + if let Some(action) = action { + self.end_hold_for_action(&action, "placed"); + } return; } // Messages from a worker arrive in order and the acknowledgement @@ -1519,6 +1818,11 @@ impl ApiWorkerSchedulerImpl { return Ok(false); } info!(?worker_id, %operation_id, %reason, ?needs_kb, "Worker declined dispatch"); + let declined_action = self + .workers + .peek(worker_id) + .and_then(|worker| worker.running_action_infos.get(operation_id)) + .map(|pending| pending.action_info.inner.unique_qualifier.clone()); self.update_action( worker_id, operation_id, @@ -1554,6 +1858,25 @@ impl ApiWorkerSchedulerImpl { worker.pause_needs_kb = needs_kb; } } + // A hold on this worker for the declined action outlives the + // decline: the worker is paused until it reports room, and is + // offered the action again then. Unless it declined the action + // idle: it is then not offered that much again until a keepalive + // says it has it free, which waits on the worker rather than on + // its actions, so the hold ends and the next pass holds another. + if let (Some(action), Some(needs_kb)) = (declined_action, needs_kb) + && self + .holds + .get(worker_id) + .is_some_and(|hold| hold.action == action) + && self.workers.peek(worker_id).is_some_and(|worker| { + worker + .idle_declined_kb + .is_some_and(|declined_kb| needs_kb >= declined_kb) + }) + { + self.end_hold(worker_id, "declined"); + } Ok(true) } @@ -1723,6 +2046,8 @@ pub struct ApiWorkerScheduler { /// See `ApiWorkerSchedulerImpl::capacity_generation`; read here without /// the lock. capacity_generation: Arc, + /// See `ApiWorkerSchedulerImpl::scheduler_id`; read here without the lock. + scheduler_id: String, #[metric(group = "platform_property_manager")] platform_property_manager: Arc, @@ -1783,11 +2108,16 @@ impl ApiWorkerScheduler { maybe_origin_event_tx: Option>, has_peers: bool, record_ttl: Duration, + scheduler_id: String, ) -> Arc { debug_assert!( !record_ttl.is_zero(), "record_ttl must be non-zero; a zero TTL degenerates the census warm-up/staleness checks" ); + debug_assert!( + reservation_hold.is_none_or(|hold| hold.hold_after_s > 0 && hold.max_hold_s > 0), + "reservation_hold.hold_after_s and max_hold_s must be above zero; the factory refuses them" + ); let local_fleet_change_notify = Arc::new(Notify::new()); let capacity_generation = Arc::new(AtomicU64::new(0)); Arc::new(Self { @@ -1811,6 +2141,8 @@ impl ApiWorkerScheduler { fleet_shape_set: BTreeSet::new(), // Without peers there is nothing to wait for. peer_fleet: (!has_peers).then(Vec::new), + peer_schedulers: Vec::new(), + scheduler_id: scheduler_id.clone(), shared: has_peers, record_ttl, peer_fleet_known_since: None, @@ -1827,9 +2159,16 @@ impl ApiWorkerScheduler { maybe_origin_event_tx, local_fleet_change_notify, capacity_generation, + scheduler_id, }) } + /// The id this scheduler publishes to its peers under. + #[must_use] + pub fn scheduler_id(&self) -> &str { + &self.scheduler_id + } + /// Whether no worker is connected here and, as far as a trusted census /// says, to any peer either. False while the census cannot be trusted, /// so a peer we cannot see is assumed to have workers. @@ -1957,12 +2296,13 @@ impl ApiWorkerScheduler { /// Records what the workers connected to peer schedulers can run, or /// `None` when that is not known. - pub async fn set_peer_fleet( - &self, - peer_fleet: Option>, - now: SystemTime, - ) { - self.inner.lock().await.set_peer_fleet(peer_fleet, now); + pub async fn set_peer_fleet(&self, peers: Option>, now: SystemTime) { + self.inner.lock().await.set_peer_fleet(peers, now); + } + + /// The workers held for a queued action right now. + pub async fn held_workers_for_test(&self) -> Vec { + self.inner.lock().await.holds.keys().cloned().collect() } /// Woken when a worker joins or leaves this scheduler. @@ -1980,28 +2320,13 @@ impl ApiWorkerScheduler { platform_properties: &PlatformProperties, full_worker_logging: bool, now: SystemTime, - asking: Option<&ActionUniqueQualifier>, + asking: Option>, ) -> MatchOutcome { self.find_worker_for_action_observed(platform_properties, full_worker_logging, now, asking) .await .0 } - /// Holds a worker for `action`, a queued action that has waited long - /// enough finding no room; see `ReservationHoldSpec`. Returns whether - /// the action holds a worker now. - pub async fn hold_worker_for( - &self, - action: &ActionUniqueQualifier, - platform_properties: &PlatformProperties, - now: SystemTime, - ) -> bool { - self.inner - .lock() - .await - .hold_worker_for(action, platform_properties, now) - } - /// A matching pass is about to read the queue; returns the pass, for /// `end_pass_holds`. pub async fn begin_pass_holds(&self) -> u64 { @@ -2024,7 +2349,7 @@ impl ApiWorkerScheduler { platform_properties: &PlatformProperties, full_worker_logging: bool, now: SystemTime, - asking: Option<&ActionUniqueQualifier>, + asking: Option>, ) -> (MatchOutcome, u64) { let start = Instant::now(); self.metrics diff --git a/nativelink-scheduler/src/awaited_action_db/mod.rs b/nativelink-scheduler/src/awaited_action_db/mod.rs index 657a5d291d6..8ac6a59f1ae 100644 --- a/nativelink-scheduler/src/awaited_action_db/mod.rs +++ b/nativelink-scheduler/src/awaited_action_db/mod.rs @@ -198,17 +198,17 @@ pub trait AwaitedActionDb: Send + Sync + MetricsComponent + Unpin + 'static { fn set_worker_registry(&mut self, _worker_registry: SharedWorkerRegistry) {} /// Tells other schedulers on the same state what the workers connected - /// here can run, and returns what the workers connected to those - /// schedulers can run. `local` holds the properties each distinct kind - /// of worker registered with, and stays visible to peers for `ttl`. - /// Only shared-state implementations need it; the default shares + /// here can run, and returns, per peer scheduler, what the workers + /// connected to it can run. `local` holds the properties each distinct + /// kind of worker registered with, and stays visible to peers for + /// `ttl`. Only shared-state implementations need it; the default shares /// nothing and sees nothing. fn exchange_fleet_capabilities( &self, _scheduler_id: &str, _local: Vec, _ttl: Duration, - ) -> impl Future, Error>> + Send { + ) -> impl Future, Error>> + Send { async { Ok(Vec::new()) } } @@ -218,3 +218,13 @@ pub trait AwaitedActionDb: Send + Sync + MetricsComponent + Unpin + 'static { false } } + +/// What one peer scheduler's workers can run, as it published them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PeerScheduler { + /// The id the peer publishes under, unique to its process. + pub scheduler_id: String, + /// The properties each distinct kind of worker connected to the peer + /// registered with. + pub workers: Vec, +} diff --git a/nativelink-scheduler/src/default_scheduler_factory.rs b/nativelink-scheduler/src/default_scheduler_factory.rs index d30924eef86..ef519f17a4d 100644 --- a/nativelink-scheduler/src/default_scheduler_factory.rs +++ b/nativelink-scheduler/src/default_scheduler_factory.rs @@ -159,6 +159,15 @@ async fn simple_scheduler_factory( policy.percent )); } + if let Some(hold) = &spec.reservation_hold + && (hold.hold_after_s == 0 || hold.max_hold_s == 0) + { + return Err(make_input_err!( + "reservation_hold.hold_after_s and max_hold_s must be above zero, got {} and {}: a zero wait holds for any action on its first miss, and a zero hold expires in the pass that took it and wakes the next", + hold.hold_after_s, + hold.max_hold_s + )); + } match spec .experimental_backend .as_ref() diff --git a/nativelink-scheduler/src/simple_scheduler.rs b/nativelink-scheduler/src/simple_scheduler.rs index 3e85f4ed419..ea25a8c66a1 100644 --- a/nativelink-scheduler/src/simple_scheduler.rs +++ b/nativelink-scheduler/src/simple_scheduler.rs @@ -54,8 +54,8 @@ use tokio::time::Duration; use tracing::{debug, error, info, info_span, warn}; use uuid::Uuid; -use crate::api_worker_scheduler::ApiWorkerScheduler; -use crate::awaited_action_db::{AwaitedActionDb, CLIENT_KEEPALIVE_DURATION}; +use crate::api_worker_scheduler::{ApiWorkerScheduler, QueuedAction}; +use crate::awaited_action_db::{AwaitedActionDb, CLIENT_KEEPALIVE_DURATION, PeerScheduler}; use crate::known_platform_property_provider::KnownPlatformPropertyProvider; use crate::match_outcome::{MatchOutcome, PropertyShape, UnsatisfiableReason}; use crate::platform_property_manager::PlatformPropertyManager; @@ -235,10 +235,6 @@ pub struct SimpleScheduler { /// burst drains in a few passes. See `unsatisfiable_fail_cap`. last_unsatisfiable_backlog: AtomicUsize, - /// How long a queued action waits, finding no room on every pass, - /// before a worker is held for it; `None` when holds are off. - hold_after: Option, - /// The clock the state manager uses. now_fn: Box SystemTime + Send + Sync>, } @@ -409,13 +405,33 @@ impl SimpleScheduler { self.do_try_match(false).await } - /// Stands in for the fleet exchange with peer schedulers. + /// Stands in for the fleet exchange with peer schedulers: one peer with + /// these workers. pub async fn set_peer_fleet_for_test(&self, peer_fleet: Vec) { + self.set_peer_schedulers_for_test(vec![PeerScheduler { + scheduler_id: "peer".to_string(), + workers: peer_fleet, + }]) + .await; + } + + /// Stands in for the fleet exchange with peer schedulers. + pub async fn set_peer_schedulers_for_test(&self, peers: Vec) { self.worker_scheduler - .set_peer_fleet(Some(peer_fleet), (self.now_fn)()) + .set_peer_fleet(Some(peers), (self.now_fn)()) .await; } + /// The id this scheduler publishes to its peers under. + pub fn scheduler_id_for_test(&self) -> &str { + self.worker_scheduler.scheduler_id() + } + + /// The workers held for a queued action right now. + pub async fn held_workers_for_test(&self) -> Vec { + self.worker_scheduler.held_workers_for_test().await + } + /// Stands in for a failed fleet exchange with peer schedulers. pub async fn set_peers_unknown_for_test(&self) { self.worker_scheduler @@ -805,7 +821,10 @@ impl SimpleScheduler { &loaded.action_info.platform_properties, full_worker_logging, now, - Some(&loaded.action_info.inner.unique_qualifier), + Some(QueuedAction { + unique_qualifier: &loaded.action_info.inner.unique_qualifier, + insert_timestamp: loaded.action_info.inner.insert_timestamp, + }), ) .await else { @@ -926,7 +945,10 @@ impl SimpleScheduler { &loaded.action_info.platform_properties, full_worker_logging, unsatisfiable_pass.now, - Some(&loaded.action_info.inner.unique_qualifier), + Some(QueuedAction { + unique_qualifier: &loaded.action_info.inner.unique_qualifier, + insert_timestamp: loaded.action_info.inner.insert_timestamp, + }), ) .await; if generation == capacity_seen || parked.is_empty() { @@ -965,24 +987,9 @@ impl SimpleScheduler { .map(|_| ()), ); } - // A worker could run it once one has room. One that has - // waited long enough for that gets a worker held for it, - // if one would fit it idle (`reservation_hold`). + // A worker could run it once one has room. The match + // considered holding a worker for it (`reservation_hold`). MatchOutcome::WaitingForCapacity => { - if let Some(hold_after) = self.hold_after - && unsatisfiable_pass - .now - .duration_since(loaded.action_info.inner.insert_timestamp) - .is_ok_and(|waited| waited >= hold_after) - { - workers - .hold_worker_for( - &loaded.action_info.inner.unique_qualifier, - &loaded.action_info.platform_properties, - unsatisfiable_pass.now, - ) - .await; - } parked.park(loaded); } // Nothing to wait for; a worker connecting starts a new pass, @@ -1245,6 +1252,9 @@ impl SimpleScheduler { Some(worker_registry.clone()), ); + // The id this scheduler publishes its fleet to its peers under, and + // takes its turn holding workers by. + let scheduler_id = Uuid::new_v4().hyphenated().to_string(); let worker_scheduler = ApiWorkerScheduler::new( state_manager.clone(), platform_property_manager.clone(), @@ -1260,6 +1270,7 @@ impl SimpleScheduler { maybe_origin_event_tx.clone(), has_peers, record_ttl, + scheduler_id.clone(), ); let worker_scheduler_clone = worker_scheduler.clone(); @@ -1359,7 +1370,6 @@ impl SimpleScheduler { // so that a peer which does fail actions sees these workers even // while this scheduler is still rolling out or has the timeout off. let task_fleet_exchange_spawn = { - let scheduler_id = Uuid::new_v4().hyphenated().to_string(); let state_manager = Arc::downgrade(&state_manager); let local_fleet_change_notify = worker_scheduler.local_fleet_change_notify().clone(); let worker_scheduler = Arc::downgrade(&worker_scheduler); @@ -1598,10 +1608,6 @@ impl SimpleScheduler { no_worker_tracker: no_worker_action_timeout .map(|timeout| Mutex::new(UnsatisfiableTracker::new(Some(timeout)))), last_unsatisfiable_backlog: AtomicUsize::new(0), - hold_after: spec - .reservation_hold - .as_ref() - .map(|hold| Duration::from_secs(hold.hold_after_s)), now_fn: scheduler_now_fn, } }); diff --git a/nativelink-scheduler/src/simple_scheduler_state_manager.rs b/nativelink-scheduler/src/simple_scheduler_state_manager.rs index ff6a0de1722..4defb3bced6 100644 --- a/nativelink-scheduler/src/simple_scheduler_state_manager.rs +++ b/nativelink-scheduler/src/simple_scheduler_state_manager.rs @@ -44,7 +44,8 @@ use opentelemetry::KeyValue; use tracing::{debug, info, trace, warn}; use super::awaited_action_db::{ - AwaitedAction, AwaitedActionDb, AwaitedActionSubscriber, SortedAwaitedActionState, + AwaitedAction, AwaitedActionDb, AwaitedActionSubscriber, PeerScheduler, + SortedAwaitedActionState, }; use crate::worker_registry::{ORPHANED_ACTION_TIMEOUT, SharedWorkerRegistry, WorkerLiveness}; @@ -790,7 +791,7 @@ where scheduler_id: &str, local: Vec, ttl: Duration, - ) -> Result, Error> { + ) -> Result, Error> { self.action_db .exchange_fleet_capabilities(scheduler_id, local, ttl) .await diff --git a/nativelink-scheduler/src/store_awaited_action_db.rs b/nativelink-scheduler/src/store_awaited_action_db.rs index 41897e0ad45..01d486798ee 100644 --- a/nativelink-scheduler/src/store_awaited_action_db.rs +++ b/nativelink-scheduler/src/store_awaited_action_db.rs @@ -44,7 +44,7 @@ use tracing::{error, warn}; use crate::awaited_action_db::{ AwaitedAction, AwaitedActionDb, AwaitedActionSubscriber, CLIENT_KEEPALIVE_DURATION, - SortedAwaitedAction, SortedAwaitedActionState, + PeerScheduler, SortedAwaitedAction, SortedAwaitedActionState, }; use crate::worker_registry::{ORPHANED_ACTION_TIMEOUT, SharedWorkerRegistry, WorkerLiveness}; @@ -1254,7 +1254,7 @@ where scheduler_id: &str, local: Vec, ttl: Duration, - ) -> Result, Error> { + ) -> Result, Error> { self.store .update_data( UpdateFleetCapabilities(FleetCapabilities { @@ -1276,7 +1276,10 @@ where stream .try_filter(|record| futures::future::ready(record.scheduler_id != scheduler_id)) .try_fold(Vec::new(), |mut peers, record| async move { - peers.extend(record.workers); + peers.push(PeerScheduler { + scheduler_id: record.scheduler_id, + workers: record.workers, + }); Ok(peers) }) .await diff --git a/nativelink-scheduler/tests/fleet_capabilities_test.rs b/nativelink-scheduler/tests/fleet_capabilities_test.rs index ff2c8c23496..40d099977a9 100644 --- a/nativelink-scheduler/tests/fleet_capabilities_test.rs +++ b/nativelink-scheduler/tests/fleet_capabilities_test.rs @@ -23,7 +23,7 @@ use nativelink_config::stores::RedisSpec; use nativelink_error::{Code, Error}; use nativelink_macro::nativelink_test; use nativelink_redis_tester::FakeRedisBackend; -use nativelink_scheduler::awaited_action_db::AwaitedActionDb; +use nativelink_scheduler::awaited_action_db::{AwaitedActionDb, PeerScheduler}; use nativelink_scheduler::simple_scheduler::SimpleScheduler; use nativelink_scheduler::store_awaited_action_db::StoreAwaitedActionDb; use nativelink_scheduler::worker::Worker; @@ -88,18 +88,29 @@ async fn schedulers_see_each_others_fleet_but_not_their_own() -> Result<(), Erro let peers = db_a .exchange_fleet_capabilities("a", vec![shape(0)], TTL) .await?; - assert_eq!(peers, Vec::::new()); + assert_eq!(peers, Vec::::new()); let peers = db_b .exchange_fleet_capabilities("b", vec![shape(1), shape(2)], TTL) .await?; - assert_eq!(peers, vec![shape(0)]); + assert_eq!( + peers, + vec![PeerScheduler { + scheduler_id: "a".to_string(), + workers: vec![shape(0)], + }] + ); - let mut peers = db_a + let peers = db_a .exchange_fleet_capabilities("a", vec![shape(0)], TTL) .await?; - peers.sort_by_key(|p| p.properties["gpu_count"].as_str().into_owned()); - assert_eq!(peers, vec![shape(1), shape(2)]); + assert_eq!( + peers, + vec![PeerScheduler { + scheduler_id: "b".to_string(), + workers: vec![shape(1), shape(2)], + }] + ); // Each record expires on its own once its scheduler stops refreshing it. let expiries = fake_redis_backend.expiries.lock().unwrap(); @@ -120,7 +131,14 @@ async fn republishing_replaces_the_previous_record() -> Result<(), Error> { db_b.exchange_fleet_capabilities("b", vec![], TTL).await?; let peers = db_a.exchange_fleet_capabilities("a", vec![], TTL).await?; - assert_eq!(peers, Vec::::new()); + assert_eq!( + peers, + vec![PeerScheduler { + scheduler_id: "b".to_string(), + workers: vec![], + }], + "the peer is still there, with no workers" + ); Ok(()) } diff --git a/nativelink-scheduler/tests/reservation_hold_test.rs b/nativelink-scheduler/tests/reservation_hold_test.rs index c6a7224e8e0..4594574efc6 100644 --- a/nativelink-scheduler/tests/reservation_hold_test.rs +++ b/nativelink-scheduler/tests/reservation_hold_test.rs @@ -22,13 +22,17 @@ use nativelink_config::schedulers::{PropertyType, ReservationHoldSpec, SimpleSpe use nativelink_error::{Error, ResultExt}; use nativelink_macro::nativelink_test; use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::{ - UpdateForWorker, update_for_worker, + UpdateForWorker, WorkerLoad, update_for_worker, }; +use nativelink_scheduler::api_worker_scheduler::hold_owner; +use nativelink_scheduler::awaited_action_db::PeerScheduler; use nativelink_scheduler::default_scheduler_factory::memory_awaited_action_db_factory; use nativelink_scheduler::simple_scheduler::SimpleScheduler; use nativelink_scheduler::worker::Worker; use nativelink_scheduler::worker_scheduler::WorkerScheduler; -use nativelink_util::action_messages::{ActionResult, ActionStage, OperationId, WorkerId}; +use nativelink_util::action_messages::{ + ActionResult, ActionStage, ActionUniqueQualifier, OperationId, WorkerId, +}; use nativelink_util::common::DigestInfo; use nativelink_util::instant_wrapper::MockInstantWrapped; use nativelink_util::operation_state_manager::{ @@ -49,14 +53,36 @@ const MAX_HOLD_S: u64 = 300; /// Each worker advertises this much memory and runs up to four actions. const WORKER_KB: u64 = 100_000; const SMALL_KB: u64 = 10_000; -/// Fits a worker only when nothing else runs on it. +/// Fits a worker only when nothing else runs on it: beside one small +/// action there is room for it once that one finishes, beside two there +/// is not until both have. const LARGE_KB: u64 = 95_000; /// The client listeners of the queued actions, kept so that dropping one /// does not drop its action. type Listeners = Vec>; -fn make_scheduler(max_held_workers: usize) -> Arc { +#[derive(Clone, Copy)] +struct Options { + max_held_workers: usize, + live_memory_veto: bool, + client_action_timeout_s: u64, +} + +impl Default for Options { + fn default() -> Self { + Self { + max_held_workers: 0, + live_memory_veto: false, + // The clock jumps past a hold here, and no client polls + // meanwhile; neither the client timeout nor the backend's + // retention window may retire a queued action over the jump. + client_action_timeout_s: 100_000, + } + } +} + +fn make_scheduler(options: Options) -> Arc { MockClock::set_time(Duration::from_secs(NOW_TIME)); let spec = SimpleSpec { supported_platform_properties: Some(HashMap::from([( @@ -66,12 +92,10 @@ fn make_scheduler(max_held_workers: usize) -> Arc { reservation_hold: Some(ReservationHoldSpec { hold_after_s: HOLD_AFTER_S, max_hold_s: MAX_HOLD_S, - max_held_workers, + max_held_workers: options.max_held_workers, }), - // The clock jumps past a hold here, and no client polls meanwhile; - // neither the client timeout nor the backend's retention window - // (below) may retire a queued action over the jump. - client_action_timeout_s: 100_000, + live_memory_veto: options.live_memory_veto.then(|| "memory_kb".to_string()), + client_action_timeout_s: options.client_action_timeout_s, ..SimpleSpec::default() }; let task_change_notify = Arc::new(Notify::new()); @@ -86,13 +110,18 @@ fn make_scheduler(max_held_workers: usize) -> Arc { scheduler } -async fn add_worker( +fn worker_id(name: &str) -> WorkerId { + WorkerId(name.to_string()) +} + +async fn add_worker_with( scheduler: &SimpleScheduler, name: &str, + admits_when_idle: bool, ) -> Result, Error> { let (tx, mut rx) = mpsc::channel(64); - let worker = Worker::new( - WorkerId(name.to_string()), + let mut worker = Worker::new( + worker_id(name), PlatformProperties::new(HashMap::from([( "memory_kb".to_string(), PlatformPropertyValue::Minimum(WORKER_KB), @@ -101,6 +130,7 @@ async fn add_worker( NOW_TIME, /* max_inflight_tasks */ 4, ); + worker.admits_when_idle = admits_when_idle; scheduler .add_worker(worker) .await @@ -114,24 +144,32 @@ async fn add_worker( Ok(rx) } +async fn add_worker( + scheduler: &SimpleScheduler, + name: &str, +) -> Result, Error> { + add_worker_with(scheduler, name, false).await +} + async fn add_action( scheduler: &SimpleScheduler, listeners: &mut Listeners, digest_byte: u8, kb: u64, -) -> Result<(), Error> { +) -> Result { let mut action_info = make_base_action_info( UNIX_EPOCH + MockClock::time(), DigestInfo::new([digest_byte; 32], 512), ); Arc::make_mut(&mut action_info).platform_properties = HashMap::from([("memory_kb".to_string(), kb.to_string())]); + let unique_qualifier = action_info.unique_qualifier.clone(); let listener = scheduler .add_action(OperationId::default(), action_info) .await?; listeners.push(listener); tokio::task::yield_now().await; - Ok(()) + Ok(unique_qualifier) } /// The dispatches a worker received since last asked: operation id and the @@ -157,6 +195,10 @@ fn dispatches(rx: &mut mpsc::Receiver) -> Vec<(String, u64)> { seen } +fn memory_of(dispatched: &[(String, u64)]) -> Vec { + dispatched.iter().map(|(_, kb)| *kb).collect() +} + async fn complete( scheduler: &SimpleScheduler, worker: &str, @@ -164,7 +206,7 @@ async fn complete( ) -> Result<(), Error> { scheduler .update_action( - &WorkerId(worker.to_string()), + &worker_id(worker), &OperationId::from(operation_id), UpdateOperationType::UpdateWithActionStage(ActionStage::Completed( ActionResult::default(), @@ -175,6 +217,34 @@ async fn complete( Ok(()) } +/// The worker acknowledges the dispatch. +async fn acknowledge( + scheduler: &SimpleScheduler, + worker: &str, + operation_id: &str, +) -> Result<(), Error> { + scheduler + .worker_dispatch_accepted(&worker_id(worker), &OperationId::from(operation_id)) + .await?; + tokio::task::yield_now().await; + Ok(()) +} + +/// A keepalive from the worker reporting this much memory free. +async fn report_free(scheduler: &SimpleScheduler, worker: &str, free_kb: u64) -> Result<(), Error> { + scheduler + .worker_keep_alive_received( + &worker_id(worker), + MockClock::time().as_secs(), + Some(WorkerLoad { + free_memory_kb: free_kb, + }), + ) + .await?; + tokio::task::yield_now().await; + Ok(()) +} + /// Lets the clock run and runs a pass, as the matcher would. async fn pass_after(scheduler: &SimpleScheduler, seconds: u64) -> Result<(), Error> { MockClock::advance(Duration::from_secs(seconds)); @@ -183,29 +253,46 @@ async fn pass_after(scheduler: &SimpleScheduler, seconds: u64) -> Result<(), Err Ok(()) } -/// Two workers each running one small action, and a large action that fits -/// neither beside it. Returns the small actions' operation ids on a and b. +async fn held(scheduler: &SimpleScheduler) -> Vec { + let mut held: Vec = scheduler + .held_workers_for_test() + .await + .into_iter() + .map(|id| id.0) + .collect(); + held.sort(); + held +} + +/// Two workers each running two small actions, and a large action that +/// fits neither until both of its small actions finish. Returns the small +/// actions' operation ids on a and on b, and the large action. async fn two_busy_workers_and_a_large_action( scheduler: &SimpleScheduler, listeners: &mut Listeners, a: &mut mpsc::Receiver, b: &mut mpsc::Receiver, -) -> Result<(String, String), Error> { - add_action(scheduler, listeners, 1, SMALL_KB).await?; - add_action(scheduler, listeners, 2, SMALL_KB).await?; +) -> Result<(Vec, Vec, ActionUniqueQualifier), Error> { + for digest in 1..5 { + add_action(scheduler, listeners, digest, SMALL_KB).await?; + } let (on_a, on_b) = (dispatches(a), dispatches(b)); assert_eq!( (on_a.len(), on_b.len()), - (1, 1), - "one small action per worker" + (2, 2), + "two small actions per worker" ); - add_action(scheduler, listeners, 3, LARGE_KB).await?; + let large = add_action(scheduler, listeners, 9, LARGE_KB).await?; assert_eq!( (dispatches(a).len(), dispatches(b).len()), (0, 0), - "the large action fits neither worker beside a small one" + "the large action fits neither worker beside two small ones" ); - Ok((on_a[0].0.clone(), on_b[0].0.clone())) + Ok(( + on_a.into_iter().map(|(id, _)| id).collect(), + on_b.into_iter().map(|(id, _)| id).collect(), + large, + )) } /// Once a worker is held, two small actions both land on the other one. @@ -216,35 +303,39 @@ async fn held_and_free( a: &mut mpsc::Receiver, b: &mut mpsc::Receiver, ) -> Result<(&'static str, &'static str), Error> { - add_action(scheduler, listeners, 4, SMALL_KB).await?; - add_action(scheduler, listeners, 5, SMALL_KB).await?; + add_action(scheduler, listeners, 10, SMALL_KB).await?; + add_action(scheduler, listeners, 11, SMALL_KB).await?; let (on_a, on_b) = (dispatches(a).len(), dispatches(b).len()); - match (on_a, on_b) { - (0, 2) => Ok(("a", "b")), - (2, 0) => Ok(("b", "a")), + let (held_worker, free) = match (on_a, on_b) { + (0, 2) => ("a", "b"), + (2, 0) => ("b", "a"), other => panic!("one worker should be held and take nothing, got {other:?}"), - } + }; + assert_eq!(held(scheduler).await, vec![held_worker.to_string()]); + Ok((held_worker, free)) } /// A large action waits behind small ones on every worker. Once it has /// waited `hold_after_s`, one worker is held: new small actions all go to -/// the other, and when the held worker's own action finishes the large one -/// runs there. The hold then ends and the worker takes small actions again. +/// the other, and when the held worker's own actions finish the large one +/// runs there. The hold ends once the worker takes it, and the worker +/// takes small actions again. #[nativelink_test] async fn a_starving_reservation_gets_a_worker_held_then_runs_there() -> Result<(), Error> { - let scheduler = make_scheduler(0); + let scheduler = make_scheduler(Options::default()); let mut listeners = Listeners::new(); let mut a = add_worker(&scheduler, "a").await?; let mut b = add_worker(&scheduler, "b").await?; - let (small_on_a, small_on_b) = + let (small_on_a, small_on_b, _) = two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; // Not long enough yet: nothing is held, a small action goes anywhere. pass_after(&scheduler, HOLD_AFTER_S - 1).await?; - add_action(&scheduler, &mut listeners, 9, SMALL_KB).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + add_action(&scheduler, &mut listeners, 12, SMALL_KB).await?; let (on_a, on_b) = (dispatches(&mut a), dispatches(&mut b)); assert_eq!(on_a.len() + on_b.len(), 1, "nothing is held yet"); - // Put the fleet back to one small action per worker. + // Put the fleet back to two small actions per worker. let (third_worker, third) = if on_a.is_empty() { ("b", on_b[0].0.clone()) } else { @@ -259,66 +350,106 @@ async fn a_starving_reservation_gets_a_worker_held_then_runs_there() -> Result<( ); pass_after(&scheduler, 1).await?; - let (held, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; - let (held_rx, free_rx) = if held == "a" { + let (held_worker, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let (held_rx, free_rx) = if held_worker == "a" { (&mut a, &mut b) } else { (&mut b, &mut a) }; - let small_on_held = if held == "a" { small_on_a } else { small_on_b }; + let small_on_held = if held_worker == "a" { + small_on_a + } else { + small_on_b + }; - // The held worker's own action finishes: the large action runs there. - complete(&scheduler, held, &small_on_held).await?; + // One of the held worker's actions finishes: not enough room yet, and + // nothing else is offered the worker either. + complete(&scheduler, held_worker, &small_on_held[0]).await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!(dispatches(held_rx).len(), 0); + assert_eq!(held(&scheduler).await, vec![held_worker.to_string()]); + + // The other finishes: the large action runs there. The hold lasts + // until the worker says it took it. + complete(&scheduler, held_worker, &small_on_held[1]).await?; scheduler.do_try_match_for_test().await?; tokio::task::yield_now().await; let on_held = dispatches(held_rx); assert_eq!( - on_held.iter().map(|(_, kb)| *kb).collect::>(), + memory_of(&on_held), vec![LARGE_KB], "the held worker runs the action it was held for" ); assert_eq!(dispatches(free_rx).len(), 0); + assert_eq!( + held(&scheduler).await, + vec![held_worker.to_string()], + "the hold outlives the dispatch" + ); + acknowledge(&scheduler, held_worker, &on_held[0].0).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); - // The hold is over. Once the large action is done, the worker is idle - // and the free one has a slot left, so two small actions both run only - // if the formerly held worker takes one. - complete(&scheduler, held, &on_held[0].0).await?; - add_action(&scheduler, &mut listeners, 6, SMALL_KB).await?; - add_action(&scheduler, &mut listeners, 7, SMALL_KB).await?; + // Once the large action is done the worker is idle and the free one + // is full, so two small actions both run only if it takes them. + complete(&scheduler, held_worker, &on_held[0].0).await?; + add_action(&scheduler, &mut listeners, 13, SMALL_KB).await?; + add_action(&scheduler, &mut listeners, 14, SMALL_KB).await?; assert_eq!( - dispatches(held_rx).len() + dispatches(free_rx).len(), - 2, + (dispatches(held_rx).len(), dispatches(free_rx).len()), + (2, 0), "the worker is offered everything again once its action ran" ); Ok(()) } +/// Small actions queued behind a full fleet are only waiting their turn: +/// one action finishing anywhere makes room for one of them. However long +/// they wait, no worker is held. +#[nativelink_test] +async fn an_action_one_completion_makes_room_for_gets_no_hold() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + for digest in 1..9 { + add_action(&scheduler, &mut listeners, digest, SMALL_KB).await?; + } + assert_eq!((dispatches(&mut a).len(), dispatches(&mut b).len()), (4, 4)); + for digest in 9..12 { + add_action(&scheduler, &mut listeners, digest, SMALL_KB).await?; + } + pass_after(&scheduler, HOLD_AFTER_S).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + // Nor for a reservation that fits beside three small actions but not + // beside four: one finishing makes room for it. + add_action(&scheduler, &mut listeners, 12, WORKER_KB - 3 * SMALL_KB).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + Ok(()) +} + /// A hold that never fits is given up after `max_hold_s`, and the action is /// not held for again straight away: the worker takes other work. #[nativelink_test] async fn a_hold_expires_and_the_worker_takes_other_work() -> Result<(), Error> { - let scheduler = make_scheduler(0); + let scheduler = make_scheduler(Options::default()); let mut listeners = Listeners::new(); let mut a = add_worker(&scheduler, "a").await?; let mut b = add_worker(&scheduler, "b").await?; two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; pass_after(&scheduler, HOLD_AFTER_S).await?; - let (held, free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; - let (held_rx, free_rx) = if held == "a" { + let (held_worker, free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let (held_rx, free_rx) = if held_worker == "a" { (&mut a, &mut b) } else { (&mut b, &mut a) }; - // Fill the free worker (three running) to its four slots. - add_action(&scheduler, &mut listeners, 6, SMALL_KB).await?; - assert_eq!( - (dispatches(held_rx).len(), dispatches(free_rx).len()), - (0, 1) - ); - // Now a small action has nowhere to go: the free worker is full and + // The free worker is full now; a small action has nowhere to go, since // the held worker takes nothing else. - add_action(&scheduler, &mut listeners, 7, SMALL_KB).await?; + add_action(&scheduler, &mut listeners, 12, SMALL_KB).await?; assert_eq!( (dispatches(held_rx).len(), dispatches(free_rx).len()), (0, 0) @@ -328,43 +459,302 @@ async fn a_hold_expires_and_the_worker_takes_other_work() -> Result<(), Error> { // next pass places the small action on the worker, which is not held // for the large action again while its cooldown lasts. pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); scheduler.do_try_match_for_test().await?; tokio::task::yield_now().await; assert_eq!( - dispatches(held_rx) - .iter() - .map(|(_, kb)| *kb) - .collect::>(), + memory_of(&dispatches(held_rx)), vec![SMALL_KB], "the expired hold's worker takes the small action, not the large one" ); assert_eq!(dispatches(free_rx).len(), 0, "{free} was full"); - // Cooldown over, and the large action still waits: a worker is held - // for it again, and a small action finds no room on it. + // Cooldown over (as long as the hold, since the worker was busy when + // it expired), and the large action still waits: a worker is held for + // it again. + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!(held(&scheduler).await.len(), 1); + Ok(()) +} + +/// With the live veto on, a held worker that drains but never reports room +/// for the action keeps the hold to its expiry while idle. Each such expiry +/// in a row doubles how long the action is left unheld. +#[nativelink_test] +async fn an_expiry_on_an_idle_worker_doubles_the_cooldown() -> Result<(), Error> { + let scheduler = make_scheduler(Options { + live_memory_veto: true, + ..Options::default() + }); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + let (small_on_a, small_on_b, _) = + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + let (held_worker, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let held_rx = if held_worker == "a" { &mut a } else { &mut b }; + let small_on_held = if held_worker == "a" { + small_on_a + } else { + small_on_b + }; + + // The held worker reports less free than the action asks and then + // drains: the ledger would place it, the veto does not. + report_free(&scheduler, held_worker, LARGE_KB / 2).await?; + for operation_id in &small_on_held { + complete(&scheduler, held_worker, operation_id).await?; + } + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!(dispatches(held_rx).len(), 0, "vetoed, not placed"); + assert_eq!(held(&scheduler).await, vec![held_worker.to_string()]); + + // Expiry on an idle worker: the cooldown is twice the hold. + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); pass_after(&scheduler, MAX_HOLD_S).await?; - add_action(&scheduler, &mut listeners, 8, SMALL_KB).await?; assert_eq!( - dispatches(held_rx).len() + dispatches(free_rx).len(), - 0, - "one worker full, the other held again" + held(&scheduler).await, + Vec::::new(), + "still cooling down after one hold's worth" ); + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!(held(&scheduler).await.len(), 1, "held again after two"); Ok(()) } -/// The large action finds room elsewhere first: an idle worker joins and -/// takes it at once, the hold ends as placed, and the held worker takes -/// small actions again. +/// The held worker declines the action for load once it is sent: the hold +/// stays, and when the worker reports room the action is offered it again. +#[nativelink_test] +async fn a_declined_dispatch_keeps_the_hold() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + // Workers that admit when idle: a decline for load from one is about + // what else it runs, not a verdict on the action. + let mut a = add_worker_with(&scheduler, "a", true).await?; + let mut b = add_worker_with(&scheduler, "b", true).await?; + let (small_on_a, small_on_b, _) = + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + let (held_worker, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let held_rx = if held_worker == "a" { &mut a } else { &mut b }; + let small_on_held = if held_worker == "a" { + small_on_a + } else { + small_on_b + }; + for operation_id in &small_on_held { + complete(&scheduler, held_worker, operation_id).await?; + } + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + let sent = dispatches(held_rx); + assert_eq!(memory_of(&sent), vec![LARGE_KB]); + + scheduler + .worker_dispatch_declined( + &worker_id(held_worker), + &OperationId::from(sent[0].0.as_str()), + "load".to_string(), + Some(LARGE_KB), + ) + .await?; + tokio::task::yield_now().await; + assert_eq!( + held(&scheduler).await, + vec![held_worker.to_string()], + "the hold outlives the decline" + ); + // Paused until it reports room; meanwhile the matcher offers it + // nothing, and a pass does not release the hold either. + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!(dispatches(held_rx).len(), 0); + assert_eq!(held(&scheduler).await, vec![held_worker.to_string()]); + + report_free(&scheduler, held_worker, WORKER_KB).await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + let sent = dispatches(held_rx); + assert_eq!(memory_of(&sent), vec![LARGE_KB], "offered again"); + acknowledge(&scheduler, held_worker, &sent[0].0).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + Ok(()) +} + +/// A worker that declined that much while idle is not offered the action +/// again until it reports room, so it is never the one held; and a held +/// worker that declines the action idle loses the hold to another. +#[nativelink_test] +async fn a_worker_that_declined_idle_is_passed_over() -> Result<(), Error> { + // A decline for load, and the verdict it leaves on an idle worker, + // exist with the live veto on. + let scheduler = make_scheduler(Options { + live_memory_veto: true, + ..Options::default() + }); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + for digest in 1..5 { + add_action(&scheduler, &mut listeners, digest, SMALL_KB).await?; + } + let (on_a, on_b) = (dispatches(&mut a), dispatches(&mut b)); + assert_eq!((on_a.len(), on_b.len()), (2, 2)); + + // An idle worker joins; the large action goes to it at once, and it + // declines it. Paused by the decline, it is offered nothing until its + // next keepalive, and that much never again until it reports room. + let mut c = add_worker(&scheduler, "c").await?; + let large = add_action(&scheduler, &mut listeners, 9, LARGE_KB).await?; + let sent = dispatches(&mut c); + assert_eq!(memory_of(&sent), vec![LARGE_KB]); + scheduler + .worker_dispatch_declined( + &worker_id("c"), + &OperationId::from(sent[0].0.as_str()), + "load".to_string(), + Some(LARGE_KB), + ) + .await?; + tokio::task::yield_now().await; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(dispatches(&mut c).len(), 0, "not offered the action again"); + let held_now = held(&scheduler).await; + assert_eq!(held_now.len(), 1); + assert_ne!( + held_now[0], "c", + "the idle worker that declined is not held" + ); + drop(large); + + // The held worker drains and declines the action idle: the hold moves. + let held_worker = held_now[0].clone(); + let held_rx = if held_worker == "a" { &mut a } else { &mut b }; + let other = if held_worker == "a" { "b" } else { "a" }; + let running = if held_worker == "a" { on_a } else { on_b }; + for (operation_id, _) in &running { + complete(&scheduler, &held_worker, operation_id).await?; + } + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + let sent = dispatches(held_rx); + assert_eq!(memory_of(&sent), vec![LARGE_KB]); + scheduler + .worker_dispatch_declined( + &worker_id(&held_worker), + &OperationId::from(sent[0].0.as_str()), + "load".to_string(), + Some(LARGE_KB), + ) + .await?; + tokio::task::yield_now().await; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!( + held(&scheduler).await, + vec![other.to_string()], + "the hold moved to the other busy worker" + ); + Ok(()) +} + +/// Of two equally busy workers, the one with more memory free as last +/// reported is held. +#[nativelink_test] +async fn the_worker_with_more_memory_free_is_held() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + report_free(&scheduler, "a", 30_000).await?; + report_free(&scheduler, "b", 60_000).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(held(&scheduler).await, vec!["b".to_string()]); + Ok(()) +} + +/// A hold ends when its worker drains or leaves, and the next pass holds +/// another. +#[nativelink_test] +async fn a_hold_ends_when_its_worker_drains_or_leaves() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + let held_now = held(&scheduler).await; + assert_eq!(held_now.len(), 1); + let (held_worker, other) = if held_now[0] == "a" { + ("a", "b") + } else { + ("b", "a") + }; + + scheduler + .set_drain_worker(&worker_id(held_worker), true) + .await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!(held(&scheduler).await, vec![other.to_string()]); + + scheduler.remove_worker(&worker_id(other)).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!( + held(&scheduler).await, + Vec::::new(), + "a draining worker is not held" + ); + Ok(()) +} + +/// A hold ends when its action leaves the queue: here the client gives up +/// on it and the client timeout retires it. +#[nativelink_test] +async fn a_hold_ends_when_its_action_leaves_the_queue() -> Result<(), Error> { + let scheduler = make_scheduler(Options { + client_action_timeout_s: 100, + ..Options::default() + }); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + // The large action's client goes away. + drop(listeners.pop()); + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(held(&scheduler).await.len(), 1); + pass_after(&scheduler, 100).await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!( + held(&scheduler).await, + Vec::::new(), + "released once no pass meets the action" + ); + Ok(()) +} + +/// The large action is placed elsewhere first: an idle worker joins and +/// takes it at once; the hold ends once that worker takes it, and the held +/// worker takes small actions again. #[nativelink_test] async fn a_hold_ends_when_its_action_runs_elsewhere() -> Result<(), Error> { - let scheduler = make_scheduler(0); + let scheduler = make_scheduler(Options::default()); let mut listeners = Listeners::new(); let mut a = add_worker(&scheduler, "a").await?; let mut b = add_worker(&scheduler, "b").await?; two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; pass_after(&scheduler, HOLD_AFTER_S).await?; - let (held, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; - let (held_rx, free_rx) = if held == "a" { + let (held_worker, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let (held_rx, free_rx) = if held_worker == "a" { (&mut a, &mut b) } else { (&mut b, &mut a) @@ -373,44 +763,132 @@ async fn a_hold_ends_when_its_action_runs_elsewhere() -> Result<(), Error> { let mut c = add_worker(&scheduler, "c").await?; scheduler.do_try_match_for_test().await?; tokio::task::yield_now().await; + let on_c = dispatches(&mut c); assert_eq!( - dispatches(&mut c) - .iter() - .map(|(_, kb)| *kb) - .collect::>(), + memory_of(&on_c), vec![LARGE_KB], "an idle worker that fits the action takes it at once" ); assert_eq!(dispatches(held_rx).len(), 0); + assert_eq!( + held(&scheduler).await, + vec![held_worker.to_string()], + "held until the new worker takes it" + ); + acknowledge(&scheduler, "c", &on_c[0].0).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); - // The hold ended as placed. The new worker has no memory left beside - // the large action, the free worker one slot, and the formerly held - // worker three: five small actions run four deep only if it takes - // part again, and one deep if it were still held. - for digest in 20..25 { - add_action(&scheduler, &mut listeners, digest, SMALL_KB).await?; - } - let total = dispatches(held_rx).len() + dispatches(free_rx).len() + dispatches(&mut c).len(); + // The new worker has no memory left beside the large action, the free + // worker is full, and the formerly held worker has two slots: two + // small actions run only if it takes them. + add_action(&scheduler, &mut listeners, 20, SMALL_KB).await?; + add_action(&scheduler, &mut listeners, 21, SMALL_KB).await?; assert_eq!( - total, 4, + ( + dispatches(held_rx).len(), + dispatches(free_rx).len(), + dispatches(&mut c).len() + ), + (2, 0, 0), "the formerly held worker takes small actions again" ); Ok(()) } -/// With the cap at one hold, a second starving action waits unheld. +/// At most the cap is held: one worker with the cap at one, and one on a +/// two-worker fleet with the cap at a tenth of the fleet. #[nativelink_test] async fn at_most_the_cap_is_held() -> Result<(), Error> { - let scheduler = make_scheduler(1); + for max_held_workers in [1, 0] { + let scheduler = make_scheduler(Options { + max_held_workers, + ..Options::default() + }); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + add_action(&scheduler, &mut listeners, 10, LARGE_KB).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!( + held(&scheduler).await.len(), + 1, + "cap {max_held_workers}: one worker held, the other still takes work" + ); + add_action(&scheduler, &mut listeners, 11, SMALL_KB).await?; + add_action(&scheduler, &mut listeners, 12, SMALL_KB).await?; + assert_eq!(dispatches(&mut a).len() + dispatches(&mut b).len(), 2); + } + Ok(()) +} + +/// Scheduler replicas sharing a queue take turns holding, by a hash of the +/// action over the replicas whose workers would fit it: a peer whose +/// workers would not fit it is never the owner, and when the peer would, +/// this scheduler holds only for the actions that fall to it. +#[nativelink_test] +async fn replicas_take_turns_holding() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); let mut listeners = Listeners::new(); let mut a = add_worker(&scheduler, "a").await?; let mut b = add_worker(&scheduler, "b").await?; - two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; - add_action(&scheduler, &mut listeners, 10, LARGE_KB).await?; + let shape = |memory_kb: u64| { + PlatformProperties::new(HashMap::from([( + "memory_kb".to_string(), + PlatformPropertyValue::Minimum(memory_kb), + )])) + }; + // A peer with workers too small for the large action: this scheduler + // holds for it whatever the hash says. + scheduler + .set_peer_schedulers_for_test(vec![PeerScheduler { + scheduler_id: "peer".to_string(), + workers: vec![shape(LARGE_KB - 1)], + }]) + .await; + let (_, _, large) = + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; pass_after(&scheduler, HOLD_AFTER_S).await?; - // One worker held, the other still takes small actions. - add_action(&scheduler, &mut listeners, 4, SMALL_KB).await?; - add_action(&scheduler, &mut listeners, 5, SMALL_KB).await?; - assert_eq!(dispatches(&mut a).len() + dispatches(&mut b).len(), 2); + assert_eq!(held(&scheduler).await.len(), 1); + + // A peer whose workers would fit it: whichever of the two the hash + // names holds, and the other does not. + let local = scheduler.scheduler_id_for_test().to_string(); + let mut owners = vec![local.as_str(), "peer"]; + owners.sort_unstable(); + let local_owns = hold_owner(&large, &owners) == local; + scheduler + .set_peer_schedulers_for_test(vec![PeerScheduler { + scheduler_id: "peer".to_string(), + workers: vec![shape(WORKER_KB)], + }]) + .await; + // The existing hold stands; a fresh one goes by the owner. Expire it + // and let the cooldown pass, then see whether it comes back. + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!( + held(&scheduler).await.len(), + usize::from(local_owns), + "held here only when the hash names this scheduler (owner: {})", + hold_owner(&large, &owners) + ); + // Another action, which the hash sends to the other owner, shows the + // turn-taking rather than a fixed verdict. + let other = loop { + let digest = u8::try_from(30 + listeners.len()).unwrap_or(250); + let action = add_action(&scheduler, &mut listeners, digest, LARGE_KB).await?; + if (hold_owner(&action, &owners) == local) != local_owns { + break action; + } + }; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!( + held(&scheduler).await.len(), + usize::from(!local_owns), + "the other action falls to the other owner ({})", + hold_owner(&other, &owners) + ); Ok(()) } diff --git a/nativelink-scheduler/tests/worker_running_operations_report_test.rs b/nativelink-scheduler/tests/worker_running_operations_report_test.rs index d6eb1aeff8d..09be2229e0d 100644 --- a/nativelink-scheduler/tests/worker_running_operations_report_test.rs +++ b/nativelink-scheduler/tests/worker_running_operations_report_test.rs @@ -118,6 +118,7 @@ async fn dispatched() -> Result { None, false, Duration::from_secs(15), + "scheduler".to_string(), ); let worker_id = WorkerId::from(String::from("worker")); let (tx, mut rx) = mpsc::channel(8); diff --git a/nativelink-service/tests/worker_api_server_test.rs b/nativelink-service/tests/worker_api_server_test.rs index f7f07711a60..0e0056788c2 100644 --- a/nativelink-service/tests/worker_api_server_test.rs +++ b/nativelink-service/tests/worker_api_server_test.rs @@ -202,6 +202,7 @@ async fn setup_api_server_with( None, false, // has_peers Duration::from_secs(15), // record_ttl (unused while has_peers = false) + "scheduler".to_string(), ); let mut schedulers: HashMap> = HashMap::new(); diff --git a/nativelink-util/src/metrics.rs b/nativelink-util/src/metrics.rs index e6f4fe09dd7..c166d465fbc 100644 --- a/nativelink-util/src/metrics.rs +++ b/nativelink-util/src/metrics.rs @@ -1198,7 +1198,7 @@ pub static SCHEDULER_METRICS: LazyLock = LazyLock::new(|| worker_holds: meter .u64_counter("scheduler.worker.holds") .with_description( - "Workers held for a queued action that kept finding no room, by outcome: held when the hold began, placed when the action ran, released when the action or the worker left, expired when max_hold_s passed first", + "Workers held for a queued action that kept finding no room, by outcome: held when the hold began, placed when a worker took the action, released when the action or the worker left, declined when the held worker declined the action idle, expired when max_hold_s passed first", ) .with_unit("{hold}") .build(), @@ -1408,9 +1408,11 @@ pub fn record_parked_dispatched(count: u64) { } } -/// Records a worker hold beginning (`held`) or ending: `placed` when the -/// action ran, `released` when the action left the queue or the worker -/// left, `expired` when `max_hold_s` passed first. +/// Records a worker hold beginning (`held`) or ending: `placed` when a +/// worker took the action, `released` when the action left the queue or +/// the worker left or drained, `declined` when the held worker declined +/// the action idle and another is held instead, `expired` when `max_hold_s` +/// passed first. pub fn record_worker_hold(outcome: &'static str) { SCHEDULER_METRICS .worker_holds diff --git a/web/apps/docs/content/docs/explanations/scheduler-internals.mdx b/web/apps/docs/content/docs/explanations/scheduler-internals.mdx index 495262b753d..63d53af8546 100644 --- a/web/apps/docs/content/docs/explanations/scheduler-internals.mdx +++ b/web/apps/docs/content/docs/explanations/scheduler-internals.mdx @@ -190,17 +190,26 @@ Parking does not help an action whose reservation only fits a worker with nothing else on it. Every pass tries it first and finds each worker partly busy, then fills the room that opens with newer, smaller actions, so no worker ever drains for it while the fleet reads as free. **`reservation_hold`** -is for that action: once it has waited `hold_after_s` finding no room while -some connected worker would fit it idle, the pass holds one such worker -(fewest actions running, then most memory left) and offers it nothing else; -as its actions finish the held action is placed there the moment the ledger -and the live veto allow. The hold ends when the action runs, anywhere, when -it leaves the queue, when the worker leaves or drains, or after `max_hold_s`, -whichever is first; an expired hold is not repeated for that action until as -long again has passed, so an action that never fits does not keep workers -idle in turns. At most `max_held_workers` are held at once (a tenth of the -connected workers by default), and a hold is local to the scheduler the -worker is connected to. `scheduler.worker.holds` counts holds by outcome and +is for that action: once it has waited `hold_after_s` and fits no busy worker +until more than one of that worker's actions finish (an action one completion +would make room for is only waiting its turn, which the pass order gives it), +the pass holds a worker that would fit it idle (fewest actions running, then +most memory free as last reported, never one that declined that much while +idle) and offers it nothing else; as its actions finish the held action is +placed there the moment the ledger and the live veto allow. The hold stays +through the dispatch until the worker takes the action; a worker that declines +it sends it back to the queue with the hold kept, unless it declined idle, in +which case the hold moves to another worker. The hold ends when a worker takes +the action, anywhere, when it leaves the queue, when the worker leaves or +drains, or after `max_hold_s`, whichever is first; an expired hold is not +repeated for that action until as long again has passed, twice as long for +each expiry in a row on a worker that was idle, so an action that never fits +does not keep workers idle in turns. At most `max_held_workers` are held at +once per replica (a tenth of the workers connected to it by default). A hold is +local to the scheduler replica the worker is connected to; replicas sharing a +queue take turns per action, by a hash of the action over the replicas whose +workers would fit it as the fleet census last read them, so one worker is held +for it fleet-wide. `scheduler.worker.holds` counts holds by outcome and `scheduler.worker.held` is how many workers are held right now. An action no connected worker would fit even idle gets no hold; that is the unsatisfiable path, or a sizing class larger than the workers less their overhead. diff --git a/web/apps/docs/content/docs/reference/metrics.mdx b/web/apps/docs/content/docs/reference/metrics.mdx index f55acde22ad..7987cac4a60 100644 --- a/web/apps/docs/content/docs/reference/metrics.mdx +++ b/web/apps/docs/content/docs/reference/metrics.mdx @@ -5,7 +5,7 @@ full: true --- {/* AUTOGENERATED. Do not edit by hand. - Source: nativelink-util/src/metrics.rs @ v1.7.6 (e8eabb5e0) + Source: nativelink-util/src/metrics.rs @ v1.7.6 (965bb3e21) Regenerate from web/: bun --filter @nativelink/docs gen:metrics-reference */} NativeLink **v1.7.6** declares 49 OpenTelemetry instruments. This page is @@ -221,7 +221,7 @@ Histogram of stage durations in seconds | Prometheus series | `nativelink_execution_stage_duration` | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:544`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L544) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) | **Histogram buckets:** 15 explicit boundaries, from `0.001` to `3600.0` `s`. Anything past the last boundary lands in the @@ -240,7 +240,7 @@ Histogram of total execution durations in seconds | Prometheus series | `nativelink_execution_total_duration` | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:569`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L569) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) | **Histogram buckets:** 13 explicit boundaries, from `0.01` to `7200.0` `s`. Anything past the last boundary lands in the @@ -259,7 +259,7 @@ Histogram of queue wait times in seconds | Prometheus series | `nativelink_execution_queue_time` | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:594`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L594) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) | **Histogram buckets:** 12 explicit boundaries, from `0.001` to `600.0` `s`. Anything past the last boundary lands in the @@ -292,7 +292,7 @@ Total number of completed executions | Prometheus series | `nativelink_execution_completed_count_total` | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:620`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L620) | -| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:152`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L152), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:162`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L162) | +| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:163`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L163) | ### `execution.stage.transitions` @@ -306,7 +306,7 @@ Number of stage transitions | Prometheus series | `nativelink_execution_stage_transitions_total` | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:626`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L626) | -| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:141`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L141) | +| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:142`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L142) | ### `execution.cpu.time` @@ -320,7 +320,7 @@ CPU time an action used, as sampled by the worker. | Prometheus series | `nativelink_execution_cpu_time` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:632`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L632) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2155`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2155) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2480`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2480) | **Histogram buckets:** 12 explicit boundaries, from `0.1` to `7200.0` `s`. Anything past the last boundary lands in the @@ -339,7 +339,7 @@ Peak memory an action used, as sampled by the worker. | Prometheus series | `nativelink_execution_peak_memory` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:641`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L641) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2147`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2147) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2472`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2472) | **Histogram buckets:** 8 explicit boundaries, from `1_048_576.0` to `68_719_476_736.0` `By`. Anything past the last boundary lands in the @@ -358,7 +358,7 @@ Histogram of output sizes in bytes | Prometheus series | `nativelink_execution_output_size` | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:657`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L657) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) | **Histogram buckets:** 8 explicit boundaries, from `1_024.0` to `10_737_418_240.0` `By`. Anything past the last boundary lands in the @@ -377,7 +377,7 @@ Counter for execution retries | Prometheus series | `nativelink_execution_retry_count_total` | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:673`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L673) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L174) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:175`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L175) | ## Worker fleet metrics @@ -399,7 +399,7 @@ Workers currently connected. | Prometheus series | `nativelink_worker_connected_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:886`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L886) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:494`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L494), [`nativelink-scheduler/src/api_worker_scheduler.rs:1696`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1696) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:581`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L581), [`nativelink-scheduler/src/api_worker_scheduler.rs:2019`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2019) | ### `worker.connections` @@ -413,7 +413,7 @@ Workers that have joined, cumulative. | Prometheus series | `nativelink_worker_connections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:892`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L892) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:494`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L494) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:581`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L581) | ### `worker.disconnections` @@ -427,7 +427,7 @@ Workers that have left, cumulative, by reason. | Prometheus series | `nativelink_worker_disconnections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:898`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L898) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1696`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1696) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2019`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2019) | ### `worker.input.relinks` @@ -469,7 +469,7 @@ Dispatches requeued untried, by reason. | Prometheus series | `nativelink_scheduler_dispatch_requeues` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:914`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L914) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1389`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1389), [`nativelink-scheduler/src/api_worker_scheduler.rs:1586`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1586), [`nativelink-scheduler/src/api_worker_scheduler.rs:1912`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1912), [`nativelink-scheduler/src/api_worker_scheduler.rs:2243`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2243), [`nativelink-scheduler/src/api_worker_scheduler.rs:2274`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2274), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L679) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1680`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1680), [`nativelink-scheduler/src/api_worker_scheduler.rs:1909`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1909), [`nativelink-scheduler/src/api_worker_scheduler.rs:2251`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2251), [`nativelink-scheduler/src/api_worker_scheduler.rs:2568`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2568), [`nativelink-scheduler/src/api_worker_scheduler.rs:2599`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2599), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:680`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L680) | ### `worker.keepalives` @@ -483,7 +483,7 @@ Keepalives received, cumulative. | Prometheus series | `nativelink_worker_keepalives` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L919) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:452`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L452) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:539`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L539) | ### `worker.keepalive.gap` @@ -497,7 +497,7 @@ Seconds between consecutive messages from a worker. | Prometheus series | `nativelink_worker_keepalive_gap` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:925`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L925) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:386`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L386) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:473`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L473) | **Histogram buckets:** 12 explicit boundaries, from `1.0` to `60.0` `s`. Anything past the last boundary lands in the @@ -516,7 +516,7 @@ Connected workers currently paused or draining. | Prometheus series | `nativelink_worker_state_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:936`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L936) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:435`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L435), [`nativelink-scheduler/src/api_worker_scheduler.rs:488`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L488), [`nativelink-scheduler/src/api_worker_scheduler.rs:491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L491), [`nativelink-scheduler/src/api_worker_scheduler.rs:602`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L602), [`nativelink-scheduler/src/api_worker_scheduler.rs:1350`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1350), [`nativelink-scheduler/src/api_worker_scheduler.rs:1392`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1392), [`nativelink-scheduler/src/api_worker_scheduler.rs:1531`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1531) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:522`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L522), [`nativelink-scheduler/src/api_worker_scheduler.rs:575`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L575), [`nativelink-scheduler/src/api_worker_scheduler.rs:578`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L578), [`nativelink-scheduler/src/api_worker_scheduler.rs:696`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L696), [`nativelink-scheduler/src/api_worker_scheduler.rs:1641`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1641), [`nativelink-scheduler/src/api_worker_scheduler.rs:1683`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1683), [`nativelink-scheduler/src/api_worker_scheduler.rs:1835`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1835) | ## gRPC server metrics @@ -569,7 +569,7 @@ while the queue is non-empty means matching is the bottleneck. | Prometheus series | `nativelink_scheduler_matching_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1117`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1117) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:462`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L462) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:478`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L478) | **Histogram buckets:** 13 explicit boundaries, from `0.0001` to `10.0` `s`. Anything past the last boundary lands in the @@ -588,7 +588,7 @@ Matching passes, by result. | Prometheus series | `nativelink_scheduler_matching_passes` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1126`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1126) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:462`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L462) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:478`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L478) | ### `scheduler.unsatisfiable.queued` @@ -604,7 +604,7 @@ instance, judged against that instance's own workers. | Prometheus series | `nativelink_scheduler_unsatisfiable_queued` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1132`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1132) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1066`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1066) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1073`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1073) | ### `scheduler.unsatisfiable.failed` @@ -619,7 +619,7 @@ the properties that could not be satisfied. | Prometheus series | `nativelink_scheduler_unsatisfiable_failed` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1140`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1140) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:564`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L564) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:580`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L580) | ### `scheduler.queue.depth` @@ -633,7 +633,7 @@ Queued actions the last matching pass read. | Prometheus series | `nativelink_scheduler_queue_depth` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1146`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1146) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1067`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1067) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1074`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1074) | ### `scheduler.queue.age` @@ -647,7 +647,7 @@ Queued actions the last matching pass read, by age bucket. | Prometheus series | `nativelink_scheduler_queue_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1152`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1152) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1068`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1068) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1075`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1075) | ### `scheduler.queue.oldest_age` @@ -661,7 +661,7 @@ Age of the oldest action the last matching pass read, in seconds. | Prometheus series | `nativelink_scheduler_queue_oldest_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1160`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1160) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1068`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1068) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1075`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1075) | ### `scheduler.executing.age` @@ -675,7 +675,7 @@ Executing actions the last executing sweep read, by age bucket. | Prometheus series | `nativelink_scheduler_executing_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1168`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1168) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1338`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1338) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1349`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1349) | ### `scheduler.executing.oldest_age` @@ -690,7 +690,7 @@ executing, in seconds. | Prometheus series | `nativelink_scheduler_executing_oldest_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1176`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1176) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1338`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1338) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1349`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1349) | ### `scheduler.queue.retired` @@ -704,7 +704,7 @@ Queued actions retired for having no client. | Prometheus series | `nativelink_scheduler_queue_retired` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1184`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1184) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:783`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L783) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:784`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L784) | ### `scheduler.matching.parked_dispatched` @@ -719,7 +719,7 @@ opened before the pass ended, ahead of the newer actions it was reading. | Prometheus series | `nativelink_scheduler_matching_parked_dispatched` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1190) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1043`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1043) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1050`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1050) | ### `scheduler.worker.holds` @@ -734,7 +734,7 @@ began or ended the hold. | Prometheus series | `nativelink_scheduler_worker_holds` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1198`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1198) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:621`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L621), [`nativelink-scheduler/src/api_worker_scheduler.rs:707`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L707) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:719`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L719), [`nativelink-scheduler/src/api_worker_scheduler.rs:932`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L932) | ### `scheduler.worker.held` @@ -748,7 +748,7 @@ Workers held for such an action right now. | Prometheus series | `nativelink_scheduler_worker_held` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1206`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1206) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:622`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L622), [`nativelink-scheduler/src/api_worker_scheduler.rs:708`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L708) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:720`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L720), [`nativelink-scheduler/src/api_worker_scheduler.rs:933`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L933) | ### `scheduler.awaited_action.orphans` @@ -762,7 +762,7 @@ Listed operations whose record was gone on read, by site. | Prometheus series | `nativelink_scheduler_awaited_action_orphans` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1212`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1212) | -| Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:604`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L604), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:614`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L614), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:731`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L731), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1662`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1662), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1685`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1685) | +| Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:620`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L620), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:615`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L615), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:732`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L732), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1663`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1663), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1686`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1686) | ### `scheduler.sweep.failures` @@ -776,7 +776,7 @@ Abandoned-queue sweeps that failed before finishing. | Prometheus series | `nativelink_scheduler_sweep_failures` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | | Declared at | [`nativelink-util/src/metrics.rs:1220`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1220) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1290`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1290), [`nativelink-scheduler/src/simple_scheduler.rs:1349`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1349) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1301`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1301), [`nativelink-scheduler/src/simple_scheduler.rs:1360`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1360) | ### `scheduler.hints.resolutions` @@ -796,7 +796,7 @@ Actions sized by the historical resource scheduler, by source. Global object-store metrics. -Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1463`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1463), backed by the `StoreMetrics` struct. +Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1465`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1465), backed by the `StoreMetrics` struct. ### `store.object.throttles` @@ -809,14 +809,14 @@ Throttled responses, by provider and operation. | Unit | `{response}` | | Prometheus series | `nativelink_store_object_throttles` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1467`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1467) | +| Declared at | [`nativelink-util/src/metrics.rs:1469`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1469) | | Emitted from | [`nativelink-store/src/azure_blob_store.rs:268`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/azure_blob_store.rs#L268) | ## Store tier metrics Global tiered-store metrics. -Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1491), backed by the `StoreTierMetrics` struct. +Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1493`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1493), backed by the `StoreTierMetrics` struct. Emitted from the `fast_slow` store wrapper only. A deployment without a `fast_slow` store emits none of these. @@ -831,7 +831,7 @@ Reads served per tier, by result. | Unit | `{operation}` | | Prometheus series | `nativelink_store_tier_operations` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1495`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1495) | +| Declared at | [`nativelink-util/src/metrics.rs:1497`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1497) | | Emitted from | [`nativelink-store/src/fast_slow_store.rs:348`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L348), [`nativelink-store/src/fast_slow_store.rs:839`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L839), [`nativelink-store/src/fast_slow_store.rs:855`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L855), [`nativelink-store/src/fast_slow_store.rs:865`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L865), [`nativelink-store/src/fast_slow_store.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L882), [`nativelink-store/src/fast_slow_store.rs:912`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L912), [`nativelink-store/src/fast_slow_store.rs:934`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L934) | ### `store.tier.io` @@ -845,14 +845,14 @@ Bytes moved per tier and direction. | Unit | `By` | | Prometheus series | `nativelink_store_tier_io` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1501`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1501) | +| Declared at | [`nativelink-util/src/metrics.rs:1503`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1503) | | Emitted from | [`nativelink-store/src/fast_slow_store.rs:357`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L357), [`nativelink-store/src/fast_slow_store.rs:856`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L856), [`nativelink-store/src/fast_slow_store.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L913), [`nativelink-store/src/fast_slow_store.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L935) | ## Health check metrics Global health-check metrics. -Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1542`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1542), backed by the `HealthMetrics` struct. +Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1544`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1544), backed by the `HealthMetrics` struct. Emitted each time the health check endpoint runs, one observation per registered health indicator. @@ -867,14 +867,14 @@ Health check results, by namespace and status. | Unit | `{check}` | | Prometheus series | `nativelink_health_checks` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1546`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1546) | +| Declared at | [`nativelink-util/src/metrics.rs:1548`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1548) | | Emitted from | [`nativelink-util/src/health_utils.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/health_utils.rs#L248) | ## Connection pool metrics Global connection-pool metrics. -Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1573`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1573), backed by the `ConnectionMetrics` struct. +Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1575`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1575), backed by the `ConnectionMetrics` struct. Emitted by the gRPC connection manager and the Redis store as they hand out and re-establish connections. A deployment that uses neither emits none of these. @@ -889,7 +889,7 @@ Free slots at the moment a connection was taken. | Unit | `{connection}` | | Prometheus series | `nativelink_connection_pool_available` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1577`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1577) | +| Declared at | [`nativelink-util/src/metrics.rs:1579`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1579) | | Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L351) | **Histogram buckets:** 11 explicit boundaries, from @@ -908,7 +908,7 @@ Acquisitions, by pool and result. | Unit | `{acquisition}` | | Prometheus series | `nativelink_connection_pool_acquisitions` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1586`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1586) | +| Declared at | [`nativelink-util/src/metrics.rs:1588`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1588) | | Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L351) | ### `connection.reconnects` @@ -922,7 +922,7 @@ Reconnects, by pool. | Unit | `{reconnect}` | | Prometheus series | `nativelink_connection_reconnects` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1592`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1592) | +| Declared at | [`nativelink-util/src/metrics.rs:1594`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1594) | | Emitted from | [`nativelink-store/src/redis_store.rs:319`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L319) | ## Attributes