diff --git a/nativelink-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs index bebe74a0ee8..ad00f642c80 100644 --- a/nativelink-config/src/schedulers.rs +++ b/nativelink-config/src/schedulers.rs @@ -345,6 +345,26 @@ pub struct SimpleSpec { #[serde(default)] pub memory_escalation: Option, + /// Hold a worker for a queued action that keeps finding no room. An + /// action whose reservation only fits an idle worker starves under + /// load: every pass tries it first and finds each worker partly busy, + /// then fills the room that opens with newer, smaller actions, so no + /// worker ever drains for it. With this set, once such an action has + /// waited `hold_after_s` and fits no busy worker until more than one + /// of that worker's actions finish (one finishing would make room for + /// an action that is only waiting its turn), the scheduler picks a + /// worker that would fit it idle and offers it nothing else until a + /// worker takes the action, it leaves the queue, or `max_hold_s` + /// passes. An action no connected worker would fit even idle gets no + /// hold. Each scheduler replica holds only workers connected to it; + /// replicas sharing a queue take turns per action, by a hash of the + /// action over the replicas whose workers would fit it as the fleet + /// census last read them, so one worker is held for it fleet-wide. + /// While a replica cannot read the census it holds on its own. + /// Default: unset (off) + #[serde(default)] + pub reservation_hold: Option, + /// The storage backend to use for the scheduler. /// Default: memory pub experimental_backend: Option, @@ -571,6 +591,58 @@ const fn default_memory_escalation_percent() -> u64 { 200 } +#[derive(Deserialize, Serialize, Debug, Clone, Copy)] +#[serde(deny_unknown_fields)] +#[cfg_attr(feature = "dev-schema", derive(JsonSchema))] +pub struct ReservationHoldSpec { + /// Seconds a queued action has to have waited, finding no room on + /// every pass, before a worker is held for it. Must be above zero. + /// Default: 30 + #[serde( + default = "default_reservation_hold_after_s", + deserialize_with = "convert_duration_with_shellexpand" + )] + pub hold_after_s: u64, + + /// Seconds a hold lasts at most, capped at a week. A hold that long + /// without the action fitting means the worker never reported room for + /// it even drained; the hold is given up so the worker is not idle for + /// good, and the action is not held for again until as long has + /// passed, twice as long for each such expiry in a row on a worker + /// that was idle (up to sixteen times), since then nothing kept the + /// action off the worker but the action itself. Must be above zero. + /// Default: 300 + #[serde( + default = "default_reservation_max_hold_s", + deserialize_with = "convert_duration_with_shellexpand" + )] + pub max_hold_s: u64, + + /// Most workers held at once on this scheduler replica. 0 is one tenth + /// of the workers connected to it, and at least one. + /// Default: 0 + #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + pub max_held_workers: usize, +} + +impl Default for ReservationHoldSpec { + fn default() -> Self { + Self { + hold_after_s: default_reservation_hold_after_s(), + max_hold_s: default_reservation_max_hold_s(), + max_held_workers: 0, + } + } +} + +const fn default_reservation_hold_after_s() -> u64 { + 30 +} + +const fn default_reservation_max_hold_s() -> u64 { + 300 +} + #[derive(Deserialize, Serialize, Debug, Default, Clone, Copy)] #[serde(deny_unknown_fields)] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] diff --git a/nativelink-scheduler/BUILD.bazel b/nativelink-scheduler/BUILD.bazel index 334a8d97511..e17cff43c66 100644 --- a/nativelink-scheduler/BUILD.bazel +++ b/nativelink-scheduler/BUILD.bazel @@ -86,6 +86,7 @@ rust_test_suite( "tests/property_modifier_scheduler_test.rs", "tests/queue_order_test.rs", "tests/redis_store_awaited_action_db_test.rs", + "tests/reservation_hold_test.rs", "tests/simple_scheduler_state_manager_test.rs", "tests/simple_scheduler_test.rs", "tests/store_awaited_action_db_test.rs", diff --git a/nativelink-scheduler/src/api_worker_scheduler.rs b/nativelink-scheduler/src/api_worker_scheduler.rs index eb28ab2b331..64d7439e449 100644 --- a/nativelink-scheduler/src/api_worker_scheduler.rs +++ b/nativelink-scheduler/src/api_worker_scheduler.rs @@ -12,9 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use core::cmp::Reverse; +use core::hash::{Hash, Hasher}; use core::ops::{Deref, DerefMut}; use core::sync::atomic::{AtomicU64, Ordering}; use core::time::Duration; +use std::collections::hash_map::DefaultHasher; use std::collections::{BTreeSet, HashMap, HashSet}; use std::sync::Arc; use std::time::{Instant, SystemTime, UNIX_EPOCH}; @@ -22,7 +25,9 @@ use std::time::{Instant, SystemTime, UNIX_EPOCH}; use async_lock::Mutex; use futures::{StreamExt, future}; use lru::LruCache; -use nativelink_config::schedulers::{MemoryEscalationSpec, WorkerAllocationStrategy}; +use nativelink_config::schedulers::{ + MemoryEscalationSpec, ReservationHoldSpec, WorkerAllocationStrategy, +}; use nativelink_error::{Code, Error, ResultExt, make_err, make_input_err}; use nativelink_metric::{ MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, @@ -34,11 +39,12 @@ use nativelink_proto::com::github::trace_machina::nativelink::events::{ use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::{ ActionResourceUsage, ResourceOutcome, WorkerLoad, }; -use nativelink_util::action_messages::{ActionStage, OperationId, WorkerId}; +use nativelink_util::action_messages::{ActionStage, ActionUniqueQualifier, OperationId, WorkerId}; use nativelink_util::metrics::{ WorkerDisconnectReason, record_dispatch_requeue, record_execution_cpu_time, - record_execution_peak_memory, record_worker_connected, record_worker_disconnected, - record_worker_keepalive, record_worker_keepalive_gap, record_worker_state, + record_execution_peak_memory, record_held_workers, record_worker_connected, + record_worker_disconnected, record_worker_hold, record_worker_keepalive, + record_worker_keepalive_gap, record_worker_state, }; use nativelink_util::operation_state_manager::{ Decline, Escalation, UpdateOperationType, WorkerStateManager, @@ -133,6 +139,7 @@ pub struct SchedulerMetrics { pub dispatches_unacknowledged: AtomicU64, } +use crate::awaited_action_db::PeerScheduler; use crate::match_outcome::{ MatchOutcome, PropertyShape, UnsatisfiableReason, explain_unsatisfiable, }; @@ -176,6 +183,95 @@ impl MetricsComponent for Workers { } } +/// A worker kept for one queued action that found no room anywhere. +#[derive(Debug, Clone)] +struct Hold { + /// The action the worker is kept for. + action: ActionUniqueQualifier, + /// When the hold is given up. + until: SystemTime, + /// The last matching pass that read the action, by `pass_generation`. + /// A pass that read the whole queue without meeting it means the + /// action is gone, and so is the reason to hold. + seen_in: u64, + /// Whether the action is on its way to a worker, which took it off the + /// queue: the passes meanwhile do not meet it, and the hold waits for + /// the worker to take it or send it back. + dispatched: bool, +} + +/// An action whose hold expired: it is not held for again until `until`. +#[derive(Debug, Clone)] +struct HoldCooldown { + until: SystemTime, + /// Expiries in a row on a worker that was idle, each doubling the + /// cooldown, up to `MAX_IDLE_EXPIRY_DOUBLINGS`. + idle_expiries: u32, + /// The last matching pass that read the action; see `Hold::seen_in`. + seen_in: u64, +} + +/// The longest a hold lasts whatever `max_hold_s` says, so that the time +/// it ends at can always be computed. +const HOLD_CEILING_S: u64 = 7 * 24 * 60 * 60; + +/// How many times in a row an expiry on an idle worker doubles the wait +/// before the action is held for again: sixteen times `max_hold_s` at most. +const MAX_IDLE_EXPIRY_DOUBLINGS: u32 = 4; + +fn hold_duration(spec: ReservationHoldSpec) -> Duration { + Duration::from_secs(spec.max_hold_s.min(HOLD_CEILING_S)) +} + +/// A queued action a matching pass asks a worker for: which action, so a +/// worker held for it recognises it, and since when it has waited, for +/// whether to hold a worker for it. +#[derive(Debug, Clone, Copy)] +pub struct QueuedAction<'a> { + pub unique_qualifier: &'a ActionUniqueQualifier, + pub insert_timestamp: SystemTime, +} + +/// Which of `owners`, sorted, distinct and not empty, holds a worker for +/// `action`: the same one on every scheduler that sees the same owners, +/// by a hash of the action. Replicas run one binary, so the hash agrees. +#[must_use] +pub fn hold_owner<'a>(action: &ActionUniqueQualifier, owners: &[&'a str]) -> &'a str { + let mut hasher = DefaultHasher::new(); + action.hash(&mut hasher); + let count = u64::try_from(owners.len()).unwrap_or(u64::MAX); + let index = usize::try_from(hasher.finish() % count).unwrap_or(0); + owners[index] +} + +/// Whether `platform_properties` would fit `worker` once any one of its +/// running actions finished: every `Minimum` it asks is within what the +/// worker has left plus the largest reservation of that property among +/// the actions running on it. +fn fits_after_one_finishes(platform_properties: &PlatformProperties, worker: &Worker) -> bool { + platform_properties.properties.iter().all(|(name, value)| { + let PlatformPropertyValue::Minimum(needed) = value else { + return true; + }; + let left = match worker.platform_properties.properties.get(name) { + Some(PlatformPropertyValue::Minimum(left)) => *left, + _ => 0, + }; + let largest_running = worker + .running_action_infos + .values() + .filter_map(|pending| { + match pending.action_info.platform_properties.properties.get(name) { + Some(PlatformPropertyValue::Minimum(reserved)) => Some(*reserved), + _ => None, + } + }) + .max() + .unwrap_or(0); + *needed <= left.saturating_add(largest_running) + }) +} + /// A collection of workers that are available to run tasks. #[derive(MetricsComponent)] struct ApiWorkerSchedulerImpl { @@ -194,6 +290,19 @@ struct ApiWorkerSchedulerImpl { /// How a memory kill is turned into a retry with a larger reservation; /// unset means the kill fails the action. memory_escalation: Option, + /// How a queued action that keeps finding no room gets a worker held + /// for it; unset means never. + reservation_hold: Option, + /// Workers held for one queued action each, by worker, and the same + /// holds by action. A held worker is offered nothing but its action. + holds: HashMap, + holds_by_action: HashMap, + /// Actions whose hold expired, and until when they are not held for + /// again, so one that never fits does not keep a worker idle in turns. + hold_cooldowns: HashMap, + /// Counts matching passes, so a hold can record the last pass that + /// met its action; see `Hold::seen_in`. + pass_generation: u64, /// A channel to notify the matching engine that the worker pool has changed. worker_change_notify: Arc, /// Bumped with every notification, under the same lock as placement, so @@ -228,6 +337,14 @@ struct ApiWorkerSchedulerImpl { /// judged unsatisfiable, since a peer might have a worker for it. peer_fleet: Option>, + /// The peers as the census last read them, sorted by id, for deciding + /// which scheduler holds a worker for an action; empty while the census + /// is unknown. + peer_schedulers: Vec, + + /// The id this scheduler publishes to its peers under. + scheduler_id: String, + /// Whether this scheduler shares its state with peers. A non-shared db is /// the whole fleet, so peer-absence is authoritative at once; a shared one /// applies the census warm-up / staleness checks below before it trusts @@ -471,6 +588,7 @@ impl ApiWorkerSchedulerImpl { /// Note: The caller is responsible for any rescheduling of any tasks that might be /// running. fn remove_worker(&mut self, worker_id: &WorkerId) -> Option { + self.end_hold(worker_id, "released"); // Remove from capability index self.capability_index.remove_worker(worker_id); // Pop before recomputing the shapes, or the departing worker still @@ -518,19 +636,26 @@ impl ApiWorkerSchedulerImpl { .collect() } - fn set_peer_fleet(&mut self, peer_fleet: Option>, now: SystemTime) { + fn set_peer_fleet(&mut self, peers: Option>, now: SystemTime) { // Peers list their workers in no particular order and may share // shapes, so sort and dedup before comparing, or an unchanged fleet // would look changed on every exchange and empty `static_verdicts`. - let peer_fleet = peer_fleet.map(|peer_fleet| { - let mut keyed: Vec<_> = peer_fleet - .into_iter() - .map(|totals| (PropertyShape::from(&totals), totals)) + let peer_fleet = peers.as_ref().map(|peers| { + let mut keyed: Vec<_> = peers + .iter() + .flat_map(|peer| peer.workers.iter()) + .map(|totals| (PropertyShape::from(totals), totals.clone())) .collect(); keyed.sort_unstable_by(|a, b| a.0.cmp(&b.0)); keyed.dedup_by(|a, b| a.0 == b.0); keyed.into_iter().map(|(_, totals)| totals).collect() }); + // Which scheduler holds a worker for which action goes by the + // peers' ids as well as their shapes, so the peers are kept as read + // even when the union of shapes did not move. + let mut peer_schedulers = peers.unwrap_or_default(); + peer_schedulers.sort_unstable_by(|a, b| a.scheduler_id.cmp(&b.scheduler_id)); + self.peer_schedulers = peer_schedulers; // Freshness bookkeeping: update the warm-up / // staleness clocks BEFORE the unchanged-shapes early return, so a // successful exchange that happens to return the same shapes still @@ -571,10 +696,318 @@ impl ApiWorkerSchedulerImpl { record_worker_state("draining", is_draining); } worker.is_draining = is_draining; + if is_draining { + // A draining worker takes nothing, its holder included. + self.end_hold(worker_id, "released"); + } self.capacity_changed(); Ok(()) } + /// Ends the hold on `worker_id`, if any, for `outcome`. The worker is + /// room the fleet did not have a moment ago, so the matcher is woken. + /// An action placed starts over with its cooldown. + fn end_hold(&mut self, worker_id: &WorkerId, outcome: &'static str) { + let Some(hold) = self.holds.remove(worker_id) else { + return; + }; + self.holds_by_action.remove(&hold.action); + if outcome == "placed" { + self.hold_cooldowns.remove(&hold.action); + } + info!(?worker_id, action = ?hold.action, outcome, "Worker hold ended"); + record_worker_hold(outcome); + record_held_workers(self.holds.len()); + self.capacity_changed(); + } + + /// Ends the hold for `action`, if any, for `outcome`. + fn end_hold_for_action(&mut self, action: &ActionUniqueQualifier, outcome: &'static str) { + if let Some(worker_id) = self.holds_by_action.get(action).cloned() { + self.end_hold(&worker_id, outcome); + } + } + + /// Records that the held action was sent to a worker (`true`), or came + /// back from one untried (`false`); see `Hold::dispatched`. + fn note_hold_dispatched(&mut self, action: &ActionUniqueQualifier, dispatched: bool) { + if let Some(worker_id) = self.holds_by_action.get(action) + && let Some(hold) = self.holds.get_mut(worker_id) + { + hold.dispatched = dispatched; + } + } + + /// A queued action found no room. Once it has waited `hold_after_s`, + /// if it fits no busy worker until more than one of that worker's + /// actions finish, and it is this scheduler's turn to hold for it, a + /// worker is held for it here, under the lock the match ran under. + fn consider_hold( + &mut self, + asking: Option>, + platform_properties: &PlatformProperties, + now: SystemTime, + full_worker_logging: bool, + ) { + let (Some(spec), Some(asking)) = (self.reservation_hold, asking) else { + return; + }; + let action = asking.unique_qualifier; + if self.holds_by_action.contains_key(action) { + return; + } + let waited = now + .duration_since(asking.insert_timestamp) + .unwrap_or_default(); + if waited < Duration::from_secs(spec.hold_after_s) { + return; + } + if self + .hold_cooldowns + .get(action) + .is_some_and(|cooldown| now < cooldown.until) + { + return; + } + let cap = if spec.max_held_workers == 0 { + (self.workers.len() / 10).max(1) + } else { + spec.max_held_workers + }; + if self.holds.len() >= cap { + return; + } + if !self.only_an_idle_worker_fits(platform_properties) { + if full_worker_logging { + info!("No hold: one action finishing on some worker would make room for this one"); + } + return; + } + if !self.owns_hold(action, platform_properties) { + if full_worker_logging { + info!("No hold here: a peer scheduler holds a worker for this action"); + } + return; + } + self.hold_worker_for(action, platform_properties, now, spec); + } + + /// Whether `platform_properties` fits no matching worker that has an + /// action running until more than one of its actions finish: on each + /// such worker some `Minimum` the action asks exceeds what the worker + /// has left plus the most any one of its running actions reserved. An + /// action one completion would make room for is only waiting its turn, + /// which the pass order gives it; a hold for it would idle slots for + /// nothing. False too when no matching worker is busy: the idle ones + /// refuse it for a reason a hold does not change. + fn only_an_idle_worker_fits(&self, platform_properties: &PlatformProperties) -> bool { + let mut any_busy = false; + for (_, worker) in self.workers.iter() { + if worker.is_draining + || worker.running_action_infos.is_empty() + || !platform_properties.is_satisfied_by(&worker.total_platform_properties, false) + { + continue; + } + any_busy = true; + if fits_after_one_finishes(platform_properties, worker) { + return false; + } + } + any_busy + } + + /// Whether `action`, which found no room, is this scheduler's to hold a + /// worker for. Every replica reads the same queue in the same order, so + /// left to themselves they would each drain a worker of their own for + /// the same action. The schedulers with a worker that would fit it idle + /// (this one, and the peers as the census last read them) take turns + /// by a hash of the action; while the census cannot be read, this one + /// holds. + fn owns_hold( + &self, + action: &ActionUniqueQualifier, + platform_properties: &PlatformProperties, + ) -> bool { + let fits_here = self.workers.iter().any(|(_, worker)| { + platform_properties.is_satisfied_by(&worker.total_platform_properties, false) + }); + if !fits_here { + return false; + } + if self.peer_fleet.is_none() { + return true; + } + let mut owners: Vec<&str> = self + .peer_schedulers + .iter() + .filter(|peer| { + peer.workers + .iter() + .any(|totals| platform_properties.is_satisfied_by(totals, false)) + }) + .map(|peer| peer.scheduler_id.as_str()) + .collect(); + if owners.is_empty() { + return true; + } + owners.push(self.scheduler_id.as_str()); + owners.sort_unstable(); + owners.dedup(); + hold_owner(action, &owners) == self.scheduler_id + } + + /// Holds a worker for `action`: of the workers neither draining nor + /// held that would fit it idle and have not declined that much while + /// idle, the one with the fewest actions running and then the most + /// memory free, as last reported or else as the ledger has it. Nothing + /// is held when no connected worker would fit the action even idle, + /// which is not a matter of waiting. + fn hold_worker_for( + &mut self, + action: &ActionUniqueQualifier, + platform_properties: &PlatformProperties, + now: SystemTime, + spec: ReservationHoldSpec, + ) { + let needed_kb = self.live_memory_veto_kb(platform_properties); + let memory_property = self.live_memory_veto.as_deref(); + let memory_free = |worker: &Worker| { + worker + .last_load + .map(|load| load.free_memory_kb) + .or_else(|| { + match memory_property + .and_then(|property| worker.platform_properties.properties.get(property)) + { + Some(PlatformPropertyValue::Minimum(kb)) => Some(*kb), + _ => None, + } + }) + .unwrap_or(0) + }; + let holds = &self.holds; + let chosen = self + .workers + .iter() + .filter(|(worker_id, worker)| !worker.is_draining && !holds.contains_key(*worker_id)) + .filter(|(_, worker)| { + platform_properties.is_satisfied_by(&worker.total_platform_properties, false) + }) + // A worker that declined this much while idle is not offered + // it again until it reports that much free; a hold on it would + // wait for the worker, not for its actions. + .filter(|(_, worker)| { + !matches!( + (needed_kb, worker.idle_declined_kb), + (Some(needed_kb), Some(declined_kb)) if needed_kb >= declined_kb + ) + }) + .min_by_key(|(_, worker)| { + ( + worker.running_action_infos.len(), + Reverse(memory_free(worker)), + ) + }) + .map(|(worker_id, _)| worker_id.clone()); + let Some(worker_id) = chosen else { + return; + }; + info!( + ?worker_id, + ?action, + max_hold_s = spec.max_hold_s, + "Holding a worker for a queued action that fits only an idle one" + ); + self.holds.insert( + worker_id.clone(), + Hold { + action: action.clone(), + until: now.checked_add(hold_duration(spec)).unwrap_or(now), + seen_in: self.pass_generation, + dispatched: false, + }, + ); + self.holds_by_action.insert(action.clone(), worker_id); + record_worker_hold("held"); + record_held_workers(self.holds.len()); + } + + /// A matching pass begins; returns its generation, which the holds it + /// meets record and `end_pass_holds` compares against. + const fn begin_pass_holds(&mut self) -> u64 { + self.pass_generation += 1; + self.pass_generation + } + + /// Ends the holds past their time, whose action is then not held for + /// again for as long (twice as long for each such expiry in a row on a + /// worker that was idle, up to a bound: nothing kept the action off it + /// but the action itself), and the ones no pass since `pass` began has + /// met, whose action is gone. A hold whose action is in flight to a + /// worker is not met by passes either and waits for the worker to + /// answer. Judged by generation rather than a flag so that two passes + /// in flight at once (a test entry point beside the matcher's own loop) + /// cannot clear each other's marks. A cooldown no pass has met since is + /// dropped the same way. + fn end_pass_holds(&mut self, pass: u64, now: SystemTime) { + let ended: Vec<(WorkerId, &'static str)> = self + .holds + .iter() + .filter_map(|(worker_id, hold)| { + if hold.until <= now { + Some((worker_id.clone(), "expired")) + } else if hold.seen_in < pass && !hold.dispatched { + Some((worker_id.clone(), "released")) + } else { + None + } + }) + .collect(); + for (worker_id, outcome) in ended { + if outcome == "expired" + && let Some(spec) = self.reservation_hold + && let Some(hold) = self.holds.get(&worker_id) + { + let idle = self + .workers + .peek(&worker_id) + .is_some_and(|worker| worker.running_action_infos.is_empty()); + let cooldown = + self.hold_cooldowns + .entry(hold.action.clone()) + .or_insert(HoldCooldown { + until: now, + idle_expiries: 0, + seen_in: pass, + }); + if idle { + cooldown.idle_expiries = + (cooldown.idle_expiries + 1).min(MAX_IDLE_EXPIRY_DOUBLINGS); + } + cooldown.until = now + .checked_add(hold_duration(spec) * (1 << cooldown.idle_expiries)) + .unwrap_or(now); + } + self.end_hold(&worker_id, outcome); + } + self.hold_cooldowns + .retain(|_, cooldown| cooldown.seen_in >= pass); + } + + /// Marks the hold and the cooldown for `action`, if any, as met by the + /// pass under way. + fn note_held_action_seen(&mut self, action: &ActionUniqueQualifier) { + if let Some(worker_id) = self.holds_by_action.get(action) + && let Some(hold) = self.holds.get_mut(worker_id) + { + hold.seen_in = self.pass_generation; + } + if let Some(cooldown) = self.hold_cooldowns.get_mut(action) { + cooldown.seen_in = self.pass_generation; + } + } + /// Whether peer-absence can be trusted to fail an action right now — a /// shared census must be known, warm and fresh; a non-shared db is trusted /// as soon as it is known. See the free `peer_census_trusted`. @@ -671,6 +1104,7 @@ impl ApiWorkerSchedulerImpl { platform_properties: &PlatformProperties, candidates: &HashSet, full_worker_logging: bool, + asking: Option<&ActionUniqueQualifier>, ) -> Option { // Check function for availability AND dynamic Minimum property verification. // The index only does presence checks for Minimum properties since their @@ -690,6 +1124,16 @@ impl ApiWorkerSchedulerImpl { return false; } + // A held worker takes nothing but the action it is held for. + if let Some(hold) = self.holds.get(*worker_id) + && asking != Some(&hold.action) + { + if full_worker_logging { + info!("Worker {worker_id} is held for another action"); + } + return false; + } + if let (Some(needed_kb), Some(declined_kb)) = (needed_kb, w.idle_declined_kb) && needed_kb >= declined_kb { @@ -782,7 +1226,31 @@ impl ApiWorkerSchedulerImpl { platform_properties: &PlatformProperties, full_worker_logging: bool, now: SystemTime, + asking: Option>, ) -> MatchOutcome { + let outcome = self.match_worker_for_action( + platform_properties, + full_worker_logging, + now, + asking.map(|action| action.unique_qualifier), + ); + if matches!(outcome, MatchOutcome::WaitingForCapacity) { + self.consider_hold(asking, platform_properties, now, full_worker_logging); + } + outcome + } + + /// The match itself; `inner_find_worker_for_action` adds the hold. + fn match_worker_for_action( + &mut self, + platform_properties: &PlatformProperties, + full_worker_logging: bool, + now: SystemTime, + asking: Option<&ActionUniqueQualifier>, + ) -> MatchOutcome { + if let Some(action) = asking { + self.note_held_action_seen(action); + } if self.workers.is_empty() { if full_worker_logging { info!("No workers available to match!"); @@ -798,9 +1266,12 @@ impl ApiWorkerSchedulerImpl { let candidates = self .capability_index .find_matching_workers(platform_properties, full_worker_logging); - if let Some(worker_id) = - self.find_available_worker(platform_properties, &candidates, full_worker_logging) - { + if let Some(worker_id) = self.find_available_worker( + platform_properties, + &candidates, + full_worker_logging, + asking, + ) { return MatchOutcome::Matched(worker_id); } Some(candidates) @@ -1046,6 +1517,28 @@ impl ApiWorkerSchedulerImpl { operation_id: &OperationId, update: UpdateOperationType, ) -> Result<(), Error> { + // Any word from the worker about an action it was sent, other than + // sending it back, is the worker taking it: a hold for it ends as + // placed. A decline or a disconnect requeues the action, and its + // hold waits for it in the queue again. + let held_for = self + .workers + .peek(worker_id) + .and_then(|worker| worker.running_action_infos.get(operation_id)) + .map(|pending| pending.action_info.inner.unique_qualifier.clone()) + .filter(|action| self.holds_by_action.contains_key(action)); + if let Some(action) = held_for { + if matches!( + update, + UpdateOperationType::UpdateWithDecline(_) + | UpdateOperationType::UpdateWithDisconnect + ) { + self.note_hold_dispatched(&action, false); + } else { + self.end_hold_for_action(&action, "placed"); + } + } + let worker = self.workers.get_mut(worker_id).err_tip(|| { format!("Worker {worker_id} does not exist in SimpleScheduler::update_action") })?; @@ -1245,6 +1738,10 @@ impl ApiWorkerSchedulerImpl { .await; return Result::<(), _>::Err(err).merge(evicted).merge(requeued); } + // A worker held for the action keeps its hold until the worker + // the action went to takes it; a decline sends the action back + // to the queue, and the hold waits for it there again. + self.note_hold_dispatched(&action_info.inner.unique_qualifier, true); Ok(()) } else { warn!( @@ -1274,6 +1771,13 @@ impl ApiWorkerSchedulerImpl { return; }; if worker.mark_accepted(operation_id) { + let action = worker + .running_action_infos + .get(operation_id) + .map(|pending| pending.action_info.inner.unique_qualifier.clone()); + if let Some(action) = action { + self.end_hold_for_action(&action, "placed"); + } return; } // Messages from a worker arrive in order and the acknowledgement @@ -1314,6 +1818,11 @@ impl ApiWorkerSchedulerImpl { return Ok(false); } info!(?worker_id, %operation_id, %reason, ?needs_kb, "Worker declined dispatch"); + let declined_action = self + .workers + .peek(worker_id) + .and_then(|worker| worker.running_action_infos.get(operation_id)) + .map(|pending| pending.action_info.inner.unique_qualifier.clone()); self.update_action( worker_id, operation_id, @@ -1349,6 +1858,25 @@ impl ApiWorkerSchedulerImpl { worker.pause_needs_kb = needs_kb; } } + // A hold on this worker for the declined action outlives the + // decline: the worker is paused until it reports room, and is + // offered the action again then. Unless it declined the action + // idle: it is then not offered that much again until a keepalive + // says it has it free, which waits on the worker rather than on + // its actions, so the hold ends and the next pass holds another. + if let (Some(action), Some(needs_kb)) = (declined_action, needs_kb) + && self + .holds + .get(worker_id) + .is_some_and(|hold| hold.action == action) + && self.workers.peek(worker_id).is_some_and(|worker| { + worker + .idle_declined_kb + .is_some_and(|declined_kb| needs_kb >= declined_kb) + }) + { + self.end_hold(worker_id, "declined"); + } Ok(true) } @@ -1518,6 +2046,8 @@ pub struct ApiWorkerScheduler { /// See `ApiWorkerSchedulerImpl::capacity_generation`; read here without /// the lock. capacity_generation: Arc, + /// See `ApiWorkerSchedulerImpl::scheduler_id`; read here without the lock. + scheduler_id: String, #[metric(group = "platform_property_manager")] platform_property_manager: Arc, @@ -1569,6 +2099,7 @@ impl ApiWorkerScheduler { allocation_strategy: WorkerAllocationStrategy, live_memory_veto: Option, memory_escalation: Option, + reservation_hold: Option, worker_change_notify: Arc, worker_timeout_s: u64, unacknowledged_kill_timeout_s: u64, @@ -1577,11 +2108,16 @@ impl ApiWorkerScheduler { maybe_origin_event_tx: Option>, has_peers: bool, record_ttl: Duration, + scheduler_id: String, ) -> Arc { debug_assert!( !record_ttl.is_zero(), "record_ttl must be non-zero; a zero TTL degenerates the census warm-up/staleness checks" ); + debug_assert!( + reservation_hold.is_none_or(|hold| hold.hold_after_s > 0 && hold.max_hold_s > 0), + "reservation_hold.hold_after_s and max_hold_s must be above zero; the factory refuses them" + ); let local_fleet_change_notify = Arc::new(Notify::new()); let capacity_generation = Arc::new(AtomicU64::new(0)); Arc::new(Self { @@ -1591,6 +2127,11 @@ impl ApiWorkerScheduler { allocation_strategy, live_memory_veto, memory_escalation, + reservation_hold, + holds: HashMap::new(), + holds_by_action: HashMap::new(), + hold_cooldowns: HashMap::new(), + pass_generation: 0, worker_change_notify, capacity_generation: capacity_generation.clone(), worker_registry: worker_registry.clone(), @@ -1600,6 +2141,8 @@ impl ApiWorkerScheduler { fleet_shape_set: BTreeSet::new(), // Without peers there is nothing to wait for. peer_fleet: (!has_peers).then(Vec::new), + peer_schedulers: Vec::new(), + scheduler_id: scheduler_id.clone(), shared: has_peers, record_ttl, peer_fleet_known_since: None, @@ -1616,9 +2159,16 @@ impl ApiWorkerScheduler { maybe_origin_event_tx, local_fleet_change_notify, capacity_generation, + scheduler_id, }) } + /// The id this scheduler publishes to its peers under. + #[must_use] + pub fn scheduler_id(&self) -> &str { + &self.scheduler_id + } + /// Whether no worker is connected here and, as far as a trusted census /// says, to any peer either. False while the census cannot be trusted, /// so a peer we cannot see is assumed to have workers. @@ -1746,12 +2296,13 @@ impl ApiWorkerScheduler { /// Records what the workers connected to peer schedulers can run, or /// `None` when that is not known. - pub async fn set_peer_fleet( - &self, - peer_fleet: Option>, - now: SystemTime, - ) { - self.inner.lock().await.set_peer_fleet(peer_fleet, now); + pub async fn set_peer_fleet(&self, peers: Option>, now: SystemTime) { + self.inner.lock().await.set_peer_fleet(peers, now); + } + + /// The workers held for a queued action right now. + pub async fn held_workers_for_test(&self) -> Vec { + self.inner.lock().await.holds.keys().cloned().collect() } /// Woken when a worker joins or leaves this scheduler. @@ -1769,12 +2320,25 @@ impl ApiWorkerScheduler { platform_properties: &PlatformProperties, full_worker_logging: bool, now: SystemTime, + asking: Option>, ) -> MatchOutcome { - self.find_worker_for_action_observed(platform_properties, full_worker_logging, now) + self.find_worker_for_action_observed(platform_properties, full_worker_logging, now, asking) .await .0 } + /// A matching pass is about to read the queue; returns the pass, for + /// `end_pass_holds`. + pub async fn begin_pass_holds(&self) -> u64 { + self.inner.lock().await.begin_pass_holds() + } + + /// The pass read the whole queue: release the holds no pass since it + /// began has met, and the ones past their time. + pub async fn end_pass_holds(&self, pass: u64, now: SystemTime) { + self.inner.lock().await.end_pass_holds(pass, now); + } + /// `find_worker_for_action`, also returning the capacity generation read /// under the same lock as the placement. A pass compares it with the one /// it saw last: a change means a worker gained room since, and an older @@ -1785,6 +2349,7 @@ impl ApiWorkerScheduler { platform_properties: &PlatformProperties, full_worker_logging: bool, now: SystemTime, + asking: Option>, ) -> (MatchOutcome, u64) { let start = Instant::now(); self.metrics @@ -1793,8 +2358,12 @@ impl ApiWorkerScheduler { let mut inner = self.inner.lock().await; let worker_count = inner.workers.len() as u64; - let result = - inner.inner_find_worker_for_action(platform_properties, full_worker_logging, now); + let result = inner.inner_find_worker_for_action( + platform_properties, + full_worker_logging, + now, + asking, + ); let generation = inner.capacity_generation.load(Ordering::Relaxed); // Track workers iterated (worst case is all workers) diff --git a/nativelink-scheduler/src/awaited_action_db/mod.rs b/nativelink-scheduler/src/awaited_action_db/mod.rs index 657a5d291d6..8ac6a59f1ae 100644 --- a/nativelink-scheduler/src/awaited_action_db/mod.rs +++ b/nativelink-scheduler/src/awaited_action_db/mod.rs @@ -198,17 +198,17 @@ pub trait AwaitedActionDb: Send + Sync + MetricsComponent + Unpin + 'static { fn set_worker_registry(&mut self, _worker_registry: SharedWorkerRegistry) {} /// Tells other schedulers on the same state what the workers connected - /// here can run, and returns what the workers connected to those - /// schedulers can run. `local` holds the properties each distinct kind - /// of worker registered with, and stays visible to peers for `ttl`. - /// Only shared-state implementations need it; the default shares + /// here can run, and returns, per peer scheduler, what the workers + /// connected to it can run. `local` holds the properties each distinct + /// kind of worker registered with, and stays visible to peers for + /// `ttl`. Only shared-state implementations need it; the default shares /// nothing and sees nothing. fn exchange_fleet_capabilities( &self, _scheduler_id: &str, _local: Vec, _ttl: Duration, - ) -> impl Future, Error>> + Send { + ) -> impl Future, Error>> + Send { async { Ok(Vec::new()) } } @@ -218,3 +218,13 @@ pub trait AwaitedActionDb: Send + Sync + MetricsComponent + Unpin + 'static { false } } + +/// What one peer scheduler's workers can run, as it published them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PeerScheduler { + /// The id the peer publishes under, unique to its process. + pub scheduler_id: String, + /// The properties each distinct kind of worker connected to the peer + /// registered with. + pub workers: Vec, +} diff --git a/nativelink-scheduler/src/default_scheduler_factory.rs b/nativelink-scheduler/src/default_scheduler_factory.rs index d30924eef86..ef519f17a4d 100644 --- a/nativelink-scheduler/src/default_scheduler_factory.rs +++ b/nativelink-scheduler/src/default_scheduler_factory.rs @@ -159,6 +159,15 @@ async fn simple_scheduler_factory( policy.percent )); } + if let Some(hold) = &spec.reservation_hold + && (hold.hold_after_s == 0 || hold.max_hold_s == 0) + { + return Err(make_input_err!( + "reservation_hold.hold_after_s and max_hold_s must be above zero, got {} and {}: a zero wait holds for any action on its first miss, and a zero hold expires in the pass that took it and wakes the next", + hold.hold_after_s, + hold.max_hold_s + )); + } match spec .experimental_backend .as_ref() diff --git a/nativelink-scheduler/src/simple_scheduler.rs b/nativelink-scheduler/src/simple_scheduler.rs index 4cb5cccd305..ea25a8c66a1 100644 --- a/nativelink-scheduler/src/simple_scheduler.rs +++ b/nativelink-scheduler/src/simple_scheduler.rs @@ -54,8 +54,8 @@ use tokio::time::Duration; use tracing::{debug, error, info, info_span, warn}; use uuid::Uuid; -use crate::api_worker_scheduler::ApiWorkerScheduler; -use crate::awaited_action_db::{AwaitedActionDb, CLIENT_KEEPALIVE_DURATION}; +use crate::api_worker_scheduler::{ApiWorkerScheduler, QueuedAction}; +use crate::awaited_action_db::{AwaitedActionDb, CLIENT_KEEPALIVE_DURATION, PeerScheduler}; use crate::known_platform_property_provider::KnownPlatformPropertyProvider; use crate::match_outcome::{MatchOutcome, PropertyShape, UnsatisfiableReason}; use crate::platform_property_manager::PlatformPropertyManager; @@ -405,13 +405,33 @@ impl SimpleScheduler { self.do_try_match(false).await } - /// Stands in for the fleet exchange with peer schedulers. + /// Stands in for the fleet exchange with peer schedulers: one peer with + /// these workers. pub async fn set_peer_fleet_for_test(&self, peer_fleet: Vec) { + self.set_peer_schedulers_for_test(vec![PeerScheduler { + scheduler_id: "peer".to_string(), + workers: peer_fleet, + }]) + .await; + } + + /// Stands in for the fleet exchange with peer schedulers. + pub async fn set_peer_schedulers_for_test(&self, peers: Vec) { self.worker_scheduler - .set_peer_fleet(Some(peer_fleet), (self.now_fn)()) + .set_peer_fleet(Some(peers), (self.now_fn)()) .await; } + /// The id this scheduler publishes to its peers under. + pub fn scheduler_id_for_test(&self) -> &str { + self.worker_scheduler.scheduler_id() + } + + /// The workers held for a queued action right now. + pub async fn held_workers_for_test(&self) -> Vec { + self.worker_scheduler.held_workers_for_test().await + } + /// Stands in for a failed fleet exchange with peer schedulers. pub async fn set_peers_unknown_for_test(&self) { self.worker_scheduler @@ -801,6 +821,10 @@ impl SimpleScheduler { &loaded.action_info.platform_properties, full_worker_logging, now, + Some(QueuedAction { + unique_qualifier: &loaded.action_info.inner.unique_qualifier, + insert_timestamp: loaded.action_info.inner.insert_timestamp, + }), ) .await else { @@ -883,6 +907,7 @@ impl SimpleScheduler { let maybe_origin_event_tx = self.maybe_origin_event_tx.as_ref(); let mut parked = Parked::default(); let mut placement = Placement::default(); + let hold_pass = workers.begin_pass_holds().await; let mut capacity_seen = workers.capacity_generation(); let mut queued_seen: u64 = 0; // How long each action read has waited since its client submitted @@ -920,6 +945,10 @@ impl SimpleScheduler { &loaded.action_info.platform_properties, full_worker_logging, unsatisfiable_pass.now, + Some(QueuedAction { + unique_qualifier: &loaded.action_info.inner.unique_qualifier, + insert_timestamp: loaded.action_info.inner.insert_timestamp, + }), ) .await; if generation == capacity_seen || parked.is_empty() { @@ -958,8 +987,11 @@ impl SimpleScheduler { .map(|_| ()), ); } - // A worker could run it once one has room. - MatchOutcome::WaitingForCapacity => parked.park(loaded), + // A worker could run it once one has room. The match + // considered holding a worker for it (`reservation_hold`). + MatchOutcome::WaitingForCapacity => { + parked.park(loaded); + } // Nothing to wait for; a worker connecting starts a new pass, // unless the no-worker clock is running. MatchOutcome::NoWorkersConnected => { @@ -1010,6 +1042,11 @@ impl SimpleScheduler { .await, ); } + // The whole queue was read: a held action the pass did not meet + // is gone, and a hold past its time is given up. + workers + .end_pass_holds(hold_pass, unsatisfiable_pass.now) + .await; record_parked_dispatched(placement.dispatched); if placement.attempted > 0 || parked.cap_dropped > 0 { debug!( @@ -1215,12 +1252,16 @@ impl SimpleScheduler { Some(worker_registry.clone()), ); + // The id this scheduler publishes its fleet to its peers under, and + // takes its turn holding workers by. + let scheduler_id = Uuid::new_v4().hyphenated().to_string(); let worker_scheduler = ApiWorkerScheduler::new( state_manager.clone(), platform_property_manager.clone(), spec.allocation_strategy, spec.live_memory_veto.clone(), spec.memory_escalation.clone(), + spec.reservation_hold, worker_change_notify.clone(), worker_timeout_s, unacknowledged_kill_timeout_s, @@ -1229,6 +1270,7 @@ impl SimpleScheduler { maybe_origin_event_tx.clone(), has_peers, record_ttl, + scheduler_id.clone(), ); let worker_scheduler_clone = worker_scheduler.clone(); @@ -1328,7 +1370,6 @@ impl SimpleScheduler { // so that a peer which does fail actions sees these workers even // while this scheduler is still rolling out or has the timeout off. let task_fleet_exchange_spawn = { - let scheduler_id = Uuid::new_v4().hyphenated().to_string(); let state_manager = Arc::downgrade(&state_manager); let local_fleet_change_notify = worker_scheduler.local_fleet_change_notify().clone(); let worker_scheduler = Arc::downgrade(&worker_scheduler); diff --git a/nativelink-scheduler/src/simple_scheduler_state_manager.rs b/nativelink-scheduler/src/simple_scheduler_state_manager.rs index ff6a0de1722..4defb3bced6 100644 --- a/nativelink-scheduler/src/simple_scheduler_state_manager.rs +++ b/nativelink-scheduler/src/simple_scheduler_state_manager.rs @@ -44,7 +44,8 @@ use opentelemetry::KeyValue; use tracing::{debug, info, trace, warn}; use super::awaited_action_db::{ - AwaitedAction, AwaitedActionDb, AwaitedActionSubscriber, SortedAwaitedActionState, + AwaitedAction, AwaitedActionDb, AwaitedActionSubscriber, PeerScheduler, + SortedAwaitedActionState, }; use crate::worker_registry::{ORPHANED_ACTION_TIMEOUT, SharedWorkerRegistry, WorkerLiveness}; @@ -790,7 +791,7 @@ where scheduler_id: &str, local: Vec, ttl: Duration, - ) -> Result, Error> { + ) -> Result, Error> { self.action_db .exchange_fleet_capabilities(scheduler_id, local, ttl) .await diff --git a/nativelink-scheduler/src/store_awaited_action_db.rs b/nativelink-scheduler/src/store_awaited_action_db.rs index 41897e0ad45..01d486798ee 100644 --- a/nativelink-scheduler/src/store_awaited_action_db.rs +++ b/nativelink-scheduler/src/store_awaited_action_db.rs @@ -44,7 +44,7 @@ use tracing::{error, warn}; use crate::awaited_action_db::{ AwaitedAction, AwaitedActionDb, AwaitedActionSubscriber, CLIENT_KEEPALIVE_DURATION, - SortedAwaitedAction, SortedAwaitedActionState, + PeerScheduler, SortedAwaitedAction, SortedAwaitedActionState, }; use crate::worker_registry::{ORPHANED_ACTION_TIMEOUT, SharedWorkerRegistry, WorkerLiveness}; @@ -1254,7 +1254,7 @@ where scheduler_id: &str, local: Vec, ttl: Duration, - ) -> Result, Error> { + ) -> Result, Error> { self.store .update_data( UpdateFleetCapabilities(FleetCapabilities { @@ -1276,7 +1276,10 @@ where stream .try_filter(|record| futures::future::ready(record.scheduler_id != scheduler_id)) .try_fold(Vec::new(), |mut peers, record| async move { - peers.extend(record.workers); + peers.push(PeerScheduler { + scheduler_id: record.scheduler_id, + workers: record.workers, + }); Ok(peers) }) .await diff --git a/nativelink-scheduler/tests/fleet_capabilities_test.rs b/nativelink-scheduler/tests/fleet_capabilities_test.rs index ff2c8c23496..40d099977a9 100644 --- a/nativelink-scheduler/tests/fleet_capabilities_test.rs +++ b/nativelink-scheduler/tests/fleet_capabilities_test.rs @@ -23,7 +23,7 @@ use nativelink_config::stores::RedisSpec; use nativelink_error::{Code, Error}; use nativelink_macro::nativelink_test; use nativelink_redis_tester::FakeRedisBackend; -use nativelink_scheduler::awaited_action_db::AwaitedActionDb; +use nativelink_scheduler::awaited_action_db::{AwaitedActionDb, PeerScheduler}; use nativelink_scheduler::simple_scheduler::SimpleScheduler; use nativelink_scheduler::store_awaited_action_db::StoreAwaitedActionDb; use nativelink_scheduler::worker::Worker; @@ -88,18 +88,29 @@ async fn schedulers_see_each_others_fleet_but_not_their_own() -> Result<(), Erro let peers = db_a .exchange_fleet_capabilities("a", vec![shape(0)], TTL) .await?; - assert_eq!(peers, Vec::::new()); + assert_eq!(peers, Vec::::new()); let peers = db_b .exchange_fleet_capabilities("b", vec![shape(1), shape(2)], TTL) .await?; - assert_eq!(peers, vec![shape(0)]); + assert_eq!( + peers, + vec![PeerScheduler { + scheduler_id: "a".to_string(), + workers: vec![shape(0)], + }] + ); - let mut peers = db_a + let peers = db_a .exchange_fleet_capabilities("a", vec![shape(0)], TTL) .await?; - peers.sort_by_key(|p| p.properties["gpu_count"].as_str().into_owned()); - assert_eq!(peers, vec![shape(1), shape(2)]); + assert_eq!( + peers, + vec![PeerScheduler { + scheduler_id: "b".to_string(), + workers: vec![shape(1), shape(2)], + }] + ); // Each record expires on its own once its scheduler stops refreshing it. let expiries = fake_redis_backend.expiries.lock().unwrap(); @@ -120,7 +131,14 @@ async fn republishing_replaces_the_previous_record() -> Result<(), Error> { db_b.exchange_fleet_capabilities("b", vec![], TTL).await?; let peers = db_a.exchange_fleet_capabilities("a", vec![], TTL).await?; - assert_eq!(peers, Vec::::new()); + assert_eq!( + peers, + vec![PeerScheduler { + scheduler_id: "b".to_string(), + workers: vec![], + }], + "the peer is still there, with no workers" + ); Ok(()) } diff --git a/nativelink-scheduler/tests/reservation_hold_test.rs b/nativelink-scheduler/tests/reservation_hold_test.rs new file mode 100644 index 00000000000..4594574efc6 --- /dev/null +++ b/nativelink-scheduler/tests/reservation_hold_test.rs @@ -0,0 +1,894 @@ +// Copyright 2026 The NativeLink Authors. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::time::Duration; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::UNIX_EPOCH; + +use mock_instant::thread_local::MockClock; +use nativelink_config::schedulers::{PropertyType, ReservationHoldSpec, SimpleSpec}; +use nativelink_error::{Error, ResultExt}; +use nativelink_macro::nativelink_test; +use nativelink_proto::com::github::trace_machina::nativelink::remote_execution::{ + UpdateForWorker, WorkerLoad, update_for_worker, +}; +use nativelink_scheduler::api_worker_scheduler::hold_owner; +use nativelink_scheduler::awaited_action_db::PeerScheduler; +use nativelink_scheduler::default_scheduler_factory::memory_awaited_action_db_factory; +use nativelink_scheduler::simple_scheduler::SimpleScheduler; +use nativelink_scheduler::worker::Worker; +use nativelink_scheduler::worker_scheduler::WorkerScheduler; +use nativelink_util::action_messages::{ + ActionResult, ActionStage, ActionUniqueQualifier, OperationId, WorkerId, +}; +use nativelink_util::common::DigestInfo; +use nativelink_util::instant_wrapper::MockInstantWrapped; +use nativelink_util::operation_state_manager::{ + ActionStateResult, ClientStateManager, UpdateOperationType, +}; +use nativelink_util::platform_properties::{PlatformProperties, PlatformPropertyValue}; +use pretty_assertions::assert_eq; +use tokio::sync::{Notify, mpsc}; +use utils::scheduler_utils::make_base_action_info; + +mod utils { + pub(crate) mod scheduler_utils; +} + +const NOW_TIME: u64 = 10000; +const HOLD_AFTER_S: u64 = 30; +const MAX_HOLD_S: u64 = 300; +/// Each worker advertises this much memory and runs up to four actions. +const WORKER_KB: u64 = 100_000; +const SMALL_KB: u64 = 10_000; +/// Fits a worker only when nothing else runs on it: beside one small +/// action there is room for it once that one finishes, beside two there +/// is not until both have. +const LARGE_KB: u64 = 95_000; + +/// The client listeners of the queued actions, kept so that dropping one +/// does not drop its action. +type Listeners = Vec>; + +#[derive(Clone, Copy)] +struct Options { + max_held_workers: usize, + live_memory_veto: bool, + client_action_timeout_s: u64, +} + +impl Default for Options { + fn default() -> Self { + Self { + max_held_workers: 0, + live_memory_veto: false, + // The clock jumps past a hold here, and no client polls + // meanwhile; neither the client timeout nor the backend's + // retention window may retire a queued action over the jump. + client_action_timeout_s: 100_000, + } + } +} + +fn make_scheduler(options: Options) -> Arc { + MockClock::set_time(Duration::from_secs(NOW_TIME)); + let spec = SimpleSpec { + supported_platform_properties: Some(HashMap::from([( + "memory_kb".to_string(), + PropertyType::Minimum, + )])), + reservation_hold: Some(ReservationHoldSpec { + hold_after_s: HOLD_AFTER_S, + max_hold_s: MAX_HOLD_S, + max_held_workers: options.max_held_workers, + }), + live_memory_veto: options.live_memory_veto.then(|| "memory_kb".to_string()), + client_action_timeout_s: options.client_action_timeout_s, + ..SimpleSpec::default() + }; + let task_change_notify = Arc::new(Notify::new()); + let (scheduler, _worker_scheduler) = SimpleScheduler::new_with_callback( + &spec, + memory_awaited_action_db_factory(100_000, &task_change_notify, MockInstantWrapped::default), + || async move {}, + task_change_notify, + MockInstantWrapped::default, + None, + ); + scheduler +} + +fn worker_id(name: &str) -> WorkerId { + WorkerId(name.to_string()) +} + +async fn add_worker_with( + scheduler: &SimpleScheduler, + name: &str, + admits_when_idle: bool, +) -> Result, Error> { + let (tx, mut rx) = mpsc::channel(64); + let mut worker = Worker::new( + worker_id(name), + PlatformProperties::new(HashMap::from([( + "memory_kb".to_string(), + PlatformPropertyValue::Minimum(WORKER_KB), + )])), + tx, + NOW_TIME, + /* max_inflight_tasks */ 4, + ); + worker.admits_when_idle = admits_when_idle; + scheduler + .add_worker(worker) + .await + .err_tip(|| "Failed to add worker")?; + tokio::task::yield_now().await; + let first = rx.try_recv().expect("worker got no connection result"); + assert!(matches!( + first.update, + Some(update_for_worker::Update::ConnectionResult(_)) + )); + Ok(rx) +} + +async fn add_worker( + scheduler: &SimpleScheduler, + name: &str, +) -> Result, Error> { + add_worker_with(scheduler, name, false).await +} + +async fn add_action( + scheduler: &SimpleScheduler, + listeners: &mut Listeners, + digest_byte: u8, + kb: u64, +) -> Result { + let mut action_info = make_base_action_info( + UNIX_EPOCH + MockClock::time(), + DigestInfo::new([digest_byte; 32], 512), + ); + Arc::make_mut(&mut action_info).platform_properties = + HashMap::from([("memory_kb".to_string(), kb.to_string())]); + let unique_qualifier = action_info.unique_qualifier.clone(); + let listener = scheduler + .add_action(OperationId::default(), action_info) + .await?; + listeners.push(listener); + tokio::task::yield_now().await; + Ok(unique_qualifier) +} + +/// The dispatches a worker received since last asked: operation id and the +/// memory each reserved. +fn dispatches(rx: &mut mpsc::Receiver) -> Vec<(String, u64)> { + let mut seen = Vec::new(); + while let Ok(msg) = rx.try_recv() { + if let Some(update_for_worker::Update::StartAction(start)) = msg.update { + let memory_kb = start + .platform + .as_ref() + .and_then(|platform| { + platform + .properties + .iter() + .find(|p| p.name == "memory_kb") + .and_then(|p| p.value.parse().ok()) + }) + .expect("the dispatch carries memory_kb"); + seen.push((start.operation_id, memory_kb)); + } + } + seen +} + +fn memory_of(dispatched: &[(String, u64)]) -> Vec { + dispatched.iter().map(|(_, kb)| *kb).collect() +} + +async fn complete( + scheduler: &SimpleScheduler, + worker: &str, + operation_id: &str, +) -> Result<(), Error> { + scheduler + .update_action( + &worker_id(worker), + &OperationId::from(operation_id), + UpdateOperationType::UpdateWithActionStage(ActionStage::Completed( + ActionResult::default(), + )), + ) + .await?; + tokio::task::yield_now().await; + Ok(()) +} + +/// The worker acknowledges the dispatch. +async fn acknowledge( + scheduler: &SimpleScheduler, + worker: &str, + operation_id: &str, +) -> Result<(), Error> { + scheduler + .worker_dispatch_accepted(&worker_id(worker), &OperationId::from(operation_id)) + .await?; + tokio::task::yield_now().await; + Ok(()) +} + +/// A keepalive from the worker reporting this much memory free. +async fn report_free(scheduler: &SimpleScheduler, worker: &str, free_kb: u64) -> Result<(), Error> { + scheduler + .worker_keep_alive_received( + &worker_id(worker), + MockClock::time().as_secs(), + Some(WorkerLoad { + free_memory_kb: free_kb, + }), + ) + .await?; + tokio::task::yield_now().await; + Ok(()) +} + +/// Lets the clock run and runs a pass, as the matcher would. +async fn pass_after(scheduler: &SimpleScheduler, seconds: u64) -> Result<(), Error> { + MockClock::advance(Duration::from_secs(seconds)); + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + Ok(()) +} + +async fn held(scheduler: &SimpleScheduler) -> Vec { + let mut held: Vec = scheduler + .held_workers_for_test() + .await + .into_iter() + .map(|id| id.0) + .collect(); + held.sort(); + held +} + +/// Two workers each running two small actions, and a large action that +/// fits neither until both of its small actions finish. Returns the small +/// actions' operation ids on a and on b, and the large action. +async fn two_busy_workers_and_a_large_action( + scheduler: &SimpleScheduler, + listeners: &mut Listeners, + a: &mut mpsc::Receiver, + b: &mut mpsc::Receiver, +) -> Result<(Vec, Vec, ActionUniqueQualifier), Error> { + for digest in 1..5 { + add_action(scheduler, listeners, digest, SMALL_KB).await?; + } + let (on_a, on_b) = (dispatches(a), dispatches(b)); + assert_eq!( + (on_a.len(), on_b.len()), + (2, 2), + "two small actions per worker" + ); + let large = add_action(scheduler, listeners, 9, LARGE_KB).await?; + assert_eq!( + (dispatches(a).len(), dispatches(b).len()), + (0, 0), + "the large action fits neither worker beside two small ones" + ); + Ok(( + on_a.into_iter().map(|(id, _)| id).collect(), + on_b.into_iter().map(|(id, _)| id).collect(), + large, + )) +} + +/// Once a worker is held, two small actions both land on the other one. +/// Returns the names of the held and the free worker. +async fn held_and_free( + scheduler: &SimpleScheduler, + listeners: &mut Listeners, + a: &mut mpsc::Receiver, + b: &mut mpsc::Receiver, +) -> Result<(&'static str, &'static str), Error> { + add_action(scheduler, listeners, 10, SMALL_KB).await?; + add_action(scheduler, listeners, 11, SMALL_KB).await?; + let (on_a, on_b) = (dispatches(a).len(), dispatches(b).len()); + let (held_worker, free) = match (on_a, on_b) { + (0, 2) => ("a", "b"), + (2, 0) => ("b", "a"), + other => panic!("one worker should be held and take nothing, got {other:?}"), + }; + assert_eq!(held(scheduler).await, vec![held_worker.to_string()]); + Ok((held_worker, free)) +} + +/// A large action waits behind small ones on every worker. Once it has +/// waited `hold_after_s`, one worker is held: new small actions all go to +/// the other, and when the held worker's own actions finish the large one +/// runs there. The hold ends once the worker takes it, and the worker +/// takes small actions again. +#[nativelink_test] +async fn a_starving_reservation_gets_a_worker_held_then_runs_there() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + let (small_on_a, small_on_b, _) = + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + + // Not long enough yet: nothing is held, a small action goes anywhere. + pass_after(&scheduler, HOLD_AFTER_S - 1).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + add_action(&scheduler, &mut listeners, 12, SMALL_KB).await?; + let (on_a, on_b) = (dispatches(&mut a), dispatches(&mut b)); + assert_eq!(on_a.len() + on_b.len(), 1, "nothing is held yet"); + // Put the fleet back to two small actions per worker. + let (third_worker, third) = if on_a.is_empty() { + ("b", on_b[0].0.clone()) + } else { + ("a", on_a[0].0.clone()) + }; + complete(&scheduler, third_worker, &third).await?; + scheduler.do_try_match_for_test().await?; + assert_eq!( + (dispatches(&mut a).len(), dispatches(&mut b).len()), + (0, 0), + "the room that opened is still too small for the large action" + ); + + pass_after(&scheduler, 1).await?; + let (held_worker, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let (held_rx, free_rx) = if held_worker == "a" { + (&mut a, &mut b) + } else { + (&mut b, &mut a) + }; + let small_on_held = if held_worker == "a" { + small_on_a + } else { + small_on_b + }; + + // One of the held worker's actions finishes: not enough room yet, and + // nothing else is offered the worker either. + complete(&scheduler, held_worker, &small_on_held[0]).await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!(dispatches(held_rx).len(), 0); + assert_eq!(held(&scheduler).await, vec![held_worker.to_string()]); + + // The other finishes: the large action runs there. The hold lasts + // until the worker says it took it. + complete(&scheduler, held_worker, &small_on_held[1]).await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + let on_held = dispatches(held_rx); + assert_eq!( + memory_of(&on_held), + vec![LARGE_KB], + "the held worker runs the action it was held for" + ); + assert_eq!(dispatches(free_rx).len(), 0); + assert_eq!( + held(&scheduler).await, + vec![held_worker.to_string()], + "the hold outlives the dispatch" + ); + acknowledge(&scheduler, held_worker, &on_held[0].0).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + + // Once the large action is done the worker is idle and the free one + // is full, so two small actions both run only if it takes them. + complete(&scheduler, held_worker, &on_held[0].0).await?; + add_action(&scheduler, &mut listeners, 13, SMALL_KB).await?; + add_action(&scheduler, &mut listeners, 14, SMALL_KB).await?; + assert_eq!( + (dispatches(held_rx).len(), dispatches(free_rx).len()), + (2, 0), + "the worker is offered everything again once its action ran" + ); + Ok(()) +} + +/// Small actions queued behind a full fleet are only waiting their turn: +/// one action finishing anywhere makes room for one of them. However long +/// they wait, no worker is held. +#[nativelink_test] +async fn an_action_one_completion_makes_room_for_gets_no_hold() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + for digest in 1..9 { + add_action(&scheduler, &mut listeners, digest, SMALL_KB).await?; + } + assert_eq!((dispatches(&mut a).len(), dispatches(&mut b).len()), (4, 4)); + for digest in 9..12 { + add_action(&scheduler, &mut listeners, digest, SMALL_KB).await?; + } + pass_after(&scheduler, HOLD_AFTER_S).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + // Nor for a reservation that fits beside three small actions but not + // beside four: one finishing makes room for it. + add_action(&scheduler, &mut listeners, 12, WORKER_KB - 3 * SMALL_KB).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + Ok(()) +} + +/// A hold that never fits is given up after `max_hold_s`, and the action is +/// not held for again straight away: the worker takes other work. +#[nativelink_test] +async fn a_hold_expires_and_the_worker_takes_other_work() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + let (held_worker, free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let (held_rx, free_rx) = if held_worker == "a" { + (&mut a, &mut b) + } else { + (&mut b, &mut a) + }; + + // The free worker is full now; a small action has nowhere to go, since + // the held worker takes nothing else. + add_action(&scheduler, &mut listeners, 12, SMALL_KB).await?; + assert_eq!( + (dispatches(held_rx).len(), dispatches(free_rx).len()), + (0, 0) + ); + + // The hold runs out; the pass that notices wakes the matcher, and the + // next pass places the small action on the worker, which is not held + // for the large action again while its cooldown lasts. + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!( + memory_of(&dispatches(held_rx)), + vec![SMALL_KB], + "the expired hold's worker takes the small action, not the large one" + ); + assert_eq!(dispatches(free_rx).len(), 0, "{free} was full"); + + // Cooldown over (as long as the hold, since the worker was busy when + // it expired), and the large action still waits: a worker is held for + // it again. + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!(held(&scheduler).await.len(), 1); + Ok(()) +} + +/// With the live veto on, a held worker that drains but never reports room +/// for the action keeps the hold to its expiry while idle. Each such expiry +/// in a row doubles how long the action is left unheld. +#[nativelink_test] +async fn an_expiry_on_an_idle_worker_doubles_the_cooldown() -> Result<(), Error> { + let scheduler = make_scheduler(Options { + live_memory_veto: true, + ..Options::default() + }); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + let (small_on_a, small_on_b, _) = + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + let (held_worker, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let held_rx = if held_worker == "a" { &mut a } else { &mut b }; + let small_on_held = if held_worker == "a" { + small_on_a + } else { + small_on_b + }; + + // The held worker reports less free than the action asks and then + // drains: the ledger would place it, the veto does not. + report_free(&scheduler, held_worker, LARGE_KB / 2).await?; + for operation_id in &small_on_held { + complete(&scheduler, held_worker, operation_id).await?; + } + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!(dispatches(held_rx).len(), 0, "vetoed, not placed"); + assert_eq!(held(&scheduler).await, vec![held_worker.to_string()]); + + // Expiry on an idle worker: the cooldown is twice the hold. + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!( + held(&scheduler).await, + Vec::::new(), + "still cooling down after one hold's worth" + ); + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!(held(&scheduler).await.len(), 1, "held again after two"); + Ok(()) +} + +/// The held worker declines the action for load once it is sent: the hold +/// stays, and when the worker reports room the action is offered it again. +#[nativelink_test] +async fn a_declined_dispatch_keeps_the_hold() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + // Workers that admit when idle: a decline for load from one is about + // what else it runs, not a verdict on the action. + let mut a = add_worker_with(&scheduler, "a", true).await?; + let mut b = add_worker_with(&scheduler, "b", true).await?; + let (small_on_a, small_on_b, _) = + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + let (held_worker, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let held_rx = if held_worker == "a" { &mut a } else { &mut b }; + let small_on_held = if held_worker == "a" { + small_on_a + } else { + small_on_b + }; + for operation_id in &small_on_held { + complete(&scheduler, held_worker, operation_id).await?; + } + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + let sent = dispatches(held_rx); + assert_eq!(memory_of(&sent), vec![LARGE_KB]); + + scheduler + .worker_dispatch_declined( + &worker_id(held_worker), + &OperationId::from(sent[0].0.as_str()), + "load".to_string(), + Some(LARGE_KB), + ) + .await?; + tokio::task::yield_now().await; + assert_eq!( + held(&scheduler).await, + vec![held_worker.to_string()], + "the hold outlives the decline" + ); + // Paused until it reports room; meanwhile the matcher offers it + // nothing, and a pass does not release the hold either. + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!(dispatches(held_rx).len(), 0); + assert_eq!(held(&scheduler).await, vec![held_worker.to_string()]); + + report_free(&scheduler, held_worker, WORKER_KB).await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + let sent = dispatches(held_rx); + assert_eq!(memory_of(&sent), vec![LARGE_KB], "offered again"); + acknowledge(&scheduler, held_worker, &sent[0].0).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + Ok(()) +} + +/// A worker that declined that much while idle is not offered the action +/// again until it reports room, so it is never the one held; and a held +/// worker that declines the action idle loses the hold to another. +#[nativelink_test] +async fn a_worker_that_declined_idle_is_passed_over() -> Result<(), Error> { + // A decline for load, and the verdict it leaves on an idle worker, + // exist with the live veto on. + let scheduler = make_scheduler(Options { + live_memory_veto: true, + ..Options::default() + }); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + for digest in 1..5 { + add_action(&scheduler, &mut listeners, digest, SMALL_KB).await?; + } + let (on_a, on_b) = (dispatches(&mut a), dispatches(&mut b)); + assert_eq!((on_a.len(), on_b.len()), (2, 2)); + + // An idle worker joins; the large action goes to it at once, and it + // declines it. Paused by the decline, it is offered nothing until its + // next keepalive, and that much never again until it reports room. + let mut c = add_worker(&scheduler, "c").await?; + let large = add_action(&scheduler, &mut listeners, 9, LARGE_KB).await?; + let sent = dispatches(&mut c); + assert_eq!(memory_of(&sent), vec![LARGE_KB]); + scheduler + .worker_dispatch_declined( + &worker_id("c"), + &OperationId::from(sent[0].0.as_str()), + "load".to_string(), + Some(LARGE_KB), + ) + .await?; + tokio::task::yield_now().await; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(dispatches(&mut c).len(), 0, "not offered the action again"); + let held_now = held(&scheduler).await; + assert_eq!(held_now.len(), 1); + assert_ne!( + held_now[0], "c", + "the idle worker that declined is not held" + ); + drop(large); + + // The held worker drains and declines the action idle: the hold moves. + let held_worker = held_now[0].clone(); + let held_rx = if held_worker == "a" { &mut a } else { &mut b }; + let other = if held_worker == "a" { "b" } else { "a" }; + let running = if held_worker == "a" { on_a } else { on_b }; + for (operation_id, _) in &running { + complete(&scheduler, &held_worker, operation_id).await?; + } + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + let sent = dispatches(held_rx); + assert_eq!(memory_of(&sent), vec![LARGE_KB]); + scheduler + .worker_dispatch_declined( + &worker_id(&held_worker), + &OperationId::from(sent[0].0.as_str()), + "load".to_string(), + Some(LARGE_KB), + ) + .await?; + tokio::task::yield_now().await; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!( + held(&scheduler).await, + vec![other.to_string()], + "the hold moved to the other busy worker" + ); + Ok(()) +} + +/// Of two equally busy workers, the one with more memory free as last +/// reported is held. +#[nativelink_test] +async fn the_worker_with_more_memory_free_is_held() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + report_free(&scheduler, "a", 30_000).await?; + report_free(&scheduler, "b", 60_000).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(held(&scheduler).await, vec!["b".to_string()]); + Ok(()) +} + +/// A hold ends when its worker drains or leaves, and the next pass holds +/// another. +#[nativelink_test] +async fn a_hold_ends_when_its_worker_drains_or_leaves() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + let held_now = held(&scheduler).await; + assert_eq!(held_now.len(), 1); + let (held_worker, other) = if held_now[0] == "a" { + ("a", "b") + } else { + ("b", "a") + }; + + scheduler + .set_drain_worker(&worker_id(held_worker), true) + .await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!(held(&scheduler).await, vec![other.to_string()]); + + scheduler.remove_worker(&worker_id(other)).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!( + held(&scheduler).await, + Vec::::new(), + "a draining worker is not held" + ); + Ok(()) +} + +/// A hold ends when its action leaves the queue: here the client gives up +/// on it and the client timeout retires it. +#[nativelink_test] +async fn a_hold_ends_when_its_action_leaves_the_queue() -> Result<(), Error> { + let scheduler = make_scheduler(Options { + client_action_timeout_s: 100, + ..Options::default() + }); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + // The large action's client goes away. + drop(listeners.pop()); + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(held(&scheduler).await.len(), 1); + pass_after(&scheduler, 100).await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + assert_eq!( + held(&scheduler).await, + Vec::::new(), + "released once no pass meets the action" + ); + Ok(()) +} + +/// The large action is placed elsewhere first: an idle worker joins and +/// takes it at once; the hold ends once that worker takes it, and the held +/// worker takes small actions again. +#[nativelink_test] +async fn a_hold_ends_when_its_action_runs_elsewhere() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + let (held_worker, _free) = held_and_free(&scheduler, &mut listeners, &mut a, &mut b).await?; + let (held_rx, free_rx) = if held_worker == "a" { + (&mut a, &mut b) + } else { + (&mut b, &mut a) + }; + + let mut c = add_worker(&scheduler, "c").await?; + scheduler.do_try_match_for_test().await?; + tokio::task::yield_now().await; + let on_c = dispatches(&mut c); + assert_eq!( + memory_of(&on_c), + vec![LARGE_KB], + "an idle worker that fits the action takes it at once" + ); + assert_eq!(dispatches(held_rx).len(), 0); + assert_eq!( + held(&scheduler).await, + vec![held_worker.to_string()], + "held until the new worker takes it" + ); + acknowledge(&scheduler, "c", &on_c[0].0).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + + // The new worker has no memory left beside the large action, the free + // worker is full, and the formerly held worker has two slots: two + // small actions run only if it takes them. + add_action(&scheduler, &mut listeners, 20, SMALL_KB).await?; + add_action(&scheduler, &mut listeners, 21, SMALL_KB).await?; + assert_eq!( + ( + dispatches(held_rx).len(), + dispatches(free_rx).len(), + dispatches(&mut c).len() + ), + (2, 0, 0), + "the formerly held worker takes small actions again" + ); + Ok(()) +} + +/// At most the cap is held: one worker with the cap at one, and one on a +/// two-worker fleet with the cap at a tenth of the fleet. +#[nativelink_test] +async fn at_most_the_cap_is_held() -> Result<(), Error> { + for max_held_workers in [1, 0] { + let scheduler = make_scheduler(Options { + max_held_workers, + ..Options::default() + }); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + add_action(&scheduler, &mut listeners, 10, LARGE_KB).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!( + held(&scheduler).await.len(), + 1, + "cap {max_held_workers}: one worker held, the other still takes work" + ); + add_action(&scheduler, &mut listeners, 11, SMALL_KB).await?; + add_action(&scheduler, &mut listeners, 12, SMALL_KB).await?; + assert_eq!(dispatches(&mut a).len() + dispatches(&mut b).len(), 2); + } + Ok(()) +} + +/// Scheduler replicas sharing a queue take turns holding, by a hash of the +/// action over the replicas whose workers would fit it: a peer whose +/// workers would not fit it is never the owner, and when the peer would, +/// this scheduler holds only for the actions that fall to it. +#[nativelink_test] +async fn replicas_take_turns_holding() -> Result<(), Error> { + let scheduler = make_scheduler(Options::default()); + let mut listeners = Listeners::new(); + let mut a = add_worker(&scheduler, "a").await?; + let mut b = add_worker(&scheduler, "b").await?; + let shape = |memory_kb: u64| { + PlatformProperties::new(HashMap::from([( + "memory_kb".to_string(), + PlatformPropertyValue::Minimum(memory_kb), + )])) + }; + // A peer with workers too small for the large action: this scheduler + // holds for it whatever the hash says. + scheduler + .set_peer_schedulers_for_test(vec![PeerScheduler { + scheduler_id: "peer".to_string(), + workers: vec![shape(LARGE_KB - 1)], + }]) + .await; + let (_, _, large) = + two_busy_workers_and_a_large_action(&scheduler, &mut listeners, &mut a, &mut b).await?; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!(held(&scheduler).await.len(), 1); + + // A peer whose workers would fit it: whichever of the two the hash + // names holds, and the other does not. + let local = scheduler.scheduler_id_for_test().to_string(); + let mut owners = vec![local.as_str(), "peer"]; + owners.sort_unstable(); + let local_owns = hold_owner(&large, &owners) == local; + scheduler + .set_peer_schedulers_for_test(vec![PeerScheduler { + scheduler_id: "peer".to_string(), + workers: vec![shape(WORKER_KB)], + }]) + .await; + // The existing hold stands; a fresh one goes by the owner. Expire it + // and let the cooldown pass, then see whether it comes back. + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!(held(&scheduler).await, Vec::::new()); + pass_after(&scheduler, MAX_HOLD_S).await?; + assert_eq!( + held(&scheduler).await.len(), + usize::from(local_owns), + "held here only when the hash names this scheduler (owner: {})", + hold_owner(&large, &owners) + ); + // Another action, which the hash sends to the other owner, shows the + // turn-taking rather than a fixed verdict. + let other = loop { + let digest = u8::try_from(30 + listeners.len()).unwrap_or(250); + let action = add_action(&scheduler, &mut listeners, digest, LARGE_KB).await?; + if (hold_owner(&action, &owners) == local) != local_owns { + break action; + } + }; + pass_after(&scheduler, HOLD_AFTER_S).await?; + assert_eq!( + held(&scheduler).await.len(), + usize::from(!local_owns), + "the other action falls to the other owner ({})", + hold_owner(&other, &owners) + ); + Ok(()) +} diff --git a/nativelink-scheduler/tests/worker_running_operations_report_test.rs b/nativelink-scheduler/tests/worker_running_operations_report_test.rs index 32c2f129fc1..09be2229e0d 100644 --- a/nativelink-scheduler/tests/worker_running_operations_report_test.rs +++ b/nativelink-scheduler/tests/worker_running_operations_report_test.rs @@ -109,6 +109,7 @@ async fn dispatched() -> Result { WorkerAllocationStrategy::default(), None, None, + None, Arc::new(Notify::new()), WORKER_TIMEOUT_S, 60, @@ -117,6 +118,7 @@ async fn dispatched() -> Result { None, false, Duration::from_secs(15), + "scheduler".to_string(), ); let worker_id = WorkerId::from(String::from("worker")); let (tx, mut rx) = mpsc::channel(8); diff --git a/nativelink-service/tests/worker_api_server_test.rs b/nativelink-service/tests/worker_api_server_test.rs index bc9fcffb335..0e0056788c2 100644 --- a/nativelink-service/tests/worker_api_server_test.rs +++ b/nativelink-service/tests/worker_api_server_test.rs @@ -193,6 +193,7 @@ async fn setup_api_server_with( WorkerAllocationStrategy::default(), None, None, + None, tasks_or_worker_change_notify, worker_timeout, 60, // unacknowledged_kill_timeout_s @@ -201,6 +202,7 @@ async fn setup_api_server_with( None, false, // has_peers Duration::from_secs(15), // record_ttl (unused while has_peers = false) + "scheduler".to_string(), ); let mut schedulers: HashMap> = HashMap::new(); @@ -525,6 +527,7 @@ pub async fn going_away_with_drain_stops_dispatch_and_keeps_the_worker_until_the &PlatformProperties::new(HashMap::new()), false, make_system_time(BASE_NOW_S), + None, ) .await; assert!( @@ -955,6 +958,7 @@ pub async fn workers_only_allow_max_tasks() -> Result<(), Box Result<(), Box = LazyLock::new(|| .with_unit("{action}") .build(), + worker_holds: meter + .u64_counter("scheduler.worker.holds") + .with_description( + "Workers held for a queued action that kept finding no room, by outcome: held when the hold began, placed when a worker took the action, released when the action or the worker left, declined when the held worker declined the action idle, expired when max_hold_s passed first", + ) + .with_unit("{hold}") + .build(), + + held_workers: meter + .u64_gauge("scheduler.worker.held") + .with_description("Workers this scheduler holds for a queued action right now") + .with_unit("{worker}") + .build(), + awaited_action_orphans: meter .u64_counter("scheduler.awaited_action.orphans") .with_description( @@ -1262,6 +1277,11 @@ pub struct SchedulerOtlpMetrics { /// Actions a pass parked for want of room and dispatched when room /// opened before the pass ended, ahead of the newer actions it was reading. pub parked_dispatched: metrics::Counter, + /// Workers held for a queued action that kept finding no room, by what + /// began or ended the hold. + pub worker_holds: metrics::Counter, + /// Workers held for such an action right now. + pub held_workers: metrics::Gauge, /// Listed operations whose record was gone on read, by site. pub awaited_action_orphans: metrics::Counter, /// Abandoned-queue sweeps that failed before finishing. @@ -1395,6 +1415,24 @@ pub fn record_parked_dispatched(count: u64) { } } +/// Records a worker hold beginning (`held`) or ending: `placed` when a +/// worker took the action, `released` when the action left the queue or +/// the worker left or drained, `declined` when the held worker declined +/// the action idle and another is held instead, `expired` when `max_hold_s` +/// passed first. +pub fn record_worker_hold(outcome: &'static str) { + SCHEDULER_METRICS + .worker_holds + .add(1, &[KeyValue::new(SCHEDULER_HOLD_OUTCOME, outcome)]); +} + +/// Records how many workers are held right now. +pub fn record_held_workers(count: usize) { + SCHEDULER_METRICS + .held_workers + .record(u64::try_from(count).unwrap_or(u64::MAX), &[]); +} + /// Records an operation the queue listed whose record was gone when read; /// `site` is where it was met (`matching`, `sweep`). pub fn record_awaited_action_orphan(site: &'static str) { diff --git a/web/apps/docs/content/docs/explanations/scheduler-internals.mdx b/web/apps/docs/content/docs/explanations/scheduler-internals.mdx index a87f651c88d..63d53af8546 100644 --- a/web/apps/docs/content/docs/explanations/scheduler-internals.mdx +++ b/web/apps/docs/content/docs/explanations/scheduler-internals.mdx @@ -186,6 +186,34 @@ the parked actions a worker was actually told to run that way; a non-zero rate is normal on a busy fleet and means the head of the queue is being served. +Parking does not help an action whose reservation only fits a worker with +nothing else on it. Every pass tries it first and finds each worker partly +busy, then fills the room that opens with newer, smaller actions, so no +worker ever drains for it while the fleet reads as free. **`reservation_hold`** +is for that action: once it has waited `hold_after_s` and fits no busy worker +until more than one of that worker's actions finish (an action one completion +would make room for is only waiting its turn, which the pass order gives it), +the pass holds a worker that would fit it idle (fewest actions running, then +most memory free as last reported, never one that declined that much while +idle) and offers it nothing else; as its actions finish the held action is +placed there the moment the ledger and the live veto allow. The hold stays +through the dispatch until the worker takes the action; a worker that declines +it sends it back to the queue with the hold kept, unless it declined idle, in +which case the hold moves to another worker. The hold ends when a worker takes +the action, anywhere, when it leaves the queue, when the worker leaves or +drains, or after `max_hold_s`, whichever is first; an expired hold is not +repeated for that action until as long again has passed, twice as long for +each expiry in a row on a worker that was idle, so an action that never fits +does not keep workers idle in turns. At most `max_held_workers` are held at +once per replica (a tenth of the workers connected to it by default). A hold is +local to the scheduler replica the worker is connected to; replicas sharing a +queue take turns per action, by a hash of the action over the replicas whose +workers would fit it as the fleet census last read them, so one worker is held +for it fleet-wide. `scheduler.worker.holds` counts holds by outcome and +`scheduler.worker.held` is how many workers are held right now. An action no +connected worker would fit even idle gets no hold; that is the unsatisfiable +path, or a sizing class larger than the workers less their overhead. + When a worker reports an action killed for memory, by its own reservation enforcement or by the kernel near the limit, **`memory_escalation`** requeues the action with the reservation multiplied by `percent` (default 200) instead diff --git a/web/apps/docs/content/docs/reference/metrics.mdx b/web/apps/docs/content/docs/reference/metrics.mdx index cd7ed6fc12c..d2b70e632ca 100644 --- a/web/apps/docs/content/docs/reference/metrics.mdx +++ b/web/apps/docs/content/docs/reference/metrics.mdx @@ -5,10 +5,10 @@ full: true --- {/* AUTOGENERATED. Do not edit by hand. - Source: nativelink-util/src/metrics.rs @ v1.7.6 (3d614c607) + Source: nativelink-util/src/metrics.rs @ v1.7.6 (9aa3e3e90) Regenerate from web/: bun --filter @nativelink/docs gen:metrics-reference */} -NativeLink **v1.7.6** declares 47 OpenTelemetry instruments. This page is +NativeLink **v1.7.6** declares 49 OpenTelemetry instruments. This page is generated from the declarations in [`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs) and from every `.add()` and `.record()` call site across the workspace, so the @@ -61,6 +61,8 @@ wrong. | [`scheduler.executing.oldest_age`](#scheduler-executing-oldest-age) | `nativelink_scheduler_executing_oldest_age` (unreferenced) | Gauge | Conditionally | scheduler process only | | [`scheduler.queue.retired`](#scheduler-queue-retired) | `nativelink_scheduler_queue_retired` (unreferenced) | Counter | Conditionally | scheduler process only | | [`scheduler.matching.parked_dispatched`](#scheduler-matching-parked-dispatched) | `nativelink_scheduler_matching_parked_dispatched` (unreferenced) | Counter | Conditionally | scheduler process only | +| [`scheduler.worker.holds`](#scheduler-worker-holds) | `nativelink_scheduler_worker_holds` (unreferenced) | Counter | Conditionally | scheduler process only | +| [`scheduler.worker.held`](#scheduler-worker-held) | `nativelink_scheduler_worker_held` (unreferenced) | Gauge | Conditionally | scheduler process only | | [`scheduler.awaited_action.orphans`](#scheduler-awaited-action-orphans) | `nativelink_scheduler_awaited_action_orphans` (unreferenced) | Counter | Conditionally | scheduler process only | | [`scheduler.sweep.failures`](#scheduler-sweep-failures) | `nativelink_scheduler_sweep_failures` (unreferenced) | Counter | Conditionally | scheduler process only | | [`scheduler.hints.resolutions`](#scheduler-hints-resolutions) | `nativelink_scheduler_hints_resolutions` (unreferenced) | Counter | Conditionally | scheduler process only | @@ -106,7 +108,7 @@ Every instrument declared in `metrics.rs` has at least one call site. Global cache metrics instruments. -Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:448`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L448), backed by the `CacheMetrics` struct. +Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:449`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L449), backed by the `CacheMetrics` struct. Every cache instrument is emitted from one place: the `cache_metrics` store wrapper. A store that is not wrapped emits nothing, no matter how the collector is configured. @@ -121,7 +123,7 @@ Histogram of cache operation durations in milliseconds | Unit | `ms` | | Prometheus series | `nativelink_cache_operation_duration` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:452`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L452) | +| Declared at | [`nativelink-util/src/metrics.rs:453`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L453) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:73`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L73) | **Histogram buckets:** 19 explicit boundaries, from @@ -140,7 +142,7 @@ Counter of cache operations by type and result | Unit | — | | Prometheus series | `nativelink_cache_operations_total` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:486`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L486) | +| Declared at | [`nativelink-util/src/metrics.rs:487`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L487) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:127`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L127), [`nativelink-store/src/cache_metrics_store.rs:132`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L132), [`nativelink-store/src/cache_metrics_store.rs:143`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L143), [`nativelink-store/src/cache_metrics_store.rs:168`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L168), [`nativelink-store/src/cache_metrics_store.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L174), [`nativelink-store/src/cache_metrics_store.rs:200`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L200), [`nativelink-store/src/cache_metrics_store.rs:206`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L206), [`nativelink-store/src/cache_metrics_store.rs:219`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L219), [`nativelink-store/src/cache_metrics_store.rs:225`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L225), [`nativelink-store/src/cache_metrics_store.rs:246`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L246), [`nativelink-store/src/cache_metrics_store.rs:254`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L254), [`nativelink-store/src/cache_metrics_store.rs:260`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L260) | ### `cache.io` @@ -154,7 +156,7 @@ Counter of bytes read/written during cache operations | Unit | `By` | | Prometheus series | `nativelink_cache_io_total` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L491) | +| Declared at | [`nativelink-util/src/metrics.rs:492`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L492) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:80`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L80), [`nativelink-store/src/cache_metrics_store.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L248) | ### `cache.size` @@ -168,7 +170,7 @@ Current total size of all cached data in bytes | Unit | `By` | | Prometheus series | `nativelink_cache_size` | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:497`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L497) | +| Declared at | [`nativelink-util/src/metrics.rs:498`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L498) | | Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) | ### `cache.entries` @@ -182,7 +184,7 @@ Current number of entries in cache | Unit | `{entry}` | | Prometheus series | `nativelink_cache_entries` | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:503`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L503) | +| Declared at | [`nativelink-util/src/metrics.rs:504`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L504) | | Emitted from | [`nativelink-util/src/evicting_map.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/evicting_map.rs#L468) | ### `cache.item.size` @@ -196,14 +198,14 @@ Histogram of individual cache entry sizes in bytes | Unit | `By` | | Prometheus series | `nativelink_cache_item_size` | | Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | -| Declared at | [`nativelink-util/src/metrics.rs:509`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L509) | +| Declared at | [`nativelink-util/src/metrics.rs:510`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L510) | | Emitted from | [`nativelink-store/src/cache_metrics_store.rs:83`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/cache_metrics_store.rs#L83) | ## Execution metrics Global remote execution metrics instruments. -Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:549`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L549), backed by the `ExecutionMetrics` struct. +Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:550`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L550), backed by the `ExecutionMetrics` struct. Execution instruments are emitted from the scheduler process as actions change state, plus the per-action CPU time and peak memory a worker reports back when it finishes. Worker processes emit no execution metrics of their own. @@ -218,8 +220,8 @@ Histogram of stage durations in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_stage_duration` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:553`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L553) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | +| Declared at | [`nativelink-util/src/metrics.rs:554`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L554) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) | **Histogram buckets:** 15 explicit boundaries, from `0.001` to `3600.0` `s`. Anything past the last boundary lands in the @@ -237,8 +239,8 @@ Histogram of total execution durations in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_total_duration` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:578`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L578) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | +| Declared at | [`nativelink-util/src/metrics.rs:579`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L579) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) | **Histogram buckets:** 13 explicit boundaries, from `0.01` to `7200.0` `s`. Anything past the last boundary lands in the @@ -256,8 +258,8 @@ Histogram of queue wait times in seconds | Unit | `s` | | Prometheus series | `nativelink_execution_queue_time` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:603`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L603) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | +| Declared at | [`nativelink-util/src/metrics.rs:604`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L604) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) | **Histogram buckets:** 12 explicit boundaries, from `0.001` to `600.0` `s`. Anything past the last boundary lands in the @@ -275,7 +277,7 @@ Current number of actions in each stage | Unit | `{action}` | | Prometheus series | `nativelink_execution_active_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:623`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L623) | +| Declared at | [`nativelink-util/src/metrics.rs:624`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L624) | | Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:425`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L425), [`nativelink-scheduler/src/memory_awaited_action_db.rs:672`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L672), [`nativelink-scheduler/src/memory_awaited_action_db.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L679), [`nativelink-scheduler/src/memory_awaited_action_db.rs:824`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L824), [`nativelink-scheduler/src/store_awaited_action_db.rs:716`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/store_awaited_action_db.rs#L716) | ### `execution.completed.count` @@ -289,8 +291,8 @@ Total number of completed executions | Unit | `{action}` | | Prometheus series | `nativelink_execution_completed_count_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:629`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L629) | -| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:152`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L152), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:162`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L162) | +| Declared at | [`nativelink-util/src/metrics.rs:630`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L630) | +| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:698`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L698), [`nativelink-scheduler/src/memory_awaited_action_db.rs:710`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L710), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:163`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L163) | ### `execution.stage.transitions` @@ -303,8 +305,8 @@ Number of stage transitions | Unit | `{transition}` | | Prometheus series | `nativelink_execution_stage_transitions_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:635`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L635) | -| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:141`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L141) | +| Declared at | [`nativelink-util/src/metrics.rs:636`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L636) | +| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:665`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/memory_awaited_action_db.rs#L665), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:142`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L142) | ### `execution.cpu.time` @@ -317,8 +319,8 @@ CPU time an action used, as sampled by the worker. | Unit | `s` | | Prometheus series | `nativelink_execution_cpu_time` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:641`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L641) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1911`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1911) | +| Declared at | [`nativelink-util/src/metrics.rs:642`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L642) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2480`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2480) | **Histogram buckets:** 12 explicit boundaries, from `0.1` to `7200.0` `s`. Anything past the last boundary lands in the @@ -336,8 +338,8 @@ Peak memory an action used, as sampled by the worker. | Unit | `By` | | Prometheus series | `nativelink_execution_peak_memory` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:650`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L650) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1903`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1903) | +| Declared at | [`nativelink-util/src/metrics.rs:651`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L651) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2472`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2472) | **Histogram buckets:** 8 explicit boundaries, from `1_048_576.0` to `68_719_476_736.0` `By`. Anything past the last boundary lands in the @@ -355,8 +357,8 @@ Histogram of output sizes in bytes | Unit | `By` | | Prometheus series | `nativelink_execution_output_size` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:666`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L666) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:153`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L153) | +| Declared at | [`nativelink-util/src/metrics.rs:667`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L667) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:154`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L154) | **Histogram buckets:** 8 explicit boundaries, from `1_024.0` to `10_737_418_240.0` `By`. Anything past the last boundary lands in the @@ -374,14 +376,14 @@ Counter for execution retries | Unit | `{retry}` | | Prometheus series | `nativelink_execution_retry_count_total` | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:682`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L682) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:174`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L174) | +| Declared at | [`nativelink-util/src/metrics.rs:683`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L683) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:175`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L175) | ## Worker fleet metrics Global worker fleet metrics instruments. -Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:891`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L891), backed by the `WorkerMetrics` struct. +Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:892`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L892), backed by the `WorkerMetrics` struct. Worker fleet instruments are emitted from the scheduler process as workers connect, heartbeat, pause, drain and leave. They describe the pool as this scheduler instance sees it. @@ -396,8 +398,8 @@ Workers currently connected. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_connected_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:895`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L895) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:464`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L464), [`nativelink-scheduler/src/api_worker_scheduler.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1491) | +| Declared at | [`nativelink-util/src/metrics.rs:896`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L896) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:581`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L581), [`nativelink-scheduler/src/api_worker_scheduler.rs:2019`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2019) | ### `worker.connections` @@ -410,8 +412,8 @@ Workers that have joined, cumulative. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_connections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:901`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L901) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:464`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L464) | +| Declared at | [`nativelink-util/src/metrics.rs:902`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L902) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:581`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L581) | ### `worker.disconnections` @@ -424,8 +426,8 @@ Workers that have left, cumulative, by reason. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_disconnections` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:907`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L907) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1491`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1491) | +| Declared at | [`nativelink-util/src/metrics.rs:908`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L908) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:2019`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2019) | ### `worker.input.relinks` @@ -438,7 +440,7 @@ Input hardlinks made again after their source was evicted mid-staging, by kind. | Unit | — | | Prometheus series | `nativelink_worker_input_relinks` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L913) | +| Declared at | [`nativelink-util/src/metrics.rs:914`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L914) | | Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1038), [`nativelink-worker/src/running_actions_manager.rs:1172`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1172), [`nativelink-worker/src/running_actions_manager.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1190) | ### `worker.prepare.retries` @@ -452,7 +454,7 @@ Input fetches tried again on the worker after a transient store failure: `retrie | Unit | — | | Prometheus series | `nativelink_worker_prepare_retries` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:918`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L918) | +| Declared at | [`nativelink-util/src/metrics.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L919) | | Emitted from | [`nativelink-worker/src/running_actions_manager.rs:1014`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-worker/src/running_actions_manager.rs#L1014) | ### `scheduler.dispatch.requeues` @@ -466,8 +468,8 @@ Dispatches requeued untried, by reason. | Unit | — | | Prometheus series | `nativelink_scheduler_dispatch_requeues` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:923`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L923) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1187`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1187), [`nativelink-scheduler/src/api_worker_scheduler.rs:1381`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1381), [`nativelink-scheduler/src/api_worker_scheduler.rs:1701`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1701), [`nativelink-scheduler/src/api_worker_scheduler.rs:1999`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1999), [`nativelink-scheduler/src/api_worker_scheduler.rs:2030`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2030), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:679`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L679) | +| Declared at | [`nativelink-util/src/metrics.rs:924`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L924) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:1680`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1680), [`nativelink-scheduler/src/api_worker_scheduler.rs:1909`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1909), [`nativelink-scheduler/src/api_worker_scheduler.rs:2251`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2251), [`nativelink-scheduler/src/api_worker_scheduler.rs:2568`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2568), [`nativelink-scheduler/src/api_worker_scheduler.rs:2599`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L2599), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:680`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L680) | ### `worker.keepalives` @@ -480,8 +482,8 @@ Keepalives received, cumulative. | Unit | `{keepalive}` | | Prometheus series | `nativelink_worker_keepalives` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:928`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L928) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:422`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L422) | +| Declared at | [`nativelink-util/src/metrics.rs:929`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L929) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:539`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L539) | ### `worker.keepalive.gap` @@ -494,8 +496,8 @@ Seconds between consecutive messages from a worker. | Unit | `s` | | Prometheus series | `nativelink_worker_keepalive_gap` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:934`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L934) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:356`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L356) | +| Declared at | [`nativelink-util/src/metrics.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L935) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:473`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L473) | **Histogram buckets:** 12 explicit boundaries, from `1.0` to `60.0` `s`. Anything past the last boundary lands in the @@ -513,8 +515,8 @@ Connected workers currently paused or draining. | Unit | `{worker}` | | Prometheus series | `nativelink_worker_state_count` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:945`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L945) | -| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:405`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L405), [`nativelink-scheduler/src/api_worker_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L458), [`nativelink-scheduler/src/api_worker_scheduler.rs:461`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L461), [`nativelink-scheduler/src/api_worker_scheduler.rs:571`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L571), [`nativelink-scheduler/src/api_worker_scheduler.rs:1148`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1148), [`nativelink-scheduler/src/api_worker_scheduler.rs:1190`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1190), [`nativelink-scheduler/src/api_worker_scheduler.rs:1326`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1326) | +| Declared at | [`nativelink-util/src/metrics.rs:946`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L946) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:522`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L522), [`nativelink-scheduler/src/api_worker_scheduler.rs:575`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L575), [`nativelink-scheduler/src/api_worker_scheduler.rs:578`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L578), [`nativelink-scheduler/src/api_worker_scheduler.rs:696`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L696), [`nativelink-scheduler/src/api_worker_scheduler.rs:1641`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1641), [`nativelink-scheduler/src/api_worker_scheduler.rs:1683`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1683), [`nativelink-scheduler/src/api_worker_scheduler.rs:1835`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L1835) | ## gRPC server metrics @@ -523,7 +525,7 @@ Global gRPC serving metrics. One duration histogram covers rate, errors and latency: the count gives rate, the status attribute separates errors, and the buckets give latency. -Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1069`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1069), backed by the `RpcMetrics` struct. +Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:1070`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1070), backed by the `RpcMetrics` struct. Emitted by the `OtlpLayer` tower middleware on every gRPC server NativeLink starts, so a CAS, a scheduler and a worker API endpoint each report their own. @@ -538,7 +540,7 @@ Duration of inbound gRPC calls, by service, method and status. | Unit | `s` | | Prometheus series | `nativelink_rpc_server_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1073`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1073) | +| Declared at | [`nativelink-util/src/metrics.rs:1074`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1074) | | Emitted from | [`nativelink-util/src/telemetry.rs:371`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/telemetry.rs#L371) | **Histogram buckets:** 15 explicit boundaries, from @@ -550,7 +552,7 @@ measurement. Global scheduler metrics. -Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1122`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1122), backed by the `SchedulerOtlpMetrics` struct. +Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:1123`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1123), backed by the `SchedulerOtlpMetrics` struct. Emitted from the scheduler process once per matching pass, whether or not the pass assigned anything. @@ -566,8 +568,8 @@ while the queue is non-empty means matching is the bottleneck. | Unit | `s` | | Prometheus series | `nativelink_scheduler_matching_duration` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1126`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1126) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L458) | +| Declared at | [`nativelink-util/src/metrics.rs:1127`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1127) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:478`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L478) | **Histogram buckets:** 13 explicit boundaries, from `0.0001` to `10.0` `s`. Anything past the last boundary lands in the @@ -585,8 +587,8 @@ Matching passes, by result. | Unit | `{pass}` | | Prometheus series | `nativelink_scheduler_matching_passes` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1135`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1135) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:458`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L458) | +| Declared at | [`nativelink-util/src/metrics.rs:1136`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1136) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:478`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L478) | ### `scheduler.unsatisfiable.queued` @@ -601,8 +603,8 @@ instance, judged against that instance's own workers. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_unsatisfiable_queued` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1141`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1141) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1036`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1036) | +| Declared at | [`nativelink-util/src/metrics.rs:1142`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1142) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1073`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1073) | ### `scheduler.unsatisfiable.failed` @@ -616,8 +618,8 @@ the properties that could not be satisfied. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_unsatisfiable_failed` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1149`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1149) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:560`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L560) | +| Declared at | [`nativelink-util/src/metrics.rs:1150`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1150) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:580`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L580) | ### `scheduler.queue.depth` @@ -630,8 +632,8 @@ Queued actions the last matching pass read. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_depth` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1155`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1155) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1037`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1037) | +| Declared at | [`nativelink-util/src/metrics.rs:1156`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1156) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1074`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1074) | ### `scheduler.queue.age` @@ -644,8 +646,8 @@ Queued actions the last matching pass read, by age bucket. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1161`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1161) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1038) | +| Declared at | [`nativelink-util/src/metrics.rs:1162`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1162) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1075`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1075) | ### `scheduler.queue.oldest_age` @@ -658,8 +660,8 @@ Age of the oldest action the last matching pass read, in seconds. | Unit | `s` | | Prometheus series | `nativelink_scheduler_queue_oldest_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1169`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1169) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1038`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1038) | +| Declared at | [`nativelink-util/src/metrics.rs:1170`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1170) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1075`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1075) | ### `scheduler.executing.age` @@ -672,8 +674,8 @@ Executing actions the last executing sweep read, by age bucket. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_executing_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1177`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1177) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1307`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1307) | +| Declared at | [`nativelink-util/src/metrics.rs:1178`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1178) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1349`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1349) | ### `scheduler.executing.oldest_age` @@ -687,8 +689,8 @@ executing, in seconds. | Unit | `s` | | Prometheus series | `nativelink_scheduler_executing_oldest_age` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1185`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1185) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1307`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1307) | +| Declared at | [`nativelink-util/src/metrics.rs:1186`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1186) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1349`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1349) | ### `scheduler.queue.retired` @@ -701,8 +703,8 @@ Queued actions retired for having no client. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_queue_retired` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1193`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1193) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:783`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L783) | +| Declared at | [`nativelink-util/src/metrics.rs:1194`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1194) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:784`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L784) | ### `scheduler.matching.parked_dispatched` @@ -716,8 +718,37 @@ opened before the pass ended, ahead of the newer actions it was reading. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_matching_parked_dispatched` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1199`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1199) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1013`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1013) | +| Declared at | [`nativelink-util/src/metrics.rs:1200`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1200) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1050`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1050) | + +### `scheduler.worker.holds` + +Workers held for a queued action that kept finding no room, by what +began or ended the hold. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `worker_holds` | +| Unit | `{hold}` | +| Prometheus series | `nativelink_scheduler_worker_holds` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:1208`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1208) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:719`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L719), [`nativelink-scheduler/src/api_worker_scheduler.rs:932`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L932) | + +### `scheduler.worker.held` + +Workers held for such an action right now. + +| Property | Value | +| --- | --- | +| Instrument type | `Gauge` | +| Field | `held_workers` | +| Unit | `{worker}` | +| Prometheus series | `nativelink_scheduler_worker_held` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:1216`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1216) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:720`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L720), [`nativelink-scheduler/src/api_worker_scheduler.rs:933`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/api_worker_scheduler.rs#L933) | ### `scheduler.awaited_action.orphans` @@ -730,8 +761,8 @@ Listed operations whose record was gone on read, by site. | Unit | `{operation}` | | Prometheus series | `nativelink_scheduler_awaited_action_orphans` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1207`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1207) | -| Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:600`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L600), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:614`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L614), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:731`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L731), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1662`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1662), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1685`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1685) | +| Declared at | [`nativelink-util/src/metrics.rs:1222`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1222) | +| Emitted from | [`nativelink-scheduler/src/admin.rs:107`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/admin.rs#L107), [`nativelink-scheduler/src/simple_scheduler.rs:620`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L620), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:615`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L615), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:732`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L732), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1663`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1663), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:1686`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L1686) | ### `scheduler.sweep.failures` @@ -744,8 +775,8 @@ Abandoned-queue sweeps that failed before finishing. | Unit | `{sweep}` | | Prometheus series | `nativelink_scheduler_sweep_failures` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1215`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1215) | -| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1259`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1259), [`nativelink-scheduler/src/simple_scheduler.rs:1318`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1318) | +| Declared at | [`nativelink-util/src/metrics.rs:1230`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1230) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:1301`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1301), [`nativelink-scheduler/src/simple_scheduler.rs:1360`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/simple_scheduler.rs#L1360) | ### `scheduler.hints.resolutions` @@ -758,14 +789,14 @@ Actions sized by the historical resource scheduler, by source. | Unit | `{action}` | | Prometheus series | `nativelink_scheduler_hints_resolutions` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. scheduler process only | -| Declared at | [`nativelink-util/src/metrics.rs:1221`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1221) | +| Declared at | [`nativelink-util/src/metrics.rs:1236`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1236) | | Emitted from | [`nativelink-scheduler/src/historical_resource_scheduler.rs:643`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L643), [`nativelink-scheduler/src/historical_resource_scheduler.rs:647`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L647), [`nativelink-scheduler/src/historical_resource_scheduler.rs:650`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-scheduler/src/historical_resource_scheduler.rs#L650) | ## STORE_METRICS Global object-store metrics. -Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1434`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1434), backed by the `StoreMetrics` struct. +Declared as `STORE_METRICS` in [`nativelink-util/src/metrics.rs:1472`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1472), backed by the `StoreMetrics` struct. ### `store.object.throttles` @@ -778,14 +809,14 @@ Throttled responses, by provider and operation. | Unit | `{response}` | | Prometheus series | `nativelink_store_object_throttles` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1438`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1438) | +| Declared at | [`nativelink-util/src/metrics.rs:1476`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1476) | | Emitted from | [`nativelink-store/src/azure_blob_store.rs:268`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/azure_blob_store.rs#L268) | ## Store tier metrics Global tiered-store metrics. -Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1465`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1465), backed by the `StoreTierMetrics` struct. +Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1503`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1503), backed by the `StoreTierMetrics` struct. Emitted from the `fast_slow` store wrapper only. A deployment without a `fast_slow` store emits none of these. @@ -800,7 +831,7 @@ Reads served per tier, by result. | Unit | `{operation}` | | Prometheus series | `nativelink_store_tier_operations` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1469`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1469) | +| Declared at | [`nativelink-util/src/metrics.rs:1507`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1507) | | Emitted from | [`nativelink-store/src/fast_slow_store.rs:348`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L348), [`nativelink-store/src/fast_slow_store.rs:839`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L839), [`nativelink-store/src/fast_slow_store.rs:855`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L855), [`nativelink-store/src/fast_slow_store.rs:865`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L865), [`nativelink-store/src/fast_slow_store.rs:882`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L882), [`nativelink-store/src/fast_slow_store.rs:912`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L912), [`nativelink-store/src/fast_slow_store.rs:934`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L934) | ### `store.tier.io` @@ -814,14 +845,14 @@ Bytes moved per tier and direction. | Unit | `By` | | Prometheus series | `nativelink_store_tier_io` (not referenced by any shipped rule or dashboard) | | Emitted | Conditionally. only through a `fast_slow` store | -| Declared at | [`nativelink-util/src/metrics.rs:1475`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1475) | +| Declared at | [`nativelink-util/src/metrics.rs:1513`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1513) | | Emitted from | [`nativelink-store/src/fast_slow_store.rs:357`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L357), [`nativelink-store/src/fast_slow_store.rs:856`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L856), [`nativelink-store/src/fast_slow_store.rs:913`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L913), [`nativelink-store/src/fast_slow_store.rs:935`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/fast_slow_store.rs#L935) | ## Health check metrics Global health-check metrics. -Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1516`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1516), backed by the `HealthMetrics` struct. +Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1554`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1554), backed by the `HealthMetrics` struct. Emitted each time the health check endpoint runs, one observation per registered health indicator. @@ -836,14 +867,14 @@ Health check results, by namespace and status. | Unit | `{check}` | | Prometheus series | `nativelink_health_checks` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1520`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1520) | +| Declared at | [`nativelink-util/src/metrics.rs:1558`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1558) | | Emitted from | [`nativelink-util/src/health_utils.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/health_utils.rs#L248) | ## Connection pool metrics Global connection-pool metrics. -Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1547`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1547), backed by the `ConnectionMetrics` struct. +Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1585`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1585), backed by the `ConnectionMetrics` struct. Emitted by the gRPC connection manager and the Redis store as they hand out and re-establish connections. A deployment that uses neither emits none of these. @@ -858,7 +889,7 @@ Free slots at the moment a connection was taken. | Unit | `{connection}` | | Prometheus series | `nativelink_connection_pool_available` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1551`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1551) | +| Declared at | [`nativelink-util/src/metrics.rs:1589`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1589) | | Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L351) | **Histogram buckets:** 11 explicit boundaries, from @@ -877,7 +908,7 @@ Acquisitions, by pool and result. | Unit | `{acquisition}` | | Prometheus series | `nativelink_connection_pool_acquisitions` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1560`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1560) | +| Declared at | [`nativelink-util/src/metrics.rs:1598`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1598) | | Emitted from | [`nativelink-store/src/redis_store.rs:587`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L587), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/connection_manager.rs#L351) | ### `connection.reconnects` @@ -891,7 +922,7 @@ Reconnects, by pool. | Unit | `{reconnect}` | | Prometheus series | `nativelink_connection_reconnects` (not referenced by any shipped rule or dashboard) | | Emitted | Yes. whenever the code path runs | -| Declared at | [`nativelink-util/src/metrics.rs:1566`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1566) | +| Declared at | [`nativelink-util/src/metrics.rs:1604`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-util/src/metrics.rs#L1604) | | Emitted from | [`nativelink-store/src/redis_store.rs:319`](https://github.com/TraceMachina/nativelink/blob/v1.7.6/nativelink-store/src/redis_store.rs#L319) | ## Attributes @@ -945,6 +976,7 @@ underscores. | --- | --- | --- | --- | | `scheduler.match.result` | `scheduler_match_result` | `SCHEDULER_MATCH_RESULT` | open (free-form string) | | `scheduler.unsatisfiable.properties` | `scheduler_unsatisfiable_properties` | `SCHEDULER_UNSATISFIABLE_PROPERTIES` | open (free-form string) | +| `scheduler.hold.outcome` | `scheduler_hold_outcome` | `SCHEDULER_HOLD_OUTCOME` | open (free-form string) | | `scheduler.action.age` | `scheduler_action_age` | `SCHEDULER_ACTION_AGE` | open (free-form string) | ### Store tier attributes